diff --git a/.github/workflows/report-embeds.yml b/.github/workflows/report-embeds.yml index 5d9c46446c..ff2e157cee 100644 --- a/.github/workflows/report-embeds.yml +++ b/.github/workflows/report-embeds.yml @@ -1,84 +1,35 @@ name: Report embeds -# Validates the W&B report-embed registry (scripts/report-embeds/registry.yaml) -# and its use in docs pages. Static checks run on PRs; URL liveness runs weekly -# and files an issue on failure (it never blocks PRs — wandb.ai rate-limits -# crawlers, so network results are flaky by design). +# Weekly liveness check of the live W&B report embeds () in docs +# pages, discovered by scanning the .mdx sources. Files an issue if a report has +# gone dead. Deliberately does not run on PRs: wandb.ai rate-limits crawlers, so +# network results are too flaky to gate merges on. on: - pull_request: - types: [opened, synchronize, reopened] - paths: - - "scripts/report-embeds/**" - - "snippets/WandbReport.jsx" - - "**/*.mdx" schedule: - cron: "17 6 * * 1" # Weekly, Monday 06:17 UTC workflow_dispatch: -concurrency: - group: report-embeds-${{ github.ref }} - cancel-in-progress: true - permissions: contents: read - issues: write # scheduled liveness files an issue on failure + issues: write # files an issue on failure jobs: - static: - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 # needed to diff for the registry-changed gate below - - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - run: pip install pyyaml - - - name: Unit tests - run: python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v - - - name: Registry + MDX consistency - run: python3 scripts/report-embeds/check_embeds.py --mode static --github - - - name: Did the registry change? - id: registry-changed - if: github.event_name == 'pull_request' - uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 - with: - files: scripts/report-embeds/registry.yaml - - - name: Liveness for registry-touching PRs - if: >- - github.event_name == 'pull_request' && - steps.registry-changed.outputs.any_changed == 'true' - run: python3 scripts/report-embeds/check_embeds.py --mode liveness --github - liveness: - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" - - - run: pip install pyyaml - - - name: Full check (static + liveness) + - name: Check embeds id: check continue-on-error: true - run: python3 scripts/report-embeds/check_embeds.py --mode all --output-md ./embed-report.md - + run: python3 scripts/report-embeds/check_embeds.py --output-md ./embed-report.md - name: Create issue from file if: steps.check.outcome == 'failure' uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6 with: - title: Fix broken or unregistered report embeds in docs.wandb.ai + title: Fix broken report embeds in docs.wandb.ai content-filepath: ./embed-report.md labels: report, automated issue diff --git a/AGENTS.md b/AGENTS.md index 975fe7a6cd..407d41611c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,7 +99,7 @@ To check or apply style, read the relevant pass file(s) and use them as guidance response="A concise example ARIA response" /> ``` -- **Live report embeds**: To show a live, interactive W&B report on a page, import `/snippets/WandbReport.jsx` and render it with `src` (the report URL), `title` (an accessible description), and optional `height` (500–800; default 640): +- **Live report embeds**: import `/snippets/WandbReport.jsx` and render it with `src` (report URL), `title` (accessible description), and optional `height` (500–800; default 640). The component renders its own "View Report" button, so no separate link is needed. ```mdx import { WandbReport } from '/snippets/WandbReport.jsx'; @@ -111,13 +111,12 @@ To check or apply style, read the relevant pass file(s) and use them as guidance /> ``` - Rules for every embed: - - **Always pair it with prose and a plain Markdown link** to the same report in the surrounding text, stating what the reader should take from it. Agents, the llms.txt export, and the translation pipeline read MDX source, where the iframe is opaque; the link is also the fallback wherever third-party frames are blocked. This is enforced by CI (`scripts/report-embeds/check_embeds.py`). - - **Register the report** in `scripts/report-embeds/registry.yaml` — see [`scripts/report-embeds/README.md`](scripts/report-embeds/README.md) for the fields and workflow. - - **The report must be viewable by anonymous visitors**: a report in a public project, or one shared via a view-only link (`Share` → "anyone with the link can view"; see [View-only report links](/models/reports/cross-project-reports#view-only-report-links)). The URL — including any `?accessToken=` — ships in public source and git history, so treat the report as public forever and never embed sensitive data. - - **Regular reports only, not Fully Connected articles.** FC articles keep their full blog chrome in a frame and look broken. Regular reports render in a slim embed view. - - **Keep it skinny and sparse**: prefer purpose-built reports (ideally a single panel grid), one or two per page maximum. Each iframe boots the full W&B app, is a fixed height with inner scroll (no auto-resize), and renders its own light theme regardless of the docs dark-mode toggle — so article-length reports show as a tall scroll region, not the whole page at once. - - English sources only — never add embeds to `ja/`, `ko/`, or `fr/` copies. + - **Regular reports only, not Fully Connected articles** — FC articles keep their full blog chrome in a frame and look broken. + - **Anonymous-viewable** — a public-project report or a [view-only link](/models/reports/cross-project-reports#view-only-report-links). The URL (with any `?accessToken=`) ships in public source and git history, so treat the report as public forever. + - **One or two per page**, skinny and purpose-built — each iframe boots the full W&B app at a fixed height, in its own light theme. + - **English sources only** — not `ja/`, `ko/`, `fr/`. + + A weekly CI job (`scripts/report-embeds/check_embeds.py`) verifies each embedded report still renders and files an issue if one breaks. It does not gate PRs. ## Working with the repository diff --git a/scripts/report-embeds/README.md b/scripts/report-embeds/README.md index e02e9aa2bf..85250047dd 100644 --- a/scripts/report-embeds/README.md +++ b/scripts/report-embeds/README.md @@ -1,70 +1,42 @@ # Report embeds -Tooling that keeps live W&B report embeds honest. A report embedded with the -[``](../../snippets/WandbReport.jsx) component is database content: -it can drift or vanish without a docs PR. This directory tracks every embed in a -registry and validates it in CI. - -## Files - -- `registry.yaml` — one entry per embedded report (id, URL, owner, purpose, - pages, last-reviewed date). Hand-curated. An empty `reports: []` is valid. -- `check_embeds.py` — the validator (schema + usage consistency + URL liveness). -- `tests/test_check_embeds.py` — unit tests (no network). - -CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflows/report-embeds.yml). - -## Add an embedded report - -1. Confirm the report is viewable by anonymous visitors — a report in a public - project, or one shared via a view-only ("magic") link - (`Share` → "anyone with the link can view"). The URL, including any - `?accessToken=`, ships in public source and git history, so treat the report - as public forever. Never embed sensitive data. -2. Add an entry to `registry.yaml`. `id` is the trailing `Vmlldzo...` token in - the report URL. -3. On the page, add both the component and a plain Markdown link to the same - report in the surrounding prose: - - ```mdx - import { WandbReport } from '/snippets/WandbReport.jsx'; - - The following [sweep report](https://wandb.ai/ENTITY/PROJECT/reports/Slug--VmlldzoXXXXXXX) - shows ... . - - - ``` - - The prose link is required: agents, the llms.txt export, and the translation - pipeline read MDX source, where the iframe is opaque. - -4. Validate locally: - - ```bash - pip install pyyaml - python3 scripts/report-embeds/check_embeds.py --mode static - ``` +`check_embeds.py` keeps live W&B report embeds honest. It finds every +[``](../../snippets/WandbReport.jsx) by scanning the English `.mdx` +sources (no registry), then checks each one: it sits on a page, has a +recognizable report URL, and still renders anonymously over the network. + +## Embed a report + +Use a **regular W&B report, not a Fully Connected article** (FC articles keep +their full blog chrome in a frame and look broken), viewable anonymously — in a +public project or shared via a view-only link (`Share` → "anyone with the link +can view"). The URL, including any `?accessToken=`, ships in public source and +git history, so treat the report as public forever. Add the component (it +renders a "View Report" button, so no separate link is needed): + +```mdx +import { WandbReport } from '/snippets/WandbReport.jsx'; + + +``` ## Run the checks ```bash -python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v -python3 scripts/report-embeds/check_embeds.py --mode static # schema + MDX, no network -python3 scripts/report-embeds/check_embeds.py --mode liveness # anonymous HTTP per URL -python3 scripts/report-embeds/check_embeds.py --mode all # both +python3 scripts/report-embeds/check_embeds.py # scan + liveness +python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' # unit tests ``` -Static errors (registry drift, missing prose link, unregistered embed) fail CI on -PRs. Liveness runs on a weekly schedule and files an issue on failure — it does -not block PRs, because `wandb.ai` rate-limits crawlers and network results are -flaky (this is also why `wandb.ai` is excluded from the site-wide lychee check). +CI ([`report-embeds.yml`](../../.github/workflows/report-embeds.yml)) runs this +weekly and files an issue if a report has gone dead. It does not run on PRs — +`wandb.ai` rate-limits crawlers, so network results are too flaky to gate merges. ## Component coupling `check_embeds.py` keys on the literal tag `WandbReport` and the `src` prop -(`COMPONENT_RE` / `SRC_ATTR_RE` at the top of the file). If -`snippets/WandbReport.jsx` renames either, update those constants. +(`COMPONENT_RE` / `SRC_ATTR_RE`). If `snippets/WandbReport.jsx` renames either, +update those constants. diff --git a/scripts/report-embeds/check_embeds.py b/scripts/report-embeds/check_embeds.py index c52907fcdd..8b687f55f6 100644 --- a/scripts/report-embeds/check_embeds.py +++ b/scripts/report-embeds/check_embeds.py @@ -1,376 +1,213 @@ #!/usr/bin/env python3 -"""Validate the W&B report-embed registry and its use in docs pages. +"""Check the live W&B report embeds () in docs pages. -Three modes: - static - registry schema + usage consistency (no network) - liveness - anonymous HTTP check that each registered URL still renders - all - both +Embeds are found by scanning the English .mdx sources (no registry): each is +checked for placement and a recognizable report URL, then fetched anonymously +to confirm the report still renders. -Dependencies: PyYAML only (HTTP uses the standard library). - -The tag name `WandbReport` and the `src` prop below are a contract with -snippets/WandbReport.jsx. If the component renames either, update COMPONENT_RE -and SRC_ATTR_RE here (see scripts/report-embeds/README.md). +COMPONENT_RE / SRC_ATTR_RE are a contract with snippets/WandbReport.jsx — keep +them in sync if the component's tag or `src` prop is renamed. """ from __future__ import annotations import argparse -import datetime import re -import sys import time import urllib.error import urllib.request from dataclasses import dataclass from pathlib import Path -from typing import Iterable, Iterator - -import yaml +from typing import Iterator REPO_ROOT = Path(__file__).resolve().parents[2] -DEFAULT_REGISTRY = Path(__file__).resolve().parent / "registry.yaml" # A report URL ends in `--` or `---` where starts with "Vmlldz". REPORT_ID_RE = re.compile(r"-{2,3}(Vmlldz[A-Za-z0-9]+)") -# Match a opening tag (self-closing or not), including -# multi-line attribute lists. `[^>]` also matches newlines. +# A opening tag (self-closing or not), across newlines. COMPONENT_RE = re.compile(r"]*?(?:/>|>)") -# Extract the src value: src="...", src='...', src={"..."}, src={'...'}, src={`...`} +# The src value: src="...", src='...', or src={"..."} / {'...'} / {`...`}. SRC_ATTR_RE = re.compile(r"""src\s*=\s*\{?\s*['"`]([^'"`]+)['"`]""") -# Fenced code blocks and MDX comments hold example/illustrative markup, not live -# embeds — mask them before scanning so a documented usage or a -# report link shown as sample code isn't treated as the real thing. FENCE_RE = re.compile(r"```.*?```", re.DOTALL) MDX_COMMENT_RE = re.compile(r"\{/\*.*?\*/\}", re.DOTALL) # Locale content is managed by a separate translation pipeline (AGENTS.md); -# the validator only governs English sources. +# only English sources are governed here. SKIP_PREFIXES = ( "ja/", "ko/", "fr/", "snippets/ja/", "snippets/ko/", "snippets/fr/", "node_modules/", ".git/", ) -# Snippets are shared partials, not pages; an embed there makes the registry's -# `pages` field ambiguous. Embeds belong on pages only. -SNIPPETS_PREFIX = "snippets/" +SNIPPETS_PREFIX = "snippets/" # embeds belong on pages, not shared partials LOGIN_MARKERS = ("/login", "/signin", "/site/login", "/authorize") -MAX_EMBEDS_PER_PAGE = 2 - @dataclass class Finding: - level: str # "error" | "warning" code: str message: str path: str | None = None line: int | None = None -# --------------------------------------------------------------------------- # -# Registry -# --------------------------------------------------------------------------- # -def load_registry(path: Path) -> tuple[list[dict], list[Finding]]: - """Parse and schema-validate the registry. Returns (entries, findings).""" - findings: list[Finding] = [] - try: - raw = yaml.safe_load(path.read_text(encoding="utf-8")) - except FileNotFoundError: - return [], [Finding("error", "REGISTRY_MISSING", f"Registry not found: {path}")] - except yaml.YAMLError as exc: - return [], [Finding("error", "REGISTRY_UNREADABLE", f"Invalid YAML in {path}: {exc}")] - - if not isinstance(raw, dict) or raw.get("reports") is None: - return [], [Finding("error", "REGISTRY_SHAPE", "Registry must have a top-level `reports:` list")] - entries = raw["reports"] - if not isinstance(entries, list): - return [], [Finding("error", "REGISTRY_SHAPE", "`reports` must be a list")] - - today = datetime.date.today() - seen_ids: set[str] = set() - valid: list[dict] = [] - - for i, entry in enumerate(entries): - loc = f"registry.yaml reports[{i}]" - if not isinstance(entry, dict): - findings.append(Finding("error", "ENTRY_SHAPE", f"{loc}: entry must be a mapping")) - continue - - rid = entry.get("id") - url = entry.get("url") - pages = entry.get("pages") - - for field in ("id", "url", "owner", "purpose", "pages", "last_reviewed"): - if entry.get(field) in (None, "", []): - findings.append(Finding("error", "MISSING_FIELD", f"{loc}: required field `{field}` is missing or empty")) - - if isinstance(rid, str): - if not re.fullmatch(r"Vmlldz[A-Za-z0-9]+", rid): - findings.append(Finding("error", "BAD_ID", f"{loc}: id `{rid}` must match Vmlldz[A-Za-z0-9]+")) - if rid in seen_ids: - findings.append(Finding("error", "DUPLICATE_ID", f"{loc}: duplicate id `{rid}`")) - seen_ids.add(rid) - - if isinstance(url, str): - if not url.startswith("https://wandb.ai/"): - findings.append(Finding("error", "BAD_URL_HOST", f"{loc}: url must start with https://wandb.ai/")) - if "/reports/" not in url: - findings.append(Finding("error", "BAD_URL_PATH", f"{loc}: url must contain /reports/")) - url_ids = REPORT_ID_RE.findall(url) - if isinstance(rid, str) and rid not in url_ids: - findings.append(Finding("error", "URL_ID_MISMATCH", f"{loc}: url must contain --{rid}")) - - if isinstance(pages, list): - for p in pages: - if not isinstance(p, str) or not p.endswith(".mdx"): - findings.append(Finding("error", "BAD_PAGE", f"{loc}: page `{p}` must be a repo-relative .mdx path")) - continue - if any(p.startswith(pre) for pre in SKIP_PREFIXES): - findings.append(Finding("error", "LOCALE_PAGE", f"{loc}: page `{p}` is a locale/excluded path; register English sources only")) - elif not (REPO_ROOT / p).is_file(): - findings.append(Finding("error", "PAGE_NOT_FOUND", f"{loc}: page `{p}` does not exist")) - - reviewed = entry.get("last_reviewed") - if reviewed is not None: - d = reviewed if isinstance(reviewed, datetime.date) else None - if d is None: - try: - d = datetime.date.fromisoformat(str(reviewed)) - except ValueError: - findings.append(Finding("error", "BAD_DATE", f"{loc}: last_reviewed `{reviewed}` is not an ISO date (YYYY-MM-DD)")) - if d is not None and d > today: - findings.append(Finding("error", "FUTURE_DATE", f"{loc}: last_reviewed `{d}` is in the future")) - - height = entry.get("height") - if height is not None and not (isinstance(height, int) and height > 0): - findings.append(Finding("error", "BAD_HEIGHT", f"{loc}: height must be a positive integer")) - - valid.append(entry) - - return valid, findings - - -# --------------------------------------------------------------------------- # -# MDX scanning -# --------------------------------------------------------------------------- # +# --- MDX scanning --- def iter_mdx(root: Path) -> Iterator[Path]: for path in sorted(root.rglob("*.mdx")): rel = path.relative_to(root).as_posix() - if any(rel.startswith(pre) for pre in SKIP_PREFIXES): - continue - yield path - - -def _line_of(text: str, index: int) -> int: - return text.count("\n", 0, index) + 1 + if not any(rel.startswith(pre) for pre in SKIP_PREFIXES): + yield path def extract_embeds(text: str) -> list[tuple[str, int]]: """Return (src, line) for every whose src can be parsed.""" - out: list[tuple[str, int]] = [] + out = [] for m in COMPONENT_RE.finditer(text): src_m = SRC_ATTR_RE.search(m.group(0)) if src_m: - out.append((src_m.group(1), _line_of(text, m.start()))) + out.append((src_m.group(1), text.count("\n", 0, m.start()) + 1)) return out -def strip_embeds(text: str) -> str: - return COMPONENT_RE.sub("", text) - - -def _blank(match: re.Match) -> str: - # Replace matched span with same-length whitespace so byte offsets — and - # therefore reported line numbers — are preserved. - return re.sub(r"[^\n]", " ", match.group(0)) - - def mask_noncontent(text: str) -> str: - return MDX_COMMENT_RE.sub(_blank, FENCE_RE.sub(_blank, text)) + # Blank fenced code + MDX comments with same-length whitespace (keeps line + # numbers stable) so example markup isn't scanned as a live embed. + def blank(m: re.Match) -> str: + return re.sub(r"[^\n]", " ", m.group(0)) + return MDX_COMMENT_RE.sub(blank, FENCE_RE.sub(blank, text)) -def check_mdx(root: Path, entries: list[dict]) -> list[Finding]: +def scan(root: Path) -> tuple[list[Finding], list[tuple[str, str, int]]]: + """Return (findings, embeds); embeds is (url, page, line) deduped by URL.""" findings: list[Finding] = [] - by_id = {e["id"]: e for e in entries if isinstance(e.get("id"), str)} - # id -> set of pages that actually embed it (English sources) - embedded_on: dict[str, set[str]] = {rid: set() for rid in by_id} - + embeds: list[tuple[str, str, int]] = [] + seen: set[str] = set() for path in iter_mdx(root): rel = path.relative_to(root).as_posix() - text = mask_noncontent(path.read_text(encoding="utf-8")) - embeds = extract_embeds(text) - if not embeds: + occurrences = extract_embeds(mask_noncontent(path.read_text(encoding="utf-8"))) + if not occurrences: continue - if rel.startswith(SNIPPETS_PREFIX): - findings.append(Finding("error", "EMBED_IN_SNIPPET", " must be used on a page, not in a shared snippet", rel, embeds[0][1])) + findings.append(Finding("EMBED_IN_SNIPPET", " must be used on a page, not a shared snippet", rel, occurrences[0][1])) continue + for src, line in occurrences: + if not REPORT_ID_RE.search(src): + findings.append(Finding("BAD_EMBED_SRC", f"src `{src}` is not a recognizable report URL (needs --Vmlldz...)", rel, line)) + elif src not in seen: + seen.add(src) + embeds.append((src, rel, line)) + return findings, embeds - if len(embeds) > MAX_EMBEDS_PER_PAGE: - findings.append(Finding("warning", "TOO_MANY_EMBEDS", f"{len(embeds)} embeds on one page; each boots the full W&B app (cap is {MAX_EMBEDS_PER_PAGE})", rel, embeds[0][1])) - prose = strip_embeds(text) - for src, line in embeds: - ids = REPORT_ID_RE.findall(src) - rid = ids[0] if ids else None - if rid is None: - findings.append(Finding("error", "BAD_EMBED_SRC", f"src `{src}` is not a recognizable report URL (needs --Vmlldz...)", rel, line)) - continue - if rid not in by_id: - findings.append(Finding("error", "EMBED_NOT_IN_REGISTRY", f"report {rid} is embedded here but absent from registry.yaml", rel, line)) - else: - embedded_on[rid].add(rel) - if rel not in (by_id[rid].get("pages") or []): - findings.append(Finding("error", "PAGE_NOT_LISTED", f"registry entry for {rid} does not list this page in `pages`", rel, line)) - # Prose-link rule: the id must appear somewhere outside the component. - if rid not in prose: - findings.append(Finding("error", "MISSING_PROSE_LINK", f"no plain link to report {rid} in the page prose (agents/llms.txt read source, where the iframe is opaque)", rel, line)) - - for rid, entry in by_id.items(): - listed = set(entry.get("pages") or []) - actual = embedded_on[rid] - if not actual: - findings.append(Finding("warning", "ORPHAN_ENTRY", f"registry entry {rid} is not embedded by any English page")) - for stale in sorted(listed - actual): - findings.append(Finding("error", "STALE_PAGES_FIELD", f"registry entry {rid} lists `{stale}` but that page has no matching embed")) - - return findings - - -# --------------------------------------------------------------------------- # -# Liveness -# --------------------------------------------------------------------------- # +# --- Liveness --- def check_url(url: str, *, timeout: int, attempts: int, delay: float) -> Finding | None: """Anonymous GET; require a final 200 that is not a login redirect.""" - req = urllib.request.Request( - url, - headers={ - "User-Agent": "Mozilla/5.0 (compatible; wandb-docs-embed-check/1.0)", - "Accept": "text/html,application/xhtml+xml", - }, - ) + req = urllib.request.Request(url, headers={ + "User-Agent": "Mozilla/5.0 (compatible; wandb-docs-embed-check/1.0)", + "Accept": "text/html,application/xhtml+xml", + }) last_err = "" for attempt in range(1, attempts + 1): try: with urllib.request.urlopen(req, timeout=timeout) as resp: - final_url = resp.geturl() - status = resp.getcode() - if status != 200: - return Finding("error", "URL_NOT_200", f"{url}: final status {status}") - if any(marker in final_url for marker in LOGIN_MARKERS): - return Finding("error", "URL_LOGIN_REDIRECT", f"{url}: redirected to login ({final_url}) — magic link likely revoked") + if resp.getcode() != 200: + return Finding("URL_NOT_200", f"{url}: final status {resp.getcode()}") + if any(mk in resp.geturl() for mk in LOGIN_MARKERS): + return Finding("URL_LOGIN_REDIRECT", f"{url}: redirected to login ({resp.geturl()}) — link likely revoked") return None except urllib.error.HTTPError as exc: if exc.code in (429, 500, 502, 503, 504) and attempt < attempts: last_err = f"HTTP {exc.code}" time.sleep(delay * (2 ** attempt)) continue - return Finding("error", "URL_NOT_200", f"{url}: HTTP {exc.code}") + return Finding("URL_NOT_200", f"{url}: HTTP {exc.code}") except (urllib.error.URLError, TimeoutError, OSError) as exc: last_err = str(getattr(exc, "reason", exc)) if attempt < attempts: time.sleep(delay * (2 ** attempt)) continue - return Finding("error", "URL_UNREACHABLE", f"{url}: unreachable after {attempts} attempts ({last_err})") + return Finding("URL_UNREACHABLE", f"{url}: unreachable after {attempts} attempts ({last_err})") -def check_liveness(entries: list[dict], *, timeout: int = 30, attempts: int = 3, delay: float = 1.0) -> list[Finding]: +def check_liveness(embeds: list[tuple[str, str, int]], *, timeout: int = 30, attempts: int = 3, delay: float = 1.0) -> list[Finding]: findings: list[Finding] = [] - for entry in entries: - url = entry.get("url") - if not isinstance(url, str): - continue + for url, rel, line in embeds: finding = check_url(url, timeout=timeout, attempts=attempts, delay=delay) if finding: + finding.path, finding.line = rel, line findings.append(finding) - time.sleep(delay) # be polite; registry is small + time.sleep(delay) # be polite; the embed set is small return findings -# --------------------------------------------------------------------------- # -# Output -# --------------------------------------------------------------------------- # -def emit_github_annotations(findings: Iterable[Finding]) -> None: - for f in findings: - kind = "error" if f.level == "error" else "warning" - loc = "" - if f.path: - loc = f"file={f.path}" - if f.line: - loc += f",line={f.line}" - print(f"::{kind} {loc}::[{f.code}] {f.message}") - - -def render_markdown(findings: list[Finding], *, checked_urls: int, embeds: int) -> str: - errors = [f for f in findings if f.level == "error"] - warnings = [f for f in findings if f.level == "warning"] - lines = ["# Report embed check", ""] +# --- Output + CLI --- +def _loc(f: Finding) -> str: + return (f.path or "-") + (f":{f.line}" if f.path and f.line else "") + + +def render_markdown(findings: list[Finding], *, embeds: int) -> str: if not findings: - lines.append(f"✅ All checks passed ({embeds} embed(s), {checked_urls} URL(s) checked).") - return "\n".join(lines) + "\n" - lines.append(f"Found {len(errors)} error(s) and {len(warnings)} warning(s).") - lines.append("") - lines.append("| Level | Code | Location | Detail |") - lines.append("| --- | --- | --- | --- |") - for f in errors + warnings: - where = f.path or "registry.yaml" - if f.line: - where += f":{f.line}" - lines.append(f"| {f.level} | {f.code} | {where} | {f.message} |") + return f"# Report embed check\n\n✅ All {embeds} embed(s) render anonymously.\n" + lines = ["# Report embed check", "", f"Found {len(findings)} problem(s).", "", "| Code | Location | Detail |", "| --- | --- | --- |"] + lines += [f"| {f.code} | {_loc(f)} | {f.message} |" for f in findings] return "\n".join(lines) + "\n" -# --------------------------------------------------------------------------- # -# Main -# --------------------------------------------------------------------------- # def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--mode", choices=["static", "liveness", "all"], default="static") - parser.add_argument("--registry", type=Path, default=DEFAULT_REGISTRY) parser.add_argument("--root", type=Path, default=REPO_ROOT) - parser.add_argument("--github", action="store_true", help="emit GitHub Actions annotations") parser.add_argument("--output-md", type=Path, help="write a Markdown report to this file") - parser.add_argument("--strict", action="store_true", help="treat warnings as failures") args = parser.parse_args(argv) - entries, findings = load_registry(args.registry) - if any(f.code in ("REGISTRY_MISSING", "REGISTRY_UNREADABLE", "REGISTRY_SHAPE") for f in findings): - for f in findings: - print(f"ERROR [{f.code}] {f.message}", file=sys.stderr) - return 2 - - embeds_seen = 0 - if args.mode in ("static", "all"): - findings += check_mdx(args.root, entries) - if args.mode in ("liveness", "all"): - findings += check_liveness(entries) - - # Count embeds for the summary line (cheap re-scan of registry pages only). - embeds_seen = sum(len(e.get("pages") or []) for e in entries) - checked_urls = len(entries) if args.mode in ("liveness", "all") else 0 - - errors = [f for f in findings if f.level == "error"] - warnings = [f for f in findings if f.level == "warning"] - - if args.github: - emit_github_annotations(findings) - else: - for f in errors + warnings: - where = f" {f.path}:{f.line}" if f.path and f.line else (f" {f.path}" if f.path else "") - print(f"{f.level.upper()} [{f.code}]{where}: {f.message}") + findings, embeds = scan(args.root) + findings += check_liveness(embeds) + for f in findings: + print(f"ERROR [{f.code}] {_loc(f)}: {f.message}") if args.output_md: - args.output_md.write_text(render_markdown(findings, checked_urls=checked_urls, embeds=embeds_seen), encoding="utf-8") - + args.output_md.write_text(render_markdown(findings, embeds=len(embeds)), encoding="utf-8") if not findings: - print(f"OK: {len(entries)} registry entrie(s), {embeds_seen} embed page reference(s).") - - if errors: - return 1 - if warnings and args.strict: - return 1 - return 0 + print(f"OK: {len(embeds)} report embed(s) checked.") + return 1 if findings else 0 + + +# --- Tests — python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' --- +import tempfile +import unittest + + +class Tests(unittest.TestCase): + def _scan(self, files: dict[str, str]): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + for rel, content in files.items(): + fp = root / rel + fp.parent.mkdir(parents=True, exist_ok=True) + fp.write_text(content, encoding="utf-8") + return scan(root) + + def test_extract(self): + self.assertEqual(extract_embeds(''), [("x--Vmlldzo1", 1)]) + self.assertEqual(extract_embeds('a\n\n'), [("y---Vmlldzo2", 2)]) + + def test_valid_and_dedup(self): + src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" + embed = f'\n' + findings, embeds = self._scan({"models/a.mdx": embed, "models/b.mdx": embed}) + self.assertEqual(findings, []) + self.assertEqual(embeds, [(src, "models/a.mdx", 1)]) # deduped by URL + + def test_bad_src(self): + findings, embeds = self._scan({"models/x.mdx": '\n'}) + self.assertEqual([f.code for f in findings], ["BAD_EMBED_SRC"]) + self.assertEqual(embeds, []) + + def test_snippet_errors_and_masked_ignored(self): + src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" + embed = f'\n' + snippet, _ = self._scan({"snippets/_includes/t.mdx": embed}) + self.assertEqual([f.code for f in snippet], ["EMBED_IN_SNIPPET"]) + for masked in (f"```mdx\n{embed}```\n", "{/* " + embed + "*/}\n"): + self.assertEqual(self._scan({"models/x.mdx": masked}), ([], [])) if __name__ == "__main__": diff --git a/scripts/report-embeds/registry.yaml b/scripts/report-embeds/registry.yaml deleted file mode 100644 index 9ebd03e0a0..0000000000 --- a/scripts/report-embeds/registry.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# --------------------------------------------------------------------------- -# Registry of W&B Reports embedded in docs pages via . -# --------------------------------------------------------------------------- -# Every in an English .mdx page MUST have an entry -# here, and the page MUST also link the report URL in prose (so agents, -# llms.txt consumers, and the translation pipeline — which read MDX source, -# where the iframe is opaque — still get the link). -# -# Validated by scripts/report-embeds/check_embeds.py (CI: report-embeds.yml). -# An empty `reports: []` is a valid, passing state. -# -# To add an embedded report: -# 1. Add an entry below. `id` is the trailing --Vmlldzo... token in the URL. -# 2. On the page, add AND a plain Markdown link to -# the same report URL in the surrounding prose. -# 3. Run: python3 scripts/report-embeds/check_embeds.py --mode static -# -# Only reports viewable by anonymous visitors work embedded: a report in a -# public project, or one shared via a view-only ("magic") link. The URL — -# including any ?accessToken= — ships in public source and git history, so -# treat every embedded report as public forever. Never embed sensitive data. -# --------------------------------------------------------------------------- - -reports: - - id: Vmlldzo0NjE4MzM - url: https://wandb.ai/stacey/saferlife/reports/SafeLife-Benchmark-Experiments--Vmlldzo0NjE4MzM - owner: "@jmulhausen" - purpose: "Public demo (stacey/saferlife): live benchmark report for the Reports overview" - pages: - - models/reports.mdx - last_reviewed: 2026-07-21 - height: 720 - - - id: Vmlldzo2MzA5Mg - url: https://wandb.ai/stacey/lyft/reports/LIDAR-Point-Clouds-of-Driving-Scenes--Vmlldzo2MzA5Mg - owner: "@jmulhausen" - purpose: "Public demo (stacey/lyft): interactive 3D LIDAR point clouds for the Log media page" - pages: - - models/track/log/media.mdx - last_reviewed: 2026-07-21 - height: 720 diff --git a/scripts/report-embeds/tests/test_check_embeds.py b/scripts/report-embeds/tests/test_check_embeds.py deleted file mode 100644 index eda475c941..0000000000 --- a/scripts/report-embeds/tests/test_check_embeds.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Unit tests for the report-embed validator. No network.""" - -from __future__ import annotations - -import datetime -import importlib.util -import sys -import tempfile -import unittest -from pathlib import Path - -MODULE_PATH = Path(__file__).resolve().parents[1] / "check_embeds.py" - - -def _load(): - spec = importlib.util.spec_from_file_location("check_embeds", MODULE_PATH) - mod = importlib.util.module_from_spec(spec) - assert spec.loader - # Register before exec so dataclass type introspection can resolve the - # module (required on Python 3.9; harmless on newer versions). - sys.modules[spec.name] = mod - spec.loader.exec_module(mod) - return mod - - -m = _load() - -TODAY = datetime.date.today().isoformat() - - -def _entry(**over): - base = { - "id": "Vmlldzo12345", - "url": "https://wandb.ai/wandb/docs-embeds/reports/Foo--Vmlldzo12345", - "owner": "@me", - "purpose": "demo", - "pages": ["models/sweeps/visualize-sweep-results.mdx"], - "last_reviewed": TODAY, - "height": 700, - } - base.update(over) - return base - - -class RegistrySchemaTests(unittest.TestCase): - def _load_yaml(self, doc: str): - with tempfile.TemporaryDirectory() as d: - p = Path(d) / "registry.yaml" - p.write_text(doc, encoding="utf-8") - return m.load_registry(p) - - def test_empty_registry_is_valid(self): - _, findings = self._load_yaml("reports: []\n") - self.assertEqual(findings, []) - - def test_missing_reports_key_is_shape_error(self): - _, findings = self._load_yaml("something: else\n") - self.assertTrue(any(f.code == "REGISTRY_SHAPE" for f in findings)) - - def test_top_level_list_is_shape_error_not_crash(self): - # A top-level list/string must yield REGISTRY_SHAPE, not AttributeError. - _, findings = self._load_yaml("- a\n- b\n") - self.assertTrue(any(f.code == "REGISTRY_SHAPE" for f in findings)) - _, findings = self._load_yaml("just a string\n") - self.assertTrue(any(f.code == "REGISTRY_SHAPE" for f in findings)) - - def test_bad_id_pattern(self): - import yaml - doc = yaml.safe_dump({"reports": [_entry(id="notanid", url="https://wandb.ai/w/p/reports/Foo--notanid")]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "BAD_ID" for f in findings)) - - def test_url_id_mismatch(self): - import yaml - doc = yaml.safe_dump({"reports": [_entry(url="https://wandb.ai/w/p/reports/Foo--Vmlldzo99999")]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "URL_ID_MISMATCH" for f in findings)) - - def test_missing_required_field(self): - import yaml - e = _entry() - del e["owner"] - doc = yaml.safe_dump({"reports": [e]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "MISSING_FIELD" for f in findings)) - - def test_future_date(self): - import yaml - future = (datetime.date.today() + datetime.timedelta(days=5)).isoformat() - doc = yaml.safe_dump({"reports": [_entry(last_reviewed=future)]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "FUTURE_DATE" for f in findings)) - - def test_bad_host(self): - import yaml - doc = yaml.safe_dump({"reports": [_entry(url="https://evil.example/reports/Foo--Vmlldzo12345")]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "BAD_URL_HOST" for f in findings)) - - -class ExtractionTests(unittest.TestCase): - def test_self_closing(self): - text = '' - self.assertEqual(m.extract_embeds(text), [("https://wandb.ai/w/p/reports/Foo--Vmlldzo1", 1)]) - - def test_multiline_and_brace_src(self): - text = ( - "intro\n" - "\n" - ) - embeds = m.extract_embeds(text) - self.assertEqual(len(embeds), 1) - self.assertEqual(embeds[0][0], "https://wandb.ai/w/p/reports/Foo---Vmlldzo2") - self.assertEqual(embeds[0][1], 2) - - def test_strip_removes_component(self): - text = 'before after' - self.assertNotIn("Vmlldzo3", m.strip_embeds(text)) - self.assertIn("before", m.strip_embeds(text)) - - -class MdxConsistencyTests(unittest.TestCase): - def _run(self, files: dict[str, str], entries: list[dict]): - with tempfile.TemporaryDirectory() as d: - root = Path(d) - for rel, content in files.items(): - fp = root / rel - fp.parent.mkdir(parents=True, exist_ok=True) - fp.write_text(content, encoding="utf-8") - return m.check_mdx(root, entries) - - def test_valid_embed_with_prose_link_passes(self): - page = ( - "See the [sweep report](https://wandb.ai/w/p/reports/Foo--Vmlldzo12345).\n\n" - '\n' - ) - findings = self._run({"models/sweeps/visualize-sweep-results.mdx": page}, [_entry()]) - self.assertEqual([f for f in findings if f.level == "error"], []) - - def test_missing_prose_link(self): - page = '\n' - findings = self._run({"models/sweeps/visualize-sweep-results.mdx": page}, [_entry()]) - self.assertTrue(any(f.code == "MISSING_PROSE_LINK" for f in findings)) - - def test_unregistered_embed(self): - page = ( - "[link](https://wandb.ai/w/p/reports/Foo--Vmlldzo99999)\n" - '\n' - ) - findings = self._run({"models/sweeps/visualize-sweep-results.mdx": page}, []) - self.assertTrue(any(f.code == "EMBED_NOT_IN_REGISTRY" for f in findings)) - - def test_orphan_entry_warns(self): - findings = self._run({"models/sweeps/visualize-sweep-results.mdx": "no embeds here\n"}, [_entry()]) - self.assertTrue(any(f.code == "ORPHAN_ENTRY" and f.level == "warning" for f in findings)) - - def test_too_many_embeds_warns(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - link = f"[r]({src})\n" - page = link + "".join(f'\n' for i in range(3)) - entry = _entry(pages=["models/x.mdx"]) - findings = self._run({"models/x.mdx": page}, [entry]) - self.assertTrue(any(f.code == "TOO_MANY_EMBEDS" and f.level == "warning" for f in findings)) - - def test_locale_files_ignored(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = f'\n' # no prose link — would error if scanned - # Same embed on the English page (valid) plus a locale copy (must be skipped). - english = f"[r]({src})\n" + page - findings = self._run( - {"models/x.mdx": english, "ja/models/x.mdx": page}, - [_entry(pages=["models/x.mdx"])], - ) - self.assertEqual([f for f in findings if f.level == "error"], []) - - def test_embed_in_code_fence_ignored(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = "Usage example:\n\n```mdx\n" + f'\n' + "```\n" - findings = self._run({"models/x.mdx": page}, []) - self.assertEqual(findings, []) - - def test_embed_in_mdx_comment_ignored(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = "{/* staged, fill in real URL later:\n" + f'\n' + "*/}\n" - findings = self._run({"models/x.mdx": page}, []) - self.assertEqual(findings, []) - - def test_embed_in_snippet_errors(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - findings = self._run( - {"snippets/_includes/thing.mdx": f"[r]({src})\n\n"}, - [_entry(pages=["snippets/_includes/thing.mdx"])], - ) - self.assertTrue(any(f.code == "EMBED_IN_SNIPPET" for f in findings)) - - -if __name__ == "__main__": - unittest.main() diff --git a/snippets/WandbReport.jsx b/snippets/WandbReport.jsx index 066e0004df..178198faec 100644 --- a/snippets/WandbReport.jsx +++ b/snippets/WandbReport.jsx @@ -3,7 +3,7 @@ export const WandbReport = ({ src, title, height = 640 }) => { // (sometimes three hyphens before the id), optionally with a ?accessToken=... // query when shared via a view-only ("magic") link. Only report pages allow // framing — project and run pages block it — so reject anything else and fall - // back to the link-only card rather than rendering a frame that won't load. + // back to the button-only card rather than rendering a frame that won't load. const isReportUrl = typeof src === 'string' && /^https:\/\/wandb\.ai\/[^/]+\/[^/]+\/reports\/.*-{2,3}Vmlldz[A-Za-z0-9]+/.test(src); @@ -27,8 +27,8 @@ export const WandbReport = ({ src, title, height = 640 }) => { // regular report renders in a slim embed view. A Fully Connected article // ignores the param and keeps its full blog chrome (nav bar, breadcrumb, // author, stars), so it looks broken in a frame — embed regular reports, not - // FC articles. The human-facing caption link below omits the param and points - // at the normal report URL. + // FC articles. The View Report button below omits the param and points at the + // normal report URL. const frameSrc = isReportUrl ? `${src}${src.includes('?') ? '&' : '?'}jupyter=true` : src; // Colors come from the CSS custom properties in scripts/css-minify/colors.css @@ -57,19 +57,37 @@ export const WandbReport = ({ src, title, height = 640 }) => { )}
+ {/* "View Report" button matching the in-content button-links + (.colab-link / .try-product-link in scripts/css-minify/buttons.css), + styled inline so the component stays self-contained. It links to the + plain report URL (no ?jupyter=true), so it doubles as the fallback + wherever the frame can't load. */} - View this report in the W&B app → + View Report