From e157d4895712a79b4e010351499ba79bf96a05bd Mon Sep 17 00:00:00 2001 From: John Mulhausen Date: Wed, 22 Jul 2026 12:58:19 -0400 Subject: [PATCH 1/7] report-embeds: drop the registry, scan MDX for embeds instead check_embeds.py already discovers every by scanning the .mdx sources, so registry.yaml was a redundant second place to maintain the same list. Remove it and the registry-validation logic. - Delete scripts/report-embeds/registry.yaml. - check_embeds.py: discover embeds by scanning MDX; keep the sibling prose-link, per-page-cap, and snippet checks (static) plus URL liveness on the discovered URLs. Drop all registry loading/schema logic. No more PyYAML dependency (standard library only). - report-embeds.yml: static (prose link) on PRs; liveness weekly, filing an issue on failure. No registry-change gating, no pip install. - Update tests, README, and the AGENTS.md rule accordingly. Co-Authored-By: Claude Fable 5 --- .github/workflows/report-embeds.yml | 44 +--- AGENTS.md | 3 +- scripts/report-embeds/README.md | 37 ++-- scripts/report-embeds/check_embeds.py | 190 +++++------------- scripts/report-embeds/registry.yaml | 41 ---- .../report-embeds/tests/test_check_embeds.py | 134 +++--------- 6 files changed, 110 insertions(+), 339 deletions(-) delete mode 100644 scripts/report-embeds/registry.yaml diff --git a/.github/workflows/report-embeds.yml b/.github/workflows/report-embeds.yml index 5d9c46446c..1e6e94fe86 100644 --- a/.github/workflows/report-embeds.yml +++ b/.github/workflows/report-embeds.yml @@ -1,9 +1,12 @@ name: Report embeds -# Validates the W&B report-embed registry (scripts/report-embeds/registry.yaml) -# and its use in docs pages. Static checks run on PRs; URL liveness runs weekly -# and files an issue on failure (it never blocks PRs — wandb.ai rate-limits -# crawlers, so network results are flaky by design). +# Checks live W&B report embeds () in docs pages. Embeds are +# discovered by scanning the .mdx sources — there is no registry. +# - On PRs: static check (every embed has a sibling prose link, sits on a page, +# and stays under the per-page cap). +# - Weekly: URL liveness, filing an issue on failure. Liveness never blocks PRs — +# wandb.ai rate-limits crawlers, so network results are flaky by design (this +# is also why wandb.ai is excluded from the site-wide lychee check). on: pull_request: @@ -30,55 +33,30 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 # needed to diff for the registry-changed gate below - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" - - - run: pip install pyyaml - - name: Unit tests run: python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v - - - name: Registry + MDX consistency + - name: Embed consistency (prose link, per-page cap) run: python3 scripts/report-embeds/check_embeds.py --mode static --github - - name: Did the registry change? - id: registry-changed - if: github.event_name == 'pull_request' - uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 - with: - files: scripts/report-embeds/registry.yaml - - - name: Liveness for registry-touching PRs - if: >- - github.event_name == 'pull_request' && - steps.registry-changed.outputs.any_changed == 'true' - run: python3 scripts/report-embeds/check_embeds.py --mode liveness --github - liveness: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" - - - run: pip install pyyaml - - - name: Full check (static + liveness) + - name: Liveness check id: check continue-on-error: true - run: python3 scripts/report-embeds/check_embeds.py --mode all --output-md ./embed-report.md - + run: python3 scripts/report-embeds/check_embeds.py --mode liveness --output-md ./embed-report.md - name: Create issue from file if: steps.check.outcome == 'failure' uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6 with: - title: Fix broken or unregistered report embeds in docs.wandb.ai + title: Fix broken report embeds in docs.wandb.ai content-filepath: ./embed-report.md labels: report, automated issue diff --git a/AGENTS.md b/AGENTS.md index 975fe7a6cd..91cabac242 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,8 +112,7 @@ To check or apply style, read the relevant pass file(s) and use them as guidance ``` Rules for every embed: - - **Always pair it with prose and a plain Markdown link** to the same report in the surrounding text, stating what the reader should take from it. Agents, the llms.txt export, and the translation pipeline read MDX source, where the iframe is opaque; the link is also the fallback wherever third-party frames are blocked. This is enforced by CI (`scripts/report-embeds/check_embeds.py`). - - **Register the report** in `scripts/report-embeds/registry.yaml` — see [`scripts/report-embeds/README.md`](scripts/report-embeds/README.md) for the fields and workflow. + - **Always pair it with prose and a plain Markdown link** to the same report in the surrounding text, stating what the reader should take from it. Agents, the llms.txt export, and the translation pipeline read MDX source, where the iframe is opaque; the link is also the fallback wherever third-party frames are blocked. This is enforced by CI (`scripts/report-embeds/check_embeds.py`, which discovers embeds by scanning the `.mdx` sources — see [`scripts/report-embeds/README.md`](scripts/report-embeds/README.md)). - **The report must be viewable by anonymous visitors**: a report in a public project, or one shared via a view-only link (`Share` → "anyone with the link can view"; see [View-only report links](/models/reports/cross-project-reports#view-only-report-links)). The URL — including any `?accessToken=` — ships in public source and git history, so treat the report as public forever and never embed sensitive data. - **Regular reports only, not Fully Connected articles.** FC articles keep their full blog chrome in a frame and look broken. Regular reports render in a slim embed view. - **Keep it skinny and sparse**: prefer purpose-built reports (ideally a single panel grid), one or two per page maximum. Each iframe boots the full W&B app, is a fixed height with inner scroll (no auto-resize), and renders its own light theme regardless of the docs dark-mode toggle — so article-length reports show as a tall scroll region, not the whole page at once. diff --git a/scripts/report-embeds/README.md b/scripts/report-embeds/README.md index e02e9aa2bf..fc6b7f81c1 100644 --- a/scripts/report-embeds/README.md +++ b/scripts/report-embeds/README.md @@ -2,28 +2,26 @@ Tooling that keeps live W&B report embeds honest. A report embedded with the [``](../../snippets/WandbReport.jsx) component is database content: -it can drift or vanish without a docs PR. This directory tracks every embed in a -registry and validates it in CI. +it can drift or vanish without a docs PR. `check_embeds.py` finds every embed by +scanning the `.mdx` sources (no registry to maintain) and checks it. ## Files -- `registry.yaml` — one entry per embedded report (id, URL, owner, purpose, - pages, last-reviewed date). Hand-curated. An empty `reports: []` is valid. -- `check_embeds.py` — the validator (schema + usage consistency + URL liveness). +- `check_embeds.py` — scans English `.mdx` for `` embeds and checks + them (sibling prose link + per-page cap + URL liveness). - `tests/test_check_embeds.py` — unit tests (no network). CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflows/report-embeds.yml). ## Add an embedded report -1. Confirm the report is viewable by anonymous visitors — a report in a public - project, or one shared via a view-only ("magic") link - (`Share` → "anyone with the link can view"). The URL, including any - `?accessToken=`, ships in public source and git history, so treat the report - as public forever. Never embed sensitive data. -2. Add an entry to `registry.yaml`. `id` is the trailing `Vmlldzo...` token in - the report URL. -3. On the page, add both the component and a plain Markdown link to the same +1. Confirm the report is a **regular W&B report, not a Fully Connected article** + (FC articles keep their full blog chrome in a frame and look broken), and that + it is viewable by anonymous visitors — a report in a public project, or one + shared via a view-only ("magic") link (`Share` → "anyone with the link can + view"). The URL, including any `?accessToken=`, ships in public source and git + history, so treat the report as public forever. Never embed sensitive data. +2. On the page, add both the component and a plain Markdown link to the same report in the surrounding prose: ```mdx @@ -42,10 +40,9 @@ CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflo The prose link is required: agents, the llms.txt export, and the translation pipeline read MDX source, where the iframe is opaque. -4. Validate locally: +3. Validate locally: ```bash - pip install pyyaml python3 scripts/report-embeds/check_embeds.py --mode static ``` @@ -53,15 +50,15 @@ CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflo ```bash python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v -python3 scripts/report-embeds/check_embeds.py --mode static # schema + MDX, no network +python3 scripts/report-embeds/check_embeds.py --mode static # prose link + cap, no network python3 scripts/report-embeds/check_embeds.py --mode liveness # anonymous HTTP per URL python3 scripts/report-embeds/check_embeds.py --mode all # both ``` -Static errors (registry drift, missing prose link, unregistered embed) fail CI on -PRs. Liveness runs on a weekly schedule and files an issue on failure — it does -not block PRs, because `wandb.ai` rate-limits crawlers and network results are -flaky (this is also why `wandb.ai` is excluded from the site-wide lychee check). +Static errors (missing prose link, embed in a shared snippet) fail CI on PRs. +Liveness runs on a weekly schedule and files an issue on failure — it does not +block PRs, because `wandb.ai` rate-limits crawlers and network results are flaky +(this is also why `wandb.ai` is excluded from the site-wide lychee check). ## Component coupling diff --git a/scripts/report-embeds/check_embeds.py b/scripts/report-embeds/check_embeds.py index c52907fcdd..395dbfa58c 100644 --- a/scripts/report-embeds/check_embeds.py +++ b/scripts/report-embeds/check_embeds.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 -"""Validate the W&B report-embed registry and its use in docs pages. +"""Check W&B report embeds used in docs pages. + +Embeds are discovered by scanning the English `.mdx` sources for the +`` component — there is no registry to keep in sync. Three modes: - static - registry schema + usage consistency (no network) - liveness - anonymous HTTP check that each registered URL still renders + static - every embed has a sibling prose link, sits on a page (not a shared + snippet), and pages stay under the per-page cap (no network) + liveness - anonymous HTTP check that each embedded report URL still renders all - both -Dependencies: PyYAML only (HTTP uses the standard library). +No third-party dependencies (standard library only). The tag name `WandbReport` and the `src` prop below are a contract with snippets/WandbReport.jsx. If the component renames either, update COMPONENT_RE @@ -16,7 +20,6 @@ from __future__ import annotations import argparse -import datetime import re import sys import time @@ -26,10 +29,7 @@ from pathlib import Path from typing import Iterable, Iterator -import yaml - REPO_ROOT = Path(__file__).resolve().parents[2] -DEFAULT_REGISTRY = Path(__file__).resolve().parent / "registry.yaml" # A report URL ends in `--` or `---` where starts with "Vmlldz". REPORT_ID_RE = re.compile(r"-{2,3}(Vmlldz[A-Za-z0-9]+)") @@ -45,14 +45,13 @@ MDX_COMMENT_RE = re.compile(r"\{/\*.*?\*/\}", re.DOTALL) # Locale content is managed by a separate translation pipeline (AGENTS.md); -# the validator only governs English sources. +# the checker only governs English sources. SKIP_PREFIXES = ( "ja/", "ko/", "fr/", "snippets/ja/", "snippets/ko/", "snippets/fr/", "node_modules/", ".git/", ) -# Snippets are shared partials, not pages; an embed there makes the registry's -# `pages` field ambiguous. Embeds belong on pages only. +# Snippets are shared partials, not pages. Embeds belong on pages only. SNIPPETS_PREFIX = "snippets/" LOGIN_MARKERS = ("/login", "/signin", "/site/login", "/authorize") @@ -68,87 +67,11 @@ class Finding: line: int | None = None -# --------------------------------------------------------------------------- # -# Registry -# --------------------------------------------------------------------------- # -def load_registry(path: Path) -> tuple[list[dict], list[Finding]]: - """Parse and schema-validate the registry. Returns (entries, findings).""" - findings: list[Finding] = [] - try: - raw = yaml.safe_load(path.read_text(encoding="utf-8")) - except FileNotFoundError: - return [], [Finding("error", "REGISTRY_MISSING", f"Registry not found: {path}")] - except yaml.YAMLError as exc: - return [], [Finding("error", "REGISTRY_UNREADABLE", f"Invalid YAML in {path}: {exc}")] - - if not isinstance(raw, dict) or raw.get("reports") is None: - return [], [Finding("error", "REGISTRY_SHAPE", "Registry must have a top-level `reports:` list")] - entries = raw["reports"] - if not isinstance(entries, list): - return [], [Finding("error", "REGISTRY_SHAPE", "`reports` must be a list")] - - today = datetime.date.today() - seen_ids: set[str] = set() - valid: list[dict] = [] - - for i, entry in enumerate(entries): - loc = f"registry.yaml reports[{i}]" - if not isinstance(entry, dict): - findings.append(Finding("error", "ENTRY_SHAPE", f"{loc}: entry must be a mapping")) - continue - - rid = entry.get("id") - url = entry.get("url") - pages = entry.get("pages") - - for field in ("id", "url", "owner", "purpose", "pages", "last_reviewed"): - if entry.get(field) in (None, "", []): - findings.append(Finding("error", "MISSING_FIELD", f"{loc}: required field `{field}` is missing or empty")) - - if isinstance(rid, str): - if not re.fullmatch(r"Vmlldz[A-Za-z0-9]+", rid): - findings.append(Finding("error", "BAD_ID", f"{loc}: id `{rid}` must match Vmlldz[A-Za-z0-9]+")) - if rid in seen_ids: - findings.append(Finding("error", "DUPLICATE_ID", f"{loc}: duplicate id `{rid}`")) - seen_ids.add(rid) - - if isinstance(url, str): - if not url.startswith("https://wandb.ai/"): - findings.append(Finding("error", "BAD_URL_HOST", f"{loc}: url must start with https://wandb.ai/")) - if "/reports/" not in url: - findings.append(Finding("error", "BAD_URL_PATH", f"{loc}: url must contain /reports/")) - url_ids = REPORT_ID_RE.findall(url) - if isinstance(rid, str) and rid not in url_ids: - findings.append(Finding("error", "URL_ID_MISMATCH", f"{loc}: url must contain --{rid}")) - - if isinstance(pages, list): - for p in pages: - if not isinstance(p, str) or not p.endswith(".mdx"): - findings.append(Finding("error", "BAD_PAGE", f"{loc}: page `{p}` must be a repo-relative .mdx path")) - continue - if any(p.startswith(pre) for pre in SKIP_PREFIXES): - findings.append(Finding("error", "LOCALE_PAGE", f"{loc}: page `{p}` is a locale/excluded path; register English sources only")) - elif not (REPO_ROOT / p).is_file(): - findings.append(Finding("error", "PAGE_NOT_FOUND", f"{loc}: page `{p}` does not exist")) - - reviewed = entry.get("last_reviewed") - if reviewed is not None: - d = reviewed if isinstance(reviewed, datetime.date) else None - if d is None: - try: - d = datetime.date.fromisoformat(str(reviewed)) - except ValueError: - findings.append(Finding("error", "BAD_DATE", f"{loc}: last_reviewed `{reviewed}` is not an ISO date (YYYY-MM-DD)")) - if d is not None and d > today: - findings.append(Finding("error", "FUTURE_DATE", f"{loc}: last_reviewed `{d}` is in the future")) - - height = entry.get("height") - if height is not None and not (isinstance(height, int) and height > 0): - findings.append(Finding("error", "BAD_HEIGHT", f"{loc}: height must be a positive integer")) - - valid.append(entry) - - return valid, findings +@dataclass +class Embed: + url: str + path: str # repo-relative page that embeds it + line: int # --------------------------------------------------------------------------- # @@ -190,52 +113,45 @@ def mask_noncontent(text: str) -> str: return MDX_COMMENT_RE.sub(_blank, FENCE_RE.sub(_blank, text)) -def check_mdx(root: Path, entries: list[dict]) -> list[Finding]: +def scan(root: Path) -> tuple[list[Finding], list[Embed]]: + """Scan English .mdx for embeds. + + Returns (static findings, embeds). `embeds` is deduplicated by URL for the + liveness pass; the static findings cover every occurrence. + """ findings: list[Finding] = [] - by_id = {e["id"]: e for e in entries if isinstance(e.get("id"), str)} - # id -> set of pages that actually embed it (English sources) - embedded_on: dict[str, set[str]] = {rid: set() for rid in by_id} + embeds: list[Embed] = [] + seen_urls: set[str] = set() for path in iter_mdx(root): rel = path.relative_to(root).as_posix() text = mask_noncontent(path.read_text(encoding="utf-8")) - embeds = extract_embeds(text) - if not embeds: + occurrences = extract_embeds(text) + if not occurrences: continue if rel.startswith(SNIPPETS_PREFIX): - findings.append(Finding("error", "EMBED_IN_SNIPPET", " must be used on a page, not in a shared snippet", rel, embeds[0][1])) + findings.append(Finding("error", "EMBED_IN_SNIPPET", " must be used on a page, not in a shared snippet", rel, occurrences[0][1])) continue - if len(embeds) > MAX_EMBEDS_PER_PAGE: - findings.append(Finding("warning", "TOO_MANY_EMBEDS", f"{len(embeds)} embeds on one page; each boots the full W&B app (cap is {MAX_EMBEDS_PER_PAGE})", rel, embeds[0][1])) + if len(occurrences) > MAX_EMBEDS_PER_PAGE: + findings.append(Finding("warning", "TOO_MANY_EMBEDS", f"{len(occurrences)} embeds on one page; each boots the full W&B app (cap is {MAX_EMBEDS_PER_PAGE})", rel, occurrences[0][1])) prose = strip_embeds(text) - for src, line in embeds: + for src, line in occurrences: ids = REPORT_ID_RE.findall(src) rid = ids[0] if ids else None if rid is None: findings.append(Finding("error", "BAD_EMBED_SRC", f"src `{src}` is not a recognizable report URL (needs --Vmlldz...)", rel, line)) continue - if rid not in by_id: - findings.append(Finding("error", "EMBED_NOT_IN_REGISTRY", f"report {rid} is embedded here but absent from registry.yaml", rel, line)) - else: - embedded_on[rid].add(rel) - if rel not in (by_id[rid].get("pages") or []): - findings.append(Finding("error", "PAGE_NOT_LISTED", f"registry entry for {rid} does not list this page in `pages`", rel, line)) # Prose-link rule: the id must appear somewhere outside the component. if rid not in prose: findings.append(Finding("error", "MISSING_PROSE_LINK", f"no plain link to report {rid} in the page prose (agents/llms.txt read source, where the iframe is opaque)", rel, line)) + if src not in seen_urls: + seen_urls.add(src) + embeds.append(Embed(src, rel, line)) - for rid, entry in by_id.items(): - listed = set(entry.get("pages") or []) - actual = embedded_on[rid] - if not actual: - findings.append(Finding("warning", "ORPHAN_ENTRY", f"registry entry {rid} is not embedded by any English page")) - for stale in sorted(listed - actual): - findings.append(Finding("error", "STALE_PAGES_FIELD", f"registry entry {rid} lists `{stale}` but that page has no matching embed")) - - return findings + return findings, embeds # --------------------------------------------------------------------------- # @@ -275,16 +191,14 @@ def check_url(url: str, *, timeout: int, attempts: int, delay: float) -> Finding return Finding("error", "URL_UNREACHABLE", f"{url}: unreachable after {attempts} attempts ({last_err})") -def check_liveness(entries: list[dict], *, timeout: int = 30, attempts: int = 3, delay: float = 1.0) -> list[Finding]: +def check_liveness(embeds: list[Embed], *, timeout: int = 30, attempts: int = 3, delay: float = 1.0) -> list[Finding]: findings: list[Finding] = [] - for entry in entries: - url = entry.get("url") - if not isinstance(url, str): - continue - finding = check_url(url, timeout=timeout, attempts=attempts, delay=delay) + for embed in embeds: + finding = check_url(embed.url, timeout=timeout, attempts=attempts, delay=delay) if finding: + finding.path, finding.line = embed.path, embed.line findings.append(finding) - time.sleep(delay) # be polite; registry is small + time.sleep(delay) # be polite; the embed set is small return findings @@ -302,7 +216,7 @@ def emit_github_annotations(findings: Iterable[Finding]) -> None: print(f"::{kind} {loc}::[{f.code}] {f.message}") -def render_markdown(findings: list[Finding], *, checked_urls: int, embeds: int) -> str: +def render_markdown(findings: list[Finding], *, embeds: int, checked_urls: int) -> str: errors = [f for f in findings if f.level == "error"] warnings = [f for f in findings if f.level == "warning"] lines = ["# Report embed check", ""] @@ -314,7 +228,7 @@ def render_markdown(findings: list[Finding], *, checked_urls: int, embeds: int) lines.append("| Level | Code | Location | Detail |") lines.append("| --- | --- | --- | --- |") for f in errors + warnings: - where = f.path or "registry.yaml" + where = f.path or "-" if f.line: where += f":{f.line}" lines.append(f"| {f.level} | {f.code} | {where} | {f.message} |") @@ -327,28 +241,21 @@ def render_markdown(findings: list[Finding], *, checked_urls: int, embeds: int) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--mode", choices=["static", "liveness", "all"], default="static") - parser.add_argument("--registry", type=Path, default=DEFAULT_REGISTRY) parser.add_argument("--root", type=Path, default=REPO_ROOT) parser.add_argument("--github", action="store_true", help="emit GitHub Actions annotations") parser.add_argument("--output-md", type=Path, help="write a Markdown report to this file") parser.add_argument("--strict", action="store_true", help="treat warnings as failures") args = parser.parse_args(argv) - entries, findings = load_registry(args.registry) - if any(f.code in ("REGISTRY_MISSING", "REGISTRY_UNREADABLE", "REGISTRY_SHAPE") for f in findings): - for f in findings: - print(f"ERROR [{f.code}] {f.message}", file=sys.stderr) - return 2 + static_findings, embeds = scan(args.root) - embeds_seen = 0 + findings: list[Finding] = [] if args.mode in ("static", "all"): - findings += check_mdx(args.root, entries) + findings += static_findings + checked_urls = 0 if args.mode in ("liveness", "all"): - findings += check_liveness(entries) - - # Count embeds for the summary line (cheap re-scan of registry pages only). - embeds_seen = sum(len(e.get("pages") or []) for e in entries) - checked_urls = len(entries) if args.mode in ("liveness", "all") else 0 + findings += check_liveness(embeds) + checked_urls = len(embeds) errors = [f for f in findings if f.level == "error"] warnings = [f for f in findings if f.level == "warning"] @@ -361,10 +268,11 @@ def main(argv: list[str] | None = None) -> int: print(f"{f.level.upper()} [{f.code}]{where}: {f.message}") if args.output_md: - args.output_md.write_text(render_markdown(findings, checked_urls=checked_urls, embeds=embeds_seen), encoding="utf-8") + args.output_md.write_text(render_markdown(findings, embeds=len(embeds), checked_urls=checked_urls), encoding="utf-8") if not findings: - print(f"OK: {len(entries)} registry entrie(s), {embeds_seen} embed page reference(s).") + checked = f", {checked_urls} URL(s) checked" if args.mode in ("liveness", "all") else "" + print(f"OK: {len(embeds)} report embed(s) found{checked}.") if errors: return 1 diff --git a/scripts/report-embeds/registry.yaml b/scripts/report-embeds/registry.yaml deleted file mode 100644 index 9ebd03e0a0..0000000000 --- a/scripts/report-embeds/registry.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# --------------------------------------------------------------------------- -# Registry of W&B Reports embedded in docs pages via . -# --------------------------------------------------------------------------- -# Every in an English .mdx page MUST have an entry -# here, and the page MUST also link the report URL in prose (so agents, -# llms.txt consumers, and the translation pipeline — which read MDX source, -# where the iframe is opaque — still get the link). -# -# Validated by scripts/report-embeds/check_embeds.py (CI: report-embeds.yml). -# An empty `reports: []` is a valid, passing state. -# -# To add an embedded report: -# 1. Add an entry below. `id` is the trailing --Vmlldzo... token in the URL. -# 2. On the page, add AND a plain Markdown link to -# the same report URL in the surrounding prose. -# 3. Run: python3 scripts/report-embeds/check_embeds.py --mode static -# -# Only reports viewable by anonymous visitors work embedded: a report in a -# public project, or one shared via a view-only ("magic") link. The URL — -# including any ?accessToken= — ships in public source and git history, so -# treat every embedded report as public forever. Never embed sensitive data. -# --------------------------------------------------------------------------- - -reports: - - id: Vmlldzo0NjE4MzM - url: https://wandb.ai/stacey/saferlife/reports/SafeLife-Benchmark-Experiments--Vmlldzo0NjE4MzM - owner: "@jmulhausen" - purpose: "Public demo (stacey/saferlife): live benchmark report for the Reports overview" - pages: - - models/reports.mdx - last_reviewed: 2026-07-21 - height: 720 - - - id: Vmlldzo2MzA5Mg - url: https://wandb.ai/stacey/lyft/reports/LIDAR-Point-Clouds-of-Driving-Scenes--Vmlldzo2MzA5Mg - owner: "@jmulhausen" - purpose: "Public demo (stacey/lyft): interactive 3D LIDAR point clouds for the Log media page" - pages: - - models/track/log/media.mdx - last_reviewed: 2026-07-21 - height: 720 diff --git a/scripts/report-embeds/tests/test_check_embeds.py b/scripts/report-embeds/tests/test_check_embeds.py index eda475c941..f12aa57eb7 100644 --- a/scripts/report-embeds/tests/test_check_embeds.py +++ b/scripts/report-embeds/tests/test_check_embeds.py @@ -1,8 +1,7 @@ -"""Unit tests for the report-embed validator. No network.""" +"""Unit tests for the report-embed checker. No network.""" from __future__ import annotations -import datetime import importlib.util import sys import tempfile @@ -25,78 +24,6 @@ def _load(): m = _load() -TODAY = datetime.date.today().isoformat() - - -def _entry(**over): - base = { - "id": "Vmlldzo12345", - "url": "https://wandb.ai/wandb/docs-embeds/reports/Foo--Vmlldzo12345", - "owner": "@me", - "purpose": "demo", - "pages": ["models/sweeps/visualize-sweep-results.mdx"], - "last_reviewed": TODAY, - "height": 700, - } - base.update(over) - return base - - -class RegistrySchemaTests(unittest.TestCase): - def _load_yaml(self, doc: str): - with tempfile.TemporaryDirectory() as d: - p = Path(d) / "registry.yaml" - p.write_text(doc, encoding="utf-8") - return m.load_registry(p) - - def test_empty_registry_is_valid(self): - _, findings = self._load_yaml("reports: []\n") - self.assertEqual(findings, []) - - def test_missing_reports_key_is_shape_error(self): - _, findings = self._load_yaml("something: else\n") - self.assertTrue(any(f.code == "REGISTRY_SHAPE" for f in findings)) - - def test_top_level_list_is_shape_error_not_crash(self): - # A top-level list/string must yield REGISTRY_SHAPE, not AttributeError. - _, findings = self._load_yaml("- a\n- b\n") - self.assertTrue(any(f.code == "REGISTRY_SHAPE" for f in findings)) - _, findings = self._load_yaml("just a string\n") - self.assertTrue(any(f.code == "REGISTRY_SHAPE" for f in findings)) - - def test_bad_id_pattern(self): - import yaml - doc = yaml.safe_dump({"reports": [_entry(id="notanid", url="https://wandb.ai/w/p/reports/Foo--notanid")]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "BAD_ID" for f in findings)) - - def test_url_id_mismatch(self): - import yaml - doc = yaml.safe_dump({"reports": [_entry(url="https://wandb.ai/w/p/reports/Foo--Vmlldzo99999")]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "URL_ID_MISMATCH" for f in findings)) - - def test_missing_required_field(self): - import yaml - e = _entry() - del e["owner"] - doc = yaml.safe_dump({"reports": [e]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "MISSING_FIELD" for f in findings)) - - def test_future_date(self): - import yaml - future = (datetime.date.today() + datetime.timedelta(days=5)).isoformat() - doc = yaml.safe_dump({"reports": [_entry(last_reviewed=future)]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "FUTURE_DATE" for f in findings)) - - def test_bad_host(self): - import yaml - doc = yaml.safe_dump({"reports": [_entry(url="https://evil.example/reports/Foo--Vmlldzo12345")]}) - _, findings = self._load_yaml(doc) - self.assertTrue(any(f.code == "BAD_URL_HOST" for f in findings)) - class ExtractionTests(unittest.TestCase): def test_self_closing(self): @@ -122,77 +49,80 @@ def test_strip_removes_component(self): self.assertIn("before", m.strip_embeds(text)) -class MdxConsistencyTests(unittest.TestCase): - def _run(self, files: dict[str, str], entries: list[dict]): +class ScanTests(unittest.TestCase): + def _run(self, files: dict[str, str]): with tempfile.TemporaryDirectory() as d: root = Path(d) for rel, content in files.items(): fp = root / rel fp.parent.mkdir(parents=True, exist_ok=True) fp.write_text(content, encoding="utf-8") - return m.check_mdx(root, entries) + return m.scan(root) def test_valid_embed_with_prose_link_passes(self): page = ( "See the [sweep report](https://wandb.ai/w/p/reports/Foo--Vmlldzo12345).\n\n" '\n' ) - findings = self._run({"models/sweeps/visualize-sweep-results.mdx": page}, [_entry()]) + findings, embeds = self._run({"models/sweeps/visualize-sweep-results.mdx": page}) self.assertEqual([f for f in findings if f.level == "error"], []) + self.assertEqual(len(embeds), 1) + self.assertEqual(embeds[0].url, "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345") def test_missing_prose_link(self): page = '\n' - findings = self._run({"models/sweeps/visualize-sweep-results.mdx": page}, [_entry()]) + findings, _ = self._run({"models/sweeps/visualize-sweep-results.mdx": page}) self.assertTrue(any(f.code == "MISSING_PROSE_LINK" for f in findings)) - def test_unregistered_embed(self): - page = ( - "[link](https://wandb.ai/w/p/reports/Foo--Vmlldzo99999)\n" - '\n' - ) - findings = self._run({"models/sweeps/visualize-sweep-results.mdx": page}, []) - self.assertTrue(any(f.code == "EMBED_NOT_IN_REGISTRY" for f in findings)) - - def test_orphan_entry_warns(self): - findings = self._run({"models/sweeps/visualize-sweep-results.mdx": "no embeds here\n"}, [_entry()]) - self.assertTrue(any(f.code == "ORPHAN_ENTRY" and f.level == "warning" for f in findings)) + def test_bad_embed_src(self): + page = '\n' + findings, embeds = self._run({"models/x.mdx": page}) + self.assertTrue(any(f.code == "BAD_EMBED_SRC" for f in findings)) + self.assertEqual(embeds, []) def test_too_many_embeds_warns(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" link = f"[r]({src})\n" page = link + "".join(f'\n' for i in range(3)) - entry = _entry(pages=["models/x.mdx"]) - findings = self._run({"models/x.mdx": page}, [entry]) + findings, _ = self._run({"models/x.mdx": page}) self.assertTrue(any(f.code == "TOO_MANY_EMBEDS" and f.level == "warning" for f in findings)) + def test_duplicate_url_deduped_for_liveness(self): + src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" + link = f"[r]({src})\n" + # Same report embedded on two pages: two prose links, one unique URL. + findings, embeds = self._run({ + "models/a.mdx": link + f'\n', + "models/b.mdx": link + f'\n', + }) + self.assertEqual([f for f in findings if f.level == "error"], []) + self.assertEqual(len(embeds), 1) + def test_locale_files_ignored(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" page = f'\n' # no prose link — would error if scanned - # Same embed on the English page (valid) plus a locale copy (must be skipped). english = f"[r]({src})\n" + page - findings = self._run( - {"models/x.mdx": english, "ja/models/x.mdx": page}, - [_entry(pages=["models/x.mdx"])], - ) + findings, _ = self._run({"models/x.mdx": english, "ja/models/x.mdx": page}) self.assertEqual([f for f in findings if f.level == "error"], []) def test_embed_in_code_fence_ignored(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" page = "Usage example:\n\n```mdx\n" + f'\n' + "```\n" - findings = self._run({"models/x.mdx": page}, []) + findings, embeds = self._run({"models/x.mdx": page}) self.assertEqual(findings, []) + self.assertEqual(embeds, []) def test_embed_in_mdx_comment_ignored(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" page = "{/* staged, fill in real URL later:\n" + f'\n' + "*/}\n" - findings = self._run({"models/x.mdx": page}, []) + findings, embeds = self._run({"models/x.mdx": page}) self.assertEqual(findings, []) + self.assertEqual(embeds, []) def test_embed_in_snippet_errors(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - findings = self._run( - {"snippets/_includes/thing.mdx": f"[r]({src})\n\n"}, - [_entry(pages=["snippets/_includes/thing.mdx"])], + findings, _ = self._run( + {"snippets/_includes/thing.mdx": f'[r]({src})\n\n'}, ) self.assertTrue(any(f.code == "EMBED_IN_SNIPPET" for f in findings)) From b6ce12ca0638d9a89f42e1aaaa061c1ad79525d4 Mon Sep 17 00:00:00 2001 From: John Mulhausen Date: Wed, 22 Jul 2026 13:05:54 -0400 Subject: [PATCH 2/7] report-embeds: drop per-page-cap check and trim the checker Remove the unrequested TOO_MANY_EMBEDS cap (and its test). With no warning-level findings left, drop the warning/`--strict` machinery and the Finding.level field, and slim the output paths. Also drop the locale-ignored test (locale skipping stays; it's just untested here). check_embeds.py 377->245 lines, tests 201->88. Co-Authored-By: Claude Fable 5 --- .github/workflows/report-embeds.yml | 5 +- scripts/report-embeds/README.md | 4 +- scripts/report-embeds/check_embeds.py | 112 ++++++------------ .../report-embeds/tests/test_check_embeds.py | 75 +++--------- 4 files changed, 56 insertions(+), 140 deletions(-) diff --git a/.github/workflows/report-embeds.yml b/.github/workflows/report-embeds.yml index 1e6e94fe86..28feb21a6b 100644 --- a/.github/workflows/report-embeds.yml +++ b/.github/workflows/report-embeds.yml @@ -2,8 +2,7 @@ name: Report embeds # Checks live W&B report embeds () in docs pages. Embeds are # discovered by scanning the .mdx sources — there is no registry. -# - On PRs: static check (every embed has a sibling prose link, sits on a page, -# and stays under the per-page cap). +# - On PRs: static check (every embed has a sibling prose link and sits on a page). # - Weekly: URL liveness, filing an issue on failure. Liveness never blocks PRs — # wandb.ai rate-limits crawlers, so network results are flaky by design (this # is also why wandb.ai is excluded from the site-wide lychee check). @@ -38,7 +37,7 @@ jobs: python-version: "3.12" - name: Unit tests run: python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v - - name: Embed consistency (prose link, per-page cap) + - name: Embed consistency (prose link) run: python3 scripts/report-embeds/check_embeds.py --mode static --github liveness: diff --git a/scripts/report-embeds/README.md b/scripts/report-embeds/README.md index fc6b7f81c1..ecd864be43 100644 --- a/scripts/report-embeds/README.md +++ b/scripts/report-embeds/README.md @@ -8,7 +8,7 @@ scanning the `.mdx` sources (no registry to maintain) and checks it. ## Files - `check_embeds.py` — scans English `.mdx` for `` embeds and checks - them (sibling prose link + per-page cap + URL liveness). + them (sibling prose link + URL liveness). - `tests/test_check_embeds.py` — unit tests (no network). CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflows/report-embeds.yml). @@ -55,7 +55,7 @@ python3 scripts/report-embeds/check_embeds.py --mode liveness # anonymous HTTP python3 scripts/report-embeds/check_embeds.py --mode all # both ``` -Static errors (missing prose link, embed in a shared snippet) fail CI on PRs. +Static errors (missing prose link, embed in a shared snippet, unrecognizable src) fail CI on PRs. Liveness runs on a weekly schedule and files an issue on failure — it does not block PRs, because `wandb.ai` rate-limits crawlers and network results are flaky (this is also why `wandb.ai` is excluded from the site-wide lychee check). diff --git a/scripts/report-embeds/check_embeds.py b/scripts/report-embeds/check_embeds.py index 395dbfa58c..9105e8e670 100644 --- a/scripts/report-embeds/check_embeds.py +++ b/scripts/report-embeds/check_embeds.py @@ -5,8 +5,7 @@ `` component — there is no registry to keep in sync. Three modes: - static - every embed has a sibling prose link, sits on a page (not a shared - snippet), and pages stay under the per-page cap (no network) + static - every embed has a sibling prose link and sits on a page (no network) liveness - anonymous HTTP check that each embedded report URL still renders all - both @@ -21,7 +20,6 @@ import argparse import re -import sys import time import urllib.error import urllib.request @@ -55,34 +53,23 @@ SNIPPETS_PREFIX = "snippets/" LOGIN_MARKERS = ("/login", "/signin", "/site/login", "/authorize") -MAX_EMBEDS_PER_PAGE = 2 - @dataclass class Finding: - level: str # "error" | "warning" code: str message: str path: str | None = None line: int | None = None -@dataclass -class Embed: - url: str - path: str # repo-relative page that embeds it - line: int - - # --------------------------------------------------------------------------- # # MDX scanning # --------------------------------------------------------------------------- # def iter_mdx(root: Path) -> Iterator[Path]: for path in sorted(root.rglob("*.mdx")): rel = path.relative_to(root).as_posix() - if any(rel.startswith(pre) for pre in SKIP_PREFIXES): - continue - yield path + if not any(rel.startswith(pre) for pre in SKIP_PREFIXES): + yield path def _line_of(text: str, index: int) -> int: @@ -113,15 +100,15 @@ def mask_noncontent(text: str) -> str: return MDX_COMMENT_RE.sub(_blank, FENCE_RE.sub(_blank, text)) -def scan(root: Path) -> tuple[list[Finding], list[Embed]]: +def scan(root: Path) -> tuple[list[Finding], list[tuple[str, str, int]]]: """Scan English .mdx for embeds. - Returns (static findings, embeds). `embeds` is deduplicated by URL for the - liveness pass; the static findings cover every occurrence. + Returns (findings, embeds). `embeds` is (url, page, line) deduplicated by URL + for the liveness pass; findings cover the static checks. """ findings: list[Finding] = [] - embeds: list[Embed] = [] - seen_urls: set[str] = set() + embeds: list[tuple[str, str, int]] = [] + seen: set[str] = set() for path in iter_mdx(root): rel = path.relative_to(root).as_posix() @@ -129,27 +116,23 @@ def scan(root: Path) -> tuple[list[Finding], list[Embed]]: occurrences = extract_embeds(text) if not occurrences: continue - if rel.startswith(SNIPPETS_PREFIX): - findings.append(Finding("error", "EMBED_IN_SNIPPET", " must be used on a page, not in a shared snippet", rel, occurrences[0][1])) + findings.append(Finding("EMBED_IN_SNIPPET", " must be used on a page, not in a shared snippet", rel, occurrences[0][1])) continue - if len(occurrences) > MAX_EMBEDS_PER_PAGE: - findings.append(Finding("warning", "TOO_MANY_EMBEDS", f"{len(occurrences)} embeds on one page; each boots the full W&B app (cap is {MAX_EMBEDS_PER_PAGE})", rel, occurrences[0][1])) - prose = strip_embeds(text) for src, line in occurrences: ids = REPORT_ID_RE.findall(src) rid = ids[0] if ids else None if rid is None: - findings.append(Finding("error", "BAD_EMBED_SRC", f"src `{src}` is not a recognizable report URL (needs --Vmlldz...)", rel, line)) + findings.append(Finding("BAD_EMBED_SRC", f"src `{src}` is not a recognizable report URL (needs --Vmlldz...)", rel, line)) continue # Prose-link rule: the id must appear somewhere outside the component. if rid not in prose: - findings.append(Finding("error", "MISSING_PROSE_LINK", f"no plain link to report {rid} in the page prose (agents/llms.txt read source, where the iframe is opaque)", rel, line)) - if src not in seen_urls: - seen_urls.add(src) - embeds.append(Embed(src, rel, line)) + findings.append(Finding("MISSING_PROSE_LINK", f"no plain link to report {rid} in the page prose (agents/llms.txt read source, where the iframe is opaque)", rel, line)) + if src not in seen: + seen.add(src) + embeds.append((src, rel, line)) return findings, embeds @@ -170,33 +153,31 @@ def check_url(url: str, *, timeout: int, attempts: int, delay: float) -> Finding for attempt in range(1, attempts + 1): try: with urllib.request.urlopen(req, timeout=timeout) as resp: - final_url = resp.geturl() - status = resp.getcode() - if status != 200: - return Finding("error", "URL_NOT_200", f"{url}: final status {status}") - if any(marker in final_url for marker in LOGIN_MARKERS): - return Finding("error", "URL_LOGIN_REDIRECT", f"{url}: redirected to login ({final_url}) — magic link likely revoked") + if resp.getcode() != 200: + return Finding("URL_NOT_200", f"{url}: final status {resp.getcode()}") + if any(marker in resp.geturl() for marker in LOGIN_MARKERS): + return Finding("URL_LOGIN_REDIRECT", f"{url}: redirected to login ({resp.geturl()}) — magic link likely revoked") return None except urllib.error.HTTPError as exc: if exc.code in (429, 500, 502, 503, 504) and attempt < attempts: last_err = f"HTTP {exc.code}" time.sleep(delay * (2 ** attempt)) continue - return Finding("error", "URL_NOT_200", f"{url}: HTTP {exc.code}") + return Finding("URL_NOT_200", f"{url}: HTTP {exc.code}") except (urllib.error.URLError, TimeoutError, OSError) as exc: last_err = str(getattr(exc, "reason", exc)) if attempt < attempts: time.sleep(delay * (2 ** attempt)) continue - return Finding("error", "URL_UNREACHABLE", f"{url}: unreachable after {attempts} attempts ({last_err})") + return Finding("URL_UNREACHABLE", f"{url}: unreachable after {attempts} attempts ({last_err})") -def check_liveness(embeds: list[Embed], *, timeout: int = 30, attempts: int = 3, delay: float = 1.0) -> list[Finding]: +def check_liveness(embeds: list[tuple[str, str, int]], *, timeout: int = 30, attempts: int = 3, delay: float = 1.0) -> list[Finding]: findings: list[Finding] = [] - for embed in embeds: - finding = check_url(embed.url, timeout=timeout, attempts=attempts, delay=delay) + for url, rel, line in embeds: + finding = check_url(url, timeout=timeout, attempts=attempts, delay=delay) if finding: - finding.path, finding.line = embed.path, embed.line + finding.path, finding.line = rel, line findings.append(finding) time.sleep(delay) # be polite; the embed set is small return findings @@ -205,33 +186,23 @@ def check_liveness(embeds: list[Embed], *, timeout: int = 30, attempts: int = 3, # --------------------------------------------------------------------------- # # Output # --------------------------------------------------------------------------- # +def _loc(f: Finding) -> str: + return (f.path or "-") + (f":{f.line}" if f.path and f.line else "") + + def emit_github_annotations(findings: Iterable[Finding]) -> None: for f in findings: - kind = "error" if f.level == "error" else "warning" - loc = "" - if f.path: - loc = f"file={f.path}" - if f.line: - loc += f",line={f.line}" - print(f"::{kind} {loc}::[{f.code}] {f.message}") + loc = (f"file={f.path}" + (f",line={f.line}" if f.line else "")) if f.path else "" + print(f"::error {loc}::[{f.code}] {f.message}") def render_markdown(findings: list[Finding], *, embeds: int, checked_urls: int) -> str: - errors = [f for f in findings if f.level == "error"] - warnings = [f for f in findings if f.level == "warning"] lines = ["# Report embed check", ""] if not findings: lines.append(f"✅ All checks passed ({embeds} embed(s), {checked_urls} URL(s) checked).") return "\n".join(lines) + "\n" - lines.append(f"Found {len(errors)} error(s) and {len(warnings)} warning(s).") - lines.append("") - lines.append("| Level | Code | Location | Detail |") - lines.append("| --- | --- | --- | --- |") - for f in errors + warnings: - where = f.path or "-" - if f.line: - where += f":{f.line}" - lines.append(f"| {f.level} | {f.code} | {where} | {f.message} |") + lines += [f"Found {len(findings)} problem(s).", "", "| Code | Location | Detail |", "| --- | --- | --- |"] + lines += [f"| {f.code} | {_loc(f)} | {f.message} |" for f in findings] return "\n".join(lines) + "\n" @@ -244,28 +215,21 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--root", type=Path, default=REPO_ROOT) parser.add_argument("--github", action="store_true", help="emit GitHub Actions annotations") parser.add_argument("--output-md", type=Path, help="write a Markdown report to this file") - parser.add_argument("--strict", action="store_true", help="treat warnings as failures") args = parser.parse_args(argv) static_findings, embeds = scan(args.root) - findings: list[Finding] = [] - if args.mode in ("static", "all"): - findings += static_findings + findings: list[Finding] = list(static_findings) if args.mode in ("static", "all") else [] checked_urls = 0 if args.mode in ("liveness", "all"): findings += check_liveness(embeds) checked_urls = len(embeds) - errors = [f for f in findings if f.level == "error"] - warnings = [f for f in findings if f.level == "warning"] - if args.github: emit_github_annotations(findings) else: - for f in errors + warnings: - where = f" {f.path}:{f.line}" if f.path and f.line else (f" {f.path}" if f.path else "") - print(f"{f.level.upper()} [{f.code}]{where}: {f.message}") + for f in findings: + print(f"ERROR [{f.code}] {_loc(f)}: {f.message}") if args.output_md: args.output_md.write_text(render_markdown(findings, embeds=len(embeds), checked_urls=checked_urls), encoding="utf-8") @@ -274,11 +238,7 @@ def main(argv: list[str] | None = None) -> int: checked = f", {checked_urls} URL(s) checked" if args.mode in ("liveness", "all") else "" print(f"OK: {len(embeds)} report embed(s) found{checked}.") - if errors: - return 1 - if warnings and args.strict: - return 1 - return 0 + return 1 if findings else 0 if __name__ == "__main__": diff --git a/scripts/report-embeds/tests/test_check_embeds.py b/scripts/report-embeds/tests/test_check_embeds.py index f12aa57eb7..ee7cd6fa90 100644 --- a/scripts/report-embeds/tests/test_check_embeds.py +++ b/scripts/report-embeds/tests/test_check_embeds.py @@ -15,8 +15,6 @@ def _load(): spec = importlib.util.spec_from_file_location("check_embeds", MODULE_PATH) mod = importlib.util.module_from_spec(spec) assert spec.loader - # Register before exec so dataclass type introspection can resolve the - # module (required on Python 3.9; harmless on newer versions). sys.modules[spec.name] = mod spec.loader.exec_module(mod) return mod @@ -31,22 +29,12 @@ def test_self_closing(self): self.assertEqual(m.extract_embeds(text), [("https://wandb.ai/w/p/reports/Foo--Vmlldzo1", 1)]) def test_multiline_and_brace_src(self): - text = ( - "intro\n" - "\n" - ) - embeds = m.extract_embeds(text) - self.assertEqual(len(embeds), 1) - self.assertEqual(embeds[0][0], "https://wandb.ai/w/p/reports/Foo---Vmlldzo2") - self.assertEqual(embeds[0][1], 2) + text = 'x\n\n' + self.assertEqual(m.extract_embeds(text), [("https://wandb.ai/w/p/reports/Foo---Vmlldzo2", 2)]) def test_strip_removes_component(self): text = 'before after' self.assertNotIn("Vmlldzo3", m.strip_embeds(text)) - self.assertIn("before", m.strip_embeds(text)) class ScanTests(unittest.TestCase): @@ -60,18 +48,15 @@ def _run(self, files: dict[str, str]): return m.scan(root) def test_valid_embed_with_prose_link_passes(self): - page = ( - "See the [sweep report](https://wandb.ai/w/p/reports/Foo--Vmlldzo12345).\n\n" - '\n' - ) - findings, embeds = self._run({"models/sweeps/visualize-sweep-results.mdx": page}) - self.assertEqual([f for f in findings if f.level == "error"], []) - self.assertEqual(len(embeds), 1) - self.assertEqual(embeds[0].url, "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345") + src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" + page = f"See the [report]({src}).\n\n\n" + findings, embeds = self._run({"models/x.mdx": page}) + self.assertEqual(findings, []) + self.assertEqual(embeds, [(src, "models/x.mdx", 3)]) def test_missing_prose_link(self): page = '\n' - findings, _ = self._run({"models/sweeps/visualize-sweep-results.mdx": page}) + findings, _ = self._run({"models/x.mdx": page}) self.assertTrue(any(f.code == "MISSING_PROSE_LINK" for f in findings)) def test_bad_embed_src(self): @@ -80,50 +65,22 @@ def test_bad_embed_src(self): self.assertTrue(any(f.code == "BAD_EMBED_SRC" for f in findings)) self.assertEqual(embeds, []) - def test_too_many_embeds_warns(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - link = f"[r]({src})\n" - page = link + "".join(f'\n' for i in range(3)) - findings, _ = self._run({"models/x.mdx": page}) - self.assertTrue(any(f.code == "TOO_MANY_EMBEDS" and f.level == "warning" for f in findings)) - def test_duplicate_url_deduped_for_liveness(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - link = f"[r]({src})\n" - # Same report embedded on two pages: two prose links, one unique URL. - findings, embeds = self._run({ - "models/a.mdx": link + f'\n', - "models/b.mdx": link + f'\n', - }) - self.assertEqual([f for f in findings if f.level == "error"], []) - self.assertEqual(len(embeds), 1) - - def test_locale_files_ignored(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = f'\n' # no prose link — would error if scanned - english = f"[r]({src})\n" + page - findings, _ = self._run({"models/x.mdx": english, "ja/models/x.mdx": page}) - self.assertEqual([f for f in findings if f.level == "error"], []) - - def test_embed_in_code_fence_ignored(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = "Usage example:\n\n```mdx\n" + f'\n' + "```\n" - findings, embeds = self._run({"models/x.mdx": page}) + page = f"[r]({src})\n\n" + findings, embeds = self._run({"models/a.mdx": page, "models/b.mdx": page}) self.assertEqual(findings, []) - self.assertEqual(embeds, []) + self.assertEqual(len(embeds), 1) - def test_embed_in_mdx_comment_ignored(self): + def test_masked_regions_ignored(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = "{/* staged, fill in real URL later:\n" + f'\n' + "*/}\n" - findings, embeds = self._run({"models/x.mdx": page}) - self.assertEqual(findings, []) - self.assertEqual(embeds, []) + embed = f'\n' + for page in (f"```mdx\n{embed}```\n", "{/* later:\n" + embed + "*/}\n"): + self.assertEqual(self._run({"models/x.mdx": page}), ([], [])) def test_embed_in_snippet_errors(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - findings, _ = self._run( - {"snippets/_includes/thing.mdx": f'[r]({src})\n\n'}, - ) + findings, _ = self._run({"snippets/_includes/t.mdx": f'[r]({src})\n\n'}) self.assertTrue(any(f.code == "EMBED_IN_SNIPPET" for f in findings)) From 040d240401c40138bdeec83bf0d5350d1e5c3c33 Mon Sep 17 00:00:00 2001 From: John Mulhausen Date: Wed, 22 Jul 2026 13:12:37 -0400 Subject: [PATCH 3/7] WandbReport: render a "View Report" button; drop the prose-link rule The report URL is just `src`, so the component can render the link itself instead of asking authors to add one. - WandbReport.jsx: replace the caption text link with a right-justified "View Report" button styled like the in-content button-links (.colab-link / .try-product-link), inline so the component stays self-contained. It links to the plain report URL (no ?jupyter=true). - check_embeds.py: drop the MISSING_PROSE_LINK check and the now-unused strip_embeds(). Static mode now just checks placement + a recognizable report URL; liveness unchanged. - Drop the prose-link tests; update README and the AGENTS.md rule. The `src` URL stays in the MDX source, so agents and the llms.txt export still see the report URL. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 3 +- scripts/report-embeds/README.md | 17 ++++------ scripts/report-embeds/check_embeds.py | 14 ++------ .../report-embeds/tests/test_check_embeds.py | 19 +++-------- snippets/WandbReport.jsx | 32 +++++++++++++++---- 5 files changed, 40 insertions(+), 45 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 91cabac242..b0e05f3c03 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,8 +111,9 @@ To check or apply style, read the relevant pass file(s) and use them as guidance /> ``` + The component renders a "View Report" button linking to the report, so no separate prose link is needed. Still, introduce each embed in the surrounding text so the page reads well and states what the reader should take from it. CI (`scripts/report-embeds/check_embeds.py`) discovers embeds by scanning the `.mdx` sources and checks placement and URL liveness — see [`scripts/report-embeds/README.md`](scripts/report-embeds/README.md). + Rules for every embed: - - **Always pair it with prose and a plain Markdown link** to the same report in the surrounding text, stating what the reader should take from it. Agents, the llms.txt export, and the translation pipeline read MDX source, where the iframe is opaque; the link is also the fallback wherever third-party frames are blocked. This is enforced by CI (`scripts/report-embeds/check_embeds.py`, which discovers embeds by scanning the `.mdx` sources — see [`scripts/report-embeds/README.md`](scripts/report-embeds/README.md)). - **The report must be viewable by anonymous visitors**: a report in a public project, or one shared via a view-only link (`Share` → "anyone with the link can view"; see [View-only report links](/models/reports/cross-project-reports#view-only-report-links)). The URL — including any `?accessToken=` — ships in public source and git history, so treat the report as public forever and never embed sensitive data. - **Regular reports only, not Fully Connected articles.** FC articles keep their full blog chrome in a frame and look broken. Regular reports render in a slim embed view. - **Keep it skinny and sparse**: prefer purpose-built reports (ideally a single panel grid), one or two per page maximum. Each iframe boots the full W&B app, is a fixed height with inner scroll (no auto-resize), and renders its own light theme regardless of the docs dark-mode toggle — so article-length reports show as a tall scroll region, not the whole page at once. diff --git a/scripts/report-embeds/README.md b/scripts/report-embeds/README.md index ecd864be43..4045063b20 100644 --- a/scripts/report-embeds/README.md +++ b/scripts/report-embeds/README.md @@ -8,7 +8,7 @@ scanning the `.mdx` sources (no registry to maintain) and checks it. ## Files - `check_embeds.py` — scans English `.mdx` for `` embeds and checks - them (sibling prose link + URL liveness). + them (recognizable report URL, on a page, and URL liveness). - `tests/test_check_embeds.py` — unit tests (no network). CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflows/report-embeds.yml). @@ -21,15 +21,13 @@ CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflo shared via a view-only ("magic") link (`Share` → "anyone with the link can view"). The URL, including any `?accessToken=`, ships in public source and git history, so treat the report as public forever. Never embed sensitive data. -2. On the page, add both the component and a plain Markdown link to the same - report in the surrounding prose: +2. On the page, add the component. It renders a "View Report" button linking to + the report, so no separate prose link is needed. The `src` URL is in the MDX + source, so agents and the llms.txt export still see it. ```mdx import { WandbReport } from '/snippets/WandbReport.jsx'; - The following [sweep report](https://wandb.ai/ENTITY/PROJECT/reports/Slug--VmlldzoXXXXXXX) - shows ... . - ``` - The prose link is required: agents, the llms.txt export, and the translation - pipeline read MDX source, where the iframe is opaque. - 3. Validate locally: ```bash @@ -50,12 +45,12 @@ CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflo ```bash python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v -python3 scripts/report-embeds/check_embeds.py --mode static # prose link + cap, no network +python3 scripts/report-embeds/check_embeds.py --mode static # URL + placement, no network python3 scripts/report-embeds/check_embeds.py --mode liveness # anonymous HTTP per URL python3 scripts/report-embeds/check_embeds.py --mode all # both ``` -Static errors (missing prose link, embed in a shared snippet, unrecognizable src) fail CI on PRs. +Static errors (embed in a shared snippet, unrecognizable src) fail CI on PRs. Liveness runs on a weekly schedule and files an issue on failure — it does not block PRs, because `wandb.ai` rate-limits crawlers and network results are flaky (this is also why `wandb.ai` is excluded from the site-wide lychee check). diff --git a/scripts/report-embeds/check_embeds.py b/scripts/report-embeds/check_embeds.py index 9105e8e670..1461d28a99 100644 --- a/scripts/report-embeds/check_embeds.py +++ b/scripts/report-embeds/check_embeds.py @@ -5,7 +5,7 @@ `` component — there is no registry to keep in sync. Three modes: - static - every embed has a sibling prose link and sits on a page (no network) + static - every embed sits on a page and has a recognizable report URL (no network) liveness - anonymous HTTP check that each embedded report URL still renders all - both @@ -86,10 +86,6 @@ def extract_embeds(text: str) -> list[tuple[str, int]]: return out -def strip_embeds(text: str) -> str: - return COMPONENT_RE.sub("", text) - - def _blank(match: re.Match) -> str: # Replace matched span with same-length whitespace so byte offsets — and # therefore reported line numbers — are preserved. @@ -120,16 +116,10 @@ def scan(root: Path) -> tuple[list[Finding], list[tuple[str, str, int]]]: findings.append(Finding("EMBED_IN_SNIPPET", " must be used on a page, not in a shared snippet", rel, occurrences[0][1])) continue - prose = strip_embeds(text) for src, line in occurrences: - ids = REPORT_ID_RE.findall(src) - rid = ids[0] if ids else None - if rid is None: + if not REPORT_ID_RE.search(src): findings.append(Finding("BAD_EMBED_SRC", f"src `{src}` is not a recognizable report URL (needs --Vmlldz...)", rel, line)) continue - # Prose-link rule: the id must appear somewhere outside the component. - if rid not in prose: - findings.append(Finding("MISSING_PROSE_LINK", f"no plain link to report {rid} in the page prose (agents/llms.txt read source, where the iframe is opaque)", rel, line)) if src not in seen: seen.add(src) embeds.append((src, rel, line)) diff --git a/scripts/report-embeds/tests/test_check_embeds.py b/scripts/report-embeds/tests/test_check_embeds.py index ee7cd6fa90..b9a3d69523 100644 --- a/scripts/report-embeds/tests/test_check_embeds.py +++ b/scripts/report-embeds/tests/test_check_embeds.py @@ -32,10 +32,6 @@ def test_multiline_and_brace_src(self): text = 'x\n\n' self.assertEqual(m.extract_embeds(text), [("https://wandb.ai/w/p/reports/Foo---Vmlldzo2", 2)]) - def test_strip_removes_component(self): - text = 'before after' - self.assertNotIn("Vmlldzo3", m.strip_embeds(text)) - class ScanTests(unittest.TestCase): def _run(self, files: dict[str, str]): @@ -47,17 +43,12 @@ def _run(self, files: dict[str, str]): fp.write_text(content, encoding="utf-8") return m.scan(root) - def test_valid_embed_with_prose_link_passes(self): + def test_valid_embed_passes(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = f"See the [report]({src}).\n\n\n" + page = f'\n' findings, embeds = self._run({"models/x.mdx": page}) self.assertEqual(findings, []) - self.assertEqual(embeds, [(src, "models/x.mdx", 3)]) - - def test_missing_prose_link(self): - page = '\n' - findings, _ = self._run({"models/x.mdx": page}) - self.assertTrue(any(f.code == "MISSING_PROSE_LINK" for f in findings)) + self.assertEqual(embeds, [(src, "models/x.mdx", 1)]) def test_bad_embed_src(self): page = '\n' @@ -67,7 +58,7 @@ def test_bad_embed_src(self): def test_duplicate_url_deduped_for_liveness(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = f"[r]({src})\n\n" + page = f'\n' findings, embeds = self._run({"models/a.mdx": page, "models/b.mdx": page}) self.assertEqual(findings, []) self.assertEqual(len(embeds), 1) @@ -80,7 +71,7 @@ def test_masked_regions_ignored(self): def test_embed_in_snippet_errors(self): src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - findings, _ = self._run({"snippets/_includes/t.mdx": f'[r]({src})\n\n'}) + findings, _ = self._run({"snippets/_includes/t.mdx": f'\n'}) self.assertTrue(any(f.code == "EMBED_IN_SNIPPET" for f in findings)) diff --git a/snippets/WandbReport.jsx b/snippets/WandbReport.jsx index 066e0004df..178198faec 100644 --- a/snippets/WandbReport.jsx +++ b/snippets/WandbReport.jsx @@ -3,7 +3,7 @@ export const WandbReport = ({ src, title, height = 640 }) => { // (sometimes three hyphens before the id), optionally with a ?accessToken=... // query when shared via a view-only ("magic") link. Only report pages allow // framing — project and run pages block it — so reject anything else and fall - // back to the link-only card rather than rendering a frame that won't load. + // back to the button-only card rather than rendering a frame that won't load. const isReportUrl = typeof src === 'string' && /^https:\/\/wandb\.ai\/[^/]+\/[^/]+\/reports\/.*-{2,3}Vmlldz[A-Za-z0-9]+/.test(src); @@ -27,8 +27,8 @@ export const WandbReport = ({ src, title, height = 640 }) => { // regular report renders in a slim embed view. A Fully Connected article // ignores the param and keeps its full blog chrome (nav bar, breadcrumb, // author, stars), so it looks broken in a frame — embed regular reports, not - // FC articles. The human-facing caption link below omits the param and points - // at the normal report URL. + // FC articles. The View Report button below omits the param and points at the + // normal report URL. const frameSrc = isReportUrl ? `${src}${src.includes('?') ? '&' : '?'}jupyter=true` : src; // Colors come from the CSS custom properties in scripts/css-minify/colors.css @@ -57,19 +57,37 @@ export const WandbReport = ({ src, title, height = 640 }) => { )}
+ {/* "View Report" button matching the in-content button-links + (.colab-link / .try-product-link in scripts/css-minify/buttons.css), + styled inline so the component stays self-contained. It links to the + plain report URL (no ?jupyter=true), so it doubles as the fallback + wherever the frame can't load. */} - View this report in the W&B app → + View Report
From 65924fcb80bd6eacc333566601fdd3be97b6234d Mon Sep 17 00:00:00 2001 From: John Mulhausen Date: Wed, 22 Jul 2026 13:17:34 -0400 Subject: [PATCH 4/7] report-embeds: drop --mode (always scan + liveness); trim docs check_embeds.py no longer takes --mode; every run does the static checks and the liveness check. The workflow collapses to a single job that blocks on PRs and files an issue on the weekly run. Also trim README.md and the AGENTS.md embed rules to match what's left after removing the registry, the prose-link requirement, and modes. Co-Authored-By: Claude Fable 5 --- .github/workflows/report-embeds.yml | 34 ++++-------- AGENTS.md | 13 ++--- scripts/report-embeds/README.md | 79 ++++++++++----------------- scripts/report-embeds/check_embeds.py | 28 +++------- 4 files changed, 54 insertions(+), 100 deletions(-) diff --git a/.github/workflows/report-embeds.yml b/.github/workflows/report-embeds.yml index 28feb21a6b..1c6caadf91 100644 --- a/.github/workflows/report-embeds.yml +++ b/.github/workflows/report-embeds.yml @@ -1,11 +1,8 @@ name: Report embeds -# Checks live W&B report embeds () in docs pages. Embeds are -# discovered by scanning the .mdx sources — there is no registry. -# - On PRs: static check (every embed has a sibling prose link and sits on a page). -# - Weekly: URL liveness, filing an issue on failure. Liveness never blocks PRs — -# wandb.ai rate-limits crawlers, so network results are flaky by design (this -# is also why wandb.ai is excluded from the site-wide lychee check). +# Checks live W&B report embeds () in docs pages, discovered by +# scanning the .mdx sources. Each run checks placement + URL liveness. +# On PRs it blocks; on the weekly run it files an issue instead (never blocks). on: pull_request: @@ -24,11 +21,10 @@ concurrency: permissions: contents: read - issues: write # scheduled liveness files an issue on failure + issues: write # the weekly run files an issue on failure jobs: - static: - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + check: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -37,23 +33,13 @@ jobs: python-version: "3.12" - name: Unit tests run: python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v - - name: Embed consistency (prose link) - run: python3 scripts/report-embeds/check_embeds.py --mode static --github - - liveness: - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - name: Liveness check + - name: Check embeds id: check - continue-on-error: true - run: python3 scripts/report-embeds/check_embeds.py --mode liveness --output-md ./embed-report.md + # Block PRs; on schedule/dispatch don't fail the run — file an issue below. + continue-on-error: ${{ github.event_name != 'pull_request' }} + run: python3 scripts/report-embeds/check_embeds.py --github --output-md ./embed-report.md - name: Create issue from file - if: steps.check.outcome == 'failure' + if: github.event_name != 'pull_request' && steps.check.outcome == 'failure' uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6 with: title: Fix broken report embeds in docs.wandb.ai diff --git a/AGENTS.md b/AGENTS.md index b0e05f3c03..62795767a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,7 +99,7 @@ To check or apply style, read the relevant pass file(s) and use them as guidance response="A concise example ARIA response" /> ``` -- **Live report embeds**: To show a live, interactive W&B report on a page, import `/snippets/WandbReport.jsx` and render it with `src` (the report URL), `title` (an accessible description), and optional `height` (500–800; default 640): +- **Live report embeds**: import `/snippets/WandbReport.jsx` and render it with `src` (report URL), `title` (accessible description), and optional `height` (500–800; default 640). The component renders its own "View Report" button, so no separate link is needed. ```mdx import { WandbReport } from '/snippets/WandbReport.jsx'; @@ -111,13 +111,12 @@ To check or apply style, read the relevant pass file(s) and use them as guidance /> ``` - The component renders a "View Report" button linking to the report, so no separate prose link is needed. Still, introduce each embed in the surrounding text so the page reads well and states what the reader should take from it. CI (`scripts/report-embeds/check_embeds.py`) discovers embeds by scanning the `.mdx` sources and checks placement and URL liveness — see [`scripts/report-embeds/README.md`](scripts/report-embeds/README.md). + - **Regular reports only, not Fully Connected articles** — FC articles keep their full blog chrome in a frame and look broken. + - **Anonymous-viewable** — a public-project report or a [view-only link](/models/reports/cross-project-reports#view-only-report-links). The URL (with any `?accessToken=`) ships in public source and git history, so treat the report as public forever. + - **One or two per page**, skinny and purpose-built — each iframe boots the full W&B app at a fixed height, in its own light theme. + - **English sources only** — not `ja/`, `ko/`, `fr/`. - Rules for every embed: - - **The report must be viewable by anonymous visitors**: a report in a public project, or one shared via a view-only link (`Share` → "anyone with the link can view"; see [View-only report links](/models/reports/cross-project-reports#view-only-report-links)). The URL — including any `?accessToken=` — ships in public source and git history, so treat the report as public forever and never embed sensitive data. - - **Regular reports only, not Fully Connected articles.** FC articles keep their full blog chrome in a frame and look broken. Regular reports render in a slim embed view. - - **Keep it skinny and sparse**: prefer purpose-built reports (ideally a single panel grid), one or two per page maximum. Each iframe boots the full W&B app, is a fixed height with inner scroll (no auto-resize), and renders its own light theme regardless of the docs dark-mode toggle — so article-length reports show as a tall scroll region, not the whole page at once. - - English sources only — never add embeds to `ja/`, `ko/`, or `fr/` copies. + CI enforces placement and liveness via `scripts/report-embeds/check_embeds.py`. ## Working with the repository diff --git a/scripts/report-embeds/README.md b/scripts/report-embeds/README.md index 4045063b20..3af15bc00f 100644 --- a/scripts/report-embeds/README.md +++ b/scripts/report-embeds/README.md @@ -1,62 +1,41 @@ # Report embeds -Tooling that keeps live W&B report embeds honest. A report embedded with the -[``](../../snippets/WandbReport.jsx) component is database content: -it can drift or vanish without a docs PR. `check_embeds.py` finds every embed by -scanning the `.mdx` sources (no registry to maintain) and checks it. - -## Files - -- `check_embeds.py` — scans English `.mdx` for `` embeds and checks - them (recognizable report URL, on a page, and URL liveness). -- `tests/test_check_embeds.py` — unit tests (no network). - -CI wiring lives in [`.github/workflows/report-embeds.yml`](../../.github/workflows/report-embeds.yml). - -## Add an embedded report - -1. Confirm the report is a **regular W&B report, not a Fully Connected article** - (FC articles keep their full blog chrome in a frame and look broken), and that - it is viewable by anonymous visitors — a report in a public project, or one - shared via a view-only ("magic") link (`Share` → "anyone with the link can - view"). The URL, including any `?accessToken=`, ships in public source and git - history, so treat the report as public forever. Never embed sensitive data. -2. On the page, add the component. It renders a "View Report" button linking to - the report, so no separate prose link is needed. The `src` URL is in the MDX - source, so agents and the llms.txt export still see it. - - ```mdx - import { WandbReport } from '/snippets/WandbReport.jsx'; - - - ``` - -3. Validate locally: - - ```bash - python3 scripts/report-embeds/check_embeds.py --mode static - ``` +`check_embeds.py` keeps live W&B report embeds honest. It finds every +[``](../../snippets/WandbReport.jsx) by scanning the English `.mdx` +sources (no registry), then checks each one: it sits on a page, has a +recognizable report URL, and still renders anonymously over the network. + +## Embed a report + +Use a **regular W&B report, not a Fully Connected article** (FC articles keep +their full blog chrome in a frame and look broken), viewable anonymously — in a +public project or shared via a view-only link (`Share` → "anyone with the link +can view"). The URL, including any `?accessToken=`, ships in public source and +git history, so treat the report as public forever. Add the component (it +renders a "View Report" button, so no separate link is needed): + +```mdx +import { WandbReport } from '/snippets/WandbReport.jsx'; + + +``` ## Run the checks ```bash -python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v -python3 scripts/report-embeds/check_embeds.py --mode static # URL + placement, no network -python3 scripts/report-embeds/check_embeds.py --mode liveness # anonymous HTTP per URL -python3 scripts/report-embeds/check_embeds.py --mode all # both +python3 scripts/report-embeds/check_embeds.py # scan + liveness +python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' ``` -Static errors (embed in a shared snippet, unrecognizable src) fail CI on PRs. -Liveness runs on a weekly schedule and files an issue on failure — it does not -block PRs, because `wandb.ai` rate-limits crawlers and network results are flaky -(this is also why `wandb.ai` is excluded from the site-wide lychee check). +CI ([`report-embeds.yml`](../../.github/workflows/report-embeds.yml)) runs this on +PRs that touch embeds and weekly (filing an issue if a report has gone dead). ## Component coupling `check_embeds.py` keys on the literal tag `WandbReport` and the `src` prop -(`COMPONENT_RE` / `SRC_ATTR_RE` at the top of the file). If -`snippets/WandbReport.jsx` renames either, update those constants. +(`COMPONENT_RE` / `SRC_ATTR_RE`). If `snippets/WandbReport.jsx` renames either, +update those constants. diff --git a/scripts/report-embeds/check_embeds.py b/scripts/report-embeds/check_embeds.py index 1461d28a99..cbeea706f5 100644 --- a/scripts/report-embeds/check_embeds.py +++ b/scripts/report-embeds/check_embeds.py @@ -2,12 +2,9 @@ """Check W&B report embeds used in docs pages. Embeds are discovered by scanning the English `.mdx` sources for the -`` component — there is no registry to keep in sync. - -Three modes: - static - every embed sits on a page and has a recognizable report URL (no network) - liveness - anonymous HTTP check that each embedded report URL still renders - all - both +`` component — there is no registry to keep in sync. Every run +checks placement and a recognizable report URL, then verifies over the network +that each report still renders anonymously. No third-party dependencies (standard library only). @@ -186,10 +183,10 @@ def emit_github_annotations(findings: Iterable[Finding]) -> None: print(f"::error {loc}::[{f.code}] {f.message}") -def render_markdown(findings: list[Finding], *, embeds: int, checked_urls: int) -> str: +def render_markdown(findings: list[Finding], *, embeds: int) -> str: lines = ["# Report embed check", ""] if not findings: - lines.append(f"✅ All checks passed ({embeds} embed(s), {checked_urls} URL(s) checked).") + lines.append(f"✅ All {embeds} embed(s) render anonymously.") return "\n".join(lines) + "\n" lines += [f"Found {len(findings)} problem(s).", "", "| Code | Location | Detail |", "| --- | --- | --- |"] lines += [f"| {f.code} | {_loc(f)} | {f.message} |" for f in findings] @@ -201,19 +198,13 @@ def render_markdown(findings: list[Finding], *, embeds: int, checked_urls: int) # --------------------------------------------------------------------------- # def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--mode", choices=["static", "liveness", "all"], default="static") parser.add_argument("--root", type=Path, default=REPO_ROOT) parser.add_argument("--github", action="store_true", help="emit GitHub Actions annotations") parser.add_argument("--output-md", type=Path, help="write a Markdown report to this file") args = parser.parse_args(argv) - static_findings, embeds = scan(args.root) - - findings: list[Finding] = list(static_findings) if args.mode in ("static", "all") else [] - checked_urls = 0 - if args.mode in ("liveness", "all"): - findings += check_liveness(embeds) - checked_urls = len(embeds) + findings, embeds = scan(args.root) + findings += check_liveness(embeds) if args.github: emit_github_annotations(findings) @@ -222,11 +213,10 @@ def main(argv: list[str] | None = None) -> int: print(f"ERROR [{f.code}] {_loc(f)}: {f.message}") if args.output_md: - args.output_md.write_text(render_markdown(findings, embeds=len(embeds), checked_urls=checked_urls), encoding="utf-8") + args.output_md.write_text(render_markdown(findings, embeds=len(embeds)), encoding="utf-8") if not findings: - checked = f", {checked_urls} URL(s) checked" if args.mode in ("liveness", "all") else "" - print(f"OK: {len(embeds)} report embed(s) found{checked}.") + print(f"OK: {len(embeds)} report embed(s) checked.") return 1 if findings else 0 From b2b56406ff4c8d8cb0401d11a1826bbf38d01fe3 Mon Sep 17 00:00:00 2001 From: John Mulhausen Date: Wed, 22 Jul 2026 13:20:48 -0400 Subject: [PATCH 5/7] report-embeds: fold tests into check_embeds.py One file, not two. The tests move into a `Tests` class in check_embeds.py and call the functions directly, dropping the import-by-path scaffolding. Run them with: python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' Reroute the two references (the workflow's unit-test step and the README) to the new command and delete scripts/report-embeds/tests/. Co-Authored-By: Claude Fable 5 --- .github/workflows/report-embeds.yml | 2 +- scripts/report-embeds/README.md | 2 +- scripts/report-embeds/check_embeds.py | 42 ++++++++++ .../report-embeds/tests/test_check_embeds.py | 79 ------------------- 4 files changed, 44 insertions(+), 81 deletions(-) delete mode 100644 scripts/report-embeds/tests/test_check_embeds.py diff --git a/.github/workflows/report-embeds.yml b/.github/workflows/report-embeds.yml index 1c6caadf91..bbf4395601 100644 --- a/.github/workflows/report-embeds.yml +++ b/.github/workflows/report-embeds.yml @@ -32,7 +32,7 @@ jobs: with: python-version: "3.12" - name: Unit tests - run: python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' -v + run: python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' -v - name: Check embeds id: check # Block PRs; on schedule/dispatch don't fail the run — file an issue below. diff --git a/scripts/report-embeds/README.md b/scripts/report-embeds/README.md index 3af15bc00f..ad83577be3 100644 --- a/scripts/report-embeds/README.md +++ b/scripts/report-embeds/README.md @@ -28,7 +28,7 @@ import { WandbReport } from '/snippets/WandbReport.jsx'; ```bash python3 scripts/report-embeds/check_embeds.py # scan + liveness -python3 -m unittest discover -s scripts/report-embeds/tests -p 'test_*.py' +python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' # unit tests ``` CI ([`report-embeds.yml`](../../.github/workflows/report-embeds.yml)) runs this on diff --git a/scripts/report-embeds/check_embeds.py b/scripts/report-embeds/check_embeds.py index cbeea706f5..bfda6969ed 100644 --- a/scripts/report-embeds/check_embeds.py +++ b/scripts/report-embeds/check_embeds.py @@ -221,5 +221,47 @@ def main(argv: list[str] | None = None) -> int: return 1 if findings else 0 +# --------------------------------------------------------------------------- # +# Tests — python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' +# --------------------------------------------------------------------------- # +import tempfile +import unittest + + +class Tests(unittest.TestCase): + def _scan(self, files: dict[str, str]): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + for rel, content in files.items(): + fp = root / rel + fp.parent.mkdir(parents=True, exist_ok=True) + fp.write_text(content, encoding="utf-8") + return scan(root) + + def test_extract(self): + self.assertEqual(extract_embeds(''), [("x--Vmlldzo1", 1)]) + self.assertEqual(extract_embeds('a\n\n'), [("y---Vmlldzo2", 2)]) + + def test_valid_and_dedup(self): + src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" + embed = f'\n' + findings, embeds = self._scan({"models/a.mdx": embed, "models/b.mdx": embed}) + self.assertEqual(findings, []) + self.assertEqual(embeds, [(src, "models/a.mdx", 1)]) # deduped by URL + + def test_bad_src(self): + findings, embeds = self._scan({"models/x.mdx": '\n'}) + self.assertEqual([f.code for f in findings], ["BAD_EMBED_SRC"]) + self.assertEqual(embeds, []) + + def test_snippet_errors_and_masked_ignored(self): + src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" + embed = f'\n' + snippet, _ = self._scan({"snippets/_includes/t.mdx": embed}) + self.assertEqual([f.code for f in snippet], ["EMBED_IN_SNIPPET"]) + for masked in (f"```mdx\n{embed}```\n", "{/* " + embed + "*/}\n"): + self.assertEqual(self._scan({"models/x.mdx": masked}), ([], [])) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/report-embeds/tests/test_check_embeds.py b/scripts/report-embeds/tests/test_check_embeds.py deleted file mode 100644 index b9a3d69523..0000000000 --- a/scripts/report-embeds/tests/test_check_embeds.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Unit tests for the report-embed checker. No network.""" - -from __future__ import annotations - -import importlib.util -import sys -import tempfile -import unittest -from pathlib import Path - -MODULE_PATH = Path(__file__).resolve().parents[1] / "check_embeds.py" - - -def _load(): - spec = importlib.util.spec_from_file_location("check_embeds", MODULE_PATH) - mod = importlib.util.module_from_spec(spec) - assert spec.loader - sys.modules[spec.name] = mod - spec.loader.exec_module(mod) - return mod - - -m = _load() - - -class ExtractionTests(unittest.TestCase): - def test_self_closing(self): - text = '' - self.assertEqual(m.extract_embeds(text), [("https://wandb.ai/w/p/reports/Foo--Vmlldzo1", 1)]) - - def test_multiline_and_brace_src(self): - text = 'x\n\n' - self.assertEqual(m.extract_embeds(text), [("https://wandb.ai/w/p/reports/Foo---Vmlldzo2", 2)]) - - -class ScanTests(unittest.TestCase): - def _run(self, files: dict[str, str]): - with tempfile.TemporaryDirectory() as d: - root = Path(d) - for rel, content in files.items(): - fp = root / rel - fp.parent.mkdir(parents=True, exist_ok=True) - fp.write_text(content, encoding="utf-8") - return m.scan(root) - - def test_valid_embed_passes(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = f'\n' - findings, embeds = self._run({"models/x.mdx": page}) - self.assertEqual(findings, []) - self.assertEqual(embeds, [(src, "models/x.mdx", 1)]) - - def test_bad_embed_src(self): - page = '\n' - findings, embeds = self._run({"models/x.mdx": page}) - self.assertTrue(any(f.code == "BAD_EMBED_SRC" for f in findings)) - self.assertEqual(embeds, []) - - def test_duplicate_url_deduped_for_liveness(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - page = f'\n' - findings, embeds = self._run({"models/a.mdx": page, "models/b.mdx": page}) - self.assertEqual(findings, []) - self.assertEqual(len(embeds), 1) - - def test_masked_regions_ignored(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - embed = f'\n' - for page in (f"```mdx\n{embed}```\n", "{/* later:\n" + embed + "*/}\n"): - self.assertEqual(self._run({"models/x.mdx": page}), ([], [])) - - def test_embed_in_snippet_errors(self): - src = "https://wandb.ai/w/p/reports/Foo--Vmlldzo12345" - findings, _ = self._run({"snippets/_includes/t.mdx": f'\n'}) - self.assertTrue(any(f.code == "EMBED_IN_SNIPPET" for f in findings)) - - -if __name__ == "__main__": - unittest.main() From 443bc112fd1bea5a78503bc539c6bb3d3a097fc0 Mon Sep 17 00:00:00 2001 From: John Mulhausen Date: Wed, 22 Jul 2026 13:26:06 -0400 Subject: [PATCH 6/7] report-embeds: run only on the weekly cron, never on PRs wandb.ai rate-limits crawlers, so liveness results are too flaky to gate merges on. Drop the pull_request trigger (and with it the unit-test step, --github annotations, paths filter, and concurrency group); the workflow is now just the weekly liveness check that files an issue on failure. Update the README and AGENTS.md notes to match (no PR gating). Co-Authored-By: Claude Fable 5 --- .github/workflows/report-embeds.yml | 30 +++++++++-------------------- AGENTS.md | 2 +- scripts/report-embeds/README.md | 5 +++-- 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/.github/workflows/report-embeds.yml b/.github/workflows/report-embeds.yml index bbf4395601..ff2e157cee 100644 --- a/.github/workflows/report-embeds.yml +++ b/.github/workflows/report-embeds.yml @@ -1,45 +1,33 @@ name: Report embeds -# Checks live W&B report embeds () in docs pages, discovered by -# scanning the .mdx sources. Each run checks placement + URL liveness. -# On PRs it blocks; on the weekly run it files an issue instead (never blocks). +# Weekly liveness check of the live W&B report embeds () in docs +# pages, discovered by scanning the .mdx sources. Files an issue if a report has +# gone dead. Deliberately does not run on PRs: wandb.ai rate-limits crawlers, so +# network results are too flaky to gate merges on. on: - pull_request: - types: [opened, synchronize, reopened] - paths: - - "scripts/report-embeds/**" - - "snippets/WandbReport.jsx" - - "**/*.mdx" schedule: - cron: "17 6 * * 1" # Weekly, Monday 06:17 UTC workflow_dispatch: -concurrency: - group: report-embeds-${{ github.ref }} - cancel-in-progress: true - permissions: contents: read - issues: write # the weekly run files an issue on failure + issues: write # files an issue on failure jobs: - check: + liveness: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" - - name: Unit tests - run: python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' -v - name: Check embeds id: check - # Block PRs; on schedule/dispatch don't fail the run — file an issue below. - continue-on-error: ${{ github.event_name != 'pull_request' }} - run: python3 scripts/report-embeds/check_embeds.py --github --output-md ./embed-report.md + continue-on-error: true + run: python3 scripts/report-embeds/check_embeds.py --output-md ./embed-report.md - name: Create issue from file - if: github.event_name != 'pull_request' && steps.check.outcome == 'failure' + if: steps.check.outcome == 'failure' uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6 with: title: Fix broken report embeds in docs.wandb.ai diff --git a/AGENTS.md b/AGENTS.md index 62795767a9..407d41611c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,7 +116,7 @@ To check or apply style, read the relevant pass file(s) and use them as guidance - **One or two per page**, skinny and purpose-built — each iframe boots the full W&B app at a fixed height, in its own light theme. - **English sources only** — not `ja/`, `ko/`, `fr/`. - CI enforces placement and liveness via `scripts/report-embeds/check_embeds.py`. + A weekly CI job (`scripts/report-embeds/check_embeds.py`) verifies each embedded report still renders and files an issue if one breaks. It does not gate PRs. ## Working with the repository diff --git a/scripts/report-embeds/README.md b/scripts/report-embeds/README.md index ad83577be3..85250047dd 100644 --- a/scripts/report-embeds/README.md +++ b/scripts/report-embeds/README.md @@ -31,8 +31,9 @@ python3 scripts/report-embeds/check_embeds.py # scan + liveness python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' # unit tests ``` -CI ([`report-embeds.yml`](../../.github/workflows/report-embeds.yml)) runs this on -PRs that touch embeds and weekly (filing an issue if a report has gone dead). +CI ([`report-embeds.yml`](../../.github/workflows/report-embeds.yml)) runs this +weekly and files an issue if a report has gone dead. It does not run on PRs — +`wandb.ai` rate-limits crawlers, so network results are too flaky to gate merges. ## Component coupling From e34265e9a14fdfe9957a6e80b076a2616e687b75 Mon Sep 17 00:00:00 2001 From: John Mulhausen Date: Wed, 22 Jul 2026 13:34:08 -0400 Subject: [PATCH 7/7] report-embeds: trim check_embeds.py (267 -> 214 lines) - Remove the now-dead --github flag and emit_github_annotations() (the cron workflow only uses --output-md), and the unused Iterable import. - Fold _line_of and _blank into their one caller; tighten scan(), render_markdown, and the docstring; collapse the section banners. No behavior change; retry/backoff logic and tests are untouched. Co-Authored-By: Claude Fable 5 --- scripts/report-embeds/check_embeds.py | 125 ++++++++------------------ 1 file changed, 36 insertions(+), 89 deletions(-) diff --git a/scripts/report-embeds/check_embeds.py b/scripts/report-embeds/check_embeds.py index bfda6969ed..8b687f55f6 100644 --- a/scripts/report-embeds/check_embeds.py +++ b/scripts/report-embeds/check_embeds.py @@ -1,16 +1,12 @@ #!/usr/bin/env python3 -"""Check W&B report embeds used in docs pages. +"""Check the live W&B report embeds () in docs pages. -Embeds are discovered by scanning the English `.mdx` sources for the -`` component — there is no registry to keep in sync. Every run -checks placement and a recognizable report URL, then verifies over the network -that each report still renders anonymously. +Embeds are found by scanning the English .mdx sources (no registry): each is +checked for placement and a recognizable report URL, then fetched anonymously +to confirm the report still renders. -No third-party dependencies (standard library only). - -The tag name `WandbReport` and the `src` prop below are a contract with -snippets/WandbReport.jsx. If the component renames either, update COMPONENT_RE -and SRC_ATTR_RE here (see scripts/report-embeds/README.md). +COMPONENT_RE / SRC_ATTR_RE are a contract with snippets/WandbReport.jsx — keep +them in sync if the component's tag or `src` prop is renamed. """ from __future__ import annotations @@ -22,32 +18,27 @@ import urllib.request from dataclasses import dataclass from pathlib import Path -from typing import Iterable, Iterator +from typing import Iterator REPO_ROOT = Path(__file__).resolve().parents[2] # A report URL ends in `--` or `---` where starts with "Vmlldz". REPORT_ID_RE = re.compile(r"-{2,3}(Vmlldz[A-Za-z0-9]+)") -# Match a opening tag (self-closing or not), including -# multi-line attribute lists. `[^>]` also matches newlines. +# A opening tag (self-closing or not), across newlines. COMPONENT_RE = re.compile(r"]*?(?:/>|>)") -# Extract the src value: src="...", src='...', src={"..."}, src={'...'}, src={`...`} +# The src value: src="...", src='...', or src={"..."} / {'...'} / {`...`}. SRC_ATTR_RE = re.compile(r"""src\s*=\s*\{?\s*['"`]([^'"`]+)['"`]""") -# Fenced code blocks and MDX comments hold example/illustrative markup, not live -# embeds — mask them before scanning so a documented usage or a -# report link shown as sample code isn't treated as the real thing. FENCE_RE = re.compile(r"```.*?```", re.DOTALL) MDX_COMMENT_RE = re.compile(r"\{/\*.*?\*/\}", re.DOTALL) # Locale content is managed by a separate translation pipeline (AGENTS.md); -# the checker only governs English sources. +# only English sources are governed here. SKIP_PREFIXES = ( "ja/", "ko/", "fr/", "snippets/ja/", "snippets/ko/", "snippets/fr/", "node_modules/", ".git/", ) -# Snippets are shared partials, not pages. Embeds belong on pages only. -SNIPPETS_PREFIX = "snippets/" +SNIPPETS_PREFIX = "snippets/" # embeds belong on pages, not shared partials LOGIN_MARKERS = ("/login", "/signin", "/site/login", "/authorize") @@ -59,9 +50,7 @@ class Finding: line: int | None = None -# --------------------------------------------------------------------------- # -# MDX scanning -# --------------------------------------------------------------------------- # +# --- MDX scanning --- def iter_mdx(root: Path) -> Iterator[Path]: for path in sorted(root.rglob("*.mdx")): rel = path.relative_to(root).as_posix() @@ -69,81 +58,61 @@ def iter_mdx(root: Path) -> Iterator[Path]: yield path -def _line_of(text: str, index: int) -> int: - return text.count("\n", 0, index) + 1 - - def extract_embeds(text: str) -> list[tuple[str, int]]: """Return (src, line) for every whose src can be parsed.""" - out: list[tuple[str, int]] = [] + out = [] for m in COMPONENT_RE.finditer(text): src_m = SRC_ATTR_RE.search(m.group(0)) if src_m: - out.append((src_m.group(1), _line_of(text, m.start()))) + out.append((src_m.group(1), text.count("\n", 0, m.start()) + 1)) return out -def _blank(match: re.Match) -> str: - # Replace matched span with same-length whitespace so byte offsets — and - # therefore reported line numbers — are preserved. - return re.sub(r"[^\n]", " ", match.group(0)) - - def mask_noncontent(text: str) -> str: - return MDX_COMMENT_RE.sub(_blank, FENCE_RE.sub(_blank, text)) + # Blank fenced code + MDX comments with same-length whitespace (keeps line + # numbers stable) so example markup isn't scanned as a live embed. + def blank(m: re.Match) -> str: + return re.sub(r"[^\n]", " ", m.group(0)) + return MDX_COMMENT_RE.sub(blank, FENCE_RE.sub(blank, text)) def scan(root: Path) -> tuple[list[Finding], list[tuple[str, str, int]]]: - """Scan English .mdx for embeds. - - Returns (findings, embeds). `embeds` is (url, page, line) deduplicated by URL - for the liveness pass; findings cover the static checks. - """ + """Return (findings, embeds); embeds is (url, page, line) deduped by URL.""" findings: list[Finding] = [] embeds: list[tuple[str, str, int]] = [] seen: set[str] = set() - for path in iter_mdx(root): rel = path.relative_to(root).as_posix() - text = mask_noncontent(path.read_text(encoding="utf-8")) - occurrences = extract_embeds(text) + occurrences = extract_embeds(mask_noncontent(path.read_text(encoding="utf-8"))) if not occurrences: continue if rel.startswith(SNIPPETS_PREFIX): - findings.append(Finding("EMBED_IN_SNIPPET", " must be used on a page, not in a shared snippet", rel, occurrences[0][1])) + findings.append(Finding("EMBED_IN_SNIPPET", " must be used on a page, not a shared snippet", rel, occurrences[0][1])) continue - for src, line in occurrences: if not REPORT_ID_RE.search(src): findings.append(Finding("BAD_EMBED_SRC", f"src `{src}` is not a recognizable report URL (needs --Vmlldz...)", rel, line)) - continue - if src not in seen: + elif src not in seen: seen.add(src) embeds.append((src, rel, line)) - return findings, embeds -# --------------------------------------------------------------------------- # -# Liveness -# --------------------------------------------------------------------------- # +# --- Liveness --- def check_url(url: str, *, timeout: int, attempts: int, delay: float) -> Finding | None: """Anonymous GET; require a final 200 that is not a login redirect.""" - req = urllib.request.Request( - url, - headers={ - "User-Agent": "Mozilla/5.0 (compatible; wandb-docs-embed-check/1.0)", - "Accept": "text/html,application/xhtml+xml", - }, - ) + req = urllib.request.Request(url, headers={ + "User-Agent": "Mozilla/5.0 (compatible; wandb-docs-embed-check/1.0)", + "Accept": "text/html,application/xhtml+xml", + }) last_err = "" for attempt in range(1, attempts + 1): try: with urllib.request.urlopen(req, timeout=timeout) as resp: if resp.getcode() != 200: return Finding("URL_NOT_200", f"{url}: final status {resp.getcode()}") - if any(marker in resp.geturl() for marker in LOGIN_MARKERS): - return Finding("URL_LOGIN_REDIRECT", f"{url}: redirected to login ({resp.geturl()}) — magic link likely revoked") + if any(mk in resp.geturl() for mk in LOGIN_MARKERS): + return Finding("URL_LOGIN_REDIRECT", f"{url}: redirected to login ({resp.geturl()}) — link likely revoked") return None except urllib.error.HTTPError as exc: if exc.code in (429, 500, 502, 503, 504) and attempt < attempts: @@ -170,60 +139,38 @@ def check_liveness(embeds: list[tuple[str, str, int]], *, timeout: int = 30, att return findings -# --------------------------------------------------------------------------- # -# Output -# --------------------------------------------------------------------------- # +# --- Output + CLI --- def _loc(f: Finding) -> str: return (f.path or "-") + (f":{f.line}" if f.path and f.line else "") -def emit_github_annotations(findings: Iterable[Finding]) -> None: - for f in findings: - loc = (f"file={f.path}" + (f",line={f.line}" if f.line else "")) if f.path else "" - print(f"::error {loc}::[{f.code}] {f.message}") - - def render_markdown(findings: list[Finding], *, embeds: int) -> str: - lines = ["# Report embed check", ""] if not findings: - lines.append(f"✅ All {embeds} embed(s) render anonymously.") - return "\n".join(lines) + "\n" - lines += [f"Found {len(findings)} problem(s).", "", "| Code | Location | Detail |", "| --- | --- | --- |"] + return f"# Report embed check\n\n✅ All {embeds} embed(s) render anonymously.\n" + lines = ["# Report embed check", "", f"Found {len(findings)} problem(s).", "", "| Code | Location | Detail |", "| --- | --- | --- |"] lines += [f"| {f.code} | {_loc(f)} | {f.message} |" for f in findings] return "\n".join(lines) + "\n" -# --------------------------------------------------------------------------- # -# Main -# --------------------------------------------------------------------------- # def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=REPO_ROOT) - parser.add_argument("--github", action="store_true", help="emit GitHub Actions annotations") parser.add_argument("--output-md", type=Path, help="write a Markdown report to this file") args = parser.parse_args(argv) findings, embeds = scan(args.root) findings += check_liveness(embeds) - if args.github: - emit_github_annotations(findings) - else: - for f in findings: - print(f"ERROR [{f.code}] {_loc(f)}: {f.message}") - + for f in findings: + print(f"ERROR [{f.code}] {_loc(f)}: {f.message}") if args.output_md: args.output_md.write_text(render_markdown(findings, embeds=len(embeds)), encoding="utf-8") - if not findings: print(f"OK: {len(embeds)} report embed(s) checked.") - return 1 if findings else 0 -# --------------------------------------------------------------------------- # -# Tests — python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' -# --------------------------------------------------------------------------- # +# --- Tests — python3 -m unittest discover -s scripts/report-embeds -p 'check_embeds.py' --- import tempfile import unittest