From 323b26902f039b24fc2dc60bbbe0f162fe78aff0 Mon Sep 17 00:00:00 2001 From: saagpatel Date: Sat, 27 Jun 2026 20:44:23 -0700 Subject: [PATCH] feat(portfolio-truth): add --portfolio-truth-allow-empty-notion carry-forward The nightly portfolio-maintenance job has failed every night since 2026-06-23: it runs `report --portfolio-truth` with no NOTION_TOKEN, and the data-safety guard correctly refuses to overwrite existing truth (142 Notion context rows) with a 0-row headless run. This froze portfolio-truth-latest.json and degraded the personal-ops ghra hub spoke. Add an opt-in --portfolio-truth-allow-empty-notion flag: when live Notion is unavailable, carry forward the prior artifact's per-project advisory context (load_prior_notion_context) instead of dropping to zero, so the refresh updates risk/activity/git signals without losing advisory data. The flag suppresses the drop guard (operator explicitly opted into empty-Notion publishing); source_summary records notion_context_carried_forward for provenance; a logger.warning fires on carry-forward so a real Notion regression stays observable. Default (flag off) behavior is unchanged: the guard still protects against silent Notion drop. Tests: reconstructor units, publish-level carry-forward + guard-still-fires + flag-suppresses-guard, CLI report-subcommand parse, and end-to-end via main(). --- src/cli.py | 21 +++ src/portfolio_truth_publish.py | 18 +- src/portfolio_truth_reconcile.py | 55 ++++++ tests/test_portfolio_truth.py | 283 +++++++++++++++++++++++++++++++ 4 files changed, 375 insertions(+), 2 deletions(-) diff --git a/src/cli.py b/src/cli.py index 434bf83..2af0871 100644 --- a/src/cli.py +++ b/src/cli.py @@ -433,6 +433,16 @@ def _build_report_subparser(subparsers: argparse._SubParsersAction) -> None: # p.add_argument( "--portfolio-truth", action="store_true", help="Generate canonical portfolio truth snapshot" ) + p.add_argument( + "--portfolio-truth-allow-empty-notion", + action="store_true", + help=( + "When live Notion context is unavailable, carry forward the previously " + "published Notion context instead of refusing to publish. For headless or " + "scheduled refreshes that update risk/activity signals without dropping " + "advisory context to zero." + ), + ) p.add_argument( "--portfolio-context-recovery", action="store_true", @@ -674,6 +684,16 @@ def build_parser() -> argparse.ArgumentParser: "risk factor (requires a prior `audit report --ghas-alerts` run)" ), ) + parser.add_argument( + "--portfolio-truth-allow-empty-notion", + action="store_true", + help=( + "When live Notion context is unavailable, carry forward the previously " + "published Notion context instead of refusing to publish. For headless or " + "scheduled refreshes that update risk/activity signals without dropping " + "advisory context to zero." + ), + ) parser.add_argument( "--portfolio-context-recovery", action="store_true", @@ -5390,6 +5410,7 @@ def _run_portfolio_truth_mode(args) -> None: catalog_path=Path(args.catalog) if args.catalog else None, legacy_registry_path=legacy_registry_path, include_notion=True, + allow_empty_notion=getattr(args, "portfolio_truth_allow_empty_notion", False), release_count_by_name=release_count_by_name, security_alerts_by_name=security_alerts_by_name, ) diff --git a/src/portfolio_truth_publish.py b/src/portfolio_truth_publish.py index 2a6f473..3be17cb 100644 --- a/src/portfolio_truth_publish.py +++ b/src/portfolio_truth_publish.py @@ -5,7 +5,10 @@ from dataclasses import dataclass from pathlib import Path -from src.portfolio_truth_reconcile import build_portfolio_truth_snapshot +from src.portfolio_truth_reconcile import ( + build_portfolio_truth_snapshot, + load_prior_notion_context, +) from src.portfolio_truth_render import render_portfolio_report_markdown, render_registry_markdown from src.portfolio_truth_types import truth_latest_path from src.portfolio_truth_validate import ( @@ -82,6 +85,7 @@ def publish_portfolio_truth( catalog_path: Path | None = None, legacy_registry_path: Path | None = None, include_notion: bool = True, + allow_empty_notion: bool = False, release_count_by_name: dict[str, int] | None = None, security_alerts_by_name: dict[str, dict] | None = None, ) -> PortfolioTruthPublishResult: @@ -91,11 +95,16 @@ def publish_portfolio_truth( registry_output=registry_output, portfolio_report_output=portfolio_report_output, ) + latest_path = truth_latest_path(output_dir) + notion_context_fallback = ( + load_prior_notion_context(latest_path) if allow_empty_notion else None + ) build_result = build_portfolio_truth_snapshot( workspace_root=workspace_root, catalog_path=catalog_path, legacy_registry_path=legacy_registry_path, include_notion=include_notion, + notion_context_fallback=notion_context_fallback, release_count_by_name=release_count_by_name, security_alerts_by_name=security_alerts_by_name, ) @@ -103,11 +112,11 @@ def publish_portfolio_truth( snapshot_stamp = build_result.snapshot.generated_at.strftime("%Y-%m-%dT%H%M%SZ") snapshot_path = output_dir / f"portfolio-truth-{snapshot_stamp}.json" - latest_path = truth_latest_path(output_dir) _guard_against_notion_context_drop( build_result.snapshot.source_summary, latest_path=latest_path, include_notion=include_notion, + allow_empty_notion=allow_empty_notion, ) latest_name = latest_path.name snapshot_json = json.dumps(build_result.snapshot.to_dict(), indent=2) + "\n" @@ -200,8 +209,13 @@ def _guard_against_notion_context_drop( *, latest_path: Path, include_notion: bool, + allow_empty_notion: bool = False, ) -> None: """Avoid overwriting local truth when Notion bootstrap silently disappears.""" + if allow_empty_notion: + # Operator explicitly opted into publishing without live Notion (a headless or + # scheduled refresh); prior advisory is carried forward where available. + return if not include_notion or not _notion_project_context_configured(): return current_rows = _int_value(source_summary.get("notion_context_rows")) diff --git a/src/portfolio_truth_reconcile.py b/src/portfolio_truth_reconcile.py index ca50dbf..ef15711 100644 --- a/src/portfolio_truth_reconcile.py +++ b/src/portfolio_truth_reconcile.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import logging from collections import Counter from dataclasses import dataclass @@ -180,6 +181,7 @@ def build_portfolio_truth_snapshot( catalog_path: Path | None = None, legacy_registry_path: Path | None = None, include_notion: bool = True, + notion_context_fallback: dict[str, dict[str, str]] | None = None, now: datetime | None = None, release_count_by_name: dict[str, int] | None = None, security_alerts_by_name: dict[str, dict] | None = None, @@ -188,6 +190,18 @@ def build_portfolio_truth_snapshot( catalog_data = load_portfolio_catalog(catalog_path) legacy_rows = load_legacy_registry_rows(legacy_registry_path) notion_context = load_safe_notion_project_context() if include_notion else {} + notion_context_carried_forward = False + if include_notion and not notion_context and notion_context_fallback: + # Live Notion was unavailable; carry forward the prior published context so + # a headless refresh updates risk/activity signals without dropping advisory + # data to zero. The caller opts in via publish_portfolio_truth(allow_empty_notion=True). + notion_context = notion_context_fallback + notion_context_carried_forward = True + logger.warning( + "Live Notion context unavailable; carrying forward %d project rows " + "from the prior portfolio-truth artifact.", + len(notion_context), + ) workspace_projects = discover_workspace_projects( workspace_root, @@ -217,6 +231,7 @@ def build_portfolio_truth_snapshot( "catalog_warnings": list(catalog_data.get("warnings") or []), "legacy_registry_rows": len(legacy_rows), "notion_context_rows": len(notion_context), + "notion_context_carried_forward": notion_context_carried_forward, "context_quality_counts": dict( Counter(project.derived.context_quality for project in projects) ), @@ -251,6 +266,46 @@ def build_portfolio_truth_snapshot( ) +def load_prior_notion_context(latest_path: Path) -> dict[str, dict[str, str]]: + """Reconstruct a Notion project-context map from a previously published + portfolio-truth artifact, keyed identically to live Notion context + (``_normalize(display_name)`` -> ``{portfolio_call, momentum, current_state}``). + + Used to carry advisory context forward on a headless refresh when a live + Notion token is unavailable, rather than overwriting local truth with zero + rows. Only projects that actually carried Notion advisory are returned, so + the resulting row count reflects real carried context. Returns an empty map + when the artifact is missing or malformed. + """ + try: + data = json.loads(latest_path.read_text()) + except (OSError, json.JSONDecodeError): + return {} + projects = data.get("projects") + if not isinstance(projects, list): + return {} + context: dict[str, dict[str, str]] = {} + for project in projects: + if not isinstance(project, dict): + continue + identity = project.get("identity") + advisory = project.get("advisory") + if not isinstance(identity, dict) or not isinstance(advisory, dict): + continue + display_name = str(identity.get("display_name", "")).strip() + portfolio_call = str(advisory.get("notion_portfolio_call", "")).strip() + momentum = str(advisory.get("notion_momentum", "")).strip() + current_state = str(advisory.get("notion_current_state", "")).strip() + if not display_name or not (portfolio_call or momentum or current_state): + continue + context[_normalize(display_name)] = { + "portfolio_call": portfolio_call, + "momentum": momentum, + "current_state": current_state, + } + return context + + def _duplicate_display_names(projects: list[PortfolioTruthProject]) -> list[str]: return sorted( name diff --git a/tests/test_portfolio_truth.py b/tests/test_portfolio_truth.py index 9e5a80c..e487d76 100644 --- a/tests/test_portfolio_truth.py +++ b/tests/test_portfolio_truth.py @@ -1096,6 +1096,289 @@ def test_publish_refuses_to_drop_existing_notion_context( assert json.loads(latest_path.read_text())["source_summary"]["notion_context_rows"] == 137 +def test_load_prior_notion_context_rebuilds_from_artifact(tmp_path: Path) -> None: + from src.portfolio_truth_reconcile import load_prior_notion_context + from src.registry_parser import _normalize + + latest_path = tmp_path / "portfolio-truth-latest.json" + latest_path.write_text( + json.dumps( + { + "source_summary": {"notion_context_rows": 2}, + "projects": [ + { + "identity": {"display_name": "CryptForge"}, + "advisory": { + "notion_portfolio_call": "Finish", + "notion_momentum": "Post-Build Review Done", + "notion_current_state": "Parked", + }, + }, + { + "identity": {"display_name": "NoNotion"}, + "advisory": { + "notion_portfolio_call": "", + "notion_momentum": "", + "notion_current_state": "", + }, + }, + ], + } + ) + + "\n" + ) + + context = load_prior_notion_context(latest_path) + + assert context[_normalize("CryptForge")] == { + "portfolio_call": "Finish", + "momentum": "Post-Build Review Done", + "current_state": "Parked", + } + assert _normalize("NoNotion") not in context + assert len(context) == 1 + + +def test_load_prior_notion_context_missing_or_malformed_returns_empty(tmp_path: Path) -> None: + from src.portfolio_truth_reconcile import load_prior_notion_context + + assert load_prior_notion_context(tmp_path / "absent.json") == {} + malformed = tmp_path / "malformed.json" + malformed.write_text("{ not json") + assert load_prior_notion_context(malformed) == {} + + +def test_publish_allow_empty_notion_carries_forward_prior_context( + portfolio_workspace: Path, + portfolio_catalog: Path, + legacy_registry: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + registry_output = portfolio_workspace / "project-registry.md" + report_output = portfolio_workspace / "PORTFOLIO-AUDIT-REPORT.md" + + # Live Notion unavailable (token lost) - the exact condition that breaks the nightly job. + monkeypatch.setattr( + "src.portfolio_truth_sources.load_notion_project_context", + lambda _config_dir: None, + ) + monkeypatch.setattr( + "src.portfolio_truth_publish._notion_project_context_configured", + lambda: True, + ) + + # Discover a real workspace project name to seed prior advisory against. + discovered = build_portfolio_truth_snapshot( + workspace_root=portfolio_workspace, + catalog_path=portfolio_catalog, + legacy_registry_path=legacy_registry, + include_notion=False, + ) + target_name = discovered.snapshot.projects[0].identity.display_name + + # Seed the prior artifact with the per-project advisory a tokened run produced. + latest_path = output_dir / "portfolio-truth-latest.json" + latest_path.write_text( + json.dumps( + { + "source_summary": {"notion_context_rows": 1}, + "projects": [ + { + "identity": {"display_name": target_name}, + "advisory": { + "notion_portfolio_call": "Ship", + "notion_momentum": "Active", + "notion_current_state": "Building", + }, + } + ], + } + ) + + "\n" + ) + + result = publish_portfolio_truth( + workspace_root=portfolio_workspace, + output_dir=output_dir, + registry_output=registry_output, + portfolio_report_output=report_output, + catalog_path=portfolio_catalog, + legacy_registry_path=legacy_registry, + include_notion=True, + allow_empty_notion=True, + ) + + published = json.loads(result.latest_path.read_text()) + summary = published["source_summary"] + assert summary["notion_context_rows"] == 1 + assert summary["notion_context_carried_forward"] is True + target = next( + item for item in published["projects"] if item["identity"]["display_name"] == target_name + ) + assert target["advisory"]["notion_portfolio_call"] == "Ship" + assert target["advisory"]["notion_current_state"] == "Building" + + +def test_publish_without_allow_empty_notion_still_guards( + portfolio_workspace: Path, + portfolio_catalog: Path, + legacy_registry: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Opt-in: with the flag off, the data-safety guard must still fire. + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / "portfolio-truth-latest.json").write_text( + json.dumps({"source_summary": {"notion_context_rows": 137}}) + "\n" + ) + monkeypatch.setattr( + "src.portfolio_truth_sources.load_notion_project_context", + lambda _config_dir: None, + ) + monkeypatch.setattr( + "src.portfolio_truth_publish._notion_project_context_configured", + lambda: True, + ) + + with pytest.raises(RuntimeError, match="0 Notion context rows"): + publish_portfolio_truth( + workspace_root=portfolio_workspace, + output_dir=output_dir, + registry_output=portfolio_workspace / "project-registry.md", + portfolio_report_output=portfolio_workspace / "PORTFOLIO-AUDIT-REPORT.md", + catalog_path=portfolio_catalog, + legacy_registry_path=legacy_registry, + include_notion=True, + allow_empty_notion=False, + ) + + +def test_report_subcommand_parses_allow_empty_notion_flag() -> None: + # The nightly job runs `audit report --portfolio-truth`; the new flag + # must be accepted on that exact path and default to opt-in off. + from src.cli import build_subcommand_parser + + parser = build_subcommand_parser() + enabled = parser.parse_args( + ["report", "testuser", "--portfolio-truth", "--portfolio-truth-allow-empty-notion"] + ) + assert enabled.portfolio_truth_allow_empty_notion is True + default = parser.parse_args(["report", "testuser", "--portfolio-truth"]) + assert default.portfolio_truth_allow_empty_notion is False + + +def test_cli_portfolio_truth_allow_empty_notion_carries_forward( + portfolio_workspace: Path, + portfolio_catalog: Path, + legacy_registry: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + + discovered = build_portfolio_truth_snapshot( + workspace_root=portfolio_workspace, + catalog_path=portfolio_catalog, + legacy_registry_path=legacy_registry, + include_notion=False, + ) + target_name = discovered.snapshot.projects[0].identity.display_name + + (output_dir / "portfolio-truth-latest.json").write_text( + json.dumps( + { + "source_summary": {"notion_context_rows": 1}, + "projects": [ + { + "identity": {"display_name": target_name}, + "advisory": { + "notion_portfolio_call": "Ship", + "notion_momentum": "Active", + "notion_current_state": "Building", + }, + } + ], + } + ) + + "\n" + ) + monkeypatch.setattr( + "src.portfolio_truth_sources.load_notion_project_context", + lambda _config_dir: None, + ) + monkeypatch.setattr( + "src.portfolio_truth_publish._notion_project_context_configured", + lambda: True, + ) + argv = [ + "audit", + "testuser", + "--portfolio-truth", + "--portfolio-truth-allow-empty-notion", + "--workspace-root", + str(portfolio_workspace), + "--output-dir", + str(output_dir), + "--catalog", + str(portfolio_catalog), + "--registry", + str(legacy_registry), + ] + monkeypatch.setattr("sys.argv", argv) + + # Must NOT raise SystemExit from the Notion-drop guard - carry-forward keeps rows > 0. + main() + + published = json.loads((output_dir / "portfolio-truth-latest.json").read_text()) + assert published["source_summary"]["notion_context_rows"] == 1 + assert published["source_summary"]["notion_context_carried_forward"] is True + + +def test_publish_allow_empty_notion_without_prior_context_publishes_zero( + portfolio_workspace: Path, + portfolio_catalog: Path, + legacy_registry: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Flag set, but nothing to carry forward (prior has a count yet no per-project + # advisory): the operator opted into empty-Notion publishing, so the guard must + # not block and the run publishes with zero carried rows. + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / "portfolio-truth-latest.json").write_text( + json.dumps({"source_summary": {"notion_context_rows": 137}, "projects": []}) + "\n" + ) + monkeypatch.setattr( + "src.portfolio_truth_sources.load_notion_project_context", + lambda _config_dir: None, + ) + monkeypatch.setattr( + "src.portfolio_truth_publish._notion_project_context_configured", + lambda: True, + ) + + result = publish_portfolio_truth( + workspace_root=portfolio_workspace, + output_dir=output_dir, + registry_output=portfolio_workspace / "project-registry.md", + portfolio_report_output=portfolio_workspace / "PORTFOLIO-AUDIT-REPORT.md", + catalog_path=portfolio_catalog, + legacy_registry_path=legacy_registry, + include_notion=True, + allow_empty_notion=True, + ) + + published = json.loads(result.latest_path.read_text()) + assert published["source_summary"]["notion_context_rows"] == 0 + assert published["source_summary"]["notion_context_carried_forward"] is False + + def test_context_recovery_plan_freezes_and_filters_targets( portfolio_workspace: Path, portfolio_catalog: Path,