diff --git a/.cz.yaml b/.cz.yaml index 67bf9ff0..caf6bea7 100644 --- a/.cz.yaml +++ b/.cz.yaml @@ -2,6 +2,6 @@ commitizen: name: cz_conventional_commits tag_format: $version - version: 2.0.1 + version: 2.1.3 version_files: - src/histdatacom/__init__.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..a7595c51 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +** +!Dockerfile +!pyproject.toml +!README.md +!LICENSE +!container +!container/constraints.txt +!src +!src/** diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac02cc7a..c6af21bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,28 @@ jobs: run: ./actionlint -color shell: bash + docs: + name: Build documentation + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v7.0.0 + + - name: Set up Python + uses: actions/setup-python@v6.3.0 + with: + python-version: "3.13" + cache: pip + + - name: Install documentation dependencies + run: | + python -m pip install --upgrade pip "setuptools>=77" wheel + python -m pip install -e ".[docs]" + + - name: Build documentation with warnings as errors + run: python -m sphinx -W --keep-going -b html docs docs/_build/html + test: name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -57,7 +79,7 @@ jobs: uses: actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: ${{ matrix.python-version }} cache: pip @@ -67,21 +89,8 @@ jobs: python -m pip install --upgrade pip "setuptools>=77" wheel python -m pip install -e ".[test]" - - name: Run tests with coverage - run: python -m pytest --cov=histdatacom --cov-report=term-missing:skip-covered --cov-report=xml --cov-report=html - - - name: Enforce coverage threshold - run: python -m coverage report - - - name: Upload coverage reports - if: always() - uses: actions/upload-artifact@v7.0.1 - with: - name: coverage-${{ matrix.os }}-py${{ matrix.python-version }} - path: | - coverage.xml - htmlcov/ - if-no-files-found: warn + - name: Run tests + run: python -m pytest - name: Compile source and tests run: python -m compileall -q src tests scripts samples test.py snippets.py @@ -110,6 +119,42 @@ jobs: PY shell: bash + production-coverage: + name: Production coverage + if: github.event_name == 'pull_request' && github.base_ref == 'main' && github.head_ref == 'dev' + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v7.0.0 + + - name: Set up Python + uses: actions/setup-python@v6.3.0 + with: + python-version: "3.13" + cache: pip + + - name: Install test dependencies + run: | + python -m pip install --upgrade pip "setuptools>=77" wheel + python -m pip install -e ".[test]" + + - name: Run production coverage + run: python -m pytest --cov=histdatacom --cov-report=term-missing:skip-covered --cov-report=xml --cov-report=html + + - name: Enforce coverage threshold + run: python -m coverage report + + - name: Upload production coverage report + if: always() + uses: actions/upload-artifact@v7.0.1 + with: + name: production-coverage + path: | + coverage.xml + htmlcov/ + if-no-files-found: warn + build: name: Build package artifacts runs-on: ubuntu-latest @@ -119,7 +164,7 @@ jobs: uses: actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: "3.13" cache: pip @@ -175,6 +220,8 @@ jobs: "jupyter", "pandas", "pyarrow", + "statsmodels", + "arch", "temporalio", ): metadata.version(distribution_name) @@ -184,6 +231,8 @@ jobs: "ipywidgets", "pandas", "pyarrow", + "statsmodels", + "arch", "temporalio", ): if importlib.util.find_spec(module_name) is None: @@ -215,7 +264,7 @@ jobs: uses: actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: "3.13" cache: pip diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 00000000..e407ce3f --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,105 @@ +name: Container + +on: + pull_request: + paths: + - ".dockerignore" + - ".github/workflows/container.yml" + - "Dockerfile" + - "container/constraints.txt" + - "scripts/smoke_container.py" + - "tests/unit/test_container_distribution.py" + - "tests/unit/test_smoke_container.py" + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name != 'push' }} + +jobs: + validate: + name: Build and smoke native image + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Check out source + uses: actions/checkout@v7.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.2.0 + + - name: Build and smoke application image + run: | + python scripts/smoke_container.py \ + --image histdatacom:ci-smoke \ + --version "${GITHUB_REF_NAME}" \ + --revision "${GITHUB_SHA}" + shell: bash + + publish: + name: Publish multi-platform image + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + needs: validate + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + id-token: write + packages: write + + steps: + - name: Check out source + uses: actions/checkout@v7.0.0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.2.0 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve image metadata + id: metadata + uses: docker/metadata-action@v6.2.0 + with: + images: ghcr.io/${{ github.repository }} + flavor: latest=auto + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix=sha- + + - name: Resolve build timestamp + id: build + run: echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "${GITHUB_OUTPUT}" + shell: bash + + - name: Build and publish image + uses: docker/build-push-action@v7.3.0 + with: + build-args: | + IMAGE_CREATED=${{ steps.build.outputs.created }} + IMAGE_REVISION=${{ github.sha }} + IMAGE_VERSION=${{ github.ref_name }} + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + labels: ${{ steps.metadata.outputs.labels }} + platforms: linux/amd64,linux/arm64 + provenance: mode=max + push: true + sbom: true + tags: ${{ steps.metadata.outputs.tags }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3f5dc225..ef001f34 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Release on: push: tags: - - "v*" + - "*.*.*" workflow_dispatch: inputs: release_target: @@ -20,6 +20,11 @@ on: required: true default: false type: boolean + testpypi_run_id: + description: "Required for PyPI: successful dev Release run that published the exact files to TestPyPI" + required: false + default: "" + type: string include_bundled_platform_wheels: description: "Build private/offline bundled platform wheels; build-only dry runs only" required: true @@ -66,9 +71,36 @@ jobs: fi shell: bash + - name: Validate PyPI promotion input + env: + RELEASE_TARGET: ${{ inputs.release_target }} + TESTPYPI_DRY_RUN_CONFIRMED: ${{ inputs.testpypi_dry_run_confirmed }} + TESTPYPI_RUN_ID: ${{ inputs.testpypi_run_id }} + run: | + if [[ "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then + exit 0 + fi + + if [[ "${RELEASE_TARGET}" == "pypi" ]]; then + if [[ "${TESTPYPI_DRY_RUN_CONFIRMED}" != "true" ]]; then + echo "Set testpypi_dry_run_confirmed=true only after this version passes TestPyPI." >&2 + exit 1 + fi + + if [[ ! "${TESTPYPI_RUN_ID}" =~ ^[0-9]+$ ]]; then + echo "Set testpypi_run_id to the successful dev Release run that published this version to TestPyPI." >&2 + exit 1 + fi + elif [[ -n "${TESTPYPI_RUN_ID}" ]]; then + echo "testpypi_run_id is accepted only for a PyPI promotion." >&2 + exit 1 + fi + shell: bash + build-metadata: name: Build metadata release artifacts needs: validate-release-inputs + if: github.event_name != 'workflow_dispatch' || inputs.release_target != 'pypi' runs-on: ubuntu-latest steps: @@ -76,7 +108,7 @@ jobs: uses: actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: "3.13" cache: pip @@ -149,7 +181,7 @@ jobs: uses: actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: "3.13" cache: pip @@ -193,7 +225,7 @@ jobs: shell: bash - name: Generate bundled wheel provenance attestation - uses: actions/attest@v4.1.0 + uses: actions/attest@v4.1.1 with: subject-path: dist-platform/*.whl @@ -241,7 +273,7 @@ jobs: uses: actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: ${{ matrix.python-version }} cache: pip @@ -311,7 +343,7 @@ jobs: uses: actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: "3.13" cache: pip @@ -349,7 +381,7 @@ jobs: shell: bash - name: Generate release provenance attestation - uses: actions/attest@v4.1.0 + uses: actions/attest@v4.1.1 with: subject-path: dist/* @@ -394,31 +426,270 @@ jobs: publish-pypi: name: Publish to PyPI - needs: assemble-release-artifacts + needs: validate-release-inputs runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' && inputs.release_target == 'pypi' && github.ref == 'refs/heads/main' environment: name: pypi url: https://pypi.org/p/histdatacom permissions: + actions: read contents: read id-token: write steps: - - name: Require TestPyPI dry-run confirmation - if: ${{ inputs.testpypi_dry_run_confirmed != true }} + - name: Check out production source and tags + uses: actions/checkout@v7.0.0 + with: + fetch-depth: 0 + + - name: Validate successful TestPyPI source run + id: candidate + env: + GH_TOKEN: ${{ github.token }} + TESTPYPI_RUN_ID: ${{ inputs.testpypi_run_id }} run: | - echo "Set testpypi_dry_run_confirmed=true only after this version passes TestPyPI." - exit 1 + set -euo pipefail + mkdir -p release-reports + + gh api \ + "/repos/${GITHUB_REPOSITORY}/actions/runs/${TESTPYPI_RUN_ID}" \ + > release-reports/testpypi-source-run.json + jq -e --arg repository "${GITHUB_REPOSITORY}" ' + .status == "completed" + and .conclusion == "success" + and .event == "workflow_dispatch" + and .head_branch == "dev" + and .path == ".github/workflows/release.yml" + and .repository.full_name == $repository + ' release-reports/testpypi-source-run.json >/dev/null + + gh api \ + "/repos/${GITHUB_REPOSITORY}/actions/runs/${TESTPYPI_RUN_ID}/jobs?per_page=100" \ + > release-reports/testpypi-source-jobs.json + jq -e ' + [.jobs[] | select( + .name == "Publish to TestPyPI" and .conclusion == "success" + )] | length == 1 + ' release-reports/testpypi-source-jobs.json >/dev/null + + gh api \ + "/repos/${GITHUB_REPOSITORY}/actions/runs/${TESTPYPI_RUN_ID}/artifacts?name=histdatacom-dist&per_page=100" \ + > release-reports/testpypi-source-artifacts.json + jq -e ' + [.artifacts[] | select( + .name == "histdatacom-dist" and (.expired | not) + )] | length == 1 + ' release-reports/testpypi-source-artifacts.json >/dev/null + + version="$(python - <<'PY' + import ast + from pathlib import Path + + module = ast.parse( + Path("src/histdatacom/__init__.py").read_text(encoding="utf-8") + ) + for node in module.body: + if ( + isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) + and target.id == "__version__" + for target in node.targets + ) + ): + print(ast.literal_eval(node.value)) + break + else: + raise SystemExit("package version is missing") + PY + )" + source_sha="$(jq -r '.head_sha' release-reports/testpypi-source-run.json)" + tag_sha="$(git rev-parse "refs/tags/${version}^{commit}")" + git merge-base --is-ancestor "${tag_sha}" "${source_sha}" + + unexpected_paths=0 + while IFS= read -r path; do + case "${path}" in + .github/workflows/release.yml|RELEASE.md|tests/unit/test_release_workflow.py) + ;; + *) + echo "TestPyPI source run changes packaged or unreviewed path after tag ${version}: ${path}" >&2 + unexpected_paths=1 + ;; + esac + done < <(git diff --name-only "${tag_sha}" "${source_sha}") + if [[ "${unexpected_paths}" != "0" ]]; then + exit 1 + fi + + artifact_id="$(jq -r ' + [.artifacts[] | select( + .name == "histdatacom-dist" and (.expired | not) + )][0].id + ' release-reports/testpypi-source-artifacts.json)" + artifact_digest="$(jq -r ' + [.artifacts[] | select( + .name == "histdatacom-dist" and (.expired | not) + )][0].digest + ' release-reports/testpypi-source-artifacts.json)" + { + echo "version=${version}" + echo "source_sha=${source_sha}" + echo "tag_sha=${tag_sha}" + echo "artifact_id=${artifact_id}" + echo "artifact_digest=${artifact_digest}" + } >> "${GITHUB_OUTPUT}" shell: bash - - name: Download release artifacts - uses: actions/download-artifact@v8.0.1 - with: - name: histdatacom-dist - path: dist + - name: Download exact TestPyPI release artifacts + env: + GH_TOKEN: ${{ github.token }} + TESTPYPI_RUN_ID: ${{ inputs.testpypi_run_id }} + run: | + gh run download "${TESTPYPI_RUN_ID}" \ + --repo "${GITHUB_REPOSITORY}" \ + --name histdatacom-dist \ + --dir dist + shell: bash + + - name: Verify exact TestPyPI artifact hashes + env: + ARTIFACT_DIGEST: ${{ steps.candidate.outputs.artifact_digest }} + ARTIFACT_ID: ${{ steps.candidate.outputs.artifact_id }} + RELEASE_VERSION: ${{ steps.candidate.outputs.version }} + SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }} + TAG_SHA: ${{ steps.candidate.outputs.tag_sha }} + TESTPYPI_RUN_ID: ${{ inputs.testpypi_run_id }} + run: | + python - <<'PY' + from __future__ import annotations + + import hashlib + import json + import os + from pathlib import Path + import time + from urllib.error import HTTPError + from urllib.request import urlopen + + + version = os.environ["RELEASE_VERSION"] + dist = Path("dist") + files = sorted( + path for path in dist.iterdir() if path.suffix == ".whl" + ) + sorted(dist.glob("*.tar.gz")) + expected_names = { + f"histdatacom-{version}-py3-none-any.whl", + f"histdatacom-{version}.tar.gz", + } + if {path.name for path in files} != expected_names: + raise SystemExit(f"unexpected distribution set: {files}") + local_hashes = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in files + } + + url = f"https://test.pypi.org/pypi/histdatacom/{version}/json" + payload = None + for attempt in range(30): + try: + with urlopen(url, timeout=30) as response: + payload = json.load(response) + break + except HTTPError as error: + if error.code != 404 or attempt == 29: + raise + time.sleep(10) + if payload is None: + raise SystemExit("TestPyPI version metadata did not become available") + remote_hashes = { + item["filename"]: item["digests"]["sha256"] + for item in payload["urls"] + } + if remote_hashes != local_hashes: + raise SystemExit( + f"TestPyPI artifact hashes differ: local={local_hashes}, " + f"remote={remote_hashes}" + ) + + report = { + "schema_version": "histdatacom.pypi-artifact-promotion.v1", + "version": version, + "testpypi_run_id": int(os.environ["TESTPYPI_RUN_ID"]), + "source_sha": os.environ["SOURCE_SHA"], + "tag_sha": os.environ["TAG_SHA"], + "artifact_id": int(os.environ["ARTIFACT_ID"]), + "artifact_digest": os.environ["ARTIFACT_DIGEST"], + "files": local_hashes, + "testpypi_verified": True, + "pypi_verified": False, + } + path = Path("release-reports/pypi-artifact-promotion.json") + path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + PY + shell: bash - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@v1.14.0 with: print-hash: true + + - name: Verify exact PyPI artifact hashes + env: + RELEASE_VERSION: ${{ steps.candidate.outputs.version }} + run: | + python - <<'PY' + from __future__ import annotations + + import json + import os + from pathlib import Path + import time + from urllib.error import HTTPError + from urllib.request import urlopen + + + path = Path("release-reports/pypi-artifact-promotion.json") + report = json.loads(path.read_text(encoding="utf-8")) + version = os.environ["RELEASE_VERSION"] + url = f"https://pypi.org/pypi/histdatacom/{version}/json" + payload = None + for attempt in range(30): + try: + with urlopen(url, timeout=30) as response: + payload = json.load(response) + break + except HTTPError as error: + if error.code != 404 or attempt == 29: + raise + time.sleep(10) + if payload is None: + raise SystemExit("PyPI version metadata did not become available") + remote_hashes = { + item["filename"]: item["digests"]["sha256"] + for item in payload["urls"] + } + if remote_hashes != report["files"]: + raise SystemExit( + f"PyPI artifact hashes differ: expected={report['files']}, " + f"remote={remote_hashes}" + ) + report["pypi_verified"] = True + path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + PY + shell: bash + + - name: Upload PyPI promotion evidence + if: always() + uses: actions/upload-artifact@v7.0.1 + with: + name: histdatacom-pypi-promotion-report + path: release-reports/pypi-artifact-promotion.json + if-no-files-found: warn diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index f697104f..1dec1bc9 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -46,7 +46,7 @@ jobs: uses: actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: "3.13" cache: pip @@ -86,12 +86,12 @@ jobs: uses: actions/checkout@v7.0.0 - name: Initialize CodeQL - uses: github/codeql-action/init@v4.36.2 + uses: github/codeql-action/init@v4.36.3 with: languages: python build-mode: none - name: Analyze with CodeQL - uses: github/codeql-action/analyze@v4.36.2 + uses: github/codeql-action/analyze@v4.36.3 with: category: "/language:python" diff --git a/.gitignore b/.gitignore index acd9d683..c26ef38c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ venv/ *.csv *.txt !whitelist.txt +!container/constraints.txt *.zip !tests/fixtures/**/*.csv .info @@ -24,6 +25,7 @@ output.pstats result.json out.gv htmlcov/ +docs/_build/ coverage.xml pip-audit.json .coverage diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a38a5602..aed8f686 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -87,41 +87,6 @@ repos: verbose: True stages: - pre-commit - - id: coverage-run - name: coverage-run - entry: scripts/run_dev_tool.py coverage - language: system - args: ["run"] - pass_filenames: false - always_run: True - types: - - python - - id: coverage-combine - name: coverage-combine - entry: scripts/run_dev_tool.py coverage - language: system - args: ["combine"] - pass_filenames: false - always_run: True - types: - - python - - id: coverage-report - name: coverage-report - entry: scripts/run_dev_tool.py coverage - language: system - args: ["report"] - pass_filenames: false - always_run: True - verbose: True - types: - - python - - id: coverage-rm - name: coverage-rm - entry: scripts/run_dev_tool.py - language: system - args: ["--remove", ".coverage"] - pass_filenames: false - always_run: True repo: local - repo: https://github.com/commitizen-tools/commitizen rev: v4.16.3 diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..fd497d03 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,17 @@ +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.13" + +sphinx: + configuration: docs/conf.py + fail_on_warning: true + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f72b0f9..f928f4b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,71 @@ ## Unreleased +### Added + +- **reconstruction**: add a versioned broker-neutral v2.1 certification policy, + hash/schema/subject-verified campaign runner, JSON-pointer observation + extraction, atomic machine/human dossier publication, and installed + `reconstruction certify` command while preserving legacy V1 evidence replay + (#449). +- **reconstruction**: add exact paired nanosecond plan bounds for small + representative-window campaigns while retaining complete touched-month + source hashing, and report candidate amplification against aggregate + estimated inputs instead of mixing all ensemble members with one raw-source + denominator (#449). +- **reconstruction**: add content-addressed full-range plan sets and public + `plan-set`/`preflight-set` operations so resource-safe daily windows can span + the common history without exceeding the independent 64 MiB plan-artifact + bound (#449). +- **reconstruction**: add atomic content-addressed write/readback verification + for compact final-product activity manifests used by certification and + downstream bar reconciliation (#449). + +- **reconstruction**: expose the first-party reconstruction pipeline through + an installed CLI family and typed Python facade, with explicit information + mode/nonclaim requests, preflight resources and refusals, Temporal and local + execution, aligned status/cancel/resume receipts, bounded lineage previews, + replay verification, and stable exit codes (#467). +- **data-quality**: make weekend and expected-session-closure remediation + guidance profile-aware with bounded calendar-policy context while preserving + stable public weekend hint codes (#344). +- **data-quality**: preserve value-level quality-profile provenance during + resolution across defaults, named profiles, files, YAML, API options, and CLI + overrides, including previous source/value evidence (#367). + +### Fixed + +- **reconstruction**: preserve Arrow partition-row order for equal-timestamp + source ticks, externalize proposal and carving batch evidence into bounded + content-addressed ledgers, enforce live RSS limits, and truncate very large + cross-currency refusal lists with a deterministic count and digest; inject + one bounded, source-grid-aligned anchor from the sparsest declared leg into + missing proposal legs so independently sampled modern streams have genuine + exact-time triangle support without replacing immutable observations, and + try every declared synthetic projection target before refusing a feasible + cross-currency point (#449). +- **reconstruction**: resolve exact nanosecond plan bounds to source months with + integer time conversion so the last nanosecond before a month boundary cannot + round into the following partition (#449). +- **reconstruction**: compact high-cardinality activity provenance into bounded + retained IDs plus explicit occurrence counts and ordered SHA-256 evidence + instead of refusing ordinary reconstructed products at 256 IDs (#449). +- **reconstruction**: de-duplicate source partitions and strong artifact + verification across adaptive full-range plan shards, with stat-identity hash + and qualified-input caching plus compact streaming aggregation, so split + months report unique raw rows and neither construction nor public preflight + repeatedly materializes immutable corpora or retains every full shard graph + in memory (#449). +- **reconstruction**: retain scientifically unsupported full-range spans as + bounded refusal-only plan shards with zero workflow/output estimates, so + public planning stays exactly contiguous without converting missing context + into executable work (#449). +- **reconstruction**: reconcile end-to-end window runtime and peak stage + resources into run reports instead of exposing atomic-commit runtime as if it + represented the whole seven-stage execution (#449). +- **ci**: reserve coverage for one required `dev`-to-`main` production + promotion job instead of running it during routine pushes, issue closure, + and every Python/OS test-matrix job (#420). + ## 1.3.2 (2026-07-03) ### Added diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..595303c2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,83 @@ +ARG PYTHON_BASE="python:3.13-slim-bookworm@sha256:9d7f287598e1a5a978c015ee176d8216435aaf335ed69ac3c38dd1bbb10e8d64" + +FROM ${PYTHON_BASE} AS builder + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /build + +COPY pyproject.toml README.md LICENSE ./ +COPY container/constraints.txt ./container/constraints.txt +COPY src ./src + +RUN python -m pip install \ + --constraint container/constraints.txt \ + setuptools \ + wheel \ + && python -m pip wheel \ + --constraint container/constraints.txt \ + --no-build-isolation \ + --wheel-dir /wheels \ + . \ + && python -m venv /opt/histdatacom \ + && /opt/histdatacom/bin/python -m pip install \ + --no-index \ + --find-links=/wheels \ + histdatacom \ + && rm -rf /wheels + +FROM ${PYTHON_BASE} AS runtime + +ARG IMAGE_CREATED="1970-01-01T00:00:00Z" +ARG IMAGE_REVISION="unknown" +ARG IMAGE_VERSION="0.0.0+container" + +LABEL org.opencontainers.image.created="${IMAGE_CREATED}" \ + org.opencontainers.image.description="HistData.com FX data acquisition, quality, orchestration, and reconstruction CLI" \ + org.opencontainers.image.documentation="https://github.com/dmidlo/histdata.com-tools/blob/main/docs/container.md" \ + org.opencontainers.image.licenses="MIT" \ + org.opencontainers.image.revision="${IMAGE_REVISION}" \ + org.opencontainers.image.source="https://github.com/dmidlo/histdata.com-tools" \ + org.opencontainers.image.title="histdatacom" \ + org.opencontainers.image.version="${IMAGE_VERSION}" + +ENV HISTDATACOM_RUNTIME_HOME=/workspace/runtime \ + HISTDATACOM_RUNTIME_WORKSPACE=/workspace \ + HISTDATACOM_TEMPORAL_CACHE_DIR=/workspace/cache/temporal-cli \ + HOME=/workspace \ + PATH=/opt/histdatacom/bin:${PATH} \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + XDG_CACHE_HOME=/workspace/cache \ + XDG_STATE_HOME=/workspace/runtime + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends tini=0.19.0-1+b3 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 10001 histdatacom \ + && useradd \ + --system \ + --uid 10001 \ + --gid 10001 \ + --home-dir /workspace \ + --shell /usr/sbin/nologin \ + histdatacom \ + && install --directory \ + --owner 10001 \ + --group 10001 \ + /workspace \ + /workspace/cache \ + /workspace/cache/temporal-cli \ + /workspace/data \ + /workspace/runtime + +COPY --from=builder /opt/histdatacom /opt/histdatacom + +WORKDIR /workspace +USER 10001:10001 +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/bin/tini", "--", "histdatacom"] +CMD ["--help"] diff --git a/README.md b/README.md index bc379d2f..b7a4e1a3 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,26 @@ Works on macOS, Linux, and Windows. - [Clean and Failing Examples](#clean-and-failing-examples) - [Warning, Error, and Exit Policy](#warning-error-and-exit-policy) - [Data Analytics](#data-analytics) + - [Point-in-Time Market Context](#point-in-time-market-context) - [Feed-Regime Detection](#feed-regime-detection) + - [Historical Feed-Observation Operators](#historical-feed-observation-operators) + - [Reverse-Degradation Benchmark](#reverse-degradation-benchmark) + - [Empirical Reference-Motif Index](#empirical-reference-motif-index) + - [Real Modern Reference-Motif Library](#real-modern-reference-motif-library) + - [Empirical Motif Candidate Generation](#empirical-motif-candidate-generation) + - [Historical Candidate Carving](#historical-candidate-carving) + - [Cross-Currency Reconciliation](#cross-currency-reconciliation) + - [Calibrated Reconstruction Ensembles](#calibrated-reconstruction-ensembles) + - [Live Broker Delivery Capture](#live-broker-delivery-capture) + - [Broker Delivery Fingerprints](#broker-delivery-fingerprints) + - [Broker-Conditioned Reconstruction](#broker-conditioned-reconstruction) + - [Atomic Reconstruction Persistence](#atomic-reconstruction-persistence) + - [Reconstruction Activity Semantics](#reconstruction-activity-semantics) + - [Derived Reconstruction Candlesticks](#derived-reconstruction-candlesticks) + - [Reconstructed-History Strategy Sensitivity](#reconstructed-history-strategy-sensitivity) + - [EURUSD Triangle Reconstruction Certification](#eurusd-triangle-reconstruction-certification) + - [Temporal Reconstruction Orchestration](#temporal-reconstruction-orchestration) + - [Public Reconstruction CLI and API](#public-reconstruction-cli-and-api) - [Orchestration Runtime](#orchestration-runtime) - [Runtime Model and Install Surface](#runtime-model-and-install-surface) - [Binary Provisioning and PyPI Packaging](#binary-provisioning-and-pypi-packaging) @@ -56,6 +75,7 @@ Works on macOS, Linux, and Windows. - [Full Script Example](#full-script-example) - [Setup](#setup) - [TLDR for all platforms](#tldr-for-all-platforms) + - [Container Image](#container-image) - [Developer Setup](#developer-setup) - [Vanilla Python Setup](#vanilla-python-setup) - [Vanilla MacOS and Linux](#vanilla-macos-and-linux) @@ -100,7 +120,8 @@ usage: histdatacom [-h] [-A] [-U] [--by BY] [--version] [-V] [-D] [-X] [-C] [--config PATH] [-p PAIR [PAIR ...]] [--pair-groups GROUP [GROUP ...]] [-f FORMAT [FORMAT ...]] [-t TIMEFRAME [TIMEFRAME ...]] [-s START_YEARMONTH] - [-e END_YEARMONTH] [-I] [-d] [-b BATCH_SIZE] + [-e END_YEARMONTH] [-r EXPRESSION] [--random-seed INTEGER] + [-z IANA_ZONE] [-I] [-d] [-b BATCH_SIZE] [-c CPU_UTILIZATION] [--data-directory DATA_DIRECTORY] [-v] [--orchestration-start] [--no-orchestration-start] [--submit-only] [--no-overlap] @@ -117,6 +138,7 @@ usage: histdatacom [-h] [-A] [-U] [--by BY] [--version] [-V] [-D] [-X] [-C] [--quality-preflight-profile-preview-format {json,text,markdown}] [--quality-preflight-validation-report PATH] [--quality-preflight-run-validation] + [--quality-preflight-validation-evidence PATH] [--quality-preflight-evidence PATH] [--quality-preflight-evidence-max-age-seconds SECONDS] [--quality-preflight-evidence-stale-ok] @@ -163,6 +185,17 @@ Config: -e, --end_yearmonth END_YEARMONTH set an end year and month for data. e.g. -e 2020-00 or -e 2022-04 + -r, --random-window EXPRESSION + select deterministic duration/session tick windows; + random selection requires --random-seed, while session + expressions with both -s and -e return all matching + occurrences + --random-seed INTEGER + seed required for reproducible random-window selection + -z, --timezone, --output-timezone IANA_ZONE + append datetime_local to API results in an IANA + timezone; canonical cache and Influx timestamps remain + UTC Influxdb: -I, --import_to_influxdb @@ -249,6 +282,9 @@ Data quality: --quality-preflight-run-validation run bounded quality preflight validation commands before rendering evidence + --quality-preflight-validation-evidence PATH + write bounded machine-readable validation evidence to + PATH and reference it from quality preflight evidence --quality-preflight-evidence PATH use a saved quality preflight JSON report as evidence before a large cache-backed --quality run @@ -306,6 +342,7 @@ Commands: groups List instrument groups and major triangles jobs Inspect and control orchestrated work quality Inspect local data quality evidence + reconstruction Plan, run, and inspect reconstruction runtime Inspect and manage the orchestration runtime Run `histdatacom analytics --help` for analytics commands. @@ -313,6 +350,7 @@ Run `histdatacom cleanup --help` for cleanup commands. Run `histdatacom groups --help` for group discovery commands. Run `histdatacom jobs --help` for job telemetry commands. Run `histdatacom quality --help` for quality commands. +Run `histdatacom reconstruction --help` for reconstruction. ``` Maintainers: this help excerpt is generated from `ArgParser.format_help()` at a @@ -457,6 +495,15 @@ canonical `.data` cache files, and removes transient ZIP/CSV sources after each cache is ready. It is intentionally limited to cache-capable ASCII tick quote datasets, and it does not merge the caches into memory. +The `.data` cache is also the canonical training substrate. Cache builds +materialize one flat row-aligned ASCII tick table with observed bid/ask data, +explicit row identity, timestamp features, data-quality issue columns, +classification codes, training controls, and nullable `synth_*` placeholders. +The durable training row key is `series_id`, `period`, and `row_id`; +`datetime`/`timestamp_utc_ms` remains an observed time-axis feature and is not +the only identity value. This lets later training stages mask or bucket +timestamps without losing deterministic row identity. + #### clean up transient source artifacts without removing internal caches ```sh @@ -496,6 +543,9 @@ histdatacom: - tick-data-quotes start_yearmonth: 2022-01 end_yearmonth: 2022-03 + # Optional deterministic tick projection: + # random_window: 2d + # random_seed: 1729 data_directory: /data/histdata request_bundle_out: requests/eurusd-cache-bundle.json request_json_out: requests/eurusd-cache.json @@ -535,6 +585,10 @@ histdatacom: target: data/ASCII/T/eurusd bucket: month report: reports/eurusd-feed-regimes.json + epoch_artifact: reports/eurusd-feed-epochs.v1.json + min_evidence_periods: 12 + min_segment_periods: 3 + min_boundary_support: 0.75 json: true jobs: command: submit @@ -605,6 +659,64 @@ date ranges are for year and month and can be specified in the following ways: |2202.04| |2202_04| +##### Deterministic random and session windows + +Use `-r/--random-window EXPRESSION` to project a bounded subset of ASCII tick +data. Random selection always requires an explicit non-negative +`--random-seed`; the expression, seed, requested symbols, repository inventory, +and optional `-s`/`-e` bounds reproduce the same half-open `[start, end)` +selection regardless of input order or worker parallelism. + +```sh +# One reproducible 90-minute interval inside common EURUSD/GBPUSD support. +histdatacom -C -I -p eurusd gbpusd -f ascii -t tick-data-quotes \ + -r 90m --random-seed 1729 + +# Every London sampling window in January 2024; no seed is needed when a +# session expression has both bounds. Equal start/end months are valid here. +histdatacom -I -p eurusd -f ascii -t tick-data-quotes \ + -s 2024-01 -e 2024-01 -r ldn +``` + +Duration units are case-sensitive: `y` is a calendar year, `q` a calendar +quarter, `M` a calendar month, `w` a week, `d` a day, `h` an hour, and `m` a +minute. Year, quarter, and month selections are calendar-aligned; fixed +durations are UTC-minute-aligned. Supported forms include `1y`, `1q`, `1M`, +`2w`, `2d`, `6h`, `90m`, `ldn`, `ldn-ny`, `syd-syd`, `hk-3d`, `hk-3d-hk`, +`45m-auk`, `1h-auk-1h`, and `30m-ldn-1w-syd-1h`. Unsupported mixtures fail +closed. + +Session codes use explicit 08:00–17:00 local-clock profiles and IANA timezone +rules: + +| Code | Sampling location | IANA timezone | +|---|---|---| +| `fra` | Frankfurt/Paris | `Europe/Paris` | +| `ldn` | London | `Europe/London` | +| `ny` | New York | `America/New_York` | +| `chi` | Chicago | `America/Chicago` | +| `la` | San Francisco/Los Angeles | `America/Los_Angeles` | +| `auk` | Auckland/Wellington | `Pacific/Auckland` | +| `syd` | Sydney | `Australia/Sydney` | +| `tyo` | Tokyo | `Asia/Tokyo` | +| `hk` | Hong Kong/Singapore | `Asia/Hong_Kong` | + +These profiles are reproducible sampling windows, not claims about centralized +FX exchange hours. DST follows each IANA zone. A single session selects its +open through close; ordered session pairs span from the first open through the +second close, wrapping to the next local day when needed. Equal session anchors +such as `syd-syd` end at the following session close. Hour/minute tokens outside +anchors add padding; day/week/month/quarter/year tokens between anchors add the +bridge duration. + +HistData archives and canonical `.data` caches remain complete monthly evidence. +The planner resolves one compact, versioned selection against the common +repository coverage of all requested symbols and schedules only intersecting +months. It never truncates or rewrites ZIP, CSV, or cache artifacts. Exact +filtering happens only while materializing API results or streaming rows to +InfluxDB; empty or off-support selections fail instead of silently substituting +another interval. + ##### to fetch a single year's data, leave out the month - note: unless you're fetching data for the current year, a year-only tick request @@ -737,8 +849,16 @@ pip install "histdatacom[influx]" - ascii is the only format accepted for influxdb import. - all histdata.com datetime data is in EST (Eastern Standard Time) with no adjustments for daylight savings. -- Influxdb does not adjust for timezone and all datetime data is recorded as UTC epoch timestamps (nano-seconds since midnight 00:00, January, 1st, 1970) +- InfluxDB does not attach a display timezone; all datetime data is written as UTC + epoch timestamps with millisecond precision. - this tool converts histdata.com ESTnoDST to UTC Epoch milli-second timestamps as part of the import-to-influx process +- `-z/--timezone` never changes Influx timestamps. Select a display timezone in + the Influx query or UI instead. +- enriched ASCII tick `.data` columns are projected onto the same tick point: + observed quote fields, quality issue flags, quality/classification codes, + training controls, and populated synthetic fields when present. Duplicate + tick timestamps keep distinct Influx point identity with deterministic + `row_id` tags; the `.data` row key remains the source of truth. ```txt histdatacom -I -p eurusd -f ascii -t tick-data-quotes -s start -e now @@ -798,6 +918,26 @@ the configured data directory. Targets can be plain HistData CSV files, HistData ZIP archives, directories containing those files, or the canonical `.data` cache file. +Quality reports remain audit artifacts. Training-facing quality semantics are +row columns on enriched ASCII tick `.data` caches, including explicit +`dq_issue_*` indicators, quality counts, classification labels, training +usability/weight, and synthetic placeholders. Report and bounded summary +surfaces should derive from or stay consistent with those row-level columns, so +downstream training code does not need to parse report JSON or join separate +quality tables to assemble a training row. + +When the engine intentionally skips a target-rule evaluation—for example, a +semantic scan of a ZIP whose matching extracted CSV is preferred—the report +adds optional `metadata.quality_engine` reconciliation metadata. Its +`histdatacom.quality-skip-events.v1` event list records a stable reason code, +rule ID, target kind, and publish-safe format/timeframe/symbol/period axis; it +never records local paths or row samples. Events and aggregate reason/rule +counts are deterministically bounded with explicit omission metadata. The same +contract is projected into bounded runtime payloads as `quality_engine`, and +the human summary reports planned, executed, and skipped evaluation totals. +The existing `skipped_duplicate_archive_rule_evaluation_count` remains for +compatible consumers. + Use `--repo-quality` when the same quality run should also update the local repo helper file with bounded per-instrument quality summaries: @@ -831,6 +971,7 @@ histdatacom --quality-preflight \ --quality-checks ticks \ --quality-preflight-report reports/major-triangles-tick-preflight.json \ --quality-preflight-markdown-report reports/major-triangles-tick-preflight.md \ + --quality-preflight-validation-evidence reports/major-triangles-validation.json \ --quality-preflight-profile-preview-output reports/major-triangles-quality-profile.md \ --quality-preflight-profile-preview-format markdown ``` @@ -873,11 +1014,30 @@ run repository gates. For release notes or GitHub issue evidence, pass `--quality-preflight-validation-report PATH` to merge command status from a closure/readiness JSON report. Use `--quality-preflight-validation-report latest` to resolve the newest compatible -JSON report under `.histdatacom/closure-readiness` without running gates. Pass +JSON report under `.histdatacom/closure-readiness` without running gates. The +closure report's release-independent `full-tests` result is recognized as +full-pytest evidence; importing it never starts tests or coverage. Pass `--quality-preflight-run-validation` to run only the bounded local validation -bundle: focused quality-preflight tests and `git diff --check`. Full pytest, -pre-commit, publishing, and GitHub issue closure remain explicit -closure/release workflow responsibilities. +bundle: focused quality-preflight tests, the README help-sync check, and +`git diff --check`. It does not run full pytest/coverage or pre-commit. + +Pass `--quality-preflight-validation-evidence PATH` to write the validation +rows as a dedicated bounded JSON artifact and register it under +`evidence.artifacts.validation_evidence`. The artifact includes its schema and +generated timestamp plus each configured command's status, exit code, duration, +summary, and publish-safe output-artifact path when available. Its registry +entry records the publish-safe path, SHA-256, and byte size. Using the artifact +option alone is the dry inspection path: commands remain planned/`not-run` and +no repository gates execute. Combine it with an imported report to serialize +existing closure receipts, or with `--quality-preflight-run-validation` for the +bounded checks. Output summaries are truncated and publish-safe; full logs stay +in separately referenced artifacts. + +This application evidence supplements normal issue/PR validation notes and can +be consumed by CI, but it does not replace CI or automate GitHub issues, pull +requests, comments, or labels. Full repository gates, publishing, TestPyPI, +PyPI, and GitHub issue closure remain explicit closure/release workflow +responsibilities. When launching a large cache-backed `--quality` run, pass the saved report with `--quality-preflight-evidence PATH`. If no matching evidence is available, the @@ -963,7 +1123,7 @@ Supported groups: | `domain` | symbol metadata, quote conventions, calendar/session tags, cross-instrument consistency | | `modeling` | advisory modeling-readiness checks for leakage risk, spread-cost assumptions, and target horizon feasibility | | `provenance` | optional orchestration manifest/status lineage checks for artifact paths, sizes, checksums, cache metadata, stale caches, and orphan files | -| `fingerprint` | deterministic INFO-only time-series fingerprints for target axis, coverage, timestamp topology, tick distributions, calendar regimes, microstructure dynamics, lag dependence, stationarity/drift diagnostics, and bounded tick spread conditioning | +| `fingerprint` | deterministic INFO-only target and run-scoped time-series fingerprints for coverage, topology, distributions, regimes, dynamics, dependence, stationarity, decomposition, and cross-series FX relationships | `fingerprint.series` payloads include a `calendar_regimes` section for readable ASCII tick targets. It counts session states, active/clock sessions, @@ -973,6 +1133,30 @@ profile metadata used for classification, so incomplete/static calendar profiles remain advisory and visible rather than becoming hidden failures. Tick fingerprints also include bounded `conditional_distributions` for spread by active session and special tag when spread data is available. + +Calendar profiles can set `weekend_activity_policy` to `strict`, `advisory`, or +`allowed`, and `expected_session_closure_policy` to `expected` or `unexpected`. +Topology remediation keeps the stable `verify_weekend_session_policy` code but +adds bounded policy context with the profile name/source/version, EST-no-DST to +UTC basis, completeness, and the active treatment. Strict profiles produce an +inspection action, advisory profiles request assumption review, and allowed +weekend activity is rendered as a contextual note rather than a run-level next +action. Expected session closures remain contextual unless the profile +explicitly marks them `unexpected`. + +```json +{ + "rules": { + "domain.calendar_sessions": { + "calendar_profile": { + "weekend_activity_policy": "strict", + "expected_session_closure_policy": "expected" + } + } + } +} +``` + Tick fingerprints include `microstructure_dynamics` for interarrival times, spread changes, spread jumps, stale quote runs, bursts, and one-sided movement. These sections record their calculation basis and topology limitations, so @@ -989,12 +1173,753 @@ reasons, sample counts, configured windows, rounding policy, zero-variance markers, and deterministic transform recommendations such as `log_return`, `differencing`, and `session_conditioning`. These diagnostics are descriptive fingerprint facts only; nonstationarity does not fail a quality run. +Readable ASCII tick fingerprints also include a tick-only `decomposition` +section. It reuses the stationarity result, source-calendar hour/day/session +classification, and profile-configured rolling windows to emit deterministic +linear-trend, seasonal-bucket, residual, smoothing-window, and two-segment +structural-break proxies. Buckets and structural candidates are bounded by the +profile histogram limit; insufficient samples, skipped windows, zero variance, +and stationarity limitations remain explicit advisory metadata. These are +descriptive proxies, not fitted forecasting models, and no retired bar or M1 +schema is emitted. + +The decomposition section embeds a `period`-grain training projection with the +stable identity fields `series_id`, `period`, and `row_id`. API consumers can +call `decomposition_training_projection(...)` for its flat scalar values and +`project_decomposition_onto_training_frame(...)` to repeat those period facts +onto an already enriched ASCII tick frame without parsing a report or performing +a side join. Row count and identity columns are preserved. + +Fingerprint runs also emit `cross_series_fingerprint` metadata using +`histdatacom.cross-series-fingerprint.v1`. It groups related ASCII tick series +by timeframe and period, then reports bounded symbol membership, full timestamp +grid overlap, missing counts, unequal coverage ranges and limiting legs, +pairwise return correlations, inverse and triangular consistency, and stale +forward-fill risk. Panel coverage also reports union/common periods, missing +period counts, unequal symbol ranges, and the legs that limit the common start +or end. The group topology rollup includes row/parsed counts, +duplicate and non-monotonic timestamps, suspicious and expected-session gaps, +weekend activity, and source/cache provenance. Legacy raw `.data` caches are +enriched in memory before this projection, and any row-level evidence uses +`series_id`, `period`, and `row_id`; timestamp alone is never treated as durable +identity. Full reports expose `metadata.cross_series_fingerprint`, bounded +runtime payloads expose `fingerprint_cross_series`, and the CLI renders a +concise `Cross-series fingerprint` section. + +Before trusting cache-backed fingerprints after a cache migration, enrichment +change, manual cache copy, or unexpected source/cache timestamp change, run an +opt-in cache/source parity assessment. Parity is disabled by default, so normal +fingerprints retain their cache-first selection and cost. Enable it in a quality +profile and run the ordinary fingerprint check against the source CSV or ZIP: + +```json +{ + "schema_version": "histdatacom.quality-profile.v1", + "name": "fingerprint-cache-source-parity", + "rules": { + "fingerprint.series": { + "cache_source_parity": { + "enabled": true, + "mismatch_limit": 16 + } + } + } +} +``` + +```sh +histdatacom --quality \ + --quality-target data/DAT_ASCII_EURUSD_T_201202.csv \ + --quality-checks fingerprint \ + --quality-profile fingerprint-cache-source-parity.json \ + --quality-report reports/eurusd-fingerprint-parity.json +``` + +The advisory `cache_source_parity` target section compares bounded coverage, +topology, calendar-regime, conditioned-spread, row-identity, +duplicate-timestamp, quality-report, and Influx-projection evidence. It keeps +raw source, raw legacy cache, enriched training cache, quality report, and +Influx projection as separate bases; legacy caches are enriched in memory and +are never rewritten. Stable mismatch codes and basis metadata roll up into +`metadata.time_series_fingerprint_cache_source_parity_summary`, bounded payload +key `fingerprint_parity`, and the CLI `Fingerprint cache/source parity` section. +Source paths and ZIP members remain publication-safe, mismatch fields and target +summaries are bounded, and no raw rows or full duplicate payloads are emitted. +Missing sources or caches are reported as `not_compared`; stale caches are +compared and marked advisory rather than accepted by the normal freshness path. + +Every supported ASCII tick fingerprint also emits bounded +`synthetic_constraints` for the deterministic reference-set generator. The protocol separates +`defects_to_avoid`, `stylized_facts_to_preserve`, and +`source_artifacts_to_parameterize`; records stable comparison modes and +tolerances; and exposes advisory hints for session mix, spread regimes, gap +topology, expected closures, stationarity transforms, cache provenance, and +durable row identity. Its canonical generator input is the enriched `.data` +frame, not nested report JSON. Legacy raw caches are enriched in memory before +row issue columns are counted. + +Python consumers can call `synthetic_constraints_from_training_frame(...)` +with an enriched Polars frame and its fingerprint, then call +`generate_synthetic_ticks_from_reference(...)`. The generator applies a seeded, +contiguous empirical block bootstrap to paired midpoint log returns and spreads, +filters rows marked by `dq_issue_*`, and enforces explicit inspection and output +row limits for very large periods. It refuses a missing fingerprint or +pre-populated synthetic columns unless replacement is explicitly requested. + +Generated values augment the same rows only in `synth_bid`, `synth_ask`, +`synth_spread`, `synth_mid`, `synth_method_code`, `synth_confidence`, and +`synth_usable`. Observed `bid`/`ask`, timestamp features, duplicate-timestamp +rows, and durable `series_id`/`period`/`row_id` identity remain unchanged. Every +successful generation is automatically projected into a market-only candidate +cache, run through the ordinary `fingerprint.series` rule, and compared with +the reference through the synthetic constraint validator. Statistical mismatch +remains advisory rather than a hard data-quality gate. + +Generate an enriched synthetic cache and optionally retain its ordinary +candidate fingerprint report: + +```sh +histdatacom quality synthetic-generate \ + --reference-cache data/ASCII/T/EURUSD/2012/01/.data \ + --reference-report reports/reference-quality.json \ + --output-cache generated/ASCII/T/EURUSD/2012/01/.data \ + --candidate-report reports/generated-quality.json \ + --seed 17 \ + --json +``` + +The command will not overwrite observed columns or an existing output cache by +default. `--max-reference-rows` and `--max-generated-rows` bound work; rows +beyond the generation limit remain present with null synthetic values and +`synth_usable=false`. Volume synthesis, raw M1/OHLC generation, and derived +candlestick output remain outside this ASCII tick feature and follow later +issues #80 and #18. + +Validate an exported candidate tick dataset by running the normal fingerprint +quality path for both reference and candidate, then comparing their saved +reports: + +```sh +histdatacom quality synthetic-validate \ + --reference-report reports/reference-quality.json \ + --candidate-report reports/candidate-quality.json \ + --json +``` + +The advisory `histdatacom.synthetic-fingerprint-validation.v1` result reports +matched, mismatched, and missing target axes; candidate defect violations; +bounded stylized-fact mismatches; output/identity contract drift; and stable +mismatch codes. The validator itself does not mutate data or turn statistical +drift into a hard quality gate. Full reports expose +`metadata.time_series_fingerprint_synthetic_constraint_summary`, bounded +payloads expose `fingerprint_synthetic_constraints`, and the ordinary +fingerprint CLI summary renders `Synthetic fingerprint constraints`. + +Classical baseline diagnostics are a separate, opt-in layer after the +fingerprint substrate. They are disabled by default and do not change quality +status. Enable the low-dependency baseline families through the ordinary +fingerprint profile: + +```json +{ + "schema_version": "histdatacom.quality-profile.v1", + "name": "classical-baselines", + "rules": { + "fingerprint.series": { + "classical_baselines": { + "enabled": true, + "evaluation_fraction": 0.2, + "minimum_training_rows": 20, + "minimum_evaluation_rows": 5, + "rolling_windows": [5, 20], + "session_seasonal_enabled": true, + "rounding_digits": 12 + } + } + } +} +``` + +The `histdatacom.classical-baselines.v1` section evaluates observed midpoint +values with a chronological holdout ordered by `series_id`, `period`, and +`row_id`. It never shuffles, never requires timestamp as durable identity, and +uses walk-forward forecasts where prior observed values become available only +after their row. Initial models are naive/random-walk, rolling mean, rolling +median, and session-conditioned seasonal-naive when the enriched session class +and calendar fingerprint support it. Metrics are emitted only after the +configured training and evaluation minima produce a valid split. + +Stationarity status, rolling drift, distribution shift, skipped windows, +zero-variance markers, and the `log_return`, `differencing`, and +`session_conditioning` recommendations remain explicit evaluation guards. +Recommended transforms are reported but are not silently applied, and +nonstationarity never becomes a hard quality failure. The fitted curriculum now +includes exponential smoothing, explicit-order AR/ARMA/ARIMA, +SARIMA/ARIMAX/SARIMAX, structural state-space and Kalman diagnostics, symmetric +ARCH/GARCH, and family-neutral comparison. Automatic model selection and +forecasting leaderboards remain deliberately deferred. + +Python consumers can call +`classical_baseline_diagnostics_from_training_frame(...)`, then +`project_classical_baseline_onto_training_frame(...)` to repeat the bounded +period-level readiness, split, best-model, and error scalars onto the same +enriched rows. The projection preserves observed bid/ask and duplicate +timestamp rows, requires only `series_id`/`period`/`row_id` identity, and flows +through the existing same-point Influx projection. Full reports expose +`metadata.time_series_fingerprint_classical_baseline_summary`, bounded payloads +use `fingerprint_classical_baselines`, and the fingerprint CLI renders +`Classical fingerprint baselines`. + +The broader fitted-model curriculum begins with a separate opt-in input and +evaluation contract. It regularizes the enriched ASCII tick frame without +restoring raw M1 as an independent input or fitting ETS/ARIMA/state-space/GARCH +models prematurely: + +```json +{ + "schema_version": "histdatacom.quality-profile.v1", + "name": "classical-model-input", + "rules": { + "fingerprint.series": { + "classical_model_input": { + "enabled": true, + "frequency_ms": 60000, + "alignment_epoch_ms": 0, + "closed_side": "left", + "label_side": "left", + "midpoint_aggregation": "last", + "spread_aggregation": "last", + "minimum_observations_per_bin": 1, + "expected_closure_policy": "mark", + "unexpected_missing_policy": "mark", + "transform": "level", + "differencing_order": 0, + "seasonal_differencing_order": 0, + "seasonal_period": 0, + "horizons": [1], + "fold_kind": "expanding", + "minimum_training_observations": 20, + "minimum_evaluation_observations": 5, + "step_size": 5, + "rolling_window": 0, + "embargo_observations": 0, + "rounding_digits": 12, + "resources": { + "max_source_rows": 1000000, + "max_regularized_observations": 100000, + "max_folds": 64, + "max_horizons": 16, + "max_candidate_orders": 32, + "max_fit_attempts": 64, + "max_wall_time_seconds": 300, + "max_memory_bytes": 536870912, + "max_retained_diagnostics": 64 + } + } + } + } +} +``` + +`build_classical_model_input(...)` produces a bounded regular-grid derived +view and the `histdatacom.classical-model-input.v1` contract. UTC bins are +left-closed and left-labeled (`[start,end)`), aligned to an explicit epoch, and +never cross a source period. Midpoint and spread support explicit +first/last/mean/median aggregation. Derived midpoint open/high/low/close values +are descriptive fields in the regularized view; they do not make raw bar data +canonical again. Multiple ticks, duplicate timestamps, minimum bin support, +source-row bounds, truncation, and rounding all remain explicit. + +Empty bins are never forward-filled. Calendar classification separates expected +weekend/session closures from unexpected missing observations; both remain null +on the canonical grid. The `omit` closure policy suppresses closure-target +evaluation folds without compressing elapsed grid time or forecast horizons. Level, log-level, +return, log-return, first differencing, and seasonal differencing are applied +only when configured. The contract records invalid domains, warm-up loss, +inverse-transform requirements, and the requirement to report forecast errors +on the original requested scale. + +Expanding and rolling folds are chronological, never shuffled, and record +training/evaluation boundaries, configured horizons, step size, embargo, and +incomplete targets through `histdatacom.classical-model-fold.v1`. Shared +`histdatacom.classical-model-fit-result.v1` and +`histdatacom.classical-model-evaluation-result.v1` schemas define bounded +fitted/converged/limited/skipped/timeout/numerical/dependency/failure metadata +for later model families without including backend exception text, full +forecasts, residual histories, or fitted objects. + +`project_classical_model_input_onto_training_frame(...)` augments the same +enriched tick rows with registered nullable `cm_input_*`, `cm_fold_*`, and +`cm_evaluation_*` scalar columns. Projection uses `series_id`, `period`, and +`row_id` as durable identity and repeats a completed-bin value only on rows at +or after its UTC close. Masked timestamps remain identifiable, duplicate +timestamps remain distinct, observed bid/ask and `synth_*` fields are preserved, +and post-observation evaluation values are marked diagnostic-only. The same +columns flow through the ordinary Polars cache and same-point Influx projection; +consumers do not need report parsing or model side-table joins. + +No rich numerical dependency is added to the core package by this contract. +Statsmodels and ARCH providers belong in the optional `models` extra; the +low-dependency fingerprint and baseline paths remain usable when those +providers are absent. Full reports use +`time_series_fingerprint_classical_model_input_summary`, bounded payloads use +`fingerprint_classical_model_input`, and console output renders +`Classical model input contracts`. + +The first fitted family is also opt-in and requires the `models` extra: + +```sh +pip install "histdatacom[models]" +``` + +```json +{ + "schema_version": "histdatacom.quality-profile.v1", + "name": "exponential-smoothing", + "rules": { + "fingerprint.series": { + "classical_model_input": { + "enabled": true, + "frequency_ms": 60000, + "transform": "level", + "horizons": [1, 5, 20], + "fold_kind": "expanding", + "minimum_training_observations": 40, + "minimum_evaluation_observations": 5, + "step_size": 20 + }, + "exponential_smoothing": { + "enabled": true, + "projection_specification_id": "hw-add", + "projection_horizon": 1, + "baseline_rolling_windows": [5, 20], + "specifications": [ + { + "specification_id": "ses", + "family": "ses", + "error": "add" + }, + { + "specification_id": "holt-damped", + "family": "holt", + "error": "add", + "trend": "add", + "damped_trend": true + }, + { + "specification_id": "hw-add", + "family": "holt_winters", + "error": "add", + "trend": "add", + "seasonal": "add", + "seasonal_periods": 24, + "initialization_method": "estimated", + "optimized": true, + "max_iterations": 200 + }, + { + "specification_id": "ets-aaa", + "family": "ets", + "error": "add", + "trend": "add", + "seasonal": "add", + "seasonal_periods": 24 + } + ] + } + } + } +} +``` + +`histdatacom.exponential-smoothing.v1` consumes only the regular grid and +rolling-origin folds produced by the model-input contract. Explicit +specifications cover simple exponential smoothing, Holt trend, damped Holt, +additive or multiplicative Holt-Winters, and Statsmodels ETS error/trend/ +seasonal combinations. Initialization, smoothing parameters, optimizer method, +iteration limit, and parameter bounds are configurable; multiplicative forms +reject non-positive transformed training segments. There is no automatic +configuration search or automatic winner. + +Expected closures and unexpected missing bins stay distinct and null. A fit +uses only the trailing contiguous observed grid segment at an origin; neither +kind of gap is forward-filled or removed from elapsed horizon time. Configured +level, log-level, return, log-return, ordinary differencing, and seasonal +differencing policies come from the input contract. Forecasts and metrics are +inverted to the original scale, while warm-up, invalid-domain, insufficient +seasonal-cycle, skipped-target, convergence-warning, timeout, numerical, and +dependency limitations remain bounded advisory metadata. + +Each explicit model is evaluated on the same configured folds and horizons as +the naive/random-walk, rolling mean, rolling median, and session-seasonal +references. Reports expose per-fold forecasts and errors, per-horizon aggregate +metrics, convergence and failure counts, deterministic model IDs, backend +version, fitted scalar parameters, and enforced fit-attempt/time/memory/ +retention limits. Fitted objects, residual vectors, backend exception text, and +wall-clock measurements are never serialized. + +`project_exponential_smoothing_onto_training_frame(...)` adds registered +nullable scalar `cm_ets_*` columns to the same enriched rows using +`series_id`/`period`/`row_id`. Forecasts appear only when the origin bin has +closed. Actuals and realized errors appear only at their later diagnostic +availability row and carry explicit diagnostic-only and training-eligibility +flags. The configured projection has bounded width, preserves observed and +`synth_*` namespaces, survives duplicate or masked timestamps, and flows through +the same Polars cache and Influx point. Full reports use +`time_series_fingerprint_exponential_smoothing_summary`, bounded payloads use +`fingerprint_exponential_smoothing`, and console output renders +`Exponential-smoothing models`. + +The second fitted family adds explicit-order AR, ARMA, and ARIMA models: + +```json +{ + "rules": { + "fingerprint.series": { + "classical_model_input": { + "enabled": true, + "frequency_ms": 60000, + "transform": "level", + "horizons": [1, 5], + "minimum_training_observations": 40, + "step_size": 20 + }, + "autoregressive": { + "enabled": true, + "projection_specification_ids": ["ar-2", "arma-1-1", "arima-1-1-1"], + "projection_horizon": 1, + "compare_exponential_smoothing": true, + "specifications": [ + {"specification_id": "ar-2", "family": "ar", "p": 2, "trend": "c"}, + {"specification_id": "arma-1-1", "family": "arma", "p": 1, "q": 1, "trend": "c"}, + {"specification_id": "arima-1-1-1", "family": "arima", "p": 1, "d": 1, "q": 1} + ] + } + } + } +} +``` + +AR and ARMA are first-class configured families even though Statsmodels uses a +shared ARIMA backend. Orders, trend policy, initialization, estimation method, +stationarity/invertibility enforcement, fixed scalar parameters, and iteration +limits are explicit. Integrated model differencing (`d`) is distinct from the +input contract's configured transform/differencing; fingerprint stationarity +recommendations are advisory and are never applied automatically. Missing bins +reset fitting to the trailing contiguous observed segment and are never filled. + +Every model is refit independently at each rolling origin, preventing residual, +state, fitted-value, or future-value leakage. Forecasts and errors are returned +on the original requested scale. Per-horizon MAE/RMSE/bias, forecast coverage, +convergence/failure rates, fitted-parameter stability, roots, conditioning, and +comparisons with lightweight and available exponential-smoothing references are +bounded report diagnostics. There is no automatic order search or winner. + +`project_autoregressive_onto_training_frame(...)` adds fixed-width nullable +scalar columns under `cm_ar_*`, `cm_arma_*`, and `cm_arima_*`, joined only by +`series_id`/`period`/`row_id`. Forecast availability, realized diagnostic +availability, fit/reason codes, orders, roots, and training eligibility remain +point-in-time explicit. Full reports use +`time_series_fingerprint_autoregressive_summary`, bounded payloads use +`fingerprint_autoregressive`, and console output renders `Autoregressive +models`. Computational cost is proportional to configured specifications times +rolling origins; #421 candidate, fit-attempt, observation, time, memory, and +diagnostic limits bound that work. + +The third fitted family adds explicit seasonal and exogenous autoregressive +models. It uses the same regular grid, transforms, chronological folds, +original-scale inversion, and resource policy as the earlier families: + +```json +{ + "rules": { + "fingerprint.series": { + "classical_model_input": { + "enabled": true, + "frequency_ms": 60000, + "horizons": [1, 5], + "minimum_training_observations": 80, + "step_size": 20 + }, + "seasonal_exogenous": { + "enabled": true, + "projection_specification_ids": ["sarima-hour", "arimax-clock", "sarimax-hour-clock"], + "projection_horizon": 1, + "regressor_profile": { + "allow_partial_calendar": true, + "require_complete_calendar_for": [], + "max_regressors": 16 + }, + "specifications": [ + { + "specification_id": "sarima-hour", + "family": "sarima", + "p": 1, + "seasonal_p": 1, + "seasonal_period": 60, + "seasonal_cycle_ms": 3600000 + }, + { + "specification_id": "arimax-clock", + "family": "arimax", + "p": 1, + "regressor_names": ["source_hour_sin", "source_hour_cos"] + }, + { + "specification_id": "sarimax-hour-clock", + "family": "sarimax", + "p": 1, + "seasonal_p": 1, + "seasonal_period": 60, + "seasonal_cycle_ms": 3600000, + "regressor_names": ["source_hour_sin", "source_hour_cos"] + } + ] + } + } + } +} +``` + +SARIMA, ARIMAX, and SARIMAX remain separate configured families even though +they share Statsmodels' state-space SARIMAX backend. Nonseasonal and seasonal +orders, the seasonal period and its elapsed-millisecond cycle, trend, +initialization, optimizer, stationarity/invertibility enforcement, fixed scalar +parameters, and iteration limits are explicit. A runtime cycle check prevents a +configuration written for one sampling frequency from being silently reused at +another frequency. There is no automatic order, regressor, or winner selection. + +Exogenous columns come only from the deterministic calendar classifier. The +stable vocabulary covers source-hour and weekday cycles, market/session states, +session overlaps, rollover, Sunday open, Friday close, London fix, month/quarter/ +year end, holiday/event presence, and explicitly registered `tag:*` values. +Column order, vocabulary, forecast-time availability, missingness, calendar +profile completeness, and provenance are recorded. Unknown regressors and +observed or future-derived market values are rejected. Holiday/event regressors +may be marked partial under the bundled advisory calendar, or required to have +a complete operator-supplied calendar profile. + +Each model is refit independently at every #421 rolling origin. Future calendar +values are classified from future grid timestamps without reading future quote +values; missing bins and expected closures remain null and reset fitting to a +trailing contiguous segment. Reports contain bounded parameters, residual +summaries, roots, conditioning, convergence/failure rates, horizon metrics, +regime-conditioned errors, lightweight baseline references, and optional #422/ +#423 references using descriptive shared-fold semantics only. + +`project_seasonal_exogenous_onto_training_frame(...)` augments the same enriched +tick frame with 123 registered nullable scalar columns under `cm_sarima_*`, +`cm_arimax_*`, and `cm_sarimax_*`. These include order and regressor codes, +origin/target/fold/horizon identity, forecast availability, convergence and +reason codes, and separately gated realized diagnostics. They preserve observed +and `synth_*` namespaces and flow through the existing Polars cache and Influx +projection. Full reports use +`time_series_fingerprint_seasonal_exogenous_summary`, bounded payloads use +`fingerprint_seasonal_exogenous`, and console output renders `Seasonal and +exogenous models`. + +The fourth fitted family adds explicit structural state-space models and +leakage-safe Kalman state diagnostics. It consumes the same #421 regular grid, +aggregation, transform, rolling-origin folds, horizons, and resource policy: + +```json +{ + "rules": { + "fingerprint.series": { + "classical_model_input": { + "enabled": true, + "frequency_ms": 60000, + "horizons": [1, 5], + "minimum_training_observations": 80, + "step_size": 20 + }, + "state_space": { + "enabled": true, + "projection_specification_id": "local-level", + "projection_horizon": 1, + "max_state_dimension": 64, + "max_component_count": 8, + "max_prediction_only_gap": 240, + "max_retained_states": 16, + "specifications": [ + { + "specification_id": "local-level", + "family": "local_level" + }, + { + "specification_id": "local-linear-trend", + "family": "local_linear_trend", + "stochastic_trend": true + }, + { + "specification_id": "structural-hourly", + "family": "structural", + "seasonal_period": 60, + "seasonal_cycle_ms": 3600000 + } + ] + } + } + } +} +``` + +Local level, local linear trend, and configurable structural models remain +first-class specifications. Level/trend/irregular, seasonal, cycle, and +autoregressive components; stochastic flags; initialization; optimizer; fixed +parameters; and iteration limits are explicit. There is no automatic component +search or winner. Seasonal periods carry an elapsed-millisecond cycle so a +configuration cannot silently move to a different sampling frequency. + +Missing regular-grid observations are passed to the Statsmodels +`UnobservedComponents` backend as missing observations. The Kalman system +performs one prediction-only transition per grid step: expected closures and +true missing observations remain distinct in the #421 input contract, neither +is forward-filled, and long prediction-only runs are bounded and reported. +Each model is independently refit at every forecast origin. Filtered states use +only that origin's training segment. Smoothed states are retrospective +diagnostics for the same bounded segment, are never used to forecast, and are +never training-eligible. + +Reports include bounded likelihood/AIC/BIC, innovations, state uncertainty, +parameters, convergence/failure rates, prediction-only transition counts, +horizon metrics, lightweight baselines, and optional #422/#423/#424 references. +Comparisons are descriptive shared-fold references only. Full reports use +`time_series_fingerprint_state_space_summary`, bounded payloads use +`fingerprint_state_space`, and console output renders `State-space and Kalman +models`. + +`project_state_space_onto_training_frame(...)` augments the same enriched tick +rows with 52 registered nullable scalar columns: 32 under `cm_state_space_*` +and 20 under `cm_kalman_*`. Forecast, actual/error, fit/reason, fold, horizon, +filtered-state, smoothed-state, uncertainty, availability, retrospective, and +training-eligibility fields join only by `series_id`/`period`/`row_id`. They +preserve observed and `synth_*` namespaces and serialize through the existing +Polars cache and Influx projection. + +The fifth fitted family provides explicit symmetric ARCH(q) and GARCH(p,q) +conditional-variance models through the optional `arch` backend. It requires a +return-bearing #421 input contract (`transform: return` or `log_return`) and +keeps input definition, mean model, innovation and variance orders, +distribution, scaling, variance initialization, covariance type, parameter +bounds, and iteration limits explicit: + +```json +{ + "rules": { + "fingerprint.series": { + "classical_model_input": { + "enabled": true, + "transform": "return", + "horizons": [1, 5], + "minimum_training_observations": 80 + }, + "volatility": { + "enabled": true, + "projection_specification_ids": ["arch-5", "garch-1-1"], + "projection_horizon": 1, + "realized_variance_proxy": "squared_return", + "annualization_periods": 252, + "specifications": [ + { + "specification_id": "arch-5", + "family": "arch", + "input_definition": "raw_return", + "mean_model": "zero", + "distribution": "normal", + "innovation_order": 5 + }, + { + "specification_id": "garch-1-1", + "family": "garch", + "input_definition": "raw_return", + "mean_model": "constant", + "distribution": "students_t", + "innovation_order": 1, + "variance_order": 1 + } + ] + } + } + } +} +``` + +Raw returns, log returns, per-origin demeaned returns, and explicitly referenced +preceding mean-model residuals are separate contracts. Missing grid bins are +never filled: fitting uses only the trailing contiguous segment and records the +reset count. Fits guard finite inputs, minimum history, scaling, positive +variance, parameter bounds, persistence, unconditional variance, optimizer +status, and numerical failures with stable reason codes and without backend +exception text. + +Forecast evaluation keeps return-mean, conditional-variance, and volatility +errors separate. Realized variance currently uses the explicit deterministic +`squared_return` proxy; rolling-variance and EWMA references use the same folds. +Multiple horizons, rolling stability, convergence/failure rates, bounded +standardized-residual diagnostics, and preceding-model references are reported, +but there is no automatic winner. GJR-GARCH and EGARCH have registry entries for +future extension and are not silently fitted by this family. + +`project_volatility_onto_training_frame(...)` augments the same enriched tick +rows with 78 registered nullable scalar columns under `cm_arch_*` and +`cm_garch_*`. Forecasts are attached at origin availability; actual returns and +realized diagnostics are attached only after target availability. Durable +`series_id`/`period`/`row_id` joins preserve observed and `synth_*` fields and +round-trip through the existing Polars cache and Influx projection. Full reports +use `time_series_fingerprint_volatility_summary`, bounded payloads use +`fingerprint_volatility`, and console output renders `ARCH and GARCH volatility +models`. + +The opt-in family-neutral comparison layer consumes those saved bounded +evaluation artifacts; it does not refit models. Enable it with +`fingerprint.series.classical_model_comparison.enabled: true` after enabling the +model-input contract and the families to compare. Each record carries the +dataset/fingerprint, regularization contract, fold set, target metric, scale, +horizon, period, specification, and explicit reference baseline. A comparison +is ineligible when any compatible-identity requirement differs or when bounded +fold evidence is incomplete. Mean/level, return-mean, conditional-variance, and +absolute-return-volatility metrics remain separate. + +Skill is descriptive and reference-relative: ratio reduction for MAE/RMSE/bias +and baseline-minus-model for QLIKE. Negative skill is preserved. Missing or +near-zero references produce stable reason codes instead of silently changing +the baseline. Rolling error and parameter drift, convergence and failure rates, +regime context, resource-limit terminations, and representative reason counts +are bounded. Failed fits remain in accounting denominators. No `winner`, +`best_model`, production recommendation, automatic order search, or +hyperparameter search is emitted. + +`project_classical_model_comparison_onto_training_frame(...)` adds 43 nullable +diagnostic scalars under `cm_comparison_*`, `cm_skill_*`, and `cm_stability_*`. +They join by `series_id`/`period`/`row_id`, remain null before target-time +availability, preserve duplicate timestamps and observed/`synth_*` columns, and +are explicitly retrospective and not training-eligible. Full reports use +`time_series_fingerprint_classical_model_comparison_summary`, bounded payloads +use `fingerprint_classical_model_comparison`, and console output renders +`Classical model comparison`. + Every series fingerprint also includes a bounded `fingerprint_audit` section. It records expected, emitted, and intentionally skipped fingerprint sections, stable skip/eligibility reason codes, calendar-profile completeness, tick-spread -conditioning eligibility, dynamics readiness, and stationarity readiness. This -is machine-readable contract metadata for report consumers; the full fingerprint -sections remain the source of the detailed statistics. +conditioning eligibility, dynamics readiness, stationarity readiness, and +decomposition readiness. This is machine-readable contract metadata for report +consumers; the full fingerprint sections remain the source of the detailed +statistics. + +Topology attention targets include bounded `inspection_context` when a mapped +remediation action has focused evidence. Invalid timestamps expose row positions +and parse-failure counts; non-monotonic timestamps expose offending transitions; +exact duplicate rows expose their timestamp values and occurrence counts; +suspicious gaps expose largest boundaries, durations, and expected-session +classification; and weekend activity exposes timestamp/session buckets. These +records do not include raw quote rows or absolute paths. Each context section +reports included, omitted, and truncated counts with full `limit_metadata`. +Expected session closures remain non-actionable context and only accompany a +suspicious-gap drill-down. Set +`fingerprint.series.topology_inspection_sample_limit` in a quality profile from +`0` through `5` to control the per-section sample count. + Quality JSON reports and CLI summaries also include bounded regime and readiness summaries when fingerprint findings are present. Use `time_series_fingerprint_regime_summary` to scan dominant session states, active @@ -1009,7 +1934,10 @@ status, ACF basis, configured lag coverage, computed/skipped lag counts, skipped-lag reason counts, and per-series sample counts. It also includes stationarity status, calculation basis, sample counts, configured rolling windows, computed/skipped window counts, skipped-window reasons, rounding -policy, zero-variance markers, and recommended transforms. Use +policy, zero-variance markers, and recommended transforms. The readiness surface +also carries decomposition basis, sample/window coverage, +stationarity dependency status, trend direction, structural-break candidate +counts, limitations, and its training-projection contract. Use `time_series_fingerprint_readiness_risk` when you need a bounded, deterministic triage list of targets and sections most likely to block downstream fingerprint use. It ranks existing readiness, topology, dependence, regime, cache-source, @@ -1032,6 +1960,38 @@ The command reads report JSON only; it does not rescan market data. Use `--target-limit`, `--section-limit`, and `--reason-limit` to control the bounded machine JSON and matching human output. +Add `--next-work` when the question is which fingerprint product gap to address +next rather than which targets are risky: + +```sh +histdatacom quality fingerprint-readiness \ + --report reports/quality.json \ + --next-work \ + --alternate-limit 2 \ + --json +``` + +The `histdatacom.fingerprint-next-work.v1` result combines already-saved +readiness-risk evidence with the current implemented/planned fingerprint +registry. It emits one recommendation, bounded alternates, publish-safe input +report identities, representative target axes, reason codes, known +prerequisites and downstream consumers, confidence/basis metadata, and +issue-ready acceptance-criteria suggestions. Existing section-readiness and +report-surface gaps rank ahead of later planned capabilities. Use the ordinary +readiness-risk output for target-level diagnosis; use `--next-work` for the +bounded cross-report product recommendation. + +The recommendation basis also records whether the saved evidence confirms the +enriched single-row ASCII tick training substrate, legacy-cache projections, +durable row-identity columns, duplicate timestamps, unequal cross-series ranges, +and triangle comparisons. `ascii/T` remains the only base grain; legacy M1 +targets are counted as ignored non-base evidence and cannot become a platform +or M1 implementation recommendation. The command never rescans market data, +changes quality status, or creates, edits, closes, or ranks GitHub issues. Issue +references are static capability metadata only. `--target-limit` also bounds +representative recommendation axes, while `--alternate-limit` bounds alternate +recommendations; both emit explicit truncation metadata. + Bounded report and fingerprint summary payloads include `limit_metadata` and expanded `payload_limits` entries with requested, default, effective, minimum, maximum, and unbounded limit fields. The legacy `limit` field remains present @@ -1046,7 +2006,8 @@ histdatacom quality fingerprint-schema --json Use `histdatacom quality fingerprint-schema` for a concise human-readable summary, or add `--quality-profile profiles/strict-ci.json` to reflect profile-overridden fingerprint knobs such as quantiles, lags, rolling windows, -histogram bins, max rows, rounding, and distribution-attention thresholds. This +histogram bins, max rows, rounding, topology-inspection samples, and +distribution-attention thresholds. This discovery command is for downstream parsers, validators, and schema review: it lists schema versions, metadata keys, target capabilities, implemented/planned sections, basis/status/reason vocabularies, and publish-safe example fragments. @@ -1184,6 +2145,73 @@ payloads, and quality preflight sample evidence, opt in with: The embedded audit reuses the standalone remediation-catalog audit schema, keeps known source-code coverage separate from observed report coverage, and remains advisory; it does not change finding severities or quality exit policy. +The audit also records how each static finding was attributed to a rule: +`exact` for an explicit literal, constant, or class rule; `inferred` for a +single-rule helper chain, local rule object, module rule, or unambiguous +finding-code prefix; and `unresolved` when multiple rule callers remain +possible. Unresolved entries retain the source-family fallback and include a +stable `attribution_reason` such as `ambiguous_helper_rules`. Bounded source +family, helper, and finding-prefix counts make the remaining ambiguity +auditable without executing quality rules or reading market data. + +Inspect that attribution directly through the standalone command: + +```sh +histdatacom quality remediation-catalog --json +``` + +The concise renderer includes exact, inferred, and unresolved occurrence +counts plus attribution status and reason on ranked gaps. These fields improve +catalog planning evidence only; they do not add remediation mappings or change +gap severity. Ranked gaps also include `actionability` and +`actionability_reason`. Actionable defects sort ahead of policy/profile +decisions, unsupported formats or capabilities, expected context, attribution +or diagnostic blockers, and unsafe-to-automate cases. The summary preserves the +ordinary mapped/unmapped counts and adds boundary-aware actionable, intentional, +attribution-blocked, and diagnostic-blocked counts. Unknown warning/error gaps +remain actionable by default, so boundary classification cannot silently hide a +new defect. + +Every audit also derives a bounded `remediation_plan` from the complete ranked +gap set. Plan items are re-ranked by deterministic fixability rather than raw +severity alone and include the original catalog-gap rank, actionability, +severity counts, exact-or-family selector proposal, draft hint-code slug, +suggested action kind, fixability score/level/confidence with a reason trail, +fields still requiring maintainer judgment, and bounded source/report evidence. +Observed report-only gaps use their reported rule/finding identity and remain +first-class plan candidates. Attribution and diagnostic blockers are explicitly +marked `blocked`; policy, support, expected-context, unsafe, and informational +boundaries remain visible with low fixability instead of being presented as +automatic catalog edits. The `histdatacom.quality-remediation-plan.v1` artifact +is advisory: it never edits the remediation catalog, creates GitHub work, or +changes finding severity and exit policy. + +To translate findings in a saved quality report into concrete user-data repair +steps, use the separate non-mutating repair-plan command: + +```sh +histdatacom quality repair-plan \ + --report reports/quality.json +``` + +Add `--json` for the bounded `histdatacom.quality-repair-plan.v1` artifact. +`--item-limit` controls included findings and `--evidence-limit` controls the +publish-safe diagnostic values retained per item; both surfaces include total, +included, omitted, and truncation metadata. The initial operation vocabulary +covers invalid archive/member renames, missing or unexpected member rebuilds, +extra-member inspection, CRC/corrupt archive replacement, and read-access +restoration. Exact report evidence produces an exact proposal, incomplete +evidence produces `needs_context`, and unmapped or out-of-scope findings remain +explicitly `unsupported`. + +The repair plan is advisory and manual-only. It does not expose an `--apply` +mode and never renames files, rewrites ZIPs, removes members, changes +permissions, downloads replacements, or changes report severity and exit +policy. This is distinct from remediation-catalog `remediation_plan` output: +the catalog plan helps maintainers add missing hint mappings, while +`quality repair-plan` helps users interpret already observed findings and +mapped hints without changing their data. + The same reporting surface can be enabled without a profile file by passing `--quality-remediation-catalog-audit` with `--quality`, `--repo-quality`, or `--quality-preflight`. When the flag is combined with `--quality-profile`, the @@ -1249,10 +2277,21 @@ deterministic and includes the active profile source, source path, configured rule IDs, configured modeling assumptions, reporting keys, and the resolved `reporting.remediation_catalog_audit.enabled` value after CLI overrides. It also includes a `profile_explanation` section with input channels such as -built-in defaults, YAML config, profile file, API options, and CLI overrides; -per-value source rows; and a bounded effective diff from the built-in default -profile. The `text` and `markdown` renderers are presentation layers over that -same explanation data. +built-in defaults, named profiles, YAML config, profile files, API options, and +CLI overrides; per-value source rows; and a bounded effective diff from the +built-in default profile. Resolution preserves those facts before the profile +is normalized, so an override row records its previous source and value instead +of reconstructing them from the final JSON. The `text` and `markdown` renderers +are presentation layers over that same explanation data. + +Python callers that need the same first-class contract can use +`resolve_quality_profile()`, `load_quality_profile_file_resolution()`, and +`apply_quality_profile_overrides()` from `histdatacom.data_quality`. The +returned `QualityProfileResolution.profile` remains the normal validated +`QualityProfile`; `value_sources`, `input_channels`, and `to_payload()` expose +the deterministic provenance contract. Existing `quality_profile_from_*()` and +`load_quality_profile_file()` callers continue to receive `QualityProfile` +directly. ```sh histdatacom --quality \ @@ -1447,19 +2486,88 @@ engineering, dashboards, and modeling decisions. They are separate from `histdatacom --quality`: analytics reports do not produce clean/warning/failed statuses and do not downgrade repository quality metadata. +#### Point-in-Time Market Context + +The `histdatacom.market_context` domain stores approved macro, central-bank, +news, and shock evidence as immutable versioned timelines rather than repeated +tick columns. Every event vintage retains source/version and retrieval +metadata, content hashes, licensing and redistribution constraints, affected +currencies/symbols, confidence, limitations, normalized source time, explicit +pre/post windows, and revision lineage. + +Ex-ante queries require an as-of time and cannot expose schedules, actuals, or +revisions before the exact vintage was available. Ex-post queries retain all +vintages. Bounded window joins return compact context/calendar sidecars over +`ReconstructionWindowV1`; they never persist the full analytical enrichment +frame. Missing, incomplete, and out-of-coverage context remain explicit rather +than becoming invented event labels. + +Calendar sidecars reuse the existing session, holiday, rollover, fix, and +month/quarter/year-end classifier. The shared source-adapter seam retains +provenance and licensing but does not authorize or scrape a paid news corpus. +The production corpus uses documented official ONS, ECB, Bank of England, and +Federal Reserve sources plus a small cited operator shock catalog: + +```bash +histdatacom analytics market-context-corpus \ + --artifact-dir .histdatacom/market-context \ + --start-date 2002-03-01 \ + --end-date 2026-06-30 +``` + +The command writes immutable content-addressed raw snapshots, a directly +loadable timeline, and a self-contained corpus with source hashes, licenses, +coverage/missingness, duplicate counts, runtime, and peak memory. Installed +helpers replay the raw snapshots, refuse unsupported reconstruction context, +return the carving query contract, and project bounded benchmark event state. + +See [`docs/market-context-contracts.md`](docs/market-context-contracts.md) for +the source selection, licenses, artifacts, replay, coverage/preflight, +timezone and revision rules, information-audit integration, streaming limits, +and trust gates. + +#### CFTC positioning state + +CFTC Commitments of Traders is a separate persistent weekly positioning +sidecar, not a `MarketContextEventV1` window and not a repeated tick column. +The installed campaign retains Legacy and TFF, futures-only and combined, +EUR/GBP/EURGBP contract identities, official release/correction evidence, +compressed-history consistency, immutable refresh diffs, and fail-closed +ex-ante semantics: + +```bash +histdatacom analytics cftc-positioning-corpus \ + --artifact-dir data/.histdatacom/analytics/cftc-positioning \ + --start-date 2002-03-01 \ + --end-date 2026-06-30 +``` + +Window queries expose bounded latest-known snapshots, age, mapping status, and +point-in-time-safe net/open-interest/change/rolling features. Current PRE rows +cannot masquerade as original vintages; nominal publication estimates fail +strict ex-ante use. Companion receipts bind the query into the information +audit, benchmark, motif selection, planning, and carving without changing +their immutable v1 schemas. + +See [`docs/cftc-positioning-contracts.md`](docs/cftc-positioning-contracts.md) +for source selection and acknowledgement, field/family/scope mappings, quote +direction, publication/restatement rules, artifact replay/diff behavior, +coverage, resource limits, consumer seams, and explicit nonclaims. + #### Feed-Regime Detection -`histdatacom analytics feed-regimes` profiles local ASCII tick artifacts by -month or year, then segments long histories into feed-behavior eras such as -sparse, transitional, and dense periods. The report includes tick density, -inter-arrival intervals, quote update cadence, zero-change runs, spread -statistics, quiet-gap counts, regime boundaries, and summary metadata. +`histdatacom analytics feed-regimes` projects canonical ASCII tick fingerprints +into a versioned feed-epoch definition. Epochs represent evidence-backed +changes in the technological observation process, not calendar eras or market +regimes. Boundaries include uncertainty intervals and deterministic stability +evidence under sampling, missing-period, and feature-removal perturbations. ```sh histdatacom analytics feed-regimes \ --target data/ASCII/T/eurusd \ --bucket month \ - --report reports/eurusd-feed-regimes.json + --report reports/eurusd-feed-regimes.json \ + --epoch-artifact reports/eurusd-feed-epochs.v1.json ``` Use `--json` to print the full machine-readable payload to stdout: @@ -1468,10 +2576,602 @@ Use `--json` to print the full machine-readable payload to stdout: histdatacom analytics feed-regimes --target data/ --json ``` -Use these outputs to choose modeling windows, session filters, feature -normalization strategies, or dashboard annotations. Treat surprising regimes as -research signals; run `histdatacom --quality` separately when you need -readability, timestamp consistency, ZIP integrity, or pass/fail validation. +Only stability-passing definitions are valid downstream observation-model +inputs. Periods inside a boundary uncertainty interval are assigned to an +explicit transition instead of being forced into either neighboring epoch. The +artifact records every fingerprint, source, feature-provenance, conditioning, +quality, and config hash needed for replay. + +Feed-epoch fitting is a bounded control-plane operation. Streaming +reconstruction references the definition ID and carries only compact epoch or +transition assignments; it does not persist fingerprint panels or the wide +analytical frame per tick. See +[`docs/feed-epoch-contracts.md`](docs/feed-epoch-contracts.md) for the schema, +trust gate, resource limits, and streaming integration. + +For the real three-symbol technology-epoch fit, v2 scans monthly Arrow caches +column-wise, uses explicit calendar/open/active-time denominators, and applies +robust multivariate PELT plus family-specific holdouts: + +```sh +histdatacom analytics feed-epochs-v2 \ + --target data/ASCII/T/eurusd data/ASCII/T/gbpusd data/ASCII/T/eurgbp \ + --artifact-dir data/.histdatacom/feed-epochs-v2 +``` + +The command writes separate compact definition, bounded evidence, and runtime +artifacts. It does not create an augmented cache or claim that a detected +boundary is a market regime, recovered quote, vendor cause, or broker profile. + +#### Historical Feed-Observation Operators + +`ObservationOperatorV1` turns a bounded market-event surface into a sparse, +quantized delivery-observation surface using stability-passing feed epochs. +The operator supports conditioned thinning, unchanged-quote filtering, +timestamp and price quantization, batching, duplicates, burst/rate caps, +outages, and reconnect behavior through versioned parameters with explicit +support and uncertainty. + +The bare fitting boundary consumes canonical feed-epoch projections or paired +controlled-calibration evidence. Canonical sparse history does not identify a +true dense-event denominator, so unsupported thinning parameters remain +visibly unsupported and use a neutral identity behavior rather than being +presented as direct observations. Sparse conditioned strata follow a fixed +state/session/event-to-global fallback hierarchy or fail closed. + +`ObservationCalibrationCampaignV2` adds the real-evidence trust boundary. It +fits relative active-time retention and supported delivery mechanisms by +symbol, technology epoch, update type, and session, then applies the operator +to dense reference caches in chronological calibration, validation, and final +holdout blocks. Every parameter carries support, uncertainty, source hashes, +and an identifiability or refusal reason. Calendar closure, archive gaps, +unchanged filtering, batching, quantization, duplicates, outages, and reconnect +behavior remain distinct diagnostics. + +```sh +histdatacom analytics observation-calibrate-v2 \ + --definition data/.histdatacom/feed-epochs-v2/feed-epochs-v2-definition.json \ + --evidence data/.histdatacom/feed-epochs-v2/feed-epochs-v2-evidence.json \ + --artifact-dir data/.histdatacom/observation-calibration-v2 +``` + +The campaign cannot become application-ready when retention is merely identity +because the dense denominator is unknown, when a default required mechanism is +unsupported, or when a final holdout fails. Requesting an optional unsupported +mechanism also fails closed. Dense and degraded window rows stay process-local; +the persisted campaign contains aggregate evidence and the compact replayable +operator only. + +Observation rendering does not mutate `SyntheticEventV1`. Inputs retain their +market-event IDs and produce separate operator-lineaged delivery observations. +Forward application preserves protected historical anchors exactly; the +separate `degrade()` interface lets #436 degrade modern holdouts while +protecting only explicitly selected controls. + +Application uses `ReconstructionWindowV1`, aligned timestamp/batch quanta, +declared halo metadata, required bounded carry state after the source window, +deterministic hash decisions, and input/output amplification limits. The +compact operator JSON is durable and hash-replayable; fit panels and window +output observations remain bounded intermediates rather than augmented +permanent cache columns. See +[`docs/observation-operator-contracts.md`](docs/observation-operator-contracts.md) +for the contracts, trust gates, fallback semantics, and streaming boundary. + +#### Reverse-Degradation Benchmark + +`ReverseDegradationBenchmarkV1` is the generator-neutral validation harness for +reconstruction work. It streams dense modern reference events through a +versioned historical observation operator, evaluates transparent no-fill, +interpolation, resampling, and existing empirical-overlay controls alongside +candidate generators, and retains only bounded online aggregates. + +The immutable benchmark manifest subdivides the existing withheld validation +boundary into ordered validation and final-holdout periods without changing the +upstream information-mode v1 schema. A valid experiment covers multiple feed +epochs and degradation severities, reports symbol/epoch/session/event/sparsity +slices, records uncertainty and ensemble support, and carries cross-series, +strategy, convergence, failure, memory, scratch, and output-cost hooks. + +Hard historical-constraint or protected-anchor violations always block +promotion regardless of soft statistical fit. Scorecards compare every method +with no fill but explicitly set `automatic_winner` to false and never emit a +winner candidate. Dense, degraded, reconstructed, and rejected intermediates +remain process-local; only the compact manifest and scorecard are intended to +persist. See +[`docs/reverse-degradation-benchmark-contracts.md`](docs/reverse-degradation-benchmark-contracts.md) +for the complete interfaces, metric semantics, resource bounds, and trust +gates. + +The real-data promotion policy is separately frozen and packaged before any +promotable candidate campaign. It distinguishes hard campaign/candidate gates +from visible advisory evidence, fails closed when hard observations are +missing, and never selects an automatic winner. See +[`docs/reverse-degradation-benchmark-corpus.md`](docs/reverse-degradation-benchmark-corpus.md) +for the predeclared thresholds, evidence ordering, provisional motif boundary, +and scientific nonclaims. + +The installed `histdatacom analytics reverse-degradation-benchmark-corpus` +command now builds the real EURUSD/GBPUSD/EURGBP Arrow partitions, replays +source and selected-window hashes, executes all declared degradation families, +runs dense/no-fill/interpolation/motif/negative controls, and writes a compact +content-addressed manifest, motif index, leakage audit, resource audit, and +scorecard. Required fitted-operator or replay failures abort the campaign; +dense and holdout event rows remain process-local. + +#### Empirical Reference-Motif Index + +`ReferenceMotifIndexV1` projects bounded windows from the augmented ASCII tick +surface into compact event-time offsets, bid/ask deltas, quote-transition +marks, conditioning coordinates, transformation limits, and complete source +lineage. It consumes the enriched evidence without copying the full 521-column +row into every motif event. + +Only eligible training windows may enter the artifact. Chronological +calibration, validation, and final-holdout windows are excluded, while +cross-split source overlap and normalized near-duplicate shapes fail closed. +Index selection is stable under input reordering, retrieval follows an explicit +exact-to-global support hierarchy, and matches expose distance, cell support, +fallback level, and deterministic fragment-ID tie-breaking. + +Ex-ante queries require an as-of timestamp and hide motifs whose observations +or artifacts were not yet available. Selected motifs bind directly into the +existing reconstruction information audit as training-split empirical-motif +inputs. Index persistence is atomic and content-addressed through an +`ArtifactRef`; augmented panels remain intermediates. See +[`docs/reference-motif-index-contracts.md`](docs/reference-motif-index-contracts.md) +for split, leakage, compact-layout, retrieval, resource, and trust semantics. + +#### Real Modern Reference-Motif Library + +The installed `histdatacom analytics modern-reference-motif-library` command +builds the first production index from 24 hash-verified monthly EURUSD, +GBPUSD, and EURGBP Arrow caches in stable `technology_epoch_04`. Its fixed +chronological profile keeps 201901--202301 for training and blocks 202307, +202401, and 202510 as calibration, validation, and final holdout. + +The builder prefilters normalized cross-split near duplicates, reruns the +fail-closed leakage audit, retains a deterministic compact 256-fragment train +index, aggregates support/backoff coverage, exercises explicit unsupported +refusal, and runs the unchanged #463 real benchmark twice. The installed +readers verify the content-addressed index, manifest, leakage, coverage, +qualification, and resource artifacts. Dense source and holdout rows never +enter those files. See +[`docs/modern-reference-motif-library.md`](docs/modern-reference-motif-library.md) +for the source profile, feature schema, corrected event-clock/transition +semantics, qualification gates, CLI, and nonclaims. + +#### Empirical Motif Candidate Generation + +`generate_empirical_motif_candidates()` proposes zero, one, or many narrow +`SyntheticEventV1` rows between immutable historical anchors. Cardinality and +cadence come from the conditioned delivery regime; selected empirical paths +are transformed only inside their declared time/price support and detrended +onto an anchor-to-anchor bridge so fragment seams cannot accumulate jumps. + +Seeds and event identity depend on semantic run, member, anchor, motif, and +configuration inputs—not retries, workers, windows, or storage estimates. +Each event maps to a recoverable transform containing its source motif, +support/backoff, scale, seed, condition query, and source artifact lineage. +Sparse evidence, closed sessions, zero-width intervals, unsafe quotes, and +resource overruns produce explicit empty/refused decisions. + +Candidate rows remain bounded, process-local streaming intermediates. Batch +metadata states that hard carving, broker conditioning, and final persistence +have not run. The included benchmark adapter lets the existing +reverse-degradation harness compare this generator with all controls without +selecting an automatic winner. See +[`docs/empirical-motif-generation-contracts.md`](docs/empirical-motif-generation-contracts.md) +for determinism, seam, lineage, resource, and stage-boundary details. + +#### Historical Candidate Carving + +`carve_empirical_motif_candidates()` is the first stage allowed to accept +candidate-only motif rows. A versioned constraint set applies immutable-anchor, +resource, fingerprint-validation, context-support, quarantine, and session +closure rules before conditioned motif eligibility, intensity thinning, or +spread projection. Missing support refuses rather than inventing liquidity. + +News, rollover, crisis, and other explicit state tags can change acceptance +rates, eligible motifs, and spread envelopes. Incompatible motifs may use a +same-position candidate from an explicitly supplied substitution batch; +otherwise they are rejected. Deterministic scores exclude retry, worker, and +window identity, so adjacent window outputs union to the single-window result. + +Accepted rows carry the final constraint-set ID and compact lineage back to the +candidate event, batch, and motif transform. Projected lineage retains original +quotes and candidate/output content hashes. Rejected rows are discarded; only +reconciling reason counts and bounded examples remain. See +[`docs/historical-carving-contracts.md`](docs/historical-carving-contracts.md) +for precedence, evidence binding, refusal, identity, and streaming semantics. + +#### Cross-Currency Reconciliation + +`plan_cross_currency_windows()` intersects explicit per-symbol coverage and +plans only complete synchronized windows. Missing legs, unequal leading or +trailing periods, and spans without common support remain recorded exclusions; +they are never filled or silently shortened. + +`reconcile_cross_currency_window()` applies versioned triangle and inverse +relationships at exact nanosecond event times. It never forward-fills another +instrument. Duplicate timestamps pair by deterministic event ordinal, while +asynchronous support and stale-join risk remain measured. Only synthetic +quotes may be projected; immutable observations are content-hashed and must +remain unchanged. The first certified relationship is `EURUSD / GBPUSD ~= +EURGBP`. + +Residuals, support, projections, and infeasibility are stratified by session, +event, and feed epoch. Every passing generation group still requires the same +content-bound validation after broker conditioning. A partition manifest can +commit only when that final validation covers the complete all-symbol +synchronization unit and exact output content. The existing #331 diagnostic +also consumes reconciled streams directly without a permanent cache +roundtrip. See +[`docs/cross-currency-reconciliation-contracts.md`](docs/cross-currency-reconciliation-contracts.md) +for projection, refusal, validation, compatibility, and atomic-commit details. + +#### Calibrated Reconstruction Ensembles + +`plan_reconstruction_ensemble()` derives stable member IDs and semantic seeds +from exact source/configuration hashes, not workers, retries, row order, or +retention rank. Reverse-degradation windows then measure member intervals by +feed epoch, session, event state, symbol, horizon, and sparsity. Validation +cells fit bounded adjustments; final-holdout cells alone report achieved +coverage, failures, refusals, and substantive diversity. + +Logical-content hashes exclude member/seed/lineage identity, so identical +market paths are diagnosed as collapsed and ID-only or metric-free differences +cannot count as useful diversity. The primary member is explicitly a compact +validation-medoid representative, not historical truth, an automatic winner, +or a default generator. + +Storage estimates cover all-member computation and scratch while durable +output is limited to a configured retained subset. Omitted members can be +regenerated only from the frozen plan after every source and configuration +SHA-256 matches. Reports contain bounded summaries rather than event rows. +Motif-match similarity remains uncalibrated transformation evidence; generated +tick confidence stays null. See +[`docs/reconstruction-ensemble-calibration-contracts.md`](docs/reconstruction-ensemble-calibration-contracts.md) +for calibration, confidence, diversity, retention, and replay semantics. + +#### Live Broker Delivery Capture + +`histdatacom.broker_capture` records a broker feed as versioned measurement +evidence rather than guessing modern delivery style from historical vendor +data. Adapter messages retain optional broker/exchange timestamps with explicit +precision, exact price lexemes, batch/message identity, honest size/activity +semantics, quote and lifecycle events, and raw-message hashes where permitted. +The collector adds adjacent UTC wall and monotonic receive clocks plus explicit +clock-correction events. + +Canonical JSONL partitions are appended, fsynced, rotated, hashed, and exposed +only after atomic compact-manifest publication. Quota, immutable retention, and +high-watermark backpressure refuse predictably. Partial/orphan artifacts remain +detectable but undiscoverable as completed data, while verified replay checks +sidecars, bytes, hashes, rows, counts, and sequence before sending events +through the same consumer interface used during live collection. + +The core adapter protocol never inspects private broker configuration and the +public contracts reject credential-shaped metadata. A real adapter still +requires an explicit broker/protocol/licensing decision; no redesign is needed. +See [`docs/broker-capture-contracts.md`](docs/broker-capture-contracts.md) for +clock, security, storage, replay, fixture, and fingerprint eligibility gates. + +#### Broker Delivery Fingerprints + +Qualified broker captures are converted into compact immutable delivery +profiles with `fit_broker_delivery_fingerprint()`. Fitting verifies capture +health and hashes in a first streaming pass, then performs bounded deterministic +aggregation in a second pass and rechecks the logical content hash. It does not +persist augmented capture rows or materialize tick-sized intermediates. + +Profiles describe cadence, quote intensity, spread and spread changes, +duplicate/stale/burst behavior, source timestamp and price precision, batching, +outage/reconnect/clock behavior, and conditional behavior by symbol, session, +overlap, special window, holiday, market event, and lifecycle state. Every cell +records observed support plus an explicit supported, ordered-backoff, or +unsupported decision. Every metric has support, bounded samples, uncertainty, +extrema, quantiles, units, and limitations. + +`compare_broker_delivery_fingerprints()` produces bounded, stratified drift +evidence without a global similarity score or automatic winner. Successors are +new effective-dated artifacts that reference—but never mutate—the old profile, +so prior synthetic lineage remains reproducible. See +[`docs/broker-delivery-fingerprint-contracts.md`](docs/broker-delivery-fingerprint-contracts.md) +for eligibility, streaming, condition, drift, persistence, and #445 handoff +semantics. + +#### Broker-Conditioned Reconstruction + +`condition_broker_proposal()` applies a versioned, bounded broker-delivery +strength to cadence, burst/quiet/outage, spread, and precision coordinates before +motif retrieval. Exact profile cells and recorded backoff are honored; missing, +unsupported, ineffective, or mismatched-drift selections refuse without issuing +a conditioned query. + +After historical carving and cross-currency reconciliation, +`render_broker_delivery()` applies deterministic precision, rounding, batching, +stale-quote, exact-duplicate, timestamp, and spread presentation to synthetic +rows only. Observed anchors remain unchanged. The entire group is withheld until +local constraints, post-broker cross-currency validation, and the #331 +cross-instrument quality path pass. Compact manifests retain content hashes, +event lineage, profile/effective-period/drift evidence, action counts, config, +and optional paired benchmark comparison IDs; augmented tick intermediates are +not made durable. + +See +[`docs/broker-delivery-transfer-contracts.md`](docs/broker-delivery-transfer-contracts.md) +for proposal, renderer, refusal, validation, benchmark, and streaming/persistence +semantics. + +#### Atomic Reconstruction Persistence + +`publish_reconstruction_group()` turns one fully validated broker-rendered +symbol group into the final narrow archive. It requires an independent exact +set of immutable observed anchors plus a primary/retained-member storage +preflight. Only the 26-column `SyntheticEventV1` schema is written; the +521-column analytical frame and rejected candidates remain scratch data. + +Zstandard Parquet files are partitioned by schema, run, broker fingerprint, +ensemble member, symbol group, symbol, and UTC event date. Files and compact +source/constraint/quality/replay/retention manifests are validated below a +hidden transaction directory, then the complete synchronized unit is promoted +with one atomic same-filesystem rename. Discovery sees only committed +directories. Repeating an identical publication is idempotent, while anchor +drift, truncation, checksum/schema/count mismatches, or different physical +writer settings fail closed. + +Arrow batches and lazy Polars scans support symbol/time file pruning, column +projection, and event-time predicate pushdown. The optional `query` extra adds +DuckDB for direct Parquet inspection: + +```sh +pip install "histdatacom[arrow,query]" +``` + +See +[`docs/reconstruction-persistence-contracts.md`](docs/reconstruction-persistence-contracts.md) +for layout, atomic commit, replay, preflight, query, cleanup, and #447 Temporal +handoff semantics. + +#### Reconstruction Activity Semantics + +Final reconstructed rows are quote deliveries, not centralized FX trades. +`summarize_reconstruction_activity_streams()` and +`summarize_committed_reconstruction_activity()` derive deterministic activity +metadata without widening the immutable 26-column `SyntheticEventV1` schema or +persisting the 521-column analytical frame. The committed reader projects only +the 19 event, price, origin, confidence, and lineage columns needed by the +online accumulator and processes configurable bounded Arrow batches. + +Every symbol can emit separate observed-only, synthetic-only, and merged +slices. Each slice records quote-event/update counts, exposure duration, tick +intensity, interarrival cadence, price-change and stale-quote transitions, +spread-based liquidity proxies, optional event-confidence support, exact units, +aggregation rules, bounded provenance, and a content hash. Volume handling is +explicitly `unavailable`, `omitted`, source/broker supplied, or a synthetic +activity proxy; the contracts always set `centralized_traded_volume_claim` to +false and refuse source-size states when the final event schema has no such +fields. + +Activity evidence is bound to an information manifest and either ex-post or +ex-ante mode. Ex-ante summaries require an as-of timestamp, while ex-post +summaries reject one. Validation reuses the existing reverse-degradation +scorecard's event-count, intensity, interarrival, burst/quiet, and spread +metrics plus calibration support; it never selects an automatic winner. +Explicit sum, boundary-carry, recomputation, and support-weighted-mean rules +form the derived-bar handoff for #18, with volume remaining unavailable unless +separately sourced. + +See +[`docs/reconstruction-activity-semantics.md`](docs/reconstruction-activity-semantics.md) +for metric definitions, information and volume policy, provenance, streaming, +benchmark, and derived-bar contracts. + +#### Derived Reconstruction Candlesticks + +`publish_derived_bars()` creates an optional export product from a verified +committed reconstruction manifest. It never reads raw HistData M1 rows or the +521-column analytical frame. The immutable 26-column event product remains the +source of truth, while derived bars use a separate versioned 64-column schema +and compact manifest. Each row carries the derivation policy ID and rounding +precision; the manifest records any requested time-window bounds. + +Version one supports UTC Unix-epoch-aligned `1m`, `5m`, `15m`, `30m`, `1h`, +`4h`, and `1d` half-open bins. Bid, ask, midpoint, and spread OHLC values follow +canonical `(event_time_ns, event_sequence, event_id)` order. Observed-only, +synthetic-only diagnostic, and merged-product scopes are explicit. Empty bins +and market closures emit no rows, query-cut edge bins are flagged partial, and +no price, liquidity, or volume is forward-filled. + +The streaming accumulator carries the previous quote into the next non-empty +bar for price-change/stale transition accounting, then projects #80 counts, +duration, intensity, stale rate, mean spread, and confidence support. Volume is +always null with state `unavailable`. Each row retains event bounds, origin +support, bounded generator/broker/constraint lineage, and an event-content +hash. + +```python +from histdatacom.synthetic import ( + ActivitySliceScope, + DerivedBarPolicyV1, + publish_derived_bars, + scan_derived_bars_polars, +) + +published = publish_derived_bars( + "data/exports", + "data/reconstruction-products/.../commits/.../manifest.json", + policy=DerivedBarPolicyV1( + intervals=("1m", "5m", "1h"), + scopes=(ActivitySliceScope.MERGED,), + ), +) +bars = scan_derived_bars_polars( + published.manifest_path, + columns=("symbol", "bar_start_ns", "mid_close", "event_count"), +) +``` + +Monthly Parquet partitions are written below hidden scratch, replay-verified, +and promoted with one same-filesystem rename. Column/time/symbol/scope/interval +projection and pruning are available through Arrow batches and lazy Polars +scans. Raw M1 download/import remains rejected. + +See [`docs/derived-bar-contracts.md`](docs/derived-bar-contracts.md) for the +complete interval, OHLC, activity, lineage, partial/empty-bin, storage, +verification, and downstream reconciliation contract. + +#### Reconstructed-History Strategy Sensitivity + +`evaluate_strategy_sensitivity()` applies one content-addressed strategy, +execution, cost, latency, horizon, and resource policy to multiple exact +time-aligned source cases. Supported cases include untouched observed history, +degraded modern holdouts, reconstructed ensemble members, +broker-conditioned/unconditioned streams, and verified derived bars. Case +identity includes the source artifact, symbol, half-open window, information +audit, ensemble member, broker profile, and—when applicable—bar scope and +interval. + +The evaluator is streaming and bounded. It retains only strategy state, +pending signals, and online aggregates; quotes, individual outcomes, the +521-column analytical frame, and strategy columns are not persisted. Results +are stratified by feed epoch, session, event state, sparsity, broker profile, +ensemble member, and horizon. Reports include failure, no-trade, +missing-support, and refusal rates, member/window dispersion, and explicit +reverse-degradation evidence showing whether reconstructed execution response +moves toward the dense reference relative to the degraded input. + +`ReferenceMomentumStrategyV1` is a transparent lagged-midpoint fixture for +alignment and accounting tests. It is not a recommended strategy. Version one +uses normalized exposure, crosses bid/ask, and makes latency, quote-wait, +slippage, and per-side fixed costs explicit. Reports are compact derived +metadata and always set profit claims, investment recommendations, event-schema +augmentation, and automatic winner selection to false. + +Ex-post cases require an explicit `invalid-for-backtest` reason. Mixed ex-ante +and ex-post plans require a plan-level reason as well; the label permits only a +descriptive historical counterfactual and never converts it into point-in-time +strategy evidence. + +See +[`docs/strategy-sensitivity-contracts.md`](docs/strategy-sensitivity-contracts.md) +for input identities, information-mode gates, source adapters, accounting, +stratification, restoration, terminal states, and resource bounds. + +#### EURUSD Triangle Reconstruction Certification + +`modern_reference_triangle_certification_policy()` predeclares the current +v2.1.0 scientific, operational, reporting, repository, and release contract for +the EURGBP/EURUSD/GBPUSD product over common support beginning at `200203`. It +fixes `modern_reference` delivery with the `unconditioned_reference` claim and +explicitly excludes broker adaptation. The common end month, source-readiness +contracts, scientific thresholds, and peak-memory/scratch/runtime/storage and +candidate-amplification budgets participate in the deterministic policy +identity. The older broker-bound `eurusd_triangle_certification_policy()` and +V1 dossiers remain readable for evidence replay but are not the #449 release +path. + +Certification consumes compact, verified report artifacts bound to that exact +policy identity, so evidence cannot be reused after scope or threshold drift. +Each scalar observation names the exact artifact identities that support it, +and every requirement declares the exact artifact kinds it needs. Missing +artifacts remain `missing`; measured threshold violations are `failed`; neither +can become a pass through a summary boolean. V2 rejects every broker-named +artifact instead of silently turning synthetic reference output into a broker +claim. + +`evaluate_modern_reference_reconstruction_certification()` covers all fifteen +#449 gate groups plus the individual source-readiness and operations seams and +returns one bounded `ReconstructionCertificationDossierV2`. The dossier can be +`incomplete`, `failed`, `ready-for-promotion`, or `certified`. +`ready-for-promotion` is narrowly reserved for the state where every check has +passed except the single coverage observation. Coverage is still run exactly +once, only during the explicit `dev`-to-`main` promotion. The TestPyPI local +simple-registry preflight and all non-coverage evidence must already pass. + +`histdatacom reconstruction certify --spec CAMPAIGN.json --output-directory +DIR` executes the public campaign. It verifies every declared JSON artifact's +SHA-256, schema, and subject identity, then extracts each observation through a +declared JSON pointer. Observation values cannot be written inline in the +campaign spec, and promotion-only coverage is refused on an ordinary `dev` +campaign. Publication atomically writes canonical machine JSON, deterministic +Markdown, the frozen campaign manifest, methodology evidence, and a bounded +campaign receipt. The dossier contains no tick rows or analytical-frame +columns and never claims historical truth, selects an automatic winner, makes +an investment recommendation, or authorizes release before every gate passes. + +See +[`docs/reconstruction-certification-contracts.md`](docs/reconstruction-certification-contracts.md) +for the gate mapping, artifact binding, state machine, publication semantics, +and required real-data execution sequence. + +#### Temporal Reconstruction Orchestration + +`ReconstructionRunWorkflow` plans deterministic memory-weighted waves of +all-symbol `ReconstructionWindowWorkflow` children. Each window executes the +source/enrichment, proposal, carving, cross-series, delivery-projection, +validation, and atomic-commit boundaries sequentially through activity-side +stage handlers. Workflow history carries only bounded commands, counters, +checkpoints, and strong artifact references. + +Stage receipts and manifest-store compare-and-swap snapshots make worker loss, +activity retry, duplicate completion, and process restart resumable without +duplicating committed rows. Cancellation removes only disposable window +scratch. The final report independently verifies every committed publication +and reconciles storage counts and scope with workflow checkpoints. + +Default workers install the seven versioned first-party handlers. The reference +path uses explicit modern-reference identity delivery and generic v2 +persistence, so no application registration or fake broker fingerprint is +needed. Validation keeps a byte-identical staged-manifest mirror and a separate +transaction descriptor, allowing a retry after the atomic rename but before +the commit receipt to recover the already committed publication safely. + +See +[`docs/reconstruction-temporal-orchestration.md`](docs/reconstruction-temporal-orchestration.md) +for adapter registration, queue/resource policy, backpressure, recovery, +cancellation, report reconciliation, and fault-injection guarantees. + +#### Public Reconstruction CLI and API + +`histdatacom reconstruction` and `ReconstructionClient` expose the same typed +plan, operator-request, preflight, submission, receipt, status, cancel, resume, +output-list, bounded-preview, and integrity-replay contracts. The installed +command requires an explicit ex-post or ex-ante information mode plus the +machine-readable acknowledgement that reconstructed output is plausible +counterfactual evidence—not recovered historical truth. + +Full-range planning also has a bounded plan-set surface. `plan-set` begins with +bounded month groups and deterministically bisects any group whose execution, +retention, or artifact-size preflight refuses it; `preflight-set` freshly +verifies every resulting shard identity, artifact hash, exact contiguity, +refusal, and resource bound and reconciles the parent aggregate. This preserves +both the per-window runtime budget and the 64 MiB plan-artifact limit instead +of weakening either one for long historical ranges. Shared monthly source +partitions count once in the parent inventory totals, and repeated strong-ref +verification is cached only while the file's device, inode, size, modification +time, and change time remain identical. A span with no scientifically +supported window is retained as a refusal-only shard with zero workflows and +zero output estimates; acknowledging refusals makes that no-op safe to skip, +but never turns the unsupported span into reconstructed output. + +Only ASCII/T and the complete EURGBP/EURUSD/GBPUSD triangle are accepted. M1, +bar, partial-triangle, and broker-only requests fail before execution. Temporal +is the production path; `--local` is an explicit first-party handler smoke and +checkpoint-recovery mode, never an automatic fallback. Operation receipts bind +each workflow handle to its actual reconstruction status store, and resumed +attempts preserve scientific/checkpoint identity while using fresh parent and +child Temporal IDs. + +Committed outputs can be listed, previewed with bounded origin/anchor/generator/ +confidence/constraint-decision lineage, and replay-verified from either public +surface. The CLI returns distinct invalid-plan, refusal, runtime, validation, +and success exit codes. + +See +[`docs/reconstruction-public-interfaces.md`](docs/reconstruction-public-interfaces.md) +for exact JSON contracts, commands, Python examples, recovery semantics, and +the exit-code table. --- @@ -1664,6 +3364,10 @@ available. - Waited orchestration `-A` / `-U` repository requests keep the output contract: API calls return the available-data dictionary, and CLI calls render the repository table. - `--build-cache` / `options.build_cache` builds canonical `.data` cache files for cache-capable ASCII datasets, removes transient ZIP/CSV sources after each cache is ready, and does not merge caches into memory. - API calls with `options.api_return_type` return the requested `polars`, `pandas`, or `arrow` object after a completed orchestration job by materializing cache artifacts on disk. +- API calls with `options.output_timezone` append a timezone-aware + `datetime_local` view after cache materialization. Canonical `datetime` and + `timestamp_utc_ms` values remain UTC epoch milliseconds, and no localized + value is persisted. - If orchestration is unavailable, CLI calls exit nonzero with a clear error and API calls raise `OrchestrationUnavailableError`. - `-v` emits high-level orchestration lifecycle logs; `-vv` adds worker, workflow, and activity detail; `-vvv` enables trace-level package logging and @@ -1678,8 +3382,16 @@ defaults: ```python options.orchestration_wait_result = True options.api_return_type = "polars" +options.output_timezone = "America/New_York" # optional IANA output view ``` +The equivalent command/config option is `-z/--timezone IANA_ZONE`; YAML accepts +either `timezone` or `output_timezone`. Unknown timezone names fail before the +orchestration job is submitted. The returned `datetime_local` datatype carries +the selected timezone in Polars, pandas, and Arrow. Daylight-saving transitions +follow that output zone, but HistData source timestamps remain interpreted as +fixed EST without daylight-saving adjustments. + Set `options.orchestration_wait_result = False` to submit a job and receive job metadata instead of a materialized API return object. Set `options.orchestration_start = False` when a caller requires a pre-started @@ -1888,6 +3600,8 @@ artifacts and returns a dataframe or table. - *to use InfluxDB imports or notebook tooling, install the corresponding extras* - `pip install "histdatacom[influx]"` - `pip install "histdatacom[jupyter]"` +- *to fit optional Statsmodels classical-model families* + - `pip install "histdatacom[models]"` - ***All datetime is returned as milliseconds since January 1, 1970 (midnight UTC/GMT)*** @@ -1908,6 +3622,7 @@ options = Options() ```python options.api_return_type = "polars" # "polars", "pandas", or "arrow" +options.output_timezone = "America/New_York" # optional datetime_local column options.formats = {"ascii"} # Must be {"ascii"} options.timeframes = {"tick-data-quotes"} # can be tick-data-quotes or tick-data-quotes options.pairs = {"eurusd"} @@ -2123,6 +3838,8 @@ InfluxDB import and notebook support are optional: ```sh pip install "histdatacom[influx]" pip install "histdatacom[jupyter]" +pip install "histdatacom[models]" +pip install "histdatacom[query]" pip install "histdatacom[all]" ``` @@ -2136,6 +3853,24 @@ to install latest development version pip install git+https://github.com/dmidlo/histdata.com-tools.git ``` +### Container Image + +Version tags publish a non-root Linux AMD64/ARM64 image to GHCR. Keep data, +runtime state, and the verified Temporal cache in one named workspace volume: + +```sh +docker volume create histdatacom-workspace +docker run --rm \ + --mount type=volume,source=histdatacom-workspace,target=/workspace \ + ghcr.io/dmidlo/histdata.com-tools:2.1.0 \ + --version +``` + +The image is a one-shot CLI, not a persistent service. See the maintained +[container guide](docs/container.md) for builds, data operations, fixed +UID/GID ownership, first-run Temporal provisioning, lifecycle constraints, +verification, publication policy, and cleanup. + ### Developer Setup Use a project virtual environment for local development. Do not install @@ -2168,19 +3903,31 @@ hooks; do not rely on user-local Python packages to satisfy `histdatacom`, The dependency surfaces are split by purpose: -- `.[test]` installs pytest, coverage, pandas, pyarrow, InfluxDB support, - notebook execution support, and test-only support around the base Temporal SDK - dependency. +- `.[docs]` installs the pinned Sphinx, MyST, and Read the Docs theme + toolchain. +- `.[test]` installs pytest, coverage, pandas, pyarrow, DuckDB, InfluxDB + support, notebook execution support, and test-only support around the base + Temporal SDK dependency. - `.[lint]` installs pre-commit and direct lint/type/doc hygiene tools. - `.[release]` installs build and publish tooling. - `.[dev]` is the aggregate local contributor environment with test, lint, release, and optional integration dependencies. -The `dev`, `lint`, `test`, and `release` extras pin direct developer tools -where reproducibility matters. Runtime dependencies keep compatibility lower -bounds rather than lock-file pins because `histdatacom` is a published PyPI -library. The active lint baseline is Black, Ruff, mypy, generic file checks, -Pyroma, ShellCheck, Commitizen, and the local CLI/coverage smoke hooks. The +Build the same warning-as-error documentation tree used by CI and Read the +Docs with: + +```sh +python -m pip install -e ".[docs]" +python -m sphinx -W --keep-going -b html docs docs/_build/html +``` + +Open `docs/_build/html/index.html` to inspect the generated site locally. + +The `dev`, `docs`, `lint`, `test`, and `release` extras pin direct developer +tools where reproducibility matters. Runtime dependencies keep compatibility +lower bounds rather than lock-file pins because `histdatacom` is a published +PyPI library. The active lint baseline is Black, Ruff, mypy, generic file checks, +Pyroma, ShellCheck, Commitizen, and the local CLI smoke hook. The previous flake8 plugin stack was intentionally replaced with Ruff so local installs and hook behavior do not drift independently. @@ -2231,10 +3978,15 @@ modules. Future test work should raise `fail_under` when the baseline improves; do not lower it unless a PR explains the production risk and links the follow-up issue. -CI runs pytest through `pytest-cov`, enforces the `.coveragerc` threshold, and -uploads `coverage.xml` plus the `htmlcov/` report for every Python and OS matrix -leg. The first-pass gate is total-only. Per-package or domain thresholds belong -with the broader testing work tracked in issues #9 and #68. +Routine development and the Python/OS CI matrix run the full test suite without +coverage. Coverage runs once, in the dedicated `Production coverage` job, only +when a pull request promotes `dev` into `main`. That required production gate +runs pytest through `pytest-cov`, enforces the `.coveragerc` threshold, and +uploads one `coverage.xml` plus `htmlcov/` artifact. Ordinary commits, pushes, +issue closure, non-production pull requests, workflow dispatches, and pushes to +`main` do not execute coverage. The first-pass gate is total-only. Per-package +or domain thresholds belong with the broader testing work tracked in issues #9 +and #68. The live Temporal runtime smoke is not collected by default pytest because it requires a real Temporal executable and starts local worker processes. Bundled diff --git a/RELEASE.md b/RELEASE.md index 66046165..2972d9c2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -5,9 +5,9 @@ validate both source distributions and wheels before publishing. ## Release Decision Tree -Local publishing is the authoritative release path today. GitHub Actions -publishing is future architecture and remains guarded until the repository is -ready to move deployment out of the local maintainer workflow. +GitHub Actions Trusted Publishing is the authoritative registry-publishing +path. The local workflow remains the build, simple-registry preflight, install +verification, and emergency credential-backed fallback path. 1. Build and validate locally from any reviewed branch with `bash pypi.sh build`. @@ -15,27 +15,32 @@ ready to move deployment out of the local maintainer workflow. This builds the artifacts, serves them through a local simple index, and verifies the installed package exactly like the TestPyPI install harness without requiring upload credentials. -3. TestPyPI dry run: check out `dev`, confirm the tree is clean, then run - `bash pypi.sh testpypi`. Verify install behavior from TestPyPI before any - production release. -4. PyPI production release: merge or fast-forward the reviewed release state to - `main`, confirm the same version passed TestPyPI from `dev`, then run - `bash pypi.sh pypi` from `main`. -5. Emergency branch overrides are explicit: set +3. TestPyPI dry run: manually dispatch `release.yml` from `dev` with + `release_target=testpypi`, approve the protected `testpypi` environment, + and verify the exact installed version from TestPyPI. +4. Production promotion: open the explicit `dev` to `main` pull request so the + required `Production coverage` job runs once, then merge only after it + passes. +5. PyPI production release: manually dispatch `release.yml` from `main` with + `release_target=pypi`, `testpypi_dry_run_confirmed=true`, and the successful + TestPyPI workflow run ID in `testpypi_run_id`. The workflow validates that + run, downloads its + exact `histdatacom-dist` artifact, verifies the TestPyPI file hashes, and + publishes those same files to PyPI without rebuilding them. +6. Emergency local branch overrides are explicit: set `HISTDATACOM_ALLOW_RELEASE_BRANCH_MISMATCH=1` only after reviewing why the normal `dev` -> TestPyPI and `main` -> PyPI branch policy cannot be used. -Tag pushes are intentionally build-only in the release workflow. The dormant -GitHub Actions PyPI publishing path is reachable only from manual -`workflow_dispatch`, the `pypi` environment approval gate, the `main` branch, -and the explicit TestPyPI dry-run confirmation input. +SemVer tag pushes such as `2.1.0` are intentionally build-only in the release +workflow. Registry +publishing is reachable only from manual `workflow_dispatch`, the matching +protected environment approval gate, and the required branch. Production also +requires the explicit TestPyPI confirmation and successful TestPyPI run ID. ## Trusted Publishing Configuration -Trusted Publishing is the target GitHub Actions deployment model, not the -current publishing path. Keep the configuration notes below current so the -future migration is straightforward, but publish with `pypi.sh` until the -Actions release path is deliberately enabled. +Trusted Publishing is the active GitHub Actions deployment model. Keep the +configuration below synchronized with the workflow and protected environments. The GitHub repository has protected `testpypi` and `pypi` environments with required reviewer approval. Configure the matching Trusted Publishers in @@ -63,9 +68,10 @@ Local GPG detached signatures remain part of the local fallback path for now. GitHub release path does not use local GPG keys; it relies on GitHub artifact attestations and PyPI Trusted Publishing attestations. -## Local Publishing +## Local Publishing Fallback -The existing local workflow is the current publishing workflow: +The credential-backed local workflow remains available for an explicitly +reviewed emergency or registry outage: ```sh bash pypi.sh build @@ -310,17 +316,20 @@ Pull request CI uses workflow concurrency to cancel stale runs when a branch is updated. Release workflow concurrency does not cancel in-progress runs, so a manual publish cannot be interrupted by a later tag or dispatch event. -The release workflow builds artifacts on `v*` tags and manual dispatches. It -does not publish automatically on tag push. The dormant publish jobs are branch -guarded to match local policy: TestPyPI is only dispatchable from `dev`, and -PyPI is only dispatchable from `main`. Publishing through GitHub Actions also -requires a matching protected environment approval and OIDC Trusted Publishing -configured on the target index. - -Actions publishing remains a guarded release path, not the authoritative local -publish path. The publish jobs consume `histdatacom-dist`, which is assembled -only from the metadata-only universal wheel and source distribution. Bundled -platform wheels are an explicit private/offline dry-run path: set +The release workflow builds artifacts on SemVer-shaped `*.*.*` tags and manual +dispatches. It +does not publish automatically on tag push. The publish jobs are branch guarded: +TestPyPI is only dispatchable from `dev`, and PyPI is only dispatchable from +`main`. Publishing also requires a matching protected environment approval and +OIDC Trusted Publishing configured on the target index. + +The TestPyPI job consumes `histdatacom-dist`, which is assembled only from the +metadata-only universal wheel and source distribution. The PyPI job requires a +successful TestPyPI release run, validates its workflow, branch, job, tag +ancestry, permitted release-control-only post-tag diff, live artifact, and +public TestPyPI hashes, then downloads and publishes that exact prior artifact. +It does not rebuild production distributions. Bundled platform wheels are an +explicit private/offline dry-run path: set `include_bundled_platform_wheels=true` only with `release_target=build-only`, and set `bundled_platform_wheel_size_confirmed=true` only after confirming the private/offline purpose and artifact-size policy. diff --git a/container/constraints.txt b/container/constraints.txt new file mode 100644 index 00000000..1cadcca2 --- /dev/null +++ b/container/constraints.txt @@ -0,0 +1,25 @@ +# Runtime and build dependency lock for the default container image. +# Keep this compatible with pyproject.toml and update it through reviewed +# dependency changes before rebuilding a release tag. +certifi==2026.6.17 +charset-normalizer==3.4.9 +idna==3.18 +markdown-it-py==4.2.0 +mdurl==0.1.2 +nexus-rpc==1.4.0 +packaging==26.2 +polars==1.42.1 +polars-runtime-32==1.42.1 +protobuf==7.35.1 +Pygments==2.20.0 +pytz==2026.2 +PyYAML==6.0.3 +requests==2.34.2 +rich==15.0.0 +Rx==3.2.0 +setuptools==83.0.0 +temporalio==1.30.0 +types-protobuf==7.34.1.20260518 +typing-extensions==4.16.0 +urllib3==2.7.0 +wheel==0.47.0 diff --git a/docs/broker-capture-contracts.md b/docs/broker-capture-contracts.md new file mode 100644 index 00000000..2c283ddc --- /dev/null +++ b/docs/broker-capture-contracts.md @@ -0,0 +1,215 @@ +# Live broker delivery capture contracts + +The `histdatacom.broker_capture` domain records how a broker feed was delivered +to one collector. It is measurement evidence for a later broker-delivery +fingerprint; it is not historical reconstruction, synthetic output, a claim +about the whole FX market, or the final Parquet product. + +Version one deliberately has no real broker implementation. A real adapter +requires an explicit broker, protocol, SDK, licensing, and configuration +decision. That adapter implements the frozen public message iterator without +changing the capture, storage, replay, or downstream consumer contracts. + +## Contract boundary + +| Contract | Responsibility | +| --- | --- | +| `BrokerAdapterMessageV1` | Credential-free quote, lifecycle, health, source-time, precision, size/activity, batch, and raw-message-hash evidence emitted by an adapter. | +| `BrokerCaptureSessionV1` | Adapter/protocol/collector/clock identity, hashed account and host identity, public environment/server identity, and configuration hash. | +| `BrokerCaptureEventV1` | Collector-assigned contiguous sequence plus adjacent UTC wall-clock and monotonic receive timestamps. | +| `BrokerCaptureStoragePolicyV1` | Partition rotation, session quota, high watermark, immutable retention ceiling, manifest reserve, and fsync behavior. | +| `BrokerCapturePartitionManifestV1` | Completed JSONL artifact hash, size, sequence/time bounds, and event-kind counts. | +| `BrokerCaptureSessionManifestV1` | Atomic catalog of completed partitions and explicit open/completed/failed capture health. | +| `BrokerCaptureReplaySummaryV1` | Reconciled replay counts, sequence bounds, and logical event-content hash. | + +Every durable contract carries a schema version and verifies its derived ID on +read. Contract JSON is canonical, bounded, and contains no event batch inside a +manifest. + +## Adapter and collector boundary + +`BrokerCaptureAdapterV1` exposes only: + +- a public adapter ID; +- an adapter semantic version; +- `iter_messages()`, which yields `BrokerAdapterMessageV1` values in observed + order. + +The adapter may privately own credentials and network state, but the collector +does not inspect adapter attributes, configuration dictionaries, exceptions, +or raw payloads. A blocking WebSocket, SDK, file-descriptor, or bridge-backed +implementation can all satisfy the iterator seam. Async/thread integration is +an adapter concern and does not change downstream contracts. + +`LiveBrokerCaptureSourceV1` samples `BrokerCaptureClockV1` at the collector +boundary for each adapter message and assigns a zero-based contiguous capture +sequence. It rejects adapter-generated clock corrections: only the collector +can assert that its wall and monotonic clocks diverged. + +## Time and ordering evidence + +Each captured event records two different receive clocks: + +- `receive_time_utc_ns` is an epoch timestamp suitable for alignment; +- `receive_time_monotonic_ns` is the ordering/duration clock that cannot be + corrected by NTP or an operator clock change. + +The collector compares the change in wall time with the change in monotonic +time. A difference at or above the configured threshold inserts a first-class +`clock_correction` event with the measured offset change before the triggering +message. UTC values are therefore allowed to move backward; monotonic values +and capture sequence are not. + +An optional source timestamp is separately labeled as `broker_event`, +`exchange_event`, or `adapter_receive` and requires an explicit precision in +nanoseconds. Absence is `unavailable`, not zero or an inferred broker time. + +## Quote precision, sizes, and activity + +Numeric bid/ask values support downstream calculations. Optional `bid_text` +and `ask_text` preserve the exact plain-decimal lexemes needed to measure feed +precision and trailing-zero/rounding behavior. Their provenance is either +`source_lexeme` or `adapter_rendered`; absent text is explicitly +`unavailable`. + +Optional bid/ask sizes must declare `quoted_size` or `broker_specific` meaning. +Optional activity must declare `message_count`, `broker_activity`, or +`liquidity_proxy`. No field is labeled as centralized FX transaction volume. + +`source_batch_id`, source sequence, source message identity, and raw-message +SHA-256 are retained where supplied. Exact duplicate messages therefore share +a message ID while their distinct capture sequences produce distinct capture +event IDs. + +## Lifecycle and capture health + +Lifecycle and health are data, not log-only side effects. Version one includes: + +- process start, stop, and explicit restart; +- connection open and close; +- reconnect; +- subscription add and remove; +- heartbeat; +- known gap; +- outage start and end; +- collector clock correction; +- quote. + +This lets the fingerprint fitter distinguish a quiet market, a known broker +outage, a collector failure, a reconnect discontinuity, and a clock problem. A session manifest is +`open`, `completed`, or `failed`; a failed session retains bounded reason codes +without copying exception messages or private adapter state. + +## Credential and publication safety + +The public contracts have no token, password, credential, cookie, +authorization header, private key, or raw account-ID field. Account and host +identity are optional SHA-256 digests. Adapter configuration is represented +only by a SHA-256 digest computed over an operator-approved public +configuration projection. + +All public metadata is recursively checked for sensitive key names, bearer +tokens, credential-bearing URLs, private-key headers, non-JSON values, +non-finite numbers, and size overflow. Error messages identify only the field +location; they never echo the rejected value. Raw message bodies are not +stored. An adapter may supply a content hash when licensing and source behavior +allow it. + +This is stricter than the repository's path-oriented publication sanitizer: +capture contracts prevent credential-shaped values from entering rows or +manifests in the first place. + +## Append-only storage and crash behavior + +Capture data uses dependency-free canonical UTF-8 JSON Lines. This preserves +raw delivery evidence without coupling the base package to Arrow; #446 +implements the separate final reconstructed Parquet product. + +One session directory has this shape: + +```text +broker-capture-session-/ + session.manifest.json + partition-000000.jsonl + partition-000000.manifest.json + partition-000001.jsonl.partial +``` + +The writer performs each rotation in this order: + +1. append and fsync events to a `.jsonl.partial` file; +2. fsync and rename the data file to its final name; +3. hash it and atomically publish its partition manifest; +4. atomically replace the session manifest so the partition becomes + discoverable. + +A crash before step four leaves partial or orphan evidence, never an advertised +completed partition. `inspect_broker_capture_session()` reports partial data, +unadvertised final data, and orphan sidecars. Discovery reads only atomically +published session manifests. Replay verifies every sidecar plus data size, +SHA-256, UTF-8/line completeness, contract IDs, session, contiguous sequence, +monotonic ordering, counts, and time bounds. + +## Rotation, quota, retention, and backpressure + +Rotation can occur on event count, bytes, or monotonic duration. Before an +append, the writer conservatively accounts for current disk use, the next +canonical line, and manifest reserve. + +- hard session quota raises `BrokerCaptureQuotaError`; +- the high watermark raises `BrokerCaptureBackpressureError`; +- the partition ceiling raises `BrokerCaptureRetentionError`; +- v1 retention never deletes committed evidence to make room; +- a single overlarge row refuses before a partial file is created. + +The synchronous v1 collector therefore fails predictably rather than buffering +without bound. A future production control plane may pause/reconnect an adapter, +but it must honor these limits and cannot silently drop or overwrite captured +events. + +## Live/replay parity and the fingerprint interface + +Both `LiveBrokerCaptureSourceV1` and `BrokerCaptureReplaySourceV1` implement +`BrokerCaptureEventSourceV1`. `consume_broker_capture_source()` sends either +source through the same `BrokerCaptureEventConsumerV1.on_event()` interface. +The persistence sink is called before consumers, so an event that failed to +persist is never presented as captured fingerprint input. + +The synthetic fixture covers exact duplicates, unchanged/stale quotes, a burst, +a quiet gap, outage start/end, disconnect/reconnect, subscriptions, process +lifecycle, heartbeat, source batching, exact price lexemes, and wall-clock +drift. The same consumer sees byte-replayed events equal to those observed +during live fixture collection. + +Before the fitter creates a broker delivery fingerprint, it requires: + +- a supported schema and adapter/collector version; +- a completed session manifest; +- clean partial/orphan inspection; +- verified sidecars and data hashes; +- acceptable clock-correction and capture-health evidence; +- sufficient conditioned support; +- an immutable capture and configuration identity. + +An open or failed capture may be replayed for diagnosis, but it is not silently +eligible for fingerprint fitting. + +The implemented two-pass fitter, condition cells, support/backoff policy, +compact statistics, drift evidence, supersession, and immutable artifact rules +are specified in +[`broker-delivery-fingerprint-contracts.md`](broker-delivery-fingerprint-contracts.md). + +## Issue boundaries + +- #431 supplies the separate reconstructed-event contract. +- #433 governs information safety for reconstruction; broker capture remains + timestamped external evidence with explicit availability. +- #443 implements capture and verified replay only. +- #444 supplies immutable broker-delivery fingerprints and stratified drift + detection from qualified captures. +- #445 applies a selected fingerprint during proposal and delivery rendering. +- #446 implements the final reconstructed Parquet product publisher. + +Changing timestamp meaning, message identity, quote-lexeme semantics, +credential boundary, storage publication order, replay integrity, or consumer +interface requires a new schema or collector version. diff --git a/docs/broker-delivery-fingerprint-contracts.md b/docs/broker-delivery-fingerprint-contracts.md new file mode 100644 index 00000000..8e7d111c --- /dev/null +++ b/docs/broker-delivery-fingerprint-contracts.md @@ -0,0 +1,147 @@ +# Broker delivery fingerprint contracts + +`histdatacom.broker_capture` fits qualified live-capture evidence into a compact, +immutable description of one broker observation/delivery system. The artifact +does not copy augmented tick rows, assert market-wide truth, reconstruct history, +select a model winner, or apply broker style to a synthetic stream. Application +belongs to #445. + +## Streaming and storage boundary + +Fitting makes two bounded streaming passes over capture JSONL: + +1. `assess_broker_capture_eligibility()` verifies manifest state, partial/orphan + inspection, partition sidecars, bytes, hashes, row contracts, sequence, + clock health, minimum event support, adapter policy, and collector version. +2. `fit_broker_delivery_fingerprint()` replays the same immutable content through + bounded aggregators and requires the second logical-content SHA-256 to equal + the health-pass hash. + +No pass materializes a capture or writes augmented capture rows. Distribution +support, sums, squared sums, extrema, and bounded deterministic bottom-hash +samples are retained in memory. Input event, capture, cell, sample, context, and +comparison limits fail closed instead of truncating the evidence. This gives the +downstream streaming reconstruction pipeline a small profile artifact rather +than another tick-sized intermediate dataset. + +Cadence uses session-local monotonic receive time. Separate capture sessions are +never bridged, because monotonic clock origins are process-local. Calendar and +market-event conditioning use UTC receive time. Exact source price lexemes—not +binary-float rendering—supply decimal-place and trailing-zero behavior. + +## Eligibility contract + +`BrokerCaptureEligibilityV1` records a deterministic decision for every capture: + +- `eligible`: clean, complete, verified, supported, and without limitations; +- `limited`: fit is allowed but nonfatal clock or manifest limitations remain; +- `ineligible`: fitting is refused with bounded reason codes. + +Hard failures include incomplete capture, unsupported adapter/collector policy, +fatal manifest limitations, partial/orphan evidence, integrity failure, +insufficient events or quotes, excessive clock corrections, excessive correction +magnitude, and unexplained UTC regression. A verified decision binds the capture +manifest ID, fit-config ID, event/quote counts, clock findings, UTC support, and +logical event-content SHA-256. + +Every fitted profile also retains `BrokerDeliveryCaptureEvidenceV1` per input +session: manifest and eligibility IDs, logical content hash, a digest over the +ordered partition IDs and artifact hashes, partition/event counts, and wall-time +support. The top-level identity binds adapter ID/version/config hash, protocol, +environment, server, hashed account, collector ID/version, and the complete fit +policy. + +## Condition cells and explicit support + +`BrokerDeliveryFingerprintV1.cells` always includes a global cell and may include +these deterministic dimensions: + +| Dimension | Evidence source | +| --- | --- | +| `symbol` | capture quote | +| `session` | canonical calendar classifier | +| `overlap` | canonical session overlaps | +| `special` | rollover/fix/open/close calendar tags | +| `holiday` | versioned calendar holiday tags | +| `event` | calendar event tags and optional versioned market-context timeline | +| `lifecycle` | bounded post-reconnect, post-outage, and post-restart quote windows | + +The fitter emits both context-only and symbol-plus-context cells. Every cell has +an observed quote count and one of three states: + +- `supported`: its own support meets `min_cell_support`; +- `backed_off`: its own evidence remains visible, but `effective_condition_id` + selects the first qualified parent in the declared ordered chain; +- `unsupported`: neither the cell nor a declared parent has enough support. + +A symbol-plus-context cell backs off to symbol, then context, then global. A +single-dimension cell backs off to global. Backoff order is identity-bearing and +must not be sorted or inferred by a downstream consumer. + +## Fitted behavior + +Each metric carries its own observation support, retained sample count, estimate, +uncertainty interval, extrema, configured quantiles, units, and limitations. +Version one fits: + +- event and quote inter-arrival cadence plus per-session intensity; +- bid/ask spread and signed/absolute spread changes; +- burst, quiet, unchanged/stale, transition, and exact-duplicate rates; +- source timestamp precision and exact-lexeme price precision/trailing zeros; +- contiguous source-batch quote counts; +- known gap/outage duration and absolute clock-correction magnitude; +- event rates for quote, reconnect, gap, outage, process restart, and clock + correction; +- selected quote/lifecycle event-transition rates; +- the same quote behavior under the supported condition cells above. + +Means use all observed finite values. Quantiles use the deterministic bounded +sample and declare that limitation when sampled. Rate bounds use Wilson +intervals; distribution means use a normal mean interval. These intervals are +bounded diagnostics, not a claim of independent identically distributed ticks. + +## Drift comparison + +`compare_broker_delivery_fingerprints()` compares the union of explicit +condition/metric rows. It reports `stable`, `sampling_noise`, `material_drift`, +or `unsupported` using minimum support, combined uncertainty, and configured +relative plus metric-specific absolute effect thresholds. + +Material rows are retained first when `max_comparisons` bounds the output. The +artifact records the full candidate count and whether rows were truncated. +Status counts reconcile only the retained rows. There is deliberately no global +similarity score, automatic winner, or collapse of session/event strata into one +number. Cadence, spread, timestamp precision, price precision, stale/burst +behavior, reconnect/outage behavior, and their conditional cells remain +inspectable separately. + +## Versioning, supersession, and persistence + +A profile ID is derived from the complete canonical payload. A successor must +match the predecessor's broker/adapter/collector identity, begin later, and +record `supersedes_fingerprint_id`. It creates a new artifact; the predecessor +is never edited. Existing `SyntheticEventV1.broker_profile_id` lineage therefore +continues to resolve to the original profile even after drift causes a successor +to become effective. + +`write_broker_delivery_fingerprint()` atomically publishes canonical JSON and +returns an `ArtifactRef` with byte size and SHA-256. Rewriting identical bytes is +idempotent. Different content at the same path is refused. Loading reconstructs +all contracts and rechecks every derived identity. + +## Broker-transfer handoff + +`histdatacom.synthetic.broker_transfer` selects only a profile whose effective +interval, capture lineage, eligibility, support state, and broker identity +satisfy its reconstruction run. For a supported cell it uses that cell; for +`backed_off` it follows the recorded `effective_condition_id`; for `unsupported` +it refuses that conditioned claim. + +The selected fingerprint ID must remain on generated-event lineage and the +reconstruction manifest. Proposal conditioning and final delivery rendering may +read the profile, but they may not reinterpret capture clock semantics, invent +support for sparse cells, mutate the profile, overwrite prior manifests, or +treat this delivery fingerprint as a historical price-path generator. See +[`broker-delivery-transfer-contracts.md`](broker-delivery-transfer-contracts.md) +for the implemented selection, rendering, validation, benchmark, and streaming +boundaries. diff --git a/docs/broker-delivery-transfer-contracts.md b/docs/broker-delivery-transfer-contracts.md new file mode 100644 index 00000000..a8687332 --- /dev/null +++ b/docs/broker-delivery-transfer-contracts.md @@ -0,0 +1,128 @@ +# Broker delivery transfer contracts + +`histdatacom.synthetic.broker_transfer` applies an effective-dated broker +delivery fingerprint as a bounded observation-style transform. It does not +generate historical price paths, mutate observed anchors, or persist augmented +tick-sized intermediates. The transfer has two explicit stages: + +1. `condition_broker_proposal()` changes the delivery-related coordinates of a + `ReferenceMotifQueryV1` before motif retrieval and candidate generation. +2. `render_broker_delivery()` applies delivery precision, rounding, timestamp + batching, stale-quote, exact-duplicate, and spread behavior to synthetic rows + only after historical carving and cross-currency reconciliation. + +This order keeps the historical episode as content and the broker profile as +delivery style. A renderer never uses the broker profile to invent directional +price movement. + +## Profile selection and refusal + +`select_broker_profile()` requires the requested condition cell to exist. A +supported cell is used directly. A backed-off cell follows only the fingerprint's +recorded `effective_condition_id`. An absent or unsupported cell refuses; there +is no implicit nearest-neighbor or global fallback for the requested condition. +Global-only delivery measurements can supplement an otherwise supported cell, +and every resolved metric records the condition ID that supplied it. + +Selection also verifies that `selected_at_utc_ns` falls inside the fingerprint's +effective interval. If drift evidence is supplied, the comparison must include +the selected fingerprint. The selection embeds the fingerprint ID, predecessor +ID, effective period, comparison ID, material-drift count, resolved metrics, and +per-metric source cells. That evidence is carried into every render manifest. + +## Proposal conditioning + +Proposal conditioning operates on the motif query rather than rewriting a +generated candidate. The versioned `BrokerTransferConfigV1.strength` is a convex +blend: zero preserves the historical query and one requests the measured broker +target. The cadence target combines active/ordinary quote inter-arrival time +with measured burst, quiet, and outage behavior. Timestamp precision, price +precision, and spread are separate coordinates. Cadence and spread are changed +only when the historical query supplies a compatible coordinate; timestamp and +price precision have explicit one-nanosecond and configured input-decimal +baselines. Session, event, symbol, and reconnect eligibility remains explicit in +the selected fingerprint cell. + +An unsupported proposal returns a `BrokerConditionedProposalV1` with no +conditioned query. The before and after metrics are identical so callers can +audit that no retrieval or generation request was emitted. + +## Delivery rendering + +Rendering accepts a passing `CrossCurrencyReconciledGroupV1` and its exact +`ReconstructionRunV1`, `ReconstructionWindowV1`, and +`HistoricalCarvingConstraintSetV1`. Programmer-level scope mismatches are +rejected at the API boundary. Profile-support, resource, rendering, and +validation failures return an explicit refused result before output is exposed. +The configured `max_events_per_group` is checked before event materialization. + +Observed events are reused byte-for-byte. Synthetic rows may receive: + +- bounded timestamp quantization or batching inside their historical anchor + interval; +- source-like price precision and deterministic rounding; +- bounded spread projection without negative spread; +- deterministic stale-quote or exact-duplicate presentation; and +- the selected broker fingerprint ID on event lineage. + +All stochastic-looking decisions are content-addressed. Repeating the same +inputs, fingerprint, condition selection, and config produces the same output +and identifiers. Transfer strength, maximum timestamp movement, spread +multiplier, batch size, decimal limits, feature switches, and rounding are +versioned in `BrokerTransferConfigV1`. + +The renderer refuses the entire group if any row would escape its anchor +interval, reorder timestamps, alter an observation, violate a historical hard +constraint, acquire a negative spread, lose required quarantine state, or use +untraceable profile lineage. It never publishes partial rows. + +## Final validation and manifests + +An applied group must pass three gates after rendering: + +1. local anchor, ordering, spread, constraint, quarantine, and lineage checks; +2. `validate_cross_currency_output(..., stage=post_broker)`; and +3. the cross-instrument quality path from #331 via + `cross_currency_quality_report()`. + +`BrokerTransferManifestV1` records input/output content hashes, the complete +transfer config, selections and effective periods, action counts, event and +lineage counts, a lineage hash, post-broker validation identity/status, #331 +quality status/hash, and optional paired benchmark comparison IDs. Durable event +rows remain out of the manifest; persistence belongs to the later atomic +partition-publishing stage. + +`BrokerRenderedGroupV1` is therefore a process-local handoff containing streams, +per-event lineage, validation evidence, and compact manifest metadata. A refused +result contains reasons but no streams, lineage, validation report, or quality +payload. + +## Benchmark comparison + +`compare_broker_benchmark_results()` pairs broker-conditioned and unconditioned +candidates on the same reverse-degradation scenarios. It reports bounded metric +deltas and scenario coverage through `BrokerBenchmarkComparisonV1`. The contract +sets `automatic_winner` to false and never emits a winner candidate. Promotion +semantics remain a separate policy decision after the comparison meaning is +proven. + +## Streaming and persistence boundary + +The normal pipeline carries one synchronized group at a time: + +```text +historical anchors + -> broker-conditioned motif query + -> candidate generation + -> historical carving + -> cross-currency reconciliation + -> broker delivery rendering + -> local + post-broker + #331 validation + -> downstream atomic partition writer +``` + +Only the final validated synthetic tick partitions and compact lineage/manifests +need durable storage. The implemented #446 persistence layer consumes this +handoff. Proposal queries, candidate surfaces, reconciled groups, and rendered +groups remain bounded in-flight artifacts unless a diagnostic retention policy +explicitly preserves them. diff --git a/docs/cftc-positioning-contracts.md b/docs/cftc-positioning-contracts.md new file mode 100644 index 00000000..e3282740 --- /dev/null +++ b/docs/cftc-positioning-contracts.md @@ -0,0 +1,256 @@ +# CFTC positioning-state contracts + +The CFTC positioning domain supplies immutable weekly Commitments of Traders +(COT) state to reconstruction. It is deliberately separate from +`MarketContextEventV1`: a Tuesday measurement is persistent futures positioning +published later, not an instantaneous spot-FX event. + +The durable source of truth is a bounded latest-known-state sidecar. COT values +are never repeated onto every tick row, interpolated into invented observations, +or treated as spot volume, sentiment truth, or a causal shock label. + +## Versioned contract boundary + +| Contract | Responsibility | +| --- | --- | +| `CftcPositioningFetchProfileV1` | Date/code selection plus page, source-byte, row, runtime, memory, timeout, and staleness limits. | +| `CftcPositioningRawSourceV1` | Query parameters, deterministic ordering, retrieval time, dataset/family/scope identity, adapter version, response hash, size, URI, redistribution policy, and limitations. | +| `CftcPositioningSymbolMappingV1` | Versioned contract codes, direct/two-leg status, quote direction, support start, official CFTC metadata, and CME citation URIs. | +| `CftcReleaseEvidenceV1` | Date-only report measurement, publication time, knowledge time, confidence, restatement detection, notes, and evidence source. | +| `CftcPositioningSnapshotV1` | One immutable family/scope/contract/report-date position vector and source-row hash. | +| `CftcPositioningCorpusV1` | Sources, mappings, snapshots, coverage, compressed-history consistency, limits, and deterministic identity. | +| `CftcPositioningDiffV1` | Bounded added, removed, and content-changed logical keys between immutable refreshes. | +| `CftcPositioningQueryV1` | Latest eligible state, mapping kind, snapshot IDs, age, refusal status, and point-in-time-derived values for one window. | +| `CftcPositioningConsumerBindingV1` | Companion lineage receipt for benchmark, motif selection, planning, or carving without changing their immutable v1 schemas. | + +All contracts use `histdatacom.cftc-positioning-*.v1` schema identifiers. The +feature is a v2.1.0 minor addition; existing market-context v1 contracts remain +readable and are not extended with incompatible fields. + +## Official sources and reuse + +Production acquisition selects these official CFTC resources: + +- PRE Legacy raw dataset [`srt6-5q2f`](https://publicreporting.cftc.gov/resource/srt6-5q2f.json); +- PRE Traders in Financial Futures (TFF) raw dataset [`udgc-27he`](https://publicreporting.cftc.gov/resource/udgc-27he.json); +- the CFTC [release schedule](https://www.cftc.gov/MarketReports/CommitmentsofTraders/ReleaseSchedule/index.htm); +- [historical special announcements](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalSpecialAnnouncements/index.htm); +- the [historical compressed-file index](https://www.cftc.gov/MarketReports/CommitmentsofTraders/HistoricalCompressed/index.htm); +- CFTC release [9147-25](https://www.cftc.gov/PressRoom/PressReleases/9147-25), which records the 2025 shutdown backlog's actual publication dates; and +- the CFTC [web/reuse policy](https://www.cftc.gov/WebPolicy/index.htm). + +The retained consolidated consistency archives are +`deacot1986_2016.zip`, `deahistfo_1995_2016.zip`, +`fin_fut_txt_2006_2016.zip`, and `fin_com_txt_2006_2016.zip`. They are +current corrected history, not original-vintage proof. + +CFTC states that United States government information on its site is public +domain, while asking users to acknowledge the CFTC as the source. Corpus users +should use an acknowledgement such as “Source: U.S. Commodity Futures Trading +Commission.” Third-party marks and linked content are not covered by that +statement. + +Quote direction is recorded in immutable mapping contracts using the official +CFTC dataset metadata together with CME's +[FX quote-convention guide](https://www.cmegroup.com/education/courses/introduction-to-fx/understanding-fx-quote-conventions) +and [EUR/GBP Rule 301](https://www.cmegroup.com/rulebook/CME/III/300/301/301.pdf). +The mapping artifact retains those citation URIs, direction, codes, and notes. +The live corpus does not mirror or redistribute CME pages, and reconstruction +does not depend on CME web availability after the versioned mapping is loaded. + +## Report families, scopes, and fields + +Legacy and TFF are separate classification schemas. A TFF participant category +must not be interpreted as a Legacy category, back-cast before TFF support, or +pooled with Legacy. Futures-only and futures-plus-options-combined reports are +also separate scopes; combining both would double-count overlapping state. + +The logical duplicate key is exactly: + +```text +(report_family, report_scope, cftc_contract_market_code, report_date) +``` + +PRE `id` is retained as evidence but is not trusted as a cross-dataset primary +key. Identical duplicate logical rows are counted; contradictory rows fail the +build. + +Common source fields include `report_date_as_yyyy_mm_dd`, +`cftc_contract_market_code`, `futonly_or_combined`, +`contract_market_name`, `market_and_exchange_names`, and +`open_interest_all`. Numeric position, percentage-of-open-interest, change, +trader-count, and concentration fields are retained under their normalized PRE +names. Legacy examples include `noncomm_positions_long_all` and +`noncomm_positions_short_all`; TFF examples include +`lev_money_positions_long_all` and `lev_money_positions_short_all`. + +## Symbol and direction mapping + +| Window state | CFTC code(s) | Mapping | Direction / interpretation | +| --- | --- | --- | --- | +| EURUSD | `099741` | direct | USD per EUR; EUR FX futures positioning. | +| GBPUSD | `096742` | direct | USD per GBP; British Pound futures positioning. | +| EURGBP before 2014-06-10 | `099741` + `096742` | two-leg | Retain both EURUSD and GBPUSD leg snapshot IDs; do not label or pool them as direct EURGBP COT. | +| EURGBP from 2014-06-10 | `299741` | direct | GBP per EUR; direct EUR/GBP contract state. | + +The pre-2014 two-leg record is identity and conditioning evidence only. It does +not subtract heterogeneous participant totals into a fictitious direct +contract. + +## Measurement, publication, knowledge, and restatement time + +`report_date` is the CFTC measurement date and remains date-only. +`measurement_start_ns` is a deterministic midnight-UTC boundary used only for +ordering and age calculations; it is not claimed as an observed market +timestamp. `publication_at_ns` records a publication time when evidence +supports one, `knowledge_at_ns` records when that publication could be used, +and `valid_from_ns` is derived from that knowledge time. A separate +`restatement_detected_at_ns` preserves correction discovery. + +Availability confidence is one of `verified`, `nominal`, `unknown`, +`correction_qualified`, or `restatement_qualified`. The ordinary Friday 15:30 +America/New_York rule is only nominal, including DST conversion, and is never +strict-ex-ante eligible. The 2025 backlog dates are verified from CFTC release +9147-25. Special-announcement fixtures cover July 2015 holiday/premature +publication cases and the documented 2010 and 2018 corrections. + +PRE and compressed history expose current corrected state. They do not retain +every original published value. A row therefore remains `current_state_only` +or `restated_current_state` unless an original vintage is separately verified. +A known release time cannot make a current corrected value strict-ex-ante +eligible; the query returns `restatement_incomplete`. + +## Acquisition, immutable artifacts, and crash recovery + +```bash +histdatacom analytics cftc-positioning-corpus \ + --artifact-dir data/.histdatacom/analytics/cftc-positioning \ + --start-date 2002-03-01 \ + --end-date 2026-06-30 +``` + +PRE requests record the exact `$where`, `$order`, `$limit`, and `$offset` +parameters. Ordering is report date, contract code, report scope, and PRE ID. +Pagination is contiguous and bounded. Every response records retrieval time, +resolved URI, dataset/family identity, adapter version +`cftc-pre-positioning-adapter-v1`, content type, bytes, SHA-256, reuse policy, +and limitations. + +The writer creates content-addressed raw sources plus corpus, coverage, and +archive-consistency JSON. Writes use a flushed temporary file followed by an +atomic rename, so a crash cannot leave a partial final-named artifact. +`read_cftc_positioning_corpus()` verifies filename and content hashes. +`replay_cftc_positioning_corpus()` rebuilds from retained responses and refuses +an identity change. + +A refresh may pass `--previous-corpus`. It never overwrites the prior corpus; +it writes a `CftcPositioningDiffV1` with bounded added, removed, and changed +logical keys plus both snapshot IDs. Retrieval-time changes alone do not become +false row restatements because row comparison uses canonical source-row hashes. + +Default limits are 100,000 selected rows, 128 PRE pages per dataset, 128 MiB +per response, 256 MiB total configured source bytes, 600 seconds, and 2 GiB +peak resident memory. ZIP consistency checks validate expansion size and stream +CSV rows rather than decoding whole archive members. + +## Coverage and consistency evidence + +Coverage is partitioned by year, family, scope, and contract. Each slice +records rows, first/last report date, missing weekly intervals, duplicate keys, +contract and market names, availability-confidence counts, restatement counts, +source hashes, source bytes, and processing time. It does not infer that a +missing week means “neutral positioning.” + +Compressed-history evidence reports selected rows, PRE matches, PRE +missingness, open-interest mismatches, and contract/name changes for every +family/scope archive. Equality proves current-state consistency only; it does +not reconstruct the original publication vintage. + +## Query, derived values, and refusal semantics + +`query_cftc_positioning_corpus()` selects the latest eligible report at or +before the reconstruction window. It returns only bounded snapshots and keeps +family/scope/code identities. It reports per-snapshot age and one of `ready`, +`missing`, `stale`, `not_available_as_of`, `unsupported`, or +`restatement_incomplete`. + +Derived values include participant net, net/open-interest, change from the +prior eligible family/scope/contract snapshot, and a trailing 52-report +standard score. History is cut off at the selected report, and no value pools +families, scopes, or the two EURGBP legs. Nominal or unknown publication +history is excluded from strict ex-ante selection. + +`preflight_cftc_positioning_corpus()` returns structured refusal evidence. +`require_cftc_positioning_corpus()` raises +`CftcPositioningPreflightError`; it never substitutes a neutral state. + +## Information audit and consumers + +`cftc_positioning_information_inputs()` emits one +`ReconstructionInformationInputV1` per selected snapshot. Verified original +vintages use point-in-time scope. Current corrected state is revision-scoped; +when it is learned after the reconstruction time in an ex-post run, it is +explicitly labeled `full_period_summary` with bounded allowed lookahead. The +existing #433 leakage audit therefore sees the same temporal nonclaim as the +query. + +`CftcPositioningConsumerBindingV1` retains corpus, query, snapshot, information +input, run, window, and consumer artifact IDs. Installed helpers: + +- project a compact state label into benchmark `event_state`; +- add the state label to motif event tags while keeping positioning metrics in + the companion receipt because motif v1 has a strict metric allowlist; +- validate run/window/artifact continuity for planning and carving; and +- write binding and held-out benchmark-smoke artifacts. + +No custom notebook or permanent tick-row augmentation is required. + +## Real closure campaign + +The #468 campaign covered 2002-03-01 through 2026-06-30. It retained 24 +official responses totaling 92,920,837 bytes and produced 11,324 distinct +snapshots in 232 year/family/scope/contract coverage slices with zero duplicate +logical keys. The final refresh ran in 87.581906 seconds and recorded a +414,351,360-byte peak resident-memory measurement after the archive parser was +changed from whole-member decoding to streaming. + +Coverage includes 1,270 Legacy reports per scope for EUR `099741` and GBP +`096742` from 2002-03-05, 1,047 TFF reports per scope from its supported +2006-06-13 start, and 514 direct EURGBP `299741` reports per family/scope from +2014-06-10. The four consolidated archives supplied 4,060 overlapping rows; +all 4,060 matched PRE and none had an open-interest mismatch. + +The final logical corpus ID is +`cftc-positioning-corpus-277495b704e33d13a1340b19e3b0f7f80dbeef7970281c77b626a75945a96414`; +its self-contained artifact SHA-256 is +`887a47840090cdab1982fe910a4bdf8c1fcc9af256ab687bceae1b8dd1cbd3e0`. +An independent raw-source replay of the preceding same-schema refresh +reproduced its exact 11,324-snapshot corpus ID. The next immutable live refresh +reported zero added, zero removed, and zero changed logical rows while +retaining the distinct retrieval-evidence corpus IDs in diff +`cftc-positioning-diff-ee030e5f2b73bd2e56ab5267a12eefc3c5098bcbaadcc3d46b3b0d8bd89fbccb`. + +The real held-out smoke consumed the first 4,096 EURUSD events from the local +December 2024 HistData Arrow cache (source SHA-256 +`68f9938c2e302ffada2add14393f8ba072da07c5addceda0f1e600d076d7a954`). +Its window selected the four 2024-11-26 Legacy/TFF and +futures-only/combined snapshots. Artifact reload produced the identical output +SHA-256 +`82fd59e316354e33468ef1a66a929601084d74b2eec569777af62b589f080f5b`. +The smoke ID is +`cftc-positioning-smoke-7ee89ee3b63ac9883f064dbf4de4c8d38ec4183022ee5ec32eaa256f1b49d8c0`. + +The real corpus separately returns `restatement_incomplete` for strict ex-ante +use of current PRE state even when a delayed publication date is verified, and +preflight returns an explicit unsupported refusal for a non-triangle symbol. + +## Explicit limitations and nonclaims + +- COT is futures positioning/open interest, not decentralized spot-FX volume. +- Participant categories are not individual-trader identities or sentiment + truth. +- Position changes are not causal shock labels. +- Weekly values are not interpolated into invented observations. +- Absence, stale state, or unsupported history is not a neutral position. +- No automatic positioning winner, strategy, or trading recommendation is + produced. diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..5f80d411 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,34 @@ +"""Sphinx configuration for the HistData.com Tools documentation.""" + +from importlib import metadata + +project = "HistData.com Tools" +author = "David Midlo" +copyright = "2026, David Midlo" + +release = metadata.version("histdatacom") +version = ".".join(release.split(".")[:2]) + +extensions = ["myst_parser"] +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} +root_doc = "index" +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +myst_heading_anchors = 4 +myst_enable_extensions = ["colon_fence", "deflist", "fieldlist"] + +html_theme = "sphinx_rtd_theme" +html_theme_options = { + "collapse_navigation": False, + "navigation_depth": 4, +} +html_context = { + "display_github": True, + "github_user": "dmidlo", + "github_repo": "histdata.com-tools", + "github_version": "dev", + "conf_py_path": "/docs/", +} diff --git a/docs/container.md b/docs/container.md new file mode 100644 index 00000000..5103d8e3 --- /dev/null +++ b/docs/container.md @@ -0,0 +1,126 @@ +# Container image + +The application image packages `histdatacom` as a one-shot command-line tool. +It is not an InfluxDB image, a separately managed Temporal service, or a +long-lived API server. Each `docker run` invocation performs one CLI operation +and then exits. + +## Pull or build + +Version tags publish matching Linux AMD64 and ARM64 images to GitHub Container +Registry: + +```sh +docker pull ghcr.io/dmidlo/histdata.com-tools:2.1.0 +``` + +Build the current checkout for the native platform with the strict repository +build context: + +```sh +docker build --tag histdatacom:local . +``` + +The `.dockerignore` file is an allow-list. Only `pyproject.toml`, package source, +the reviewed container dependency constraints, the license, and the package +README enter the build context. Repository data, Git history, virtual +environments, generated reports, caches, and local credentials are excluded by +construction. The base image uses a multi-platform digest and +`container/constraints.txt` fixes the default image's Python build and runtime +dependency graph for reviewed release-tag rebuilds. + +## Persistent workspace + +Use one named volume at `/workspace` for downloaded data, cache files, runtime +state, logs, manifests, SQLite state, and the checksum-verified Temporal binary +cache: + +```sh +docker volume create histdatacom-workspace + +docker run --rm \ + --mount type=volume,source=histdatacom-workspace,target=/workspace \ + ghcr.io/dmidlo/histdata.com-tools:2.1.0 \ + --version + +docker run --rm \ + --mount type=volume,source=histdatacom-workspace,target=/workspace \ + ghcr.io/dmidlo/histdata.com-tools:2.1.0 \ + -C -p eurusd -f ascii -t tick-data-quotes -s 2024-01 +``` + +The image intentionally has no Dockerfile `VOLUME` declaration, so Docker does +not silently create anonymous storage. A named volume is recommended because a +new volume copies the image workspace ownership and initial directories. For a +host bind mount, make the host directory writable by container UID and GID +`10001:10001` before running the image. Do not solve a permission mismatch by +running the market-data process as root. + +The default relative data directory is `/workspace/data`. Container environment +defaults also keep orchestration state in `/workspace/runtime` and the Temporal +cache in `/workspace/cache/temporal-cli`. Explicit CLI options and supported +environment overrides remain available when a different layout is required. + +## Temporal lifecycle and networking + +The normal image does not preload a platform binary. The first operation that +starts orchestration downloads the pinned Temporal CLI artifact over HTTPS, +verifies its archive checksum, records provenance, and saves it in the mounted +cache. Later containers using the same workspace volume reuse that verified +entry. Air-gapped use therefore requires a previously provisioned cache or the +project's private/offline bundled-wheel path. + +Temporal server and worker processes are children of the one-shot operation. +They are started and stopped as part of a waited CLI job. `--keep-runtime` is +not useful across `docker run` boundaries because the container process exits; +durable job/runtime state remains in the workspace, but processes do not. The +image uses `tini` for signal forwarding and zombie reaping so `docker stop` +delivers a clean termination path. + +No runtime ports are published by default. A normal waited job talks to its +container-local runtime. Interactive runtime diagnostics can be executed in a +single container, but treating that container as a production service is +outside the image contract. + +The default image installs the base Python package dependency set. Optional +Python extras such as `models`, `influx`, `query`, and `jupyter` are not silently +added to this lean image. A purpose-specific image must extend the Dockerfile +and lock the selected extra dependencies explicitly. + +## Verification + +The local smoke builds the native image, inspects its runtime configuration and +OCI labels, verifies UID/GID and workspace writes, exercises CLI version/help, +and removes its temporary image: + +```sh +python scripts/smoke_container.py +``` + +The connected smoke additionally provisions the real pinned Temporal binary, +runs `runtime start`, `runtime doctor`, and `runtime stop` within one container, +and proves the verified cache survives into a second container through a named +volume. It removes the temporary image and volume by default: + +```sh +python scripts/smoke_container.py --deep-runtime +``` + +Use `--keep-image` or `--keep-workspace-volume` only when debugging retained +state. The smoke report names retained resources. + +## Publication and cleanup + +Container validation is isolated from ordinary `dev` pushes. The Container +workflow runs for pull requests that change the container surface, for manual +dispatch, and for `v*` tags. Pull-request and manual runs build without +publishing. A version tag publishes AMD64 and ARM64 manifests to GHCR with +SemVer and commit tags, OCI metadata, provenance, and an SBOM. The workflow does +not run coverage; coverage remains a `dev`-to-`main` promotion gate. + +Remove local state explicitly when it is no longer required: + +```sh +docker volume rm histdatacom-workspace +docker image rm ghcr.io/dmidlo/histdata.com-tools:2.1.0 +``` diff --git a/docs/cross-currency-reconciliation-contracts.md b/docs/cross-currency-reconciliation-contracts.md new file mode 100644 index 00000000..bc731816 --- /dev/null +++ b/docs/cross-currency-reconciliation-contracts.md @@ -0,0 +1,171 @@ +# Cross-Currency Reconciliation Contracts + +Cross-currency reconciliation turns individually carved event streams into one +synchronized reconstruction unit. It is a generation-time invariant, not a +repair performed after independent symbol products have already been accepted. + +The first certified relationship is: + +```text +EURUSD / GBPUSD ~= EURGBP +``` + +The contracts are generic enough to represent inverse pairs as well, but this +issue does not add every available currency relationship. + +## Common-coverage planning + +`plan_cross_currency_windows()` consumes explicit half-open coverage for every +symbol in a `ReconstructionRunV1`. It intersects that coverage with the +requested interval and delegates only the common span to the existing +worker-count-independent window planner. + +The returned `CrossCurrencyWindowPlanV1` records: + +- available or explicitly missing status for every symbol; +- source periods used to establish each coverage range; +- per-symbol leading and trailing exclusions; +- group-level spans without common support; +- the limiting common start and end; and +- the complete all-symbol `ReconstructionWindowV1` sequence. + +A missing leg or empty intersection produces a deterministic refused plan with +no windows. Unequal raw histories are therefore visible evidence and cannot be +silently shortened or filled. + +## Relationship and projection policy + +`CrossCurrencyRelationshipV1` supports two algebraic forms: + +```text +triangle: numerator / denominator ~= direct +inverse: left * right ~= 1 +``` + +Each relationship declares a deterministic projection priority. The EURUSD +triangle projects EURGBP first, then EURUSD, then GBPUSD. Only synthetic events +are eligible. If every event at a supported point is observed, the quotes stay +unchanged even when the relationship is infeasible. + +Projection changes the complete executable bid/ask envelope, not just the +midpoint. Triangle bid uses `numerator.bid / denominator.ask`; triangle ask uses +`numerator.ask / denominator.bid`. Inverse bid/ask use the opposite side of the +reciprocal quote. All generator, motif, anchor, feed-epoch, constraint, and +confidence lineage remains intact. `CrossCurrencyProjectionLineageV1` +content-binds the input and output quotes because `SyntheticEventV1.event_id` +intentionally does not include bid, ask, or confidence. A projection may +therefore retain the semantic event ID while still providing cryptographic +proof of the value change and its spread change. + +The hard projection limit, residual tolerance, combined-spread multiplier, and +rounding precision are versioned in +`CrossCurrencyReconciliationConfigV1`. A required midpoint move beyond the hard +limit refuses the group. It is never clipped to a more convenient value. + +## Exact event-time support + +Reconciliation does not forward-fill, backward-fill, or interpolate another +instrument merely to manufacture simultaneous support. + +Events are compared only when every leg has the exact same nanosecond event +time. Multiple events at one timestamp are paired by `(event_sequence, +event_id)` ordinal. Excess duplicates and asynchronous timestamps remain in +their original symbol streams and are counted in topology evidence. + +The validation report includes union/common/asynchronous timestamp counts, +duplicate-event counts, and stale-forward-fill risk runs. The risk metric +describes what an unsafe downstream join could do; the reconciliation engine +does not perform that join. + +## Anchor preservation and refusal + +Every observed event in the input streams is hashed before reconciliation and +rechecked afterward. Observed quotes, timestamps, sequences, identity, and +source lineage must match byte-for-byte at the contract level. + +The group refuses when: + +- a required symbol stream is missing; +- a relationship has no exact event-time support; +- an observed-only residual exceeds its spread-aware tolerance; +- the required synthetic projection exceeds its configured hard limit; +- a projected quote would be invalid; or +- post-projection validation or anchor preservation fails. + +Refused results retain the available unchanged streams and bounded failure +reasons. They cannot be presented as partially successful triangle products. + +## Conditioned residual evidence + +`CrossCurrencyConditionV1` maps half-open event-time intervals to session, +event, and feed-epoch keys. Every supported relationship point contributes to +all three dimensions. If no external condition exists, the event's own +feed-epoch IDs are used and other dimensions are marked `unclassified`. + +`CrossCurrencyResidualSliceV1` reports support, projections, infeasibility, and +pre/post maximum residuals for each relationship/dimension/key. This keeps +coherence evidence stratified instead of hiding weak regimes behind one global +average. + +## Generation and final validation gates + +`reconcile_cross_currency_window()` always emits a generation-stage +`CrossCurrencyValidationReportV1`. A passing +`CrossCurrencyReconciledGroupV1` is eligible to proceed to ensemble and broker +conditioning, but it is not sufficient for publication. + +`validate_cross_currency_output()` uses the same exact-time, spread-aware, +anchor-preserving rules at either of two explicit stages: + +```text +generation +post_broker +``` + +The post-broker stage is mandatory even if the broker transformation claims not +to affect prices. It binds the complete final stream content hash and detects a +quote change that would otherwise retain the same semantic event and stream +IDs. + +`validate_cross_currency_atomic_manifest()` requires: + +- a passing `post_broker` validation; +- identical run, window, synchronization-unit, member, and symbol scope; +- one stream and manifest count for every symbol; and +- an exact match between final stream content and validation content. + +The existing `PartitionManifestV1` remains the smallest publication unit. A +single leg or stale validation report cannot authorize a commit. Physical +Parquet staging and atomic promotion are implemented downstream by #446. + +## Existing diagnostic compatibility + +`cross_currency_quality_report()` adapts the reconciled group directly to the +existing #331 `HistDataCrossInstrumentConsistencyRule`; no permanent cache +roundtrip is required. The compatibility adapter buckets nanosecond times to +the existing HistData millisecond diagnostic grain, retains collisions as +duplicates, and never forward-fills. + +The native reconciliation report remains authoritative at nanosecond event +time. The #331 report supplies the established triangle, inverse, grid, and +stale-join audit surface over the same reconstructed group. + +## Streaming and issue boundaries + +The group object is process-local. `metadata()` hashes output and projection +content and does not inline event or lineage rows into workflow history. +Window workers may write larger streams behind artifact references, then pass +only the bounded validation and group metadata forward. + +- #442's implemented calibration layer consumes passing synchronized groups, + retains bounded representative members, and hash-gates regeneration; see + [`reconstruction-ensemble-calibration-contracts.md`](reconstruction-ensemble-calibration-contracts.md). +- #445 applies broker delivery conditioning and must rerun `post_broker` + validation. +- #446 writes final Parquet and commits only through the atomic manifest gate. +- #447 maps these artifact boundaries onto retryable Temporal activities; see + [`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + +Changing relationship meaning, projection priority semantics, event-time join +policy, hard-limit interpretation, validation stages, or atomic gate behavior +requires a new schema version. diff --git a/docs/data-quality/report-compatibility.md b/docs/data-quality/report-compatibility.md index 07182df2..cdedd44f 100644 --- a/docs/data-quality/report-compatibility.md +++ b/docs/data-quality/report-compatibility.md @@ -28,6 +28,220 @@ fields, finding fields, bounded runtime payload keys, or artifact metadata fields. A schema-version change is also required when severity or status values change meaning. +## Intentional Rule Skips + +Reports with intentionally skipped target-rule evaluations include optional +`metadata.quality_engine` metadata using +`histdatacom.quality-engine.v1`. Its planned, executed, and skipped target-rule +counts satisfy: + +```text +planned_target_rule_evaluation_count + = target_rule_evaluation_count + skipped_rule_evaluation_count +``` + +`quality_engine.skip_events` uses +`histdatacom.quality-skip-events.v1`. Each event contains a stable reason code, +rule ID, target kind, and publish-safe data format, timeframe, symbol, and +period axis. Events never contain absolute paths or row samples. The event list +is deterministically limited to 128 entries; reason, rule, and target-kind +count maps are limited to 64 entries and expose complete truncation accounting +under `limit_metadata`. + +The legacy +`quality_engine.skipped_duplicate_archive_rule_evaluation_count` remains +available. Full report consumers can use the structured events from report +metadata, and bounded runtime consumers receive the same contract at the +top-level `quality_engine` key. Reports without intentional rule skips do not +add `quality_engine`, preserving the prior optional-metadata behavior. + +## Fingerprint Topology Inspection Context + +Fingerprint topology-attention target summaries may include optional +`inspection_context` using +`histdatacom.timestamp-topology-inspection.v1`. Each evidence section contains +`total_count`, `included_count`, `omitted_count`, `truncated`, a bounded +`samples` list, and complete sample `limit_metadata`. Actionable sections link +to the same stable remediation identity used by `quality_next_actions` through +`code`, `action_kind`, `rule_id`, `flag`, and the copied target axis. Expected +session closures are +marked non-actionable and only provide context for suspicious gaps. + +This is a compatible optional v1 metadata addition. The default payload remains +publish-safe: timestamp and row-position evidence is allowed, while absolute +paths, credentials, complete quote rows, and raw row excerpts are not. + +## Cross-Series Fingerprint + +Fingerprint runs may include optional `metadata.cross_series_fingerprint` +using `histdatacom.cross-series-fingerprint.v1`. The bounded runtime equivalent +is `fingerprint_cross_series`. Group and pair lists are deterministic and expose +complete limit/truncation metadata. Group rows contain publish-safe target axes, +symbol membership, timestamp-grid and coverage-range summaries, topology and +cache provenance counts, and bounded return-correlation results. Consistency +samples may include `series_id`, `period`, `row_id`, `source_row_number`, and +`event_seq`, but never absolute paths or raw quote rows. + +This is a compatible optional v1 report addition. Consumers that do not use +run-scoped fingerprints can ignore both keys. + +## Exponential-Smoothing Fingerprint + +Fingerprint findings may include the opt-in +`time_series_fingerprint.exponential_smoothing` section using +`histdatacom.exponential-smoothing.v1`. Full reports summarize it under +`metadata.time_series_fingerprint_exponential_smoothing_summary`; the bounded +runtime equivalent is `fingerprint_exponential_smoothing`, and the text summary +heading is `Exponential-smoothing models`. + +Configuration, fit, forecast, evaluation, training-projection, and run-summary +objects carry their own `histdatacom.exponential-smoothing-*.v1` schema +versions. Model specifications and target summaries are deterministic and +bounded. Scalar fitted parameters may be present in bounded fit samples, but +fitted backend objects, residual vectors, exception text, and measured +wall-clock durations are intentionally absent. Consumers must treat model +status, convergence, errors, and baseline comparisons as advisory; no field +selects an automatic winner or changes the data-quality exit decision. + +The same annotation engine projects a configured specification and horizon as +nullable `cm_ets_*` scalar columns on enriched tick rows. Forecast values are +point-in-time available at the origin-bin close. Realized actual/error values +are post-observation diagnostics with separate availability and eligibility +flags. Durable identity remains `series_id`, `period`, and `row_id`; timestamp +is never the sole join key. + +This is a compatible optional v1 report and enriched-column addition. Core +installs and profiles that do not enable the family omit these report keys and +do not import the optional numerical backend. + +## Autoregressive-Family Fingerprint + +Fingerprint findings may also include the opt-in +`time_series_fingerprint.autoregressive` section using +`histdatacom.autoregressive.v1`. Full reports summarize it under +`metadata.time_series_fingerprint_autoregressive_summary`; the bounded runtime +equivalent is `fingerprint_autoregressive`, and the text heading is +`Autoregressive models`. + +Configuration, fit, forecast, evaluation, training-projection, and summary +objects use stable `histdatacom.autoregressive-*.v1` schemas. AR, ARMA, and +ARIMA remain explicit families with explicit orders. Automatic order search and +winner selection are absent. Backend failures, convergence, roots, +conditioning, parameters, fold errors, baseline references, resource limits, +and Statsmodels version are bounded advisory metadata. + +Configured projections add nullable scalars under `cm_ar_*`, `cm_arma_*`, and +`cm_arima_*` on enriched tick rows. Forecast fields become available at their +origin; realized errors are separately marked post-observation diagnostics. +Durable identity remains `series_id`, `period`, and `row_id`, and consumers do +not need a side-table join. Older consumers may ignore the optional report keys +and the additive enriched columns. + +## Seasonal/Exogenous-Family Fingerprint + +Fingerprint findings may include the opt-in +`time_series_fingerprint.seasonal_exogenous` section using +`histdatacom.seasonal-exogenous.v1`. Full reports summarize it under +`metadata.time_series_fingerprint_seasonal_exogenous_summary`; the bounded +runtime equivalent is `fingerprint_seasonal_exogenous`, and the text heading is +`Seasonal and exogenous models`. + +Configuration, regressor, fit, forecast, evaluation, training-projection, and +summary objects use stable `histdatacom.seasonal-exogenous-*.v1` schemas. +SARIMA, ARIMAX, and SARIMAX have explicit nonseasonal/seasonal orders and an +explicit seasonal cycle tied to the model-input sampling frequency. The flat +regressor contract records deterministic column order, vocabulary, +known-in-advance availability, missingness, and calendar-profile provenance. +Observed future market values, automatic order/regressor search, and automatic +winner selection are absent. + +Configured projections add 123 nullable scalar columns under `cm_sarima_*`, +`cm_arimax_*`, and `cm_sarimax_*` on enriched tick rows. Forecast fields become +available at their origin; realized errors remain separately flagged +post-observation diagnostics. Durable identity remains `series_id`, `period`, +and `row_id`. Consumers may ignore the optional additive report keys and +columns. + +## Volatility-Family Fingerprint + +Fingerprint findings may include the opt-in +`time_series_fingerprint.volatility` section using +`histdatacom.volatility.v1`. Full reports summarize it under +`metadata.time_series_fingerprint_volatility_summary`; the bounded runtime +equivalent is `fingerprint_volatility`, and the text heading is +`ARCH and GARCH volatility models`. + +Configuration, fit, forecast, evaluation, training-projection, and summary +objects use stable `histdatacom.volatility-*.v1` schemas. Symmetric ARCH and +GARCH orders, return input, mean model or preceding residual reference, +innovation distribution, scale, variance initialization, covariance type, +parameter bounds, and resource limits are explicit. Conditional-mean, +conditional-variance, and volatility metrics are separate; the realized +variance proxy is named, and comparison remains descriptive with no automatic +winner. Asymmetric models are registry-only in this contract. + +Configured projections add 78 nullable scalar columns under `cm_arch_*` and +`cm_garch_*` on enriched tick rows. Forecast fields become available at their +origin; realized-return and variance diagnostics remain separately flagged +post-observation fields. Durable identity remains `series_id`, `period`, and +`row_id`. Consumers may ignore these optional additive report keys and columns. + +## Classical-Model Comparison + +Fingerprint findings may include the opt-in +`time_series_fingerprint.classical_model_comparison` section using +`histdatacom.classical-model-comparison.v1`. It is generated only from saved +bounded evaluation artifacts and cannot trigger fits. Full reports summarize it +under `metadata.time_series_fingerprint_classical_model_comparison_summary`; +the bounded runtime key is `fingerprint_classical_model_comparison`, and the +text heading is `Classical model comparison`. + +Compatibility is explicit across dataset/fingerprint, regularization contract, +fold set, target metric, scale, transform, frequency, missingness, horizon, and +period. Incompatible or incomplete evidence stays visible but ineligible. Skill +uses the configured mean or variance reference without silently substituting a +baseline; negative skill, missing references, near-zero reference errors, and +incomplete folds have stable reason codes. Conditional-mean, +conditional-variance, and volatility metrics are never pooled. + +Fit accounting preserves attempted, fitted, converged, limited, skipped, +timed-out, numerically invalid, dependency-unavailable, failed, and +resource-limited counts. Stability is advisory and distinguishes insufficient +folds, stable behavior, structural parameter shifts, isolated failures, and +persistent error degradation. Fingerprint regime/stationarity/decomposition +signals are context only, not causal claims. Diagnostics are bounded and omit +fitted objects, raw rows, residual vectors, paths, and backend exception text. + +The augmented row contract adds 43 nullable scalar columns under +`cm_comparison_*`, `cm_skill_*`, and `cm_stability_*`. They are retrospective, +target-time gated, explicitly not training-eligible, and joined only by durable +`series_id`, `period`, and `row_id`. No winner, best-model field, production +recommendation, or normative automatic selection is part of this schema. + +## Synthetic Tick Generation + +`histdatacom.synthetic-tick-generation.v1` is a bounded diagnostic artifact, +not a new quality-report top level. Its configuration and automatic candidate +validation use `histdatacom.synthetic-tick-generation-configuration.v1` and +`histdatacom.synthetic-tick-generation-validation.v1`. The generated candidate +report remains an ordinary `histdatacom.quality-report.v1` containing the same +`fingerprint.series` payload and `histdatacom.synthetic-fingerprint-validation.v1` +comparison semantics used for external candidates. + +The row schema is additive: the seven existing nullable `synth_*` columns are +populated on the enriched ASCII tick row, while observed bid/ask, duplicate +timestamps, and `series_id`/`period`/`row_id` identity are preserved. Consumers +that do not use synthetic values may continue to ignore those columns. The +generator never adds a separate table, changes timestamp identity, or restores +M1 as an independent base grain. + +Generation diagnostics are deterministic for the same reference fingerprint, +configuration, and reference rows. Bounded evidence includes only counts, +stable IDs, configuration, and a limited transition-index sample; it excludes +raw quote rows, fitted objects, absolute report paths, and exception text. +Statistical fingerprint mismatch remains advisory and cannot change the +candidate quality status. + ## Golden Fixtures Representative payload fixtures live under @@ -41,12 +255,67 @@ The golden suite covers: - corrupt ZIP detailed report; - coverage-manifest failure detailed report; - canonical cache target detailed report; +- deterministic synthetic tick generation diagnostics and candidate-validation evidence; +- duplicate ZIP/CSV quality-engine skip detailed report; +- bounded runtime payload with structured quality-engine skips; +- fingerprint detailed and bounded reports with action-linked topology + inspection context; - run-scoped finding detailed report; - bounded runtime payload with quality-report artifact metadata. The fixtures intentionally use stable `quality-fixtures/...` paths instead of machine-local absolute paths. +Remediation catalog audit payloads may add bounded attribution evidence while +remaining compatible with `histdatacom.quality-remediation-catalog-audit.v1`. +Current attribution fields include `attribution_status`, +`attribution_reason`, status/reason counts, source-helper counts, and +finding-code-prefix counts. Consumers must treat `exact`, `inferred`, and +`unresolved` as advisory audit states; they do not replace `rule_id`, alter +finding severity, or change quality exit decisions. Runtime-only report gaps +use `runtime_report` because their rule ID comes from the saved report rather +than static source discovery. + +Remediation coverage and catalog-audit groups also include deterministic +`actionability` and `actionability_reason` fields. The stable actionability +vocabulary is `remediable_defect`, `policy_or_profile_decision`, +`unsupported_format_or_capability`, `expected_artifact_or_context`, +`needs_rule_attribution`, `needs_diagnostic_context`, `unsafe_to_automate`, and +`informational_only`. Boundary-aware summary counts are advisory additions: +ordinary mapped/unmapped and warning/error counts remain unchanged, while +actionable, intentional-boundary, attribution-blocked, and +diagnostic-context-blocked counts explain which gaps should be worked first. +Unknown warning/error codes default to `remediable_defect`; only deterministic +rule, finding-code, severity, mapping, or attribution evidence can move a gap +behind actionable defects. These fields do not change finding severity, quality +status, or exit policy. + +Catalog audits may also add a `remediation_plan` using +`histdatacom.quality-remediation-plan.v1`. The section is derived from +`ranked_gaps` and has independent bounded-sequence metadata under +`payload_limits.remediation_plan`. Consumers should use `items`, +`plan_item_count`, `included_plan_item_count`, `omitted_plan_item_count`, and +`truncated` together rather than assuming every candidate is embedded. Each +item preserves its `catalog_gap_rank` while adding a fixability-oriented `rank`, +suggested selector/action/hint-code metadata, explicit `missing_fields`, and +bounded `evidence`. Fixability scores and proposals are deterministic advisory +planning aids; they are not applied remediation mappings and do not mutate +reports, catalogs, files, or repository state. + +Standalone repair-plan output uses +`histdatacom.quality-repair-plan.v1`. It is derived from a saved +`histdatacom.quality-report.v1` and does not alter or replace the source report. +Consumers should use `plan_item_count`, `included_plan_item_count`, +`omitted_plan_item_count`, `truncated`, and `payload_limits.items` together. +Each item preserves the finding code, rule ID, severity, mapped hint and action +kind, publish-safe target identity, proposed operation, preconditions, +evidence requirements, bounded evidence, missing context, and confidence +basis. Stable proposal states are `proposed`, `needs_context`, and +`unsupported`; operation execution remains `manual_only` or `unsupported`. +The top-level `mode`, `apply_supported`, `mutating_operations_performed`, and +`safety` fields are normative: version 1 is advisory and performs no file, +archive, permission, network, report, or catalog mutation. + ## Update Workflow Do not update golden fixtures as a side effect of routine test runs. When a diff --git a/docs/derived-bar-contracts.md b/docs/derived-bar-contracts.md new file mode 100644 index 00000000..cde9626f --- /dev/null +++ b/docs/derived-bar-contracts.md @@ -0,0 +1,240 @@ +# Derived Reconstruction Bar Contracts + +Derived candlesticks are optional views over a verified final reconstruction +product. They are not raw evidence and are not an alternative persistence path +for the enriched analytical frame. + +## Product boundary + +The sole version-one input is a committed +`ReconstructionProductManifestV1`. Before aggregation begins, +`verify_reconstruction_publication()` checks the input manifest, exact +26-column `SyntheticEventV1` Parquet schemas, byte hashes, logical hashes, row +counts, ordering, origin support, and artifact confinement. + +The bar reader requests only the 19 event fields required for price, origin, +confidence, and bounded lineage calculations. Projection, symbol pruning, and +event-time predicates are pushed into Arrow dataset scans. Raw `ascii/T` +caches, raw HistData M1 files, and the 521-column analytical frame are never +opened by this module. + +The durable derived product contains: + +- a compact `DerivedBarProductManifestV1`; +- exact `DerivedBarV1` rows in Zstandard Parquet; +- monthly partitions by interval, scope, and symbol; +- physical byte evidence and logical content evidence. + +The original reconstruction manifest ID, publication ID, logical event hash, +run ID, and ensemble member ID bind every derived product to its source. + +## Versioned contracts + +| Contract | Purpose | +| --- | --- | +| `DerivedBarIntervalV1` | Supported duration, UTC alignment, and half-open bin semantics. | +| `DerivedBarPolicyV1` | Intervals, scopes, empty/partial/transition policies, rounding, and resource limits. | +| `DerivedBarV1` | One non-empty, provenance-bearing price and activity bar. | +| `DerivedBarPartitionV1` | Monthly Parquet axis, row/time bounds, logical hash, byte hash, and row-group evidence. | +| `DerivedBarProductManifestV1` | Source binding, policy, partitions, counts, writer evidence, and publication identity. | + +Unknown schema versions, relabeled derived fields, ID drift, unsupported +intervals, or inconsistent physical/logical evidence fail closed. + +## Intervals and ordering + +Version one supports exactly: + +| Code | Duration | +| --- | ---: | +| `1m` | 60 seconds | +| `5m` | 300 seconds | +| `15m` | 900 seconds | +| `30m` | 1,800 seconds | +| `1h` | 3,600 seconds | +| `4h` | 14,400 seconds | +| `1d` | 86,400 seconds | + +All intervals are aligned to the Unix epoch in UTC. Bins are half-open +`[bar_start_ns, bar_end_ns)`. Alternative time zones, exchange-local daily +boundaries, and arbitrary durations require a future schema version. + +Events are consumed per symbol in canonical +`(event_time_ns, event_sequence, event_id)` order. Multiple rows at one +timestamp remain distinct and their `event_sequence` determines open and close +position. A repeated or reversed position is refused. + +## Price fields + +Every non-empty bar contains independent OHLC fields for: + +- bid; +- ask; +- midpoint, calculated per event as `(bid + ask) / 2`; +- spread, calculated per event as `ask - bid`. + +Mean spread is event-support weighted. Prices must be finite and positive, +spread must be non-negative, and OHLC extrema must enclose open and close. +Rounding is deterministic and part of the policy identity. +Each row carries that policy ID and rounding precision so its arithmetic can be +validated even when it is read apart from the compact manifest. + +No interpolation, forward fill, price repair, or synthetic bar-only value is +allowed. + +## Scopes + +The policy can request any explicit combination of: + +- `observed`: immutable observed events only; +- `synthetic`: accepted generated events only, for diagnostics; +- `merged`: the final practical observed-plus-synthetic product. + +Each row records total, observed, and synthetic support. Counts must reconcile, +and origin-only scopes refuse support from the other origin. Downstream users +must select a scope explicitly; the default policy publishes merged bars only. + +## Activity projection + +The bar layer implements the handoff declared by #80: + +| Field | Bar operation | +| --- | --- | +| `event_count` / `quote_update_count` | Count delivered quote events. | +| `activity_duration_ns` | Recompute from first and last in-bar event times. | +| `tick_intensity_per_second` | Recompute count divided by positive activity duration. | +| `price_change_count` | Count changed quote transitions with boundary carry. | +| `stale_quote_count` | Count unchanged quote transitions with boundary carry. | +| `stale_quote_rate` | Recompute over supported transitions. | +| `mean_spread` | Event-support-weighted mean. | +| `mean_event_confidence` | Mean over rows where confidence exists, with support count. | +| `volume` | Always null in version one. | + +One event is one delivered quote update. Neither event count nor activity is +labeled centralized traded volume. Every row uses volume state `unavailable` +and asserts `centralized_traded_volume_claim=false`. + +The previous quote in the same symbol/scope/interval sequence is carried into +the next non-empty bar solely for transition classification. This includes a +transition across omitted empty bins or a market closure. The previous event is +not added to the next bar's price extrema, event count, activity duration, or +content hash. + +## Empty, closure, and partial bins + +Empty bins emit no rows. This applies to ordinary quiet gaps and expected +market closures. The bar product therefore does not invent liquidity or use +forward-filled rows to make a visually continuous grid. + +Partial flags describe explicit query cuts: + +- `is_partial_start=true` when `start_ns` falls inside the emitted bin; +- `is_partial_end=true` when `end_ns` falls inside the emitted bin. + +The requested nullable `query_start_ns` and `query_end_ns` are also durable +manifest fields and participate in publication identity, including when a +bound falls exactly on a bar edge. + +Without explicit query bounds, the committed source product defines the +requested coverage and edge rows are not labeled partial merely because the +first quote occurs after the clock boundary. The event bounds remain available +for downstream support decisions. + +## Provenance + +Each bar retains: + +- source product manifest, run, member, symbol, scope, and interval identity; +- first and last event IDs and event timestamps; +- observed and synthetic support counts; +- bounded source-version IDs; +- bounded generator, generator-version, generator-config, reference, motif, + feed-epoch, broker-profile, and constraint-set IDs; +- a SHA-256 digest over the canonical in-bar event projection; +- a deterministic `bar_id` over the full logical row. + +Lineage cardinality is bounded by policy. Exceeding the limit refuses the bar +instead of silently truncating provenance. + +## Streaming and resource behavior + +Aggregation retains one numeric/provenance state per active +symbol/scope/interval plus the previous quote needed for transition carry. It +does not retain input events or the analytical frame. The number of intervals +is fixed at seven, scopes at three, symbols are policy-bounded, provenance is +policy-bounded, and the total emitted bar count has an explicit ceiling. + +Publication buffers a bounded number of bar rows per active monthly writer and +writes bounded Parquet row groups. Input `batch_size`, output buffer size, and +row-group size are physical execution choices; changing them cannot change bar +IDs or the logical product hash. + +## Atomic persistence + +`stage_derived_bar_publication()` writes below: + +```text +derived-bar-products/ + schema=/ + source=/ + policy=/ + .scratch/publication.tmp-*/ +``` + +The staging directory is not discoverable. After every Parquet file and the +manifest pass validation, `commit_derived_bar_publication()` promotes the +complete directory into `commits/` with one same-filesystem +rename. Repeating an identical commit is idempotent. A matching logical +publication with different physical evidence is refused rather than silently +overwritten. + +Monthly relative paths are: + +```text +interval=/scope=/symbol=/ + bar_month=YYYY-MM/part-00000.parquet +``` + +The logical publication ID is independent of input/output chunk sizes. The +manifest separately records PyArrow version, Python runtime, compression, row +group size, partition byte hashes, and row-group counts. + +## Verification and reading + +`verify_derived_bar_publication()` checks: + +- committed-directory identity; +- exact manifest-declared artifact set; +- absence of symlinked files/directories; +- exact 64-column Arrow schema; +- byte size and SHA-256; +- row-group bounds; +- partition symbol/scope/interval/month axes; +- row ordering and contract identities; +- row/time counts and partition logical hashes; +- product logical hash and compact-manifest identity. + +`iter_derived_bar_batches()` supports column, symbol, scope, interval, and time +projection/pruning. `scan_derived_bars_polars()` provides the equivalent lazy +Polars surface. Time predicates use interval-overlap semantics (`bar_end_ns > +start_ns` and `bar_start_ns < end_ns`), so a leading partial bar is not silently +discarded. Both readers verify the publication before exposing rows. + +## Downstream use + +#448 must compare strategy/execution sensitivity using an explicit scope, +interval, product manifest ID, and time-aligned window. It must not compare a +merged bar series with an observed-only series without labeling the different +support. + +#449 must reconcile: + +- source reconstruction and derived-product identities; +- bar event/origin counts against #80 activity semantics; +- deterministic replay across batch and partition choices; +- raw-anchor preservation in the source event product; +- absence of raw M1 inputs, analytical-frame persistence, fabricated volume, + and undeclared artifacts. + +Derived bars are useful views, not historical truth and not evidence that a +synthetic reconstruction is valid by themselves. diff --git a/docs/empirical-motif-generation-contracts.md b/docs/empirical-motif-generation-contracts.md new file mode 100644 index 00000000..6937b513 --- /dev/null +++ b/docs/empirical-motif-generation-contracts.md @@ -0,0 +1,195 @@ +# Empirical Motif Candidate-Generation Contracts + +The empirical motif generator is the first production variable-cardinality +candidate for historical reconstruction. It proposes zero, one, or many +`SyntheticEventV1` rows between two immutable observations. It does not carve, +accept, broker-condition, or persist a final synthetic tick. + +## Boundary and data flow + +One call to `generate_empirical_motif_candidates()` consumes: + +- a semantic `ReconstructionRunV1` and one owned + `ReconstructionWindowV1`; +- two observed `SyntheticEventV1` anchors from the same source, symbol, run, + and ensemble member; +- one bounded `ReferenceMotifQueryResultV1`, including its conditioning cell, + support/backoff trace, and selected empirical fragments; and +- one `EmpiricalMotifGeneratorConfigV1` bound into the run configuration. + +It returns an `EmpiricalMotifCandidateBatchV1`. Candidate rows and per-event +lineage remain process-local. The batch's `metadata()` projection is bounded +control-plane evidence and explicitly reports: + +```text +candidate_only = true +hard_carving_status = not_evaluated +broker_conditioning_status = not_applied +final_storage_status = not_persisted +``` + +The metadata projection carries counts and SHA-256 digests for candidate, +transform, and event-lineage content rather than embedding those rows or large +ID lists. The process-local batch retains the actual rows for immediate +streaming or later artifact externalization. + +The motif query's `used_at_ns` must equal the right-anchor timestamp. This +binds point-in-time retrieval evidence to the exact interval that consumes it +instead of allowing a query result from a different target boundary to be +silently reused. + +This prevents a plausible empirical proposal from being mislabeled as a valid +historical, cross-series-consistent, broker-styled, or durable result. + +## Variable cardinality and delivery regime + +Version one derives target cadence from the conditioned `tick_intensity` +metric, with `interarrival_ns` as the fallback when intensity is absent. The +strictly interior cardinality is: + +```text +floor((right_anchor_ns - left_anchor_ns - 1) / cadence_ns) +``` + +Timestamp precision is then applied relative to the left anchor. Multiple +events may quantize to the same timestamp, but their stable global ordinal is +used as `event_sequence`, so `(event_time_ns, event_sequence)` remains unique +and ordered. + +Closed session states and zero target activity produce an explicit empty +decision. A zero-width or reversed anchor interval is an explicit refusal. +The generator never invents an ordinary weekend stream merely because a +global reference motif exists. + +## Fragment selection and transforms + +The complete anchor interval is planned before window ownership is applied. +For each segment, the generator derives a seed from: + +- run ID and base seed; +- ensemble-member ID; +- generator/config ID; +- anchor-interval ID; and +- global segment ordinal. + +Worker, retry, window, scratch path, and batch placement are absent. The seed +rotates through the deterministically ranked motif matches. A fragment may be +interpolated to more or fewer output events, but the resulting segment duration +must remain inside that fragment's declared time-scale range. When a compact +fragment supplies fewer gaps than the target cardinality, its positive +empirical gap weights repeat deterministically and are normalized inside the +segment boundary. Output timing therefore retains observed burst/quiet shape +instead of reverting to a uniform clock. Requested +volatility scaling is clamped to the fragment's declared price-scale range. + +Each `EmpiricalMotifTransformationV1` records the source index, query/result, +fragment, window, series, period, source artifact SHA-256, backoff level, +support, distance, source-event count, output ordinal range, time and price +scales, clamp decision, spread-shape decision, seed, and raw motif-match +similarity. That last value has the explicit meaning +`uncalibrated-motif-match-similarity-v1`; it is retrieval evidence, not a +pointwise correctness probability. + +Generator version 1.2.0 therefore leaves each candidate +`SyntheticEventV1.confidence` null. Calibrated reconstruction uncertainty is +reported only as exact-stratum metric/horizon interval coverage by the +ensemble layer; see +[`reconstruction-ensemble-calibration-contracts.md`](reconstruction-ensemble-calibration-contracts.md). + +Each `EmpiricalMotifEventLineageV1` maps an emitted event ID to its transform, +global and segment ordinals, source/anchor progress, and requested timestamp. +The query result retained by the process-local batch supplies the full +conditioning cell, fallback trace, and source fragment. Config, transform, and +event-lineage contracts have deterministic JSON round trips. + +## Seam and quote safety + +Source mid-price and optional spread shapes are detrended to zero at both ends +of each transformed segment. They are applied around a linear bridge between +the immutable anchor mids and spreads. The virtual transform therefore equals +the historical anchors at progress zero and one, and adjacent fragments meet +on the same anchor bridge instead of accumulating translation jumps. + +Target price precision is applied after transformation. Rounded internal +events preserve the selected fragment's bid-only, ask-only, both-mark, or +unchanged transition when the quote domain permits it; the last event of a +transform remains on the common linear seam. Every candidate is then checked +for finite, positive bid/ask values and non-negative spread. One +invalid transformed quote refuses the complete anchor interval; version one +does not silently clip, swap, or partially retain an unsafe path. + +`EmpiricalMotifCandidateBatchV1.merged_stream()` delegates to the frozen +synthetic-event stream contract. The caller's observed event objects are +passed through unchanged, generated timestamps remain strictly between the +anchors, and duplicate positions are rejected by the existing stream +validator. + +## Window ownership, carry, and determinism + +Every legal window recomputes the same complete semantic anchor plan and emits +only timestamps inside its half-open core interval. A partition edge can +therefore split one anchor interval without changing any retained event ID, +timestamp, bid, ask, motif selection, seed, or transform ID. Unioning adjacent +owned outputs equals a single-window result. + +The returned `CarryStateV1` records the per-symbol watermark and last emitted +or observed event reference for downstream streaming orchestration. Large +state remains outside workflow history. Carry and window IDs do not enter +candidate identity. + +## Sparse evidence and resource refusal + +The generator consumes the reference index's exact-to-global backoff trace. A +`no_supported_cell` or `not_available_as_of` query result becomes a named +generation refusal, and every sparse/unavailable attempt remains visible in +batch metadata. + +Before motif paths are materialized, the complete interval target count is +converted into a `ReconstructionResourceEstimateV1`. The run's storage policy +checks candidate amplification, peak batch events, retained members, inflight +batches, memory, scratch, and output estimates. Violations return the rejected +estimate and all reason strings in a `resource_limit` decision. + +Generator safety caps and byte-size estimates are serialized with the config +but excluded from its semantic `config_id`. Consequently, changing legal +execution estimates changes resource evidence, not successful candidate IDs +or values. Semantic settings such as precision, closed-state handling, and the +pre-carving constraint namespace remain identity inputs. + +## Reverse-degradation comparison + +`EmpiricalMotifBenchmarkGeneratorV1` implements the shared +`BenchmarkGeneratorV1` interface. It turns adjacent degraded observations into +immutable anchors, queries the motif index under the configured information +mode, emits the anchors plus owned proposals, and passes that stream to +`generate_benchmark_candidate_window()`. + +After observed anchors and proposals merge, the adapter recomputes bid-only, +ask-only, joint, and unchanged state from the actual adjacent quote values. +Benchmark transition scores therefore cannot be improved by a stale activity +label that disagrees with the emitted stream. + +The existing reverse-degradation engine can therefore compare this generator +with no-fill, linear interpolation, resample-last, and empirical-overlay +controls across the same feed-epoch, severity, session, event, and sparsity +slices. The scorecard remains report-only: it does not automatically select a +winner or bypass hard historical constraints. + +## Downstream stages + +- [Historical carving](historical-carving-contracts.md) consumes these + candidate-only batches and owns accepted/rejected constraint decisions. +- #441's implemented synchronized cross-currency reconciliation consumes these + candidate streams after historical carving; see + [`cross-currency-reconciliation-contracts.md`](cross-currency-reconciliation-contracts.md). +- #442's implemented ensemble layer calibrates substantive member diversity, + coverage, representative retention, and hash-bound regeneration; see + [`reconstruction-ensemble-calibration-contracts.md`](reconstruction-ensemble-calibration-contracts.md). +- later broker-transfer work owns broker-style conditioning. +- #446 implements final incremental Parquet layout and atomic publication. +- #447's implemented control plane owns production Temporal activities, + backpressure, cancellation, and recovery; see + [`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + +Changing event-generation semantics, identity fields, decision meanings, or +lineage interpretation requires a new schema/generator version. diff --git a/docs/feed-epoch-contracts.md b/docs/feed-epoch-contracts.md new file mode 100644 index 00000000..42391325 --- /dev/null +++ b/docs/feed-epoch-contracts.md @@ -0,0 +1,264 @@ +# Feed-epoch contracts + +Feed epochs describe changes in the technology through which historical tick +data was observed. They are not calendar eras, market regimes, or claims about +missing historical trades. A feed epoch is fitted from canonical time-series +fingerprints and becomes usable by reconstruction only after deterministic +stability checks pass. + +This implementation replaces the earlier `sparse`, `transitional`, and `dense` +labels derived from an independent raw-tick scan. The command name +`histdatacom analytics feed-regimes` remains available for compatibility, but +its discovery and evidence now flow through the data-quality target and +fingerprint engine. + +## Active-time multivariate v2 fit + +`histdatacom analytics feed-epochs-v2` is the production research path for +issue #460. It does not reinterpret the v1 schema. It reads the real monthly +ASCII tick Arrow caches directly and writes three bounded artifacts: source +evidence, the compact epoch definition, and campaign/runtime metadata. + +Each monthly observation states three different denominators: + +- full UTC calendar-month duration; +- duration in the shared fixed-EST FX market-open policy, including the + labelled Friday-close and Sunday-open windows but excluding weekend closure; +- observed active time, defined as the sum of positive market-open + inter-arrivals at or below the configured active-gap cap. + +Rates never silently substitute one denominator or numerator for another. The +calendar rate uses all rows, the market-open rate uses only market-open rows, +and the active-window rate uses qualifying intervals divided by their summed +duration. Both filtered numerators are retained as evidence counts. The v2 +evidence also records bid-only, ask-only, joint, and unchanged transitions; +hourly-count Fano dispersion; inter-arrival dispersion and lag; timestamp +quantization plus bounded last-digit counts; price precision; duplicate, burst, +and stale rates; exact stale-run p95 and maximum lengths; spread tails and +jumps; normalized activity over the shared overlapping session windows and +source-calendar weekdays; and cross-symbol hourly activity correlation and +overlap. Hourly synchronization is explicitly a bounded activity proxy, not a +claim that quote identities or missing ticks are shared. + +The shared detector robustly scales features within each symbol, takes the +cross-symbol median for each common month, and applies multivariate PELT with a +versioned winsorized squared-error cost, penalty, and minimum segment length. +Separate per-symbol fits are retained as deviations rather than folded into the +global epochs. Boundary support and uncertainty come from penalty variants, +leave-one-symbol and leave-one-feature runs, alternate denominator-feature +exclusions, and duplicate-feature exclusion. + +PELT candidates do not automatically become epochs. A candidate must meet the +support threshold overall and within every available sensitivity family. +Candidates that fail a family are retained in `rejected_candidates` with their +family support table, but they are excluded from the intervals exposed to +downstream code. + +The definition is admitted to observation-operator fitting only when the +configured symbol, common-period, feature-coverage, and boundary-sensitivity +requirements pass. Failed or unsupported definitions remain inspectable but +are not valid reconstruction inputs. + +```sh +histdatacom analytics feed-epochs-v2 \ + --target data/ASCII/T/eurusd data/ASCII/T/gbpusd data/ASCII/T/eurgbp \ + --artifact-dir data/.histdatacom/feed-epochs-v2 \ + --json +``` + +This analysis describes changes in the *observation technology represented by +these source caches*. It does not identify market regimes, recover unobserved +historical quotes, prove a causal vendor change, or supply broker adaptation. + +## Contract boundary + +| Contract | Responsibility | +| --- | --- | +| `FeedEpochEvidenceV1` | One bounded symbol-period projection of a canonical tick fingerprint, including feature provenance and source/config hashes. | +| `FeedEpochFitConfigV1` | Feature selection, coverage requirements, boundary thresholds, sensitivity policy, and resource limits. | +| `FeedEpochBoundaryV1` | Central boundary, uncertainty interval, change score, perturbation support, and contributing features. | +| `FeedEpochIntervalV1` | One named epoch and its inclusive period bounds. | +| `FeedEpochStabilityV1` | Sampling, missing-period, and feature-removal run counts plus pass/limited/fail status. | +| `FeedEpochDefinitionV1` | Versioned epochs, uncertain transitions, stability result, and complete evidence/config lineage. | +| `FeedEpochAssignmentV1` | Deterministic assignment of a period to an epoch or uncertain transition. | + +All readers validate their schema version and deterministic identity. Modified +feature values, source hashes, config values, boundary order, interval order, +or lineage counts fail closed rather than silently producing a different +interpretation under the same ID. + +## Canonical evidence only + +`FeedEpochEvidenceV1.from_fingerprint()` accepts ASCII tick fingerprints from +the canonical `fingerprint.series` surface. It projects only bounded +observation-regime signals: + +- tick rate and inter-arrival cadence; +- observed timestamp interval and price precision; +- spread level and conditioned spread level; +- spread changes, stale quotes, bursts, duplicates, and suspicious gaps; +- source-quality penalties; +- session, holiday, event, and special-window conditioning metadata. + +The evidence records both the fingerprint hash and its source hash. Inline +canonical findings use the fingerprint identity as that source hash; a persisted +fingerprint artifact may instead supply its byte-level SHA-256. The explicit +`source_hash_basis` field prevents those two meanings from being conflated. Its +feature-provenance map states the exact fingerprint path used for every value. +Calendar/event coverage and quality limitations are retained even when they are +not themselves fitted as numeric features. + +The v1 epoch fitter does not discover paths or scan raw ticks. Public v1 CLI and API +compatibility functions delegate discovery to `discover_quality_targets()`, run +the canonical fingerprint rules, then pass the resulting evidence into the +standalone fitter. This keeps one interpretation of source freshness, sibling +caches, timestamp parsing, session closures, and known quality limitations. + +## Panel normalization and boundary fitting + +Features are robustly normalized within each symbol before the symbol-period +panel is aggregated by period. This prevents a symbol with a naturally +different quote cadence or spread from creating a false boundary merely by +entering or leaving panel coverage. Eligible features must satisfy configured +period coverage, and the period panel is normalized again before adjacent +multivariate change scores are computed. + +Candidate boundaries must: + +1. exceed the configured robust change score; +2. leave the configured minimum number of periods on both sides; and +3. remain separated from stronger neighboring candidates. + +A boundary is an uncertainty interval, not an exact instant. Its central period +comes from the full fit. Its lower and upper periods include the matched +boundaries recovered by deterministic perturbation runs. + +No calendar year is treated as a regime a priori. A stable no-change history is +a valid one-epoch result. + +Canonical histories may contain annual artifacts in older years and monthly +artifacts later. The compatibility surface never fabricates monthly evidence +from an annual fingerprint. Instead it deterministically coarsens monthly +fingerprints to annual evidence whenever annual input is present or an annual +bucket is requested. An exact annual fingerprint takes precedence over +overlapping monthly fingerprints for the same symbol-year. Each aggregate +retains the complete component fingerprint and source-hash list; definition +lineage exposes both fitted-evidence and original canonical-source counts. + +## Stability and downstream trust + +Every full fit reruns the detector under three perturbation families: + +- deterministic even/odd period sampling; +- removal of each eligible internal period; +- removal of each eligible feature. + +Each boundary records support by family and overall support. The definition +passes only when all required families are available and every boundary meets +the configured support threshold. It is `limited` when the evidence is too +short to exercise all families, and `fail` when an asserted boundary is not +stable. + +`FeedEpochDefinitionV1.valid_for_observation_models` is the downstream trust +gate. `assign()` refuses a limited or failed artifact by default. Callers must +make an explicit, visible override to inspect an unstable assignment; such an +override is not suitable for reconstruction or broker-conditioning claims. + +A period inside a boundary uncertainty interval is assigned to +`kind="transition"`, not forced into either adjacent epoch. Periods outside +those intervals receive a stable epoch assignment. + +## Resource limits and determinism + +Evidence count, selected feature count, and total sensitivity runs are bounded +by the fit config. Inputs are sorted and de-duplicated before fitting, every +sensitivity variant is enumerated deterministically, and all IDs are hashes of +canonical JSON payloads. Input order, worker count, and filesystem path order +do not affect the artifact. + +The default full-fit contract requires six periods and at least two periods on +each side of a boundary. The compatibility analytics report can describe a +two-period fixture with relaxed segment limits, but its stability is necessarily +`limited` and therefore fails the downstream trust gate. + +## CLI and artifacts + +Write the compatibility report and the compact epoch definition separately: + +```sh +histdatacom analytics feed-regimes \ + --target data/ASCII/T/ \ + --bucket month \ + --report reports/feed-regimes.json \ + --epoch-artifact reports/feed-epochs.v1.json +``` + +Fit policy can be pinned explicitly for reproducible research: + +```sh +histdatacom analytics feed-regimes \ + --target data/ASCII/T/ \ + --features log_tick_rate log_median_interarrival_ms price_precision spread_median \ + --min-evidence-periods 12 \ + --min-segment-periods 3 \ + --min-change-score 0.8 \ + --min-boundary-support 0.75 \ + --max-evidence 4096 \ + --max-sensitivity-runs 256 \ + --epoch-artifact reports/feed-epochs.v1.json +``` + +The definition artifact contains the complete source/fingerprint/config lineage +needed to replay or reject a result. The larger compatibility report embeds the +same definition alongside period profiles and summary metrics. + +## Streaming reconstruction integration + +Feed-epoch fitting is a bounded control-plane step, not a per-row augmentation +job. Reconstruction should load and validate one `FeedEpochDefinitionV1` at run +admission, bind its `definition_id` into the semantic reconstruction config, +and assign each owned window or final synthetic event to an epoch or transition +by period. + +The streaming data plane therefore carries compact fields such as: + +```text +feed_epoch_definition_id +feed_epoch_id +feed_epoch_assignment_kind +feed_epoch_boundary_id # only for uncertain transitions +``` + +It does not carry the fingerprint payload, fitting panel, sensitivity matrices, +or the 521-column analytical frame. Those are reproducible intermediates. The +final synthetic tick remains the durable row, while the epoch definition is a +small immutable sidecar referenced by the run manifest and output lineage. + +The reconstruction source manifest must record the epoch definition as a +semantic source, not as mutable workflow metadata. Changing the definition or +accepting a new version consequently changes the reconstruction run identity. +Windowing, retries, or storage tuning do not. + +## Issue boundaries + +- #321–#333 own canonical fingerprints, their persistence/constraints, and the + evidence source consumed by epoch fitting. +- #433 governs ex-post versus ex-ante information access. +- #435 consumes only stability-passing definitions and implements historical + feed-observation operators. +- #436 implements reverse-degradation scorecards over the #435 interface; see + [`reverse-degradation-benchmark-contracts.md`](reverse-degradation-benchmark-contracts.md). +- #439 consumes only stable epoch artifacts while generating candidates. +- #443's implemented + [live broker delivery capture](broker-capture-contracts.md) supplies qualified + wall/monotonic delivery evidence; #444–#445 own broker fingerprints and style + transfer rather than historical technological epochs. +- #446 implements atomic final Parquet and manifest publication. +- #447's implemented Temporal control plane carries compact artifact + references through production streaming; see + [`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + workflows. + +Changing a required field, identity derivation, stability family, transition +meaning, or assignment trust rule requires a new schema version and contract +class. diff --git a/docs/historical-carving-contracts.md b/docs/historical-carving-contracts.md new file mode 100644 index 00000000..871f20c2 --- /dev/null +++ b/docs/historical-carving-contracts.md @@ -0,0 +1,178 @@ +# Historical Carving Contracts + +Historical carving is the first stage that may promote empirical-motif +candidates into accepted synthetic events. It consumes process-local candidate +rows, applies versioned historical constraints in a fixed order, and returns +accepted rows plus bounded decision evidence. It does not perform cross-series +reconciliation, broker conditioning, or final persistence. + +## Bound inputs + +One call to `carve_empirical_motif_candidates()` binds: + +- the semantic `ReconstructionRunV1` and owned `ReconstructionWindowV1`; +- one primary `EmpiricalMotifCandidateBatchV1` and optional same-scope + substitution batches; +- the caller's immutable observed events, including both anchor IDs; +- one window-covering `MarketContextQueryV1` in the same information mode; +- one `HistoricalCarvingConstraintSetV1` listed in the run configuration; and +- when required, a `CarvingFingerprintEvidenceV1` wrapping the existing + synthetic-fingerprint validator result. + +Fingerprint evidence must report `match` and list every candidate batch it +validates. Reference and candidate quality-report IDs are part of the evidence +identity. A successful validation result therefore cannot be silently reused +for unrelated candidate rows. + +The market-context query may report `no_matching_event`; an ordinary interval +with a complete calendar profile is still supported evidence. Timeline gaps, +point-in-time unavailability, missing calendar state, or an incomplete required +profile refuse the batch. Context and motif-query information modes must agree. + +## Fixed precedence and fail-closed behavior + +Version one evaluates these rule stages in order: + +```text +1. candidate integrity +2. immutable-anchor lineage +3. resource and pathological-gap envelope +4. fingerprint validation +5. market-context support +6. source-quality quarantine +7. session closure +8. conditioned motif eligibility +9. conditioned intensity +10. conditioned spread projection +11. final local validation +``` + +The exact rule IDs and order are embedded in the constraint-set identity. +Hard rules always precede conditioned rules. A weekend/session closure or +quarantine can therefore never be undone by a news, rollover, crisis, or other +state policy. Version one has no advisory carving rule: a configured +conditioned policy affects output semantics and is included in the constraint +set ID. + +Hard failures are not repaired by projection. Every candidate must already be +a finite, positive, non-negative-spread candidate-only event strictly inside +its immutable anchor interval and the window's half-open ownership range. +Spread projection runs only after those checks and after the bound fingerprint +validator passes. + +## Conditioned policies + +`HistoricalCarvingConditionPolicyV1` matches explicit lower-case tags from: + +- the motif condition's session, special, and event tags; +- the calendar state's sessions, overlaps, special, holiday, event, and + combined tags; and +- market-context event kinds and tags whose conditioned windows overlap the + candidate timestamp. + +This supplies explicit policies for states such as `news_window`, +`daily_rollover`, and `crisis`. All matching policy IDs are retained. Their +acceptance rates and spread multipliers combine multiplicatively. The final +spread multiplier must remain inside the constraint set's hard bound. + +Intensity thinning uses a stable score derived from run/member identity, +anchor interval, timestamp, event sequence, policy IDs, and constraint-set ID. +It excludes worker, retry, window ID, storage path, and execution order. +Repeating a batch or repartitioning the same anchor interval therefore cannot +change an owned event decision. + +Spread projection keeps the candidate midpoint fixed and changes only the +spread. Price precision is explicit. A projected accepted lineage stores the +original bid and ask, candidate content hash, output content hash, exact policy +IDs, and multiplier. Projection cannot conceal the pre-projection quote. + +## Motif refusal and substitution + +A condition policy may restrict eligible motif IDs. If the primary event is +incompatible, the engine searches optional substitution batches at the exact +same `(event_time_ns, event_sequence)` position. Alternatives must share run, +window, member, symbol, anchors, and anchor interval. Selection is a stable +event-ID ordering. + +An eligible replacement retains its own candidate batch, candidate event, and +motif transformation IDs in accepted lineage. If no same-position replacement +is eligible, the candidate is rejected as `motif_incompatible`. Version one +does not invent an interpolation or mutate an incompatible motif into apparent +support. + +## Identity and accepted-event lineage + +Candidate events use the candidate-only constraint namespace. Every accepted +event is rebuilt with the final carving constraint-set ID, producing a final +carving-stage event identity while preserving generator, reference, motif, +anchor, feed-epoch, and confidence fields. + +`HistoricalCarvingEventLineageV1` connects the two identities and records: + +- candidate and output event IDs and full-row content hashes; +- candidate batch and motif transformation IDs; +- accepted, projected, substituted, or substituted-and-projected action; +- applied rule, context-event, and policy IDs; +- original and final constraint-set IDs; +- deterministic acceptance score and spread multiplier; and +- original quotes whenever projection changed them. + +This extra content binding is intentional. `SyntheticEventV1` version one does +not put bid, ask, or confidence in `event_id`; carving batch and lineage hashes +prevent a value change from hiding behind an unchanged semantic event key. + +## Bounded rejection evidence and refusal + +Rejected candidate rows are never retained by +`HistoricalCarvedCandidateBatchV1`. `RejectionSummaryV1` stores exact candidate, +accepted, and rejected counts plus deterministic reason counts. The counts must +reconcile. A configurable bounded sample stores only candidate ID, content +hash, source batch ID, position, reason, and rule ID. Batch metadata explicitly +reports `rejected_rows_retained = false`. + +An empty upstream interval remains `empty`. Insufficient fingerprint/context +support, an upstream refusal, session closure, pathological anchor gap, resource +limit, or a batch with no admissible candidates becomes `refused` with a stable +reason. Some candidates accepted and some rejected becomes `partial`. Refusal +is a valid, measurable output rather than an exception or fabricated fill. + +API misuse—such as cross-run batches, a context query for another window, or a +constraint set absent from the run—is rejected as a contract error before +carving begins. + +## Final local validation and streaming boundary + +Before accepted rows leave the stage, the local validator rechecks: + +- final constraint-set and immutable-anchor IDs; +- half-open window ownership; +- unique timestamp/sequence positions; +- valid narrow event contracts and merged-stream ordering; and +- exact preservation of every caller-supplied observed event ID. + +The batch records both the bound synthetic-fingerprint validator schema and the +local event-validator ID. `merged_stream()` delegates to +`SyntheticEventStreamV1` and passes observations through unchanged. + +Only accepted rows and accepted lineage remain process-local. `metadata()` +uses content hashes instead of embedding them. Rejected rows are discarded; +final Parquet publication is implemented as the later #446 stage. This preserves the streaming +design: candidate generation, carving, and later reconciliation can operate one +bounded window at a time without materializing a permanent augmented-history +intermediate. + +## Downstream stages + +- #441's implemented synchronized cross-currency reconciliation consumes these + accepted streams; see + [`cross-currency-reconciliation-contracts.md`](cross-currency-reconciliation-contracts.md). +- later broker-transfer work owns broker-style conditioning. +- #446 implements final incremental Parquet layout and atomic publication. +- #447's implemented control plane owns production Temporal activities, + backpressure, cancellation, and recovery; see + [`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + resume behavior. + +Changing rule meaning or order, constraint identity, decision meaning, +projection semantics, or accepted-lineage interpretation requires a new schema +and engine version. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..1d26fca8 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,75 @@ +HistData.com Tools +================== + +HistData.com Tools is a command-line utility and Python ETL package for +downloading, assessing, modeling, and reconstructing historical foreign +exchange data. This documentation collects the maintained operational guides +and versioned contract references for the project. + +Data and reconstruction foundations +----------------------------------- + +.. toctree:: + :maxdepth: 2 + + synthetic-event-contracts + reconstruction-streaming-contracts + reconstruction-information-modes + feed-epoch-contracts + observation-operator-contracts + market-context-contracts + cftc-positioning-contracts + +Empirical reconstruction pipeline +--------------------------------- + +.. toctree:: + :maxdepth: 2 + + reference-motif-index-contracts + modern-reference-motif-library + empirical-motif-generation-contracts + historical-carving-contracts + cross-currency-reconciliation-contracts + reconstruction-ensemble-calibration-contracts + reconstruction-plan-contracts + broker-capture-contracts + broker-delivery-fingerprint-contracts + broker-delivery-transfer-contracts + +Persistence, projections, and validation +---------------------------------------- + +.. toctree:: + :maxdepth: 2 + + reconstruction-persistence-contracts + reconstruction-activity-semantics + derived-bar-contracts + strategy-sensitivity-contracts + reverse-degradation-benchmark-contracts + reverse-degradation-benchmark-corpus + reconstruction-certification-contracts + reconstruction-temporal-orchestration + reconstruction-public-interfaces + +Data quality +------------ + +.. toctree:: + :maxdepth: 2 + + data-quality/report-compatibility + +Temporal runtime and operations +------------------------------- + +.. toctree:: + :maxdepth: 2 + + container + temporal-binary-provisioning + temporal-orchestration-operations + temporal-orchestration-runtime-runbook + temporal-workflow-topology + temporal-orchestration-performance diff --git a/docs/market-context-contracts.md b/docs/market-context-contracts.md new file mode 100644 index 00000000..822c6a03 --- /dev/null +++ b/docs/market-context-contracts.md @@ -0,0 +1,334 @@ +# Point-in-time market-context contracts + +The market-context domain supplies immutable macro, central-bank, news, and +calendar evidence to reconstruction without adding repeated context columns to +every tick. A versioned timeline is durable; each bounded reconstruction window +receives a compact query sidecar. + +The contracts do not authorize or scrape a paid corpus. Production corpus +support selects only the official sources documented below. Source adapters +normalize approved evidence into the common interface while retaining the +original provenance and redistribution policy. Additional sources still +require an operator license review. + +## Contract boundary + +| Contract | Responsibility | +| --- | --- | +| `MarketContextSourceV1` | Source/version, retrieval time, content hash, adapter identity, license, redistribution constraints, limitations, and bounded metadata. | +| `MarketContextEventV1` | One immutable schedule, release, revision, decision, communication, shock, or news vintage. | +| `MarketContextTimelineV1` | Ordered vintages, revision chains, declared coverage, completeness, limitations, and deterministic artifact identity. | +| `MarketContextCalendarStateV1` | Compact reuse of the existing HistData session, closure, rollover, holiday, and period-end classifier. | +| `MarketContextQueryV1` | Bounded ex-post or ex-ante sidecar for one half-open interval or reconstruction window. | +| `MarketContextSourceAdapterV1` | Shared normalization seam for operator-approved sources. | + +Every external event embeds its complete source contract. The record therefore +retains retrieval/version metadata, a content SHA-256 digest, license and +redistribution policy, source URI where appropriate, adapter version, affected +currencies/symbols, confidence, limitations, and source-time normalization. +Raw source text need not be redistributed. A licensed adapter may store only a +content hash and operator-approved normalized values. + +## Event identity and revisions + +`canonical_key` identifies the economic or news event across vintages. It is +normalized to a stable lowercase identifier. `event_id` hashes the complete +event vintage, including source provenance and values. + +An initial record has `revision_sequence=0`. Each later vintage must: + +- name the immediately preceding `event_id`; +- retain the canonical key, semantic market-event time, and first-known time; +- increment the sequence by exactly one; and +- have a strictly later availability timestamp. + +The timeline retains both the first release and every revision. It rejects +duplicate IDs, duplicate logical vintages, orphan revisions, sequence gaps, +and revisions that rewrite the original event identity. A revision never +updates an earlier object in place. + +Scheduled macro records may carry expected, actual, previous, revised-previous, +and surprise values with explicit units. When expected and actual are both +present, surprise is deterministically `actual - expected`; contradictory +values fail validation. + +## Knowledge time versus market-event time + +Three times have distinct meanings: + +- `event_time_ns` is when the target market event occurs; +- `first_known_at_ns` is when the existence or schedule was first known; and +- `available_at_ns` is when this exact vintage became usable. + +An ex-ante query requires `as_of_ns` and excludes any vintage whose first-known +or availability time is later. Ex-post queries expose all matching vintages. +This lets a schedule be visible before a release without exposing the actual or +a later revision. + +When query events are added to `ReconstructionInformationInputV1`, the +information graph uses the vintage's availability time as its semantic input +event. The target release time remains in `MarketContextEventV1`. This prevents +a known schedule from being mislabeled as realized future information while +the existing #433 audit still rejects an actual or revision used before it was +available. + +## Timezone and event-window semantics + +Source event timestamps are ISO-8601 strings paired with an IANA timezone. +Normalization verifies that an explicit offset agrees with the named zone. +Naive local times are localized with `zoneinfo`: + +- ambiguous daylight-saving folds require an explicit `source_time_fold`; +- nonexistent wall-clock times fail closed; and +- the normalized value must equal the stored UTC nanosecond timestamp. + +Each record declares `pre_event_ns` and `post_event_ns`. Its conditioned window +is exactly: + +```text +[event_time_ns - pre_event_ns, event_time_ns + post_event_ns) +``` + +This half-open rule makes boundary behavior deterministic across streaming +windows. + +Unscheduled shocks and news windows cannot claim exact precision. They require +`approximate` or `window_only` precision, a confidence value, an explicit +ambiguity reason, and limitations. + +Policy-rate transitions use `policy_rate_change`. They are not interchangeable +with `central_bank_decision`: a rate history can establish that no level +changed during a supported interval, but it cannot establish that no meeting +or unchanged-rate decision occurred. + +## Bounded streaming joins + +`query_market_context_window()` consumes `ReconstructionWindowV1` and returns a +bounded `MarketContextQueryV1`. It filters by the synchronized window's symbols, +optional currencies and event kinds, point-in-time availability, and +conditioned-window overlap. The result contains event contracts, a compact +calendar classification at the window boundary, the exact timeline/window IDs, +and limitations. + +It never emits a dataframe or repeats context over market rows. The default +limit is 512 events per query and 4,096 events per timeline. Exceeding a query +limit raises `MarketContextQueryLimitError` rather than silently truncating +evidence. Large source bodies and analytical enrichment remain outside the +query payload. + +## Calendar context + +`market_context_calendar_state()` reuses +`classify_histdata_timestamp()` and `calendar_policy_metadata()` from the data +quality domain. It carries session state, clock and active sessions, overlaps, +rollover/open/close/fix tags, holiday/event tags, month/quarter/year-end tags, +calendar profile source/version, completeness, and limitations. + +The default holiday profile remains explicitly incomplete and advisory. A +calendar state is a deterministic classification, not evidence that a news or +macro event occurred. + +## Explicit missing context + +An empty query is never converted into a neutral or invented event label. Its +status is `missing` with one stable reason: + +- `no_matching_event` inside complete declared coverage; +- `not_available_as_of` when matching evidence exists but was not yet known; +- `outside_timeline_coverage`; or +- `timeline_incomplete` when an approved source is absent or incomplete. + +Calendar state may still be available alongside missing external context. The +two facts remain separate. + +## Adapter and trust gates + +`MarketContextSourceAdapterV1` exposes only an adapter name/version and +`load_events()`. Collection verifies that every emitted event names that exact +adapter identity. `StaticMarketContextSourceAdapterV1` supplies a deterministic +fixture and normalized-import implementation without implying a production +news source. + +Context may condition activity, spread, timing, motif selection, and carving +only when its timeline and query IDs are preserved in the information graph. +It may not be added retrospectively as narrative decoration, treated as +point-in-time evidence before availability, or used to replace missing context +with an invented classification. + +Changing required fields, identity rules, revision semantics, query visibility, +or timezone behavior requires a new schema version and contract class. + +## Production event corpus + +`MarketContextCorpusV1` composes the v1 timeline with the acquisition policy, +content-addressed source evidence, source diagnostics, coverage/missingness +slices, year/currency/type counts, runtime, peak memory, and a deterministic +logical corpus ID. Runtime is reported but excluded from logical identity, so +a replay can prove identical evidence without pretending wall-clock timing is +deterministic. + +The initial production adapters deliberately use public official sources: + +| Adapter | Selection and mapping | Reuse basis | Point-in-time limitation | +| --- | --- | --- | --- | +| ONS release calendar | Published, allowlisted UK CPI, GDP, labour-market, retail-sales, and public-finance records; maps to GBP, EURGBP, and GBPUSD. | [ONS release-search API](https://developer.ons.gov.uk/search/search-releases/) under [Open Government Licence v3.0](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/). | The current search record does not preserve when the schedule was first published. A release becomes eligible only at its retained release time. Date changes without change-known times remain limitations, not invented revisions. | +| ECB key interest rate | `FM.D.U2.EUR.4F.KR.MRR_RT.LEV`, the combined minimum-bid/fixed-rate daily MRO level from 1999; maps to EUR, EURUSD, and EURGBP. The raw daily response is preserved byte-for-byte and the derived corpus suppresses unchanged consecutive levels. | [ECB Data API](https://data.ecb.europa.eu/help/api/data) under the [ECB statistics reuse policy](https://www.ecb.europa.eu/stats/ecb_statistics/governance_and_quality_framework/html/usage_policy.en.html). Derived events retain `Source: ECB statistics`; raw statistics and metadata are not modified. | The series covers both the 2000–2008 variable-rate minimum-bid regime and the fixed-rate regimes. Observations are effective state dates, not announcement timestamps. They are `window_only`, become ex-ante eligible on the following UTC day, and contain no inferred consensus. | +| Bank of England Bank Rate | Official Bank Rate history (`IUDBEDR`); maps to GBP, GBPUSD, and EURGBP. | [Bank Rate history](https://www.bankofengland.co.uk/boeapps/database/Bank-Rate.asp?hl=en-GB) under the Bank's [OGL terms](https://www.bankofengland.co.uk/legal). Third-party database series are excluded. | The table supplies change dates and levels, not original decision times or schedule vintages. Records are full-day and `window_only`; ex-ante eligibility is delayed until the following UTC day. | +| Federal Reserve FOMC | Official historical meeting pages for 2000–2020 plus the current 2021+ calendar; maps to USD, EURUSD, and GBPUSD. | Federal Reserve [historical materials](https://www.federalreserve.gov/monetarypolicy/fomc_historical.htm) and [current calendar](https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm), normally [public domain](https://www.federalreserve.gov/disclaimer.htm) with attribution. | Historical date-only meetings become eligible after the local meeting day, preventing intraday look-ahead. The current supported calendar uses the Fed's documented [2 p.m. Eastern decision time](https://www.federalreserve.gov/economy-at-a-glance-policy-rate.htm), but never invents when a schedule was first published. | +| Operator public-shock catalog | Small cited public-record windows for major shocks affecting the common triangle. | MIT-licensed normalized factual metadata; every record retains its upstream public-record citation and limitations. | It is explicitly selective. Absence is never a `no_shock` label, and date windows make no causal or exact-timing claim. Date-only windows become ex-ante eligible on the following UTC day; the SNB window retains its sourced announcement time. | + +FRED/ALFRED is not used. Its current API terms leave third-party series rights +with each data owner, bind downstream application users to the API terms, and +make the API license terminable. Those conditions do not provide the uniform +reuse basis required for an immutable redistributable corpus. The adapters use +the selected primary official sources above instead. No fixture-only or +unspecified-provider adapter qualifies as production coverage. + +## Acquisition, immutable artifacts, and replay + +Build the corpus from installed code: + +```console +histdatacom analytics market-context-corpus \ + --artifact-dir .histdatacom/market-context \ + --start-date 2002-03-01 \ + --end-date 2026-06-30 +``` + +Acquisition has explicit response, total-byte, page, event, timeout, and +runtime bounds. It sends a named user agent, fails on an empty or oversized +response, and records every exact request URI, query/offset, retrieval time, +content type, response SHA-256, provider, license URI, adapter version, and +source limitation. + +`write_market_context_corpus()` writes: + +```text +artifact directory/ +├── sources/-. +├── market-context-timeline-.json +└── market-context-corpus-.json +``` + +Files are content addressed. An existing path with different bytes is refused; +an identical path is reused. Prior refreshes are never silently replaced. +`replay_market_context_corpus()` restores every raw body from its source +evidence, verifies sizes and hashes, re-runs the production adapters, and +requires the rebuilt logical corpus ID to match. + +## Coverage and preflight + +Coverage is not inferred from event density. ECB support is bounded by the +first and last parsed daily observations and becomes incomplete if the daily +state has a gap. Bank Rate support starts with the first parsed historical +change and ends at the snapshot retrieval day; invalid source rows make the +slice incomplete. Those sources can therefore support an interval containing +no transition without treating a sparse or broken response as complete. ONS +and Federal Reserve live archives are bounded by their first and last retained +official records. The operator shock catalog is always incomplete by design. + +`preflight_market_context_corpus()` evaluates required currency/kind pairs +against these declared source intervals. `require_market_context_corpus()` +raises `MarketContextCorpusPreflightError` for an unsupported period, currency, +event kind, incomplete source, or outside-timeline request. By contrast, a +supported ordinary window with no matching event returns +`no_matching_event`; that is not a preflight failure and is not converted into +a neutral invented event. `query_market_context_corpus()` performs this +preflight automatically whenever the caller declares both an event kind and a +currency (directly or through a six-letter FX symbol). Exploratory inspection +must opt out explicitly with `require_supported=False`. + +## Carving and benchmark consumption + +`read_market_context_corpus()` strictly loads the self-contained artifact. +`query_market_context_corpus()` returns the exact `MarketContextQueryV1` +already consumed by `carve_empirical_motif_candidates()`, so no notebook or +row materialization is needed. `market_context_benchmark_event_state()` +projects the same bounded query onto the benchmark's existing compact +`event_state` string. Timeline, query, event-vintage, and information-input IDs +remain available for leakage and output-lineage audits. + +An ex-ante query hides actuals, late publications, future shocks, and later +revisions until their `available_at_ns`. Missing revision history remains an +explicit source limitation instead of being reconstructed from the latest +page. + +## CFTC Commitments of Traders boundary + +Commitments of Traders is intentionally not coerced into this event corpus. +COT is a weekly persistent positioning state measured on one date, published +later, and valid until superseded. It needs latest-known-state queries, +high-dimensional position vectors, publication-confidence evidence, and +restatement handling rather than pre/post event overlap. The sibling domain is +documented in +[`cftc-positioning-contracts.md`](cftc-positioning-contracts.md). + +COT reuses source-policy, content-addressed artifact, replay, coverage, and +preflight patterns from this corpus. It does not become permanent +augmented columns on every tick; reconstruction will consume one bounded +positioning sidecar per window. + +## Real campaign evidence + +The #461 closure campaign covered 2002-03-01 through 2026-06-30 and retained +29 official/operator source snapshots. The final audited run read 9,475,207 +source bytes in 14.843019 seconds with a 159,039,488-byte peak resident-memory +measurement. It produced 1,028 unique in-range events after detecting 15 +duplicate ONS logical records: 726 scheduled UK macro releases, 48 ECB and 47 +Bank of England policy-rate changes, 202 Federal Reserve decisions, and five +in-range curated shocks. The event timeline spans 2002-03-19 through +2026-06-30. + +Coverage keeps those policy semantics separate. ECB and Bank of England +support `policy_rate_change` over the full requested interval; this does not +claim that an unchanged-rate meeting did not occur. Federal Reserve historical +and current meeting pages support `central_bank_decision` over the interval. +ONS macro coverage begins at its first retained archive record on 2016-02-19 +and preflight refuses earlier GBP macro requirements. The only source +diagnostic was the explicitly cancelled 2020-03-18 FOMC meeting; cancellation +is retained as a diagnostic and is not emitted as a decision. + +The final logical corpus ID is +`market-context-corpus:sha256:4dcd5319d8e0e2e5a7ed7f4d2950a1bb129fb12323d09bdd2c283524fb5f952d`; +the timeline ID is +`market-context-timeline:sha256:ee2761b90c65967b925745a4e9de23b040c3b82c5ac18ac0aec393db2dad1fd1`; +and the self-contained corpus artifact SHA-256 is +`9255f8c39f999b7a54e41a59a6f1d96f02e897af8383795e464a2f8738b08e00`. +An artifact reload and raw-source replay produced the same logical corpus ID. A +2025 policy preflight passed separately for EUR/GBP rate changes and USD FOMC +decisions, while a query just before the SNB shock returned +`not_available_as_of`. A 2010 GBP macro preflight was refused. + +The retained sources emitted zero revision vintages (0 of 1,028 events, +0.00%). This is reported as source missingness, not evidence that no historical +revision occurred: the current ONS, policy-rate, and meeting archives do not +retain complete change-known vintage history. The operator revision and late +publication paths are exercised with deterministic fixtures. + +| Year | EUR policy-rate changes | GBP policy-rate changes | GBP macro releases | USD FOMC decisions | Curated shocks | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 2002 | 1 | 0 | 0 | 7 | 0 | +| 2003 | 2 | 3 | 0 | 9 | 0 | +| 2004 | 0 | 4 | 0 | 8 | 0 | +| 2005 | 1 | 1 | 0 | 8 | 0 | +| 2006 | 5 | 2 | 0 | 8 | 0 | +| 2007 | 2 | 4 | 0 | 8 | 0 | +| 2008 | 4 | 5 | 0 | 8 | 1 | +| 2009 | 4 | 3 | 0 | 8 | 0 | +| 2010 | 0 | 0 | 0 | 8 | 0 | +| 2011 | 4 | 0 | 0 | 8 | 0 | +| 2012 | 1 | 0 | 0 | 8 | 0 | +| 2013 | 2 | 0 | 0 | 8 | 0 | +| 2014 | 2 | 0 | 0 | 8 | 0 | +| 2015 | 0 | 0 | 0 | 8 | 1 | +| 2016 | 1 | 1 | 11 | 8 | 1 | +| 2017 | 0 | 1 | 15 | 8 | 0 | +| 2018 | 0 | 1 | 45 | 8 | 0 | +| 2019 | 0 | 0 | 56 | 8 | 0 | +| 2020 | 0 | 2 | 56 | 13 | 1 | +| 2021 | 0 | 1 | 91 | 8 | 0 | +| 2022 | 4 | 8 | 97 | 8 | 1 | +| 2023 | 6 | 5 | 106 | 8 | 0 | +| 2024 | 4 | 2 | 98 | 8 | 0 | +| 2025 | 4 | 4 | 102 | 9 | 0 | +| 2026 | 1 | 0 | 49 | 4 | 0 | diff --git a/docs/modern-reference-motif-library.md b/docs/modern-reference-motif-library.md new file mode 100644 index 00000000..6c76715a --- /dev/null +++ b/docs/modern-reference-motif-library.md @@ -0,0 +1,153 @@ +# Real modern reference-motif library + +The production library turns small, synchronized windows from real monthly +HistData ASCII tick caches into a compact `ReferenceMotifIndexV1`. It is the +installed build and qualification boundary for issue #464; it does not package +dense source or holdout rows. + +## Fixed production profile + +The default profile uses EURUSD, GBPUSD, and EURGBP inside the stable +`technology_epoch_04` interval established by the v2 feed-epoch campaign. +Chronological roles are fixed before extraction: + +| Role | Periods | +| --- | --- | +| train | 201901, 202001, 202101, 202201, 202301 | +| calibration | 202307 | +| validation | 202401 | +| final holdout | 202510 | + +Six synchronized ten-minute parent windows per period cover Asia, London, and +New York plus available point-in-time event windows. Each symbol contributes +at most 96 real quotes per parent. Three-event fragments are needed to support +the short anchor gaps that caused the provisional #463 candidate to refuse; +the retained index is deterministically bounded to 256 fragments and 64 query +matches. + +Every monthly `.data` digest must equal the immutable source hash in the v2 +feed-epoch lineage. The manifest records all 24 selected partitions, their +periods, split assignments, hashes, row counts, sizes, and evidence IDs. It +records only compact offsets, quote deltas, transition marks, and source-row +identities—not the dense rows themselves. + +## Feature and leakage policy + +`reference_motif_condition_from_quotes()` is shared by library extraction and +benchmark retrieval. Its fixed, versioned feature schema covers symbol, +session, weekday, event/news and CFTC state, epoch, return, range, volatility, +spread, activity, inter-arrival, timestamp/price precision, and source quality. +Thresholds are code constants and are not fitted on a withheld period. + +All calibration, validation, and final-holdout fragments are projected only +for leakage and coverage audits. A train fragment whose normalized timing, +bid/ask shape, and transition signature occurs in any later role is removed +with an explicit exclusion reason. The ordinary fail-closed index builder then +reruns its source-overlap and near-duplicate audit over the filtered set. Only +eligible train fragments can remain in the persisted index. + +Coverage is aggregated by symbol, session, epoch, event state, volatility, +activity, spread, weekday, and split. For every retained stratum, withheld +queries publish query counts, match/refusal status, backoff counts, and actual +backoff rates. A zero-distance query for an impossible epoch must return +`no_supported_cell`; the builder refuses an emitted patch in that case. + +## Candidate corrections and qualification + +The provisional benchmark exposed five implementation seams that fixtures had +not exercised: + +1. long source fragments could not support short real anchor gaps; +2. uniformly spaced output discarded the source event clock; +3. candidate event-state labels did not reflect actual bid/ask transitions; +4. rounded bid-only and ask-only marks could collapse into unchanged quotes; +5. unbounded full-library rescans made a period-scale qualification process + appear to crash. + +The qualified generator cycles the selected fragment's empirical gap weights +inside each anchor interval, materializes bid-only/ask-only/both/unchanged +marks after rounding, preserves the linear internal transform seams and raw +anchors, and recomputes benchmark transition state from the merged quote +stream. Price residual amplification is bounded to `[0.05, 1.0]`. The compact +256-fragment shortlist keeps benchmark memory and runtime bounded while the +manifest still publishes support and omissions from the complete source pool. + +Qualification reuses the unchanged policy frozen for #463. It runs the real +validation and final-holdout windows twice, compares the complete candidate +report for deterministic replay, and records transparent dense, degraded, +interpolation, and negative controls. Closure requires the empirical candidate +to be non-provisional and promotion eligible, with zero failures, refusals, +anchor violations, and unsupported emissions. + +## Installed command and artifacts + +```console +histdatacom analytics modern-reference-motif-library \ + --source-root data/ASCII/T \ + --definition data/.histdatacom/analytics/feed-epochs-v2-issue-460/feed-epochs-v2-definition.json \ + --market-context-corpus .histdatacom/market-context-461-final-v4/market-context-corpus-9255f8c39f999b7a54e41a59a6f1d96f02e897af8383795e464a2f8738b08e00.json \ + --cftc-positioning-corpus data/.histdatacom/analytics/cftc-positioning-issue-468-final/cftc-positioning-corpus-887a47840090cdab1982fe910a4bdf8c1fcc9af256ab687bceae1b8dd1cbd3e0.json \ + --benchmark-manifest data/.histdatacom/analytics/reverse-degradation-benchmark-issue-463-final-v5/reverse-degradation-manifest-d1ddf45d68ade8c1ba4abc3df5a60a26483bb3eab950d4c29f53709e9214ed24.json \ + --artifact-dir data/.histdatacom/analytics/modern-reference-motif-issue-464-final-v4 +``` + +The command writes six content-addressed JSON artifacts: + +- production index; +- source/feature/split manifest; +- leakage audit; +- support and backoff coverage; +- frozen-policy benchmark qualification; and +- runtime, peak-memory, source, scratch, and compact-storage audit. + +`read_modern_reference_motif_index()` and +`read_modern_reference_motif_artifact()` are installed hash-verifying readers. +Existing content-addressed files are reused only when their bytes are exact; +otherwise the writer fails. + +## Issue #464 reference campaign + +The authoritative local evidence directory is +`data/.histdatacom/analytics/modern-reference-motif-issue-464-final-v4`. +The library ID is +`modern-reference-motif-library:sha256:a723fb5ce639dd4363a02e5680f5b640f53d9eb4fe5652fd13834174870b3e0e` +and its installed index ID is +`reference-motif-index:sha256:b5d5e7d9580fac375c42677fe5d03be96fafc190f799364a52566af7aa5a2589`. + +The build verified 24 monthly sources containing 49,630,455 real Arrow rows +and 1,389,795,816 bytes. From 4,572 projected three-event windows it removed +28 train shapes found in later roles, audited 69,754 post-exclusion +comparisons with zero findings, and retained 256 train fragments. No +calibration, validation, or final-holdout fragment is persisted. All 1,692 +withheld queries matched through the declared `symbol_epoch_session` backoff; +2,288 source windows carry a matched market event state, and retained symbol +support is 78 EURGBP, 85 EURUSD, and 93 GBPUSD fragments. + +The empirical candidate is non-provisional and passed every frozen hard gate +with zero failures, refusals, raw-anchor violations, and unsupported emissions. +Worst real-window errors were `0.17317708333333334` for event count, +`0.2850022470073947` for inter-arrival shape, `0.6261398637127946` for path +variation, `0.11612903225838411` for spread tail, and +`0.5156356696373855` for update transitions. Deterministic replay, variable +cardinality, internal boundary continuity, immutable anchors, and unsupported +refusal all passed. + +The complete build and two benchmark replays took 43.596332 seconds at +550,666,240 bytes peak RSS, used no scratch, and wrote 856,512 compact bytes. +The six exact artifact SHA-256 values are: + +| Artifact | SHA-256 | +| --- | --- | +| index | `048dd46deabf66643fc55b9cb2a996828c88f8c099a9d9573a4d29a632bce9a3` | +| manifest | `3ab008c97a45d5930f958d2cfd7cb8d2b9d14fc6fdcb5fd84ab73440bc1adb36` | +| leakage audit | `97936f21c74703a42e1074835bea05a5c7ac1320e3df0104602d46ff9717fc2e` | +| coverage | `8b12a79da78e57ff4422fa7f394564be03d4f6c414d9a3061b3075bc9988b3ce` | +| qualification | `fb6fac22d7f1c8a9e71a4c55160147b285c643e03cc9ccdb165b77bd8f6dc927` | +| resource audit | `772bbd1c533b332edb1afbd883364e50cf09608cc3b7a3594b28881bd1920bea` | + +## Nonclaims + +- The library does not identify the exact historical ticks that were missing. +- It does not use bars, synthetic OHLC, broker style, or a neural generator. +- Event/CFTC conditioning is retrieval context, not a causal trading claim. +- A benchmark gate pass does not select a strategy or authorize publication. diff --git a/docs/observation-operator-contracts.md b/docs/observation-operator-contracts.md new file mode 100644 index 00000000..26b60c96 --- /dev/null +++ b/docs/observation-operator-contracts.md @@ -0,0 +1,290 @@ +# Historical feed-observation operators + +Historical feed-observation operators describe how an underlying market-event +surface becomes the sparse, quantized, filtered feed preserved by a +technological epoch. They model delivery technology, not market-price +dynamics, candidate generation, or broker-specific style. + +The operator layer consumes stability-passing v1 or active-time v2 +[`feed-epoch definitions`](feed-epoch-contracts.md) and bounded, +provenance-bearing fit evidence. It never persists the 521-column analytical +frame. + +## Identity boundary + +`SyntheticEventV1` remains the immutable market-event identity. Quantizing its +timestamp or price in place would silently create a different event while +retaining misleading generator/source lineage. Version one therefore uses a +separate observation boundary: + +```text +SyntheticEventV1 or another market event + -> ObservationInputEventV1 + -> ObservationOperatorV1.apply/degrade + -> ObservationOutputEventV1 +``` + +An output observation references its `source_event_id`, operator, and resolved +stratum. Its deterministic identity includes the delivered timestamp, +sequence, bid/ask, duplicate ordinal, and transformation labels. Market-event +generation and delivery observation remain independently auditable. + +## Contract surface + +| Contract | Responsibility | +| --- | --- | +| `ObservationContextV1` | Symbol, epoch/transition, state, session, and event conditioning coordinates. | +| `ObservationFitEvidenceV1` | Bounded parameter values, uncertainty, support, basis, provenance, period, and source hash. | +| `ObservationParameterEstimateV1` | One fitted value with combined support, uncertainty, estimation bases, and evidence IDs. | +| `ObservationOperatorFitConfigV1` | Support gates, fallback order, row/stratum limits, halo requirements, and diagnostic bounds. | +| `ObservationStratumV1` | One conditioning stratum, fitted parameters, support status, and explicit fallback keys. | +| `ObservationFitDiagnosticsV1` | Bounded support/residual summaries and sampled stratum diagnostics. | +| `ObservationOperatorV1` | Versioned, replayable operator artifact with complete epoch/config/source lineage. | +| `ObservationInputEventV1` | Market-event values plus observation-only context and optional protected-anchor status. | +| `ObservationOutputEventV1` | Operator-lineaged delivery observation without mutating the market event. | +| `ObservationCarryStateV1` | Prior delivered quote, rate bucket, outage state, and watermarks needed across windows. | +| `ObservationApplicationResultV1` | Bounded in-memory outputs, reason/fallback counts, samples, and next carry state. | + +All readers validate their schema version and recompute deterministic IDs. +Changed parameters, uncertainty, evidence, fallback order, lineage, output +values, or carry state cannot retain an earlier ID. + +## Base parameter family + +The version-one family exposes these fixed parameter names: + +| Parameter | Delivery behavior | +| --- | --- | +| `retention_probability` | General deterministic thinning. | +| `unchanged_retention_probability` | Filtering of unchanged delivered quotes. | +| `timestamp_quantum_ns` | Timestamp precision/quantization. | +| `price_precision_digits` | Bid/ask rounding precision. | +| `quote_transition_threshold` | Suppression of sub-threshold quote transitions. | +| `batch_window_ns` | Delivery batching into aligned time buckets. | +| `duplicate_probability` | Duplicate delivery of retained events. | +| `rate_cap_per_second` | Per-burst-window output cap. | +| `burst_window_ns` | Rate-cap/burst-compression bucket. | +| `quiet_gap_probability` | Deterministic outage-bucket selection. | +| `outage_window_ns` | Quiet-gap/outage bucket duration. | +| `reconnect_duplicate_probability` | Duplicate behavior on the first retained event after an outage. | + +Parameter support is independent. An unsupported parameter uses a recorded +neutral identity value: retain, do not quantize, do not duplicate, and do not +invent outages. This conservative behavior is visible in diagnostics and does +not turn an unavailable thinning estimate into a claimed observation. + +## Canonical versus paired evidence + +`ObservationFitEvidenceV1.from_feed_epoch_evidence()` projects the bounded +canonical evidence created by the feed-epoch layer. That surface supports +descriptive proxies for cadence, timestamp/price precision, stale quotes, +duplicates, burst windows, and quiet gaps. + +Historical sparse data does not contain the unobserved dense-event +denominator needed to identify a true thinning probability. Canonical +projection therefore records `retention_probability=1` with zero support, +the basis `identity_without_dense_denominator`, and uncertainty spanning the +admissible probability range. It cannot masquerade as a directly measured +retention rate. + +Paired calibration or controlled-degradation evidence may supply supported +input/output parameters. Every parameter still carries its own: + +- support count; +- lower and upper uncertainty bounds; +- estimation basis; +- exact evidence IDs; and +- bounded feature/source provenance. + +The implemented +[`reverse-degradation-benchmark-contracts.md`](reverse-degradation-benchmark-contracts.md) +layer owns the full experiment, splits, controls, and scorecards. This module +supplies the deterministic operator interface and controlled fixtures exercised +by that benchmark. + +## Real-evidence calibration v2 + +`ObservationCalibrationCampaignV2` makes a stricter claim than a bare v1 +operator. It uses the last stable technology epoch as the dense reference, +estimates earlier symbol/epoch targets relative to its active-time denominator, +and evaluates the fitted operator on chronologically blocked calibration, +validation, and final-holdout windows. Split periods must be distinct and span +three years. Dense and degraded rows stay process-local; only bounded aggregate +window evidence is persisted. + +The calibration surface conditions retention by update type and session. It +records timestamp precision and quote-step support directly, while spread, +activity, and volatility conditioning remain explicitly unsupported until +conditional epoch denominators exist. Batching versus timestamp quantization, +and rate cap versus retention, are recorded as bounded confounded mechanisms. +Archive gaps, outage duration, and reconnect identity remain unsupported rather +than being inferred from silence. + +Every target includes all twelve parameter names with a value, interval, +support status, estimation/refusal reason, source evidence IDs, hashes, and +distinct mechanism diagnostics. The nested `ObservationOperatorV1` contains +only supported parameters; its conservative neutral behavior does not promote +an unsupported mechanism into a calibration claim. + +Application readiness is enforced by +`ObservationCalibrationCampaignV2.require_application_ready()`. The default +required mechanisms include retention, unchanged filtering, timestamp and +price precision, duplicates, and burst cadence. A caller requesting a bounded +or unsupported mechanism is rejected. A retention estimate whose only basis is +`identity_without_dense_denominator` is always rejected, even if an older bare +v1 operator can still replay it. + +Use the artifacts emitted by `feed-epochs-v2` directly: + +```sh +histdatacom analytics observation-calibrate-v2 \ + --definition data/.histdatacom/feed-epochs-v2/feed-epochs-v2-definition.json \ + --evidence data/.histdatacom/feed-epochs-v2/feed-epochs-v2-evidence.json \ + --artifact-dir data/.histdatacom/observation-calibration-v2 +``` + +The command verifies every selected dense cache's byte size and SHA-256 against +the epoch evidence, then writes separate campaign, operator, and fit-evidence +JSON files. Per-window event count and per-source byte limits are explicit in +the profile and campaign resource evidence. Campaign wall-clock and peak-memory +ceilings are also hard readiness gates; the measured runtime and process peak +are persisted beside those limits. +It exits with status 2 when a final holdout or readiness gate fails, while +retaining the failed evidence for audit. Existing v1 operator artifacts remain +readable; v2 feed-definition identities are accepted without rewriting their +schema or IDs. + +## Explicit conditioning and fallback + +Fit evidence is aggregated through this fixed hierarchy: + +```text +symbol + epoch + state + session + event + -> symbol + epoch + state + session + -> symbol + epoch + state + -> symbol + epoch + -> epoch + -> global +``` + +Each stratum is `ready`, `limited`, or `unsupported` according to the versioned +support gates. Unsupported specific strata are retained as evidence and name +their fallback keys. Application walks the same hierarchy and fails when no +usable parent exists. Sparse strata are never silently treated as if their +specific parameters were well supported. + +Technological epochs and market states remain separate coordinates. A feed +epoch cannot be relabeled as a volatility regime, and an uncertain epoch +boundary remains an explicit transition label supplied by the epoch artifact. + +## Forward application and controlled degradation + +`ObservationOperatorV1.apply()` is the forward reconstruction interface. +Protected historical anchors are retained with their exact source timestamp, +bid, and ask. Ex-ante mode rejects protected anchors because they would expose +future historical evidence. + +`ObservationOperatorV1.degrade()` is the generator-neutral benchmark +interface. It ignores input anchor flags and protects only explicitly named +event IDs, allowing modern holdout events to be degraded without pretending +they are immutable historical anchors. + +Both methods: + +- sort input independently of caller order; +- select thinning/outage/duplicate decisions by stable hashes, not mutable RNG + state; +- retain one source event ID for every output observation; +- validate positive bid/ask domains and `ask >= bid`; +- assign deterministic within-timestamp delivery sequences; +- emit bounded reason, fallback, and sample diagnostics; and +- enforce input/output amplification limits before returning work. + +The operator never changes a market price path to improve fit. It may retain, +filter, quantize, batch, duplicate, or suppress delivery observations only +through its declared parameters. + +## Streaming windows, alignment, and carry + +Application accepts an existing `ReconstructionWindowV1`. Only source events +owned by `[core_start_ns, core_end_ns)` can emit output for that call. Halo rows +may seed state but do not acquire output ownership. + +Timestamp and batch quantization use Unix-epoch-aligned floor buckets. Window +core boundaries must be divisible by every active timestamp/batch quantum so +an event cannot be shifted into another window. Misaligned work fails before +application. + +The artifact declares `required_left_halo_ns`, whether carry is required after +the first window, and the exact carry fields. The bounded carry state records: + +```text +last_source_time_ns last_observed_time_ns last_bid last_ask +rate_bucket_start_ns rate_bucket_count +outage_bucket_start_ns outage_active reconnect_pending +``` + +The first source window may start without carry. Version one requires every +later window to provide the matching operator/symbol carry artifact; a finite +time halo cannot prove that it contains the last delivered quote across an +arbitrarily long historical gap. The declared halo remains explicit contract +metadata for operator inputs, but it is not allowed to masquerade as complete +state. Partition count, caller input order, and retry count do not participate +in observation decisions. + +## Resource bounds + +Version one admits at most 4,096 fit-evidence records, 512 strata, and 250,000 +input events per application. A source event can produce at most three output +observations, and diagnostic samples stop at 128 records. Operator artifacts +are rejected above 64 MiB before JSON parsing. Timestamp and batch quanta are +bounded to one day; burst and outage durations are bounded to 31 days. Fit +configuration may lower these ceilings but cannot raise them. + +`ObservationApplicationResultV1` contains events and is an in-memory data-plane +object. It must not be placed into Temporal workflow history. Period-scale +workers write event batches and larger carry payloads outside history, then +exchange the existing compact `ArtifactRef` contracts. + +## Artifacts and replay + +`write_observation_operator()` writes canonical JSON and returns an +`ArtifactRef` with byte size, SHA-256, schema version, operator ID, and epoch +definition ID. `read_observation_operator_artifact()` verifies size, bytes, +metadata, every nested deterministic ID, and the internal lineage hash before +returning an operator. + +Operator lineage includes: + +- the exact feed-epoch definition ID; +- fit-config ID; +- every fit-evidence ID; +- every source artifact hash and its basis; and +- the complete bounded source list used by the fit. + +The operator can therefore be replayed from its artifact and manifest hashes +without the original in-memory fingerprint panel or enriched frame. + +The durable reconstruction manifest binds the operator ID as a semantic +configuration input. Observation outputs and fitting panels remain +reconstructable intermediates; the final accepted synthetic tick and compact +operator sidecar are the intended durable products. + +## Issue boundaries + +- #434 supplies stability-passing technological epochs and canonical evidence. +- #435 owns the contracts and deterministic fit/apply/degrade implementation + documented here. +- #436 implements reverse-degradation splits, controls, metrics, and promotion + scorecards over this operator. +- #439 owns variable-cardinality market-event proposal, not delivery filtering. +- #443–#445 own live broker capture, broker fingerprints, and broker-style + transfer. +- #446 implements final atomic Parquet/manifests; #447's implemented control + plane owns production Temporal orchestration; see + [`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + orchestration. + +Changing a required field, parameter meaning, fallback level, anchor policy, +output identity, carry field, or application decision requires a new schema +version and contract class. diff --git a/docs/reconstruction-activity-semantics.md b/docs/reconstruction-activity-semantics.md new file mode 100644 index 00000000..e201ffce --- /dev/null +++ b/docs/reconstruction-activity-semantics.md @@ -0,0 +1,211 @@ +# Reconstruction activity and liquidity-proxy semantics + +## Purpose + +The accepted reconstruction product contains quote events. It does not contain +centralized spot-FX transactions, and the historical HistData tick source does +not provide a defensible traded-volume field. This contract turns event +cardinality, timing, quote transitions, and spreads into explicit activity +evidence without relabeling any of those observations as traded volume. + +The implementation lives in `histdatacom.synthetic.activity`. It consumes the +same final `SyntheticEventV1` rows that are stored by reconstruction +persistence. The 26-column event schema remains unchanged. Activity results are +compact derived metadata, while the 521-column analytical/training frame +remains an ephemeral input surface. + +## Contracts + +| Contract | Responsibility | +| --- | --- | +| `ReconstructionActivityPolicyV1` | Selected origin scopes, honest volume state, rounding, symbol/slice/provenance limits, and payload limit. | +| `ReconstructionActivityMetricV1` | One value with a fixed name, unit, aggregation rule, scientific semantics, support, optional confidence, and limitations. | +| `ReconstructionActivitySliceV1` | One symbol plus observed, synthetic, or merged population with metric coverage and bounded lineage. | +| `ReconstructionActivityBenchmarkEvidenceV1` | Activity-specific extraction from the shared reverse-degradation scorecard. | +| `ReconstructionActivityManifestV1` | Information-mode-bound collection of slices, policy, source content identity, optional product/calibration identity, benchmark evidence, and derived-bar rules. | + +All version-one identities are SHA-256 hashes of canonical JSON. Readers +recompute policy, slice, benchmark-evidence, and manifest identities. Derived +claims such as the event schema version, metric definitions, bar semantics, +absence of a centralized-volume claim, and absence of an automatic winner are +also verified on read. + +## Event and volume boundary + +One durable `SyntheticEventV1` row is one delivered quote update. Exact +duplicate or stale quote deliveries still count as activity because the feed +delivered them. A separate transition metric distinguishes rows whose bid or +ask changed from rows whose quote remained stale. + +The volume state is always explicit: + +| State | Meaning | +| --- | --- | +| `unavailable` | No supported size or volume value exists. This is the default for the current final product. | +| `omitted` | A supported upstream value was intentionally excluded from this output. | +| `observed_source_size` | A future source supplies a documented size with stable units and provenance. The current event schema cannot claim this state. | +| `broker_supplied_size` | A broker supplies quoted or broker-specific size with documented semantics. The current event schema cannot claim this state. | +| `synthetic_activity_proxy` | A model-derived activity proxy is explicitly labeled as a proxy and is never called traded volume. | + +`observed_source_size` and `broker_supplied_size` fail closed when aggregating +the current final event schema because it contains no size fields. Broker +capture retains any honest quoted/broker-specific size evidence separately; it +is not silently copied into reconstructed history. A future per-event size +requires a new event-schema version and its own support, units, provenance, and +validation contract. + +## Metrics + +Every slice carries the complete fixed metric set: + +| Metric | Unit | Definition | Projection rule | +| --- | --- | --- | --- | +| `event_count` | event | Number of durable quote-event rows. | Sum. | +| `quote_update_count` | quote update | Number of delivered quote updates; equal to event count in v1. | Sum. | +| `exposure_duration_ns` | nanosecond | Last event time minus first event time in the slice. | Recompute from target interval event bounds. | +| `tick_intensity_per_second` | event/second | Event count divided by exposure duration. Unavailable for zero-duration slices. | Recompute count divided by duration. | +| `mean_interarrival_ns` | nanosecond | Mean adjacent event-time difference within the slice scope. | Transition-support-weighted mean. | +| `min_interarrival_ns` | nanosecond | Minimum adjacent event-time difference. | Minimum. | +| `max_interarrival_ns` | nanosecond | Maximum adjacent event-time difference. | Maximum. | +| `price_change_count` | transition | Adjacent quote transitions whose bid or ask changed. | Sum with prior-quote boundary carry. | +| `stale_quote_count` | transition | Adjacent quote transitions whose bid and ask were unchanged. | Sum with prior-quote boundary carry. | +| `stale_quote_rate` | ratio | Stale transitions divided by all quote transitions. | Recompute from transition counts. | +| `mean_spread` | price | Mean ask-minus-bid spread. This is a liquidity proxy, not volume. | Event-support-weighted mean. | +| `min_spread` | price | Minimum ask-minus-bid spread. | Minimum. | +| `max_spread` | price | Maximum ask-minus-bid spread. | Maximum. | +| `mean_event_confidence` | probability | Mean populated per-event confidence. Missing confidence is not filled. | Confidence-support-weighted mean. | + +Counts and durations remain integers. Floating-point values are rounded only +when the immutable metric is finalized. Unavailable values carry zero support +and an explicit limitation rather than a zero masquerading as a measurement. + +## Origin scopes and reconciliation + +The default policy emits: + +- `observed`: immutable historical anchors only; +- `synthetic`: accepted generated events only; +- `merged`: the final delivered observed-plus-synthetic product. + +For every symbol, the merged event count must equal observed plus synthetic +counts when those origin slices are present. Events are never mutated or +copied into the manifest. Each slice retains only bounded sets of source +versions, generator IDs and versions, generator configuration IDs, reference +and motif IDs, feed-epoch IDs, broker profile IDs, constraint-set IDs, and +stream IDs, plus a hash of canonical event content. Exceeding a provenance +bound refuses the summary instead of silently truncating lineage. + +## Information modes + +Every activity manifest binds to `information_manifest_id` and +`InformationMode`: + +- `ex_post_reconstruction` rejects an `as_of_ns` value; +- `ex_ante_simulation` requires `as_of_ns`. + +The as-of value describes what information was available to the reconstruction +or simulation. It is not an upper bound on simulated event timestamps. The +existing information audit remains authoritative for motif, context, +calibration, and future-anchor use. Activity aggregation does not weaken that +audit and does not infer an information mode from event times. + +## Bounded streaming + +`summarize_reconstruction_activity()` consumes ordered event iterables using +constant numeric state per symbol/scope. Events may be interleaved across +symbols, but positions must increase strictly within each symbol. + +`summarize_reconstruction_activity_streams()` consumes compatible +`SyntheticEventStreamV1` objects and retains their stream identities. + +`summarize_committed_reconstruction_activity()` first verifies the atomic +publication and then requests only these 19 Parquet columns: + +```text +event_id, origin, symbol, event_time_ns, event_sequence, bid, ask, +run_id, ensemble_member_id, source_version_id, generator_id, +generator_version, generator_config_id, reference_id, motif_id, feed_epoch_id, +broker_profile_id, constraint_set_id, confidence +``` + +Arrow batches default to 8,192 rows and are configurable. The implementation +does not load the 521-column frame, retain event rows, or produce a tick-sized +side table. Symbol, slice, provenance-value, and serialized-payload limits fail +closed. + +Example: + +```python +from histdatacom.synthetic import ( + InformationMode, + summarize_committed_reconstruction_activity, +) + +activity = summarize_committed_reconstruction_activity( + "archive/reconstruction-products/.../commits/.../manifest.json", + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:...", +) +``` + +## Reverse-degradation and calibration evidence + +`reconstruction_activity_benchmark_evidence()` consumes the existing +`ReverseDegradationScorecardV1`. It requires activity-relevant evidence in +every score slice: + +- event-count relative error; +- intensity relative error; +- interarrival histogram L1 distance; +- burst-rate absolute error; +- quiet-rate absolute error; +- spread-mean relative error; +- spread-histogram L1 distance. + +The derived evidence reports per-metric support and mean errors, restoration +gain versus the degraded input, promotion-eligible count, calibration-support +count, and execution failures. It carries scorecard and candidate-score IDs. +It cannot name a winner. Existing hard historical constraints and benchmark +promotion gates remain authoritative. + +Event confidence and ensemble calibration are distinct. Missing per-event +confidence remains missing. A manifest may bind a separate immutable +calibration report ID; benchmark evidence records whether candidate uncertainty +support was present. + +## Derived-bar handoff + +The manifest embeds exact #18 projection semantics: + +- event and quote-update counts sum; +- duration is recomputed from events in the target bar rather than blindly + summing overlapping durations; +- tick intensity is recomputed from the target count and duration; +- transition counts sum only with the preceding-quote boundary carry needed to + prevent window double counting; +- stale rate is recomputed from transition counts; +- mean spread is weighted by event support; +- volume remains unavailable unless separately sourced. + +Derived bars must choose observed-only, synthetic-only diagnostic, or merged +product scope explicitly. Empty intervals, market closures, duplicate +timestamps, partial bars, and resumption identity remain #18 responsibilities. +No raw vendor M1 input is introduced by this activity layer. + +## Failure conditions + +Aggregation or deserialization fails closed for: + +- unsupported or relabeled schema versions; +- fabricated centralized-volume or automatic-winner claims; +- source/broker size states without size-bearing event fields; +- run/member mismatches; +- non-increasing positions within a symbol; +- unsupported metric names, units, semantics, or aggregation rules; +- missing metric coverage; +- origin/merged count mismatch; +- reversed time bounds; +- non-finite or negative metrics; +- confidence outside `[0, 1]`; +- unbounded symbol, slice, provenance, batch, or payload sizes; +- benchmark slices missing required activity metrics. diff --git a/docs/reconstruction-certification-contracts.md b/docs/reconstruction-certification-contracts.md new file mode 100644 index 00000000..f38792fc --- /dev/null +++ b/docs/reconstruction-certification-contracts.md @@ -0,0 +1,168 @@ +# Reconstruction Certification Contracts + +Certification is the fail-closed evidence boundary for the +EURUSD/GBPUSD/EURGBP reconstruction product. It aggregates compact reports; it +does not retain tick rows, analytical frames, model objects, candidate batches, +or rejected events and it does not certify output because it looks plausible. + +## Current and legacy policy versions + +`ReconstructionCertificationPolicyV2` is the issue #449 release contract. It +fixes: + +- product version `2.1.0`; +- the exact EURGBP/EURUSD/GBPUSD instrument set and ASCII tick scope; +- common source support beginning at `200203` and an execution-time end month; +- delivery mode `modern_reference` and claim `unconditioned_reference`; +- source-readiness, scientific, operational, reporting, repository, and release + checks; +- explicit resource and candidate-amplification ceilings; and +- coverage exactly once at the `dev`-to-`main` promotion boundary. + +Broker adaptation is excluded from V2. A V2 policy cannot contain a +broker-named check or required artifact, the evaluator rejects broker-named +evidence, and the dossier always records `broker_specific_claim: false`. +Broker capture, fingerprinting, and transfer remain optional, separately +qualified extensions. + +`ReconstructionCertificationPolicyV1` is retained unchanged for replay of +previously published evidence. It requires a broker fingerprint and must not be +used for the modern-reference #449 campaign. V2 was introduced rather than +changing V1 in place because removing a broker field or changing evidence kinds +would silently change the meaning of old policy identities. + +## Gate mapping + +The V2 factory `modern_reference_triangle_certification_policy()` binds every +live #449 seam: + +| Gate group | Required evidence and outcome | +| --- | --- | +| `identity-and-anchors` | Inventory readability, dimensions and hashes reconcile; no duplicate dimension, raw-hash mismatch, observed-anchor change, or missing synthetic lineage exists. | +| `information-safety` | Market-context and CFTC corpora are valid; every ex-post or supported ex-ante claim has a zero-violation point-in-time audit. | +| `reverse-degradation` | The benchmark corpus predates candidate results, thresholds are predeclared, blocked holdouts pass, and negative controls fail as expected. | +| `conditioned-scorecards` | Feed epochs, observation operators, and motifs are valid; all required strata exist and product/benchmark tolerances pass. | +| `cross-currency` | Triangle, inverse, synchronization, and stale-alignment checks pass before and after identity delivery. | +| `ensemble-evidence` | Calibration, diversity, refusal, unsupported-region, between-seed, and between-window uncertainty are reported. | +| `product-reconciliation` | Final ticks, activity, and bars reconcile; the nonclaim is published; full-range preflight, representative windows, a substantial multi-period run, and CLI/API evidence-chain parity pass. | +| `failure-resume` | Mid-run failure resumes with no missing/duplicate partition and cancellation publishes no partial partition. | +| `replay` | Logical product hashes agree across clean replay and supported concurrency. | +| `resources` | Peak memory, scratch, runtime, candidate amplification, storage, and final-row evidence meet frozen bounds. | +| `negative-tests` | Corruption, stale artifacts, missing context, invalid information mode, quota overflow, and partial groups fail closed. | +| `strategy-sensitivity` | Uncertainty is reported and no automatic winner is selected. | +| `dossier-publication` | Human methodology/limitations and the machine evidence manifest are published. | +| `repository-gates` | Test dependencies are installed, the full plain suite and hooks pass, and promotion coverage runs exactly once. | +| `testpypi-preflight` | The local simple-registry TestPyPI preflight passes before promotion. | + +Changing the source range, evidence contract, threshold, or resource budget +changes the deterministic V2 policy ID. + +## Evidence contracts + +The bounded evaluator retains four layers: + +1. `CertificationArtifactV1` records the frozen policy identity, artifact kind, + subject identity and schema, content SHA-256, safe relative path, size, + verification state, and bounded metadata. +2. `CertificationObservationV1` records one scalar measurement and the exact + artifact evidence identities that support it. +3. `CertificationCheckResultV1` applies one small comparator: equality, + less-than-or-equal, greater-than-or-equal, true, false, or zero. +4. `CertificationGateResultV1` and + `ReconstructionCertificationDossierV2` aggregate results without weakening a + failure or missing observation. + +V2 requires the evidence-kind set for a check to match its policy exactly. +Unverified, foreign-policy, missing, extra, or broker-named evidence cannot +support a pass. Booleans are not numbers, nonfinite numbers are rejected, and +exact equality requires matching scalar types. + +## Executable campaign + +`ModernReferenceCertificationCampaignSpecV1` freezes policy budgets, artifacts, +scalar-extraction locations, methodology, limitations, and whether execution is +the explicit promotion boundary. Every artifact declaration supplies: + +- a campaign-local evidence key; +- the producer artifact kind and path; +- the expected file SHA-256; +- the expected subject schema version; +- the expected subject identity and JSON pointer that contains it; and +- a safe dossier-relative path. + +Every observation supplies a measurement evidence key, a JSON pointer, and the +exact supporting evidence keys. It does not contain an `actual` value. +`run_modern_reference_certification_campaign()` reads the file, verifies its +hash, schema and subject identity, resolves the JSON pointer, verifies that the +result is a scalar, and only then creates a certification observation. + +This prevents a handwritten summary boolean from standing in for a producer +report. Producer-specific contracts still own how reports calculate their +metrics; the campaign owns identity, extraction, aggregation, and publication. + +Run the installed public surface with: + +```sh +histdatacom reconstruction --json certify \ + --spec evidence/campaign.json \ + --output-directory evidence/dossier +``` + +The campaign automatically publishes and binds its frozen machine manifest and +methodology report. A normal `dev` campaign rejects +`coverage_promotion_run_count`; only a spec explicitly marked as a promotion +boundary can carry that observation. + +## States and exit behavior + +A V2 dossier has four states: + +- `incomplete`: required evidence is missing or a blocking limitation remains; +- `failed`: a measured value violates policy; +- `ready-for-promotion`: every check except promotion-only coverage passes; and +- `certified`: every scientific, product, repository, coverage, and TestPyPI + check passes with no blocking limitation. + +A measured failure outranks missing work. A known limitation can narrow a claim +but cannot replace immutable-anchor, information-safety, benchmark, +operational, or release evidence. + +The CLI returns `0` for `ready-for-promotion` or `certified`, `3` for an +incomplete campaign, and `5` for measured certification failure. Malformed or +changed campaign inputs return the existing invalid-plan category. + +## Publication and replay + +`write_modern_reference_reconstruction_certification_dossier()` atomically +writes canonical JSON and deterministic Markdown, immediately reads the JSON +back through `ReconstructionCertificationDossierV2`, and returns strong +`ArtifactRef` values. Campaign execution also writes: + +- `evidence/campaign-spec.json`; +- `evidence/methodology.json`; and +- `campaign-result.json`. + +The dossier identity covers policy, artifacts, results, methodology, +limitations, state, delivery claim, and fixed trust assertions. It always +states that event rows and analytical-frame columns are not inline, no broker +claim or historical-truth claim is made, no automatic winner is selected, and +no investment recommendation is made. + +## Required release sequence + +1. Re-inventory and hash the complete three-symbol source scope. +2. Freeze V2 policy, scientific thresholds, and resource budgets. +3. Verify all dependency artifacts and point-in-time coverage. +4. Execute ex-post reconstruction and each separately supported ex-ante view. +5. Produce real holdout, conditioned, cross-currency, ensemble, product, + activity, bar, strategy, fault, replay, resource, negative-test, and public + interface reports. +6. Run the full plain suite and repository hooks without coverage. +7. Publish to TestPyPI from `dev` and pass the local simple-registry preflight. +8. Execute the campaign and publish a `ready-for-promotion` dossier. +9. During explicit `dev`-to-`main` promotion, run coverage exactly once, + publish the final `certified` dossier, and publish the same artifact to PyPI. + +Fixture dossiers and campaign tests prove contract, extraction, comparison, +serialization, and publication semantics. They cannot certify historical +output or replace a real reconstruction campaign. diff --git a/docs/reconstruction-ensemble-calibration-contracts.md b/docs/reconstruction-ensemble-calibration-contracts.md new file mode 100644 index 00000000..7d735d73 --- /dev/null +++ b/docs/reconstruction-ensemble-calibration-contracts.md @@ -0,0 +1,165 @@ +# Reconstruction Ensemble Calibration Contracts + +Reconstruction ensembles represent uncertainty about missing market events +without promoting one generated path to historical truth. The version-one +layer plans reproducible members, evaluates their interval coverage through +reverse degradation, diagnoses non-substantive diversity, retains only a +bounded subset, and hash-gates on-demand regeneration of omitted members. + +Dense event rows remain process-local or behind streaming artifact references. +The durable plan, calibration samples, storage estimate, report, and +regeneration request contain bounded metadata and aggregate evidence only. + +## Deterministic plan and member identity + +`plan_reconstruction_ensemble()` binds the complete plan to: + +- normalized symbols; +- exact source artifact IDs and SHA-256 digests; +- exact generator, carving, reconciliation, and ensemble configuration IDs and + SHA-256 digests; +- the versioned `EnsembleCalibrationConfigV1`; and +- a semantic base seed. + +Member IDs are derived from those inputs plus a stable one-based ordinal. +Member seeds are then derived through `ReconstructionRunV1.seed_for()`. Worker +count, retry number, partition layout, row order, scratch path, and retention +rank are absent from member identity. Replaying the same source and config +hashes produces the same plan, member IDs, and seeds; a hash mismatch fails +before regeneration. + +The configuration freezes member/retention counts, forecast horizons, nominal +and minimum achieved coverage, minimum fit support, collapse and false- +diversity tolerances, failure penalty, byte estimates, rounding, sample/slice +limits, and final payload bytes. + +## Reverse-degradation calibration samples + +`benchmark_ensemble_calibration_sample()` adapts one complete +`BenchmarkCandidateWindowV1` set from the #436 reverse-degradation harness. A +sample belongs to one exact stratum: + +```text +feed epoch, session, event state, symbol, horizon, sparsity +``` + +It records compact reference and member metrics for event count, observed +duration, mean inter-arrival, mean spread, midpoint path range, endpoint +midpoint, and downstream sensitivity. It also records a logical-content hash +for each completed member. No event rows are serialized into the sample. + +Unattempted work, failed execution, hard-constraint violations, empty streams, +and missing downstream sensitivity become explicit refused or failed member +results with bounded reason codes. Reports preserve attempts, completions, +refusals, failures, and their rates by split and exact stratum. + +## Calibration and holdout semantics + +Only `validation` samples fit interval adjustments. For each metric and +stratum, the engine forms the empirical member interval at the configured +nominal coverage, measures how far the withheld reference falls outside it, +and selects a deterministic finite-sample adjustment from those validation +nonconformity scores. + +Only `final_holdout` samples measure the resulting raw and adjusted coverage. +Each `EnsembleMetricCalibrationV1` reports fit/evaluation support, adjustment, +covered counts and rates, raw/calibrated widths, median error, and a calibrated, +miscalibrated, or insufficient-support status. An absent final-holdout cell or +insufficient validation support cannot claim calibration. + +The named confidence quantity is: + +```text +finite-sample-interval-coverage-v1 +``` + +Its scope is an exact stratum, metric, and horizon summary. It is not a +per-event probability, probability that a generated tick is historically +correct, or guarantee beyond the measured benchmark surface. + +## Substantive diversity + +`benchmark_logical_content_sha256()` and +`ensemble_logical_content_sha256()` hash ordered market content after removing +member, run, seed, source-row, and lineage identity. Input row order does not +change the hash. + +For every split/stratum cell, completed member pairs are checked for: + +- **collapse**: identical logical market content despite different members; +- **false diversity**: different content hashes with metric distance at or + below the configured tolerance; and +- substantive diversity: distinct content with measurable metric separation. + +Collapse or false-diversity rates above the versioned limits block calibrated +status. Merely changing IDs, seeds, or row ordering never establishes +diversity. + +## Representative member, not winner + +Validation samples alone rank members by normalized distance from the +validation ensemble median plus an explicit refusal/failure penalty. The first +eligible member is a compact representative primary and must be retained. This +selection is labeled: + +```text +representative_member_not_historical_truth +``` + +The report always emits `automatic_winner=false`, `winner_member_id=null`, and +`default_generator_id=null`. Final-holdout results do not choose the primary, +and the report does not select a production generator. + +## Storage and regeneration + +`estimate_reconstruction_ensemble_resources()` accounts for candidate events +across every member, peak member memory, all-member scratch, and the largest +configured retained-member outputs. It delegates refusal to the existing +`ReconstructionStoragePolicyV1` preflight, so amplification, memory, scratch, +output, batch, and retained-member quotas remain centralized. + +Only the configured retained members need durable final artifacts. The report +lists the other planned members as regenerable. A regeneration request is +accepted only when: + +- the report is calibrated; +- the plan, run, config, and report identities match; +- every source and configuration hash exactly matches the frozen plan; and +- every requested member is omitted/regenerable, not retained or unknown. + +Verification requires freshly computed hashes for the source and configuration +artifacts available to the regeneration worker. Repeating the plan's own +declared hashes without checking the available artifacts is insufficient. + +The request returns stable member IDs and seeds; it does not bypass the normal +streaming, carving, synchronization, broker, validation, or publication gates. + +## Confidence boundary with motif generation + +The empirical motif generator retains `1 / (1 + match_distance)` only as +`uncalibrated-motif-match-similarity-v1` on its transformation evidence. As of +generator version 1.2.0, candidate `SyntheticEventV1.confidence` is null. Raw +retrieval similarity is not copied into a row field that could be mistaken for +calibrated confidence. + +Any future populated event-level confidence requires its own versioned, +validated quantity and scope. The ensemble report's interval-coverage evidence +does not populate pointwise tick confidence. + +## Streaming and downstream boundaries + +The ensemble layer consumes passing carved and synchronized member windows and +the reverse-degradation evidence for the same semantic configurations. It does +not persist the wide augmented analytical cache or all candidate paths. + +- #445 may apply broker delivery conditioning only after an ensemble has + trustworthy evidence and must preserve member lineage. +- #446 implements retained-member/final Parquet layout and atomic publication. +- #447's implemented control plane owns production Temporal activities, + retries, cancellation, and backpressure; see + [`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + backpressure, and regeneration execution. + +Changing member identity, calibration split use, metric meaning, diversity +meaning, confidence scope, retention semantics, or regeneration authorization +requires a new schema or engine version. diff --git a/docs/reconstruction-information-modes.md b/docs/reconstruction-information-modes.md new file mode 100644 index 00000000..4f3cdff0 --- /dev/null +++ b/docs/reconstruction-information-modes.md @@ -0,0 +1,180 @@ +# Reconstruction information modes + +The version-one reconstruction information contracts make the distinction +between historically informed reconstruction and point-in-time simulation a +machine-checkable pre-generation gate. They do not source macro/news data, +fit models, generate events, or evaluate a strategy. + +## Modes and trust boundary + +`ex_post_reconstruction` may use explicitly labeled future anchors, +full-period summaries, global normalizations, or empirical motifs when both +the input and run policy bound the required look-ahead. A passing report is +valid for historically informed reconstruction and diagnostic +counterfactuals. It is not valid for prospective simulation or a strategy +usefulness claim. + +`ex_ante_simulation` requires zero look-ahead. External and derived inputs +must have been point-in-time available when consumed, and pre-decision stages +cannot use future events, future anchors, full-period summaries, global +normalizations, or motifs selected from future observations. A passing report +opens the point-in-time simulation and strategy-usefulness gates, subject to +the declared chronological splits. + +Any violation rejects the artifact graph. `require_reconstruction_information_audit()` +raises `InformationLeakageError` before generation and carries the complete +bounded report. A rejected report is invalid for generation, validation, and +strategy claims. + +## Contract boundary + +| Contract | Responsibility | +| --- | --- | +| `ReconstructionInformationPolicyV1` | One stable mode, maximum ex-post look-ahead, fail-closed behavior, and retained-finding limit. | +| `ReconstructionInformationInputV1` | One external or derived artifact use with event, observation, availability, consumption, vintage, revision, scope, stage, reason, parent, and split metadata. | +| `ReconstructionInformationSplitV1` | One half-open train, calibration, or validation interval. | +| `ReconstructionInformationManifestV1` | One run, policy, exact window-plan identity, complete artifact-use graph, and declared split sequence. | +| `InformationAuditFindingV1` | One deterministic rule ID with bounded evidence. | +| `InformationAuditReportV1` | Acceptance, total and retained violations, truncation state, and explicit valid/invalid downstream uses. | + +The contracts are artifact-level sidecars. They do not add hundreds of +repeated fields to every synthetic event. Event rows retain `run_id`, source, +anchor, generator, motif, feed-epoch, broker-profile, and constraint lineage. +The run binds the information policy through `configuration_ids`; the audit +report then certifies the exact run, information manifest, and window plan. + +## Construction order and streaming integration + +The policy exists before the run, avoiding a circular identity between a run +and its completed input manifest: + +```python +from histdatacom.synthetic import ( + InformationMode, + ReconstructionInformationManifestV1, + ReconstructionInformationPolicyV1, + ReconstructionRunV1, + audit_reconstruction_information, + plan_reconstruction_windows, + reconstruction_information_window_plan_id, +) + +policy = ReconstructionInformationPolicyV1( + information_mode=InformationMode.EX_ANTE_SIMULATION, +) +run = ReconstructionRunV1( + symbols=("eurusd", "gbpusd", "eurgbp"), + source_version_ids=("source:sha256:historical-v1",), + configuration_ids=(policy.policy_id, "config:sha256:generator-v1"), + ensemble_member_ids=("member-000",), + base_seed=20260713, +) +windows = plan_reconstruction_windows( + run, + ensemble_member_id="member-000", + start_ns=0, + end_ns=300, + window_size_ns=100, + right_lookahead_ns=0, +) +manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id(windows), + inputs=declared_inputs, + splits=chronological_splits, +) +report = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=windows, +) +``` + +The retained-finding limit is serialized and enforced but intentionally +excluded from `policy_id`. Changing diagnostic retention therefore does not +change `run_id`, seeds, or scientific output. Mode, maximum allowed +look-ahead, fail-closed behavior, and ordered-split requirements remain +semantic policy identity. + +`right_lookahead_ns` is an information-access channel, not merely transport +tuning. The audit hashes the complete window plan and checks every window: + +- all windows must belong to the audited run; +- every ensemble member must have the same contiguous window boundaries; +- the supplied plan must match the manifest identity; +- ex-ante windows must have zero right look-ahead; +- ex-post windows cannot exceed the policy maximum. + +Left halo remains historical context and does not require look-ahead +permission. Window ownership and batch transport remain governed by the +streaming contracts; the information audit only decides what those windows +may read and what downstream claims are valid. + +## Input graph semantics + +Every input requires: + +- external or derived kind; +- consuming stage and temporal scope; +- semantic event time; +- inclusive observation start and end; +- point-in-time availability and actual use time; +- vintage ID and revision sequence; +- explicit allowed look-ahead; +- a human-readable reason for use; +- parent input IDs for derived artifacts; +- a split assignment where model fitting, calibration, validation, or + strategy evaluation requires one. + +Revisions name the superseded input. The audit checks sequence and availability +ordering and rejects a revised value used before its release in ex-ante mode. +Derived artifacts cannot become available before their parents. Missing +parents, orphan derived inputs, and graph cycles fail closed. + +The three split contracts must appear in train, calibration, validation order +and must not overlap or regress. Model-fit inputs belong to train, calibration +inputs belong to calibration, and validation or strategy-evaluation inputs +belong to validation. Assigned observation intervals must stay inside their +declared split. + +## Stable leakage rules and bounded reports + +Rule IDs are stable `INFORMATION_*` values. The audit distinguishes, among +other failures: + +- policy, run, manifest, input-mode, and window-plan mismatches; +- undeclared or excessive look-ahead; +- unavailable revisions and other not-yet-available inputs; +- future events and future observation windows; +- ex-ante future-anchor, full-period-summary, global-normalization, and motif + selection leakage; +- missing parents and invalid revision chains; +- missing, duplicate, unordered, overlapping, or misassigned splits. + +Findings are deduplicated and sorted deterministically before retention. The +report records the full violation count, retained count, and whether evidence +was truncated. Each finding also has a deterministic content-derived ID and a +hard evidence-size limit. + +## Downstream obligations + +Later reconstruction stages must add every artifact they consume or derive to +the manifest with its real availability and use times. In particular: + +- #434 must declare feed-epoch summaries and their observation intervals; +- #435 must declare fitted observation-operator inputs and train split; +- #437's implemented market-context timeline retains macro/news vintages, + target-event times, knowledge/availability times, and revision chains; its + bounded query sidecars bind exact vintages into this information graph. See + [`market-context-contracts.md`](market-context-contracts.md). +- #438 must distinguish point-in-time motif availability from an ex-post + reference library; +- #439 and #440 must require a passing report before generation or carving; +- #448 must reject strategy claims unless the report is accepted in the + appropriate mode. + +Changing required fields, rule meaning, validity claims, or identity +derivation requires a new schema version and contract class. diff --git a/docs/reconstruction-persistence-contracts.md b/docs/reconstruction-persistence-contracts.md new file mode 100644 index 00000000..87871f77 --- /dev/null +++ b/docs/reconstruction-persistence-contracts.md @@ -0,0 +1,160 @@ +# Reconstruction persistence contracts + +`histdatacom.synthetic.persistence` is the final durable boundary for the +v2.1 reconstruction product. The legacy v1 boundary accepts a fully applied +`BrokerRenderedGroupV1`; the v2 boundary accepts an explicit +`ReconstructionDeliveredGroupV1` and does not require or impersonate a broker +fingerprint. Both require exact immutable observed anchors and a successful +storage/retention preflight. They write only the 26 fields in +`SyntheticEventV1`. The 521-column analytical frame, candidate surfaces, +individual rejected rows, and broker-render workspaces remain ephemeral. + +## Transaction and layout + +One all-symbol synchronization unit is one atomic transaction. Its layout is: + +```text +reconstruction-products/ + schema=/ + run=/ + broker=/ + member=/ + group=/ + .scratch/ + publication.tmp-*/ + commits/ + / + manifest.json + symbol=eurusd/ + event_date=2023-11-14/ + part-00000.parquet +``` + +All path components are percent-encoded. The manifest stores paths relative to +its transaction directory and rejects absolute paths, traversal, alternate +separators, or paths that do not match the symbol/date partition evidence. + +Generic delivery uses the parallel v2 axis +`schema=histdatacom.reconstruction-product.v2/run=.../delivery=/member=.../group=...`. +The modern-reference profile is an identity projection with a content-bound +identity-lineage hash; broker-conditioned delivery remains an optional, +separate adapter. + +`stage_reconstruction_publication()` writes each Parquet file through a partial +name, fsyncs it, validates it, atomically publishes the compact manifest inside +the hidden transaction directory, and validates a clean replay. The final +commit revalidates every file and promotes the complete directory with one +same-filesystem `os.replace()`. Discovery scans only `commits`; a crash before +the rename therefore cannot advertise a partial synchronized group. + +The logical publication ID does not depend on scratch paths, worker attempts, +or partition bytes. Repeating the same input under the same retention plan is +idempotent. A retry with different content or physical writer settings cannot +replace the committed publication. + +The first-party v2 handler stages below the window's disposable scratch tree, +but only after verifying that scratch and the output axis share a filesystem. +Validation retains a byte-identical manifest mirror plus a compact transaction +descriptor outside the directory being renamed. A retry after rename can +therefore verify the committed output and finish the receipt/checkpoint chain +without dereferencing the vanished staging path. + +## Exact rows and immutable anchors + +Every Parquet footer is required to expose exactly +`SYNTHETIC_EVENT_ARROW_COLUMNS`, in order and with the version-one Arrow types. +Unexpected analytical or training columns fail publication. Files use Zstandard +compression, statistics, Parquet 2.6/data-page v2, page checksums, deterministic +schema metadata, and bounded row groups. + +The caller must provide the immutable observed anchors independently of the +rendered output. Publication compares the complete serialized observed rows by +event ID. Equal IDs with different bid, ask, time, source identity, or lineage +still fail. The source manifest then records counts plus hashes of exact +observed content and IDs. + +## Compact manifests + +The top-level manifest embeds bounded summaries, never event rows: + +- partition evidence: relative path, symbol/date, counts, time bounds, stream + and source IDs, row groups, byte size, byte hash, and logical hash; +- source evidence: source versions/series/periods and exact observed-anchor + counts/content/ID hashes; +- constraint evidence: generator, configuration, constraint, and feed-epoch + IDs plus compact hashes of per-event motif/reference assignments; +- quality evidence: either broker-transfer/fingerprint evidence for v1 or + generic delivery profile/mode, final validation, benchmark and leakage + artifact IDs, observed/synthetic/identity counts, identity-lineage hash, and + delivery action counts for v2; +- replay evidence: partition-independent logical hash, physical byte aggregate, + hash algorithms, writer/runtime version, compression, row-group size, and any + canonicalized metadata exclusions; and +- retention evidence: primary member, every retained member, conservative event + counts, compressed-byte estimates, manifest overhead, and the exact #432 + storage policy ID. + +Counts reconcile across partitions, source, constraints, the selected delivery +contract, and the top-level manifest. Manifest IDs cover the compact physical evidence; +publication IDs cover the logical product identity. + +## Preflight and retention + +`estimate_reconstruction_retention()` is called before event generation with +the primary member, all retained members, conservative event counts, and +expected partition count. It estimates primary bytes, all-retained bytes, and +manifest overhead and refuses before work when output bytes, retained member +count, or partition count exceed policy. Publication additionally refuses if +actual rows or partitions exceed those conservative estimates or if staged and +final bytes exceed the bound policy. + +This preflight is the final-storage complement to #442's all-member computation +and scratch estimate. Omitted ensemble members remain reproducible compact +artifacts; they are not silently materialized as final Parquet. + +## Replay, reads, and corruption + +`verify_reconstruction_publication()` checks manifest identity and location, +file sizes and SHA-256 values, exact schemas, row-group limits, origin counts, +time bounds, per-partition logical hashes, the aggregate byte hash, exact +observed-anchor hashes, and the clean partition-independent logical replay +hash. A truncated footer, changed row, extra column, moved manifest, or altered +derived count fails closed. + +`iter_reconstruction_event_batches()` uses Arrow Dataset scanners with column +projection and event-time predicates. `scan_reconstruction_events_polars()` +returns the corresponding lazy Polars plan. `reconstruction_parquet_paths()` +first prunes whole files by manifest symbol and time bounds, which also gives +DuckDB a narrow file list. The committed files are ordinary Parquet, so both +engines retain footer-statistics and column pushdown. + +`read_reconstruction_streams()` is the integrity-first replay path. It verifies +the complete publication, reconstructs exact symbol streams, and confirms the +logical hash again. + +## Cleanup and orchestration handoff + +`cleanup_reconstruction_scratch()` removes only transaction directories named +`publication.tmp-*` below a product `.scratch` directory. It rejects paths +outside the product root and never traverses into `commits`, raw caches, source +artifacts, or unrelated operator files. + +Staged and committed manifests expose strong `ArtifactRef` values. Their bytes +are identical across promotion, so they plug directly into the #432 +`running -> staged -> validated -> committed` checkpoint transitions. The +implemented #447 control plane records a committed checkpoint only after the +atomic directory promotion succeeds; see +[`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + +## Query dependency + +Arrow persistence is available through `histdatacom[arrow]`. Polars remains a +base dependency. DuckDB is an optional query/smoke dependency exposed through: + +```sh +pip install "histdatacom[query]" +``` + +InfluxDB is not the canonical archive. A serving adapter may later project +selected committed columns into InfluxDB without changing these durable +contracts. diff --git a/docs/reconstruction-plan-contracts.md b/docs/reconstruction-plan-contracts.md new file mode 100644 index 00000000..a2324c03 --- /dev/null +++ b/docs/reconstruction-plan-contracts.md @@ -0,0 +1,178 @@ +# First-party reconstruction plan + +`SyntheticInfillPlanV1` is the public planning boundary for the v2.1 tick +reconstruction product. It resolves installed scientific artifacts, inventories +immutable ASCII tick partitions, performs compatibility and information-safety +preflight, estimates resources, and emits bounded +`ReconstructionWorkflowRequestV1` batches. It does not generate reconstructed +ticks; the resulting requests are the input to the execution work tracked by +#466. + +The scientific claim is deliberately narrow: + +> Output is a plausible counterfactual ensemble conditioned on declared +> artifacts and constraints; it is not recovered historical truth. + +Observed ASCII ticks remain immutable anchors. Their bid, ask, timestamp, +monthly partition, and zero-based Arrow row ordinal are never replaced or +renumbered. Synthetic ticks may only be added around those anchors. M1, bar, +OHLC, and other aggregated products are not accepted as planning inputs; bars +are downstream projections from committed final ticks. + +## Contracts and execution handoff + +| Public API | Responsibility | +| --- | --- | +| `ReconstructionSourcePartitionV1` | Strong hash, monthly ownership, Arrow row count, timestamp bounds, feed-epoch evidence, and immutable row-identity policy for one ASCII/T cache. | +| `ReconstructionSourceInventoryV1` | Complete Cartesian inventory of the synchronized EURUSD/GBPUSD/EURGBP triangle over the selected common periods. | +| `ReconstructionPlanConfigurationV1` | Information, generation, carving, cross-currency, ensemble, storage, delivery, window, and handler policies. | +| `ReconstructionPlanExecutionManifestV1` | Content-addressed artifact graph plus durable output/checkpoint roots and disposable scratch root. | +| `ReconstructionPlanRefusalV1` | Stable, bounded reason that a planned window cannot execute. Refused windows never become workflow tasks. | +| `ReconstructionPlanResourceSummaryV1` | Source size, candidate amplification, peak memory/scratch, retained output, partition, window, member, and request estimates. | +| `SyntheticInfillPlanV1` | Deterministic top-level identity, requests, resources, refusals, delivery mode, information mode, nonclaim, and anchor policy. | +| `load_reconstruction_stage_plan()` | Public loader used by a stage handler to resolve its exact execution manifest, configuration, source inventory, and input references. | + +Every stage command carries one strong reference to the execution manifest. +The stage loader verifies that reference, reads the public configuration and +source inventory, checks the handler name and stage-specific artifact kinds, +and rejects any input not present in the execution graph. No dataframe, tick +row, or process-private configuration object is placed in Temporal history. +Artifact, output, checkpoint, and scratch roots must be non-overlapping and +must remain outside the immutable ASCII/T source tree. Execution manifests also +reject durable input artifacts placed beneath scratch, so cleanup cannot cross +an ownership boundary. +Requests are split by ensemble member and bounded window chunks because +different members share window boundaries and therefore cannot coexist in one +request whose task cores must not overlap. + +Modern-reference delivery is the default. The orchestration stage named +`broker_transfer` becomes a deterministic delivery projection with no broker +input. `broker_conditioned` is optional and fails before source scanning unless +a verified `broker_delivery_artifact_v1` is supplied. + +## Ex-post construction from installed artifacts + +This example uses the qualified artifacts produced by #460-#464 and #468. If +`start_period` and `end_period` are omitted, the builder selects the full +continuous period range common to all three symbols. + +```python +from pathlib import Path + +from histdatacom.synthetic import ( + InformationMode, + build_synthetic_infill_plan, + load_reconstruction_stage_plan, + validate_synthetic_infill_plan_for_execution, + write_synthetic_infill_plan, +) + +repo = Path.cwd() +analytics = repo / "data/.histdatacom/analytics" +motifs = analytics / "modern-reference-motif-issue-464-final-v4" +work = repo / ".histdatacom/reconstruction-plan-v2.1" + +plan = build_synthetic_infill_plan( + repo / "data/ASCII/T", + feed_epoch_definition_path=( + analytics + / "feed-epochs-v2-issue-460/feed-epochs-v2-definition.json" + ), + observation_operator_path=( + analytics + / "observation-calibration-v2-issue-462/" + "observation-calibration-v2-operator.json" + ), + market_context_corpus_path=next( + (repo / ".histdatacom/market-context-461-final-v4").glob( + "market-context-corpus-*.json" + ) + ), + cftc_positioning_corpus_path=( + analytics + / "cftc-positioning-issue-468-final/" + "cftc-positioning-corpus-" + "887a47840090cdab1982fe910a4bdf8c1fcc9af256ab687bceae1b8dd1cbd3e0.json" + ), + benchmark_manifest_path=next( + (analytics / "reverse-degradation-benchmark-issue-463-final-v5").glob( + "reverse-degradation-manifest-*.json" + ) + ), + motif_manifest_path=next(motifs.glob("modern-reference-motif-manifest-*.json")), + motif_index_path=next(motifs.glob("modern-reference-motif-index-*.json")), + motif_qualification_path=next( + motifs.glob("modern-reference-motif-qualification-*.json") + ), + motif_leakage_audit_path=next( + motifs.glob("modern-reference-motif-leakage-audit-*.json") + ), + artifact_root=work / "artifacts", + output_root=work / "output", + checkpoint_root=work / "checkpoints", + scratch_root=work / "scratch", + information_mode=InformationMode.EX_POST_RECONSTRUCTION, +) + +validate_synthetic_infill_plan_for_execution(plan) +plan_ref = write_synthetic_infill_plan(plan, work / "artifacts") + +# The exact public object that a #466 stage handler consumes. +first_command = plan.workflow_requests[0].tasks[0].commands[0] +stage_plan = load_reconstruction_stage_plan(first_command) +assert stage_plan.configuration.configuration_id == plan.configuration_id +``` + +The persisted plan and its companion artifacts are content addressed. Repeated +construction with the same files, policies, roots, and period selection has +the same IDs. Source file paths are retained for execution, while scientific +source identity is based on the declared digest, size, row count, evidence, +and period semantics. + +## Ex-ante construction and fail-closed behavior + +Ex-ante mode always uses zero right look-ahead and requires artifacts that were +trained and available before the requested period. Merely changing the mode on +the current full-history fitted artifacts is intentionally rejected: + +```python +from histdatacom.synthetic import ( + InformationMode, + ReconstructionPlanCompatibilityError, + build_synthetic_infill_plan, +) + +try: + ex_ante_plan = build_synthetic_infill_plan( + repo / "data/ASCII/T", + information_mode=InformationMode.EX_ANTE_SIMULATION, + start_period="202001", + end_period="202001", + **the_same_artifact_and_root_arguments, + ) +except ReconstructionPlanCompatibilityError as error: + assert "observe the requested future" in str(error) +``` + +A successful ex-ante plan therefore requires a point-in-time feed-epoch +definition, observation operator, motif index, context vintages, and CFTC +states whose training/availability boundaries precede the requested window. +The information audit is run over every ensemble member's complete synchronized +window plan and rejects unavailable vintages, future observations, future +motif selection, or any undeclared look-ahead. + +## Refusals and resource preflight + +Planning distinguishes an incompatible product from a scientifically +unsupported window. Hash mismatches, partial triangles, discontinuous periods, +wrong schemas, unstable/unqualified dependencies, leakage, broker-mode errors, +and quota overflow fail the whole build. Qualified source periods that lack a +supported feed assignment, market context, or sufficiently fresh CFTC state +remain visible as deterministic window refusals. At least one executable +window is required. + +`plan.to_dry_run_json()` returns the bounded operational view: request chunks, +artifact digests and sizes, resources, and refusals without expanding source +rows. `plan.resources` records both per-concurrent-window peaks and retained +ensemble output estimates. Storage-policy preflight occurs before a workflow +request can be submitted. diff --git a/docs/reconstruction-public-interfaces.md b/docs/reconstruction-public-interfaces.md new file mode 100644 index 00000000..0aa9261d --- /dev/null +++ b/docs/reconstruction-public-interfaces.md @@ -0,0 +1,270 @@ +# Public reconstruction CLI and Python API + +The supported reconstruction boundary is the installed +`histdatacom reconstruction` command family and +`histdatacom.reconstruction.ReconstructionClient`. Both surfaces consume the +same content-addressed `SyntheticInfillPlanV1`, explicit operator request, and +operation receipt contracts. Neither surface accepts tick rows, passwords, or +large analytical frames in flags or workflow control metadata. + +## Supported input and scientific acknowledgement + +Version 2.1 accepts only: + +- HistData ASCII tick caches below an `ASCII/T` source root; +- the complete `EURGBP`, `EURUSD`, `GBPUSD` synchronized triangle; +- `ex_post_reconstruction` or `ex_ante_simulation`, selected explicitly; and +- modern-reference delivery by default, or broker-conditioned delivery only + when a strong `broker_delivery_artifact_v1` reference is supplied. + +M1, OHLC, bar, partial-triangle, and broker-only requests are unsupported and +exit as invalid plans. Every execution request must also carry the exact +machine-readable scientific nonclaim and a true acknowledgement: + +> Output is a plausible counterfactual ensemble conditioned on declared +> artifacts and constraints; it is not recovered historical truth. + +The acknowledgement records operator intent. It does not weaken information +audits, validation, immutable-anchor checks, or certification gates. + +## Construct a plan and request + +Planning starts from a JSON `ReconstructionPlanSpecV1`. Paths point to strong, +qualified artifacts; data rows remain in their source artifacts. + +```json +{ + "schema_version": "histdatacom.reconstruction-plan-spec.v1", + "source_root": "data/ASCII/T", + "feed_epoch_definition_path": "artifacts/feed-epochs-v2-definition.json", + "observation_operator_path": "artifacts/observation-operator.json", + "market_context_corpus_path": "artifacts/market-context-corpus.json", + "cftc_positioning_corpus_path": "artifacts/cftc-positioning-corpus.json", + "benchmark_manifest_path": "artifacts/reverse-degradation-manifest.json", + "motif_manifest_path": "artifacts/modern-reference-motif-manifest.json", + "motif_index_path": "artifacts/modern-reference-motif-index.json", + "motif_qualification_path": "artifacts/modern-reference-motif-qualification.json", + "motif_leakage_audit_path": "artifacts/modern-reference-motif-leakage-audit.json", + "artifact_root": "work/plan-artifacts", + "output_root": "work/output", + "checkpoint_root": "work/checkpoints", + "scratch_root": "work/scratch", + "information_mode": "ex_post_reconstruction", + "delivery_mode": "modern_reference", + "start_period": "201101", + "end_period": "201101", + "requested_start_ns": null, + "requested_end_ns": null, + "window_size_ns": 86400000000000 +} +``` + +`window_size_ns` is a positive execution bound. Use smaller synchronized +windows when dense monthly inputs would exceed the declared memory, scratch, +or output policy; it changes plan and run identity and is never a hidden +runtime override. + +`requested_start_ns` and `requested_end_ns` are an optional paired half-open +UTC interval for bounded representative-window campaigns. When present, they +must agree with any supplied `start_period` and `end_period`; the planner still +inventories and hashes every complete monthly source partition touched by the +interval. Omitting them preserves whole-month planning. + +```sh +histdatacom reconstruction --json plan --spec plan-spec.json + +histdatacom reconstruction --json request \ + --plan work/plan-artifacts/synthetic-infill-plan-.json \ + --information-mode ex_post_reconstruction \ + --acknowledge-scientific-nonclaim \ + --output work/execution-request.json + +histdatacom reconstruction --json preflight \ + --request work/execution-request.json +``` + +Ranges whose safe execution window would make one plan exceed the 64 MiB plan +artifact bound use the public plan-set surface. The source specification keeps +the exact full range and resource-safe `window_size_ns`; the command starts +with bounded month groups and deterministically bisects them on execution-plan, +retention, or artifact-size preflight failures: + +```sh +histdatacom reconstruction --json plan-set \ + --spec full-range-plan-spec.json \ + --periods-per-shard 12 + +histdatacom reconstruction --json preflight-set \ + --plan-set work/plan-artifacts/reconstruction-plan-set-.json +``` + +Every resulting shard is an ordinary `SyntheticInfillPlanV1` with its own +hash-verified inventory, request graph, refusals, and resource limits. A fully +unsupported span is represented by a refusal-only plan with no workflow +requests and zero work/output estimates; it preflights only when refusals are +explicitly allowed and cannot publish a product. Dense months may therefore +contain multiple exact-bound shards. The parent +`ReconstructionPlanSetV1` requires contiguous nanosecond bounds, stores strong +plan references, and aggregates sums only for total work and output while +retaining maxima for peak memory and scratch. Raw rows, bytes, and partitions +are de-duplicated by immutable partition identity when dense months split into +multiple shards. Fresh plan-set preflight hashes each unique stat-identified +artifact once, execution-validates every shard, and rejects changed, missing, +overlapping, or gapped content. Construction and preflight retain only compact +resource summaries and partition identities after each full shard is handled. +Large qualified context corpora are resolved once per unchanged device, inode, +size, modification-time, and change-time identity set during the operation. + +Preflight hash-verifies the plan and its declared artifacts, validates that the +operator information mode matches the immutable plan, and emits the bounded +dry-run graph, resources, refusal reasons, and validation/qualification audit +references. A plan containing refused windows is not executable unless the +request was explicitly created with `--allow-refusals`; refusal evidence is +never discarded. + +## Execute, inspect, cancel, and resume + +`run` waits for Temporal completion by default. `--submit-only` returns after +submission. `--local` is an explicit in-process smoke/recovery path through the +same seven registered first-party handlers; it never becomes a silent fallback +for a failed Temporal submission. + +```sh +# Start the local runtime, submit all plan batches, and wait. +histdatacom reconstruction --start-runtime --json run \ + --request work/execution-request.json \ + --receipt work/run-receipt.json + +# Submit and return immediately. +histdatacom reconstruction --start-runtime --json run \ + --request work/execution-request.json \ + --submit-only \ + --receipt work/submission-receipt.json + +# Explicit bounded local parity/recovery execution. +histdatacom reconstruction --json run \ + --request work/execution-request.json \ + --local --window-id \ + --receipt work/local-receipt.json +``` + +Receipts bind every Temporal handle to the exact manifest/status-store root +used at submission. This prevents reconstruction status from accidentally +reading the generic runtime store. Control commands consume receipts: + +```sh +histdatacom reconstruction --json status \ + --receipt work/submission-receipt.json --offline + +histdatacom reconstruction --start-runtime --json cancel \ + --receipt work/submission-receipt.json \ + --reason "operator request" \ + --output work/cancel-receipt.json + +histdatacom reconstruction --start-runtime --json resume \ + --receipt work/submission-receipt.json \ + --output work/resume-receipt.json +``` + +Resume does not reinterpret a reconstruction workflow as the legacy ETL +`RunRequest`. It keeps the immutable scientific request and checkpoint keys, +while assigning fresh deterministic parent and child Temporal identities to +the recovery attempt. Committed windows therefore replay idempotently and a +partially completed earlier wave cannot collide with completed child workflow +IDs. + +## List, preview, and replay output + +```sh +histdatacom reconstruction --json outputs \ + --request work/execution-request.json + +histdatacom reconstruction --json preview \ + --manifest work/output/reconstruction-products/.../manifest.json \ + --limit 20 + +histdatacom reconstruction --json replay \ + --manifest work/output/reconstruction-products/.../manifest.json +``` + +`preview` is capped at 100 events. Each row includes observed/synthetic origin, +immutable-anchor lineage, generator/motif/reference identity, confidence, the +constraint-set identity, and an explicit accepted or immutable-anchor decision. +The preview also carries the product validation and constraint manifests plus +the replay logical-content hash. `replay` verifies committed files and rebuilds +the exact streams before reporting the reconciled event count and logical hash. + +## Certify modern-reference evidence + +The certification campaign is part of the installed command family: + +```sh +histdatacom reconstruction --json certify \ + --spec evidence/campaign.json \ + --output-directory evidence/dossier +``` + +The campaign fixes the broker-neutral `modern_reference` / +`unconditioned_reference` release claim. It verifies every JSON evidence file's +SHA-256, schema version, and subject identity before extracting a scalar through +the JSON pointer declared by the campaign. Scalar values are not allowed inline +in the campaign specification. The command publishes the frozen campaign, +methodology report, canonical dossier JSON, human Markdown, and a bounded result +receipt. + +An incomplete dossier exits as a scientific refusal (`3`), a measured gate +failure exits as validation failure (`5`), and `ready-for-promotion` or +`certified` exits successfully. Ordinary `dev` campaigns cannot include the +promotion-only coverage observation. + +## Typed Python surface + +```python +from histdatacom import ReconstructionClient +from histdatacom.reconstruction import ( + InformationMode, + read_execution_request, + read_plan_spec, + write_operation_receipt, +) + +client = ReconstructionClient() +plan_set_ref = client.construct_plan_set( + read_plan_spec("full-range-plan-spec.json"), periods_per_shard=12 +) +plan_set_preflight = client.preflight_plan_set(plan_set_ref.path) +request = client.create_request( + "work/plan-artifacts/synthetic-infill-plan-.json", + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + acknowledge_scientific_nonclaim=True, +) +preflight = client.preflight(request) +receipt = client.submit(request, wait=False) +write_operation_receipt(receipt, "work/submission-receipt.json") + +restored = read_execution_request("work/execution-request.json") +outputs = client.outputs(restored) +preview = client.preview(outputs["outputs"][0]["manifest_path"], limit=20) +replay = client.replay(outputs["outputs"][0]["manifest_path"]) +certification, certification_result = client.certify( + "evidence/campaign.json", + output_directory="evidence/dossier", +) +``` + +The facade also provides asynchronous `submit_async`, `inspect_async`, and +`cancel_async` methods for callers that already own an event loop. + +## Exit codes + +| Code | Name | Meaning | +| ---: | --- | --- | +| `0` | success | Plan/request created, preflight executable, submission/control accepted, status inspected, output verified, or certification is ready/certified. | +| `2` | invalid plan | Malformed/changed plan, unsupported schema, M1/bar/partial triangle, broker-only request, or invalid arguments. | +| `3` | refused | Scientific/resource refusal, missing nonclaim acknowledgement, or incomplete certification evidence. | +| `4` | runtime failure | Temporal/runtime connectivity or unexpected operational failure. | +| `5` | validation failure | Execution did not reach validated committed state or a measured certification gate failed. | + +Unsupported requests and scientific refusals retain distinct reason codes in +JSON error/preflight output even when both stop execution before data-plane +work. diff --git a/docs/reconstruction-streaming-contracts.md b/docs/reconstruction-streaming-contracts.md new file mode 100644 index 00000000..6b2b27d6 --- /dev/null +++ b/docs/reconstruction-streaming-contracts.md @@ -0,0 +1,233 @@ +# Reconstruction streaming contracts + +The version-one reconstruction streaming contracts define how bounded event +batches move between synthetic reconstruction stages without persisting the +521-column analytical frame as a second permanent dataset. Enrichment, +candidate surfaces, and rejected rows remain process-local or quota-managed +scratch data. Workflow history contains compact metadata and `ArtifactRef` +values only. + +This is a contract layer. It does not generate events, implement production +Temporal workflows, or publish the final Parquet product. + +The implemented [feed-epoch contracts](feed-epoch-contracts.md) are one of its +bounded semantic inputs. A reconstruction run binds the compact epoch +definition ID, and streamed windows or final events carry only their epoch or +uncertain-transition assignment. Fingerprint payloads, fitting panels, and +sensitivity intermediates never become permanent per-row columns. + +The implemented +[historical feed-observation operator](observation-operator-contracts.md) is a +second bounded semantic input. Runs bind its operator ID in +`configuration_ids`; window workers use the existing ownership/halo rules and +persist only larger output/carry batches behind `ArtifactRef` values. Fitted +panels and `ObservationApplicationResultV1` objects never enter workflow +history. + +## Contract boundary + +| Contract | Responsibility | +| --- | --- | +| `ReconstructionRunV1` | Semantic sources, configuration, members, symbols, and partition-independent seed namespace. | +| `ReconstructionWindowV1` | One synchronized symbol group, half-open generation interval, left halo, and right look-ahead. | +| `EventBatchV1` | Counts, time bounds, logical content hash, and a strong artifact reference; never event rows. | +| `CarryStateV1` | Per-symbol watermarks, last-event IDs, and references to larger carry artifacts. | +| `RejectionSummaryV1` | Reconciled aggregate rejection counts without rejected candidate rows. | +| `PartitionManifestV1` | One all-symbol synchronization unit whose batch counts reconcile by symbol. | +| `ReconstructionCheckpointV1` | Chained recovery state, completed batch IDs, watermarks, and publication references. | +| `ReconstructionStoragePolicyV1` | Candidate, batch, memory, scratch, output, ensemble, heartbeat, and checkpoint limits. | +| `ReconstructionResourceEstimateV1` | The complete estimate retained when resource preflight accepts or refuses work. | +| `ReconstructionHeartbeatV1` | Bounded progress, storage use, cancellation intent, and last-valid-checkpoint identity. | + +All version-one readers verify schema versions and deterministic IDs. Unknown +JSON envelope keys may be ignored by individual readers, but derived IDs, +counts, phases, and advertised references fail closed when they do not +reconcile. + +## Semantic identity and deterministic seeds + +`ReconstructionRunV1.run_id` includes only inputs that may affect scientific +output: source versions, configuration IDs, ensemble members, symbols, and the +base seed. It deliberately excludes the storage policy, worker count, retry +count, batch size, checkpoint cadence, and window plan. + +`seed_for(member, semantic_key)` derives a 64-bit seed from stable semantic +lineage such as an anchor interval or motif decision. A window ID, worker ID, +attempt number, or scratch path is not a valid semantic key. Consequently, +changing legal execution parallelism or storage tuning does not change event +identity or generated values. + +An `EventBatchV1` ID uses logical batch scope and content evidence. The +worker-local artifact path and optional artifact metadata are excluded from +logical identity, so a retry can place identical bytes at a new scratch path +without creating a different batch. Checkpoints retain the exact artifact +paths because recovery must know which physical artifacts to validate or +clean. + +## Window ownership, halo, and look-ahead + +Each window owns exactly: + +```text +[core_start_ns, core_end_ns) +``` + +Only the owning window may generate an event at a timestamp. Its readable +input interval is: + +```text +[core_start_ns - left_halo_ns, + core_end_ns + right_lookahead_ns) +``` + +Halo and look-ahead observations provide seam, anchor, and synchronized-pair +context but never grant generation ownership. The planner produces contiguous +windows over the complete symbol group, and plan validation rejects gaps, +overlaps, member drift, run drift, or symbol-group drift. This gives later +generators an explicit edge contract while keeping the scientific need for +future anchors visible. + +Right look-ahead is also an information-access channel. The implemented +[`reconstruction-information-modes.md`](reconstruction-information-modes.md) +contract binds the exact window-plan identity into the information manifest. +Its pre-generation audit requires zero right look-ahead for ex-ante simulation +and checks ex-post windows against their declared policy limit. + +Every event batch repeats its half-open ownership bounds and rejects a first or +last event time outside them. A halo observation can therefore inform a batch +but cannot be mislabeled as output owned by that batch. + +The planner has no worker-count argument. Different legal window sizes produce +different transport windows but preserve single ownership of every event time +and use the same semantic seed namespace. + +## Resource preflight and backpressure inputs + +Before a window is admitted, its estimate records: + +- input and proposed candidate event counts; +- peak events per batch and total estimated batch count; +- retained ensemble-member count; +- maximum simultaneously in-flight batches; +- estimated peak memory, scratch, and durable output bytes. + +The storage policy checks every estimate together and raises +`ReconstructionResourceLimitError` with the rejected estimate and every +violated limit. This is an early refusal, not an out-of-memory recovery path. +The policy also declares maximum events per batch, checkpoint and heartbeat +cadence, checkpoint payload bytes, removal of uncommitted scratch on +cancellation, and the committed-only publication rule. + +The implemented Temporal reconstruction control plane enforces these limits +with deterministic memory-weighted window waves and sequential stage +consumption. See +[`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + +## Checkpoints, retries, and duplicate delivery + +Checkpoints form an optimistic-concurrency chain. Every transition names the +exact current checkpoint ID and produces the next revision with the current ID +as its parent. A transition from an older checkpoint therefore fails as stale +instead of overwriting newer progress. + +Completed batch IDs are a bounded set of logical identities. On recovery, +`pending_batches()` deduplicates repeated delivery and returns only unfinished +batches. Watermarks may advance or remain fixed but cannot move backward. +This supports process-loss recovery without advertising or appending the same +logical events twice. + +Cancellation and failure checkpoints retain enough bounded evidence to resume +but expose no committed product. Resuming transitions back to `running`, keeps +completed batch IDs and watermarks, and clears the stale staged reference after +the caller performs the policy-mandated scratch cleanup. + +## Two-phase publication protocol + +The checkpoint state machine is: + +```text +planned -> running -> staged -> validated -> committed + | | | | + +----------+----------+----------+-> cancelled / failed + cancelled / failed -> running +``` + +The phase meanings are strict: + +1. `staged` references a temporary manifest that is not discoverable. +2. `validated` retains that same temporary reference after schema, count, + checksum, and synchronization-unit validation. +3. `committed` requires a promoted manifest whose SHA-256 and byte size match + the validated staging artifact. +4. `advertised_manifest_ref` is `null` in every phase except `committed`. +5. Repeating the same final commit is a no-op; attempting to replace the + committed manifest fails. + +The state machine specifies the control behavior; the implemented +`histdatacom.synthetic.persistence` layer performs the filesystem operation. +It owns temporary paths, Parquet validation, checksums, atomic +rename/promotion, final layout, discovery, and cleanup. A writer must not record +the `committed` checkpoint until its atomic promotion has succeeded. + +## Cross-symbol synchronization + +A reconstruction window contains the complete sorted symbol group and has one +`synchronization_unit_id`. Every event batch and the partition manifest must +match that run, member, window, and synchronization unit. The manifest carries +an explicit count for every symbol, including zero-event symbols, and validates +those counts against all referenced batches. + +The manifest is therefore the smallest commit unit for later triangular and +inverse consistency work. A subset of the symbols cannot be published as +though the synchronized unit succeeded. + +## Heartbeat and Temporal payload rules + +`ReconstructionHeartbeatV1` carries phase, bounded progress counts, event +counts, scratch/output bytes, cancellation intent, and the last checkpoint ID. +It states that cancellation stops future work and resume begins from the last +valid checkpoint. It contains no dataframe, event list, candidate rows, or +statistics surface and has a hard serialized-size ceiling. + +`ArtifactRef` values used by reconstruction contracts require a non-empty +kind/path, non-negative byte size, and lowercase SHA-256 digest. Artifact +metadata is bounded and rejects inline `rows`, `records`, `events`, `table`, or +`dataframe` keys. Large carry state, batches, manifests, and rejection detail +must be written to artifact storage. + +## Issue boundaries + +- #433's implemented information-mode contracts and leakage audit govern what + these windows may read and what downstream claims are valid. +- #434's implemented feed-epoch contract supplies only stability-passing, + uncertainty-aware observation-regime assignments and complete lineage. +- #435's implemented observation operator consumes those assignments and uses + these windows, alignment rules, limits, and carry seams for deterministic + forward observation and controlled degradation. +- #436 implements the streaming reverse-degradation benchmark and scorecards + over that operator interface; see + [`reverse-degradation-benchmark-contracts.md`](reverse-degradation-benchmark-contracts.md). +- #437 implements bounded point-in-time market-context query sidecars over + these windows; see + [`market-context-contracts.md`](market-context-contracts.md). +- #439 implements variable-cardinality empirical-motif candidate generation + using these windows, seeds, budgets, and carry contracts; see + [`empirical-motif-generation-contracts.md`](empirical-motif-generation-contracts.md). +- #440 owns hard historical carving and detailed rejection decisions. +- #441 implements exact-event-time cross-currency generation and reconciliation + over these all-symbol windows and manifests; see + [`cross-currency-reconciliation-contracts.md`](cross-currency-reconciliation-contracts.md). +- #442 implements deterministic ensemble plans, all-member resource estimates, + retained-member policy, calibrated summaries, and hash-bound regeneration; + see + [`reconstruction-ensemble-calibration-contracts.md`](reconstruction-ensemble-calibration-contracts.md). +- #443 implements append-only broker capture and live/replay consumer parity; + see [`broker-capture-contracts.md`](broker-capture-contracts.md). +- #446 implements final atomic Parquet and manifest publication; see + [`reconstruction-persistence-contracts.md`](reconstruction-persistence-contracts.md). +- #447 maps the contracts onto production Temporal workflows and activities; + see + [`reconstruction-temporal-orchestration.md`](reconstruction-temporal-orchestration.md). + +Changing a required field, identity rule, ownership interval, phase meaning, +or retry guarantee requires a new schema version and contract class. diff --git a/docs/reconstruction-temporal-orchestration.md b/docs/reconstruction-temporal-orchestration.md new file mode 100644 index 00000000..ac937b67 --- /dev/null +++ b/docs/reconstruction-temporal-orchestration.md @@ -0,0 +1,243 @@ +# Temporal reconstruction orchestration + +The production reconstruction control plane runs period-scale work as one +`ReconstructionRunWorkflow` with bounded `ReconstructionWindowWorkflow` +children. Temporal orders and retries the work. Market events, augmented +frames, candidate surfaces, fitted statistics, and Parquet data never enter a +workflow argument, result, query, or heartbeat. + +The workflow-visible chain is: + +```text +source/enrichment + -> proposal + -> carving + -> cross-series reconciliation + -> delivery projection + -> validation/staging + -> atomic partition commit +``` + +Each arrow is an artifact boundary. A stage handler reads declared input and +configuration `ArtifactRef` values, operates on data outside Temporal, writes +its outputs, and returns one bounded `ReconstructionStageOutcomeV1`. The +orchestrator verifies the output bytes and atomically writes that outcome as a +stage receipt before advancing the checkpoint. + +## Contracts + +| Contract | Responsibility | +| --- | --- | +| `ReconstructionWorkflowRequestV1` | Run identity, bounded window tasks, queue names, status/report roots, and concurrency limits. | +| `ReconstructionWindowTaskV1` | One all-symbol window, resource estimate, scratch scope, and the complete ordered command list. | +| `ReconstructionStageCommandV1` | Handler name, strong input/configuration references, and a receipt path below that window's scratch directory. | +| `ReconstructionStageInvocationV1` | Activity-side handler context, exact input fingerprint, heartbeat hook, and cancellation check. | +| `ReconstructionStageOutcomeV1` | Status, output references, event counts, scratch/output bytes, and deterministic outcome identity. | +| `ReconstructionWindowStateV1` | Existing reconstruction checkpoint plus a strict prefix of completed stage outcomes. | +| `ReconstructionRunReportV1` | Reconciled workflow/storage status and committed manifest references. | + +The request has a hard one-megabyte serialized ceiling and a 512-window +ceiling. Artifact references must include byte size and lowercase SHA-256. +Inline keys such as `rows`, `events`, `records`, `table`, or `dataframe` are +rejected even when hidden inside artifact metadata. + +## First-party data-plane adapters + +Scientific code runs only in activity workers. Default worker construction +idempotently registers the versioned handlers emitted by the first-party plan: + +| Stage | First-party action | +| --- | --- | +| source/enrichment | Hash-verifies ASCII/T Arrow partitions, maps the zero-based Arrow ordinal to the positive event-contract row ID, uses immutable partition/row order for same-timestamp `event_sequence`, rejects invalid quotes, and materializes input/core Parquet plus feed-epoch, calendar, and CFTC sidecars. | +| proposal | Queries the qualified modern motif index and writes candidate rows to Parquet with a content-addressed canonical-JSON-lines batch ledger outside the compact stage manifest. | +| carving | Applies the declared hard/advisory constraints, writes a content-addressed rejection/decision ledger outside the compact stage manifest, and materializes accepted core streams. | +| cross-series reconciliation | Reconciles the complete EURUSD/GBPUSD/EURGBP group using exact event-time support. | +| delivery projection | Applies explicit modern-reference identity delivery; it never invents a broker fingerprint. | +| validation | Rechecks cross-instrument output, benchmark qualification, motif leakage, information safety, immutable anchors, retention, and storage before staging. | +| atomic commit | Promotes or recovers the complete v2 Parquet transaction. | + +Applications do not register these handlers themselves. A deliberately custom +worker may still register a different, separately versioned adapter before +constructing a worker with an explicit activity list: + +```python +from histdatacom.orchestration import register_reconstruction_stage_handler + + +def proposal_handler(invocation): + # Read invocation.command.input_manifest_refs from artifact storage. + # Stream bounded batches through the existing pure proposal code. + # Write the result before returning its strong ArtifactRef. + output_ref = build_proposal_artifact(invocation) + return invocation.completed( + output_refs=(output_ref,), + observed_event_count=observed_count, + candidate_event_count=candidate_count, + scratch_bytes=scratch_bytes, + output_bytes=output_ref.size_bytes or 0, + ) + + +register_reconstruction_stage_handler("proposal-v3", proposal_handler) +``` + +Handler registration remains process-local by design. `handler_name` is +deterministic command metadata; the callable is outside workflow history. +Configuration artifact hashes and the command ID bind the selected behavior. + +Long-running handlers call `invocation.heartbeat(...)` at bounded batch +intervals and check `invocation.cancellation_requested` before producing more +work. Those hooks expose phase, window, counts, scratch/output bytes, and the +inputs needed to calculate rate/ETA without carrying rows. + +## Recovery and idempotency + +Every window state is stored in the existing manifest/status SQLite database. +`compare_and_swap_job_snapshot()` acquires an immediate SQLite transaction and +replaces a state only when its expected snapshot ID is still current. The same +write succeeds idempotently. A stale worker with different evidence fails +closed. + +This closes the important worker-loss interval: + +```text +stage output written + -> stage receipt atomically written + -> worker dies before checkpoint + -> Temporal retries activity + -> receipt and every output hash are verified + -> outcome is reused + -> checkpoint advances once +``` + +Atomic publication also closes the later loss interval: + +```text +validated manifest mirror and transaction descriptor written + -> staging directory atomically renamed into commits + -> worker dies before the commit receipt + -> retry verifies the byte-identical manifest mirror + -> descriptor locates the expected committed identity + -> committed Parquet and manifest are fully reverified + -> the commit receipt and checkpoint advance once +``` + +The transaction descriptor is not the staged phase artifact. The staged phase +reference is a durable byte-identical mirror of the product manifest, so its +SHA-256 must equal the committed manifest even after the rename removes the +original staging path. + +If the checkpoint was already advanced, the strict outcome prefix resolves a +duplicate completion to the newer state. A different receipt, input +fingerprint, output hash, or checkpoint branch is rejected. + +The window checkpoint follows the existing protocol: + +```text +planned -> running -> staged -> validated -> committed +``` + +Only the validation outcome may supply exactly one `commit_phase=staged` +manifest. Only the atomic-commit outcome may supply exactly one +`commit_phase=committed` manifest with identical manifest bytes. Since the +window always contains the full sorted symbol group, a subset cannot reach the +committed phase independently. + +## Backpressure and resource refusal + +The parent creates deterministic waves constrained by both: + +- `max_parallel_windows`; and +- the sum of admitted `estimated_memory_bytes`. + +Stages within a window are strictly sequential, so a producer cannot outrun +its next consumer. `max_parallel_windows` may not exceed the storage policy's +in-flight-batch limit. A task above the run or lane limit is placed alone in a +preflight wave; its activity records a durable failed/refused checkpoint +without invoking a scientific handler. + +After admission, each outcome is checked against candidate, scratch, and +output limits. Underestimated actual use fails the window before another stage +starts. + +The existing lanes are sufficient: + +- parent and child workflow decisions use `orchestration`; +- window activities, stage handlers, storage verification, and report + reconciliation use `cpu_file`. + +No new unbounded queue or implicit worker pool is introduced. + +## Cancellation and scratch + +Cancellation is checked between stages and is visible inside handlers. The +last valid state becomes `cancelled`, then only that task's explicitly scoped +scratch directory is removed. The cleanup helper rejects symlinks, files, the +home directory, and `/`. Request validation also rejects nested/shared window +scratch trees, overlap with manifest or report storage, and durable stage inputs +placed below disposable scratch. + +Because cancellation cleanup removes every uncommitted receipt and artifact in +the window scratch tree, resume discards the entire disposable stage prefix and +rebuilds the window from its immutable inputs. It never retains references to +deleted scratch files. Committed manifests are terminal and are never removed +by cancellation cleanup. + +## Final reconciliation + +`reconstruction_report` receives the bounded request only; it reloads each +window checkpoint from the manifest store instead of fanning every child state +back into one large Temporal activity payload. For every committed window it: + +1. verifies the committed manifest `ArtifactRef` bytes; +2. runs `verify_reconstruction_publication()` over the manifest and Parquet; +3. matches run, window, synchronization unit, and complete symbol set; +4. sums observed and synthetic event counts from storage evidence; and +5. sums stage runtimes once and retains maxima for RSS, scratch, and candidate + amplification instead of mistaking final commit telemetry for the whole + window; and +6. atomically writes one compact run report and records it in the manifest + store. + +The report activity heartbeats between windows. A committed checkpoint without +a valid matching publication fails report reconciliation. + +## Submission and worker registration + +`submit_reconstruction_request()` adds the workspace task-queue map, starts +`ReconstructionRunWorkflow` with a deterministic workflow ID, and records the +submission in the manifest store. Default workers register: + +- `ReconstructionRunWorkflow`; +- `ReconstructionWindowWorkflow`; +- `reconstruction_window`; and +- `reconstruction_report`. + +They also install all seven first-party stage handlers before accepting +activities; no application-level registry setup is required. + +Both workflows validate in Temporal's sandbox. The activity-side module is +passed through the sandbox import boundary and is never called from workflow +decision code. + +## Fault contract + +The orchestration tests inject and verify: + +- output/receipt completion followed by worker loss before checkpoint; +- stage timeout followed by restart at the last completed stage; +- duplicate checkpoint completion; +- stale checkpoint branches; +- corrupt receipts and changed output bytes; +- cancellation with scoped scratch cleanup; +- resource and lane refusal; +- memory-weighted producer/consumer backpressure; +- manifest-store re-open after a process/server-style restart; and +- final workflow/storage scope and count reconciliation. + +The real-artifact integration gate additionally runs a qualified synchronized +triangle through the first-party handlers, compares logical and physical +hashes across concurrency settings, injects termination after the atomic +rename, injects cancellation, and proves a failing qualification prevents +commit. Set `HISTDATACOM_REAL_RECONSTRUCTION_PLAN` to a #465 plan artifact to +run that gate; no scientific handler is mocked. diff --git a/docs/reference-motif-index-contracts.md b/docs/reference-motif-index-contracts.md new file mode 100644 index 00000000..80ef4641 --- /dev/null +++ b/docs/reference-motif-index-contracts.md @@ -0,0 +1,119 @@ +# Empirical Reference-Motif Index Contracts + +The reference-motif domain turns bounded, modern augmented tick windows into a +versioned empirical library for downstream variable-cardinality reconstruction. +It indexes evidence; it does not generate, carve, or accept synthetic events. + +## Evidence boundary + +`reference_motif_source_window_from_training_frame()` reads only the augmented +row fields needed for identity, observed bid/ask values, event ordering, and +training eligibility. The hundreds of analytical and classical-model columns +remain available while selecting windows, but are not copied into motif +records. + +Each `ReferenceMotifFragmentV1` retains: + +- the source artifact reference and SHA-256; +- series, period, window, source-row, and source-event identities; +- inclusive source timestamp boundaries and duration; +- event-time offsets from the first observation; +- bid and ask deltas from the first quote; +- start, bid-only, ask-only, both-mark, and unchanged transitions; +- the complete conditioning cell and data-quality eligibility; +- the versioned admissible price/time scale and warp envelope. + +The compact sequence is therefore replayable against its exact evidence +without serializing the 521-column augmented row for every event. + +## Conditioning and fallback + +`ReferenceMotifConditionV1` covers the following coordinates: + +- symbol and currency exposure; +- technological feed epoch; +- session state, active sessions, overlap, rollover/special, holiday, and event + tags; +- return, range, volatility, spread, activity, and inter-arrival regimes; +- timestamp precision, price precision, and source-quality state; +- bounded numeric return, range, volatility, spread, intensity, inter-arrival, + precision, and source-quality metrics. + +Retrieval starts with the exact cell and follows the configured hierarchy +through symbol/epoch/session/event, state, currency, symbol, epoch, and global +cells. Every attempted level records its pattern, candidate count, available +count, minimum support, and outcome. A sparse or point-in-time-unavailable cell +cannot silently become a match. + +Within the first supported cell, weighted normalized metric distance and +categorical penalties are explicit. Results are ordered by `(distance, +fragment_id)`, making tie-breaking stable for fixed artifacts and config. + +## Split and leakage policy + +Version one requires chronological `train`, `calibration`, `validation`, and +`final_holdout` splits. Only eligible training windows may enter the index. +Other splits are counted but never serialized into the reference library. + +Before retention, the builder audits all declared source windows for: + +1. overlapping or guard-adjacent intervals from the same source artifact and + symbol across train and a withheld split; and +2. equal normalized quote-shape signatures across train and a withheld split. + +Either condition raises `ReferenceMotifLeakageError`; it is not downgraded to a +warning. Normalized signatures retain relative event timing, bid/ask shape, and +transition marks, so price- or time-scaled copies cannot cross the boundary. + +## Bounds and artifact identity + +`ReferenceMotifIndexConfigV1` bounds source windows, events per fragment, +retained fragments, matches, exclusions, artifact bytes, and fallback levels. +When eligible training windows exceed the retention budget, stable hash +priority selects a worker- and input-order-independent subset. Counts for +withheld, ineligible, and budget-omitted windows reconcile with the original +source-window count. + +`write_reference_motif_index()` uses an atomic local replacement and returns an +`ArtifactRef` containing the schema, index/config identities, counts, size, and +SHA-256. `read_reference_motif_index()` verifies the optional reference before +restoring every nested deterministic ID. + +Period-scale operation remains bounded by `max_source_windows` during build, +`max_fragments` after retention, and `max_fragments * fallback_levels` during a +query. Dense event panels are not retained beside the artifact. + +## Point-in-time use + +An ex-ante `ReferenceMotifQueryV1` requires `as_of_ns`. Retrieval hides a motif +unless both its artifact availability and its last source observation are no +later than that time. If all supported evidence is hidden, the result is +`not_available_as_of`, not an empty successful match. + +`reference_motif_information_inputs()` maps every returned motif to the #433 +information graph with: + +- `MOTIF_SELECTION` stage; +- `EMPIRICAL_MOTIF` scope; +- training split identity; +- exact observation interval and availability; +- actual ex-post lookahead, or zero ex-ante lookahead. + +This lets the existing reconstruction information audit enforce the same +point-in-time rules before #439 generation begins. + +The implemented candidate consumer is documented in +[`empirical-motif-generation-contracts.md`](empirical-motif-generation-contracts.md). +It retains the query result and compact source lineage while keeping candidate +rows process-local and separate from later carving. + +## Trust gates + +- Withheld or ineligible windows never enter the index. +- Cross-split overlap or normalized near-duplicate evidence fails closed. +- Returned fragments include source artifact, row, event, window, and quality + lineage. +- Fallback level, support, distance, and tie-breaking are observable. +- Artifact and nested contract identities are content-derived and verified on + read. +- The index does not create candidate or final synthetic events. diff --git a/docs/reverse-degradation-benchmark-contracts.md b/docs/reverse-degradation-benchmark-contracts.md new file mode 100644 index 00000000..2c8d7660 --- /dev/null +++ b/docs/reverse-degradation-benchmark-contracts.md @@ -0,0 +1,205 @@ +# Reverse-degradation reconstruction benchmark + +The version-one reverse-degradation benchmark is the falsifiability boundary +for reconstruction generators. It asks whether a candidate can restore dense +modern delivery behavior after a controlled historical feed degradation while +preserving market, timing, anchor, and lineage constraints. + +The benchmark does not select a production generator. It emits reproducible, +stratified evidence that later comparison semantics may use. + +## Streaming boundary + +The process is: + +```text +dense modern reference events + -> ObservationOperatorV1.degrade() + -> degraded BenchmarkEventV1 stream + -> transparent control or BenchmarkGeneratorV1 + -> candidate BenchmarkEventV1 stream + -> ReverseDegradationBenchmarkV1.consume_window() + -> bounded online aggregates + -> ReverseDegradationScorecardV1 +``` + +`BenchmarkEventV1` is the shared versioned interface on both sides of the +degradation/generation boundary. Adapters accept: + +- dense `ObservationInputEventV1` values; +- sparse `ObservationOutputEventV1` values; +- observed or generated `SyntheticEventV1` values; and +- the row-aligned `synth_bid`/`synth_ask` surface produced by the existing + empirical-overlay control. + +Generator configuration and degradation configuration use separate +deterministic namespaces and IDs. A benchmark manifest rejects any identity +collision. `degrade_benchmark_window()` also verifies that the operator object +is the exact operator bound by the scenario before processing events. + +Candidate-window event tuples are data-plane values. Their `metadata()` method +records only event count and bounded evidence; it never places events in +workflow history. `consume_window()` immediately reduces a complete window to +online accumulators and retains no reference, degraded, candidate, rejected, +or wide analytical rows. + +## Immutable research periods + +The existing reconstruction-information v1 schema remains immutable and keeps +its `train -> calibration -> validation` contract. The benchmark adds its own +three-period manifest: + +```text +calibration -> validation -> final_holdout +``` + +`validate_benchmark_information_boundary()` requires: + +- identical run and information-manifest IDs; +- exact reuse of the information manifest's calibration interval; and +- ordered benchmark validation/final-holdout intervals wholly contained in + the information manifest's validation interval. + +The benchmark therefore subdivides the already-withheld information boundary +without relabeling or mutating the upstream v1 contract. Scenarios may evaluate +validation and final holdout only. Every scenario must consume contiguous +windows spanning its complete split before finalization. + +## Scenario matrix + +`BenchmarkScenarioV1` binds: + +- an immutable validation or final-holdout split; +- one feed epoch; +- one degradation severity; +- one exact observation-operator ID; +- bounded degradation parameters; and +- the shared benchmark-event schema version. + +A valid manifest requires at least two feed epochs, at least two degradation +severities, and scenarios in both validation and final holdout. Epoch and +severity are separate dimensions: a historical technology epoch is not +silently treated as a sparsity level or market state. + +## Transparent controls + +Every manifest contains exactly one of each control: + +| Control | Semantics | +| --- | --- | +| `no_fill` | Pass the degraded observation stream through unchanged. | +| `linear_interpolation` | Interpolate bid and ask on an explicit regular interval without reading withheld reference prices. | +| `resample_last` | Select the last degraded observation per explicit context-aware time bucket. | +| `empirical_overlay` | Accept the existing row-aligned `synth_*` output and require exact degraded-row cardinality. | + +The interpolation and resampling intervals are part of the deterministic +generator configuration. The empirical adapter requires +`timestamp_utc_ms`, `synth_bid`, and `synth_ask`; missing or null generated +values fail closed. Controls appear in the same scorecard as candidates, so +complexity is visible relative to no fill instead of being presumed useful. + +## Online metric surface + +Fixed histograms and running aggregates cover: + +- event counts and per-second intensity; +- inter-arrival distributions; +- burst and quiet-run rates; +- spread means, distributions, and transitions; +- midpoint endpoint and range behavior; +- historical-anchor preservation; +- uncertainty-interval coverage; +- common-timestamp ensemble diversity; +- cross-series metric hooks; +- strategy-sensitivity metric hooks; and +- attempts, convergence, failures, wall time, peak memory, scratch bytes, and + durable bytes. + +Scores are stratified by the exact tuple: + +```text +symbol, feed epoch, session, event state, sparsity +``` + +Here `sparsity` is shared scenario context such as a degradation severity or +measured sparsity bucket. It is not a label for whether an event came from the +reference, degraded, control, or candidate stream. + +Every slice retains reference, degraded, and mean candidate-member counts, +metric support, and its own soft loss. Candidate summaries include both mean +and worst-slice loss, restoration gain relative to the degraded surface, +uncertainty support, anchor support, and ensemble support. Aggregate means are +explicitly advisory; they cannot replace the stratified evidence. + +## Hard gates and interpretation + +Candidate windows accept bounded hard-constraint violation counts from the +synthetic-constraint and later carving layers. Missing protected reference +anchors are also converted into an automatic hard violation. + +A reconstruction candidate is promotion-eligible only when: + +- it was attempted; +- every attempt converged; +- it has no failure metadata; and +- it has zero hard-constraint violations. + +A low soft loss can never override a hard violation. Controls are never marked +promotion-eligible. + +`ReverseDegradationScorecardV1` always serializes: + +```json +{ + "automatic_winner": false, + "winner_candidate_id": null +} +``` + +Relative-to-no-fill deltas are evidence, not a ranking. No default generator or +automatic winner exists until separate comparison semantics are established. + +## Bounds and replay + +`BenchmarkProfileV1` freezes: + +- scenario, candidate, slice, event, hook, and reason-code limits; +- fixed inter-arrival and spread histogram buckets; +- burst and quiet thresholds; +- score rounding; and +- final JSON payload bytes. + +Exceeding a bound fails before an unbounded scorecard is produced. Stable IDs +cover the profile, splits, scenarios, generator/degradation configurations, +manifest, events, execution evidence, slices, candidate scores, and final +scorecard. Replaying the same versioned inputs produces the same JSON and +scorecard ID. + +The engine accepts only time-owned events, complete configured ensemble-member +sets, complete mandatory controls, ordered contiguous windows, and complete +scenario split coverage. Window retries and durable artifact orchestration +remain the responsibility of the reconstruction streaming/checkpoint layer; +the benchmark consumes each deduplicated logical window once. + +## Issue boundaries + +- #431 supplies the narrow variable-cardinality event contracts. +- #432 supplies bounded windows, carry, resource, and artifact-reference + contracts. +- #433 supplies the immutable information-mode and leakage boundary. +- #434 supplies feed epochs and transition evidence. +- #435 supplies the fitted observation operator and controlled degradation. +- #436 supplies the benchmark contracts, controls, online engine, scorecards, + hard gate, and hooks documented here. +- #439 and later generator issues implement candidate generation behind + `BenchmarkGeneratorV1`. +- #440 supplies detailed carving violations. +- #441 supplies synchronized cross-series metrics. +- #442 consumes validation and final-holdout member windows to calibrate + metric/horizon intervals, diagnose collapsed diversity, and report bounded + failure/refusal evidence; see + [`reconstruction-ensemble-calibration-contracts.md`](reconstruction-ensemble-calibration-contracts.md). +- #448 supplies downstream strategy-sensitivity metrics. + +Changing split meaning, control semantics, metric meaning, promotion gating, +event identity, or required scorecard fields requires a new schema version. diff --git a/docs/reverse-degradation-benchmark-corpus.md b/docs/reverse-degradation-benchmark-corpus.md new file mode 100644 index 00000000..306df0d9 --- /dev/null +++ b/docs/reverse-degradation-benchmark-corpus.md @@ -0,0 +1,266 @@ +# Real reverse-degradation benchmark corpus and promotion gates + +This document defines the real-data acceptance boundary owned by issue #463. +It supplements the generator-neutral v1 engine documented in +[`reverse-degradation-benchmark-contracts.md`](reverse-degradation-benchmark-contracts.md). + +The packaged promotion policy is intentionally committed before any promotable +real candidate report. At this policy stage there is no candidate winner, +promotion result, or claim that reconstruction recovers missing historical +ticks. + +## Frozen policy artifact + +`histdatacom.synthetic` packages +`assets/reverse_degradation_promotion_gates_v1.json`. The installed loader +`load_default_benchmark_promotion_gate_policy()` verifies its schema, complete +hard/advisory surface, and content-derived `policy_id`. + +The policy records: + +- issue authority `#463` and policy version `v1`; +- `frozen_before_candidate_results=true`; +- separate campaign and candidate scopes; +- explicit comparators and thresholds; +- hard gates that fail closed when evidence is missing; and +- advisory gates that remain visible but cannot silently become hard gates. + +`BenchmarkPromotionDecisionV1` always records +`automatic_winner=false`. Passing gates makes one measured subject eligible +under this policy; it does not select a default generator or compare unrelated +campaigns. + +## Campaign hard gates + +The real campaign must demonstrate all of the following before its candidate +decisions are meaningful: + +| Evidence | Predeclared hard threshold | +| --- | ---: | +| source hash mismatches | `0` | +| split/near-neighbor holdout leakage | `0` | +| information-audit violations | `0` | +| missing required strata | `0` | +| dense identity-control failures | `0` | +| negative controls that unexpectedly pass | `0` | +| maximum bounded hook metrics per window | `64` | +| campaign runtime | `<= 900 seconds` | +| peak memory | `<= 2 GiB` | +| compact persisted artifact bytes | `<= 64 MiB` | + +Campaign support of at least eighteen real synchronized windows and at least +two deterministic ensemble members is advisory. The final report must expose +these observations even when they miss the advisory threshold. + +## Candidate hard gates + +A measured candidate fails promotion eligibility when any hard observation is +missing or violates its threshold: + +| Evidence | Predeclared hard threshold | +| --- | ---: | +| immutable anchor violations | `0` | +| failed or non-converged measured windows | `0` | +| unsupported-context emissions | `0` | +| worst event-count relative error | `<= 0.50` | +| worst inter-arrival histogram L1 | `<= 0.45` | +| worst update-transition matrix L1 | `<= 0.55` | +| worst spread-tail relative error | `<= 0.75` | +| worst realized-variation relative error | `<= 0.75` | + +Refusal-rate reporting, a p99 synchronized-triangle residual no greater than +five pips, and at least six supported uncertainty intervals are advisory. They +remain part of every decision so a nominal hard-gate pass cannot hide weak +support. + +## Evidence ordering + +The scientific order is mandatory: + +1. Commit the policy schemas, evaluator, packaged thresholds, and tests. +2. Record the policy commit and content ID in the later campaign manifest. +3. Construct immutable calibration, validation, and final-holdout partitions. +4. Run the untouched dense identity and declared negative controls. +5. Run transparent controls and any provisional empirical candidate. +6. Evaluate only the already-frozen requirements. +7. Publish compact source/split hashes, leakage audit, decisions, scorecards, + resource evidence, and deterministic replay evidence. + +Changing a threshold, comparator, scope, severity, or metric meaning requires a +new policy version and a new content ID. A candidate result produced before +that new policy commit is ineligible under the new version. + +## Installed corpus and campaign API + +`build_reverse_degradation_benchmark_corpus()` accepts only explicit prior +artifact paths. It does not refresh epochs, fit a new observation operator, or +download context while a benchmark is running. The resulting +`ReverseDegradationBenchmarkCorpusV1` binds: + +- three monthly Arrow tick caches per blocked role for EURUSD, GBPUSD, and + EURGBP; +- six synchronized UTC windows per role, with Asia, London, and New York + session coverage; +- the v2 feed-epoch definition and fitted observation-operator IDs; +- the point-in-time market-context corpus and the CFTC reconstruction-view + positioning corpus; +- nine degradation configurations and the complete metric registry; and +- the packaged policy ID and the full commit SHA that predates candidate + output. + +Each monthly source records its relative cache path, byte size, row count, and +SHA-256. Each window records the half-open UTC interval, source partition IDs, +event counts by symbol and update state, and a SHA-256 of the first bounded +event selection for every symbol. +`replay_reverse_degradation_benchmark_corpus()` re-hashes every source, +recounts Arrow rows, reselects every bounded window, and compares all window +hashes before candidate execution. Neither dense nor holdout event rows appear +in the corpus or scorecard. + +`run_reverse_degradation_benchmark_campaign()` builds a train-only provisional +motif index, executes every declared degradation, runs dense identity, +degraded/no-fill identity, linear interpolation, the existing +`EmpiricalMotifBenchmarkGeneratorV1`, and an anchor-drop negative control, then +evaluates the frozen policy. A required degradation failure aborts the +campaign rather than becoming a passing aggregate. The empirical motif report +is always marked provisional until #464 supplies the qualified library. + +The CFTC archive used by #468 represents the corrected reconstruction view, +not a fully preserved original publication vintage. The campaign therefore +queries it with `EX_POST_RECONSTRUCTION` and retains that limitation; it does +not relabel corrected state as ex-ante knowledge. Market-context event queries +remain ex-ante at each window start. + +## Installed command + +Run the complete bounded build, replay, campaign, and artifact write through +the installed CLI: + +```console +histdatacom analytics reverse-degradation-benchmark-corpus \ + --source-root data/ASCII/T \ + --definition data/.histdatacom/analytics/feed-epochs-v2-issue-460/feed-epochs-v2-definition.json \ + --observation-campaign data/.histdatacom/analytics/observation-calibration-v2-issue-462/observation-calibration-v2-campaign.json \ + --market-context-corpus .histdatacom/market-context-461-final-v4/market-context-corpus-9255f8c39f999b7a54e41a59a6f1d96f02e897af8383795e464a2f8738b08e00.json \ + --cftc-positioning-corpus data/.histdatacom/analytics/cftc-positioning-issue-468-final/cftc-positioning-corpus-887a47840090cdab1982fe910a4bdf8c1fcc9af256ab687bceae1b8dd1cbd3e0.json \ + --artifact-dir data/.histdatacom/analytics/reverse-degradation-benchmark-issue-463-final-v5 \ + --json +``` + +The defaults select `201001`, `202401`, and `202510` as calibration/training, +validation, and final holdout. All periods and resource limits are explicit CLI +options, but changing them creates a different content ID. + +The artifact directory contains five immutable JSON files: + +```text +reverse-degradation-manifest-.json +reverse-degradation-motif-index-.json +reverse-degradation-leakage-audit-.json +reverse-degradation-resource-audit-.json +reverse-degradation-scorecard-.json +``` + +Readers verify the filename digest before restoring strict contracts. Writes +are atomic and reuse identical content, while an existing content-addressed +path with different bytes is refused. + +## Executed degradation and metric surface + +Every real synchronized window executes uniform thinning, the fitted +state-dependent operator, unchanged filtering, timestamp quantization, +batching, rate caps, a missing-window stress, duplicate injection, and +symbol-specific thinning. Protected first/last anchors remain immutable in all +non-negative-control paths. + +The compact report covers multiscale counts and dispersion; inter-arrival +histograms, quantiles, duration dependence, and burst/quiet rates; quote-update +proportions and transition matrices; spread tails/jumps, stale runs, timestamp +precision, and tick-grid adherence; increments, realized variation, jump +proxies, excursions, and anchors; triangle synchronization and residuals; +context-conditioned slices; and refusal/unsupported rates. Point-process fit +is recorded as not applicable when a candidate exposes no conditional +intensity instead of fabricating a diagnostic. Six frequentist 95% uncertainty +intervals summarize window/member variation for every candidate report. + +## Candidate boundary supplied to #464 + +The #463 campaign exercised `EmpiricalMotifBenchmarkGeneratorV1` with a +provisional calibration-only motif input whose source windows could not touch +validation or final holdout. That failed result remains immutable baseline +evidence. The qualified production library, independent split manifest, and +non-provisional rerun are now owned by #464 and documented in +[`modern-reference-motif-library.md`](modern-reference-motif-library.md). + +## Issue #463 reference campaign + +The installed command produced the closure campaign on 2026-07-15. Its +immutable identities are: + +- corpus ID + `reverse-degradation-corpus:sha256:a760a010d44de2d6258b7c3d71651b00bc24eaef53092f37bd75b3ae2395c5dc`; +- provisional motif index ID + `reference-motif-index:sha256:7c56a5d1caf219df72762499743c3ca1fd3eba162c605612ffe46fa0495b6835`; +- campaign ID + `reverse-degradation-campaign:sha256:9158e185c5fdeec12a2450e1e8f9d0b3d42d735f63d1d0e9ec530340c373792d`; + and +- frozen policy ID + `benchmark-promotion-gates:sha256:f59039526fca4a70b40f836525ca08efc0bb336668a1d6676bc1c6a3ba7a186f` + from commit `0caec1480a957528ebefdff062e13012ea11e84d`. + +The corpus binds nine real monthly sources containing 11,225,291 Arrow rows +and 314,343,815 bytes. Replay verified all nine source hashes and all 54 +symbol-window hashes. Eighteen synchronized windows cover all three blocked +roles and sessions. Six windows carry point-in-time macro-release context, and +twelve explicitly record no matching event. Across the corpus, all four update +states are represented: 3,153 ask-only, 3,137 bid-only, 5,843 joint, and 162 +unchanged observations. The provisional index retained 221 train fragments +from 770 projected source windows and performed 5,310 cross-split leakage +comparisons; neighbor leakage and information-audit violations were both zero. + +All nine degradation families executed on all eighteen windows with zero +execution failures and zero protected-anchor violations. The campaign also +refuses a vacuous family: every family had to change the observable stream in +at least one real window. The affected-window counts were 12 for batching, 18 +for duplicate injection, 18 for the fitted state-dependent operator, one for +the missing-window stress, eight for the eight-events/second rate cap, 18 for +symbol-specific thinning, 12 for timestamp quantization, 12 for unchanged +filtering, and 18 for uniform thinning. The campaign passed its frozen +campaign gates in 18.089916 seconds at 555,483,136 bytes peak RSS. The five +persisted artifacts total exactly 1,093,214 bytes, matching both the scorecard +observation and the post-write filesystem measurement. + +The controls behaved falsifiably: + +- untouched dense identity passed with zero hard-metric or anchor error; +- the anchor-drop negative control failed with one immutable-anchor violation; +- degraded identity and linear interpolation failed their applicable hard + gates; and +- the provisional empirical motif emitted real proposals but was not eligible: + its candidate-window refusal rate was `0.4166666666666667`, worst event-count + relative error was `0.6653645833333334` against the hard `0.50` threshold, + and its p99 triangle residual was `6.219778927497112` pips against the + advisory five-pip target. Its inter-arrival, path-variation, and + update-transition gates also failed. + +That motif result is intentionally published as a failure, not tuned away. +It gives #464 a concrete event-count, refusal, and cross-series acceptance +surface while preserving the rule that #463 does not select a winning model. + +The reference artifact SHA-256 values are: + +| Artifact | SHA-256 | +| --- | --- | +| manifest | `d1ddf45d68ade8c1ba4abc3df5a60a26483bb3eab950d4c29f53709e9214ed24` | +| motif index | `b32f355f2d2348682684723213a617035c98943ac67ac97eeb5518b81281f7d4` | +| leakage audit | `3f532feaebabf3f3503dd719b17a4c04398cd110c283f29fcd9c60bfee6269c5` | +| resource audit | `3c12b6d3749e557d25b18106bae424b2711b8c9be6861dec879a1fc140be9d6b` | +| scorecard | `841ffd9f2cfbef2578bf9cf339b6f045650423eff7a95b4caaadb4f233858b03` | + +## Nonclaims + +- A policy pass does not identify the ticks that historically went missing. +- A policy pass does not select an automatic winner or trading strategy. +- Negative-control failure is required evidence, not an implementation error. +- Bars and M1 data are not benchmark source inputs. +- Broker capture, fingerprinting, and transfer are outside this policy. diff --git a/docs/strategy-sensitivity-contracts.md b/docs/strategy-sensitivity-contracts.md new file mode 100644 index 00000000..15da6b01 --- /dev/null +++ b/docs/strategy-sensitivity-contracts.md @@ -0,0 +1,201 @@ +# Strategy-sensitivity contracts + +The version-one strategy-sensitivity layer measures whether the behavior of one +deterministic strategy/execution specification changes when the same historical +window is presented through observed, degraded, reconstructed, +broker-conditioned, unconditioned, or derived-bar data. It is a downstream +scientific diagnostic. It does not validate a reconstruction by itself and it +does not issue a trading, investment, or model-promotion recommendation. + +## Contract surface + +| Contract | Responsibility | +| --- | --- | +| `StrategySpecificationV1` | Method, implementation version, bounded parameters, and a content-derived strategy identity. | +| `StrategyExecutionSpecificationV1` | Entry latency, maximum quote wait, per-side slippage, per-side fixed cost, quote-crossing semantics, and normalized exposure. | +| `StrategyEvaluationPolicyV1` | Multiple horizons and hard case, quote, signal, pending-signal, slice, payload, and rounding bounds. | +| `StrategyEvaluationCaseV1` | One source artifact and exact aligned half-open symbol/time window, including information mode, audit identity, member, broker, scope, and bar interval. | +| `StrategyEvaluationPlanV1` | One strategy/execution/policy applied identically to every case and alignment group. | +| `StrategyQuoteV1` | Minimal bid/ask quote plus feed epoch, session, event state, sparsity, member, and broker context. | +| `StrategySignalV1` | A current-quote-bound, content-addressed long or short decision. | +| `StrategyWindowResultV1` | Bounded window status, cadence/spread support, counts, and online slice summaries without retained quotes or outcomes. | +| `StrategySliceResultV1` | Source/epoch/session/event/sparsity/broker/member/horizon execution response. | +| `StrategyUncertaintySummaryV1` | Cross-member and cross-window dispersion for one source/regime/horizon cell. | +| `StrategyRestorationResultV1` | Reverse-degradation distance of a reconstructed execution response from the dense reference relative to the degraded input. | +| `StrategySensitivityReportV1` | Deterministic plan, window evidence, uncertainty, restoration evidence, terminal rates, and trust labels. | + +Every input and result has a versioned schema plus a content-derived identifier. +Changing field meaning, accounting, time alignment, validity semantics, or +identity derivation requires a new schema version. + +## Source surfaces and exact alignment + +`StrategySourceKind` distinguishes: + +- untouched observed history; +- intentionally degraded modern holdouts; +- reconstructed ensemble streams; +- unconditioned reconstructions; +- broker-conditioned reconstructions; and +- bars derived from committed final events. + +Cases sharing an `alignment_window_id` must have the same normalized symbol and +the same `[start_ns,end_ns)` bounds. A plan rejects a source/member/broker role +that appears twice in the same aligned window. The strategy, execution +assumptions, and evaluation horizons live once on the plan, so source cases +cannot quietly use different logic. + +Derived-bar cases must name both `source_scope` and `bar_interval_code`. Their +`source_artifact_id` is expected to be the verified derived-bar manifest. This +prevents merged bars from being compared with observed-only bars without making +the support difference explicit. + +`StrategyQuoteV1` adapters bridge the existing product contracts: + +- `from_benchmark_event()` handles observed, degraded, control, and candidate + reverse-degradation events; +- `from_synthetic_event()` handles final observed/generated reconstruction + events after the caller supplies the point-in-time session/event context; and +- `from_derived_bar()` uses the verified bar close and never invents volume. + +Quote streams must be strictly ordered by +`(event_time_ns,event_sequence,quote_id)`, stay inside the case window, and +match its symbol, ensemble member, and broker profile where declared. + +## Information modes and the invalid-for-backtest boundary + +Every case binds an `InformationAuditReportV1`. The evaluator verifies the run, +manifest, audit identity, and information mode and requires an accepted audit. +An ex-ante case that claims prospective usefulness additionally requires the +existing `valid_for_strategy_usefulness_claim` gate. + +Ex-post reconstruction is useful for historical counterfactual diagnosis but +is not point-in-time-valid strategy evidence. Every ex-post case therefore +requires an `invalid_for_backtest_reason`. Mixing ex-ante and ex-post cases in +one plan additionally requires an explicit plan-level reason. The case, plan, +window result, and final report then serialize: + +```json +{ + "valid_for_backtest": false, + "backtest_label": "invalid-for-backtest" +} +``` + +An explicit label permits a descriptive comparison; it does not make the +comparison prospective-valid. + +## Pluggable strategy boundary + +`StrategySignalEngineV1` exposes a versioned `StrategySpecificationV1` and +creates fresh `StrategySignalStateV1` state for every case. Window-local state +consumes one current quote at a time and may emit only signals bound to that +quote and decision time. The evaluator rejects implementation/specification +drift and signals that refer to another quote or time. + +`ReferenceMomentumStrategyV1` is the transparent fixture. It compares the +current midpoint with the last bounded quote at or before a configured +time-based lookback, rate-limits decisions, applies a threshold in basis points, +and emits a long or short signal. Its bounded deque is reset for each case. It +exists to validate alignment and accounting semantics, not as a recommended +trading strategy. + +## Execution response accounting + +Version one uses normalized unit exposure and always crosses the quoted market: + +- long entry at ask and exit at bid; +- short entry at bid and exit at ask; +- configured per-side slippage worsens both prices; +- configured per-side fixed costs are subtracted twice; +- entry uses the first quote at or after `decision_time + entry_latency_ns`; +- entry and horizon exits must arrive within `max_execution_wait_ns`. + +Each completed signal/horizon records only online aggregates for: + +- gross signed midpoint response in basis points; +- net executable response after spread, slippage, and fixed costs; +- cost drag; +- actual entry delay; and +- favorable-response rate. + +These are normalized response/sensitivity quantities, not currency P&L. The +report fixes `profit_claim`, `investment_recommendation`, and +`automatic_winner` to false. + +## Stratification, uncertainty, and reverse degradation + +Signal outcomes are accumulated under the decision quote's complete key: + +```text +source kind, symbol, feed epoch, session, event state, sparsity, +broker profile, ensemble member, horizon +``` + +Uncertainty summaries retain member IDs and report cross-member/cross-window +mean, range, and population standard deviation of the mean net executable +response. Different horizons remain separate cells. + +Where one aligned cell contains dense observed, degraded holdout, and +reconstructed results, the restoration result computes: + +```text +degraded_error = abs(degraded_response - dense_response) +candidate_error = abs(candidate_response - dense_response) +restoration_gain = degraded_error - candidate_error +approaches_dense_reference = candidate_error <= degraded_error +``` + +Sparsity remains visible on each result but is not part of the restoration join +because dense, degraded, and reconstructed surfaces necessarily have different +support labels. Missing dense or degraded comparators increment a bounded +`restoration_unavailable_count`; they never imply success. + +`strategy_sensitivity_benchmark_hooks()` projects a completed window into the +existing #436 `strategy_hooks` surface, including the canonical +`downstream_sensitivity` consumed by #442 ensemble calibration plus gross +response, cost drag, entry delay, and missing-support rate. Non-completed +windows refuse hook projection instead of receiving a plausible zero; their +missing hook preserves the existing ensemble-member refusal behavior. + +## Terminal states and rates + +Every planned case produces exactly one terminal window result: + +| Status | Meaning | +| --- | --- | +| `completed` | At least one signal/horizon had valid entry and exit support. | +| `no_trade` | Quotes were present but the identical strategy emitted no signal. | +| `missing_support` | The stream was absent/empty or no signal horizon could be completed. | +| `refused` | A configured quote, signal, pending-signal, slice, or payload resource ceiling stopped evaluation. | +| `failed` | A strategy plugin reported an explicit scientific/evaluation failure. | + +The final summary includes counts and rates for failure, no-trade, +missing-support, and refusal plus the missing-support outcome rate. Contract or +ordering violations raise and fail closed instead of becoming a plausible +window result. + +## Streaming and storage + +Cases are evaluated sequentially. The evaluator retains only: + +- the strategy's bounded current state; +- bounded pending signals until their largest horizon resolves; and +- constant-size numeric accumulators per bounded slice. + +It does not retain the 521-column analytical surface, whole quote streams, or +individual signal outcomes. Window metadata fixes `quotes_retained` and +`outcomes_retained` to false. The report uses +`bounded-derived-metadata`, and `event_schema_augmented` is false. Strategy +sensitivity is therefore a compact replayable side artifact, not another set of +per-tick augmented columns. + +## Downstream obligations + +- #449 must run identical strategy/execution plans across the certified + EURUSD/GBPUSD/EURGBP source surfaces and include terminal/support rates, + member uncertainty, and restoration evidence in its acceptance dossier. +- A strategy result remains one downstream criterion alongside structural, + cross-currency, carving, broker, activity, and reverse-degradation evidence. +- No caller may promote a reconstruction, strategy, or ensemble member solely + because its strategy response looks favorable. diff --git a/docs/synthetic-event-contracts.md b/docs/synthetic-event-contracts.md new file mode 100644 index 00000000..d0b968ca --- /dev/null +++ b/docs/synthetic-event-contracts.md @@ -0,0 +1,126 @@ +# Synthetic event contracts + +The version-one synthetic contracts define the narrow durable row boundary for +historical reconstruction. They do not replace the enriched ASCII tick frame. +The 521-column frame remains an in-memory computation surface; accepted output +uses the fields below. + +## Contract boundaries + +- `SyntheticEventV1` represents one immutable observation or one generated + event. +- `SyntheticEventStreamV1` represents one symbol, run, and ensemble member. +- `SyntheticEnsembleManifestV1` records compact member counts, stream IDs, and + content hashes without embedding event rows. +- `histdatacom.data_quality.synthetic_generation` remains the same-cardinality + Stage-0 control. It does not emit these variable-cardinality events. +- Window planning/checkpoints belong to #432. The implemented empirical-motif + candidate generator is documented in + [`empirical-motif-generation-contracts.md`](empirical-motif-generation-contracts.md), + while #446 implements atomic final publication/partition layout in + `histdatacom.synthetic.persistence`. + +## Event ordering and identity + +`event_time_ns` is a signed 64-bit count of UTC nanoseconds since the Unix +epoch. `event_sequence` is a non-negative signed 64-bit integer assigned +stably within one timestamp. A symbol stream rejects duplicate +`(event_time_ns, event_sequence)` pairs and sorts by time, sequence, then event +ID. Duplicate timestamps therefore remain distinct and deterministic. +First-party ASCII reconstruction assigns same-timestamp observed sequences in +immutable `(source_period, Arrow row ordinal)` order. Quote values never act as +an ordering key. + +Observed event IDs are derived from schema, symbol, event position, source +version, `source_series_id`, `source_period`, and immutable `source_row_id`. +Run and ensemble member do not alter an observed event's identity. + +Synthetic event IDs are derived from schema, event position, run/member, +ordered anchors, generator/version/configuration, source version, optional +reference/motif/feed/broker lineage, and the constraint set. Process count, +retry count, and window/partition placement are not identity inputs. + +The source/configuration version is responsible for price-mark semantics. +Changing generator semantics without changing its version/configuration is a +contract violation. + +## Origin-specific lineage + +Observed events require: + +- `source_version_id` +- `source_series_id` +- `source_period` +- positive `source_row_id` + +They reject synthetic lineage so an observed row cannot be relabeled as an +invention. + +Synthetic events require: + +- `source_version_id` +- ordered left and right anchor event IDs and `anchor_interval_id` +- `generator_id`, `generator_version`, and `generator_config_id` +- `constraint_set_id` + +Reference, motif, feed-epoch, and broker-profile IDs are nullable because not +every generator stage has used them yet. If used, they are row-aligned scalar +lineage. Confidence is nullable until a stage has a defined, versioned +calibration quantity and scope; supplied values must be finite and in +`[0, 1]`. Raw motif-match similarity is not such a quantity and is retained +only on empirical-motif transformation evidence. The #442 ensemble confidence +quantity is exact-stratum metric/horizon interval coverage and does not +populate per-event confidence. Synthetic events reject observed source-row +identity. + +## Flat Arrow and Parquet schema + +The Arrow schema contains 26 scalar columns: + +```text +schema_version event_id origin symbol event_time_ns event_sequence bid ask +run_id ensemble_member_id source_version_id source_series_id source_period +source_row_id anchor_interval_id left_anchor_event_id right_anchor_event_id +generator_id generator_version generator_config_id reference_id motif_id +feed_epoch_id broker_profile_id constraint_set_id confidence +``` + +It intentionally contains no `dq_*`, `cm_*`, or same-row `synth_*` analytical +columns. Bid and ask are finite positive Float64 values; event time, sequence, +and source row ID use signed Int64. Stream identity/count metadata is bounded +Arrow schema metadata rather than a repeated nested report. + +Arrow remains optional. Importing `histdatacom.synthetic` does not import +PyArrow; Arrow/Parquet helpers require the `histdatacom[arrow]` extra only when +called. Parquet helpers use a fixed Zstandard/version/data-page configuration +for stable output under the same pinned runtime. The file helper is +intentionally non-atomic: callers that need the final product use #446's +temporary-path validation, checksums, and atomic publication layer. + +## Schema evolution + +Version-one class names, ID derivations, required fields, ordering rules, and +Arrow types are frozen. A semantic change requires a new schema version and +new contract class. Version-one readers reject other schema versions and +Arrow schema drift. + +For JSON compatibility, missing nullable fields are treated as null and +unknown keys are ignored. This permits bounded envelope metadata to evolve +without silently changing the persisted version-one row. Deterministic IDs are +recomputed on every read; a supplied event, stream, or ensemble ID that does +not match its canonical identity fails closed. + +## Streaming use + +These classes are correctness contracts, not permission to place an entire +period in Temporal workflow history. #432 must move event batches through +artifact references and bounded checkpoints. `SyntheticEventStreamV1` is +suitable for a bounded window/partition or test artifact; final period-scale +storage must use the Arrow schema incrementally. + +The implemented window, batch, carry, checkpoint, resource, and two-phase +publication protocol is documented in +[`reconstruction-streaming-contracts.md`](reconstruction-streaming-contracts.md). +Run-bound ex-post/ex-ante modes, artifact availability, chronological splits, +window-plan look-ahead, and the fail-closed leakage gate are documented in +[`reconstruction-information-modes.md`](reconstruction-information-modes.md). diff --git a/docs/temporal-orchestration-runtime-runbook.md b/docs/temporal-orchestration-runtime-runbook.md index 79d5927d..8298e93c 100644 --- a/docs/temporal-orchestration-runtime-runbook.md +++ b/docs/temporal-orchestration-runtime-runbook.md @@ -208,7 +208,7 @@ Override the base directory with `HISTDATACOM_RUNTIME_HOME` or Each workspace gets a deterministic directory: -```txt +```text /workspaces/-/ ``` @@ -513,7 +513,7 @@ The workflow ID format is `histdatacom-`. Job snapshots are persisted under the workspace-scoped orchestration runtime manifests directory, not in HistData download/cache directories: -```txt +```text //manifests/.histdatacom/manifest-status.sqlite3 ``` @@ -584,7 +584,7 @@ Retry and resume are executable control operations, not intent-only labels. The client inspects the original job, reads the persisted `RunRequest` snapshot, and starts a deterministic replacement `HistDataRunWorkflow` with a workflow ID like: -```txt +```text histdatacom--retry--001 histdatacom--resume--001 ``` @@ -610,7 +610,7 @@ Workers use workspace-scoped task queues. Defaults: The queue name pattern is: -```txt +```text histdatacom.. ``` @@ -860,7 +860,7 @@ histdatacom --quality \ --quality-report reports/quality-clean.json ``` -```txt +```text Data quality assessment checks: ingestion status: clean @@ -915,7 +915,7 @@ histdatacom --quality \ --quality-report reports/quality-failing.json ``` -```txt +```text Data quality assessment checks: ingestion status: failed diff --git a/pyproject.toml b/pyproject.toml index caa9bb4d..821170d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,13 +50,17 @@ dependencies = [ "requests>=2.34.2", "rich>=15.0.0", "Rx>=3.2", - "temporalio>=1.10,<1.29", + "temporalio>=1.10,<1.31", + "tzdata>=2026.3", ] [project.optional-dependencies] arrow = [ "pyarrow>=24.0.0", ] +query = [ + "duckdb>=1.5.4,<2", +] influx = [ "influxdb-client>=1.50.0", ] @@ -68,18 +72,33 @@ jupyter = [ "ipywidgets", ] temporal = [ - "temporalio>=1.10,<1.29", + "temporalio>=1.10,<1.31", +] +models = [ + "arch>=8.0.0,<9", + "statsmodels>=0.14.6,<0.15", +] +docs = [ + "myst-parser==4.0.1", + "sphinx==8.1.3", + "sphinx-rtd-theme==3.1.0", ] all = [ + "arch>=8.0.0,<9", + "duckdb>=1.5.4,<2", "influxdb-client>=1.50.0", "ipywidgets", "jupyter", "pandas>=2.3.3", "pyarrow>=24.0.0", - "temporalio>=1.10,<1.29", + "statsmodels>=0.14.6,<0.15", + "temporalio>=1.10,<1.31", ] test = [ - "coverage==7.14.2", + "arch==8.0.0", + "coverage==7.15.0", + "duckdb==1.5.4", + "hypothesis==6.152.7", "ipykernel", "influxdb-client>=1.50.0", "nbclient", @@ -88,17 +107,18 @@ test = [ "pyarrow>=24.0.0", "pytest==9.1.1", "pytest-cov==7.1.0", - "temporalio>=1.10,<1.29", + "statsmodels==0.14.6", + "temporalio>=1.10,<1.31", ] lint = [ "black==26.5.1", - "commitizen==4.16.3", + "commitizen==4.16.4", "mypy==2.1.0", "pandas-stubs==3.0.3.260530; python_version >= '3.11'", "pre-commit==4.6.0", "pylint==4.0.6; python_version < '3.15'", "pyroma==5.0.1", - "ruff==0.15.18", + "ruff==0.15.20", "sh==2.3.0", "shellcheck-py==0.11.0.1", "types-PyYAML==6.0.12.20260518", @@ -106,20 +126,24 @@ lint = [ release = [ "build==1.5.0", "keyring==25.7.0", - "setuptools==82.0.1", + "setuptools==83.0.0", "twine==6.2.0", "wheel==0.47.0", ] dev = [ + "arch==8.0.0", "build==1.5.0", "black==26.5.1", - "commitizen==4.16.3", - "coverage==7.14.2", + "commitizen==4.16.4", + "coverage==7.15.0", + "duckdb==1.5.4", + "hypothesis==6.152.7", "ipykernel", "influxdb-client>=1.50.0", "ipywidgets", "jupyter", "keyring==25.7.0", + "myst-parser==4.0.1", "mypy==2.1.0", "nbclient", "nbformat", @@ -131,11 +155,14 @@ dev = [ "pyarrow>=24.0.0", "pytest==9.1.1", "pytest-cov==7.1.0", - "ruff==0.15.18", - "setuptools==82.0.1", + "ruff==0.15.20", + "setuptools==83.0.0", "sh==2.3.0", "shellcheck-py==0.11.0.1", - "temporalio>=1.10,<1.29", + "sphinx==8.1.3", + "sphinx-rtd-theme==3.1.0", + "statsmodels==0.14.6", + "temporalio>=1.10,<1.31", "twine==6.2.0", "types-PyYAML==6.0.12.20260518", "wheel==0.47.0", @@ -162,6 +189,12 @@ where = ["src"] "assets/bin/*/*", "assets/third-party/*/*", ] +"histdatacom.market_context" = [ + "assets/*.json", +] +"histdatacom.synthetic" = [ + "assets/*.json", +] [tool.setuptools.dynamic] version = { attr = "histdatacom.__version__" } diff --git a/scripts/closure_readiness.py b/scripts/closure_readiness.py index 89900117..9b6d2c94 100755 --- a/scripts/closure_readiness.py +++ b/scripts/closure_readiness.py @@ -338,7 +338,7 @@ def _elapsed(self, now: float) -> float: return round(now - self._started_at, 3) -GATE_SPECS = ( +PRE_TEST_GATE_SPECS = ( GateSpec( "readme-help-sync", (sys.executable, "scripts/sync_readme_cli_help.py", "--check"), @@ -350,13 +350,18 @@ def _elapsed(self, now: float) -> float: (sys.executable, "-m", "histdatacom", "--help"), "python -m histdatacom --help", ), - GateSpec("pytest", (sys.executable, "-m", "pytest"), "python -m pytest"), GateSpec( "pre-commit", (sys.executable, "-m", "pre_commit", "run", "--all-files"), "python -m pre_commit run --all-files", ), ) +FINAL_TEST_GATE = GateSpec( + "full-tests", + (sys.executable, "-m", "pytest"), + "python -m pytest", +) +GATE_SPECS = (*PRE_TEST_GATE_SPECS, FINAL_TEST_GATE) FORMATTER_MUTATION_GATES = frozenset({"pre-commit", "readme-help-sync"}) FORMATTER_MUTATION_SUFFIXES = frozenset( { @@ -869,8 +874,8 @@ def build_closure_verification_report( "README help sync", "git diff --check", "main CLI help smoke", - "full pytest", "full pre-commit", + "full plain pytest suite", "optional TestPyPI local simple-registry preflight", "final git status", "issue readback", @@ -2189,6 +2194,14 @@ def collect_gate_summary( "state": "not-run", "reason": "run with --run-gates to execute closure gates", "required": [gate.display for gate in GATE_SPECS], + "final_coverage": { + "state": "not-applicable", + "reason": ( + "coverage is enforced only for dev-to-main production " + "promotion" + ), + "result": {}, + }, "results": [], } results = [ @@ -2198,8 +2211,29 @@ def collect_gate_summary( runner=runner, monitored_paths=monitored_paths, ) - for gate in GATE_SPECS + for gate in PRE_TEST_GATE_SPECS ] + pre_test_payload = {"results": results} + pre_test_changed = _gate_changed_paths(pre_test_payload) + pre_test_failed = any( + _mapping(result).get("status") != "pass" for result in results + ) + if not pre_test_failed and not pre_test_changed: + results.append( + _run_gate( + repo_root, + FINAL_TEST_GATE, + runner=runner, + monitored_paths=monitored_paths, + ) + ) + final_coverage = { + "state": "not-applicable", + "reason": ( + "coverage is enforced only for dev-to-main production promotion" + ), + "result": {}, + } gate_payload = {"results": results} changed_after = _gate_changed_paths(gate_payload) gate_sources = _gate_changed_path_sources(gate_payload) @@ -2275,6 +2309,7 @@ def collect_gate_summary( "mutation_summary": mutation_summary, "required_rerun": required_rerun, "rerun": rerun_report, + "final_coverage": final_coverage, "results": results, } @@ -4677,7 +4712,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument( "--run-gates", action="store_true", - help="run pytest, pre-commit, help sync, diff check, and help smoke", + help="run the full plain test and pre-commit closure gates", ) parser.add_argument( "--rerun-standalone-formatter-mutations", @@ -6236,6 +6271,14 @@ def _pre_mutation_gates_not_run(*, enabled: bool) -> dict[str, Any]: "state": "not-run", "reason": reason, "required": [gate.display for gate in GATE_SPECS], + "final_coverage": { + "state": "not-applicable", + "reason": ( + "coverage is enforced only for dev-to-main production " + "promotion" + ), + "result": {}, + }, "results": [], }, "results": [], @@ -7012,8 +7055,8 @@ def _closure_verification_command_plan( "state": "ready", "execute_workflow": _shell_command(parts), "notes": [ - "execute-workflow will rerun closure gates before mutation", - "execute-workflow will rerun closure gates after push before close", + "execute-workflow will ensure closure gates before mutation", + "production coverage is reserved for dev-to-main promotion", ], } diff --git a/scripts/inspect_wheel.py b/scripts/inspect_wheel.py index e08d6a9c..cf92b60f 100644 --- a/scripts/inspect_wheel.py +++ b/scripts/inspect_wheel.py @@ -12,6 +12,7 @@ from zipfile import ZipFile EXPECTED_BASE_RUNTIME_ASSETS = { + "histdatacom/market_context/assets/operator_shocks_v1.json", "histdatacom/orchestration/assets/README.md", "histdatacom/orchestration/assets/manifest.json", "histdatacom/orchestration/assets/runtime-defaults.json", @@ -446,8 +447,11 @@ def inspect_wheel( f"console script missing from wheel metadata: {console_script}" ) provides_extra = set(wheel_metadata.get_all("Provides-Extra", [])) - if "temporal" not in provides_extra: - raise SystemExit("temporal optional extra missing from wheel metadata") + for required_extra in ("models", "temporal"): + if required_extra not in provides_extra: + raise SystemExit( + f"{required_extra} optional extra missing from wheel metadata" + ) classifiers = set(wheel_metadata.get_all("Classifier", [])) missing_classifiers = sorted(EXPECTED_METADATA_CLASSIFIERS - classifiers) if missing_classifiers: @@ -464,6 +468,11 @@ def inspect_wheel( dependency="temporalio", ): raise SystemExit("temporalio dependency missing from core metadata") + if not _requires_dist_core_contains( + requires_dist, + dependency="tzdata", + ): + raise SystemExit("tzdata dependency missing from core metadata") if not _requires_dist_contains( requires_dist, dependency="temporalio", @@ -476,6 +485,18 @@ def inspect_wheel( extra="all", ): raise SystemExit("temporalio dependency missing from all extra") + if not _requires_dist_contains( + requires_dist, + dependency="statsmodels", + extra="models", + ): + raise SystemExit("statsmodels dependency missing from models extra") + if not _requires_dist_contains( + requires_dist, + dependency="statsmodels", + extra="all", + ): + raise SystemExit("statsmodels dependency missing from all extra") if manifest["runtime"] != "temporal": raise SystemExit("runtime manifest does not describe Temporal") if manifest["distribution_strategy"] != ( diff --git a/scripts/smoke_container.py b/scripts/smoke_container.py new file mode 100644 index 00000000..0c304fb6 --- /dev/null +++ b/scripts/smoke_container.py @@ -0,0 +1,467 @@ +"""Build and verify the distributable histdatacom container image.""" + +from __future__ import annotations + +import argparse +import json +import secrets +import subprocess +import sys +from collections.abc import Callable, Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +DEFAULT_IMAGE = "histdatacom:local-smoke" +DEFAULT_CONTEXT = Path(__file__).resolve().parents[1] +CONTAINER_UID = 10001 +CONTAINER_GID = 10001 +WORKSPACE = "/workspace" +REQUIRED_ENV = { + "HISTDATACOM_RUNTIME_HOME": "/workspace/runtime", + "HISTDATACOM_RUNTIME_WORKSPACE": WORKSPACE, + "HISTDATACOM_TEMPORAL_CACHE_DIR": "/workspace/cache/temporal-cli", +} +REQUIRED_LABELS = { + "org.opencontainers.image.created", + "org.opencontainers.image.description", + "org.opencontainers.image.documentation", + "org.opencontainers.image.licenses", + "org.opencontainers.image.revision", + "org.opencontainers.image.source", + "org.opencontainers.image.title", + "org.opencontainers.image.version", +} + +RunCommand = Callable[..., subprocess.CompletedProcess[str]] + + +class ContainerSmokeError(RuntimeError): + """Raised when a container distribution contract is not satisfied.""" + + def __init__(self, message: str, diagnostics: Mapping[str, Any]) -> None: + super().__init__(message) + self.diagnostics = dict(diagnostics) + + +def _run( + command: Sequence[str], + *, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + """Run one command and retain bounded diagnostics on failure.""" + completed = subprocess.run( + list(command), + capture_output=True, + check=False, + text=True, + ) + if check and completed.returncode != 0: + raise ContainerSmokeError( + f"command failed with exit {completed.returncode}: " f"{' '.join(command)}", + { + "command": list(command), + "returncode": completed.returncode, + "stdout": completed.stdout[-8000:], + "stderr": completed.stderr[-8000:], + }, + ) + return completed + + +def _json_output(completed: subprocess.CompletedProcess[str]) -> Any: + """Decode a command's JSON stdout with useful failure evidence.""" + try: + return json.loads(completed.stdout) + except json.JSONDecodeError as err: + raise ContainerSmokeError( + "command did not emit valid JSON", + { + "command": list(completed.args), + "stdout": completed.stdout[-8000:], + "stderr": completed.stderr[-8000:], + }, + ) from err + + +def build_image( + image: str, + *, + context: Path = DEFAULT_CONTEXT, + version: str = "local-smoke", + revision: str = "local", + created: str | None = None, + run_command: RunCommand = _run, +) -> None: + """Build the native-platform smoke image with explicit OCI metadata.""" + timestamp = created or datetime.now(tz=UTC).replace(microsecond=0).isoformat() + run_command( + [ + "docker", + "build", + "--tag", + image, + "--build-arg", + f"IMAGE_CREATED={timestamp.replace('+00:00', 'Z')}", + "--build-arg", + f"IMAGE_REVISION={revision}", + "--build-arg", + f"IMAGE_VERSION={version}", + str(context), + ] + ) + + +def inspect_image( + image: str, + *, + run_command: RunCommand = _run, +) -> dict[str, Any]: + """Return and validate the image configuration contract.""" + payload = _json_output(run_command(["docker", "image", "inspect", image])) + if not isinstance(payload, list) or len(payload) != 1: + raise ContainerSmokeError( + "docker image inspect returned an unexpected payload", + {"image": image, "payload": payload}, + ) + inspected = payload[0] + if not isinstance(inspected, dict): + raise ContainerSmokeError( + "docker image inspect entry is not an object", + {"image": image, "payload": inspected}, + ) + config = inspected.get("Config") + if not isinstance(config, dict): + raise ContainerSmokeError( + "docker image inspect is missing Config", + {"image": image, "payload": inspected}, + ) + + env = {} + for item in config.get("Env", []): + key, separator, value = str(item).partition("=") + if separator: + env[key] = value + labels = config.get("Labels") or {} + failures: list[str] = [] + if config.get("User") != f"{CONTAINER_UID}:{CONTAINER_GID}": + failures.append("fixed non-root user is not configured") + if config.get("WorkingDir") != WORKSPACE: + failures.append("working directory is not /workspace") + if config.get("Entrypoint") != ["/usr/bin/tini", "--", "histdatacom"]: + failures.append("tini-backed histdatacom entry point is not configured") + if config.get("Cmd") != ["--help"]: + failures.append("bounded help is not the default command") + if config.get("Healthcheck") is not None: + failures.append("one-shot CLI image declares a service health check") + if config.get("Volumes") is not None: + failures.append("image declares hidden anonymous storage") + for key, expected in REQUIRED_ENV.items(): + if env.get(key) != expected: + failures.append(f"{key} does not resolve beneath /workspace") + missing_labels = sorted(REQUIRED_LABELS.difference(labels)) + if missing_labels: + failures.append("missing OCI labels: " + ", ".join(missing_labels)) + if failures: + raise ContainerSmokeError( + "container image configuration contract failed", + {"image": image, "failures": failures, "config": config}, + ) + return inspected + + +_WRITE_PROBE = """ +import json +import os +from pathlib import Path + +paths = [ + Path('/workspace/data'), + Path('/workspace/runtime'), + Path('/workspace/cache/temporal-cli'), +] +for path in paths: + marker = path / '.histdatacom-container-write-probe' + marker.write_text('ok', encoding='utf-8') + marker.unlink() +print(json.dumps({ + 'gid': os.getgid(), + 'paths': [str(path) for path in paths], + 'uid': os.getuid(), +})) +""" + + +_RUNTIME_PROBE = """ +import json +import subprocess + +def run(command): + completed = subprocess.run( + command, + capture_output=True, + check=False, + text=True, + ) + if completed.returncode != 0: + raise RuntimeError(json.dumps({ + 'command': command, + 'returncode': completed.returncode, + 'stdout': completed.stdout[-8000:], + 'stderr': completed.stderr[-8000:], + }, sort_keys=True)) + return json.loads(completed.stdout) + +start = None +doctor = None +stop = None +try: + start = run([ + 'histdatacom', 'runtime', 'start', '--json', + '--startup-timeout', '60', + ]) + doctor = run(['histdatacom', 'runtime', 'doctor', '--json']) +finally: + if start is not None: + stop = run(['histdatacom', 'runtime', 'stop', '--json']) + +print(json.dumps({ + 'doctor': doctor, + 'start': start, + 'stop': stop, +}, sort_keys=True)) +""" + + +def _run_cli_smoke( + image: str, + *, + run_command: RunCommand, +) -> dict[str, Any]: + version = run_command(["docker", "run", "--rm", image, "--version"]) + help_result = run_command(["docker", "run", "--rm", image, "--help"]) + write_probe = _json_output( + run_command( + [ + "docker", + "run", + "--rm", + "--entrypoint", + "python", + image, + "-c", + _WRITE_PROBE, + ] + ) + ) + expected_identity = {"uid": CONTAINER_UID, "gid": CONTAINER_GID} + actual_identity = { + "uid": write_probe.get("uid"), + "gid": write_probe.get("gid"), + } + if actual_identity != expected_identity: + raise ContainerSmokeError( + "container process did not run with the expected identity", + {"expected": expected_identity, "actual": actual_identity}, + ) + if not version.stdout.strip() or "usage: histdatacom" not in help_result.stdout: + raise ContainerSmokeError( + "container CLI version/help smoke failed", + { + "version": version.stdout[-2000:], + "help": help_result.stdout[-4000:], + }, + ) + return { + "help_line_count": len(help_result.stdout.splitlines()), + "version": version.stdout.strip(), + "write_probe": write_probe, + } + + +def _run_runtime_smoke( + image: str, + volume: str, + *, + run_command: RunCommand, +) -> dict[str, Any]: + mount = f"type=volume,source={volume},target={WORKSPACE}" + first_pass = _json_output( + run_command( + [ + "docker", + "run", + "--rm", + "--mount", + mount, + "--entrypoint", + "/usr/bin/tini", + image, + "--", + "python", + "-c", + _RUNTIME_PROBE, + ] + ) + ) + persisted_doctor = _json_output( + run_command( + [ + "docker", + "run", + "--rm", + "--mount", + mount, + image, + "runtime", + "doctor", + "--json", + ] + ) + ) + provisioning = persisted_doctor.get("runtime_provisioning", {}) + start = first_pass.get("start") or {} + stop = first_pass.get("stop") or {} + failures: list[str] = [] + if start.get("state") != "running": + failures.append("runtime did not reach running state") + if stop.get("state") != "stopped": + failures.append("runtime did not stop cleanly") + if not provisioning.get("cache_available"): + failures.append("verified Temporal cache did not persist") + if not provisioning.get("cache_entries"): + failures.append("Temporal cache provenance is missing") + if failures: + raise ContainerSmokeError( + "connected runtime container smoke failed", + { + "failures": failures, + "first_pass": first_pass, + "persisted_doctor": persisted_doctor, + }, + ) + return { + "platform": persisted_doctor.get("platform"), + "runtime_provisioning": provisioning, + "start_state": start.get("state"), + "stop_state": stop.get("state"), + "workspace_volume": volume, + } + + +def run_container_smoke( + *, + image: str = DEFAULT_IMAGE, + context: Path = DEFAULT_CONTEXT, + deep_runtime: bool = False, + keep_image: bool = False, + keep_workspace_volume: bool = False, + version: str = "local-smoke", + revision: str = "local", + created: str | None = None, + run_command: RunCommand = _run, +) -> dict[str, Any]: + """Build, inspect, execute, and normally clean the application image.""" + volume = f"histdatacom-smoke-{secrets.token_hex(6)}" + image_built = False + volume_created = False + report: dict[str, Any] = { + "deep_runtime": deep_runtime, + "image": image, + "status": "running", + } + try: + build_image( + image, + context=context, + version=version, + revision=revision, + created=created, + run_command=run_command, + ) + image_built = True + inspected = inspect_image(image, run_command=run_command) + report["image_id"] = inspected.get("Id") + report["platform"] = { + "architecture": inspected.get("Architecture"), + "os": inspected.get("Os"), + } + report["cli"] = _run_cli_smoke(image, run_command=run_command) + if deep_runtime: + run_command(["docker", "volume", "create", volume]) + volume_created = True + report["runtime"] = _run_runtime_smoke( + image, + volume, + run_command=run_command, + ) + report["status"] = "passed" + return report + finally: + if volume_created and not keep_workspace_volume: + run_command( + ["docker", "volume", "rm", "--force", volume], + check=False, + ) + if image_built and not keep_image: + run_command( + ["docker", "image", "rm", "--force", image], + check=False, + ) + + +def build_parser() -> argparse.ArgumentParser: + """Build the container smoke CLI parser.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", default=DEFAULT_IMAGE) + parser.add_argument("--context", type=Path, default=DEFAULT_CONTEXT) + parser.add_argument( + "--deep-runtime", + action="store_true", + help=( + "download the pinned Temporal runtime and exercise " + "start/doctor/stop with a persistent named volume" + ), + ) + parser.add_argument("--keep-image", action="store_true") + parser.add_argument("--keep-workspace-volume", action="store_true") + parser.add_argument("--version", default="local-smoke") + parser.add_argument("--revision", default="local") + parser.add_argument("--created") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the image smoke and emit one machine-readable report.""" + args = build_parser().parse_args(argv) + try: + report = run_container_smoke( + image=args.image, + context=args.context, + deep_runtime=args.deep_runtime, + keep_image=args.keep_image, + keep_workspace_volume=args.keep_workspace_volume, + version=args.version, + revision=args.revision, + created=args.created, + ) + except ContainerSmokeError as err: + print( # noqa: T201 + json.dumps( + { + "diagnostics": err.diagnostics, + "error": str(err), + "status": "failed", + }, + indent=2, + sort_keys=True, + ), + file=sys.stderr, + ) + return 1 + print(json.dumps(report, indent=2, sort_keys=True)) # noqa: T201 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_influx_docker.py b/scripts/smoke_influx_docker.py index 3180f975..dd2f71b7 100644 --- a/scripts/smoke_influx_docker.py +++ b/scripts/smoke_influx_docker.py @@ -13,7 +13,7 @@ import urllib.request from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Any +from typing import Any, cast DEFAULT_INFLUX_IMAGE = "influxdb:2.7-alpine" DEFAULT_ORG = "histdatacom" @@ -24,13 +24,16 @@ DEFAULT_STARTUP_TIMEOUT = 60.0 INFLUX_PORT = "8086/tcp" SMOKE_MEASUREMENT = "eurusd" -SMOKE_LINES = ( +SMOKE_ROWS = ( + (1328072403660, 1.3066, 1.30677, 0), + (1328072403973, 1.30658, 1.30675, 25), +) +LEGACY_SMOKE_LINES = ( "eurusd,source=histdata.com,format=ascii,timeframe=T " "bidquote=1.3066,askquote=1.30677 1328072403660", "eurusd,source=histdata.com,format=ascii,timeframe=T " "bidquote=1.30658,askquote=1.30675 1328072403973", ) -EXPECTED_FIELD_COUNT = 2 RunCommand = Callable[..., subprocess.CompletedProcess[str]] HealthWaiter = Callable[[str, float], Mapping[str, Any]] @@ -44,6 +47,56 @@ def __init__(self, message: str, diagnostics: Mapping[str, Any]) -> None: self.diagnostics = dict(diagnostics) +def build_enriched_smoke_lines() -> tuple[str, ...]: + """Return representative enriched ASCII tick line-protocol rows.""" + import polars as pl + + from histdatacom.data_quality.training_features import ( + enrich_tick_cache_with_training_features, + ) + from histdatacom.histdata_ascii import format_influx_line + + frame = enrich_tick_cache_with_training_features( + pl.DataFrame( + { + "datetime": [row[0] for row in SMOKE_ROWS], + "bid": [row[1] for row in SMOKE_ROWS], + "ask": [row[2] for row in SMOKE_ROWS], + "vol": [row[3] for row in SMOKE_ROWS], + }, + schema={ + "datetime": pl.Int64, + "bid": pl.Float64, + "ask": pl.Float64, + "vol": pl.Int32, + }, + ), + symbol="EURUSD", + data_format="ascii", + timeframe="T", + period="201202", + ) + return tuple( + format_influx_line( + SMOKE_MEASUREMENT, + "ascii", + "T", + row, + columns=frame.columns, + ) + for row in frame.iter_rows() + ) + + +def line_field_count(line: str) -> int: + """Return the number of fields in one Influx line-protocol row.""" + return len(line.split(" ", maxsplit=2)[1].split(",")) + + +SMOKE_LINES = build_enriched_smoke_lines() +EXPECTED_FIELD_COUNT = sum(line_field_count(line) for line in SMOKE_LINES) + + @dataclass(frozen=True, slots=True) class InfluxContainer: """Running InfluxDB container details.""" @@ -301,13 +354,13 @@ def _sum_query_record_values(tables: Any) -> int: def _influx_batch_writer_factory() -> Callable[[Mapping[str, Any]], Any]: from histdatacom.influx import InfluxBatchWriter - return InfluxBatchWriter + return cast(Callable[[Mapping[str, Any]], Any], InfluxBatchWriter) def _influx_client_factory() -> Callable[..., Any]: from influxdb_client import InfluxDBClient - return InfluxDBClient + return cast(Callable[..., Any], InfluxDBClient) def run_docker_influx_smoke( diff --git a/scripts/smoke_runtime_install.py b/scripts/smoke_runtime_install.py index 6ac4175b..6101dc9b 100644 --- a/scripts/smoke_runtime_install.py +++ b/scripts/smoke_runtime_install.py @@ -1093,6 +1093,9 @@ def check_package_metadata(*, expect_temporal_extra: bool) -> dict[str, Any]: temporalio_version = metadata.version("temporalio") if importlib.util.find_spec("temporalio") is None: raise SystemExit("temporalio distribution is installed but missing") + tzdata_version = metadata.version("tzdata") + if importlib.util.find_spec("tzdata") is None: + raise SystemExit("tzdata distribution is installed but missing") if RunRequest is not RuntimeRunRequest: raise SystemExit( "orchestration contract RunRequest does not match runtime contract" @@ -1104,6 +1107,7 @@ def check_package_metadata(*, expect_temporal_extra: bool) -> dict[str, Any]: "console_scripts": sorted(EXPECTED_CONSOLE_SCRIPTS), "orchestration_contracts": ["RunRequest"], "temporalio_version": temporalio_version, + "tzdata_version": tzdata_version, } diff --git a/src/histdatacom/__init__.py b/src/histdatacom/__init__.py index 4e9d214d..7720a329 100644 --- a/src/histdatacom/__init__.py +++ b/src/histdatacom/__init__.py @@ -9,7 +9,7 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from histdatacom.fx_enums import Format, Pairs, Timeframe from histdatacom.options import Options @@ -24,12 +24,46 @@ "Pairs", "Timeframe", "Format", + "ReconstructionClient", + "ReconstructionExecutionRequestV1", + "ReconstructionExitCode", + "ReconstructionOperationReceiptV1", + "ReconstructionPlanSetPreflightV1", + "ReconstructionPlanSetV1", + "ReconstructionPlanShardV1", + "ReconstructionPlanSpecV1", + "ReconstructionPreflightV1", ] -__version__ = "2.0.1" +__version__ = "2.1.3" __author__ = "David Midlo" +_RECONSTRUCTION_EXPORTS = frozenset( + { + "ReconstructionClient", + "ReconstructionExecutionRequestV1", + "ReconstructionExitCode", + "ReconstructionOperationReceiptV1", + "ReconstructionPlanSetPreflightV1", + "ReconstructionPlanSetV1", + "ReconstructionPlanShardV1", + "ReconstructionPlanSpecV1", + "ReconstructionPreflightV1", + } +) + + +def __getattr__(name: str) -> Any: + """Lazily expose the typed reconstruction facade without import cycles.""" + if name not in _RECONSTRUCTION_EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module("histdatacom.reconstruction"), name) + globals()[name] = value + return value + class APICaller(sys.modules[__name__].__class__): # type: ignore # noqa:H601 """APICaller. A Masquerade class. diff --git a/src/histdatacom/activity_stages.py b/src/histdatacom/activity_stages.py index 4bdcf2a1..6638056d 100644 --- a/src/histdatacom/activity_stages.py +++ b/src/histdatacom/activity_stages.py @@ -16,6 +16,7 @@ from typing import ( Any, Callable, + cast, Iterable, Mapping, MutableMapping, @@ -40,12 +41,22 @@ from histdatacom.histdata_ascii import ( CACHE_FILENAME, convert_polars_datetime_to_utc_ms, + filename_has_unsupported_raw_dimensions, format_influx_line, read_ascii_file_to_polars, read_polars_cache, write_polars_cache, ) from histdatacom.records import Record +from histdatacom.random_windows import ( + RANDOM_WINDOW_SELECTION_METADATA_KEY, + RandomWindowEmptySelectionError, + RandomWindowSelectionV1, + filter_polars_frame_to_random_window, + random_window_planning_yearmonths, + random_window_selection_from_metadata, + resolve_random_window_selection, +) from histdatacom.repository_quality import REPOSITORY_QUALITY_KEY from histdatacom.runtime_contracts import ( ArtifactRef, @@ -59,6 +70,7 @@ status_has_csv_artifact, ) from histdatacom.utils import ( + OUTPUT_DATETIME_COLUMN, check_installed_module, create_full_path, force_datemonth_if_only_year, @@ -67,6 +79,7 @@ get_now_utc_timestamp, get_year_from_datemonth, hash_dict, + normalize_output_timezone, ) DEFAULT_REPOSITORY_URL = ( @@ -397,6 +410,11 @@ def validate_url_work_item( _check_form_token(record.data_tk) else: check_for_valid_download(record) + _validate_url_raw_dimensions( + record.data_format, + record.data_timeframe, + url=record.url, + ) updated = _work_item_from_record(record, work_item) else: page_fetcher = fetch_page_data or fetch_histdata_page_data @@ -592,6 +610,12 @@ def apply_form_metadata_to_work_item( metadata: UrlFormMetadata, ) -> WorkItem: """Return a validated work item with parsed form metadata applied.""" + _validate_url_raw_dimensions( + metadata.data_format, + metadata.data_timeframe, + url=work_item.url, + ) + _validate_form_metadata_matches_plan(work_item, metadata) return replace( work_item, status=WorkStatus.URL_VALID, @@ -624,6 +648,7 @@ def download_archive_work_item( """Download one ZIP archive through an explicit work item.""" record = _record_from_work_item(work_item) try: + _validate_archive_raw_dimensions(record) _ensure_record_data_dir(record, args) existing = existing_archive_artifact(record) if existing is not None: @@ -665,6 +690,7 @@ def download_archive_work_item( download_file(record) download_result = archive_download_result_for_record(record) + _validate_archive_raw_dimensions(record) record.status = WorkStatus.CSV_ZIP record.write_manifest_status(base_dir=_default_download_dir(args)) updated = _work_item_from_record(record, work_item) @@ -747,6 +773,7 @@ def download_histdata_archive_to_record( post_archive=post_archive, ) filename = archive_filename_from_response(response) + _validate_downloaded_archive_filename(filename, record) content = _response_content_bytes(response) target_path = atomic_write_zip_archive( Path(record.data_dir), @@ -843,6 +870,13 @@ def atomic_write_zip_archive( work_id: str, ) -> Path: """Write a ZIP through a temp file, validate it, then rename.""" + if filename_has_unsupported_raw_dimensions(filename): + raise ArchiveDownloadError( + "UNSUPPORTED_RAW_INPUT", + "Archive filename declares a retired raw dimension.", + retryable=False, + detail={"filename": filename}, + ) target_path = data_dir / filename temp_path = target_path.with_name( f".{target_path.name}.{derive_work_id(work_id).removeprefix('work-')}.tmp" @@ -853,6 +887,9 @@ def atomic_write_zip_archive( _validate_zip_payload(temp_path) temp_path.replace(target_path) return target_path + except ArchiveDownloadError: + _unlink_path(temp_path) + raise except zipfile.BadZipFile as err: _unlink_path(temp_path) raise ArchiveDownloadError( @@ -954,6 +991,7 @@ def extract_csv_work_item( """Extract one CSV payload through an explicit work item.""" record = _record_from_work_item(work_item) try: + _validate_extraction_raw_dimensions(record) _ensure_record_data_dir(record, args) zip_persist = _zip_persist_enabled(args, record) existing = existing_extraction_artifact( @@ -1050,6 +1088,7 @@ def extract_archive_to_record( with zipfile.ZipFile(zip_path, "r") as zip_ref: member = _single_data_member(zip_ref, zip_path) filename = _safe_data_member_filename(member) + _validate_extracted_csv_filename(filename, record) record.csv_filename = filename target_path = Path(record.data_dir, filename) reused_existing = target_path.exists() @@ -1118,6 +1157,13 @@ def atomic_extract_archive_member( work_id: str, ) -> Path: """Extract one ZIP member through a temp file, then rename atomically.""" + if filename_has_unsupported_raw_dimensions(target_path.name): + raise ArchiveExtractionError( + "UNSUPPORTED_RAW_INPUT", + "Archive member filename declares a retired raw dimension.", + retryable=False, + detail={"filename": target_path.name}, + ) temp_path = target_path.with_name( f".{target_path.name}.{derive_work_id(work_id).removeprefix('work-')}.tmp" ) @@ -1257,6 +1303,7 @@ def cache_build_result_for_record( reused_existing: bool = False, ) -> CacheBuildResult: """Validate a cache artifact and return bounded cache metadata.""" + _validate_cache_raw_dimensions(record) if not record.data_dir: raise CacheBuildError( "INVALID_CACHE_REQUEST", @@ -1324,9 +1371,25 @@ def merge_cache_work_items( *, return_type: str = "polars", materialize: bool = True, + output_timezone: str = "", + random_selection: RandomWindowSelectionV1 | Mapping[str, Any] | None = None, ) -> MergeStageOutput: """Merge cache artifacts from explicit work items.""" mergeable = _mergeable_cache_items(work_items) + selection = _resolve_random_window_selection( + mergeable, + explicit=random_selection, + ) + unsupported_count = sum( + 1 + for item in work_items + if item.cache_filename == CACHE_FILENAME + and Path(item.data_dir, item.cache_filename).is_file() + and not _supports_raw_dimensions( + item.data_format, + item.data_timeframe, + ) + ) sets_to_merge = _collate_cache_sets(mergeable) merge_sets: list[CacheMergeSetSummary] = [] for cache_set in sets_to_merge: @@ -1343,6 +1406,8 @@ def merge_cache_work_items( ordered_items, return_type=return_type, already_ordered=True, + output_timezone=output_timezone, + random_selection=selection, ) if not materialize: @@ -1379,14 +1444,21 @@ def merge_cache_work_items( "record_count": len(mergeable), "set_count": len(sets_to_merge), "materialized": materialize, + "random_window_selection_id": ( + selection.selection_id if selection is not None else "" + ), }, ), ), metrics={ "record_count": len(mergeable), + "unsupported_count": unsupported_count, "set_count": len(sets_to_merge), "materialized": materialize, "sets": merge_set_metrics, + "random_window_selection": ( + selection.to_dict() if selection is not None else {} + ), }, ) return MergeStageOutput( @@ -1401,18 +1473,43 @@ def merge_cache_items( *, return_type: str, already_ordered: bool = False, + output_timezone: str = "", + random_selection: RandomWindowSelectionV1 | Mapping[str, Any] | None = None, ) -> Any: """Merge one pair/timeframe cache set into the requested API type.""" import polars as pl + unsupported = [ + item + for item in work_items + if not _supports_raw_dimensions( + item.data_format, + item.data_timeframe, + ) + ] + if unsupported: + raise ValueError("cache merge supports ASCII tick inputs only") + ordered_items = ( tuple(work_items) if already_ordered else order_cache_items(work_items) ) + selection = _resolve_random_window_selection( + ordered_items, + explicit=random_selection, + ) frames = [ - read_polars_cache(Path(item.data_dir, item.cache_filename)) + filter_polars_frame_to_random_window( + read_polars_cache(Path(item.data_dir, item.cache_filename)), + selection, + ) for item in ordered_items ] merged = pl.concat(frames) if frames else pl.DataFrame() + if selection is not None and merged.height < 1: + raise RandomWindowEmptySelectionError( + "resolved random window contains no cache rows" + ) + merged = _append_output_timezone_projection(merged, output_timezone) return _convert_cache_frame(merged, return_type) @@ -1420,11 +1517,15 @@ def merge_cache_records( records: Sequence[Any], *, return_type: str, + output_timezone: str = "", + random_selection: RandomWindowSelectionV1 | Mapping[str, Any] | None = None, ) -> Any: """Merge one legacy record set through the queue-free implementation.""" return merge_cache_items( [WorkItem.from_record(record) for record in records], return_type=return_type, + output_timezone=output_timezone, + random_selection=random_selection, ) @@ -1471,9 +1572,30 @@ def import_to_influx_work_item( batch_count = 0 line_count = 0 try: + if not _supports_raw_dimensions( + record.data_format, + record.data_timeframe, + ): + return _activity_output( + _work_item_from_record(record, work_item), + stage="import_to_influx", + status=WorkStatus.SKIPPED, + metrics={ + "batch_count": 0, + "line_count": 0, + "decision": "skipped_unsupported", + "data_format": record.data_format, + "timeframe": record.data_timeframe, + }, + message="Influx projection supports ASCII tick inputs only.", + ) + if ( record.status is not WorkStatus.INFLUX_UPLOAD - and str.lower(record.data_format) == "ascii" + and _supports_raw_dimensions( + record.data_format, + record.data_timeframe, + ) ): cache_path = Path(record.data_dir, CACHE_FILENAME) if not cache_path.exists() and status_has_csv_artifact( @@ -1524,11 +1646,29 @@ def emit_influx_cache_batches( emit_lines: LineSink, ) -> tuple[int, int]: """Emit bounded Influx line-protocol batches for one cache artifact.""" + from histdatacom.data_quality.training_features import ( + ensure_tick_training_features, + ) + + if not _supports_raw_dimensions( + work_item.data_format, + work_item.data_timeframe, + ): + raise ValueError("Influx projection supports ASCII tick inputs only") + cache_filename = work_item.cache_filename or CACHE_FILENAME cache = read_polars_cache(Path(work_item.data_dir, cache_filename)) + selection = random_window_selection_from_metadata(work_item.metadata) + cache = filter_polars_frame_to_random_window(cache, selection) + if selection is not None and cache.height < 1: + raise RandomWindowEmptySelectionError( + "resolved random window contains no rows in the planned cache" + ) + cache = ensure_tick_training_features(cache, target=work_item) batch_size = coerce_batch_size(args["batch_size"]) batch_count = 0 line_count = 0 + columns = tuple(cache.columns) for rows in iter_polars_row_batches(cache, batch_size): lines = [ format_influx_line( @@ -1536,6 +1676,7 @@ def emit_influx_cache_batches( work_item.data_format, work_item.data_timeframe, row, + columns=columns, ) for row in rows ] @@ -1559,6 +1700,8 @@ def dataset_plan_stage( zip_persist: bool = False, cache_only: bool = False, repository_ranges: Mapping[str, Any] | None = None, + random_window: str = "", + random_seed: int | None = None, ) -> DatasetPlanOutput: """Plan explicit dataset work items without queues or side effects.""" formats_input = tuple(formats or ()) @@ -1576,6 +1719,8 @@ def dataset_plan_stage( zip_persist=zip_persist, cache_only=cache_only, repository_ranges=repository_ranges, + random_window=random_window, + random_seed=random_seed, ) dimensions = valid_dataset_dimensions( formats_input, @@ -1586,6 +1731,10 @@ def dataset_plan_stage( normalized_timeframes = tuple(sorted({item[1] for item in dimensions})) normalized_pairs = normalize_dataset_pairs(pairs_input) resolved_current = current_yearmonth or get_current_datemonth_gmt_minus5() + selection = _random_window_selection_from_work_items(work_items) + selection_payload: JSONValue = ( + selection.to_dict() if selection is not None else {} + ) result = StageResult( work_id=derive_work_id( "dataset_plan", @@ -1594,6 +1743,7 @@ def dataset_plan_stage( *normalized_formats, *normalized_timeframes, *normalized_pairs, + selection.selection_id if selection is not None else "", ), stage="dataset_plan", status=WorkStatus.COMPLETED, @@ -1602,7 +1752,10 @@ def dataset_plan_stage( status=WorkStatus.COMPLETED, stage="dataset_plan", message="Dataset work items planned.", - metadata={"work_item_count": len(work_items)}, + metadata={ + "work_item_count": len(work_items), + "random_window_selection": selection_payload, + }, ), ), metrics={ @@ -1614,6 +1767,7 @@ def dataset_plan_stage( "end_yearmonth": _coerce_yearmonth(end_yearmonth) or "", "current_yearmonth": resolved_current, "cache_only": cache_only, + "random_window_selection": selection_payload, }, ) return DatasetPlanOutput(work_items=work_items, result=result) @@ -1632,6 +1786,8 @@ def plan_dataset_work_items( zip_persist: bool = False, cache_only: bool = False, repository_ranges: Mapping[str, Any] | None = None, + random_window: str = "", + random_seed: int | None = None, ) -> tuple[WorkItem, ...]: """Return deterministic URL work items for a HistData request.""" formats_input = tuple(formats or ()) @@ -1640,13 +1796,25 @@ def plan_dataset_work_items( resolved_current = current_yearmonth or get_current_datemonth_gmt_minus5() start = _coerce_yearmonth(start_yearmonth) end = _coerce_yearmonth(end_yearmonth) + normalized_pairs = normalize_dataset_pairs(pairs_input) + selection: RandomWindowSelectionV1 | None = None + if random_window: + selection = resolve_random_window_selection( + random_window, + seed=random_seed, + pairs=normalized_pairs, + repository_ranges=repository_pair_data(repository_ranges or {}), + start_yearmonth=start, + end_yearmonth=end, + current_yearmonth=resolved_current, + ) + start, end = random_window_planning_yearmonths(selection) full_scope = not start and not end if not start and not end: start = "200001" end = resolved_current planned: list[WorkItem] = [] - normalized_pairs = normalize_dataset_pairs(pairs_input) pair_ranges = _dataset_pair_ranges( normalized_pairs, repository_ranges=repository_ranges if full_scope else None, @@ -1677,7 +1845,22 @@ def plan_dataset_work_items( zip_persist=zip_persist, ) ) - return tuple(planned) + if selection is None: + return tuple(planned) + selection_payload = selection.to_dict() + planned_count = len(planned) + return tuple( + replace( + item, + work_id=derive_work_id(item.work_id, selection.selection_id), + metadata={ + **item.metadata, + RANDOM_WINDOW_SELECTION_METADATA_KEY: selection_payload, + "random_window_planned_work_item_count": planned_count, + }, + ) + for item in planned + ) def _dataset_pair_ranges( @@ -2295,7 +2478,48 @@ def _record_from_work_item(work_item: WorkItem) -> Record: def _work_item_from_record(record: Record, original: WorkItem) -> WorkItem: - return WorkItem.from_record(record, work_id=original.work_id) + return replace( + WorkItem.from_record(record, work_id=original.work_id), + metadata=dict(original.metadata), + ) + + +def _random_window_selection_from_work_items( + work_items: Sequence[WorkItem], +) -> RandomWindowSelectionV1 | None: + selections = { + selection.selection_id: selection + for item in work_items + if (selection := random_window_selection_from_metadata(item.metadata)) + is not None + } + if len(selections) > 1: + raise ValueError("cache work items contain conflicting random windows") + return next(iter(selections.values()), None) + + +def _resolve_random_window_selection( + work_items: Sequence[WorkItem], + *, + explicit: RandomWindowSelectionV1 | Mapping[str, Any] | None, +) -> RandomWindowSelectionV1 | None: + persisted = _random_window_selection_from_work_items(work_items) + supplied = ( + explicit + if isinstance(explicit, RandomWindowSelectionV1) + else ( + RandomWindowSelectionV1.from_dict(explicit) + if explicit is not None + else None + ) + ) + if ( + persisted is not None + and supplied is not None + and persisted.selection_id != supplied.selection_id + ): + raise ValueError("explicit and persisted random windows do not match") + return supplied or persisted def _activity_output( @@ -2443,7 +2667,220 @@ def _cache_build_failure( ) +def _supports_raw_dimensions(data_format: str, timeframe: str) -> bool: + """Return whether a dimension is the sole supported raw input.""" + try: + normalized_format = _format_value(data_format) + normalized_timeframe = _timeframe_key(timeframe) + except (KeyError, ValueError): + return False + return ( + normalized_format == Format.ASCII.value + and normalized_timeframe == Timeframe.T.name + and normalized_timeframe + in get_valid_format_timeframes(normalized_format) + ) + + +def _raw_dimension_detail( + data_format: str, + timeframe: str, + *, + url: str = "", +) -> dict[str, JSONValue]: + detail: dict[str, JSONValue] = { + "data_format": str(data_format), + "timeframe": str(timeframe), + "supported_data_format": Format.ASCII.value, + "supported_timeframe": Timeframe.T.name, + } + if url: + detail["url"] = url + return detail + + +def _validate_url_raw_dimensions( + data_format: str, + timeframe: str, + *, + url: str, +) -> None: + if _supports_raw_dimensions(data_format, timeframe): + return + raise UrlValidationError( + "UNSUPPORTED_RAW_INPUT", + "Only HistData ASCII tick data is accepted as a raw input.", + detail=_raw_dimension_detail(data_format, timeframe, url=url), + ) + + +def _validate_form_metadata_matches_plan( + work_item: WorkItem, + metadata: UrlFormMetadata, +) -> None: + """Reject server form dimensions that differ from the planned target.""" + mismatches: list[str] = [] + try: + if work_item.data_format and ( + _format_value(work_item.data_format) + != _format_value(metadata.data_format) + ): + mismatches.append("data_format") + if work_item.data_timeframe and ( + _timeframe_key(work_item.data_timeframe) + != _timeframe_key(metadata.data_timeframe) + ): + mismatches.append("timeframe") + except (KeyError, ValueError) as err: + raise UrlValidationError( + "UNSUPPORTED_RAW_INPUT", + "The planned target contains an unsupported raw dimension.", + detail=_raw_dimension_detail( + work_item.data_format, + work_item.data_timeframe, + url=work_item.url, + ), + ) from err + + if work_item.data_fxpair and ( + work_item.data_fxpair.lower() != metadata.data_fxpair.lower() + ): + mismatches.append("fxpair") + if work_item.data_datemonth and ( + work_item.data_datemonth != metadata.data_datemonth + ): + mismatches.append("datemonth") + if not mismatches: + return + + raise UrlValidationError( + "FORM_DIMENSION_MISMATCH", + "HistData download form dimensions do not match the planned target.", + detail={ + "url": work_item.url, + "mismatched_fields": cast(JSONValue, mismatches), + "planned_data_format": work_item.data_format, + "returned_data_format": metadata.data_format, + "planned_timeframe": work_item.data_timeframe, + "returned_timeframe": metadata.data_timeframe, + "planned_fxpair": work_item.data_fxpair, + "returned_fxpair": metadata.data_fxpair, + "planned_datemonth": work_item.data_datemonth, + "returned_datemonth": metadata.data_datemonth, + }, + ) + + +def _retired_filename_detail( + record: Record, + filename: str, +) -> dict[str, JSONValue]: + return { + **_raw_dimension_detail( + record.data_format, + record.data_timeframe, + url=record.url, + ), + "filename": filename, + } + + +def _validate_downloaded_archive_filename( + filename: str, + record: Record, +) -> None: + if not filename_has_unsupported_raw_dimensions(filename): + return + raise ArchiveDownloadError( + "UNSUPPORTED_RAW_INPUT", + "Downloaded archive filename declares a retired raw dimension.", + retryable=False, + detail=_retired_filename_detail(record, filename), + ) + + +def _validate_extracted_csv_filename( + filename: str, + record: Record, +) -> None: + if not filename_has_unsupported_raw_dimensions(filename): + return + raise ArchiveExtractionError( + "UNSUPPORTED_RAW_INPUT", + "Archive member filename declares a retired raw dimension.", + retryable=False, + detail=_retired_filename_detail(record, filename), + ) + + +def _validate_archive_raw_dimensions(record: Record) -> None: + if not _supports_raw_dimensions(record.data_format, record.data_timeframe): + raise ArchiveDownloadError( + "UNSUPPORTED_RAW_INPUT", + "Archive download supports HistData ASCII tick inputs only.", + retryable=False, + detail=_raw_dimension_detail( + record.data_format, + record.data_timeframe, + url=record.url, + ), + ) + for filename in (record.zip_filename, record.csv_filename): + if filename_has_unsupported_raw_dimensions(filename): + raise ArchiveDownloadError( + "UNSUPPORTED_RAW_INPUT", + "Local artifact filename declares a retired raw dimension.", + retryable=False, + detail=_retired_filename_detail(record, filename), + ) + + +def _validate_extraction_raw_dimensions(record: Record) -> None: + if not _supports_raw_dimensions(record.data_format, record.data_timeframe): + raise ArchiveExtractionError( + "UNSUPPORTED_RAW_INPUT", + "Archive extraction supports HistData ASCII tick inputs only.", + retryable=False, + detail=_raw_dimension_detail( + record.data_format, + record.data_timeframe, + url=record.url, + ), + ) + for filename in (record.zip_filename, record.csv_filename): + if filename_has_unsupported_raw_dimensions(filename): + raise ArchiveExtractionError( + "UNSUPPORTED_RAW_INPUT", + "Local artifact filename declares a retired raw dimension.", + retryable=False, + detail=_retired_filename_detail(record, filename), + ) + + +def _validate_cache_raw_dimensions(record: Record) -> None: + if not _supports_raw_dimensions(record.data_format, record.data_timeframe): + raise CacheBuildError( + "UNSUPPORTED_RAW_INPUT", + "Cache creation and validation support ASCII tick inputs only.", + retryable=False, + detail=_raw_dimension_detail( + record.data_format, + record.data_timeframe, + url=record.url, + ), + ) + for filename in (record.zip_filename, record.csv_filename): + if filename_has_unsupported_raw_dimensions(filename): + raise CacheBuildError( + "UNSUPPORTED_RAW_INPUT", + "Cache source filename declares a retired raw dimension.", + retryable=False, + detail=_retired_filename_detail(record, filename), + ) + + def _validate_archive_request(record: Record) -> None: + _validate_archive_raw_dimensions(record) missing = [ field_name for field_name in ( @@ -2467,6 +2904,7 @@ def _validate_archive_request(record: Record) -> None: def _validate_extraction_request(record: Record) -> None: + _validate_extraction_raw_dimensions(record) missing = [ field_name for field_name in ("data_dir", "zip_filename") @@ -2617,8 +3055,20 @@ def _response_content_bytes(response: Any) -> bytes: def _validate_zip_payload(path: Path) -> None: with zipfile.ZipFile(path, "r") as archive: bad_member = archive.testzip() + unsupported_members = [ + name + for name in archive.namelist() + if filename_has_unsupported_raw_dimensions(name) + ] if bad_member: raise zipfile.BadZipFile(f"bad member in ZIP archive: {bad_member}") + if unsupported_members: + raise ArchiveDownloadError( + "UNSUPPORTED_RAW_INPUT", + "Archive contains a member with a retired raw dimension.", + retryable=False, + detail={"filename": unsupported_members[0]}, + ) def _unlink_path(path: Path) -> None: @@ -2642,6 +3092,10 @@ def _mergeable_cache_items( for item in work_items if item.cache_filename == CACHE_FILENAME and Path(item.data_dir, item.cache_filename).is_file() + and _supports_raw_dimensions( + item.data_format, + item.data_timeframe, + ) ] @@ -2702,8 +3156,9 @@ def _existing_archive_artifact_on_disk(record: Record) -> bool: def _supports_cache(record: Record) -> bool: - return str.lower(record.data_format) == "ascii" and ( - record.data_timeframe == "T" + return _supports_raw_dimensions( + record.data_format, + record.data_timeframe, ) @@ -2724,6 +3179,7 @@ def _source_artifact_path(record: Record, filename: str) -> Path | None: def create_cache_file(record: Record, args: Mapping[str, Any]) -> None: + _validate_cache_raw_dimensions(record) zip_path = _source_artifact_path(record, record.zip_filename) csv_path = _source_artifact_path(record, record.csv_filename) @@ -2772,14 +3228,23 @@ def create_cache_file(record: Record, args: Mapping[str, Any]) -> None: def _import_source_to_polars(record: Record, source_path: Path) -> Any: + from histdatacom.data_quality.training_features import ( + enrich_tick_cache_with_training_features, + ) + try: raw_frame = read_ascii_file_to_polars( source_path, record.data_timeframe ) - return convert_polars_datetime_to_utc_ms( + normalized = convert_polars_datetime_to_utc_ms( raw_frame, record.data_timeframe, ) + return enrich_tick_cache_with_training_features( + normalized, + target=record, + source="histdata.com", + ) except ValueError as err: raise CacheBuildError( "CACHE_SOURCE_INVALID", @@ -2858,6 +3323,24 @@ def _convert_cache_frame(frame: Any, return_type: str) -> Any: raise ValueError(f"unsupported API return type: {return_type}") +def _append_output_timezone_projection(frame: Any, timezone_name: str) -> Any: + """Append a localized API timestamp while preserving canonical UTC millis.""" + normalized = normalize_output_timezone(timezone_name) + if not normalized: + return frame + if "datetime" not in frame.columns: + raise ValueError("timezone projection requires a datetime column") + + import polars as pl + + return frame.with_columns( + pl.col("datetime") + .cast(pl.Datetime("ms", "UTC")) + .dt.convert_time_zone(normalized) + .alias(OUTPUT_DATETIME_COLUMN) + ) + + def _collate_cache_sets( work_items: Sequence[WorkItem], ) -> list[MutableMapping[str, Any]]: diff --git a/src/histdatacom/api.py b/src/histdatacom/api.py index 6409e819..dc30365b 100644 --- a/src/histdatacom/api.py +++ b/src/histdatacom/api.py @@ -36,8 +36,15 @@ from histdatacom.legacy_boundary import warn_legacy_side_effect from histdatacom.observability import ProgressState, progress_increment from histdatacom.runtime_contracts import WorkItem, WorkStatus +from histdatacom.random_windows import ( + RandomWindowError, + RandomWindowSelectionV1, +) from histdatacom.scraper.scraper import Scraper -from histdatacom.utils import normalize_api_return_type +from histdatacom.utils import ( + normalize_api_return_type, + normalize_output_timezone, +) if TYPE_CHECKING: from pandas.core.frame import DataFrame @@ -72,6 +79,9 @@ def __init__( self.return_type = _resolve_api_return_type( return_type or self.args.get("api_return_type") ) + self.output_timezone = normalize_output_timezone( + str(self.args.get("output_timezone", "") or "") + ) @classmethod def _create_cache(cls, record: "Record", args: dict) -> None: @@ -241,6 +251,10 @@ def merge_caches( records_to_merge: list[Record] | None = None, *, return_type: str | None = None, + output_timezone: str | None = None, + random_selection: ( + RandomWindowSelectionV1 | Mapping[str, Any] | None + ) = None, ) -> list | PolarsDataFrame | DataFrame | Table: """Merge caches for start_yearmonth and end_yearmonth range. @@ -252,6 +266,8 @@ def merge_caches( return self.merge_records( records_to_merge or [], return_type=return_type, + output_timezone=output_timezone, + random_selection=random_selection, ) def merge_records( @@ -259,18 +275,31 @@ def merge_records( records_to_merge: list, *, return_type: str | None = None, + output_timezone: str | None = None, + random_selection: ( + RandomWindowSelectionV1 | Mapping[str, Any] | None + ) = None, ) -> list | PolarsDataFrame | DataFrame | Table: """Merge explicit cache records into the configured API return type.""" if not records_to_merge: return [] + if self.args.get("random_window") and random_selection is None: + raise RandomWindowError( + "random-window API merge requires a resolved selection" + ) resolved_return_type = _resolve_api_return_type( return_type or self.return_type ) + resolved_output_timezone = normalize_output_timezone( + self.output_timezone if output_timezone is None else output_timezone + ) merge_output = merge_cache_work_items( [WorkItem.from_record(record) for record in records_to_merge], return_type=resolved_return_type, materialize=True, + output_timezone=resolved_output_timezone, + random_selection=random_selection, ) if merge_output.result.status is WorkStatus.SKIPPED: return [] @@ -281,6 +310,7 @@ def _merge_records( tp_set_dict: dict, *, return_type: str | None = None, + output_timezone: str | None = None, ) -> None: """Sort and Merge records from a timeframe/pair set. @@ -319,6 +349,11 @@ def _merge_records( return_type=_resolve_api_return_type( return_type or self.return_type ), + output_timezone=normalize_output_timezone( + self.output_timezone + if output_timezone is None + else output_timezone + ), ) def _collate_sets_to_merge(self, records_to_merge: list) -> list: diff --git a/src/histdatacom/broker_capture/__init__.py b/src/histdatacom/broker_capture/__init__.py new file mode 100644 index 00000000..8bdea1f5 --- /dev/null +++ b/src/histdatacom/broker_capture/__init__.py @@ -0,0 +1,189 @@ +"""Public live broker delivery capture contracts and replay interfaces.""" + +from histdatacom.broker_capture.adapters import ( + BrokerCaptureAdapterV1, + BrokerCaptureClockV1, + BrokerCaptureConsumeResultV1, + BrokerCaptureEventConsumerV1, + BrokerCaptureEventSinkV1, + BrokerCaptureEventSourceV1, + LiveBrokerCaptureSourceV1, + SequenceBrokerCaptureAdapterV1, + SequenceBrokerCaptureClockV1, + SystemBrokerCaptureClockV1, + consume_broker_capture_source, +) +from histdatacom.broker_capture.contracts import ( + BROKER_ADAPTER_MESSAGE_SCHEMA_VERSION, + BROKER_CAPTURE_COLLECTOR_ID, + BROKER_CAPTURE_COLLECTOR_VERSION, + BROKER_CAPTURE_DATA_ARTIFACT_KIND, + BROKER_CAPTURE_EVENT_SCHEMA_VERSION, + BROKER_CAPTURE_PARTITION_MANIFEST_SCHEMA_VERSION, + BROKER_CAPTURE_REPLAY_SUMMARY_SCHEMA_VERSION, + BROKER_CAPTURE_SESSION_MANIFEST_SCHEMA_VERSION, + BROKER_CAPTURE_SESSION_SCHEMA_VERSION, + BROKER_CAPTURE_STORAGE_POLICY_SCHEMA_VERSION, + BrokerAdapterMessageV1, + BrokerCaptureActivitySemantics, + BrokerCaptureBackpressureMode, + BrokerCaptureEventKind, + BrokerCaptureEventV1, + BrokerCapturePartitionManifestV1, + BrokerCapturePriceTextSemantics, + BrokerCaptureReplaySummaryV1, + BrokerCaptureRetentionMode, + BrokerCaptureSessionManifestV1, + BrokerCaptureSessionState, + BrokerCaptureSessionV1, + BrokerCaptureSizeSemantics, + BrokerCaptureStoragePolicyV1, + BrokerCaptureSourceTimestampSemantics, + assert_secret_free_capture_value, + canonical_capture_json, + logical_capture_content_sha256, +) +from histdatacom.broker_capture.storage import ( + AppendOnlyBrokerCaptureWriterV1, + BrokerCaptureBackpressureError, + BrokerCaptureExistingSessionError, + BrokerCaptureIntegrityError, + BrokerCaptureQuotaError, + BrokerCaptureReplaySourceV1, + BrokerCaptureRetentionError, + BrokerCaptureSessionInspectionV1, + BrokerCaptureStorageError, + discover_broker_capture_session_manifests, + inspect_broker_capture_session, + load_broker_capture_session_manifest, + replay_broker_capture_session, + verify_broker_capture_partition_manifests, +) +from histdatacom.broker_capture.fingerprint_contracts import ( + BROKER_CAPTURE_ELIGIBILITY_SCHEMA_VERSION, + BROKER_DELIVERY_CAPTURE_EVIDENCE_SCHEMA_VERSION, + BROKER_DELIVERY_CELL_SCHEMA_VERSION, + BROKER_DELIVERY_CONDITION_SCHEMA_VERSION, + BROKER_DELIVERY_DRIFT_CONFIG_SCHEMA_VERSION, + BROKER_DELIVERY_FINGERPRINT_ARTIFACT_KIND, + BROKER_DELIVERY_FINGERPRINT_COMPARISON_SCHEMA_VERSION, + BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION, + BROKER_DELIVERY_FIT_CONFIG_SCHEMA_VERSION, + BROKER_DELIVERY_METRIC_COMPARISON_SCHEMA_VERSION, + BROKER_DELIVERY_METRIC_SCHEMA_VERSION, + BrokerCaptureEligibilityStatus, + BrokerCaptureEligibilityV1, + BrokerDeliveryCaptureEvidenceV1, + BrokerDeliveryCellV1, + BrokerDeliveryConditionV1, + BrokerDeliveryDriftConfigV1, + BrokerDeliveryDriftStatus, + BrokerDeliveryFingerprintComparisonV1, + BrokerDeliveryFingerprintV1, + BrokerDeliveryFitConfigV1, + BrokerDeliveryMetricComparisonV1, + BrokerDeliveryMetricV1, + BrokerDeliverySupportStatus, +) +from histdatacom.broker_capture.fingerprints import ( + BrokerDeliveryFingerprintArtifactError, + BrokerDeliveryFingerprintError, + BrokerDeliveryFingerprintIdentityError, + BrokerDeliveryIneligibleCaptureError, + BrokerDeliveryResourceLimitError, + assess_broker_capture_eligibility, + compare_broker_delivery_fingerprints, + fit_broker_delivery_fingerprint, + load_broker_delivery_fingerprint, + write_broker_delivery_fingerprint, +) + +__all__ = [ + "BROKER_ADAPTER_MESSAGE_SCHEMA_VERSION", + "BROKER_CAPTURE_COLLECTOR_ID", + "BROKER_CAPTURE_COLLECTOR_VERSION", + "BROKER_CAPTURE_DATA_ARTIFACT_KIND", + "BROKER_CAPTURE_EVENT_SCHEMA_VERSION", + "BROKER_CAPTURE_ELIGIBILITY_SCHEMA_VERSION", + "BROKER_CAPTURE_PARTITION_MANIFEST_SCHEMA_VERSION", + "BROKER_CAPTURE_REPLAY_SUMMARY_SCHEMA_VERSION", + "BROKER_CAPTURE_SESSION_MANIFEST_SCHEMA_VERSION", + "BROKER_CAPTURE_SESSION_SCHEMA_VERSION", + "BROKER_CAPTURE_STORAGE_POLICY_SCHEMA_VERSION", + "BROKER_DELIVERY_CAPTURE_EVIDENCE_SCHEMA_VERSION", + "BROKER_DELIVERY_CELL_SCHEMA_VERSION", + "BROKER_DELIVERY_CONDITION_SCHEMA_VERSION", + "BROKER_DELIVERY_DRIFT_CONFIG_SCHEMA_VERSION", + "BROKER_DELIVERY_FINGERPRINT_ARTIFACT_KIND", + "BROKER_DELIVERY_FINGERPRINT_COMPARISON_SCHEMA_VERSION", + "BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION", + "BROKER_DELIVERY_FIT_CONFIG_SCHEMA_VERSION", + "BROKER_DELIVERY_METRIC_COMPARISON_SCHEMA_VERSION", + "BROKER_DELIVERY_METRIC_SCHEMA_VERSION", + "AppendOnlyBrokerCaptureWriterV1", + "BrokerAdapterMessageV1", + "BrokerCaptureActivitySemantics", + "BrokerCaptureAdapterV1", + "BrokerCaptureBackpressureError", + "BrokerCaptureBackpressureMode", + "BrokerCaptureClockV1", + "BrokerCaptureConsumeResultV1", + "BrokerCaptureEventConsumerV1", + "BrokerCaptureEventKind", + "BrokerCaptureEventSinkV1", + "BrokerCaptureEventSourceV1", + "BrokerCaptureEventV1", + "BrokerCaptureEligibilityStatus", + "BrokerCaptureEligibilityV1", + "BrokerCaptureExistingSessionError", + "BrokerCaptureIntegrityError", + "BrokerCapturePartitionManifestV1", + "BrokerCapturePriceTextSemantics", + "BrokerCaptureQuotaError", + "BrokerCaptureReplaySourceV1", + "BrokerCaptureReplaySummaryV1", + "BrokerCaptureRetentionError", + "BrokerCaptureRetentionMode", + "BrokerCaptureSessionInspectionV1", + "BrokerCaptureSessionManifestV1", + "BrokerCaptureSessionState", + "BrokerCaptureSessionV1", + "BrokerCaptureSizeSemantics", + "BrokerCaptureStorageError", + "BrokerCaptureStoragePolicyV1", + "BrokerCaptureSourceTimestampSemantics", + "BrokerDeliveryCaptureEvidenceV1", + "BrokerDeliveryCellV1", + "BrokerDeliveryConditionV1", + "BrokerDeliveryDriftConfigV1", + "BrokerDeliveryDriftStatus", + "BrokerDeliveryFingerprintArtifactError", + "BrokerDeliveryFingerprintComparisonV1", + "BrokerDeliveryFingerprintError", + "BrokerDeliveryFingerprintIdentityError", + "BrokerDeliveryFingerprintV1", + "BrokerDeliveryFitConfigV1", + "BrokerDeliveryIneligibleCaptureError", + "BrokerDeliveryMetricComparisonV1", + "BrokerDeliveryMetricV1", + "BrokerDeliveryResourceLimitError", + "BrokerDeliverySupportStatus", + "LiveBrokerCaptureSourceV1", + "SequenceBrokerCaptureAdapterV1", + "SequenceBrokerCaptureClockV1", + "SystemBrokerCaptureClockV1", + "assert_secret_free_capture_value", + "assess_broker_capture_eligibility", + "canonical_capture_json", + "consume_broker_capture_source", + "compare_broker_delivery_fingerprints", + "discover_broker_capture_session_manifests", + "inspect_broker_capture_session", + "fit_broker_delivery_fingerprint", + "load_broker_capture_session_manifest", + "load_broker_delivery_fingerprint", + "logical_capture_content_sha256", + "replay_broker_capture_session", + "verify_broker_capture_partition_manifests", + "write_broker_delivery_fingerprint", +] diff --git a/src/histdatacom/broker_capture/adapters.py b/src/histdatacom/broker_capture/adapters.py new file mode 100644 index 00000000..4dbcc3e6 --- /dev/null +++ b/src/histdatacom/broker_capture/adapters.py @@ -0,0 +1,272 @@ +"""Adapter and consumer seams for live and replayed broker capture evidence.""" + +from __future__ import annotations + +import time +from collections.abc import Iterable, Iterator, Sequence +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + +from histdatacom.broker_capture.contracts import ( + BrokerAdapterMessageV1, + BrokerCaptureEventKind, + BrokerCaptureEventV1, + BrokerCaptureSessionV1, +) + + +@runtime_checkable +class BrokerCaptureAdapterV1(Protocol): + """Credential-opaque source of public broker messages. + + A real implementation owns its private connection configuration. The + collector deliberately reads only these public identifiers and the + message iterator, so credentials cannot leak through contract reflection. + """ + + @property + def adapter_id(self) -> str: + """Return the public adapter identifier.""" + + @property + def adapter_version(self) -> str: + """Return the adapter semantic version.""" + + def iter_messages(self) -> Iterable[BrokerAdapterMessageV1]: + """Yield public messages in adapter-observed order.""" + + +@runtime_checkable +class BrokerCaptureClockV1(Protocol): + """Clock seam sampled at the collector boundary for every message.""" + + def sample(self) -> tuple[int, int]: + """Return ``(UTC wall time ns, monotonic receive time ns)``.""" + + +@runtime_checkable +class BrokerCaptureEventSourceV1(Protocol): + """Common live/replay source consumed by downstream fingerprinting.""" + + @property + def session_id(self) -> str: + """Return the capture session identity.""" + + def iter_events(self) -> Iterable[BrokerCaptureEventV1]: + """Yield collector-stamped events in capture order.""" + + +@runtime_checkable +class BrokerCaptureEventSinkV1(Protocol): + """Append-only persistence sink for captured events.""" + + def append(self, event: BrokerCaptureEventV1) -> None: + """Persist one event or fail before acknowledging it.""" + + +@runtime_checkable +class BrokerCaptureEventConsumerV1(Protocol): + """Identical downstream interface used for live and replay sources.""" + + def on_event(self, event: BrokerCaptureEventV1) -> None: + """Consume one verified capture event.""" + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureConsumeResultV1: + """Small in-memory result from driving a source through consumers.""" + + session_id: str + event_count: int + event_kind_counts: dict[str, int] + first_capture_sequence: int | None + last_capture_sequence: int | None + + +class SystemBrokerCaptureClockV1: + """Production clock implementation using Python nanosecond clocks.""" + + def sample(self) -> tuple[int, int]: + """Return one adjacent wall/monotonic sample pair.""" + return time.time_ns(), time.monotonic_ns() + + +@dataclass(slots=True) +class SequenceBrokerCaptureClockV1: + """Deterministic clock fixture for drift, burst, and quiet-gap tests.""" + + samples: Sequence[tuple[int, int]] + _index: int = 0 + + def sample(self) -> tuple[int, int]: + """Return the next fixture sample and fail when underspecified.""" + if self._index >= len(self.samples): + raise RuntimeError("broker capture fixture clock is exhausted") + value = self.samples[self._index] + self._index += 1 + return value + + +@dataclass(frozen=True, slots=True) +class SequenceBrokerCaptureAdapterV1: + """Deterministic public adapter used by synthetic capture fixtures.""" + + adapter_id: str + adapter_version: str + messages: tuple[BrokerAdapterMessageV1, ...] + + def iter_messages(self) -> Iterable[BrokerAdapterMessageV1]: + """Yield configured fixture messages without mutation.""" + return iter(self.messages) + + +@dataclass(frozen=True, slots=True) +class LiveBrokerCaptureSourceV1: + """Collector-side source that stamps an adapter stream with two clocks.""" + + session: BrokerCaptureSessionV1 + adapter: BrokerCaptureAdapterV1 + clock: BrokerCaptureClockV1 + clock_correction_threshold_ns: int = 5_000_000 + + def __post_init__(self) -> None: + if self.adapter.adapter_id != self.session.adapter_id: + raise ValueError("adapter_id does not match capture session") + if self.adapter.adapter_version != self.session.adapter_version: + raise ValueError("adapter_version does not match capture session") + if ( + isinstance(self.clock_correction_threshold_ns, bool) + or not isinstance(self.clock_correction_threshold_ns, int) + or self.clock_correction_threshold_ns <= 0 + ): + raise ValueError("clock correction threshold must be positive") + + @property + def session_id(self) -> str: + """Return the capture session identity.""" + return self.session.session_id + + def iter_events(self) -> Iterable[BrokerCaptureEventV1]: + """Yield ordered events and explicit wall-clock correction evidence.""" + return self._event_iterator() + + def _event_iterator(self) -> Iterator[BrokerCaptureEventV1]: + sequence = 0 + previous_wall: int | None = None + previous_monotonic: int | None = None + for message in self.adapter.iter_messages(): + if not isinstance(message, BrokerAdapterMessageV1): + raise TypeError("broker adapter yielded a non-contract message") + if message.kind is BrokerCaptureEventKind.CLOCK_CORRECTION: + raise ValueError( + "broker adapters cannot emit collector clock corrections" + ) + wall, monotonic = self.clock.sample() + _validate_clock_sample(wall, monotonic) + if ( + previous_monotonic is not None + and monotonic < previous_monotonic + ): + raise ValueError("collector monotonic clock moved backwards") + if previous_wall is not None and previous_monotonic is not None: + offset_change = (wall - previous_wall) - ( + monotonic - previous_monotonic + ) + if abs(offset_change) >= self.clock_correction_threshold_ns: + correction_message = BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.CLOCK_CORRECTION, + source_event_time_ns=message.source_event_time_ns, + source_timestamp_semantics=( + message.source_timestamp_semantics + ), + source_timestamp_precision_ns=( + message.source_timestamp_precision_ns + ), + reason_code="wall_monotonic_divergence", + public_metadata={ + "basis": "wall_delta_minus_monotonic_delta", + "threshold_ns": self.clock_correction_threshold_ns, + }, + ) + yield BrokerCaptureEventV1( + session_id=self.session.session_id, + capture_sequence=sequence, + receive_time_utc_ns=wall, + receive_time_monotonic_ns=monotonic, + message=correction_message, + clock_offset_change_ns=offset_change, + ) + sequence += 1 + yield BrokerCaptureEventV1( + session_id=self.session.session_id, + capture_sequence=sequence, + receive_time_utc_ns=wall, + receive_time_monotonic_ns=monotonic, + message=message, + ) + sequence += 1 + previous_wall = wall + previous_monotonic = monotonic + + +def consume_broker_capture_source( + source: BrokerCaptureEventSourceV1, + *, + sink: BrokerCaptureEventSinkV1 | None = None, + consumers: Sequence[BrokerCaptureEventConsumerV1] = (), +) -> BrokerCaptureConsumeResultV1: + """Drive either a live or replay source through the identical interface.""" + expected_sequence: int | None = None + first_sequence: int | None = None + last_sequence: int | None = None + counts: dict[str, int] = {} + event_count = 0 + for event in source.iter_events(): + if event.session_id != source.session_id: + raise ValueError("capture source yielded another session") + if expected_sequence is None: + expected_sequence = event.capture_sequence + first_sequence = event.capture_sequence + if event.capture_sequence != expected_sequence: + raise ValueError("capture source sequence is not contiguous") + if sink is not None: + sink.append(event) + for consumer in consumers: + consumer.on_event(event) + counts[event.kind.value] = counts.get(event.kind.value, 0) + 1 + event_count += 1 + last_sequence = event.capture_sequence + expected_sequence += 1 + return BrokerCaptureConsumeResultV1( + session_id=source.session_id, + event_count=event_count, + event_kind_counts=dict(sorted(counts.items())), + first_capture_sequence=first_sequence, + last_capture_sequence=last_sequence, + ) + + +def _validate_clock_sample(wall: int, monotonic: int) -> None: + for name, value in ( + ("receive_time_utc_ns", wall), + ("receive_time_monotonic_ns", monotonic), + ): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value < 0 or value > 2**63 - 1: + raise ValueError(f"{name} is outside non-negative int64 range") + + +__all__ = [ + "BrokerCaptureAdapterV1", + "BrokerCaptureClockV1", + "BrokerCaptureConsumeResultV1", + "BrokerCaptureEventConsumerV1", + "BrokerCaptureEventSinkV1", + "BrokerCaptureEventSourceV1", + "LiveBrokerCaptureSourceV1", + "SequenceBrokerCaptureAdapterV1", + "SequenceBrokerCaptureClockV1", + "SystemBrokerCaptureClockV1", + "consume_broker_capture_source", +] diff --git a/src/histdatacom/broker_capture/contracts.py b/src/histdatacom/broker_capture/contracts.py new file mode 100644 index 00000000..ccc24ee3 --- /dev/null +++ b/src/histdatacom/broker_capture/contracts.py @@ -0,0 +1,1739 @@ +"""Versioned contracts for append-only live broker delivery capture. + +Broker capture is measurement evidence, not synthetic output. These contracts +keep source timestamps, collector wall-clock timestamps, monotonic ordering, +connection health, and public adapter provenance explicit while structurally +excluding credentials and raw secret-bearing configuration. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from decimal import Decimal, InvalidOperation +from enum import Enum +from pathlib import PurePosixPath +from typing import Any, TypeVar, cast + +from histdatacom.runtime_contracts import ArtifactRef, JSONValue + +BROKER_ADAPTER_MESSAGE_SCHEMA_VERSION = "histdatacom.broker-adapter-message.v1" +BROKER_CAPTURE_SESSION_SCHEMA_VERSION = "histdatacom.broker-capture-session.v1" +BROKER_CAPTURE_EVENT_SCHEMA_VERSION = "histdatacom.broker-capture-event.v1" +BROKER_CAPTURE_STORAGE_POLICY_SCHEMA_VERSION = ( + "histdatacom.broker-capture-storage-policy.v1" +) +BROKER_CAPTURE_PARTITION_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.broker-capture-partition-manifest.v1" +) +BROKER_CAPTURE_SESSION_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.broker-capture-session-manifest.v1" +) +BROKER_CAPTURE_REPLAY_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.broker-capture-replay-summary.v1" +) + +BROKER_CAPTURE_COLLECTOR_ID = "histdatacom.broker-capture" +BROKER_CAPTURE_COLLECTOR_VERSION = "1.0.0" +BROKER_CAPTURE_DATA_ARTIFACT_KIND = "broker_capture_jsonl" + +INT64_MAX = 2**63 - 1 +MAX_CAPTURE_TEXT = 1024 +MAX_CAPTURE_METADATA_BYTES = 32_768 +MAX_CAPTURE_KIND_COUNTS = 64 +MAX_CAPTURE_PARTITIONS = 65_536 +MAX_CAPTURE_MANIFEST_BYTES = 16_777_216 + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_CANONICAL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$") +_SYMBOL_RE = re.compile(r"^[A-Z0-9._:-]{3,32}$") +_SEMVER_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][A-Za-z0-9.-]+)?$") +_PRICE_LEXEME_RE = re.compile(r"^[0-9]+(?:\.[0-9]+)?$") +_SENSITIVE_KEY_PARTS = frozenset( + { + "api_key", + "apikey", + "authorization", + "cookie", + "credential", + "credentials", + "oauth", + "password", + "private_key", + "refresh_token", + "secret", + "session_key", + "token", + } +) +_SENSITIVE_VALUE_PATTERNS = ( + re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{8,}"), + re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), + re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://[^\s/:]+:[^\s/@]+@"), +) +_EnumT = TypeVar("_EnumT", bound=Enum) + + +class BrokerCaptureEventKind(str, Enum): + """First-class quote, lifecycle, health, and clock evidence.""" + + PROCESS_START = "process_start" + PROCESS_STOP = "process_stop" + PROCESS_RESTART = "process_restart" + CONNECTION_OPEN = "connection_open" + CONNECTION_CLOSE = "connection_close" + RECONNECT = "reconnect" + SUBSCRIPTION_ADD = "subscription_add" + SUBSCRIPTION_REMOVE = "subscription_remove" + QUOTE = "quote" + HEARTBEAT = "heartbeat" + GAP = "gap" + OUTAGE_START = "outage_start" + OUTAGE_END = "outage_end" + CLOCK_CORRECTION = "clock_correction" + + @classmethod + def from_value( + cls, value: str | "BrokerCaptureEventKind" + ) -> "BrokerCaptureEventKind": + """Return a strict normalized event kind.""" + return _enum_value(cls, value, "broker capture event kind") + + +class BrokerCaptureSizeSemantics(str, Enum): + """Meaning of optional bid/ask size values.""" + + UNAVAILABLE = "unavailable" + QUOTED_SIZE = "quoted_size" + BROKER_SPECIFIC = "broker_specific" + + @classmethod + def from_value( + cls, value: str | "BrokerCaptureSizeSemantics" + ) -> "BrokerCaptureSizeSemantics": + """Return strict size semantics.""" + return _enum_value(cls, value, "broker capture size semantics") + + +class BrokerCapturePriceTextSemantics(str, Enum): + """Provenance of exact quote lexemes used to measure feed precision.""" + + UNAVAILABLE = "unavailable" + SOURCE_LEXEME = "source_lexeme" + ADAPTER_RENDERED = "adapter_rendered" + + @classmethod + def from_value( + cls, value: str | "BrokerCapturePriceTextSemantics" + ) -> "BrokerCapturePriceTextSemantics": + """Return strict price-text semantics.""" + return _enum_value(cls, value, "broker capture price-text semantics") + + +class BrokerCaptureSourceTimestampSemantics(str, Enum): + """Origin of an optional source-side event timestamp.""" + + UNAVAILABLE = "unavailable" + BROKER_EVENT = "broker_event" + EXCHANGE_EVENT = "exchange_event" + ADAPTER_RECEIVE = "adapter_receive" + + @classmethod + def from_value( + cls, value: str | "BrokerCaptureSourceTimestampSemantics" + ) -> "BrokerCaptureSourceTimestampSemantics": + """Return strict source-timestamp semantics.""" + return _enum_value( + cls, value, "broker capture source-timestamp semantics" + ) + + +class BrokerCaptureActivitySemantics(str, Enum): + """Meaning of an optional broker activity field.""" + + UNAVAILABLE = "unavailable" + MESSAGE_COUNT = "message_count" + BROKER_ACTIVITY = "broker_activity" + LIQUIDITY_PROXY = "liquidity_proxy" + + @classmethod + def from_value( + cls, value: str | "BrokerCaptureActivitySemantics" + ) -> "BrokerCaptureActivitySemantics": + """Return strict activity semantics.""" + return _enum_value(cls, value, "broker capture activity semantics") + + +class BrokerCaptureSessionState(str, Enum): + """Publication state for a capture session.""" + + OPEN = "open" + COMPLETED = "completed" + FAILED = "failed" + + @classmethod + def from_value( + cls, value: str | "BrokerCaptureSessionState" + ) -> "BrokerCaptureSessionState": + """Return a strict normalized session state.""" + return _enum_value(cls, value, "broker capture session state") + + +class BrokerCaptureRetentionMode(str, Enum): + """Fail-closed retention behavior for immutable capture evidence.""" + + REFUSE = "refuse" + + @classmethod + def from_value( + cls, value: str | "BrokerCaptureRetentionMode" + ) -> "BrokerCaptureRetentionMode": + """Return the supported immutable retention mode.""" + return _enum_value(cls, value, "broker capture retention mode") + + +class BrokerCaptureBackpressureMode(str, Enum): + """High-watermark behavior for the synchronous collector seam.""" + + REFUSE = "refuse" + + @classmethod + def from_value( + cls, value: str | "BrokerCaptureBackpressureMode" + ) -> "BrokerCaptureBackpressureMode": + """Return the supported fail-closed backpressure mode.""" + return _enum_value(cls, value, "broker capture backpressure mode") + + +@dataclass(frozen=True, slots=True) +class BrokerAdapterMessageV1: + """One public, credential-free message emitted by a broker adapter.""" + + kind: BrokerCaptureEventKind + source_event_time_ns: int | None = None + source_timestamp_semantics: BrokerCaptureSourceTimestampSemantics = ( + BrokerCaptureSourceTimestampSemantics.UNAVAILABLE + ) + source_timestamp_precision_ns: int | None = None + source_sequence: int | None = None + source_message_id: str | None = None + source_batch_id: str | None = None + symbol: str | None = None + bid: float | None = None + ask: float | None = None + bid_text: str | None = None + ask_text: str | None = None + price_text_semantics: BrokerCapturePriceTextSemantics = ( + BrokerCapturePriceTextSemantics.UNAVAILABLE + ) + bid_size: float | None = None + ask_size: float | None = None + size_semantics: BrokerCaptureSizeSemantics = ( + BrokerCaptureSizeSemantics.UNAVAILABLE + ) + activity_value: float | None = None + activity_semantics: BrokerCaptureActivitySemantics = ( + BrokerCaptureActivitySemantics.UNAVAILABLE + ) + connection_id: str | None = None + subscription_id: str | None = None + gap_duration_ns: int | None = None + reason_code: str | None = None + raw_message_sha256: str | None = None + public_metadata: dict[str, JSONValue] = field(default_factory=dict) + message_id: str = "" + schema_version: str = BROKER_ADAPTER_MESSAGE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_ADAPTER_MESSAGE_SCHEMA_VERSION: + raise ValueError("unsupported broker adapter message schema") + object.__setattr__( + self, "kind", BrokerCaptureEventKind.from_value(self.kind) + ) + object.__setattr__( + self, + "source_event_time_ns", + _optional_nonnegative_int64( + self.source_event_time_ns, "source_event_time_ns" + ), + ) + object.__setattr__( + self, + "source_timestamp_semantics", + BrokerCaptureSourceTimestampSemantics.from_value( + self.source_timestamp_semantics + ), + ) + object.__setattr__( + self, + "source_timestamp_precision_ns", + _optional_nonnegative_int64( + self.source_timestamp_precision_ns, + "source_timestamp_precision_ns", + ), + ) + object.__setattr__( + self, + "source_sequence", + _optional_nonnegative_int64( + self.source_sequence, "source_sequence" + ), + ) + for name in ( + "source_message_id", + "source_batch_id", + "connection_id", + "subscription_id", + "reason_code", + ): + object.__setattr__( + self, + name, + _optional_canonical_id(getattr(self, name), name), + ) + symbol = _optional_symbol(self.symbol) + object.__setattr__(self, "symbol", symbol) + for name in ("bid", "ask", "bid_size", "ask_size", "activity_value"): + object.__setattr__( + self, + name, + _optional_nonnegative_float(getattr(self, name), name), + ) + for name in ("bid_text", "ask_text"): + object.__setattr__( + self, name, _optional_price_lexeme(getattr(self, name), name) + ) + object.__setattr__( + self, + "price_text_semantics", + BrokerCapturePriceTextSemantics.from_value( + self.price_text_semantics + ), + ) + object.__setattr__( + self, + "size_semantics", + BrokerCaptureSizeSemantics.from_value(self.size_semantics), + ) + object.__setattr__( + self, + "activity_semantics", + BrokerCaptureActivitySemantics.from_value(self.activity_semantics), + ) + object.__setattr__( + self, + "gap_duration_ns", + _optional_nonnegative_int64( + self.gap_duration_ns, "gap_duration_ns" + ), + ) + raw_hash = _optional_sha256( + self.raw_message_sha256, "raw_message_sha256" + ) + object.__setattr__(self, "raw_message_sha256", raw_hash) + metadata = _secret_free_metadata(self.public_metadata) + object.__setattr__(self, "public_metadata", metadata) + self._validate_kind_fields() + expected = _stable_id("broker-message", self.identity_payload()) + supplied = _optional_text(self.message_id) + if supplied is not None and supplied != expected: + raise ValueError("message_id does not match deterministic identity") + object.__setattr__(self, "message_id", expected) + + def _validate_kind_fields(self) -> None: + timestamp_present = self.source_event_time_ns is not None + if timestamp_present: + if ( + self.source_timestamp_semantics + is BrokerCaptureSourceTimestampSemantics.UNAVAILABLE + ): + raise ValueError( + "source timestamps require explicit timestamp semantics" + ) + if not self.source_timestamp_precision_ns: + raise ValueError( + "source timestamps require positive timestamp precision" + ) + elif ( + self.source_timestamp_semantics + is not BrokerCaptureSourceTimestampSemantics.UNAVAILABLE + or self.source_timestamp_precision_ns is not None + ): + raise ValueError( + "source timestamp metadata requires a source timestamp" + ) + quote_values = (self.bid, self.ask) + if self.kind is BrokerCaptureEventKind.QUOTE: + if self.symbol is None or any( + value is None for value in quote_values + ): + raise ValueError("quote messages require symbol, bid, and ask") + if cast(float, self.bid) <= 0 or cast(float, self.ask) <= 0: + raise ValueError("quote prices must be positive") + if cast(float, self.bid) > cast(float, self.ask): + raise ValueError("quote bid must not exceed ask") + elif any(value is not None for value in quote_values): + raise ValueError("non-quote messages cannot contain bid or ask") + + text_present = self.bid_text is not None or self.ask_text is not None + if text_present: + if self.kind is not BrokerCaptureEventKind.QUOTE: + raise ValueError("price lexemes are only valid on quotes") + if self.bid_text is None or self.ask_text is None: + raise ValueError( + "bid_text and ask_text must be supplied together" + ) + if ( + self.price_text_semantics + is BrokerCapturePriceTextSemantics.UNAVAILABLE + ): + raise ValueError( + "price lexemes require explicit price-text semantics" + ) + if Decimal(self.bid_text) != Decimal(str(self.bid)) or Decimal( + self.ask_text + ) != Decimal(str(self.ask)): + raise ValueError("price lexemes do not match numeric quotes") + elif ( + self.price_text_semantics + is not BrokerCapturePriceTextSemantics.UNAVAILABLE + ): + raise ValueError("price-text semantics require quote lexemes") + + sizes_present = self.bid_size is not None or self.ask_size is not None + if sizes_present and self.kind is not BrokerCaptureEventKind.QUOTE: + raise ValueError("sizes are only valid on quote messages") + if ( + sizes_present + and self.size_semantics is BrokerCaptureSizeSemantics.UNAVAILABLE + ): + raise ValueError("size values require explicit size semantics") + if ( + not sizes_present + and self.size_semantics + is not BrokerCaptureSizeSemantics.UNAVAILABLE + ): + raise ValueError("size semantics require at least one size value") + if ( + self.activity_value is None + and self.activity_semantics + is not BrokerCaptureActivitySemantics.UNAVAILABLE + ): + raise ValueError("activity semantics require an activity value") + if ( + self.activity_value is not None + and self.activity_semantics + is BrokerCaptureActivitySemantics.UNAVAILABLE + ): + raise ValueError("activity values require explicit semantics") + + connection_kinds = { + BrokerCaptureEventKind.CONNECTION_OPEN, + BrokerCaptureEventKind.CONNECTION_CLOSE, + BrokerCaptureEventKind.RECONNECT, + } + if self.kind in connection_kinds and self.connection_id is None: + raise ValueError( + "connection lifecycle messages require connection_id" + ) + subscription_kinds = { + BrokerCaptureEventKind.SUBSCRIPTION_ADD, + BrokerCaptureEventKind.SUBSCRIPTION_REMOVE, + } + if self.kind in subscription_kinds and ( + self.subscription_id is None or self.symbol is None + ): + raise ValueError( + "subscription messages require subscription_id and symbol" + ) + gap_kinds = { + BrokerCaptureEventKind.GAP, + BrokerCaptureEventKind.OUTAGE_END, + } + if self.kind in gap_kinds and self.gap_duration_ns is None: + raise ValueError("gap/outage-end messages require gap_duration_ns") + if ( + self.kind is BrokerCaptureEventKind.PROCESS_RESTART + and self.reason_code is None + ): + raise ValueError("process restart messages require a reason code") + if ( + self.kind is BrokerCaptureEventKind.CLOCK_CORRECTION + and self.reason_code != "wall_monotonic_divergence" + ): + raise ValueError( + "clock corrections require the collector reason code" + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields defining stable adapter-message identity.""" + return { + "schema_version": self.schema_version, + "kind": self.kind.value, + "source_event_time_ns": self.source_event_time_ns, + "source_timestamp_semantics": self.source_timestamp_semantics.value, + "source_timestamp_precision_ns": self.source_timestamp_precision_ns, + "source_sequence": self.source_sequence, + "source_message_id": self.source_message_id, + "source_batch_id": self.source_batch_id, + "symbol": self.symbol, + "bid": self.bid, + "ask": self.ask, + "bid_text": self.bid_text, + "ask_text": self.ask_text, + "price_text_semantics": self.price_text_semantics.value, + "bid_size": self.bid_size, + "ask_size": self.ask_size, + "size_semantics": self.size_semantics.value, + "activity_value": self.activity_value, + "activity_semantics": self.activity_semantics.value, + "connection_id": self.connection_id, + "subscription_id": self.subscription_id, + "gap_duration_ns": self.gap_duration_ns, + "reason_code": self.reason_code, + "raw_message_sha256": self.raw_message_sha256, + "public_metadata": dict(self.public_metadata), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible message metadata.""" + return {**self.identity_payload(), "message_id": self.message_id} + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return canonical_capture_json(self.to_dict()) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerAdapterMessageV1": + """Restore and verify a version-one adapter message.""" + _require_schema(data, BROKER_ADAPTER_MESSAGE_SCHEMA_VERSION) + return cls( + kind=BrokerCaptureEventKind.from_value(str(data.get("kind", ""))), + source_event_time_ns=cast( + int | None, data.get("source_event_time_ns") + ), + source_timestamp_semantics=( + BrokerCaptureSourceTimestampSemantics.from_value( + str(data.get("source_timestamp_semantics", "")) + ) + ), + source_timestamp_precision_ns=cast( + int | None, data.get("source_timestamp_precision_ns") + ), + source_sequence=cast(int | None, data.get("source_sequence")), + source_message_id=_mapping_optional_text(data, "source_message_id"), + source_batch_id=_mapping_optional_text(data, "source_batch_id"), + symbol=_mapping_optional_text(data, "symbol"), + bid=cast(float | None, data.get("bid")), + ask=cast(float | None, data.get("ask")), + bid_text=_mapping_optional_text(data, "bid_text"), + ask_text=_mapping_optional_text(data, "ask_text"), + price_text_semantics=BrokerCapturePriceTextSemantics.from_value( + str(data.get("price_text_semantics", "")) + ), + bid_size=cast(float | None, data.get("bid_size")), + ask_size=cast(float | None, data.get("ask_size")), + size_semantics=BrokerCaptureSizeSemantics.from_value( + str(data.get("size_semantics", "")) + ), + activity_value=cast(float | None, data.get("activity_value")), + activity_semantics=BrokerCaptureActivitySemantics.from_value( + str(data.get("activity_semantics", "")) + ), + connection_id=_mapping_optional_text(data, "connection_id"), + subscription_id=_mapping_optional_text(data, "subscription_id"), + gap_duration_ns=cast(int | None, data.get("gap_duration_ns")), + reason_code=_mapping_optional_text(data, "reason_code"), + raw_message_sha256=_mapping_optional_text( + data, "raw_message_sha256" + ), + public_metadata=_json_dict(data.get("public_metadata")), + message_id=str(data.get("message_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerAdapterMessageV1": + """Restore a message from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureSessionV1: + """Public, credential-free identity and clock context for one session.""" + + adapter_id: str + adapter_version: str + adapter_config_sha256: str + protocol: str + environment_id: str + server_id: str + started_at_utc_ns: int + started_at_monotonic_ns: int + account_id_sha256: str | None = None + host_id_sha256: str | None = None + clock_source: str = "system_wall_and_monotonic" + clock_resolution_ns: int = 1 + collector_id: str = BROKER_CAPTURE_COLLECTOR_ID + collector_version: str = BROKER_CAPTURE_COLLECTOR_VERSION + public_metadata: dict[str, JSONValue] = field(default_factory=dict) + session_id: str = "" + schema_version: str = BROKER_CAPTURE_SESSION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_CAPTURE_SESSION_SCHEMA_VERSION: + raise ValueError("unsupported broker capture session schema") + for name in ( + "adapter_id", + "protocol", + "environment_id", + "server_id", + "clock_source", + "collector_id", + ): + object.__setattr__( + self, name, _required_canonical_id(getattr(self, name), name) + ) + for name in ("adapter_version", "collector_version"): + value = _required_text(getattr(self, name)) + if not _SEMVER_RE.fullmatch(value): + raise ValueError(f"{name} must be a semantic version") + object.__setattr__(self, name, value) + object.__setattr__( + self, + "adapter_config_sha256", + _required_sha256( + self.adapter_config_sha256, "adapter_config_sha256" + ), + ) + for name in ("account_id_sha256", "host_id_sha256"): + object.__setattr__( + self, + name, + _optional_sha256(getattr(self, name), name), + ) + for name in ("started_at_utc_ns", "started_at_monotonic_ns"): + object.__setattr__( + self, + name, + _nonnegative_int64(getattr(self, name), name), + ) + object.__setattr__( + self, + "clock_resolution_ns", + _positive_int64(self.clock_resolution_ns, "clock_resolution_ns"), + ) + metadata = _secret_free_metadata(self.public_metadata) + object.__setattr__(self, "public_metadata", metadata) + expected = _stable_id("broker-capture-session", self.identity_payload()) + supplied = _optional_text(self.session_id) + if supplied is not None and supplied != expected: + raise ValueError("session_id does not match deterministic identity") + object.__setattr__(self, "session_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields defining this capture session.""" + return { + "schema_version": self.schema_version, + "adapter_id": self.adapter_id, + "adapter_version": self.adapter_version, + "adapter_config_sha256": self.adapter_config_sha256, + "protocol": self.protocol, + "environment_id": self.environment_id, + "server_id": self.server_id, + "account_id_sha256": self.account_id_sha256, + "host_id_sha256": self.host_id_sha256, + "started_at_utc_ns": self.started_at_utc_ns, + "started_at_monotonic_ns": self.started_at_monotonic_ns, + "clock_source": self.clock_source, + "clock_resolution_ns": self.clock_resolution_ns, + "collector_id": self.collector_id, + "collector_version": self.collector_version, + "public_metadata": dict(self.public_metadata), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible session metadata.""" + return {**self.identity_payload(), "session_id": self.session_id} + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return canonical_capture_json(self.to_dict()) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerCaptureSessionV1": + """Restore and verify a version-one capture session.""" + _require_schema(data, BROKER_CAPTURE_SESSION_SCHEMA_VERSION) + return cls( + adapter_id=str(data.get("adapter_id", "")), + adapter_version=str(data.get("adapter_version", "")), + adapter_config_sha256=str(data.get("adapter_config_sha256", "")), + protocol=str(data.get("protocol", "")), + environment_id=str(data.get("environment_id", "")), + server_id=str(data.get("server_id", "")), + started_at_utc_ns=cast(int, data.get("started_at_utc_ns")), + started_at_monotonic_ns=cast( + int, data.get("started_at_monotonic_ns") + ), + account_id_sha256=_mapping_optional_text(data, "account_id_sha256"), + host_id_sha256=_mapping_optional_text(data, "host_id_sha256"), + clock_source=str(data.get("clock_source", "")), + clock_resolution_ns=cast(int, data.get("clock_resolution_ns")), + collector_id=str(data.get("collector_id", "")), + collector_version=str(data.get("collector_version", "")), + public_metadata=_json_dict(data.get("public_metadata")), + session_id=str(data.get("session_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerCaptureSessionV1": + """Restore a session from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureEventV1: + """One collector-stamped broker message in stable capture order.""" + + session_id: str + capture_sequence: int + receive_time_utc_ns: int + receive_time_monotonic_ns: int + message: BrokerAdapterMessageV1 + clock_offset_change_ns: int | None = None + event_id: str = "" + schema_version: str = BROKER_CAPTURE_EVENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_CAPTURE_EVENT_SCHEMA_VERSION: + raise ValueError("unsupported broker capture event schema") + object.__setattr__( + self, + "session_id", + _required_prefixed_id(self.session_id, "broker-capture-session-"), + ) + for name in ( + "capture_sequence", + "receive_time_utc_ns", + "receive_time_monotonic_ns", + ): + object.__setattr__( + self, name, _nonnegative_int64(getattr(self, name), name) + ) + if not isinstance(self.message, BrokerAdapterMessageV1): + raise TypeError("message must be BrokerAdapterMessageV1") + correction = _optional_int64( + self.clock_offset_change_ns, "clock_offset_change_ns" + ) + object.__setattr__(self, "clock_offset_change_ns", correction) + if self.message.kind is BrokerCaptureEventKind.CLOCK_CORRECTION: + if correction is None or correction == 0: + raise ValueError( + "clock correction events require non-zero offset change" + ) + elif correction is not None: + raise ValueError("clock offset change is only valid on corrections") + expected = _stable_id("broker-capture-event", self.identity_payload()) + supplied = _optional_text(self.event_id) + if supplied is not None and supplied != expected: + raise ValueError("event_id does not match deterministic identity") + object.__setattr__(self, "event_id", expected) + + @property + def kind(self) -> BrokerCaptureEventKind: + """Return the underlying broker-message kind.""" + return self.message.kind + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields defining immutable capture-event identity.""" + return { + "schema_version": self.schema_version, + "session_id": self.session_id, + "capture_sequence": self.capture_sequence, + "receive_time_utc_ns": self.receive_time_utc_ns, + "receive_time_monotonic_ns": self.receive_time_monotonic_ns, + "message": self.message.to_dict(), + "clock_offset_change_ns": self.clock_offset_change_ns, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible event evidence.""" + return {**self.identity_payload(), "event_id": self.event_id} + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return canonical_capture_json(self.to_dict()) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerCaptureEventV1": + """Restore and verify a version-one capture event.""" + _require_schema(data, BROKER_CAPTURE_EVENT_SCHEMA_VERSION) + return cls( + session_id=str(data.get("session_id", "")), + capture_sequence=cast(int, data.get("capture_sequence")), + receive_time_utc_ns=cast(int, data.get("receive_time_utc_ns")), + receive_time_monotonic_ns=cast( + int, data.get("receive_time_monotonic_ns") + ), + message=BrokerAdapterMessageV1.from_dict( + _mapping(data.get("message")) + ), + clock_offset_change_ns=cast( + int | None, data.get("clock_offset_change_ns") + ), + event_id=str(data.get("event_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerCaptureEventV1": + """Restore an event from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureStoragePolicyV1: + """Rotation, quota, retention, and backpressure limits.""" + + max_partition_events: int = 100_000 + max_partition_bytes: int = 128 * 1024**2 + max_partition_duration_ns: int = 5 * 60 * 1_000_000_000 + max_session_bytes: int = 10 * 1024**3 + high_watermark_bytes: int = 8 * 1024**3 + max_retained_partitions: int = 4096 + manifest_reserve_bytes: int = 64 * 1024 + fsync_each_event: bool = True + retention_mode: BrokerCaptureRetentionMode = ( + BrokerCaptureRetentionMode.REFUSE + ) + backpressure_mode: BrokerCaptureBackpressureMode = ( + BrokerCaptureBackpressureMode.REFUSE + ) + policy_id: str = "" + schema_version: str = BROKER_CAPTURE_STORAGE_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_CAPTURE_STORAGE_POLICY_SCHEMA_VERSION: + raise ValueError("unsupported broker capture storage policy schema") + for name in ( + "max_partition_events", + "max_partition_bytes", + "max_partition_duration_ns", + "max_session_bytes", + "high_watermark_bytes", + "max_retained_partitions", + "manifest_reserve_bytes", + ): + object.__setattr__( + self, name, _positive_int64(getattr(self, name), name) + ) + if ( + self.max_partition_bytes + self.manifest_reserve_bytes + > self.max_session_bytes + ): + raise ValueError( + "one partition plus manifest reserve exceeds session quota" + ) + if self.high_watermark_bytes > self.max_session_bytes: + raise ValueError("high watermark exceeds session quota") + object.__setattr__( + self, + "fsync_each_event", + _strict_bool(self.fsync_each_event, "fsync_each_event"), + ) + retention = BrokerCaptureRetentionMode.from_value(self.retention_mode) + backpressure = BrokerCaptureBackpressureMode.from_value( + self.backpressure_mode + ) + if retention is not BrokerCaptureRetentionMode.REFUSE: + raise ValueError("v1 capture retention must refuse before deletion") + if backpressure is not BrokerCaptureBackpressureMode.REFUSE: + raise ValueError("v1 capture backpressure must refuse") + object.__setattr__(self, "retention_mode", retention) + object.__setattr__(self, "backpressure_mode", backpressure) + expected = _stable_id("broker-capture-policy", self.identity_payload()) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("policy_id does not match deterministic identity") + object.__setattr__(self, "policy_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields defining storage behavior.""" + return { + "schema_version": self.schema_version, + "max_partition_events": self.max_partition_events, + "max_partition_bytes": self.max_partition_bytes, + "max_partition_duration_ns": self.max_partition_duration_ns, + "max_session_bytes": self.max_session_bytes, + "high_watermark_bytes": self.high_watermark_bytes, + "max_retained_partitions": self.max_retained_partitions, + "manifest_reserve_bytes": self.manifest_reserve_bytes, + "fsync_each_event": self.fsync_each_event, + "retention_mode": self.retention_mode.value, + "backpressure_mode": self.backpressure_mode.value, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible policy metadata.""" + return {**self.identity_payload(), "policy_id": self.policy_id} + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return canonical_capture_json(self.to_dict()) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerCaptureStoragePolicyV1": + """Restore and verify a version-one storage policy.""" + _require_schema(data, BROKER_CAPTURE_STORAGE_POLICY_SCHEMA_VERSION) + return cls( + max_partition_events=cast(int, data.get("max_partition_events")), + max_partition_bytes=cast(int, data.get("max_partition_bytes")), + max_partition_duration_ns=cast( + int, data.get("max_partition_duration_ns") + ), + max_session_bytes=cast(int, data.get("max_session_bytes")), + high_watermark_bytes=cast(int, data.get("high_watermark_bytes")), + max_retained_partitions=cast( + int, data.get("max_retained_partitions") + ), + manifest_reserve_bytes=cast( + int, data.get("manifest_reserve_bytes") + ), + fsync_each_event=cast(bool, data.get("fsync_each_event")), + retention_mode=BrokerCaptureRetentionMode.from_value( + str(data.get("retention_mode", "")) + ), + backpressure_mode=BrokerCaptureBackpressureMode.from_value( + str(data.get("backpressure_mode", "")) + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerCaptureStoragePolicyV1": + """Restore a policy from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerCapturePartitionManifestV1: + """Compact immutable manifest for one completed JSONL partition.""" + + session_id: str + policy_id: str + partition_ordinal: int + data_artifact: ArtifactRef + event_count: int + first_capture_sequence: int + last_capture_sequence: int + first_receive_time_utc_ns: int + last_receive_time_utc_ns: int + first_receive_time_monotonic_ns: int + last_receive_time_monotonic_ns: int + event_kind_counts: dict[str, int] + completed: bool = True + partition_id: str = "" + schema_version: str = BROKER_CAPTURE_PARTITION_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != BROKER_CAPTURE_PARTITION_MANIFEST_SCHEMA_VERSION + ): + raise ValueError("unsupported broker capture partition manifest") + object.__setattr__( + self, + "session_id", + _required_prefixed_id(self.session_id, "broker-capture-session-"), + ) + object.__setattr__( + self, + "policy_id", + _required_prefixed_id(self.policy_id, "broker-capture-policy-"), + ) + for name in ( + "partition_ordinal", + "event_count", + "first_capture_sequence", + "last_capture_sequence", + "first_receive_time_utc_ns", + "last_receive_time_utc_ns", + "first_receive_time_monotonic_ns", + "last_receive_time_monotonic_ns", + ): + object.__setattr__( + self, name, _nonnegative_int64(getattr(self, name), name) + ) + if self.event_count <= 0: + raise ValueError("completed partitions must contain events") + if self.last_capture_sequence < self.first_capture_sequence: + raise ValueError("partition capture sequence bounds are reversed") + if ( + self.last_capture_sequence - self.first_capture_sequence + 1 + != self.event_count + ): + raise ValueError("partition capture sequences must be contiguous") + if ( + self.last_receive_time_monotonic_ns + < self.first_receive_time_monotonic_ns + ): + raise ValueError("partition monotonic receive bounds are reversed") + counts = _event_kind_counts(self.event_kind_counts) + if sum(counts.values()) != self.event_count: + raise ValueError("partition event-kind counts do not reconcile") + object.__setattr__(self, "event_kind_counts", counts) + artifact = _validated_capture_artifact(self.data_artifact) + object.__setattr__(self, "data_artifact", artifact) + if not _strict_bool(self.completed, "completed"): + raise ValueError( + "partition manifests only advertise completed data" + ) + object.__setattr__(self, "completed", True) + expected = _stable_id( + "broker-capture-partition", self.identity_payload() + ) + supplied = _optional_text(self.partition_id) + if supplied is not None and supplied != expected: + raise ValueError( + "partition_id does not match deterministic identity" + ) + object.__setattr__(self, "partition_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields defining this immutable partition.""" + return { + "schema_version": self.schema_version, + "session_id": self.session_id, + "policy_id": self.policy_id, + "partition_ordinal": self.partition_ordinal, + "data_artifact": self.data_artifact.to_dict(), + "event_count": self.event_count, + "first_capture_sequence": self.first_capture_sequence, + "last_capture_sequence": self.last_capture_sequence, + "first_receive_time_utc_ns": self.first_receive_time_utc_ns, + "last_receive_time_utc_ns": self.last_receive_time_utc_ns, + "first_receive_time_monotonic_ns": ( + self.first_receive_time_monotonic_ns + ), + "last_receive_time_monotonic_ns": ( + self.last_receive_time_monotonic_ns + ), + "event_kind_counts": dict(self.event_kind_counts), + "completed": True, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible manifest metadata.""" + return {**self.identity_payload(), "partition_id": self.partition_id} + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return canonical_capture_json(self.to_dict()) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerCapturePartitionManifestV1": + """Restore and verify a version-one partition manifest.""" + _require_schema(data, BROKER_CAPTURE_PARTITION_MANIFEST_SCHEMA_VERSION) + return cls( + session_id=str(data.get("session_id", "")), + policy_id=str(data.get("policy_id", "")), + partition_ordinal=cast(int, data.get("partition_ordinal")), + data_artifact=ArtifactRef.from_dict( + _mapping(data.get("data_artifact")) + ), + event_count=cast(int, data.get("event_count")), + first_capture_sequence=cast( + int, data.get("first_capture_sequence") + ), + last_capture_sequence=cast(int, data.get("last_capture_sequence")), + first_receive_time_utc_ns=cast( + int, data.get("first_receive_time_utc_ns") + ), + last_receive_time_utc_ns=cast( + int, data.get("last_receive_time_utc_ns") + ), + first_receive_time_monotonic_ns=cast( + int, data.get("first_receive_time_monotonic_ns") + ), + last_receive_time_monotonic_ns=cast( + int, data.get("last_receive_time_monotonic_ns") + ), + event_kind_counts={ + str(key): cast(int, value) + for key, value in _mapping( + data.get("event_kind_counts") + ).items() + }, + completed=cast(bool, data.get("completed")), + partition_id=str(data.get("partition_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerCapturePartitionManifestV1": + """Restore a partition manifest from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureSessionManifestV1: + """Bounded catalog of completed partitions and capture-health evidence.""" + + session: BrokerCaptureSessionV1 + storage_policy: BrokerCaptureStoragePolicyV1 + state: BrokerCaptureSessionState + partitions: tuple[BrokerCapturePartitionManifestV1, ...] + event_count: int + event_kind_counts: dict[str, int] + first_capture_sequence: int | None + last_capture_sequence: int | None + partial_artifact_count: int = 0 + limitations: tuple[str, ...] = () + manifest_id: str = "" + schema_version: str = BROKER_CAPTURE_SESSION_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != BROKER_CAPTURE_SESSION_MANIFEST_SCHEMA_VERSION + ): + raise ValueError("unsupported broker capture session manifest") + if not isinstance(self.session, BrokerCaptureSessionV1): + raise TypeError("session must be BrokerCaptureSessionV1") + if not isinstance(self.storage_policy, BrokerCaptureStoragePolicyV1): + raise TypeError( + "storage_policy must be BrokerCaptureStoragePolicyV1" + ) + object.__setattr__( + self, "state", BrokerCaptureSessionState.from_value(self.state) + ) + partitions = tuple( + sorted(self.partitions, key=lambda item: item.partition_ordinal) + ) + if len(partitions) > MAX_CAPTURE_PARTITIONS: + raise ValueError("capture session manifest has too many partitions") + if len({item.partition_id for item in partitions}) != len(partitions): + raise ValueError( + "capture session manifest has duplicate partitions" + ) + for expected_ordinal, partition in enumerate(partitions): + if partition.partition_ordinal != expected_ordinal: + raise ValueError( + "capture partition ordinals must be contiguous" + ) + if partition.session_id != self.session.session_id: + raise ValueError("capture partition session does not match") + if partition.policy_id != self.storage_policy.policy_id: + raise ValueError("capture partition policy does not match") + if expected_ordinal and ( + partition.first_capture_sequence + != partitions[expected_ordinal - 1].last_capture_sequence + 1 + ): + raise ValueError( + "capture partition sequences must be contiguous" + ) + object.__setattr__(self, "partitions", partitions) + event_count = _nonnegative_int64(self.event_count, "event_count") + if sum(item.event_count for item in partitions) != event_count: + raise ValueError("capture session event count does not reconcile") + object.__setattr__(self, "event_count", event_count) + counts = _event_kind_counts(self.event_kind_counts, allow_empty=True) + combined: dict[str, int] = {} + for partition in partitions: + for kind, count in partition.event_kind_counts.items(): + combined[kind] = combined.get(kind, 0) + count + combined = dict(sorted(combined.items())) + if counts != combined: + raise ValueError( + "capture session event-kind counts do not reconcile" + ) + object.__setattr__(self, "event_kind_counts", counts) + first = _optional_nonnegative_int64( + self.first_capture_sequence, "first_capture_sequence" + ) + last = _optional_nonnegative_int64( + self.last_capture_sequence, "last_capture_sequence" + ) + if partitions: + if ( + first != partitions[0].first_capture_sequence + or last != partitions[-1].last_capture_sequence + ): + raise ValueError( + "capture session sequence bounds do not reconcile" + ) + elif first is not None or last is not None: + raise ValueError( + "empty capture session cannot have sequence bounds" + ) + object.__setattr__(self, "first_capture_sequence", first) + object.__setattr__(self, "last_capture_sequence", last) + partial_count = _nonnegative_int64( + self.partial_artifact_count, "partial_artifact_count" + ) + if partial_count: + raise ValueError( + "published manifests cannot advertise partial artifacts" + ) + object.__setattr__(self, "partial_artifact_count", 0) + limitations = tuple( + sorted({_bounded_text(value) for value in self.limitations}) + ) + object.__setattr__(self, "limitations", limitations) + expected = _stable_id( + "broker-capture-manifest", self.identity_payload() + ) + supplied = _optional_text(self.manifest_id) + if supplied is not None and supplied != expected: + raise ValueError( + "manifest_id does not match deterministic identity" + ) + object.__setattr__(self, "manifest_id", expected) + if len(self.to_json().encode("utf-8")) > MAX_CAPTURE_MANIFEST_BYTES: + raise ValueError("capture session manifest exceeds bounded size") + + @property + def complete(self) -> bool: + """Return whether collection ended normally.""" + return self.state is BrokerCaptureSessionState.COMPLETED + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields defining this manifest snapshot.""" + return { + "schema_version": self.schema_version, + "session": self.session.to_dict(), + "storage_policy": self.storage_policy.to_dict(), + "state": self.state.value, + "partitions": [item.to_dict() for item in self.partitions], + "event_count": self.event_count, + "event_kind_counts": dict(self.event_kind_counts), + "first_capture_sequence": self.first_capture_sequence, + "last_capture_sequence": self.last_capture_sequence, + "partial_artifact_count": 0, + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible manifest metadata.""" + return {**self.identity_payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return canonical_capture_json(self.to_dict()) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerCaptureSessionManifestV1": + """Restore and verify a version-one session manifest.""" + _require_schema(data, BROKER_CAPTURE_SESSION_MANIFEST_SCHEMA_VERSION) + return cls( + session=BrokerCaptureSessionV1.from_dict( + _mapping(data.get("session")) + ), + storage_policy=BrokerCaptureStoragePolicyV1.from_dict( + _mapping(data.get("storage_policy")) + ), + state=BrokerCaptureSessionState.from_value( + str(data.get("state", "")) + ), + partitions=tuple( + BrokerCapturePartitionManifestV1.from_dict(item) + for item in _mapping_sequence(data.get("partitions")) + ), + event_count=cast(int, data.get("event_count")), + event_kind_counts={ + str(key): cast(int, value) + for key, value in _mapping( + data.get("event_kind_counts") + ).items() + }, + first_capture_sequence=cast( + int | None, data.get("first_capture_sequence") + ), + last_capture_sequence=cast( + int | None, data.get("last_capture_sequence") + ), + partial_artifact_count=cast( + int, data.get("partial_artifact_count", 0) + ), + limitations=tuple( + str(value) for value in _sequence(data.get("limitations")) + ), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerCaptureSessionManifestV1": + """Restore a session manifest from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureReplaySummaryV1: + """Bounded integrity and ordering evidence from one replay.""" + + session_id: str + manifest_id: str + partition_count: int + event_count: int + event_kind_counts: dict[str, int] + first_capture_sequence: int | None + last_capture_sequence: int | None + logical_content_sha256: str + summary_id: str = "" + schema_version: str = BROKER_CAPTURE_REPLAY_SUMMARY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_CAPTURE_REPLAY_SUMMARY_SCHEMA_VERSION: + raise ValueError("unsupported broker capture replay summary") + object.__setattr__( + self, + "session_id", + _required_prefixed_id(self.session_id, "broker-capture-session-"), + ) + object.__setattr__( + self, + "manifest_id", + _required_prefixed_id(self.manifest_id, "broker-capture-manifest-"), + ) + for name in ("partition_count", "event_count"): + object.__setattr__( + self, name, _nonnegative_int64(getattr(self, name), name) + ) + counts = _event_kind_counts(self.event_kind_counts, allow_empty=True) + if sum(counts.values()) != self.event_count: + raise ValueError("replay event-kind counts do not reconcile") + object.__setattr__(self, "event_kind_counts", counts) + first = _optional_nonnegative_int64( + self.first_capture_sequence, "first_capture_sequence" + ) + last = _optional_nonnegative_int64( + self.last_capture_sequence, "last_capture_sequence" + ) + if self.event_count: + if ( + first is None + or last is None + or last - first + 1 != self.event_count + ): + raise ValueError("replay capture sequences do not reconcile") + elif first is not None or last is not None: + raise ValueError("empty replay cannot have sequence bounds") + object.__setattr__(self, "first_capture_sequence", first) + object.__setattr__(self, "last_capture_sequence", last) + object.__setattr__( + self, + "logical_content_sha256", + _required_sha256( + self.logical_content_sha256, "logical_content_sha256" + ), + ) + expected = _stable_id("broker-capture-replay", self.identity_payload()) + supplied = _optional_text(self.summary_id) + if supplied is not None and supplied != expected: + raise ValueError("summary_id does not match deterministic identity") + object.__setattr__(self, "summary_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields defining replay evidence.""" + return { + "schema_version": self.schema_version, + "session_id": self.session_id, + "manifest_id": self.manifest_id, + "partition_count": self.partition_count, + "event_count": self.event_count, + "event_kind_counts": dict(self.event_kind_counts), + "first_capture_sequence": self.first_capture_sequence, + "last_capture_sequence": self.last_capture_sequence, + "logical_content_sha256": self.logical_content_sha256, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible replay evidence.""" + return {**self.identity_payload(), "summary_id": self.summary_id} + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return canonical_capture_json(self.to_dict()) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerCaptureReplaySummaryV1": + """Restore and verify version-one replay evidence.""" + _require_schema(data, BROKER_CAPTURE_REPLAY_SUMMARY_SCHEMA_VERSION) + return cls( + session_id=str(data.get("session_id", "")), + manifest_id=str(data.get("manifest_id", "")), + partition_count=cast(int, data.get("partition_count")), + event_count=cast(int, data.get("event_count")), + event_kind_counts={ + str(key): cast(int, value) + for key, value in _mapping( + data.get("event_kind_counts") + ).items() + }, + first_capture_sequence=cast( + int | None, data.get("first_capture_sequence") + ), + last_capture_sequence=cast( + int | None, data.get("last_capture_sequence") + ), + logical_content_sha256=str(data.get("logical_content_sha256", "")), + summary_id=str(data.get("summary_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerCaptureReplaySummaryV1": + """Restore replay evidence from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +def canonical_capture_json(value: Mapping[str, JSONValue]) -> str: + """Return canonical compact UTF-8 JSON for capture contracts.""" + return json.dumps( + dict(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + + +def assert_secret_free_capture_value(value: JSONValue) -> None: + """Fail closed if a public capture value resembles credential material.""" + _validate_secret_free(value, path="capture") + + +def logical_capture_content_sha256( + events: Sequence[BrokerCaptureEventV1], +) -> str: + """Hash ordered canonical event content without filesystem metadata.""" + digest = hashlib.sha256() + for event in events: + digest.update(event.to_json().encode("utf-8")) + digest.update(b"\n") + return digest.hexdigest() + + +def _validated_capture_artifact(value: ArtifactRef) -> ArtifactRef: + if not isinstance(value, ArtifactRef): + raise TypeError("data_artifact must be ArtifactRef") + if value.kind != BROKER_CAPTURE_DATA_ARTIFACT_KIND: + raise ValueError("unsupported broker capture artifact kind") + path = _relative_artifact_path(value.path) + size = _positive_int64(value.size_bytes, "data_artifact.size_bytes") + digest = _required_sha256(value.sha256, "data_artifact.sha256") + metadata = _secret_free_metadata(value.metadata) + return ArtifactRef( + kind=value.kind, + path=path, + size_bytes=size, + sha256=digest, + metadata=metadata, + ) + + +def _event_kind_counts( + values: Mapping[str, int], *, allow_empty: bool = False +) -> dict[str, int]: + if len(values) > MAX_CAPTURE_KIND_COUNTS: + raise ValueError("event-kind count map exceeds limit") + counts: dict[str, int] = {} + for key, value in values.items(): + kind = BrokerCaptureEventKind.from_value(str(key)).value + count = _nonnegative_int64(value, f"event_kind_counts.{kind}") + if count: + counts[kind] = count + if not counts and not allow_empty: + raise ValueError("event-kind counts cannot be empty") + return dict(sorted(counts.items())) + + +def _secret_free_metadata(value: Mapping[str, Any]) -> dict[str, JSONValue]: + converted = cast(dict[str, JSONValue], dict(value)) + _validate_json_value(converted, "public_metadata") + _validate_secret_free(converted, path="public_metadata") + encoded = canonical_capture_json(converted) + if len(encoded.encode("utf-8")) > MAX_CAPTURE_METADATA_BYTES: + raise ValueError("public metadata exceeds bounded size") + return dict(sorted(converted.items())) + + +def _validate_secret_free(value: JSONValue, *, path: str) -> None: + if isinstance(value, dict): + for key, item in value.items(): + normalized = re.sub(r"[^a-z0-9]+", "_", key.strip().lower()).strip( + "_" + ) + if any( + part == normalized or part in normalized.split("_") + for part in _SENSITIVE_KEY_PARTS + ): + raise ValueError( + f"sensitive metadata key is prohibited at {path}" + ) + _validate_secret_free(item, path=f"{path}.{key}") + return + if isinstance(value, list): + for index, item in enumerate(value): + _validate_secret_free(item, path=f"{path}[{index}]") + return + if isinstance(value, str): + if any(pattern.search(value) for pattern in _SENSITIVE_VALUE_PATTERNS): + raise ValueError( + f"sensitive metadata value is prohibited at {path}" + ) + + +def _validate_json_value(value: Any, path: str) -> None: + if value is None or isinstance(value, (str, bool, int)): + return + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError(f"{path} contains a non-finite float") + return + if isinstance(value, list): + for index, item in enumerate(value): + _validate_json_value(item, f"{path}[{index}]") + return + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise TypeError(f"{path} contains a non-string key") + _validate_json_value(item, f"{path}.{key}") + return + raise TypeError(f"{path} contains a non-JSON value") + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_capture_json(payload).encode("utf-8") + ).hexdigest() + return f"{prefix}-{digest}" + + +def _required_text(value: Any) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("text value is required") + return _bounded_text(text) + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return _bounded_text(text) if text else None + + +def _bounded_text(value: Any) -> str: + text = str(value).strip() + if not text or len(text) > MAX_CAPTURE_TEXT: + raise ValueError("capture text must be non-empty and bounded") + return text + + +def _required_canonical_id(value: Any, name: str) -> str: + text = _required_text(value) + if not _CANONICAL_ID_RE.fullmatch(text): + raise ValueError(f"{name} is not a canonical public identifier") + return text + + +def _optional_canonical_id(value: Any, name: str) -> str | None: + text = _optional_text(value) + if text is None: + return None + if not _CANONICAL_ID_RE.fullmatch(text): + raise ValueError(f"{name} is not a canonical public identifier") + return text + + +def _required_prefixed_id(value: Any, prefix: str) -> str: + text = _required_text(value) + if not text.startswith(prefix) or not _SHA256_RE.fullmatch( + text[len(prefix) :] + ): + raise ValueError(f"identifier must use {prefix}sha256 form") + return text + + +def _optional_symbol(value: Any) -> str | None: + text = _optional_text(value) + if text is None: + return None + symbol = text.upper() + if not _SYMBOL_RE.fullmatch(symbol): + raise ValueError("unsupported broker capture symbol") + return symbol + + +def _nonnegative_int64(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value < 0 or value > INT64_MAX: + raise ValueError(f"{name} is outside non-negative int64 range") + return value + + +def _positive_int64(value: Any, name: str) -> int: + number = _nonnegative_int64(value, name) + if number <= 0: + raise ValueError(f"{name} must be positive") + return number + + +def _optional_nonnegative_int64(value: Any, name: str) -> int | None: + return None if value is None else _nonnegative_int64(value, name) + + +def _optional_int64(value: Any, name: str) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value < -(2**63) or value > INT64_MAX: + raise ValueError(f"{name} is outside int64 range") + return value + + +def _optional_nonnegative_float(value: Any, name: str) -> float | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{name} must be numeric") + number = float(value) + if not math.isfinite(number) or number < 0: + raise ValueError(f"{name} must be finite and non-negative") + return number + + +def _optional_price_lexeme(value: Any, name: str) -> str | None: + text = _optional_text(value) + if text is None: + return None + if not _PRICE_LEXEME_RE.fullmatch(text): + raise ValueError(f"{name} must preserve a plain decimal price lexeme") + try: + number = Decimal(text) + except InvalidOperation as err: + raise ValueError(f"{name} is not a decimal price lexeme") from err + if not number.is_finite() or number <= 0: + raise ValueError(f"{name} must represent a finite positive price") + return text + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise TypeError(f"{name} must be a boolean") + return value + + +def _required_sha256(value: Any, name: str) -> str: + text = str(value or "").strip().lower() + if not _SHA256_RE.fullmatch(text): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return text + + +def _optional_sha256(value: Any, name: str) -> str | None: + text = _optional_text(value) + return None if text is None else _required_sha256(text, name) + + +def _relative_artifact_path(value: Any) -> str: + text = _required_text(value).replace("\\", "/") + path = PurePosixPath(text) + if path.is_absolute() or ".." in path.parts: + raise ValueError("capture artifact path must be relative and contained") + return str(path) + + +def _enum_value(enum_type: type[_EnumT], value: Any, label: str) -> _EnumT: + if isinstance(value, enum_type): + return value + try: + return enum_type(str(value).strip().lower()) + except ValueError as err: + raise ValueError(f"unsupported {label}") from err + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError(f"unsupported schema; expected {expected}") + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise TypeError("mapping value is required") + return cast(Mapping[str, Any], value) + + +def _sequence(value: Any) -> Sequence[Any]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise TypeError("sequence value is required") + return value + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + return tuple(_mapping(item) for item in _sequence(value)) + + +def _json_dict(value: Any) -> dict[str, JSONValue]: + mapping = _mapping(value) + return cast(dict[str, JSONValue], dict(mapping)) + + +def _mapping_optional_text(data: Mapping[str, Any], name: str) -> str | None: + return _optional_text(data.get(name)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + return _mapping(value) + + +__all__ = [ + "BROKER_ADAPTER_MESSAGE_SCHEMA_VERSION", + "BROKER_CAPTURE_COLLECTOR_ID", + "BROKER_CAPTURE_COLLECTOR_VERSION", + "BROKER_CAPTURE_DATA_ARTIFACT_KIND", + "BROKER_CAPTURE_EVENT_SCHEMA_VERSION", + "BROKER_CAPTURE_PARTITION_MANIFEST_SCHEMA_VERSION", + "BROKER_CAPTURE_REPLAY_SUMMARY_SCHEMA_VERSION", + "BROKER_CAPTURE_SESSION_MANIFEST_SCHEMA_VERSION", + "BROKER_CAPTURE_SESSION_SCHEMA_VERSION", + "BROKER_CAPTURE_STORAGE_POLICY_SCHEMA_VERSION", + "BrokerAdapterMessageV1", + "BrokerCaptureActivitySemantics", + "BrokerCaptureBackpressureMode", + "BrokerCaptureEventKind", + "BrokerCaptureEventV1", + "BrokerCapturePartitionManifestV1", + "BrokerCapturePriceTextSemantics", + "BrokerCaptureReplaySummaryV1", + "BrokerCaptureRetentionMode", + "BrokerCaptureSessionManifestV1", + "BrokerCaptureSessionState", + "BrokerCaptureSessionV1", + "BrokerCaptureSizeSemantics", + "BrokerCaptureStoragePolicyV1", + "BrokerCaptureSourceTimestampSemantics", + "assert_secret_free_capture_value", + "canonical_capture_json", + "logical_capture_content_sha256", +] diff --git a/src/histdatacom/broker_capture/fingerprint_contracts.py b/src/histdatacom/broker_capture/fingerprint_contracts.py new file mode 100644 index 00000000..ebcaf3af --- /dev/null +++ b/src/histdatacom/broker_capture/fingerprint_contracts.py @@ -0,0 +1,1654 @@ +"""Immutable contracts for broker-delivery fingerprints and drift evidence. + +These profiles describe one broker observation/delivery system over an explicit +support interval. They are compact fitted artifacts, not augmented capture +rows and not claims about the whole market. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, TypeVar, cast + +from histdatacom.broker_capture.contracts import ( + BROKER_CAPTURE_COLLECTOR_VERSION, + canonical_capture_json, +) +from histdatacom.runtime_contracts import JSONValue + +BROKER_DELIVERY_FIT_CONFIG_SCHEMA_VERSION = ( + "histdatacom.broker-delivery-fit-config.v1" +) +BROKER_CAPTURE_ELIGIBILITY_SCHEMA_VERSION = ( + "histdatacom.broker-capture-eligibility.v1" +) +BROKER_DELIVERY_CAPTURE_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.broker-delivery-capture-evidence.v1" +) +BROKER_DELIVERY_CONDITION_SCHEMA_VERSION = ( + "histdatacom.broker-delivery-condition.v1" +) +BROKER_DELIVERY_METRIC_SCHEMA_VERSION = "histdatacom.broker-delivery-metric.v1" +BROKER_DELIVERY_CELL_SCHEMA_VERSION = "histdatacom.broker-delivery-cell.v1" +BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION = ( + "histdatacom.broker-delivery-fingerprint.v1" +) +BROKER_DELIVERY_DRIFT_CONFIG_SCHEMA_VERSION = ( + "histdatacom.broker-delivery-drift-config.v1" +) +BROKER_DELIVERY_METRIC_COMPARISON_SCHEMA_VERSION = ( + "histdatacom.broker-delivery-metric-comparison.v1" +) +BROKER_DELIVERY_FINGERPRINT_COMPARISON_SCHEMA_VERSION = ( + "histdatacom.broker-delivery-fingerprint-comparison.v1" +) +BROKER_DELIVERY_FINGERPRINT_ARTIFACT_KIND = "broker_delivery_fingerprint_json" + +MAX_BROKER_DELIVERY_CAPTURES = 64 +MAX_BROKER_DELIVERY_CELLS = 2_048 +MAX_BROKER_DELIVERY_METRICS_PER_CELL = 256 +MAX_BROKER_DELIVERY_COMPARISONS = 8_192 +MAX_BROKER_DELIVERY_COMPARISON_CANDIDATES = ( + 2 * MAX_BROKER_DELIVERY_CELLS * MAX_BROKER_DELIVERY_METRICS_PER_CELL +) +MAX_BROKER_DELIVERY_REASONS = 128 +MAX_BROKER_DELIVERY_CATEGORIES = 256 +MAX_BROKER_DELIVERY_TEXT = 1_024 +MAX_BROKER_DELIVERY_EVENTS = 100_000_000 +MAX_BROKER_DELIVERY_SAMPLES = 65_536 +MAX_BROKER_DELIVERY_CONTEXT_EVENTS = 16_384 +INT64_MAX = 2**63 - 1 + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$") +_CONDITION_DIMENSIONS = frozenset( + {"symbol", "session", "overlap", "special", "holiday", "event", "lifecycle"} +) +_EnumT = TypeVar("_EnumT", bound=Enum) + + +class BrokerCaptureEligibilityStatus(str, Enum): + """Whether a capture may be used for profile fitting.""" + + ELIGIBLE = "eligible" + LIMITED = "limited" + INELIGIBLE = "ineligible" + + @classmethod + def from_value( + cls, value: str | "BrokerCaptureEligibilityStatus" + ) -> "BrokerCaptureEligibilityStatus": + return _enum_value(cls, value, "capture eligibility status") + + +class BrokerDeliverySupportStatus(str, Enum): + """Support decision for one conditioned fingerprint cell.""" + + SUPPORTED = "supported" + BACKED_OFF = "backed_off" + UNSUPPORTED = "unsupported" + + @classmethod + def from_value( + cls, value: str | "BrokerDeliverySupportStatus" + ) -> "BrokerDeliverySupportStatus": + return _enum_value(cls, value, "broker delivery support status") + + +class BrokerDeliveryDriftStatus(str, Enum): + """Evidence-backed state for one stratified metric comparison.""" + + STABLE = "stable" + SAMPLING_NOISE = "sampling_noise" + MATERIAL_DRIFT = "material_drift" + UNSUPPORTED = "unsupported" + + @classmethod + def from_value( + cls, value: str | "BrokerDeliveryDriftStatus" + ) -> "BrokerDeliveryDriftStatus": + return _enum_value(cls, value, "broker delivery drift status") + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryFitConfigV1: + """Deterministic resource, support, health, and conditioning policy.""" + + min_capture_events: int = 8 + min_quote_events: int = 3 + min_cell_support: int = 4 + max_capture_manifests: int = 16 + max_input_events: int = 10_000_000 + max_cells: int = 512 + max_samples_per_metric: int = 4_096 + max_transition_categories: int = 64 + max_market_context_events: int = 4_096 + max_market_matches_per_quote: int = 8 + burst_interval_ns: int = 100_000_000 + quiet_interval_ns: int = 5_000_000_000 + stale_max_interval_ns: int = 5_000_000_000 + post_lifecycle_quote_count: int = 8 + max_clock_correction_events: int = 32 + max_abs_clock_correction_ns: int = 60_000_000_000 + max_unexplained_wall_regressions: int = 0 + quantiles: tuple[float, ...] = (0.05, 0.25, 0.5, 0.75, 0.95, 0.99) + supported_collector_versions: tuple[str, ...] = ( + BROKER_CAPTURE_COLLECTOR_VERSION, + ) + supported_adapter_ids: tuple[str, ...] = () + fatal_limitation_prefixes: tuple[str, ...] = ( + "collector_failure", + "quota", + "retention", + "backpressure", + ) + rounding_digits: int = 9 + config_id: str = "" + schema_version: str = BROKER_DELIVERY_FIT_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_DELIVERY_FIT_CONFIG_SCHEMA_VERSION: + raise ValueError("unsupported broker delivery fit config schema") + for name, minimum, maximum in ( + ("min_capture_events", 1, MAX_BROKER_DELIVERY_EVENTS), + ("min_quote_events", 1, MAX_BROKER_DELIVERY_EVENTS), + ("min_cell_support", 1, MAX_BROKER_DELIVERY_EVENTS), + ("max_capture_manifests", 1, MAX_BROKER_DELIVERY_CAPTURES), + ("max_input_events", 1, MAX_BROKER_DELIVERY_EVENTS), + ("max_cells", 1, MAX_BROKER_DELIVERY_CELLS), + ("max_samples_per_metric", 2, MAX_BROKER_DELIVERY_SAMPLES), + ("max_transition_categories", 1, MAX_BROKER_DELIVERY_CATEGORIES), + ( + "max_market_context_events", + 1, + MAX_BROKER_DELIVERY_CONTEXT_EVENTS, + ), + ("max_market_matches_per_quote", 1, MAX_BROKER_DELIVERY_CATEGORIES), + ("burst_interval_ns", 1, INT64_MAX), + ("quiet_interval_ns", 1, INT64_MAX), + ("stale_max_interval_ns", 1, INT64_MAX), + ("post_lifecycle_quote_count", 1, 1_000_000), + ("max_clock_correction_events", 0, MAX_BROKER_DELIVERY_EVENTS), + ("max_abs_clock_correction_ns", 1, INT64_MAX), + ("max_unexplained_wall_regressions", 0, MAX_BROKER_DELIVERY_EVENTS), + ): + _bounded_int(getattr(self, name), name, minimum, maximum) + if self.burst_interval_ns >= self.quiet_interval_ns: + raise ValueError( + "burst interval must be smaller than quiet interval" + ) + quantiles = tuple(sorted(set(self.quantiles))) + if not quantiles or any(not 0.0 < value < 1.0 for value in quantiles): + raise ValueError( + "fit quantiles must lie strictly between zero and one" + ) + object.__setattr__(self, "quantiles", quantiles) + for name in ( + "supported_collector_versions", + "supported_adapter_ids", + "fatal_limitation_prefixes", + ): + object.__setattr__( + self, + name, + _bounded_text_tuple( + getattr(self, name), name, allow_empty=True + ), + ) + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between zero and sixteen") + expected = _stable_id( + "broker-delivery-fit-config", self.identity_payload() + ) + supplied = str(self.config_id or "").strip() + if supplied and supplied != expected: + raise ValueError("config_id does not match deterministic identity") + object.__setattr__(self, "config_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "min_capture_events": self.min_capture_events, + "min_quote_events": self.min_quote_events, + "min_cell_support": self.min_cell_support, + "max_capture_manifests": self.max_capture_manifests, + "max_input_events": self.max_input_events, + "max_cells": self.max_cells, + "max_samples_per_metric": self.max_samples_per_metric, + "max_transition_categories": self.max_transition_categories, + "max_market_context_events": self.max_market_context_events, + "max_market_matches_per_quote": self.max_market_matches_per_quote, + "burst_interval_ns": self.burst_interval_ns, + "quiet_interval_ns": self.quiet_interval_ns, + "stale_max_interval_ns": self.stale_max_interval_ns, + "post_lifecycle_quote_count": self.post_lifecycle_quote_count, + "max_clock_correction_events": self.max_clock_correction_events, + "max_abs_clock_correction_ns": self.max_abs_clock_correction_ns, + "max_unexplained_wall_regressions": ( + self.max_unexplained_wall_regressions + ), + "quantiles": list(self.quantiles), + "supported_collector_versions": list( + self.supported_collector_versions + ), + "supported_adapter_ids": list(self.supported_adapter_ids), + "fatal_limitation_prefixes": list(self.fatal_limitation_prefixes), + "rounding_digits": self.rounding_digits, + "sampling_policy": "deterministic-bottom-hash", + "capture_passes": ["health_and_integrity", "bounded_fit"], + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "config_id": self.config_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerDeliveryFitConfigV1": + _require_schema(data, BROKER_DELIVERY_FIT_CONFIG_SCHEMA_VERSION) + return cls( + min_capture_events=_strict_int(data.get("min_capture_events")), + min_quote_events=_strict_int(data.get("min_quote_events")), + min_cell_support=_strict_int(data.get("min_cell_support")), + max_capture_manifests=_strict_int( + data.get("max_capture_manifests") + ), + max_input_events=_strict_int(data.get("max_input_events")), + max_cells=_strict_int(data.get("max_cells")), + max_samples_per_metric=_strict_int( + data.get("max_samples_per_metric") + ), + max_transition_categories=_strict_int( + data.get("max_transition_categories") + ), + max_market_context_events=_strict_int( + data.get("max_market_context_events") + ), + max_market_matches_per_quote=_strict_int( + data.get("max_market_matches_per_quote") + ), + burst_interval_ns=_strict_int(data.get("burst_interval_ns")), + quiet_interval_ns=_strict_int(data.get("quiet_interval_ns")), + stale_max_interval_ns=_strict_int( + data.get("stale_max_interval_ns") + ), + post_lifecycle_quote_count=_strict_int( + data.get("post_lifecycle_quote_count") + ), + max_clock_correction_events=_strict_int( + data.get("max_clock_correction_events") + ), + max_abs_clock_correction_ns=_strict_int( + data.get("max_abs_clock_correction_ns") + ), + max_unexplained_wall_regressions=_strict_int( + data.get("max_unexplained_wall_regressions") + ), + quantiles=tuple( + _finite_float(value) + for value in _sequence(data.get("quantiles")) + ), + supported_collector_versions=_string_tuple( + data.get("supported_collector_versions") + ), + supported_adapter_ids=_string_tuple( + data.get("supported_adapter_ids") + ), + fatal_limitation_prefixes=_string_tuple( + data.get("fatal_limitation_prefixes") + ), + rounding_digits=_strict_int(data.get("rounding_digits")), + config_id=str(data.get("config_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureEligibilityV1: + """Immutable health and integrity decision for one capture manifest.""" + + session_id: str + manifest_id: str + config_id: str + status: BrokerCaptureEligibilityStatus + reason_codes: tuple[str, ...] + manifest_complete: bool + inspection_clean: bool + integrity_verified: bool + event_count: int + quote_count: int + clock_correction_count: int + max_abs_clock_correction_ns: int + explained_wall_regression_count: int + unexplained_wall_regression_count: int + first_receive_time_utc_ns: int | None + last_receive_time_utc_ns: int | None + logical_content_sha256: str | None + decision_id: str = "" + schema_version: str = BROKER_CAPTURE_ELIGIBILITY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_CAPTURE_ELIGIBILITY_SCHEMA_VERSION: + raise ValueError("unsupported broker capture eligibility schema") + for name in ("session_id", "manifest_id", "config_id"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "status", + BrokerCaptureEligibilityStatus.from_value(self.status), + ) + reasons = _bounded_text_tuple( + self.reason_codes, "eligibility reason", allow_empty=True + ) + if len(reasons) > MAX_BROKER_DELIVERY_REASONS: + raise ValueError("too many eligibility reasons") + object.__setattr__(self, "reason_codes", reasons) + for name in ( + "event_count", + "quote_count", + "clock_correction_count", + "explained_wall_regression_count", + "unexplained_wall_regression_count", + ): + _bounded_int( + getattr(self, name), name, 0, MAX_BROKER_DELIVERY_EVENTS + ) + _bounded_int( + self.max_abs_clock_correction_ns, + "max_abs_clock_correction_ns", + 0, + INT64_MAX, + ) + for name in ("first_receive_time_utc_ns", "last_receive_time_utc_ns"): + value = getattr(self, name) + if value is not None: + _bounded_int(value, name, 0, INT64_MAX) + if ( + self.first_receive_time_utc_ns is not None + and self.last_receive_time_utc_ns is not None + and self.last_receive_time_utc_ns < self.first_receive_time_utc_ns + and not self.clock_correction_count + ): + raise ValueError( + "reversed wall-clock support requires correction evidence" + ) + digest = _optional_sha256(self.logical_content_sha256) + object.__setattr__(self, "logical_content_sha256", digest) + if self.integrity_verified and digest is None: + raise ValueError( + "verified eligibility requires a logical content hash" + ) + if self.status is BrokerCaptureEligibilityStatus.ELIGIBLE and reasons: + raise ValueError("eligible capture cannot carry reason codes") + if ( + self.status is BrokerCaptureEligibilityStatus.INELIGIBLE + and not reasons + ): + raise ValueError("ineligible capture requires reason codes") + expected = _stable_id( + "broker-capture-eligibility", self.identity_payload() + ) + supplied = str(self.decision_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "decision_id does not match deterministic identity" + ) + object.__setattr__(self, "decision_id", expected) + + @property + def fit_allowed(self) -> bool: + return self.status is not BrokerCaptureEligibilityStatus.INELIGIBLE + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "session_id": self.session_id, + "manifest_id": self.manifest_id, + "config_id": self.config_id, + "status": self.status.value, + "reason_codes": list(self.reason_codes), + "manifest_complete": self.manifest_complete, + "inspection_clean": self.inspection_clean, + "integrity_verified": self.integrity_verified, + "event_count": self.event_count, + "quote_count": self.quote_count, + "clock_correction_count": self.clock_correction_count, + "max_abs_clock_correction_ns": self.max_abs_clock_correction_ns, + "explained_wall_regression_count": ( + self.explained_wall_regression_count + ), + "unexplained_wall_regression_count": ( + self.unexplained_wall_regression_count + ), + "first_receive_time_utc_ns": self.first_receive_time_utc_ns, + "last_receive_time_utc_ns": self.last_receive_time_utc_ns, + "logical_content_sha256": self.logical_content_sha256, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "decision_id": self.decision_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerCaptureEligibilityV1": + _require_schema(data, BROKER_CAPTURE_ELIGIBILITY_SCHEMA_VERSION) + return cls( + session_id=str(data.get("session_id", "")), + manifest_id=str(data.get("manifest_id", "")), + config_id=str(data.get("config_id", "")), + status=BrokerCaptureEligibilityStatus.from_value( + str(data.get("status", "")) + ), + reason_codes=_string_tuple(data.get("reason_codes")), + manifest_complete=_strict_bool(data.get("manifest_complete")), + inspection_clean=_strict_bool(data.get("inspection_clean")), + integrity_verified=_strict_bool(data.get("integrity_verified")), + event_count=_strict_int(data.get("event_count")), + quote_count=_strict_int(data.get("quote_count")), + clock_correction_count=_strict_int( + data.get("clock_correction_count") + ), + max_abs_clock_correction_ns=_strict_int( + data.get("max_abs_clock_correction_ns") + ), + explained_wall_regression_count=_strict_int( + data.get("explained_wall_regression_count") + ), + unexplained_wall_regression_count=_strict_int( + data.get("unexplained_wall_regression_count") + ), + first_receive_time_utc_ns=_optional_int( + data.get("first_receive_time_utc_ns") + ), + last_receive_time_utc_ns=_optional_int( + data.get("last_receive_time_utc_ns") + ), + logical_content_sha256=_optional_text( + data.get("logical_content_sha256") + ), + decision_id=str(data.get("decision_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryCaptureEvidenceV1: + """Compact strong lineage for one qualified capture.""" + + session_id: str + manifest_id: str + eligibility_decision_id: str + logical_content_sha256: str + partition_hashes_sha256: str + partition_count: int + event_count: int + first_receive_time_utc_ns: int + last_receive_time_utc_ns: int + evidence_id: str = "" + schema_version: str = BROKER_DELIVERY_CAPTURE_EVIDENCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != BROKER_DELIVERY_CAPTURE_EVIDENCE_SCHEMA_VERSION + ): + raise ValueError( + "unsupported broker delivery capture evidence schema" + ) + for name in ("session_id", "manifest_id", "eligibility_decision_id"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + for name in ("logical_content_sha256", "partition_hashes_sha256"): + object.__setattr__( + self, name, _required_sha256(getattr(self, name)) + ) + _bounded_int(self.partition_count, "partition_count", 1, INT64_MAX) + _bounded_int( + self.event_count, "event_count", 1, MAX_BROKER_DELIVERY_EVENTS + ) + start = _bounded_int( + self.first_receive_time_utc_ns, + "first_receive_time_utc_ns", + 0, + INT64_MAX, + ) + end = _bounded_int( + self.last_receive_time_utc_ns, + "last_receive_time_utc_ns", + 0, + INT64_MAX, + ) + if end < start: + raise ValueError("capture evidence wall-clock support is reversed") + expected = _stable_id( + "broker-delivery-capture-evidence", self.identity_payload() + ) + supplied = str(self.evidence_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "evidence_id does not match deterministic identity" + ) + object.__setattr__(self, "evidence_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "session_id": self.session_id, + "manifest_id": self.manifest_id, + "eligibility_decision_id": self.eligibility_decision_id, + "logical_content_sha256": self.logical_content_sha256, + "partition_hashes_sha256": self.partition_hashes_sha256, + "partition_count": self.partition_count, + "event_count": self.event_count, + "first_receive_time_utc_ns": self.first_receive_time_utc_ns, + "last_receive_time_utc_ns": self.last_receive_time_utc_ns, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerDeliveryCaptureEvidenceV1": + _require_schema(data, BROKER_DELIVERY_CAPTURE_EVIDENCE_SCHEMA_VERSION) + return cls( + session_id=str(data.get("session_id", "")), + manifest_id=str(data.get("manifest_id", "")), + eligibility_decision_id=str( + data.get("eligibility_decision_id", "") + ), + logical_content_sha256=str(data.get("logical_content_sha256", "")), + partition_hashes_sha256=str( + data.get("partition_hashes_sha256", "") + ), + partition_count=_strict_int(data.get("partition_count")), + event_count=_strict_int(data.get("event_count")), + first_receive_time_utc_ns=_strict_int( + data.get("first_receive_time_utc_ns") + ), + last_receive_time_utc_ns=_strict_int( + data.get("last_receive_time_utc_ns") + ), + evidence_id=str(data.get("evidence_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryConditionV1: + """Canonical dimensions defining one conditioned delivery cell.""" + + dimensions: dict[str, str] + condition_id: str = "" + schema_version: str = BROKER_DELIVERY_CONDITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_DELIVERY_CONDITION_SCHEMA_VERSION: + raise ValueError("unsupported broker delivery condition schema") + normalized: dict[str, str] = {} + for raw_name, raw_value in sorted(self.dimensions.items()): + name = _required_name(raw_name) + if name not in _CONDITION_DIMENSIONS: + raise ValueError( + f"unsupported broker delivery dimension: {name}" + ) + value = _required_name(raw_value) + normalized[name] = ( + value.upper() if name == "symbol" else value.lower() + ) + if len(normalized) > len(_CONDITION_DIMENSIONS): + raise ValueError( + "broker delivery condition has too many dimensions" + ) + object.__setattr__(self, "dimensions", normalized) + expected = _stable_id( + "broker-delivery-condition", self.identity_payload() + ) + supplied = str(self.condition_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "condition_id does not match deterministic identity" + ) + object.__setattr__(self, "condition_id", expected) + + @property + def key(self) -> str: + if not self.dimensions: + return "global" + return "|".join( + f"{name}={value}" for name, value in self.dimensions.items() + ) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "dimensions": dict(self.dimensions), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "condition_id": self.condition_id, + "key": self.key, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerDeliveryConditionV1": + _require_schema(data, BROKER_DELIVERY_CONDITION_SCHEMA_VERSION) + condition = cls( + dimensions={ + str(name): str(value) + for name, value in _mapping(data.get("dimensions")).items() + }, + condition_id=str(data.get("condition_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + supplied_key = str(data.get("key", "")) + if supplied_key and supplied_key != condition.key: + raise ValueError("condition key does not match dimensions") + return condition + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryMetricV1: + """One bounded estimate with sample support and uncertainty interval.""" + + name: str + kind: str + unit: str + support_count: int + sample_count: int + estimate: float | None + lower: float | None + upper: float | None + minimum: float | None = None + maximum: float | None = None + quantiles: dict[str, float] = field(default_factory=dict) + category_counts: dict[str, int] = field(default_factory=dict) + limitations: tuple[str, ...] = () + metric_id: str = "" + schema_version: str = BROKER_DELIVERY_METRIC_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_DELIVERY_METRIC_SCHEMA_VERSION: + raise ValueError("unsupported broker delivery metric schema") + for name in ("name", "kind", "unit"): + object.__setattr__(self, name, _required_name(getattr(self, name))) + support = _bounded_int( + self.support_count, "support_count", 0, MAX_BROKER_DELIVERY_EVENTS + ) + sample = _bounded_int( + self.sample_count, "sample_count", 0, MAX_BROKER_DELIVERY_SAMPLES + ) + if sample > support: + raise ValueError("metric sample_count exceeds support_count") + estimate = _optional_finite_float(self.estimate) + lower = _optional_finite_float(self.lower) + upper = _optional_finite_float(self.upper) + if estimate is None: + if lower is not None or upper is not None: + raise ValueError( + "unavailable metric cannot have uncertainty bounds" + ) + elif lower is None or upper is None or not lower <= estimate <= upper: + raise ValueError( + "metric estimate must lie inside uncertainty bounds" + ) + minimum = _optional_finite_float(self.minimum) + maximum = _optional_finite_float(self.maximum) + if (minimum is None) != (maximum is None): + raise ValueError("metric extrema must be supplied together") + if minimum is not None and maximum is not None and maximum < minimum: + raise ValueError("metric maximum precedes minimum") + quantiles = { + _required_name(str(name)): _finite_float(value) + for name, value in sorted(self.quantiles.items()) + } + categories = { + _required_name(str(name)): _bounded_int( + value, f"category {name}", 0, MAX_BROKER_DELIVERY_EVENTS + ) + for name, value in sorted(self.category_counts.items()) + } + if len(categories) > MAX_BROKER_DELIVERY_CATEGORIES: + raise ValueError("metric has too many categories") + limitations = _bounded_text_tuple( + self.limitations, "metric limitation", allow_empty=True + ) + if estimate is None and not limitations: + raise ValueError("unavailable metric requires a limitation") + object.__setattr__(self, "support_count", support) + object.__setattr__(self, "sample_count", sample) + object.__setattr__(self, "estimate", estimate) + object.__setattr__(self, "lower", lower) + object.__setattr__(self, "upper", upper) + object.__setattr__(self, "minimum", minimum) + object.__setattr__(self, "maximum", maximum) + object.__setattr__(self, "quantiles", quantiles) + object.__setattr__(self, "category_counts", categories) + object.__setattr__(self, "limitations", limitations) + expected = _stable_id("broker-delivery-metric", self.identity_payload()) + supplied = str(self.metric_id or "").strip() + if supplied and supplied != expected: + raise ValueError("metric_id does not match deterministic identity") + object.__setattr__(self, "metric_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "name": self.name, + "kind": self.kind, + "unit": self.unit, + "support_count": self.support_count, + "sample_count": self.sample_count, + "estimate": self.estimate, + "lower": self.lower, + "upper": self.upper, + "minimum": self.minimum, + "maximum": self.maximum, + "quantiles": dict(self.quantiles), + "category_counts": dict(self.category_counts), + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "metric_id": self.metric_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerDeliveryMetricV1": + _require_schema(data, BROKER_DELIVERY_METRIC_SCHEMA_VERSION) + return cls( + name=str(data.get("name", "")), + kind=str(data.get("kind", "")), + unit=str(data.get("unit", "")), + support_count=_strict_int(data.get("support_count")), + sample_count=_strict_int(data.get("sample_count")), + estimate=_optional_float(data.get("estimate")), + lower=_optional_float(data.get("lower")), + upper=_optional_float(data.get("upper")), + minimum=_optional_float(data.get("minimum")), + maximum=_optional_float(data.get("maximum")), + quantiles={ + str(name): _finite_float(value) + for name, value in _mapping(data.get("quantiles")).items() + }, + category_counts={ + str(name): _strict_int(value) + for name, value in _mapping(data.get("category_counts")).items() + }, + limitations=_string_tuple(data.get("limitations")), + metric_id=str(data.get("metric_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryCellV1: + """One conditioned profile cell and its explicit backoff decision.""" + + condition: BrokerDeliveryConditionV1 + support_count: int + support_status: BrokerDeliverySupportStatus + backoff_condition_ids: tuple[str, ...] + effective_condition_id: str | None + metrics: tuple[BrokerDeliveryMetricV1, ...] + limitations: tuple[str, ...] = () + cell_id: str = "" + schema_version: str = BROKER_DELIVERY_CELL_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_DELIVERY_CELL_SCHEMA_VERSION: + raise ValueError("unsupported broker delivery cell schema") + if not isinstance(self.condition, BrokerDeliveryConditionV1): + raise TypeError("condition must be BrokerDeliveryConditionV1") + support = _bounded_int( + self.support_count, "support_count", 0, MAX_BROKER_DELIVERY_EVENTS + ) + status = BrokerDeliverySupportStatus.from_value(self.support_status) + backoffs = _bounded_ordered_text_tuple( + self.backoff_condition_ids, "backoff condition", allow_empty=True + ) + if len(set(backoffs)) != len(backoffs): + raise ValueError("cell has duplicate backoff conditions") + effective = _optional_text(self.effective_condition_id) + if status is BrokerDeliverySupportStatus.SUPPORTED: + if effective != self.condition.condition_id: + raise ValueError("supported cell must select itself") + elif status is BrokerDeliverySupportStatus.BACKED_OFF: + if effective is None or effective not in backoffs: + raise ValueError( + "backed-off cell must select a declared parent" + ) + elif effective is not None: + raise ValueError("unsupported cell cannot select an effective cell") + metrics = tuple(sorted(self.metrics, key=lambda item: item.name)) + if len(metrics) > MAX_BROKER_DELIVERY_METRICS_PER_CELL: + raise ValueError("broker delivery cell has too many metrics") + if len({item.name for item in metrics}) != len(metrics): + raise ValueError("broker delivery cell has duplicate metrics") + limitations = _bounded_text_tuple( + self.limitations, "cell limitation", allow_empty=True + ) + object.__setattr__(self, "support_count", support) + object.__setattr__(self, "support_status", status) + object.__setattr__(self, "backoff_condition_ids", backoffs) + object.__setattr__(self, "effective_condition_id", effective) + object.__setattr__(self, "metrics", metrics) + object.__setattr__(self, "limitations", limitations) + expected = _stable_id("broker-delivery-cell", self.identity_payload()) + supplied = str(self.cell_id or "").strip() + if supplied and supplied != expected: + raise ValueError("cell_id does not match deterministic identity") + object.__setattr__(self, "cell_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "condition": self.condition.to_dict(), + "support_count": self.support_count, + "support_status": self.support_status.value, + "backoff_condition_ids": list(self.backoff_condition_ids), + "effective_condition_id": self.effective_condition_id, + "metrics": [item.to_dict() for item in self.metrics], + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "cell_id": self.cell_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerDeliveryCellV1": + _require_schema(data, BROKER_DELIVERY_CELL_SCHEMA_VERSION) + return cls( + condition=BrokerDeliveryConditionV1.from_dict( + _mapping(data.get("condition")) + ), + support_count=_strict_int(data.get("support_count")), + support_status=BrokerDeliverySupportStatus.from_value( + str(data.get("support_status", "")) + ), + backoff_condition_ids=_string_tuple( + data.get("backoff_condition_ids") + ), + effective_condition_id=_optional_text( + data.get("effective_condition_id") + ), + metrics=tuple( + BrokerDeliveryMetricV1.from_dict(item) + for item in _mapping_sequence(data.get("metrics")) + ), + limitations=_string_tuple(data.get("limitations")), + cell_id=str(data.get("cell_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryFingerprintV1: + """Immutable support-aware broker delivery profile.""" + + adapter_id: str + adapter_version: str + adapter_config_sha256: str + protocol: str + environment_id: str + server_id: str + account_id_sha256: str | None + collector_id: str + collector_version: str + fit_config: BrokerDeliveryFitConfigV1 + capture_evidence: tuple[BrokerDeliveryCaptureEvidenceV1, ...] + eligibility_decisions: tuple[BrokerCaptureEligibilityV1, ...] + support_start_utc_ns: int + support_end_utc_ns: int + effective_start_utc_ns: int + effective_end_utc_ns: int | None + cells: tuple[BrokerDeliveryCellV1, ...] + supersedes_fingerprint_id: str | None = None + limitations: tuple[str, ...] = () + fingerprint_id: str = "" + schema_version: str = BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION: + raise ValueError("unsupported broker delivery fingerprint schema") + for name in ( + "adapter_id", + "adapter_version", + "protocol", + "environment_id", + "server_id", + "collector_id", + "collector_version", + ): + object.__setattr__(self, name, _required_name(getattr(self, name))) + object.__setattr__( + self, + "adapter_config_sha256", + _required_sha256(self.adapter_config_sha256), + ) + object.__setattr__( + self, "account_id_sha256", _optional_sha256(self.account_id_sha256) + ) + if not isinstance(self.fit_config, BrokerDeliveryFitConfigV1): + raise TypeError("fit_config must be BrokerDeliveryFitConfigV1") + evidence = tuple( + sorted(self.capture_evidence, key=lambda item: item.session_id) + ) + decisions = tuple( + sorted(self.eligibility_decisions, key=lambda item: item.session_id) + ) + if ( + not evidence + or len(evidence) > self.fit_config.max_capture_manifests + ): + raise ValueError( + "fingerprint capture evidence is empty or unbounded" + ) + if len(evidence) != len(decisions): + raise ValueError("capture evidence and eligibility counts differ") + if {item.session_id for item in evidence} != { + item.session_id for item in decisions + }: + raise ValueError("capture evidence and eligibility sessions differ") + if any(not item.fit_allowed for item in decisions): + raise ValueError("fingerprint includes an ineligible capture") + object.__setattr__(self, "capture_evidence", evidence) + object.__setattr__(self, "eligibility_decisions", decisions) + start = _bounded_int( + self.support_start_utc_ns, "support_start_utc_ns", 0, INT64_MAX + ) + end = _bounded_int( + self.support_end_utc_ns, "support_end_utc_ns", 0, INT64_MAX + ) + effective_start = _bounded_int( + self.effective_start_utc_ns, "effective_start_utc_ns", 0, INT64_MAX + ) + effective_end = self.effective_end_utc_ns + if end < start: + raise ValueError("fingerprint support interval is reversed") + if effective_end is not None: + _bounded_int(effective_end, "effective_end_utc_ns", 0, INT64_MAX) + if effective_end <= effective_start: + raise ValueError("fingerprint effective interval is empty") + cells = tuple(sorted(self.cells, key=lambda item: item.condition.key)) + if not cells or len(cells) > self.fit_config.max_cells: + raise ValueError("fingerprint cells are empty or unbounded") + if len({item.condition.condition_id for item in cells}) != len(cells): + raise ValueError("fingerprint has duplicate conditions") + global_cells = [item for item in cells if not item.condition.dimensions] + if len(global_cells) != 1: + raise ValueError("fingerprint requires exactly one global cell") + cell_ids = {item.condition.condition_id for item in cells} + for cell in cells: + if any( + parent not in cell_ids for parent in cell.backoff_condition_ids + ): + raise ValueError( + "fingerprint cell references an absent backoff cell" + ) + supersedes = _optional_text(self.supersedes_fingerprint_id) + limitations = _bounded_text_tuple( + self.limitations, "fingerprint limitation", allow_empty=True + ) + object.__setattr__(self, "support_start_utc_ns", start) + object.__setattr__(self, "support_end_utc_ns", end) + object.__setattr__(self, "effective_start_utc_ns", effective_start) + object.__setattr__(self, "effective_end_utc_ns", effective_end) + object.__setattr__(self, "cells", cells) + object.__setattr__(self, "supersedes_fingerprint_id", supersedes) + object.__setattr__(self, "limitations", limitations) + expected = _stable_id( + "broker-delivery-fingerprint", self.identity_payload() + ) + supplied = str(self.fingerprint_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "fingerprint_id does not match deterministic identity" + ) + if supersedes == expected: + raise ValueError("fingerprint cannot supersede itself") + object.__setattr__(self, "fingerprint_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "adapter_id": self.adapter_id, + "adapter_version": self.adapter_version, + "adapter_config_sha256": self.adapter_config_sha256, + "protocol": self.protocol, + "environment_id": self.environment_id, + "server_id": self.server_id, + "account_id_sha256": self.account_id_sha256, + "collector_id": self.collector_id, + "collector_version": self.collector_version, + "fit_config": self.fit_config.to_dict(), + "capture_evidence": [ + item.to_dict() for item in self.capture_evidence + ], + "eligibility_decisions": [ + item.to_dict() for item in self.eligibility_decisions + ], + "support_start_utc_ns": self.support_start_utc_ns, + "support_end_utc_ns": self.support_end_utc_ns, + "effective_start_utc_ns": self.effective_start_utc_ns, + "effective_end_utc_ns": self.effective_end_utc_ns, + "cells": [item.to_dict() for item in self.cells], + "supersedes_fingerprint_id": self.supersedes_fingerprint_id, + "limitations": list(self.limitations), + "profile_claim": "broker_observation_delivery_system_only", + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "fingerprint_id": self.fingerprint_id, + } + + def to_json(self) -> str: + return str(canonical_capture_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerDeliveryFingerprintV1": + _require_schema(data, BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION) + fingerprint = cls( + adapter_id=str(data.get("adapter_id", "")), + adapter_version=str(data.get("adapter_version", "")), + adapter_config_sha256=str(data.get("adapter_config_sha256", "")), + protocol=str(data.get("protocol", "")), + environment_id=str(data.get("environment_id", "")), + server_id=str(data.get("server_id", "")), + account_id_sha256=_optional_text(data.get("account_id_sha256")), + collector_id=str(data.get("collector_id", "")), + collector_version=str(data.get("collector_version", "")), + fit_config=BrokerDeliveryFitConfigV1.from_dict( + _mapping(data.get("fit_config")) + ), + capture_evidence=tuple( + BrokerDeliveryCaptureEvidenceV1.from_dict(item) + for item in _mapping_sequence(data.get("capture_evidence")) + ), + eligibility_decisions=tuple( + BrokerCaptureEligibilityV1.from_dict(item) + for item in _mapping_sequence(data.get("eligibility_decisions")) + ), + support_start_utc_ns=_strict_int(data.get("support_start_utc_ns")), + support_end_utc_ns=_strict_int(data.get("support_end_utc_ns")), + effective_start_utc_ns=_strict_int( + data.get("effective_start_utc_ns") + ), + effective_end_utc_ns=_optional_int( + data.get("effective_end_utc_ns") + ), + cells=tuple( + BrokerDeliveryCellV1.from_dict(item) + for item in _mapping_sequence(data.get("cells")) + ), + supersedes_fingerprint_id=_optional_text( + data.get("supersedes_fingerprint_id") + ), + limitations=_string_tuple(data.get("limitations")), + fingerprint_id=str(data.get("fingerprint_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + claim = str(data.get("profile_claim", "")) + if claim and claim != "broker_observation_delivery_system_only": + raise ValueError("unsupported broker delivery profile claim") + return fingerprint + + @classmethod + def from_json(cls, text: str) -> "BrokerDeliveryFingerprintV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryDriftConfigV1: + """Bounded statistical-effect policy for stratified profile comparison.""" + + min_metric_support: int = 4 + relative_material_threshold: float = 0.25 + default_absolute_material_threshold: float = 0.0 + absolute_material_thresholds: dict[str, float] = field( + default_factory=lambda: { + "burst_interval_rate": 0.05, + "exact_duplicate_rate": 0.05, + "price_decimal_places": 0.5, + "quiet_interval_rate": 0.05, + "source_timestamp_precision_ns": 1.0, + "spread": 0.00001, + "stale_quote_rate": 0.05, + "transition_rate": 0.05, + } + ) + max_comparisons: int = 2_048 + rounding_digits: int = 9 + config_id: str = "" + schema_version: str = BROKER_DELIVERY_DRIFT_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_DELIVERY_DRIFT_CONFIG_SCHEMA_VERSION: + raise ValueError("unsupported broker delivery drift config schema") + _bounded_int( + self.min_metric_support, + "min_metric_support", + 1, + MAX_BROKER_DELIVERY_EVENTS, + ) + relative = _finite_float(self.relative_material_threshold) + default_absolute = _finite_float( + self.default_absolute_material_threshold + ) + if relative < 0 or default_absolute < 0: + raise ValueError("drift thresholds must be non-negative") + thresholds = { + _required_name(str(name)): _finite_float(value) + for name, value in sorted(self.absolute_material_thresholds.items()) + } + if any(value < 0 for value in thresholds.values()): + raise ValueError("absolute drift thresholds must be non-negative") + _bounded_int( + self.max_comparisons, + "max_comparisons", + 1, + MAX_BROKER_DELIVERY_COMPARISONS, + ) + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between zero and sixteen") + object.__setattr__(self, "relative_material_threshold", relative) + object.__setattr__( + self, "default_absolute_material_threshold", default_absolute + ) + object.__setattr__(self, "absolute_material_thresholds", thresholds) + expected = _stable_id( + "broker-delivery-drift-config", self.identity_payload() + ) + supplied = str(self.config_id or "").strip() + if supplied and supplied != expected: + raise ValueError("config_id does not match deterministic identity") + object.__setattr__(self, "config_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "min_metric_support": self.min_metric_support, + "relative_material_threshold": self.relative_material_threshold, + "default_absolute_material_threshold": ( + self.default_absolute_material_threshold + ), + "absolute_material_thresholds": dict( + self.absolute_material_thresholds + ), + "max_comparisons": self.max_comparisons, + "rounding_digits": self.rounding_digits, + "comparison_policy": "cell_and_metric_stratified_no_similarity_score", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "config_id": self.config_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerDeliveryDriftConfigV1": + _require_schema(data, BROKER_DELIVERY_DRIFT_CONFIG_SCHEMA_VERSION) + return cls( + min_metric_support=_strict_int(data.get("min_metric_support")), + relative_material_threshold=_finite_float( + data.get("relative_material_threshold") + ), + default_absolute_material_threshold=_finite_float( + data.get("default_absolute_material_threshold") + ), + absolute_material_thresholds={ + str(name): _finite_float(value) + for name, value in _mapping( + data.get("absolute_material_thresholds") + ).items() + }, + max_comparisons=_strict_int(data.get("max_comparisons")), + rounding_digits=_strict_int(data.get("rounding_digits")), + config_id=str(data.get("config_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryMetricComparisonV1: + """One support-aware metric comparison in one explicit condition.""" + + condition_id: str + condition_key: str + metric_name: str + reference_support_count: int + candidate_support_count: int + reference_estimate: float | None + candidate_estimate: float | None + absolute_difference: float | None + relative_difference: float | None + combined_uncertainty: float | None + status: BrokerDeliveryDriftStatus + reason_codes: tuple[str, ...] + comparison_id: str = "" + schema_version: str = BROKER_DELIVERY_METRIC_COMPARISON_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != BROKER_DELIVERY_METRIC_COMPARISON_SCHEMA_VERSION + ): + raise ValueError( + "unsupported broker delivery metric comparison schema" + ) + for name in ("condition_id", "condition_key", "metric_name"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + for name in ("reference_support_count", "candidate_support_count"): + _bounded_int( + getattr(self, name), name, 0, MAX_BROKER_DELIVERY_EVENTS + ) + for name in ( + "reference_estimate", + "candidate_estimate", + "absolute_difference", + "relative_difference", + "combined_uncertainty", + ): + object.__setattr__( + self, name, _optional_finite_float(getattr(self, name)) + ) + status = BrokerDeliveryDriftStatus.from_value(self.status) + reasons = _bounded_text_tuple( + self.reason_codes, "comparison reason", allow_empty=True + ) + if status is not BrokerDeliveryDriftStatus.STABLE and not reasons: + raise ValueError("non-stable comparison requires a reason") + object.__setattr__(self, "status", status) + object.__setattr__(self, "reason_codes", reasons) + expected = _stable_id( + "broker-delivery-metric-comparison", self.identity_payload() + ) + supplied = str(self.comparison_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "comparison_id does not match deterministic identity" + ) + object.__setattr__(self, "comparison_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "condition_id": self.condition_id, + "condition_key": self.condition_key, + "metric_name": self.metric_name, + "reference_support_count": self.reference_support_count, + "candidate_support_count": self.candidate_support_count, + "reference_estimate": self.reference_estimate, + "candidate_estimate": self.candidate_estimate, + "absolute_difference": self.absolute_difference, + "relative_difference": self.relative_difference, + "combined_uncertainty": self.combined_uncertainty, + "status": self.status.value, + "reason_codes": list(self.reason_codes), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "comparison_id": self.comparison_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerDeliveryMetricComparisonV1": + _require_schema(data, BROKER_DELIVERY_METRIC_COMPARISON_SCHEMA_VERSION) + return cls( + condition_id=str(data.get("condition_id", "")), + condition_key=str(data.get("condition_key", "")), + metric_name=str(data.get("metric_name", "")), + reference_support_count=_strict_int( + data.get("reference_support_count") + ), + candidate_support_count=_strict_int( + data.get("candidate_support_count") + ), + reference_estimate=_optional_float(data.get("reference_estimate")), + candidate_estimate=_optional_float(data.get("candidate_estimate")), + absolute_difference=_optional_float( + data.get("absolute_difference") + ), + relative_difference=_optional_float( + data.get("relative_difference") + ), + combined_uncertainty=_optional_float( + data.get("combined_uncertainty") + ), + status=BrokerDeliveryDriftStatus.from_value( + str(data.get("status", "")) + ), + reason_codes=_string_tuple(data.get("reason_codes")), + comparison_id=str(data.get("comparison_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerDeliveryFingerprintComparisonV1: + """Bounded stratified drift evidence with no aggregate similarity score.""" + + reference_fingerprint_id: str + candidate_fingerprint_id: str + drift_config: BrokerDeliveryDriftConfigV1 + comparison_candidate_count: int + comparisons: tuple[BrokerDeliveryMetricComparisonV1, ...] + status_counts: dict[str, int] + truncated: bool + comparison_id: str = "" + schema_version: str = BROKER_DELIVERY_FINGERPRINT_COMPARISON_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != BROKER_DELIVERY_FINGERPRINT_COMPARISON_SCHEMA_VERSION + ): + raise ValueError( + "unsupported broker delivery fingerprint comparison schema" + ) + for name in ("reference_fingerprint_id", "candidate_fingerprint_id"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if self.reference_fingerprint_id == self.candidate_fingerprint_id: + raise ValueError( + "fingerprint comparison requires distinct profiles" + ) + if not isinstance(self.drift_config, BrokerDeliveryDriftConfigV1): + raise TypeError("drift_config must be BrokerDeliveryDriftConfigV1") + candidate_count = _bounded_int( + self.comparison_candidate_count, + "comparison_candidate_count", + 0, + MAX_BROKER_DELIVERY_COMPARISON_CANDIDATES, + ) + comparisons = tuple(self.comparisons) + if len(comparisons) > self.drift_config.max_comparisons: + raise ValueError("fingerprint comparison exceeds configured bound") + if len({item.comparison_id for item in comparisons}) != len( + comparisons + ): + raise ValueError("fingerprint comparison has duplicate rows") + counts = { + str(name): _bounded_int(value, str(name), 0, candidate_count) + for name, value in sorted(self.status_counts.items()) + } + observed: dict[str, int] = {} + for item in comparisons: + observed[item.status.value] = observed.get(item.status.value, 0) + 1 + if counts != dict(sorted(observed.items())): + raise ValueError( + "fingerprint comparison status counts do not reconcile" + ) + if self.truncated != (candidate_count > len(comparisons)): + raise ValueError( + "fingerprint comparison truncation flag is inconsistent" + ) + object.__setattr__(self, "comparison_candidate_count", candidate_count) + object.__setattr__(self, "comparisons", comparisons) + object.__setattr__(self, "status_counts", counts) + expected = _stable_id( + "broker-delivery-fingerprint-comparison", self.identity_payload() + ) + supplied = str(self.comparison_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "comparison_id does not match deterministic identity" + ) + object.__setattr__(self, "comparison_id", expected) + + @property + def material_drift_count(self) -> int: + return self.status_counts.get( + BrokerDeliveryDriftStatus.MATERIAL_DRIFT.value, 0 + ) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "reference_fingerprint_id": self.reference_fingerprint_id, + "candidate_fingerprint_id": self.candidate_fingerprint_id, + "drift_config": self.drift_config.to_dict(), + "comparison_candidate_count": self.comparison_candidate_count, + "comparisons": [item.to_dict() for item in self.comparisons], + "status_counts": dict(self.status_counts), + "truncated": self.truncated, + "global_similarity_score": None, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "comparison_id": self.comparison_id} + + def to_json(self) -> str: + return str(canonical_capture_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerDeliveryFingerprintComparisonV1": + _require_schema( + data, BROKER_DELIVERY_FINGERPRINT_COMPARISON_SCHEMA_VERSION + ) + if data.get("global_similarity_score") is not None: + raise ValueError("global similarity scores are not supported") + return cls( + reference_fingerprint_id=str( + data.get("reference_fingerprint_id", "") + ), + candidate_fingerprint_id=str( + data.get("candidate_fingerprint_id", "") + ), + drift_config=BrokerDeliveryDriftConfigV1.from_dict( + _mapping(data.get("drift_config")) + ), + comparison_candidate_count=_strict_int( + data.get("comparison_candidate_count") + ), + comparisons=tuple( + BrokerDeliveryMetricComparisonV1.from_dict(item) + for item in _mapping_sequence(data.get("comparisons")) + ), + status_counts={ + str(name): _strict_int(value) + for name, value in _mapping(data.get("status_counts")).items() + }, + truncated=_strict_bool(data.get("truncated")), + comparison_id=str(data.get("comparison_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerDeliveryFingerprintComparisonV1": + return cls.from_dict(_json_mapping(text)) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_capture_json(payload).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _enum_value( + enum_type: type[_EnumT], value: str | _EnumT, name: str +) -> _EnumT: + if isinstance(value, enum_type): + return value + try: + return enum_type(str(value).strip().lower()) + except ValueError as err: + raise ValueError(f"unsupported {name}") from err + + +def _required_text(value: object) -> str: + if not isinstance(value, str): + raise TypeError("text value must be a string") + text = value.strip() + if not text or len(text) > MAX_BROKER_DELIVERY_TEXT: + raise ValueError("text value is empty or unbounded") + return text + + +def _required_name(value: object) -> str: + text = _required_text(value) + if not _NAME_RE.fullmatch(text): + raise ValueError("name contains unsupported characters") + return text + + +def _optional_text(value: object) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise TypeError("optional text value must be a string") + text = value.strip() + return _required_text(text) if text else None + + +def _required_sha256(value: object) -> str: + text = str(value or "").strip().lower() + if not _SHA256_RE.fullmatch(text): + raise ValueError("value must be a lowercase SHA-256 digest") + return text + + +def _optional_sha256(value: object) -> str | None: + text = _optional_text(value) + return None if text is None else _required_sha256(text) + + +def _bounded_int(value: object, name: str, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if not minimum <= value <= maximum: + raise ValueError(f"{name} is outside configured bounds") + return value + + +def _strict_int(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError("value must be an integer") + return value + + +def _strict_bool(value: object) -> bool: + if not isinstance(value, bool): + raise TypeError("value must be a boolean") + return value + + +def _optional_int(value: object) -> int | None: + return None if value is None else _strict_int(value) + + +def _finite_float(value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError("value must be numeric") + result = float(value) + if not math.isfinite(result): + raise ValueError("value must be finite") + return result + + +def _optional_finite_float(value: object) -> float | None: + return None if value is None else _finite_float(value) + + +def _optional_float(value: object) -> float | None: + return _optional_finite_float(value) + + +def _bounded_text_tuple( + values: Sequence[object], + name: str, + *, + allow_empty: bool, +) -> tuple[str, ...]: + normalized = tuple(sorted({_required_text(value) for value in values})) + if not allow_empty and not normalized: + raise ValueError(f"{name} values are empty") + if len(normalized) > MAX_BROKER_DELIVERY_REASONS: + raise ValueError(f"{name} values are unbounded") + return normalized + + +def _bounded_ordered_text_tuple( + values: Sequence[object], + name: str, + *, + allow_empty: bool, +) -> tuple[str, ...]: + normalized = tuple(_required_text(value) for value in values) + if not allow_empty and not normalized: + raise ValueError(f"{name} values are empty") + if len(normalized) > MAX_BROKER_DELIVERY_REASONS: + raise ValueError(f"{name} values are unbounded") + return normalized + + +def _mapping(value: object) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise TypeError("value must be a mapping") + return cast(Mapping[str, Any], value) + + +def _sequence(value: object) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise TypeError("value must be a sequence") + return value + + +def _mapping_sequence(value: object) -> tuple[Mapping[str, Any], ...]: + return tuple(_mapping(item) for item in _sequence(value)) + + +def _string_tuple(value: object) -> tuple[str, ...]: + values = _sequence(value) + if any(not isinstance(item, str) for item in values): + raise TypeError("sequence values must be strings") + return tuple(values) + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError(f"unsupported schema version; expected {expected}") + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + return _mapping(value) + + +__all__ = [ + "BROKER_CAPTURE_ELIGIBILITY_SCHEMA_VERSION", + "BROKER_DELIVERY_CAPTURE_EVIDENCE_SCHEMA_VERSION", + "BROKER_DELIVERY_CELL_SCHEMA_VERSION", + "BROKER_DELIVERY_CONDITION_SCHEMA_VERSION", + "BROKER_DELIVERY_DRIFT_CONFIG_SCHEMA_VERSION", + "BROKER_DELIVERY_FINGERPRINT_ARTIFACT_KIND", + "BROKER_DELIVERY_FINGERPRINT_COMPARISON_SCHEMA_VERSION", + "BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION", + "BROKER_DELIVERY_FIT_CONFIG_SCHEMA_VERSION", + "BROKER_DELIVERY_METRIC_COMPARISON_SCHEMA_VERSION", + "BROKER_DELIVERY_METRIC_SCHEMA_VERSION", + "BrokerCaptureEligibilityStatus", + "BrokerCaptureEligibilityV1", + "BrokerDeliveryCaptureEvidenceV1", + "BrokerDeliveryCellV1", + "BrokerDeliveryConditionV1", + "BrokerDeliveryDriftConfigV1", + "BrokerDeliveryDriftStatus", + "BrokerDeliveryFingerprintComparisonV1", + "BrokerDeliveryFingerprintV1", + "BrokerDeliveryFitConfigV1", + "BrokerDeliveryMetricComparisonV1", + "BrokerDeliveryMetricV1", + "BrokerDeliverySupportStatus", +] diff --git a/src/histdatacom/broker_capture/fingerprints.py b/src/histdatacom/broker_capture/fingerprints.py new file mode 100644 index 00000000..66687980 --- /dev/null +++ b/src/histdatacom/broker_capture/fingerprints.py @@ -0,0 +1,1425 @@ +"""Streaming fit, drift comparison, and persistence for broker fingerprints.""" + +from __future__ import annotations + +import hashlib +import heapq +import math +import os +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +from histdatacom.broker_capture.contracts import ( + BrokerCaptureEventKind, + BrokerCaptureEventV1, + BrokerCaptureSessionManifestV1, + canonical_capture_json, +) +from histdatacom.broker_capture.fingerprint_contracts import ( + BROKER_DELIVERY_FINGERPRINT_ARTIFACT_KIND, + BrokerCaptureEligibilityStatus, + BrokerCaptureEligibilityV1, + BrokerDeliveryCaptureEvidenceV1, + BrokerDeliveryCellV1, + BrokerDeliveryConditionV1, + BrokerDeliveryDriftConfigV1, + BrokerDeliveryDriftStatus, + BrokerDeliveryFingerprintComparisonV1, + BrokerDeliveryFingerprintV1, + BrokerDeliveryFitConfigV1, + BrokerDeliveryMetricComparisonV1, + BrokerDeliveryMetricV1, + BrokerDeliverySupportStatus, +) +from histdatacom.broker_capture.storage import ( + BrokerCaptureStorageError, + inspect_broker_capture_session, + replay_broker_capture_session, +) +from histdatacom.data_quality.calendar_profiles import HistDataCalendarProfile +from histdatacom.market_context.contracts import ( + MarketContextEventV1, + MarketContextTimelineV1, + market_context_calendar_state, +) +from histdatacom.runtime_contracts import ArtifactRef, JSONValue + +_TAG_RE = re.compile(r"[^A-Za-z0-9._:-]+") +_NANOSECONDS_PER_SECOND = 1_000_000_000 +_RATE_EVENT_KINDS = ( + BrokerCaptureEventKind.QUOTE, + BrokerCaptureEventKind.RECONNECT, + BrokerCaptureEventKind.GAP, + BrokerCaptureEventKind.OUTAGE_START, + BrokerCaptureEventKind.OUTAGE_END, + BrokerCaptureEventKind.PROCESS_RESTART, + BrokerCaptureEventKind.CLOCK_CORRECTION, +) +_RATE_TRANSITIONS = ( + (BrokerCaptureEventKind.QUOTE, BrokerCaptureEventKind.QUOTE), + (BrokerCaptureEventKind.RECONNECT, BrokerCaptureEventKind.QUOTE), + (BrokerCaptureEventKind.OUTAGE_END, BrokerCaptureEventKind.QUOTE), + (BrokerCaptureEventKind.PROCESS_RESTART, BrokerCaptureEventKind.QUOTE), +) + + +class BrokerDeliveryFingerprintError(RuntimeError): + """Base class for fail-closed broker fingerprint operations.""" + + +class BrokerDeliveryIneligibleCaptureError(BrokerDeliveryFingerprintError): + """At least one capture failed the declared fitting health gate.""" + + def __init__(self, eligibility: BrokerCaptureEligibilityV1) -> None: + self.eligibility = eligibility + super().__init__( + f"capture {eligibility.session_id} is ineligible: " + + ", ".join(eligibility.reason_codes) + ) + + +class BrokerDeliveryResourceLimitError(BrokerDeliveryFingerprintError): + """A configured resource limit was reached without truncating input.""" + + +class BrokerDeliveryFingerprintIdentityError(BrokerDeliveryFingerprintError): + """Capture or predecessor identities are incompatible.""" + + +class BrokerDeliveryFingerprintArtifactError(BrokerDeliveryFingerprintError): + """A fingerprint artifact failed immutable persistence verification.""" + + +@dataclass(slots=True) +class _HealthConsumer: + event_count: int = 0 + quote_count: int = 0 + clock_correction_count: int = 0 + max_abs_clock_correction_ns: int = 0 + explained_wall_regression_count: int = 0 + unexplained_wall_regression_count: int = 0 + first_receive_time_utc_ns: int | None = None + last_receive_time_utc_ns: int | None = None + _previous_wall_ns: int | None = None + + def on_event(self, event: BrokerCaptureEventV1) -> None: + self.event_count += 1 + if event.kind is BrokerCaptureEventKind.QUOTE: + self.quote_count += 1 + if event.kind is BrokerCaptureEventKind.CLOCK_CORRECTION: + self.clock_correction_count += 1 + self.max_abs_clock_correction_ns = max( + self.max_abs_clock_correction_ns, + abs(event.clock_offset_change_ns or 0), + ) + if ( + self._previous_wall_ns is not None + and event.receive_time_utc_ns < self._previous_wall_ns + ): + if event.kind is BrokerCaptureEventKind.CLOCK_CORRECTION: + self.explained_wall_regression_count += 1 + else: + self.unexplained_wall_regression_count += 1 + self._previous_wall_ns = event.receive_time_utc_ns + if self.first_receive_time_utc_ns is None: + self.first_receive_time_utc_ns = event.receive_time_utc_ns + self.last_receive_time_utc_ns = event.receive_time_utc_ns + else: + self.first_receive_time_utc_ns = min( + self.first_receive_time_utc_ns, event.receive_time_utc_ns + ) + assert self.last_receive_time_utc_ns is not None + self.last_receive_time_utc_ns = max( + self.last_receive_time_utc_ns, event.receive_time_utc_ns + ) + + +def assess_broker_capture_eligibility( + root: str | Path, + manifest: BrokerCaptureSessionManifestV1, + *, + config: BrokerDeliveryFitConfigV1 | None = None, +) -> BrokerCaptureEligibilityV1: + """Verify one capture and return a deterministic fit-health decision.""" + policy = config or BrokerDeliveryFitConfigV1() + hard_reasons: set[str] = set() + limited_reasons: set[str] = set() + inspection_clean = False + integrity_verified = False + logical_digest: str | None = None + health = _HealthConsumer() + + if not manifest.complete: + hard_reasons.add("capture_not_completed") + if ( + manifest.session.collector_version + not in policy.supported_collector_versions + ): + hard_reasons.add("unsupported_collector_version") + if ( + policy.supported_adapter_ids + and manifest.session.adapter_id not in policy.supported_adapter_ids + ): + hard_reasons.add("unsupported_adapter_id") + for limitation in manifest.limitations: + if any( + limitation.startswith(prefix) + for prefix in policy.fatal_limitation_prefixes + ): + hard_reasons.add("fatal_capture_limitation") + else: + limited_reasons.add("capture_limitation_present") + + try: + inspection = inspect_broker_capture_session( + root, manifest.session.session_id + ) + inspection_clean = inspection.clean + if not inspection_clean: + hard_reasons.add("capture_inspection_not_clean") + summary = replay_broker_capture_session( + root, manifest, consumers=(health,) + ) + integrity_verified = True + logical_digest = summary.logical_content_sha256 + except (BrokerCaptureStorageError, OSError, TypeError, ValueError): + hard_reasons.add("integrity_verification_failed") + + if health.event_count < policy.min_capture_events: + hard_reasons.add("insufficient_capture_events") + if health.quote_count < policy.min_quote_events: + hard_reasons.add("insufficient_quote_events") + if health.clock_correction_count > policy.max_clock_correction_events: + hard_reasons.add("excessive_clock_corrections") + elif health.clock_correction_count: + limited_reasons.add("clock_corrections_present") + if health.max_abs_clock_correction_ns > policy.max_abs_clock_correction_ns: + hard_reasons.add("excessive_clock_correction_magnitude") + if ( + health.unexplained_wall_regression_count + > policy.max_unexplained_wall_regressions + ): + hard_reasons.add("unexplained_wall_clock_regression") + + if hard_reasons: + status = BrokerCaptureEligibilityStatus.INELIGIBLE + reasons = tuple(sorted(hard_reasons | limited_reasons)) + elif limited_reasons: + status = BrokerCaptureEligibilityStatus.LIMITED + reasons = tuple(sorted(limited_reasons)) + else: + status = BrokerCaptureEligibilityStatus.ELIGIBLE + reasons = () + return BrokerCaptureEligibilityV1( + session_id=manifest.session.session_id, + manifest_id=manifest.manifest_id, + config_id=policy.config_id, + status=status, + reason_codes=reasons, + manifest_complete=manifest.complete, + inspection_clean=inspection_clean, + integrity_verified=integrity_verified, + event_count=health.event_count, + quote_count=health.quote_count, + clock_correction_count=health.clock_correction_count, + max_abs_clock_correction_ns=health.max_abs_clock_correction_ns, + explained_wall_regression_count=( + health.explained_wall_regression_count + ), + unexplained_wall_regression_count=( + health.unexplained_wall_regression_count + ), + first_receive_time_utc_ns=health.first_receive_time_utc_ns, + last_receive_time_utc_ns=health.last_receive_time_utc_ns, + logical_content_sha256=logical_digest, + ) + + +@dataclass(slots=True) +class _SampleAccumulator: + name: str + kind: str + unit: str + sample_limit: int + rounding_digits: int + support_count: int = 0 + total: float = 0.0 + total_squares: float = 0.0 + minimum: float | None = None + maximum: float | None = None + _samples: list[tuple[int, str, float]] = field(default_factory=list) + + def add(self, value: float, evidence_key: str) -> None: + if not math.isfinite(value): + return + self.support_count += 1 + self.total += value + self.total_squares += value * value + self.minimum = ( + value if self.minimum is None else min(self.minimum, value) + ) + self.maximum = ( + value if self.maximum is None else max(self.maximum, value) + ) + score = int.from_bytes( + hashlib.sha256( + f"{self.name}\0{evidence_key}".encode("utf-8") + ).digest(), + "big", + ) + row = (-score, evidence_key, value) + if len(self._samples) < self.sample_limit: + heapq.heappush(self._samples, row) + elif score < -self._samples[0][0]: + heapq.heapreplace(self._samples, row) + + def to_metric(self, quantiles: Sequence[float]) -> BrokerDeliveryMetricV1: + if not self.support_count: + return BrokerDeliveryMetricV1( + name=self.name, + kind=self.kind, + unit=self.unit, + support_count=0, + sample_count=0, + estimate=None, + lower=None, + upper=None, + limitations=("metric_has_no_observations",), + ) + estimate = self.total / self.support_count + if self.kind == "rate": + lower, upper = _wilson_interval(estimate, self.support_count) + elif self.support_count == 1: + lower = upper = estimate + else: + variance = max( + 0.0, + ( + self.total_squares + - self.total * self.total / self.support_count + ) + / (self.support_count - 1), + ) + half_width = 1.96 * math.sqrt(variance / self.support_count) + lower, upper = estimate - half_width, estimate + half_width + sample_values = sorted(row[2] for row in self._samples) + limitations = ( + ("deterministic_bottom_hash_sample",) + if len(sample_values) < self.support_count + else () + ) + return BrokerDeliveryMetricV1( + name=self.name, + kind=self.kind, + unit=self.unit, + support_count=self.support_count, + sample_count=len(sample_values), + estimate=_rounded(estimate, self.rounding_digits), + lower=_rounded(lower, self.rounding_digits), + upper=_rounded(upper, self.rounding_digits), + minimum=_rounded(self.minimum, self.rounding_digits), + maximum=_rounded(self.maximum, self.rounding_digits), + quantiles={ + _quantile_name(q): _rounded_required( + _quantile(sample_values, q), self.rounding_digits + ) + for q in quantiles + }, + limitations=limitations, + ) + + +@dataclass(slots=True) +class _CellAccumulator: + condition: BrokerDeliveryConditionV1 + sample_limit: int + rounding_digits: int + support_count: int = 0 + metrics: dict[str, _SampleAccumulator] = field(default_factory=dict) + active_runs: dict[str, int] = field(default_factory=dict) + + def observe_quote(self) -> None: + self.support_count += 1 + + def add( + self, + name: str, + value: float, + evidence_key: str, + *, + kind: str = "distribution", + unit: str = "ratio", + ) -> None: + metric = self.metrics.get(name) + if metric is None: + metric = _SampleAccumulator( + name=name, + kind=kind, + unit=unit, + sample_limit=self.sample_limit, + rounding_digits=self.rounding_digits, + ) + self.metrics[name] = metric + metric.add(value, evidence_key) + + def observe_run( + self, + name: str, + active: bool, + evidence_key: str, + ) -> None: + count = self.active_runs.get(name, 0) + if active: + self.active_runs[name] = count + 1 + elif count: + self.add( + f"{name}_run_length", + float(count), + evidence_key, + unit="intervals", + ) + self.active_runs[name] = 0 + + def flush_runs(self, evidence_key: str) -> None: + for name, count in sorted(self.active_runs.items()): + if count: + self.add( + f"{name}_run_length", + float(count), + evidence_key, + unit="intervals", + ) + self.active_runs.clear() + + +@dataclass(frozen=True, slots=True) +class _PreviousQuote: + monotonic_ns: int + bid: float + ask: float + spread: float + message_id: str + + +class _FingerprintConsumer: + def __init__( + self, + config: BrokerDeliveryFitConfigV1, + *, + calendar_profile: HistDataCalendarProfile | None, + context_events: Sequence[MarketContextEventV1], + ) -> None: + self.config = config + self.calendar_profile = calendar_profile + self.context_events = context_events + self.event_count = 0 + self.quote_count = 0 + self.cells: dict[str, _CellAccumulator] = {} + self._previous_event_monotonic_ns: int | None = None + self._previous_event_kind: BrokerCaptureEventKind | None = None + self._previous_quotes: dict[str, _PreviousQuote] = {} + self._batch_id: str | None = None + self._batch_run_count = 0 + self._lifecycle: str | None = None + self._lifecycle_quotes_remaining = 0 + self._session_id = "" + self._session_first_monotonic_ns: int | None = None + self._session_last_monotonic_ns: int | None = None + self._session_event_count = 0 + self._session_quote_count = 0 + self.calendar_profile_complete = True + self._ensure_cell({}) + + def start_session(self, session_id: str) -> None: + self._previous_event_monotonic_ns = None + self._previous_event_kind = None + self._previous_quotes.clear() + self._batch_id = None + self._batch_run_count = 0 + self._lifecycle = None + self._lifecycle_quotes_remaining = 0 + self._session_id = session_id + self._session_first_monotonic_ns = None + self._session_last_monotonic_ns = None + self._session_event_count = 0 + self._session_quote_count = 0 + + def end_session(self) -> None: + self._flush_batch_run(f"{self._session_id}:capture-end") + for cell in self.cells.values(): + cell.flush_runs(f"{self._session_id}:capture-end") + start = self._session_first_monotonic_ns + end = self._session_last_monotonic_ns + if start is not None and end is not None and end > start: + duration_seconds = (end - start) / _NANOSECONDS_PER_SECOND + global_cell = self._ensure_cell({}) + global_cell.add( + "event_intensity_hz", + self._session_event_count / duration_seconds, + self._session_id, + unit="events_per_second", + ) + global_cell.add( + "quote_intensity_hz", + self._session_quote_count / duration_seconds, + self._session_id, + unit="quotes_per_second", + ) + + def on_event(self, event: BrokerCaptureEventV1) -> None: + self.event_count += 1 + self._session_event_count += 1 + if self._session_first_monotonic_ns is None: + self._session_first_monotonic_ns = event.receive_time_monotonic_ns + self._session_last_monotonic_ns = event.receive_time_monotonic_ns + if self.event_count > self.config.max_input_events: + raise BrokerDeliveryResourceLimitError( + "broker capture events exceed max_input_events" + ) + global_cell = self._ensure_cell({}) + if self._previous_event_monotonic_ns is not None: + global_cell.add( + "event_interarrival_ns", + float( + event.receive_time_monotonic_ns + - self._previous_event_monotonic_ns + ), + event.event_id, + unit="ns", + ) + if self._previous_event_kind is not None: + for previous_kind, current_kind in _RATE_TRANSITIONS[ + : self.config.max_transition_categories + ]: + global_cell.add( + f"event_transition.{previous_kind.value}_to_{current_kind.value}_rate", + float( + self._previous_event_kind is previous_kind + and event.kind is current_kind + ), + event.event_id, + kind="rate", + ) + self._previous_event_monotonic_ns = event.receive_time_monotonic_ns + self._previous_event_kind = event.kind + for rate_kind in _RATE_EVENT_KINDS: + global_cell.add( + f"event_kind.{rate_kind.value}_rate", + float(event.kind is rate_kind), + event.event_id, + kind="rate", + ) + if event.message.gap_duration_ns is not None: + global_cell.add( + "outage_or_gap_duration_ns", + float(event.message.gap_duration_ns), + event.event_id, + unit="ns", + ) + if event.clock_offset_change_ns is not None: + global_cell.add( + "clock_correction_abs_ns", + float(abs(event.clock_offset_change_ns)), + event.event_id, + unit="ns", + ) + if event.kind in { + BrokerCaptureEventKind.RECONNECT, + BrokerCaptureEventKind.OUTAGE_END, + BrokerCaptureEventKind.PROCESS_RESTART, + }: + self._lifecycle = { + BrokerCaptureEventKind.RECONNECT: "post_reconnect", + BrokerCaptureEventKind.OUTAGE_END: "post_outage", + BrokerCaptureEventKind.PROCESS_RESTART: "post_restart", + }[event.kind] + self._lifecycle_quotes_remaining = ( + self.config.post_lifecycle_quote_count + ) + if event.kind is BrokerCaptureEventKind.QUOTE: + self._on_quote(event) + + def finish(self) -> None: + self.end_session() + + def _on_quote(self, event: BrokerCaptureEventV1) -> None: + message = event.message + assert message.symbol is not None + assert message.bid is not None + assert message.ask is not None + self.quote_count += 1 + self._session_quote_count += 1 + conditions = self._quote_conditions(event) + cells = [self._ensure_cell(dimensions) for dimensions in conditions] + for cell in cells: + cell.observe_quote() + spread = message.ask - message.bid + previous = self._previous_quotes.get(message.symbol) + for cell in cells: + cell.add("spread", spread, event.event_id, unit="price") + if message.source_timestamp_precision_ns is not None: + cell.add( + "source_timestamp_precision_ns", + float(message.source_timestamp_precision_ns), + event.event_id, + unit="ns", + ) + decimals = _price_decimal_places(message.bid_text, message.ask_text) + if decimals is not None: + cell.add( + "price_decimal_places", + float(decimals), + event.event_id, + unit="decimal_places", + ) + cell.add( + "price_trailing_zero_rate", + float( + _has_trailing_zero(message.bid_text, message.ask_text) + ), + event.event_id, + kind="rate", + ) + if previous is not None: + interval = event.receive_time_monotonic_ns - previous.monotonic_ns + changed = message.bid != previous.bid or message.ask != previous.ask + burst = interval <= self.config.burst_interval_ns + quiet = interval >= self.config.quiet_interval_ns + stale = ( + not changed and interval <= self.config.stale_max_interval_ns + ) + values = ( + ( + "quote_interarrival_ns", + float(interval), + "distribution", + "ns", + ), + ( + "burst_interval_rate", + float(burst), + "rate", + "ratio", + ), + ( + "quiet_interval_rate", + float(quiet), + "rate", + "ratio", + ), + ( + "stale_quote_rate", + float(stale), + "rate", + "ratio", + ), + ("transition_rate", float(changed), "rate", "ratio"), + ( + "exact_duplicate_rate", + float(message.message_id == previous.message_id), + "rate", + "ratio", + ), + ( + "spread_change", + spread - previous.spread, + "distribution", + "price", + ), + ( + "absolute_spread_change", + abs(spread - previous.spread), + "distribution", + "price", + ), + ) + for cell in cells: + for name, value, kind, unit in values: + cell.add( + name, + value, + event.event_id, + kind=kind, + unit=unit, + ) + if not quiet: + cell.add( + "active_quote_interarrival_ns", + float(interval), + event.event_id, + unit="ns", + ) + cell.observe_run("burst_interval", burst, event.event_id) + cell.observe_run("quiet_interval", quiet, event.event_id) + cell.observe_run("stale_quote", stale, event.event_id) + self._previous_quotes[message.symbol] = _PreviousQuote( + monotonic_ns=event.receive_time_monotonic_ns, + bid=message.bid, + ask=message.ask, + spread=spread, + message_id=message.message_id, + ) + self._observe_batch(message.source_batch_id, event.event_id) + if self._lifecycle_quotes_remaining: + self._lifecycle_quotes_remaining -= 1 + if not self._lifecycle_quotes_remaining: + self._lifecycle = None + + def _quote_conditions( + self, event: BrokerCaptureEventV1 + ) -> tuple[dict[str, str], ...]: + symbol = event.message.symbol + assert symbol is not None + state = market_context_calendar_state( + event.receive_time_utc_ns, + calendar_profile=self.calendar_profile, + ) + self.calendar_profile_complete = ( + self.calendar_profile_complete and state.profile_complete + ) + dimensions: list[dict[str, str]] = [{}, {"symbol": symbol}] + sessions = state.active_sessions or (state.session_state,) + for dimension, tags in ( + ("session", sessions), + ("overlap", state.overlaps), + ("special", state.special_tags), + ("holiday", state.holiday_tags), + ("event", state.event_tags), + ): + for tag in tags: + self._append_condition_pair(dimensions, symbol, dimension, tag) + context_match_count = 0 + for context_event in self.context_events: + if not _context_event_matches( + context_event, event.receive_time_utc_ns, symbol + ): + continue + tags = (f"market_{context_event.kind.value}", *context_event.tags) + for tag in tags: + context_match_count += 1 + if ( + context_match_count + > self.config.max_market_matches_per_quote + ): + raise BrokerDeliveryResourceLimitError( + "market context matches exceed max_market_matches_per_quote" + ) + self._append_condition_pair(dimensions, symbol, "event", tag) + if self._lifecycle is not None and self._lifecycle_quotes_remaining: + self._append_condition_pair( + dimensions, symbol, "lifecycle", self._lifecycle + ) + canonical = { + BrokerDeliveryConditionV1(item).key: item for item in dimensions + } + return tuple(canonical[key] for key in sorted(canonical)) + + def _append_condition_pair( + self, + conditions: list[dict[str, str]], + symbol: str, + dimension: str, + tag: str, + ) -> None: + value = _safe_tag(tag) + conditions.append({dimension: value}) + conditions.append({"symbol": symbol, dimension: value}) + + def _ensure_cell(self, dimensions: Mapping[str, str]) -> _CellAccumulator: + condition = BrokerDeliveryConditionV1(dict(dimensions)) + existing = self.cells.get(condition.condition_id) + if existing is not None: + return existing + if len(self.cells) >= self.config.max_cells: + raise BrokerDeliveryResourceLimitError( + "condition cells exceed max_cells" + ) + cell = _CellAccumulator( + condition=condition, + sample_limit=self.config.max_samples_per_metric, + rounding_digits=self.config.rounding_digits, + ) + self.cells[condition.condition_id] = cell + return cell + + def _observe_batch(self, batch_id: str | None, evidence_key: str) -> None: + if batch_id == self._batch_id: + self._batch_run_count += 1 + return + self._flush_batch_run(evidence_key) + self._batch_id = batch_id + self._batch_run_count = 1 if batch_id is not None else 0 + + def _flush_batch_run(self, evidence_key: str) -> None: + if self._batch_id is not None and self._batch_run_count: + self._ensure_cell({}).add( + "source_batch_quote_count", + float(self._batch_run_count), + evidence_key, + unit="quotes", + ) + self._batch_id = None + self._batch_run_count = 0 + + +def fit_broker_delivery_fingerprint( + root: str | Path, + manifests: Sequence[BrokerCaptureSessionManifestV1], + *, + config: BrokerDeliveryFitConfigV1 | None = None, + calendar_profile: HistDataCalendarProfile | None = None, + market_context_timeline: MarketContextTimelineV1 | None = None, + supersedes: BrokerDeliveryFingerprintV1 | None = None, + effective_start_utc_ns: int | None = None, + effective_end_utc_ns: int | None = None, +) -> BrokerDeliveryFingerprintV1: + """Fit one compact immutable profile with two verified streaming passes.""" + policy = config or BrokerDeliveryFitConfigV1() + ordered = tuple(sorted(manifests, key=lambda item: item.session.session_id)) + if not ordered: + raise ValueError("at least one capture manifest is required") + if len(ordered) > policy.max_capture_manifests: + raise BrokerDeliveryResourceLimitError( + "capture manifests exceed max_capture_manifests" + ) + if len({item.session.session_id for item in ordered}) != len(ordered): + raise ValueError("capture manifests contain duplicate sessions") + _assert_compatible_capture_identity(ordered) + context_events = _bounded_context_events(market_context_timeline, policy) + decisions: list[BrokerCaptureEligibilityV1] = [] + evidence: list[BrokerDeliveryCaptureEvidenceV1] = [] + for manifest in ordered: + decision = assess_broker_capture_eligibility( + root, manifest, config=policy + ) + if not decision.fit_allowed: + raise BrokerDeliveryIneligibleCaptureError(decision) + assert decision.logical_content_sha256 is not None + assert decision.first_receive_time_utc_ns is not None + assert decision.last_receive_time_utc_ns is not None + decisions.append(decision) + evidence.append( + BrokerDeliveryCaptureEvidenceV1( + session_id=manifest.session.session_id, + manifest_id=manifest.manifest_id, + eligibility_decision_id=decision.decision_id, + logical_content_sha256=decision.logical_content_sha256, + partition_hashes_sha256=_partition_hashes_digest(manifest), + partition_count=len(manifest.partitions), + event_count=decision.event_count, + first_receive_time_utc_ns=decision.first_receive_time_utc_ns, + last_receive_time_utc_ns=decision.last_receive_time_utc_ns, + ) + ) + total_events = sum(item.event_count for item in decisions) + if total_events > policy.max_input_events: + raise BrokerDeliveryResourceLimitError( + "qualified captures exceed max_input_events" + ) + + consumer = _FingerprintConsumer( + policy, + calendar_profile=calendar_profile, + context_events=context_events, + ) + evidence_by_session = {item.session_id: item for item in evidence} + for manifest in ordered: + consumer.start_session(manifest.session.session_id) + summary = replay_broker_capture_session( + root, manifest, consumers=(consumer,) + ) + consumer.end_session() + expected = evidence_by_session[manifest.session.session_id] + if summary.logical_content_sha256 != expected.logical_content_sha256: + raise BrokerDeliveryFingerprintArtifactError( + "capture content changed between eligibility and fitting passes" + ) + cells = _finalize_cells(consumer.cells, policy) + support_start = min(item.first_receive_time_utc_ns for item in evidence) + support_end = max(item.last_receive_time_utc_ns for item in evidence) + start = ( + support_start + if effective_start_utc_ns is None + else effective_start_utc_ns + ) + if supersedes is not None: + _assert_compatible_predecessor(ordered[0], supersedes, start) + limitations = _fit_limitations( + decisions, + cells, + timeline=market_context_timeline, + total_events=total_events, + policy=policy, + support_start_utc_ns=support_start, + support_end_utc_ns=support_end, + calendar_profile_complete=consumer.calendar_profile_complete, + ) + identity = ordered[0].session + return BrokerDeliveryFingerprintV1( + adapter_id=identity.adapter_id, + adapter_version=identity.adapter_version, + adapter_config_sha256=identity.adapter_config_sha256, + protocol=identity.protocol, + environment_id=identity.environment_id, + server_id=identity.server_id, + account_id_sha256=identity.account_id_sha256, + collector_id=identity.collector_id, + collector_version=identity.collector_version, + fit_config=policy, + capture_evidence=tuple(evidence), + eligibility_decisions=tuple(decisions), + support_start_utc_ns=support_start, + support_end_utc_ns=support_end, + effective_start_utc_ns=start, + effective_end_utc_ns=effective_end_utc_ns, + cells=cells, + supersedes_fingerprint_id=( + None if supersedes is None else supersedes.fingerprint_id + ), + limitations=limitations, + ) + + +def compare_broker_delivery_fingerprints( + reference: BrokerDeliveryFingerprintV1, + candidate: BrokerDeliveryFingerprintV1, + *, + config: BrokerDeliveryDriftConfigV1 | None = None, +) -> BrokerDeliveryFingerprintComparisonV1: + """Compare matching conditioned metrics without an aggregate score.""" + if reference.fingerprint_id == candidate.fingerprint_id: + raise ValueError("drift comparison requires distinct fingerprints") + policy = config or BrokerDeliveryDriftConfigV1() + reference_cells = {item.condition.key: item for item in reference.cells} + candidate_cells = {item.condition.key: item for item in candidate.cells} + retained_by_status: dict[ + BrokerDeliveryDriftStatus, list[BrokerDeliveryMetricComparisonV1] + ] = {status: [] for status in BrokerDeliveryDriftStatus} + candidate_count = 0 + for condition_key in sorted(set(reference_cells) | set(candidate_cells)): + reference_cell = reference_cells.get(condition_key) + candidate_cell = candidate_cells.get(condition_key) + reference_metrics = _metrics_by_name(reference_cell) + candidate_metrics = _metrics_by_name(candidate_cell) + for name in sorted(set(reference_metrics) | set(candidate_metrics)): + comparison = _compare_metric( + condition_key, + reference_cell, + candidate_cell, + reference_metrics.get(name), + candidate_metrics.get(name), + policy, + ) + candidate_count += 1 + bucket = retained_by_status[comparison.status] + if len(bucket) < policy.max_comparisons: + bucket.append(comparison) + priority = ( + BrokerDeliveryDriftStatus.MATERIAL_DRIFT, + BrokerDeliveryDriftStatus.SAMPLING_NOISE, + BrokerDeliveryDriftStatus.UNSUPPORTED, + BrokerDeliveryDriftStatus.STABLE, + ) + retained = tuple( + item for status in priority for item in retained_by_status[status] + )[: policy.max_comparisons] + counts: dict[str, int] = {} + for item in retained: + counts[item.status.value] = counts.get(item.status.value, 0) + 1 + return BrokerDeliveryFingerprintComparisonV1( + reference_fingerprint_id=reference.fingerprint_id, + candidate_fingerprint_id=candidate.fingerprint_id, + drift_config=policy, + comparison_candidate_count=candidate_count, + comparisons=retained, + status_counts=counts, + truncated=candidate_count > len(retained), + ) + + +def write_broker_delivery_fingerprint( + path: str | Path, + fingerprint: BrokerDeliveryFingerprintV1, +) -> ArtifactRef: + """Atomically publish an immutable fingerprint or verify idempotence.""" + target = Path(path) + payload = fingerprint.to_json() + "\n" + encoded = payload.encode("utf-8") + digest = hashlib.sha256(encoded).hexdigest() + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + try: + existing = target.read_bytes() + except OSError as err: + raise BrokerDeliveryFingerprintArtifactError( + "could not read existing fingerprint artifact" + ) from err + if existing != encoded: + raise BrokerDeliveryFingerprintArtifactError( + "immutable fingerprint artifact already exists with other content" + ) + else: + partial = target.with_name(target.name + ".partial") + try: + with partial.open("xb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + partial.replace(target) + _fsync_directory(target.parent) + except OSError as err: + try: + partial.unlink(missing_ok=True) + except OSError: + pass + raise BrokerDeliveryFingerprintArtifactError( + "atomic fingerprint publication failed" + ) from err + return ArtifactRef( + kind=BROKER_DELIVERY_FINGERPRINT_ARTIFACT_KIND, + path=str(target), + size_bytes=len(encoded), + sha256=digest, + metadata={ + "fingerprint_id": fingerprint.fingerprint_id, + "schema_version": fingerprint.schema_version, + }, + ) + + +def load_broker_delivery_fingerprint( + path: str | Path, +) -> BrokerDeliveryFingerprintV1: + """Load and identity-verify one immutable fingerprint artifact.""" + try: + return BrokerDeliveryFingerprintV1.from_json( + Path(path).read_text(encoding="utf-8") + ) + except (OSError, TypeError, ValueError) as err: + raise BrokerDeliveryFingerprintArtifactError( + "broker delivery fingerprint artifact is invalid" + ) from err + + +def _finalize_cells( + accumulators: Mapping[str, _CellAccumulator], + policy: BrokerDeliveryFitConfigV1, +) -> tuple[BrokerDeliveryCellV1, ...]: + supports = { + condition_id: item.support_count + for condition_id, item in accumulators.items() + } + cells: list[BrokerDeliveryCellV1] = [] + for accumulator in sorted( + accumulators.values(), key=lambda item: item.condition.key + ): + parents = _backoff_conditions(accumulator.condition) + parent_ids = tuple(item.condition_id for item in parents) + support = accumulator.support_count + effective: str | None + if support >= policy.min_cell_support: + status = BrokerDeliverySupportStatus.SUPPORTED + effective = accumulator.condition.condition_id + limitations: tuple[str, ...] = () + else: + effective = next( + ( + condition_id + for condition_id in parent_ids + if supports.get(condition_id, 0) >= policy.min_cell_support + ), + None, + ) + if effective is None: + status = BrokerDeliverySupportStatus.UNSUPPORTED + limitations = ("insufficient_support_no_qualified_backoff",) + else: + status = BrokerDeliverySupportStatus.BACKED_OFF + limitations = ("insufficient_support_using_parent_condition",) + cells.append( + BrokerDeliveryCellV1( + condition=accumulator.condition, + support_count=support, + support_status=status, + backoff_condition_ids=parent_ids, + effective_condition_id=effective, + metrics=tuple( + metric.to_metric(policy.quantiles) + for _, metric in sorted(accumulator.metrics.items()) + ), + limitations=limitations, + ) + ) + return tuple(cells) + + +def _backoff_conditions( + condition: BrokerDeliveryConditionV1, +) -> tuple[BrokerDeliveryConditionV1, ...]: + dimensions = condition.dimensions + if not dimensions: + return () + parents: list[BrokerDeliveryConditionV1] = [] + if len(dimensions) > 1 and "symbol" in dimensions: + parents.append( + BrokerDeliveryConditionV1({"symbol": dimensions["symbol"]}) + ) + non_symbol = { + key: value for key, value in dimensions.items() if key != "symbol" + } + parents.append(BrokerDeliveryConditionV1(non_symbol)) + parents.append(BrokerDeliveryConditionV1({})) + unique: dict[str, BrokerDeliveryConditionV1] = {} + for parent in parents: + if parent.condition_id != condition.condition_id: + unique.setdefault(parent.condition_id, parent) + return tuple(unique.values()) + + +def _compare_metric( + condition_key: str, + reference_cell: BrokerDeliveryCellV1 | None, + candidate_cell: BrokerDeliveryCellV1 | None, + reference: BrokerDeliveryMetricV1 | None, + candidate: BrokerDeliveryMetricV1 | None, + policy: BrokerDeliveryDriftConfigV1, +) -> BrokerDeliveryMetricComparisonV1: + condition_id = ( + reference_cell.condition.condition_id + if reference_cell is not None + else ( + candidate_cell.condition.condition_id + if candidate_cell is not None + else condition_key + ) + ) + name = reference.name if reference is not None else candidate.name # type: ignore[union-attr] + reference_support = 0 if reference is None else reference.support_count + candidate_support = 0 if candidate is None else candidate.support_count + reasons: tuple[str, ...] + if reference is None or candidate is None: + status = BrokerDeliveryDriftStatus.UNSUPPORTED + reasons = ("metric_missing_from_one_fingerprint",) + reference_estimate = None if reference is None else reference.estimate + candidate_estimate = None if candidate is None else candidate.estimate + difference = relative = uncertainty = None + elif ( + reference.support_count < policy.min_metric_support + or candidate.support_count < policy.min_metric_support + or reference.estimate is None + or candidate.estimate is None + ): + status = BrokerDeliveryDriftStatus.UNSUPPORTED + reasons = ("metric_support_below_comparison_minimum",) + reference_estimate = reference.estimate + candidate_estimate = candidate.estimate + difference = relative = uncertainty = None + else: + reference_estimate = reference.estimate + candidate_estimate = candidate.estimate + difference = abs(candidate.estimate - reference.estimate) + scale = max(abs(reference.estimate), 10 ** (-policy.rounding_digits)) + relative = difference / scale + reference_lower = ( + reference.estimate if reference.lower is None else reference.lower + ) + reference_upper = ( + reference.estimate if reference.upper is None else reference.upper + ) + candidate_lower = ( + candidate.estimate if candidate.lower is None else candidate.lower + ) + candidate_upper = ( + candidate.estimate if candidate.upper is None else candidate.upper + ) + reference_half = max( + reference.estimate - reference_lower, + reference_upper - reference.estimate, + ) + candidate_half = max( + candidate.estimate - candidate_lower, + candidate_upper - candidate.estimate, + ) + uncertainty = math.sqrt(reference_half**2 + candidate_half**2) + absolute_threshold = policy.absolute_material_thresholds.get( + name, policy.default_absolute_material_threshold + ) + if difference == 0: + status = BrokerDeliveryDriftStatus.STABLE + reasons = () + elif difference <= uncertainty: + status = BrokerDeliveryDriftStatus.SAMPLING_NOISE + reasons = ("uncertainty_intervals_overlap",) + elif ( + absolute_threshold > 0 and difference >= absolute_threshold + ) or relative >= policy.relative_material_threshold: + status = BrokerDeliveryDriftStatus.MATERIAL_DRIFT + reasons = ("difference_exceeds_uncertainty_and_effect_threshold",) + else: + status = BrokerDeliveryDriftStatus.SAMPLING_NOISE + reasons = ("effect_below_material_threshold",) + return BrokerDeliveryMetricComparisonV1( + condition_id=condition_id, + condition_key=condition_key, + metric_name=name, + reference_support_count=reference_support, + candidate_support_count=candidate_support, + reference_estimate=_rounded(reference_estimate, policy.rounding_digits), + candidate_estimate=_rounded(candidate_estimate, policy.rounding_digits), + absolute_difference=_rounded(difference, policy.rounding_digits), + relative_difference=_rounded(relative, policy.rounding_digits), + combined_uncertainty=_rounded(uncertainty, policy.rounding_digits), + status=status, + reason_codes=reasons, + ) + + +def _metrics_by_name( + cell: BrokerDeliveryCellV1 | None, +) -> dict[str, BrokerDeliveryMetricV1]: + return {} if cell is None else {item.name: item for item in cell.metrics} + + +def _assert_compatible_capture_identity( + manifests: Sequence[BrokerCaptureSessionManifestV1], +) -> None: + first = manifests[0].session + fields = ( + "adapter_id", + "adapter_version", + "adapter_config_sha256", + "protocol", + "environment_id", + "server_id", + "account_id_sha256", + "collector_id", + "collector_version", + ) + for manifest in manifests[1:]: + if any( + getattr(manifest.session, field_name) != getattr(first, field_name) + for field_name in fields + ): + raise BrokerDeliveryFingerprintIdentityError( + "capture manifests describe different broker delivery identities" + ) + + +def _assert_compatible_predecessor( + manifest: BrokerCaptureSessionManifestV1, + predecessor: BrokerDeliveryFingerprintV1, + effective_start_utc_ns: int, +) -> None: + session = manifest.session + fields = ( + "adapter_id", + "adapter_version", + "adapter_config_sha256", + "protocol", + "environment_id", + "server_id", + "account_id_sha256", + "collector_id", + "collector_version", + ) + if any( + getattr(session, field_name) != getattr(predecessor, field_name) + for field_name in fields + ): + raise BrokerDeliveryFingerprintIdentityError( + "successor profile broker identity differs from predecessor" + ) + if effective_start_utc_ns <= predecessor.effective_start_utc_ns: + raise BrokerDeliveryFingerprintIdentityError( + "successor effective start must follow predecessor effective start" + ) + + +def _bounded_context_events( + timeline: MarketContextTimelineV1 | None, + policy: BrokerDeliveryFitConfigV1, +) -> tuple[MarketContextEventV1, ...]: + if timeline is None: + return () + if len(timeline.events) > policy.max_market_context_events: + raise BrokerDeliveryResourceLimitError( + "market context timeline exceeds max_market_context_events" + ) + return tuple(timeline.events) + + +def _context_event_matches( + event: MarketContextEventV1, + timestamp_ns: int, + symbol: str, +) -> bool: + if not ( + event.event_time_ns - event.pre_event_ns + <= timestamp_ns + < event.event_time_ns + event.post_event_ns + ): + return False + if symbol in event.affected_symbols: + return True + currencies = {symbol[:3], symbol[3:6]} if len(symbol) >= 6 else set() + return bool(currencies.intersection(event.affected_currencies)) + + +def _partition_hashes_digest( + manifest: BrokerCaptureSessionManifestV1, +) -> str: + payload: dict[str, JSONValue] = { + "partitions": [ + { + "partition_id": item.partition_id, + "artifact_sha256": item.data_artifact.sha256, + } + for item in manifest.partitions + ] + } + return hashlib.sha256( + canonical_capture_json(payload).encode("utf-8") + ).hexdigest() + + +def _fit_limitations( + decisions: Sequence[BrokerCaptureEligibilityV1], + cells: Sequence[BrokerDeliveryCellV1], + *, + timeline: MarketContextTimelineV1 | None, + total_events: int, + policy: BrokerDeliveryFitConfigV1, + support_start_utc_ns: int, + support_end_utc_ns: int, + calendar_profile_complete: bool, +) -> tuple[str, ...]: + limitations = { + "profile_describes_broker_observation_delivery_not_market_truth", + "cadence_uses_monotonic_receive_time", + "calendar_conditioning_uses_utc_receive_time", + } + if any( + item.status is BrokerCaptureEligibilityStatus.LIMITED + for item in decisions + ): + limitations.add("one_or_more_captures_have_nonfatal_health_limitations") + if any( + item.support_status is not BrokerDeliverySupportStatus.SUPPORTED + for item in cells + ): + limitations.add( + "sparse_condition_cells_use_explicit_backoff_or_are_unsupported" + ) + if total_events > policy.max_samples_per_metric: + limitations.add( + "large_metric_distributions_use_deterministic_bounded_samples" + ) + if timeline is None: + limitations.add("no_versioned_market_context_timeline_supplied") + elif not timeline.complete: + limitations.add("market_context_timeline_is_incomplete") + if timeline is not None and ( + timeline.coverage_start_ns > support_start_utc_ns + or timeline.coverage_end_ns <= support_end_utc_ns + ): + limitations.add( + "market_context_timeline_does_not_cover_capture_support" + ) + if not calendar_profile_complete: + limitations.add("calendar_profile_is_incomplete") + return tuple(sorted(limitations)) + + +def _wilson_interval(rate: float, count: int) -> tuple[float, float]: + z = 1.96 + denominator = 1 + z * z / count + centre = (rate + z * z / (2 * count)) / denominator + half = ( + z + * math.sqrt(rate * (1 - rate) / count + z * z / (4 * count * count)) + / denominator + ) + return max(0.0, centre - half), min(1.0, centre + half) + + +def _quantile(values: Sequence[float], probability: float) -> float: + if len(values) == 1: + return values[0] + position = probability * (len(values) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return values[lower] + weight = position - lower + return values[lower] * (1 - weight) + values[upper] * weight + + +def _quantile_name(value: float) -> str: + return f"q{value:.6f}".rstrip("0").rstrip(".") + + +def _rounded(value: float | None, digits: int) -> float | None: + if value is None: + return None + rounded = round(float(value), digits) + return 0.0 if rounded == 0 else rounded + + +def _rounded_required(value: float, digits: int) -> float: + rounded = round(float(value), digits) + return 0.0 if rounded == 0 else rounded + + +def _price_decimal_places( + bid_text: str | None, ask_text: str | None +) -> int | None: + if bid_text is None or ask_text is None: + return None + return max(_decimal_places(bid_text), _decimal_places(ask_text)) + + +def _decimal_places(value: str) -> int: + mantissa = value.lower().split("e", 1)[0] + return len(mantissa.split(".", 1)[1]) if "." in mantissa else 0 + + +def _has_trailing_zero(bid_text: str | None, ask_text: str | None) -> bool: + return bool( + bid_text + and ask_text + and "." in bid_text + and "." in ask_text + and (bid_text.endswith("0") or ask_text.endswith("0")) + ) + + +def _safe_tag(value: str) -> str: + normalized = _TAG_RE.sub("_", str(value).strip()).strip("_").lower() + return normalized[:256] or "unknown" + + +def _fsync_directory(path: Path) -> None: + try: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError: + return + + +__all__ = [ + "BrokerDeliveryFingerprintArtifactError", + "BrokerDeliveryFingerprintError", + "BrokerDeliveryFingerprintIdentityError", + "BrokerDeliveryIneligibleCaptureError", + "BrokerDeliveryResourceLimitError", + "assess_broker_capture_eligibility", + "compare_broker_delivery_fingerprints", + "fit_broker_delivery_fingerprint", + "load_broker_delivery_fingerprint", + "write_broker_delivery_fingerprint", +] diff --git a/src/histdatacom/broker_capture/storage.py b/src/histdatacom/broker_capture/storage.py new file mode 100644 index 00000000..e8011d00 --- /dev/null +++ b/src/histdatacom/broker_capture/storage.py @@ -0,0 +1,713 @@ +"""Crash-safe append-only storage and verified replay for broker captures.""" + +from __future__ import annotations + +import hashlib +import os +from collections.abc import Iterable, Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO + +from histdatacom.broker_capture.adapters import ( + BrokerCaptureEventConsumerV1, + BrokerCaptureEventSourceV1, + consume_broker_capture_source, +) +from histdatacom.broker_capture.contracts import ( + BROKER_CAPTURE_DATA_ARTIFACT_KIND, + BrokerCaptureEventV1, + BrokerCapturePartitionManifestV1, + BrokerCaptureReplaySummaryV1, + BrokerCaptureSessionManifestV1, + BrokerCaptureSessionState, + BrokerCaptureSessionV1, + BrokerCaptureStoragePolicyV1, +) +from histdatacom.runtime_contracts import ArtifactRef + +SESSION_MANIFEST_FILENAME = "session.manifest.json" +PARTITION_DATA_TEMPLATE = "partition-{ordinal:06d}.jsonl" +PARTITION_MANIFEST_TEMPLATE = "partition-{ordinal:06d}.manifest.json" + + +class BrokerCaptureStorageError(RuntimeError): + """Base class for fail-closed capture-storage errors.""" + + +class BrokerCaptureQuotaError(BrokerCaptureStorageError): + """Hard disk quota would be exceeded.""" + + +class BrokerCaptureBackpressureError(BrokerCaptureStorageError): + """Configured high-watermark backpressure refused another event.""" + + +class BrokerCaptureRetentionError(BrokerCaptureStorageError): + """Immutable retention ceiling refused another partition.""" + + +class BrokerCaptureIntegrityError(BrokerCaptureStorageError): + """Stored capture evidence failed hash, count, or ordering verification.""" + + +class BrokerCaptureExistingSessionError(BrokerCaptureStorageError): + """A new writer would overlap an existing session directory.""" + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureSessionInspectionV1: + """Read-only detection of incomplete or unadvertised session artifacts.""" + + session_directory: str + advertised_partition_count: int + partial_artifacts: tuple[str, ...] + orphan_data_artifacts: tuple[str, ...] + orphan_manifest_artifacts: tuple[str, ...] + + @property + def clean(self) -> bool: + """Return whether every on-disk artifact is committed and advertised.""" + return not ( + self.partial_artifacts + or self.orphan_data_artifacts + or self.orphan_manifest_artifacts + ) + + +class AppendOnlyBrokerCaptureWriterV1: + """Rotate canonical JSONL partitions and publish only completed evidence.""" + + def __init__( + self, + root: str | Path, + *, + session: BrokerCaptureSessionV1, + storage_policy: BrokerCaptureStoragePolicyV1, + ) -> None: + self.root = Path(root) + self.session = session + self.storage_policy = storage_policy + self.session_directory = self.root / session.session_id + if self.session_directory.exists() and any( + self.session_directory.iterdir() + ): + raise BrokerCaptureExistingSessionError( + "capture session directory already contains evidence" + ) + self.session_directory.mkdir(parents=True, exist_ok=True) + self._partitions: list[BrokerCapturePartitionManifestV1] = [] + self._event_kind_counts: dict[str, int] = {} + self._total_events = 0 + self._last_monotonic_ns: int | None = None + self._file: BinaryIO | None = None + self._partial_path: Path | None = None + self._partition_event_count = 0 + self._partition_bytes = 0 + self._partition_first_event: BrokerCaptureEventV1 | None = None + self._partition_last_event: BrokerCaptureEventV1 | None = None + self._partition_kind_counts: dict[str, int] = {} + self._closed = False + self._manifest = self._publish_session_manifest( + BrokerCaptureSessionState.OPEN, limitations=() + ) + + @property + def manifest(self) -> BrokerCaptureSessionManifestV1: + """Return the latest compact manifest snapshot.""" + return self._manifest + + def append(self, event: BrokerCaptureEventV1) -> None: + """Append one event, rotating before limits are crossed.""" + if self._closed: + raise BrokerCaptureStorageError("capture writer is closed") + if event.session_id != self.session.session_id: + raise ValueError("capture event belongs to another session") + if event.capture_sequence != self._total_events: + raise ValueError("capture event sequence is not contiguous") + if ( + self._last_monotonic_ns is not None + and event.receive_time_monotonic_ns < self._last_monotonic_ns + ): + raise ValueError("capture event monotonic time moved backwards") + line = (event.to_json() + "\n").encode("utf-8") + if len(line) > self.storage_policy.max_partition_bytes: + raise BrokerCaptureQuotaError( + "one capture event exceeds the partition byte limit" + ) + if self._should_rotate(event, len(line)): + self._finalize_partition() + starting_partition = self._file is None + self._enforce_storage_limits(len(line), starting_partition) + if starting_partition: + self._open_partition() + assert self._file is not None + try: + self._file.write(line) + self._file.flush() + if self.storage_policy.fsync_each_event: + os.fsync(self._file.fileno()) + except OSError as err: + raise BrokerCaptureStorageError( + "capture partition append failed" + ) from err + self._partition_bytes += len(line) + self._partition_event_count += 1 + if self._partition_first_event is None: + self._partition_first_event = event + self._partition_last_event = event + self._partition_kind_counts[event.kind.value] = ( + self._partition_kind_counts.get(event.kind.value, 0) + 1 + ) + self._event_kind_counts[event.kind.value] = ( + self._event_kind_counts.get(event.kind.value, 0) + 1 + ) + self._total_events += 1 + self._last_monotonic_ns = event.receive_time_monotonic_ns + + def close( + self, + *, + completed: bool = True, + limitations: Sequence[str] = (), + ) -> BrokerCaptureSessionManifestV1: + """Finalize valid data and publish terminal capture health.""" + if self._closed: + return self._manifest + self._finalize_partition() + state = ( + BrokerCaptureSessionState.COMPLETED + if completed + else BrokerCaptureSessionState.FAILED + ) + self._manifest = self._publish_session_manifest( + state, limitations=limitations + ) + self._closed = True + return self._manifest + + def __enter__(self) -> "AppendOnlyBrokerCaptureWriterV1": + return self + + def __exit__( + self, exc_type: object, exc: object, traceback: object + ) -> None: + if exc_type is None: + self.close(completed=True) + return + limitation = ( + f"collector_failure:{getattr(exc_type, '__name__', 'unknown')}" + ) + self.close(completed=False, limitations=(limitation,)) + + def _should_rotate( + self, event: BrokerCaptureEventV1, line_bytes: int + ) -> bool: + if self._file is None or self._partition_first_event is None: + return False + if ( + self._partition_event_count + >= self.storage_policy.max_partition_events + ): + return True + if ( + self._partition_bytes + line_bytes + > self.storage_policy.max_partition_bytes + ): + return True + elapsed = ( + event.receive_time_monotonic_ns + - self._partition_first_event.receive_time_monotonic_ns + ) + return elapsed >= self.storage_policy.max_partition_duration_ns + + def _enforce_storage_limits( + self, line_bytes: int, starting_partition: bool + ) -> None: + if starting_partition and ( + len(self._partitions) >= self.storage_policy.max_retained_partitions + ): + raise BrokerCaptureRetentionError( + "capture retention ceiling reached; committed evidence was not deleted" + ) + projected = ( + _directory_size(self.session_directory) + + line_bytes + + self.storage_policy.manifest_reserve_bytes + ) + if projected > self.storage_policy.max_session_bytes: + raise BrokerCaptureQuotaError( + "capture session disk quota would be exceeded" + ) + if projected > self.storage_policy.high_watermark_bytes: + raise BrokerCaptureBackpressureError( + "capture storage high watermark requires backpressure" + ) + + def _open_partition(self) -> None: + ordinal = len(self._partitions) + partial_name = ( + PARTITION_DATA_TEMPLATE.format(ordinal=ordinal) + ".partial" + ) + partial_path = self.session_directory / partial_name + try: + self._file = partial_path.open("xb") + except OSError as err: + raise BrokerCaptureStorageError( + "could not create capture partial partition" + ) from err + self._partial_path = partial_path + self._partition_event_count = 0 + self._partition_bytes = 0 + self._partition_first_event = None + self._partition_last_event = None + self._partition_kind_counts = {} + + def _finalize_partition(self) -> None: + if self._file is None: + return + file_handle = self._file + partial_path = self._partial_path + first = self._partition_first_event + last = self._partition_last_event + if partial_path is None or first is None or last is None: + raise BrokerCaptureStorageError( + "capture partition state is incomplete" + ) + try: + file_handle.flush() + os.fsync(file_handle.fileno()) + file_handle.close() + ordinal = len(self._partitions) + final_path = ( + self.session_directory + / PARTITION_DATA_TEMPLATE.format(ordinal=ordinal) + ) + partial_path.replace(final_path) + _fsync_directory(self.session_directory) + size_bytes = final_path.stat().st_size + if size_bytes != self._partition_bytes: + raise BrokerCaptureIntegrityError( + "capture partition byte count changed before publication" + ) + artifact = ArtifactRef( + kind=BROKER_CAPTURE_DATA_ARTIFACT_KIND, + path=str(final_path.relative_to(self.root)).replace( + os.sep, "/" + ), + size_bytes=size_bytes, + sha256=_file_sha256(final_path), + metadata={ + "encoding": "utf-8", + "format": "canonical-json-lines", + "ordering": "capture_sequence", + }, + ) + partition = BrokerCapturePartitionManifestV1( + session_id=self.session.session_id, + policy_id=self.storage_policy.policy_id, + partition_ordinal=ordinal, + data_artifact=artifact, + event_count=self._partition_event_count, + first_capture_sequence=first.capture_sequence, + last_capture_sequence=last.capture_sequence, + first_receive_time_utc_ns=first.receive_time_utc_ns, + last_receive_time_utc_ns=last.receive_time_utc_ns, + first_receive_time_monotonic_ns=( + first.receive_time_monotonic_ns + ), + last_receive_time_monotonic_ns=(last.receive_time_monotonic_ns), + event_kind_counts=dict(self._partition_kind_counts), + ) + partition_manifest_path = ( + self.session_directory + / PARTITION_MANIFEST_TEMPLATE.format(ordinal=ordinal) + ) + _atomic_write_text( + partition_manifest_path, partition.to_json() + "\n" + ) + self._partitions.append(partition) + self._manifest = self._publish_session_manifest( + BrokerCaptureSessionState.OPEN, limitations=() + ) + except BrokerCaptureStorageError: + raise + except OSError as err: + raise BrokerCaptureStorageError( + "capture partition finalization failed" + ) from err + finally: + if not file_handle.closed: + file_handle.close() + self._file = None + self._partial_path = None + self._partition_event_count = 0 + self._partition_bytes = 0 + self._partition_first_event = None + self._partition_last_event = None + self._partition_kind_counts = {} + + def _publish_session_manifest( + self, + state: BrokerCaptureSessionState, + *, + limitations: Sequence[str], + ) -> BrokerCaptureSessionManifestV1: + partitions = tuple(self._partitions) + event_count = sum(item.event_count for item in partitions) + counts: dict[str, int] = {} + for item in partitions: + for kind, count in item.event_kind_counts.items(): + counts[kind] = counts.get(kind, 0) + count + manifest = BrokerCaptureSessionManifestV1( + session=self.session, + storage_policy=self.storage_policy, + state=state, + partitions=partitions, + event_count=event_count, + event_kind_counts=counts, + first_capture_sequence=( + partitions[0].first_capture_sequence if partitions else None + ), + last_capture_sequence=( + partitions[-1].last_capture_sequence if partitions else None + ), + partial_artifact_count=0, + limitations=tuple(limitations), + ) + _atomic_write_text( + self.session_directory / SESSION_MANIFEST_FILENAME, + manifest.to_json() + "\n", + ) + return manifest + + +@dataclass(frozen=True, slots=True) +class BrokerCaptureReplaySourceV1: + """Verified event source over completed, advertised capture partitions.""" + + root: Path + manifest: BrokerCaptureSessionManifestV1 + + def __init__( + self, + root: str | Path, + manifest: BrokerCaptureSessionManifestV1, + ) -> None: + object.__setattr__(self, "root", Path(root)) + object.__setattr__(self, "manifest", manifest) + + @property + def session_id(self) -> str: + """Return the replayed capture session identity.""" + return self.manifest.session.session_id + + def iter_events(self) -> Iterable[BrokerCaptureEventV1]: + """Verify hashes/counts/order and yield events partition by partition.""" + return self._event_iterator() + + def _event_iterator(self) -> Iterator[BrokerCaptureEventV1]: + verify_broker_capture_partition_manifests(self.root, self.manifest) + expected_sequence = self.manifest.first_capture_sequence + total_count = 0 + combined_counts: dict[str, int] = {} + for partition in self.manifest.partitions: + path = _contained_artifact_path( + self.root, partition.data_artifact.path + ) + if not path.is_file(): + raise BrokerCaptureIntegrityError( + "advertised capture partition is missing" + ) + if path.stat().st_size != partition.data_artifact.size_bytes: + raise BrokerCaptureIntegrityError( + "capture partition size does not match manifest" + ) + if _file_sha256(path) != partition.data_artifact.sha256: + raise BrokerCaptureIntegrityError( + "capture partition hash does not match manifest" + ) + partition_count = 0 + partition_counts: dict[str, int] = {} + first_event: BrokerCaptureEventV1 | None = None + last_event: BrokerCaptureEventV1 | None = None + try: + with path.open("rt", encoding="utf-8", newline="") as handle: + for line in handle: + if not line.endswith("\n") or not line.strip(): + raise BrokerCaptureIntegrityError( + "capture partition contains a partial JSON line" + ) + event = BrokerCaptureEventV1.from_json(line) + if event.session_id != self.session_id: + raise BrokerCaptureIntegrityError( + "capture partition contains another session" + ) + if expected_sequence is None: + expected_sequence = event.capture_sequence + if event.capture_sequence != expected_sequence: + raise BrokerCaptureIntegrityError( + "capture replay sequence is not contiguous" + ) + if ( + last_event is not None + and event.receive_time_monotonic_ns + < last_event.receive_time_monotonic_ns + ): + raise BrokerCaptureIntegrityError( + "capture replay monotonic time moved backwards" + ) + if first_event is None: + first_event = event + last_event = event + partition_count += 1 + total_count += 1 + partition_counts[event.kind.value] = ( + partition_counts.get(event.kind.value, 0) + 1 + ) + combined_counts[event.kind.value] = ( + combined_counts.get(event.kind.value, 0) + 1 + ) + expected_sequence += 1 + yield event + except UnicodeDecodeError as err: + raise BrokerCaptureIntegrityError( + "capture partition is not valid UTF-8" + ) from err + if ( + partition_count != partition.event_count + or partition_counts != partition.event_kind_counts + or first_event is None + or last_event is None + or first_event.capture_sequence + != partition.first_capture_sequence + or last_event.capture_sequence + != partition.last_capture_sequence + or first_event.receive_time_utc_ns + != partition.first_receive_time_utc_ns + or last_event.receive_time_utc_ns + != partition.last_receive_time_utc_ns + or first_event.receive_time_monotonic_ns + != partition.first_receive_time_monotonic_ns + or last_event.receive_time_monotonic_ns + != partition.last_receive_time_monotonic_ns + ): + raise BrokerCaptureIntegrityError( + "capture partition contents do not reconcile with manifest" + ) + if ( + total_count != self.manifest.event_count + or combined_counts != self.manifest.event_kind_counts + ): + raise BrokerCaptureIntegrityError( + "capture replay does not reconcile with session manifest" + ) + + +def load_broker_capture_session_manifest( + path: str | Path, +) -> BrokerCaptureSessionManifestV1: + """Load and verify one published session manifest.""" + return BrokerCaptureSessionManifestV1.from_json( + Path(path).read_text(encoding="utf-8") + ) + + +def discover_broker_capture_session_manifests( + root: str | Path, +) -> tuple[BrokerCaptureSessionManifestV1, ...]: + """Discover only atomically published session manifests.""" + capture_root = Path(root) + manifests_list: list[BrokerCaptureSessionManifestV1] = [] + for path in sorted(capture_root.glob(f"*/{SESSION_MANIFEST_FILENAME}")): + if not path.is_file(): + continue + manifest = load_broker_capture_session_manifest(path) + verify_broker_capture_partition_manifests(capture_root, manifest) + manifests_list.append(manifest) + manifests = tuple(manifests_list) + return tuple(sorted(manifests, key=lambda item: item.session.session_id)) + + +def verify_broker_capture_partition_manifests( + root: str | Path, + manifest: BrokerCaptureSessionManifestV1, +) -> BrokerCaptureSessionManifestV1: + """Verify every advertised partition sidecar against the session catalog.""" + session_directory = Path(root) / manifest.session.session_id + for partition in manifest.partitions: + sidecar_path = session_directory / PARTITION_MANIFEST_TEMPLATE.format( + ordinal=partition.partition_ordinal + ) + if not sidecar_path.is_file(): + raise BrokerCaptureIntegrityError( + "advertised capture partition manifest is missing" + ) + try: + restored = BrokerCapturePartitionManifestV1.from_json( + sidecar_path.read_text(encoding="utf-8") + ) + except (OSError, TypeError, ValueError) as err: + raise BrokerCaptureIntegrityError( + "capture partition manifest is invalid" + ) from err + if restored != partition: + raise BrokerCaptureIntegrityError( + "capture partition manifest does not match session catalog" + ) + return manifest + + +def inspect_broker_capture_session( + root: str | Path, + session_id: str, +) -> BrokerCaptureSessionInspectionV1: + """Detect partial and orphan artifacts without advertising them.""" + capture_root = Path(root) + session_directory = capture_root / session_id + manifest_path = session_directory / SESSION_MANIFEST_FILENAME + manifest = ( + load_broker_capture_session_manifest(manifest_path) + if manifest_path.is_file() + else None + ) + advertised_data = { + Path(item.data_artifact.path).name + for item in (() if manifest is None else manifest.partitions) + } + advertised_manifests = { + PARTITION_MANIFEST_TEMPLATE.format(ordinal=item.partition_ordinal) + for item in (() if manifest is None else manifest.partitions) + } + partial = tuple( + path.name for path in sorted(session_directory.glob("*.partial")) + ) + orphan_data = tuple( + path.name + for path in sorted(session_directory.glob("partition-*.jsonl")) + if path.name not in advertised_data + ) + orphan_manifests = tuple( + path.name + for path in sorted(session_directory.glob("partition-*.manifest.json")) + if path.name not in advertised_manifests + ) + return BrokerCaptureSessionInspectionV1( + session_directory=str(session_directory), + advertised_partition_count=( + 0 if manifest is None else len(manifest.partitions) + ), + partial_artifacts=partial, + orphan_data_artifacts=orphan_data, + orphan_manifest_artifacts=orphan_manifests, + ) + + +def replay_broker_capture_session( + root: str | Path, + manifest: BrokerCaptureSessionManifestV1, + *, + consumers: Sequence[BrokerCaptureEventConsumerV1] = (), +) -> BrokerCaptureReplaySummaryV1: + """Replay verified evidence through the live-compatible consumer seam.""" + digest_consumer = _LogicalDigestConsumer() + source: BrokerCaptureEventSourceV1 = BrokerCaptureReplaySourceV1( + root, manifest + ) + result = consume_broker_capture_source( + source, + consumers=(digest_consumer, *consumers), + ) + return BrokerCaptureReplaySummaryV1( + session_id=result.session_id, + manifest_id=manifest.manifest_id, + partition_count=len(manifest.partitions), + event_count=result.event_count, + event_kind_counts=result.event_kind_counts, + first_capture_sequence=result.first_capture_sequence, + last_capture_sequence=result.last_capture_sequence, + logical_content_sha256=digest_consumer.hexdigest(), + ) + + +class _LogicalDigestConsumer: + def __init__(self) -> None: + self._digest = hashlib.sha256() + + def on_event(self, event: BrokerCaptureEventV1) -> None: + self._digest.update(event.to_json().encode("utf-8")) + self._digest.update(b"\n") + + def hexdigest(self) -> str: + return self._digest.hexdigest() + + +def _atomic_write_text(path: Path, text: str) -> None: + partial_path = path.with_name(path.name + ".partial") + try: + with partial_path.open("x", encoding="utf-8", newline="\n") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + partial_path.replace(path) + _fsync_directory(path.parent) + except OSError as err: + raise BrokerCaptureStorageError( + "atomic capture manifest publication failed" + ) from err + + +def _contained_artifact_path(root: Path, relative_path: str) -> Path: + root_resolved = root.resolve() + candidate = (root / relative_path).resolve() + if not candidate.is_relative_to(root_resolved): + raise BrokerCaptureIntegrityError( + "capture artifact escaped the configured root" + ) + return candidate + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while block := handle.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def _directory_size(path: Path) -> int: + return sum( + item.stat().st_size for item in path.rglob("*") if item.is_file() + ) + + +def _fsync_directory(path: Path) -> None: + flags = getattr(os, "O_DIRECTORY", 0) | os.O_RDONLY + try: + descriptor = os.open(path, flags) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) + + +__all__ = [ + "AppendOnlyBrokerCaptureWriterV1", + "BrokerCaptureBackpressureError", + "BrokerCaptureExistingSessionError", + "BrokerCaptureIntegrityError", + "BrokerCaptureQuotaError", + "BrokerCaptureReplaySourceV1", + "BrokerCaptureRetentionError", + "BrokerCaptureSessionInspectionV1", + "BrokerCaptureStorageError", + "discover_broker_capture_session_manifests", + "inspect_broker_capture_session", + "load_broker_capture_session_manifest", + "replay_broker_capture_session", + "verify_broker_capture_partition_manifests", +] diff --git a/src/histdatacom/cli.py b/src/histdatacom/cli.py index 8c86c8d8..152a3c43 100644 --- a/src/histdatacom/cli.py +++ b/src/histdatacom/cli.py @@ -87,10 +87,10 @@ DEFAULT_QUALITY_PREFLIGHT_SAMPLE_SIZE, ) from histdatacom.data_quality.profiles import ( - QualityProfile, QualityProfileError, - load_quality_profile_file, - quality_profile_from_mapping, + apply_quality_profile_overrides, + load_quality_profile_file_resolution, + resolve_quality_profile, ) from histdatacom.fx_enums import ( Format, @@ -101,6 +101,11 @@ pair_group_names, get_valid_format_timeframes, ) +from histdatacom.random_windows import ( + RandomWindowError, + parse_random_window_expression, + random_window_requires_seed, +) from histdatacom.verbosity import normalize_verbosity from histdatacom.utils import ( get_current_datemonth_gmt_minus5, @@ -131,21 +136,9 @@ def _positive_int(value: str) -> int: return parsed -def _enable_remediation_catalog_audit( - profile: QualityProfile, -) -> QualityProfile: - """Return a profile with remediation-catalog audit reporting enabled.""" - payload = profile.to_request_payload() - reporting_value = payload.get("reporting", {}) - reporting = ( - dict(reporting_value) if isinstance(reporting_value, dict) else {} - ) - audit_value = reporting.get("remediation_catalog_audit", {}) - audit = dict(audit_value) if isinstance(audit_value, dict) else {} - audit["enabled"] = True - reporting["remediation_catalog_audit"] = audit - payload["reporting"] = reporting - return quality_profile_from_mapping(payload) +def _argv_has_option(args: Tuple[str, ...], option: str) -> bool: + """Return whether argv contains an option in split or equals form.""" + return any(arg == option or arg.startswith(f"{option}=") for arg in args) class ArgParser(argparse.ArgumentParser): # noqa:H601 @@ -173,12 +166,14 @@ def __init__(self, options: Options | None = None): " groups List instrument groups and major triangles\n" " jobs Inspect and control orchestrated work\n" " quality Inspect local data quality evidence\n" + " reconstruction Plan, run, and inspect reconstruction\n" " runtime Inspect and manage the orchestration runtime\n\n" "Run `histdatacom analytics --help` for analytics commands.\n" "Run `histdatacom cleanup --help` for cleanup commands.\n" "Run `histdatacom groups --help` for group discovery commands.\n" "Run `histdatacom jobs --help` for job telemetry commands." "\nRun `histdatacom quality --help` for quality commands." + "\nRun `histdatacom reconstruction --help` for reconstruction." ), ) # bring in the defaults arg DTO from outer class, use the @@ -186,6 +181,8 @@ def __init__(self, options: Options | None = None): self.arg_namespace = options if options is not None else Options() self._default_args = self.arg_namespace.to_dict() + self._explicit_cli_args: Tuple[str, ...] = () + self._config_file_args: Tuple[str, ...] = () self.set_defaults(**self._default_args) @classmethod @@ -371,6 +368,12 @@ def _check_quality_mode(self) -> None: "--quality-preflight-validation-report requires --quality-preflight" ) raise SystemExit(1) + if self.arg_namespace.quality_preflight_validation_evidence_path: + print( # noqa:T201 + "--quality-preflight-validation-evidence requires " + "--quality-preflight" + ) + raise SystemExit(1) if self.arg_namespace.quality_preflight_run_validation: print( # noqa:T201 "--quality-preflight-run-validation requires --quality-preflight" @@ -480,6 +483,15 @@ def _check_quality_mode(self) -> None: "file path, not '-'" ) raise SystemExit(1) + if ( + self.arg_namespace.quality_preflight_validation_evidence_path + == "-" + ): + print( # noqa:T201 + "--quality-preflight-validation-evidence requires a " + "file path, not '-'" + ) + raise SystemExit(1) if ( self.arg_namespace.quality_profile_preview and self.arg_namespace.quality_preflight_profile_preview_output_path @@ -500,6 +512,7 @@ def _check_quality_mode(self) -> None: raise SystemExit(1) elif ( self.arg_namespace.quality_preflight_validation_report_path + or self.arg_namespace.quality_preflight_validation_evidence_path or self.arg_namespace.quality_preflight_run_validation ): print( # noqa:T201 @@ -532,40 +545,80 @@ def _check_quality_mode(self) -> None: self._load_quality_profile() def _load_quality_profile(self) -> None: - """Validate and embed an operator quality profile, when configured.""" + """Resolve a profile and preserve value-level source metadata.""" + config_path = str(self.arg_namespace.config_path or "") try: if self.arg_namespace.quality_profile_path: - profile = load_quality_profile_file( - self.arg_namespace.quality_profile_path + if self.arg_namespace.from_api: + selected_by = "api_options" + elif _argv_has_option( + self._explicit_cli_args, + "--quality-profile", + ): + selected_by = "cli_override" + else: + selected_by = "yaml_config" + resolution = load_quality_profile_file_resolution( + self.arg_namespace.quality_profile_path, + config_path=config_path, + selected_by=selected_by, ) elif self.arg_namespace.quality_profile: - profile = quality_profile_from_mapping( + resolution = resolve_quality_profile( self.arg_namespace.quality_profile, - source="api-options", - ) - elif self.arg_namespace.quality_remediation_catalog_audit: - profile = quality_profile_from_mapping( - { - "reporting": { - "remediation_catalog_audit": { - "enabled": True, - } - } - }, source=( "api-options" if self.arg_namespace.from_api - else "cli-options" + else "operator-config" + ), + config_path=config_path, + selected_by=( + "api_options" + if self.arg_namespace.from_api + else "operator_config" ), ) else: - return + resolution = resolve_quality_profile( + config_path=config_path, + ) if self.arg_namespace.quality_remediation_catalog_audit: - profile = _enable_remediation_catalog_audit(profile) + if self.arg_namespace.from_api: + override_source = "api_options" + elif _argv_has_option( + self._explicit_cli_args, + "--quality-remediation-catalog-audit", + ): + override_source = "cli_override" + elif _argv_has_option( + self._config_file_args, + "--quality-remediation-catalog-audit", + ): + override_source = "yaml_config" + else: + override_source = "cli_override" + resolution = apply_quality_profile_overrides( + resolution, + { + "reporting.remediation_catalog_audit.enabled": True, + }, + source=override_source, + source_path=( + config_path if override_source == "yaml_config" else "" + ), + ) except QualityProfileError as exc: print(f"quality profile error: {exc}") # noqa:T201 raise SystemExit(1) from exc - self.arg_namespace.quality_profile = profile.to_request_payload() + self.arg_namespace.quality_profile_resolution = resolution.to_payload() + if ( + self.arg_namespace.quality_profile_path + or self.arg_namespace.quality_profile + or self.arg_namespace.quality_remediation_catalog_audit + ): + self.arg_namespace.quality_profile = ( + resolution.profile.to_request_payload() + ) def _clean_from_api_args(self) -> list: # noqa:CCR001 """Build the args list from api Options. @@ -705,6 +758,17 @@ def _clean_from_api_args(self) -> list: # noqa:CCR001 ), ] ) + if ( + self.arg_namespace.quality_preflight_validation_evidence_path + ): + args.extend( + [ + "--quality-preflight-validation-evidence", + ( + self.arg_namespace.quality_preflight_validation_evidence_path + ), + ] + ) if self.arg_namespace.quality_preflight_run_validation: args.append("--quality-preflight-run-validation") args.extend( @@ -781,6 +845,12 @@ def _clean_from_api_args(self) -> list: # noqa:CCR001 args.extend(["-s", self.arg_namespace.start_yearmonth]) if self.arg_namespace.end_yearmonth: args.extend(["-e", self.arg_namespace.end_yearmonth]) + if self.arg_namespace.random_window: + args.extend(["--random-window", self.arg_namespace.random_window]) + if self.arg_namespace.random_seed is not None: + args.extend(["--random-seed", str(self.arg_namespace.random_seed)]) + if self.arg_namespace.output_timezone: + args.extend(["--timezone", self.arg_namespace.output_timezone]) if self.arg_namespace.available_remote_data: args.append("-A") if self.arg_namespace.update_remote_data: @@ -983,6 +1053,57 @@ def _check_datetime_input(self) -> None: self._check_start_lessthan_end() self._validate_prerequisites() + def _check_random_window_input(self) -> None: + """Validate deterministic tick-window selection inputs.""" + expression = str(self.arg_namespace.random_window or "") + seed = self.arg_namespace.random_seed + if not expression: + if seed is not None: + print( + "ERROR: --random-seed requires --random-window" + ) # noqa:T201 + raise SystemExit(1) + return + if ( + self.arg_namespace.available_remote_data + or self.arg_namespace.update_remote_data + ): + print( # noqa:T201 + "ERROR: --random-window cannot be combined with repository " + "inventory modes -A or -U" + ) + raise SystemExit(1) + if ( + self.arg_namespace.data_quality + or self.arg_namespace.repo_quality_refresh + or self.arg_namespace.quality_preflight + ): + print( # noqa:T201 + "ERROR: --random-window is a tick projection option and " + "cannot be combined with data-quality modes" + ) + raise SystemExit(1) + try: + parse_random_window_expression(expression) + if seed is not None and seed > 2**63 - 1: + raise RandomWindowError( + "random-window seed exceeds signed int64 range" + ) + if ( + random_window_requires_seed( + expression, + start_yearmonth=self.arg_namespace.start_yearmonth, + end_yearmonth=self.arg_namespace.end_yearmonth, + ) + and seed is None + ): + raise RandomWindowError( + "random selection requires --random-seed" + ) + except RandomWindowError as exc: + print(f"ERROR: invalid --random-window: {exc}") # noqa:T201 + raise SystemExit(1) from exc + def _validate_prerequisites(self) -> None: """Set prereqs for behavior flags -V -D -X -I.""" if ( @@ -1216,6 +1337,8 @@ def _check_for_same_start_yearmonth(self) -> None: ValueError: -s and -e cannot be the same. SystemExit: Exit on error """ + if getattr(self.arg_namespace, "random_window", ""): + return try: start_yearmonth = self.arg_namespace.start_yearmonth start_year = get_year_from_datemonth(start_yearmonth) @@ -1520,6 +1643,37 @@ def _set_args(self) -> None: # noqa:CFQ001 " e.g. -e 2020-00 or -e 2022-04" ), ) + config_args.add_argument( + "-r", + "--random-window", + dest="random_window", + type=str, + metavar="EXPRESSION", + help=( + "select deterministic duration/session tick windows; random " + "selection requires --random-seed, while session expressions " + "with both -s and -e return all matching occurrences" + ), + ) + config_args.add_argument( + "--random-seed", + dest="random_seed", + type=_non_negative_int, + metavar="INTEGER", + help="seed required for reproducible random-window selection", + ) + config_args.add_argument( + "-z", + "--timezone", + "--output-timezone", + dest="output_timezone", + type=str, + metavar="IANA_ZONE", + help=( + "append datetime_local to API results in an IANA timezone; " + "canonical cache and Influx timestamps remain UTC" + ), + ) influx_args.add_argument( "-I", "--import_to_influxdb", @@ -1776,6 +1930,16 @@ def _set_args(self) -> None: # noqa:CFQ001 "rendering evidence" ), ) + quality_args.add_argument( + "--quality-preflight-validation-evidence", + dest="quality_preflight_validation_evidence_path", + type=str, + metavar="PATH", + help=( + "write bounded machine-readable validation evidence to PATH " + "and reference it from quality preflight evidence" + ), + ) quality_args.add_argument( "--quality-preflight-evidence", dest="quality_preflight_evidence_path", @@ -1912,11 +2076,15 @@ def _sanitize_input(self) -> None: # noqa:DAR401 if self.arg_namespace.from_api: args = self._clean_from_api_args() + self._explicit_cli_args = () + self._config_file_args = () self.parse_args(args, namespace=self.arg_namespace) else: # Get the args from sys.argv cli_args = sys.argv[1:] config_args = self._config_args_from_cli(cli_args) + self._explicit_cli_args = tuple(cli_args) + self._config_file_args = tuple(config_args) self.parse_args( [*config_args, *cli_args], namespace=self.arg_namespace, @@ -1926,6 +2094,7 @@ def _sanitize_input(self) -> None: # noqa:DAR401 self._adjust_for_repo_data_request() self._check_quality_mode() self._check_datetime_input() + self._check_random_window_input() self._check_for_ascii_if_influx() self._check_for_ascii_if_api() self._check_for_supported_format_timeframe_combination() diff --git a/src/histdatacom/cli_config.py b/src/histdatacom/cli_config.py index 71a53c3d..470bcef0 100644 --- a/src/histdatacom/cli_config.py +++ b/src/histdatacom/cli_config.py @@ -23,6 +23,7 @@ class CliConfigError(ValueError): "groups", "jobs", "quality", + "reconstruction", "runtime", "worker", } @@ -72,6 +73,12 @@ class CliConfigError(ValueError): "quality_preflight_validation_report_path": ( "quality_preflight_validation_report_path" ), + "quality_preflight_validation_evidence": ( + "quality_preflight_validation_evidence_path" + ), + "quality_preflight_validation_evidence_path": ( + "quality_preflight_validation_evidence_path" + ), "quality_target": "quality_paths", "quality_targets": "quality_paths", "quality_check_groups": "quality_check_groups", @@ -104,6 +111,9 @@ class CliConfigError(ValueError): "symbol_group": "pair_groups", "symbol_groups": "pair_groups", "keep_runtime": "orchestration_keep_runtime", + "random": "random_window", + "random_period": "random_window", + "timezone": "output_timezone", "repo_quality": "repo_quality_refresh", "schedule": "schedule_key", "verbose": "verbosity", @@ -112,6 +122,7 @@ class CliConfigError(ValueError): "analytics_command": "command", "job_command": "command", "jobs_command": "command", + "reconstruction_command": "command", "runtime_command": "command", "subcommand": "command", "worker_command": "command", @@ -161,6 +172,8 @@ class CliConfigError(ValueError): "by": "--by", "start_yearmonth": "--start_yearmonth", "end_yearmonth": "--end_yearmonth", + "random_window": "--random-window", + "random_seed": "--random-seed", "batch_size": "--batch_size", "cpu_utilization": "--cpu_utilization", "data_directory": "--data-directory", @@ -179,6 +192,9 @@ class CliConfigError(ValueError): "quality_preflight_validation_report_path": ( "--quality-preflight-validation-report" ), + "quality_preflight_validation_evidence_path": ( + "--quality-preflight-validation-evidence" + ), "quality_preflight_sample_size": "--quality-preflight-sample-size", "quality_profile_path": "--quality-profile", "quality_profile_preview_format": "--quality-profile-preview-format", @@ -192,6 +208,7 @@ class CliConfigError(ValueError): "request_bundle_out": "--request-bundle-out", "request_json_out": "--request-json-out", "schedule_key": "--schedule-key", + "output_timezone": "--timezone", } _LIST_ARGS = { "pair_groups": "--pair-groups", @@ -214,7 +231,15 @@ class CliConfigError(ValueError): | _CONTROL_BOOL_KEYS | {"verbosity"} ) -_ANALYTICS_COMMANDS = {"feed-regimes"} +_ANALYTICS_COMMANDS = { + "cftc-positioning-corpus", + "feed-epochs-v2", + "feed-regimes", + "market-context-corpus", + "modern-reference-motif-library", + "observation-calibrate-v2", + "reverse-degradation-benchmark-corpus", +} _ANALYTICS_ALIASES = { **_COMMAND_KEY_ALIASES, "path": "paths", @@ -225,12 +250,71 @@ class CliConfigError(ValueError): "json": "--json", } _ANALYTICS_SCALAR_ARGS = { + "active_gap_cap_ms": "--active-gap-cap-ms", + "activity_bin_ms": "--activity-bin-ms", + "artifact_dir": "--artifact-dir", + "benchmark_manifest": "--benchmark-manifest", + "boundary_match_tolerance_periods": "--boundary-match-tolerance-periods", "bucket": "--bucket", + "burst_interval_ms": "--burst-interval-ms", + "calibration_period": "--calibration-period", + "definition": "--definition", + "cftc_positioning_corpus": "--cftc-positioning-corpus", + "evidence": "--evidence", + "epoch_artifact": "--epoch-artifact", + "max_evidence": "--max-evidence", + "max_fragments": "--max-fragments", + "max_matches": "--max-matches", + "end_date": "--end-date", + "max_events_per_window": "--max-events-per-window", + "max_events_per_symbol": "--max-events-per-symbol", + "max_events": "--max-events", + "max_ons_pages_per_query": "--max-ons-pages-per-query", + "max_pages_per_dataset": "--max-pages-per-dataset", + "max_peak_memory_bytes": "--max-peak-memory-bytes", + "max_artifact_bytes": "--max-artifact-bytes", + "max_runtime_seconds": "--max-runtime-seconds", + "max_rows": "--max-rows", + "max_source_bytes": "--max-source-bytes", + "max_response_bytes": "--max-response-bytes", + "max_total_source_bytes": "--max-total-source-bytes", + "max_sensitivity_runs": "--max-sensitivity-runs", + "min_boundary_support": "--min-boundary-support", + "min_change_score": "--min-change-score", + "min_evidence_periods": "--min-evidence-periods", + "min_feature_coverage": "--min-feature-coverage", + "min_symbol_count": "--min-symbol-count", + "min_segment_periods": "--min-segment-periods", + "minimum_events_per_window": "--minimum-events-per-window", + "minimum_events_per_symbol": "--minimum-events-per-symbol", + "neighbor_guard_seconds": "--neighbor-guard-seconds", "quiet_gap_ms": "--quiet-gap-ms", + "penalty_multiplier": "--penalty-multiplier", "report": "--report", + "robust_clip": "--robust-clip", + "final_holdout_period": "--final-holdout-period", + "validation_period": "--validation-period", + "operator_catalog": "--operator-catalog", + "observation_campaign": "--observation-campaign", + "page_size": "--page-size", + "previous_corpus": "--previous-corpus", + "start_date": "--start-date", + "source_root": "--source-root", + "market_context_corpus": "--market-context-corpus", + "gate_policy_commit": "--gate-policy-commit", + "max_staleness_days": "--max-staleness-days", + "timeout_seconds": "--timeout-seconds", + "window_duration_seconds": "--window-duration-seconds", + "windows_per_split": "--windows-per-split", + "windows_per_period": "--windows-per-period", } _ANALYTICS_LIST_ARGS = { + "features": "--features", + "ons_queries": "--ons-query", "paths": "--target", + "sessions": "--sessions", + "sources": "--sources", + "train_periods": "--train-periods", } _ANALYTICS_ALLOWED_KEYS = ( {"command", "verbosity"} @@ -282,6 +366,57 @@ class CliConfigError(ValueError): | set(_CLEANUP_SCALAR_ARGS) | set(_CLEANUP_LIST_ARGS) ) +_RECONSTRUCTION_COMMANDS = { + "cancel", + "outputs", + "plan", + "preflight", + "preview", + "replay", + "request", + "resume", + "run", + "status", +} +_RECONSTRUCTION_ALIASES = { + **_COMMAND_KEY_ALIASES, + "acknowledge_nonclaim": "acknowledge_scientific_nonclaim", + "allow_partial": "allow_refusals", + "manifest_path": "manifest", + "plan_path": "plan", + "receipt_path": "receipt", + "request_path": "request", + "spec_path": "spec", +} +_RECONSTRUCTION_GLOBAL_TRUE_FLAG_ARGS = { + "json": "--json", + "start_runtime": "--start-runtime", +} +_RECONSTRUCTION_TRUE_FLAG_ARGS = { + "acknowledge_scientific_nonclaim": ("--acknowledge-scientific-nonclaim"), + "allow_refusals": "--allow-refusals", + "local": "--local", + "offline": "--offline", + "submit_only": "--submit-only", +} +_RECONSTRUCTION_SCALAR_ARGS = { + "information_mode": "--information-mode", + "limit": "--limit", + "manifest": "--manifest", + "output": "--output", + "plan": "--plan", + "reason": "--reason", + "receipt": "--receipt", + "request": "--request", + "spec": "--spec", + "window_id": "--window-id", +} +_RECONSTRUCTION_ALLOWED_KEYS = ( + {"command"} + | set(_RECONSTRUCTION_GLOBAL_TRUE_FLAG_ARGS) + | set(_RECONSTRUCTION_TRUE_FLAG_ARGS) + | set(_RECONSTRUCTION_SCALAR_ARGS) +) _QUALITY_COMMANDS = { "catalog", "doctor-evidence", @@ -290,6 +425,10 @@ class CliConfigError(ValueError): "fingerprint-discovery", "fingerprint-schema", "inspect-evidence", + "synthetic-fingerprint-validate", + "synthetic-generate", + "synthetic-tick-generate", + "synthetic-validate", "remediation-audit", "remediation-catalog", } @@ -308,6 +447,20 @@ class CliConfigError(ValueError): "quality_path": "target_root", "quality_profile": "quality_profile_path", "quality_profile_path": "quality_profile_path", + "reference_report": "reference_report", + "reference_cache": "reference_cache", + "candidate_report": "candidate_report", + "output_cache": "output_cache", + "block_size": "block_size", + "minimum_reference_rows": "minimum_reference_rows", + "max_reference_rows": "max_reference_rows", + "max_generated_rows": "max_generated_rows", + "max_abs_log_return": "max_abs_log_return", + "rounding_digits": "rounding_digits", + "diagnostic_sample_limit": "diagnostic_sample_limit", + "anchor_mode": "anchor_mode", + "seed": "seed", + "mismatch_limit": "mismatch_limit", "quality_preflight_evidence": "evidence_path", "quality_preflight_evidence_max_age": ("evidence_max_age_seconds"), "quality_preflight_evidence_max_age_seconds": ("evidence_max_age_seconds"), @@ -323,23 +476,41 @@ class CliConfigError(ValueError): "symbol_groups": "pair_groups", "target": "target_root", "target_axis_limit": "target_axis_limit", + "target_limit": "target_limit", } _QUALITY_TRUE_FLAG_ARGS = { "allow_stale_evidence": "--quality-preflight-evidence-stale-ok", "json": "--json", "verify": "--verify", + "overwrite_output": "--overwrite-output", + "overwrite_synthetic": "--overwrite-synthetic", } _QUALITY_SCALAR_ARGS = { "code_limit": "--code-limit", + "candidate_report": "--candidate-report", + "reference_cache": "--reference-cache", + "output_cache": "--output-cache", + "block_size": "--block-size", + "minimum_reference_rows": "--minimum-reference-rows", + "max_reference_rows": "--max-reference-rows", + "max_generated_rows": "--max-generated-rows", + "max_abs_log_return": "--max-abs-log-return", + "rounding_digits": "--rounding-digits", + "diagnostic_sample_limit": "--diagnostic-sample-limit", + "anchor_mode": "--anchor-mode", + "seed": "--seed", "evidence_max_age_seconds": ( "--quality-preflight-evidence-max-age-seconds" ), "evidence_path": "--evidence", "quality_profile_path": "--quality-profile", + "reference_report": "--reference-report", + "mismatch_limit": "--mismatch-limit", "rule_limit": "--rule-limit", "source_limit": "--source-limit", "target_root": "--target", "target_axis_limit": "--target-axis-limit", + "target_limit": "--target-limit", } _QUALITY_LIST_ARGS = { "formats": "--formats", @@ -682,6 +853,23 @@ def configured_quality_argv(args: Sequence[str]) -> list[str]: ) +def configured_reconstruction_argv(args: Sequence[str]) -> list[str]: + """Return reconstruction argv with YAML defaults injected.""" + return _configured_subcommand_argv( + args, + section_name="reconstruction", + commands=_RECONSTRUCTION_COMMANDS, + allowed_keys=_RECONSTRUCTION_ALLOWED_KEYS, + aliases=_RECONSTRUCTION_ALIASES, + global_true_flags=_RECONSTRUCTION_GLOBAL_TRUE_FLAG_ARGS, + global_scalar_args={}, + global_list_args={}, + command_true_flags=_RECONSTRUCTION_TRUE_FLAG_ARGS, + command_scalar_args=_RECONSTRUCTION_SCALAR_ARGS, + command_list_args={}, + ) + + def configured_runtime_argv(args: Sequence[str]) -> list[str]: """Return runtime argv with YAML defaults injected.""" config_path = config_path_from_cli_args(args) diff --git a/src/histdatacom/data_analytics/__init__.py b/src/histdatacom/data_analytics/__init__.py index 608a1820..fd6e6678 100644 --- a/src/histdatacom/data_analytics/__init__.py +++ b/src/histdatacom/data_analytics/__init__.py @@ -2,6 +2,26 @@ from __future__ import annotations +from histdatacom.data_analytics.feed_epochs import ( + DEFAULT_FEED_EPOCH_FEATURES, + FEED_EPOCH_ASSIGNMENT_SCHEMA_VERSION, + FEED_EPOCH_BOUNDARY_SCHEMA_VERSION, + FEED_EPOCH_DEFINITION_SCHEMA_VERSION, + FEED_EPOCH_EVIDENCE_SCHEMA_VERSION, + FEED_EPOCH_FIT_CONFIG_SCHEMA_VERSION, + FEED_EPOCH_INTERVAL_SCHEMA_VERSION, + FEED_EPOCH_STABILITY_SCHEMA_VERSION, + FeedEpochAssignmentV1, + FeedEpochBoundaryV1, + FeedEpochDefinitionV1, + FeedEpochEvidenceV1, + FeedEpochFitConfigV1, + FeedEpochIntervalV1, + FeedEpochStabilityV1, + feed_epoch_definition_to_json, + fit_feed_epochs, + write_feed_epoch_definition, +) from histdatacom.data_analytics.feed_regimes import ( ANALYTICS_REPORT_SCHEMA_VERSION, AnalyticsDiscoveryResult, @@ -15,17 +35,83 @@ format_feed_regime_console_summary, write_feed_regime_report, ) +from histdatacom.data_analytics.feed_epochs_v2 import ( + DEFAULT_ACTIVE_TIME_FEATURES, + FEED_EPOCH_ASSIGNMENT_V2_SCHEMA_VERSION, + FEED_EPOCH_BOUNDARY_V2_SCHEMA_VERSION, + FEED_EPOCH_CAMPAIGN_V2_SCHEMA_VERSION, + FEED_EPOCH_DEFINITION_V2_SCHEMA_VERSION, + FEED_EPOCH_EVIDENCE_V2_SCHEMA_VERSION, + FEED_EPOCH_FIT_CONFIG_V2_SCHEMA_VERSION, + FEED_EPOCH_INTERVAL_V2_SCHEMA_VERSION, + FEED_EPOCH_STABILITY_V2_SCHEMA_VERSION, + FeedEpochAssignmentV2, + FeedEpochBoundaryV2, + FeedEpochCampaignV2, + FeedEpochDefinitionV2, + FeedEpochEvidenceV2, + FeedEpochFitConfigV2, + FeedEpochIntervalV2, + FeedEpochStabilityV2, + FeedEpochSymbolDeviationV2, + analyze_active_time_feed_epochs, + fit_active_time_feed_epochs, + read_active_time_feed_epoch_definition, + scan_active_time_evidence, + write_active_time_feed_epoch_campaign, +) __all__ = [ "ANALYTICS_REPORT_SCHEMA_VERSION", "AnalyticsDiscoveryResult", "AnalyticsTarget", + "DEFAULT_ACTIVE_TIME_FEATURES", + "DEFAULT_FEED_EPOCH_FEATURES", + "FEED_EPOCH_ASSIGNMENT_SCHEMA_VERSION", + "FEED_EPOCH_BOUNDARY_SCHEMA_VERSION", + "FEED_EPOCH_DEFINITION_SCHEMA_VERSION", + "FEED_EPOCH_EVIDENCE_SCHEMA_VERSION", + "FEED_EPOCH_FIT_CONFIG_SCHEMA_VERSION", + "FEED_EPOCH_INTERVAL_SCHEMA_VERSION", + "FEED_EPOCH_STABILITY_SCHEMA_VERSION", + "FEED_EPOCH_ASSIGNMENT_V2_SCHEMA_VERSION", + "FEED_EPOCH_BOUNDARY_V2_SCHEMA_VERSION", + "FEED_EPOCH_CAMPAIGN_V2_SCHEMA_VERSION", + "FEED_EPOCH_DEFINITION_V2_SCHEMA_VERSION", + "FEED_EPOCH_EVIDENCE_V2_SCHEMA_VERSION", + "FEED_EPOCH_FIT_CONFIG_V2_SCHEMA_VERSION", + "FEED_EPOCH_INTERVAL_V2_SCHEMA_VERSION", + "FEED_EPOCH_STABILITY_V2_SCHEMA_VERSION", + "FeedEpochAssignmentV1", + "FeedEpochBoundaryV1", + "FeedEpochDefinitionV1", + "FeedEpochEvidenceV1", + "FeedEpochFitConfigV1", + "FeedEpochIntervalV1", + "FeedEpochStabilityV1", + "FeedEpochAssignmentV2", + "FeedEpochBoundaryV2", + "FeedEpochCampaignV2", + "FeedEpochDefinitionV2", + "FeedEpochEvidenceV2", + "FeedEpochFitConfigV2", + "FeedEpochIntervalV2", + "FeedEpochStabilityV2", + "FeedEpochSymbolDeviationV2", "FeedPeriodProfile", "FeedRegimeEra", "FeedRegimeReport", "analyze_feed_regimes", + "analyze_active_time_feed_epochs", "discover_analytics_targets", + "feed_epoch_definition_to_json", "feed_regime_report_to_json", + "fit_feed_epochs", + "fit_active_time_feed_epochs", + "read_active_time_feed_epoch_definition", "format_feed_regime_console_summary", + "write_feed_epoch_definition", "write_feed_regime_report", + "scan_active_time_evidence", + "write_active_time_feed_epoch_campaign", ] diff --git a/src/histdatacom/data_analytics/cli.py b/src/histdatacom/data_analytics/cli.py index 5e6d7f22..973405dc 100644 --- a/src/histdatacom/data_analytics/cli.py +++ b/src/histdatacom/data_analytics/cli.py @@ -5,6 +5,9 @@ import argparse import json import sys +from typing import cast + +from requests import RequestException from histdatacom.cli_config import ( CliConfigError, @@ -17,6 +20,46 @@ format_feed_regime_console_summary, write_feed_regime_report, ) +from histdatacom.data_analytics.feed_epochs import ( + FeedEpochFitConfigV1, + write_feed_epoch_definition, +) +from histdatacom.data_analytics.feed_epochs_v2 import ( + FeedEpochFitConfigV2, + analyze_active_time_feed_epochs, + read_active_time_feed_epoch_definition, + write_active_time_feed_epoch_campaign, +) +from histdatacom.market_context import ( + DEFAULT_MARKET_CONTEXT_SOURCES, + DEFAULT_ONS_QUERIES, + MarketContextFetchProfileV1, + CftcPositioningFetchProfileV1, + build_live_market_context_corpus, + build_live_cftc_positioning_corpus, + read_cftc_positioning_corpus, + write_market_context_corpus, + write_cftc_positioning_corpus, +) +from histdatacom.synthetic.observation_calibration import ( + ObservationCalibrationProfileV2, + calibrate_historical_observation_operators, + read_feed_epoch_evidence_v2, + write_observation_calibration_campaign, +) +from histdatacom.synthetic.benchmark_corpus import ( + PREDECLARED_GATE_COMMIT, + ReverseDegradationCorpusProfileV1, + build_reverse_degradation_benchmark_corpus, + run_reverse_degradation_benchmark_campaign, + write_reverse_degradation_benchmark_artifacts, +) +from histdatacom.synthetic.motif_library import ( + DEFAULT_MODERN_MOTIF_SPLIT_PERIODS, + ModernReferenceMotifProfileV1, + build_modern_reference_motif_library, + write_modern_reference_motif_artifacts, +) from histdatacom.verbosity import configure_logging @@ -65,11 +108,412 @@ def build_parser() -> argparse.ArgumentParser: metavar="PATH", help="write the machine-readable analytics report to PATH", ) + feed.add_argument( + "--epoch-artifact", + default="", + metavar="PATH", + help="write the compact versioned feed-epoch definition to PATH", + ) + feed.add_argument( + "--features", + nargs="+", + default=None, + metavar="NAME", + help="observation-regime fingerprint features used by the fitter", + ) + feed.add_argument( + "--min-evidence-periods", + type=int, + default=None, + metavar="N", + help="minimum distinct periods required for a full fit", + ) + feed.add_argument( + "--min-segment-periods", + type=int, + default=None, + metavar="N", + help="minimum periods on each side of a boundary", + ) + feed.add_argument( + "--min-feature-coverage", + type=float, + default=None, + metavar="RATE", + help="minimum period coverage required for a feature", + ) + feed.add_argument( + "--min-change-score", + type=float, + default=None, + metavar="SCORE", + help="minimum robust adjacent-period change score", + ) + feed.add_argument( + "--min-boundary-support", + type=float, + default=None, + metavar="RATE", + help="minimum deterministic perturbation support for a boundary", + ) + feed.add_argument( + "--boundary-match-tolerance-periods", + type=int, + default=None, + metavar="N", + help="period tolerance used to match perturbed boundaries", + ) + feed.add_argument( + "--max-evidence", + type=int, + default=None, + metavar="N", + help="maximum canonical symbol-period evidence records", + ) + feed.add_argument( + "--max-sensitivity-runs", + type=int, + default=None, + metavar="N", + help="maximum sampling/missingness/feature-removal fits", + ) feed.add_argument( "--json", action="store_true", help="emit the full machine-readable analytics payload", ) + active = subparsers.add_parser( + "feed-epochs-v2", + help="fit active-time multivariate epochs from ASCII tick caches", + ) + active.add_argument( + "--target", + "--path", + dest="paths", + nargs="+", + required=True, + metavar="PATH", + help="local file or directory containing monthly ASCII tick caches", + ) + active.add_argument( + "--artifact-dir", + required=True, + metavar="PATH", + help="write definition, evidence, and campaign JSON artifacts", + ) + active.add_argument( + "--features", + nargs="+", + default=None, + metavar="NAME", + help="active-time features included in the multivariate objective", + ) + active.add_argument("--min-evidence-periods", type=int, default=None) + active.add_argument("--min-segment-periods", type=int, default=None) + active.add_argument("--min-feature-coverage", type=float, default=None) + active.add_argument("--min-symbol-count", type=int, default=None) + active.add_argument("--penalty-multiplier", type=float, default=None) + active.add_argument("--robust-clip", type=float, default=None) + active.add_argument("--min-boundary-support", type=float, default=None) + active.add_argument( + "--boundary-match-tolerance-periods", type=int, default=None + ) + active.add_argument("--active-gap-cap-ms", type=int, default=None) + active.add_argument("--burst-interval-ms", type=int, default=None) + active.add_argument("--activity-bin-ms", type=int, default=None) + active.add_argument("--max-evidence", type=int, default=None) + active.add_argument("--max-sensitivity-runs", type=int, default=None) + active.add_argument( + "--json", + action="store_true", + help="emit compact campaign and artifact metadata", + ) + calibration = subparsers.add_parser( + "observation-calibrate-v2", + help="fit and hold out historical observation operators", + ) + calibration.add_argument( + "--definition", + required=True, + metavar="PATH", + help="stable feed-epochs-v2 definition artifact", + ) + calibration.add_argument( + "--evidence", + required=True, + metavar="PATH", + help="feed-epochs-v2 evidence artifact with local cache paths", + ) + calibration.add_argument( + "--artifact-dir", + required=True, + metavar="PATH", + help="write operator, fit evidence, and campaign artifacts", + ) + calibration.add_argument( + "--sessions", + nargs="+", + default=("asia", "london", "new_york"), + metavar="NAME", + help="bounded UTC session windows used for real evaluation", + ) + calibration.add_argument("--calibration-period", default="") + calibration.add_argument("--validation-period", default="") + calibration.add_argument("--final-holdout-period", default="") + calibration.add_argument("--max-events-per-window", type=int, default=4096) + calibration.add_argument( + "--minimum-events-per-window", type=int, default=512 + ) + calibration.add_argument( + "--max-source-bytes", type=int, default=2 * 1024**3 + ) + calibration.add_argument("--max-runtime-seconds", type=float, default=600.0) + calibration.add_argument( + "--max-peak-memory-bytes", type=int, default=2 * 1024**3 + ) + calibration.add_argument( + "--json", + action="store_true", + help="emit compact calibration and artifact metadata", + ) + context = subparsers.add_parser( + "market-context-corpus", + help="build an immutable official-source point-in-time event corpus", + ) + context.add_argument( + "--artifact-dir", + required=True, + metavar="PATH", + help="write content-addressed source, timeline, and corpus artifacts", + ) + context.add_argument("--start-date", required=True, metavar="YYYY-MM-DD") + context.add_argument("--end-date", required=True, metavar="YYYY-MM-DD") + context.add_argument( + "--sources", + nargs="+", + choices=DEFAULT_MARKET_CONTEXT_SOURCES, + default=DEFAULT_MARKET_CONTEXT_SOURCES, + metavar="NAME", + help="approved source families to acquire", + ) + context.add_argument( + "--ons-query", + dest="ons_queries", + nargs="+", + default=DEFAULT_ONS_QUERIES, + metavar="TEXT", + help="bounded ONS release-search queries", + ) + context.add_argument( + "--operator-catalog", + default="", + metavar="PATH", + help="optional operator-maintained catalog (default: packaged catalog)", + ) + context.add_argument("--timeout-seconds", type=float, default=30.0) + context.add_argument("--max-response-bytes", type=int, default=16 * 1024**2) + context.add_argument( + "--max-total-source-bytes", type=int, default=64 * 1024**2 + ) + context.add_argument("--max-ons-pages-per-query", type=int, default=8) + context.add_argument("--max-events", type=int, default=4096) + context.add_argument("--max-runtime-seconds", type=float, default=300.0) + context.add_argument( + "--json", + action="store_true", + help="emit compact corpus and artifact metadata", + ) + positioning = subparsers.add_parser( + "cftc-positioning-corpus", + help="build replayable CFTC weekly positioning state", + ) + positioning.add_argument( + "--artifact-dir", + required=True, + metavar="PATH", + help="write content-addressed source, corpus, and audit artifacts", + ) + positioning.add_argument( + "--start-date", required=True, metavar="YYYY-MM-DD" + ) + positioning.add_argument("--end-date", required=True, metavar="YYYY-MM-DD") + positioning.add_argument("--page-size", type=int, default=1000) + positioning.add_argument("--max-pages-per-dataset", type=int, default=128) + positioning.add_argument("--timeout-seconds", type=float, default=45.0) + positioning.add_argument( + "--max-response-bytes", type=int, default=32 * 1024**2 + ) + positioning.add_argument( + "--max-total-source-bytes", type=int, default=256 * 1024**2 + ) + positioning.add_argument("--max-rows", type=int, default=100_000) + positioning.add_argument("--max-runtime-seconds", type=float, default=600.0) + positioning.add_argument( + "--max-peak-memory-bytes", type=int, default=2 * 1024**3 + ) + positioning.add_argument("--max-staleness-days", type=int, default=14) + positioning.add_argument( + "--previous-corpus", + default="", + metavar="PATH", + help="write a logical-row refresh/restatement diff against PATH", + ) + positioning.add_argument( + "--json", + action="store_true", + help="emit compact corpus and artifact metadata", + ) + motif_library = subparsers.add_parser( + "modern-reference-motif-library", + help="build and qualify the real modern empirical motif library", + ) + motif_library.add_argument( + "--source-root", + required=True, + metavar="PATH", + help="root containing symbol/year/month/.data Arrow tick caches", + ) + motif_library.add_argument( + "--definition", + required=True, + metavar="PATH", + help="stable feed-epochs-v2 definition artifact", + ) + motif_library.add_argument( + "--market-context-corpus", + required=True, + metavar="PATH", + help="content-addressed market-context corpus artifact", + ) + motif_library.add_argument( + "--cftc-positioning-corpus", + required=True, + metavar="PATH", + help="content-addressed CFTC positioning corpus artifact", + ) + motif_library.add_argument( + "--benchmark-manifest", + required=True, + metavar="PATH", + help="frozen #463 benchmark manifest for real qualification", + ) + motif_library.add_argument( + "--artifact-dir", + required=True, + metavar="PATH", + help="write content-addressed library and qualification artifacts", + ) + motif_library.add_argument( + "--train-periods", + nargs="+", + default=DEFAULT_MODERN_MOTIF_SPLIT_PERIODS["train"], + ) + motif_library.add_argument( + "--calibration-period", + default=DEFAULT_MODERN_MOTIF_SPLIT_PERIODS["calibration"][0], + ) + motif_library.add_argument( + "--validation-period", + default=DEFAULT_MODERN_MOTIF_SPLIT_PERIODS["validation"][0], + ) + motif_library.add_argument( + "--final-holdout-period", + default=DEFAULT_MODERN_MOTIF_SPLIT_PERIODS["final_holdout"][0], + ) + motif_library.add_argument("--windows-per-period", type=int, default=6) + motif_library.add_argument( + "--window-duration-seconds", type=int, default=600 + ) + motif_library.add_argument( + "--minimum-events-per-symbol", type=int, default=32 + ) + motif_library.add_argument("--max-events-per-symbol", type=int, default=96) + motif_library.add_argument("--max-fragments", type=int, default=256) + motif_library.add_argument("--max-matches", type=int, default=64) + motif_library.add_argument( + "--neighbor-guard-seconds", type=int, default=1800 + ) + motif_library.add_argument( + "--max-source-bytes", type=int, default=2 * 1024**3 + ) + motif_library.add_argument( + "--max-runtime-seconds", type=float, default=900.0 + ) + motif_library.add_argument( + "--max-peak-memory-bytes", type=int, default=2 * 1024**3 + ) + motif_library.add_argument( + "--max-artifact-bytes", type=int, default=64 * 1024**2 + ) + motif_library.add_argument( + "--json", + action="store_true", + help="emit compact library, gate, and artifact metadata", + ) + benchmark = subparsers.add_parser( + "reverse-degradation-benchmark-corpus", + help="build and run the immutable real tick reconstruction benchmark", + ) + benchmark.add_argument( + "--source-root", + required=True, + metavar="PATH", + help="root containing symbol/year/month/.data Arrow tick caches", + ) + benchmark.add_argument( + "--definition", + required=True, + metavar="PATH", + help="feed-epochs-v2 definition artifact", + ) + benchmark.add_argument( + "--observation-campaign", + required=True, + metavar="PATH", + help="fitted observation-calibration-v2 campaign artifact", + ) + benchmark.add_argument( + "--market-context-corpus", + required=True, + metavar="PATH", + help="content-addressed market-context corpus artifact", + ) + benchmark.add_argument( + "--cftc-positioning-corpus", + required=True, + metavar="PATH", + help="content-addressed CFTC positioning corpus artifact", + ) + benchmark.add_argument( + "--artifact-dir", + required=True, + metavar="PATH", + help="write content-addressed benchmark evidence artifacts", + ) + benchmark.add_argument("--calibration-period", default="201001") + benchmark.add_argument("--validation-period", default="202401") + benchmark.add_argument("--final-holdout-period", default="202510") + benchmark.add_argument("--windows-per-split", type=int, default=6) + benchmark.add_argument("--window-duration-seconds", type=int, default=600) + benchmark.add_argument("--minimum-events-per-symbol", type=int, default=64) + benchmark.add_argument("--max-events-per-symbol", type=int, default=256) + benchmark.add_argument("--neighbor-guard-seconds", type=int, default=1800) + benchmark.add_argument("--max-source-bytes", type=int, default=2 * 1024**3) + benchmark.add_argument("--max-runtime-seconds", type=float, default=900.0) + benchmark.add_argument( + "--max-peak-memory-bytes", type=int, default=2 * 1024**3 + ) + benchmark.add_argument( + "--max-artifact-bytes", type=int, default=64 * 1024**2 + ) + benchmark.add_argument( + "--gate-policy-commit", default=PREDECLARED_GATE_COMMIT + ) + benchmark.add_argument( + "--json", + action="store_true", + help="emit compact corpus, campaign, decision, and artifact metadata", + ) return parser @@ -83,27 +527,451 @@ def main(argv: list[str] | None = None) -> int: print(f"config error: {exc}", file=sys.stderr) # noqa:T201 return 1 configure_logging(args.verbosity) + if args.analytics_command == "modern-reference-motif-library": + try: + motif_build = build_modern_reference_motif_library( + args.source_root, + feed_epoch_definition_path=args.definition, + market_context_corpus_path=args.market_context_corpus, + cftc_positioning_corpus_path=args.cftc_positioning_corpus, + benchmark_manifest_path=args.benchmark_manifest, + profile=ModernReferenceMotifProfileV1( + split_periods={ + "train": tuple(args.train_periods), + "calibration": (args.calibration_period,), + "validation": (args.validation_period,), + "final_holdout": (args.final_holdout_period,), + }, + synchronized_windows_per_period=args.windows_per_period, + window_duration_seconds=args.window_duration_seconds, + minimum_events_per_symbol=args.minimum_events_per_symbol, + max_events_per_symbol=args.max_events_per_symbol, + max_fragments=args.max_fragments, + max_matches=args.max_matches, + neighbor_guard_seconds=args.neighbor_guard_seconds, + max_source_bytes=args.max_source_bytes, + max_runtime_seconds=args.max_runtime_seconds, + max_peak_memory_bytes=args.max_peak_memory_bytes, + max_artifact_bytes=args.max_artifact_bytes, + ), + ) + motif_artifacts = write_modern_reference_motif_artifacts( + motif_build, args.artifact_dir + ) + except (OSError, RuntimeError, ValueError) as exc: + print( + f"modern motif library error: {exc}", file=sys.stderr + ) # noqa:T201 + return 1 + eligible = bool( + motif_build.qualification["candidate_promotion_eligible"] + and all(motif_build.qualification["real_window_contracts"].values()) + ) + motif_payload = { + "schema_version": motif_build.manifest["schema_version"], + "library_id": motif_build.library_id, + "index_id": motif_build.index.index_id, + "fragment_count": len(motif_build.index.fragments), + "candidate_promotion_eligible": motif_build.qualification[ + "candidate_promotion_eligible" + ], + "real_window_contracts": motif_build.qualification[ + "real_window_contracts" + ], + "artifacts": { + name: artifact.to_dict() + for name, artifact in sorted(motif_artifacts.items()) + }, + } + if args.json: + print( + json.dumps(motif_payload, indent=2, sort_keys=True) + ) # noqa:T201 + else: + print( # noqa:T201 + "Modern reference-motif library\n" + f"fragments: {len(motif_build.index.fragments)}\n" + f"candidate gate: {'pass' if eligible else 'fail'}\n" + f"manifest: {motif_artifacts['manifest'].path}\n" + f"qualification: {motif_artifacts['qualification'].path}" + ) + return 0 if eligible else 2 + if args.analytics_command == "reverse-degradation-benchmark-corpus": + try: + benchmark_corpus = build_reverse_degradation_benchmark_corpus( + args.source_root, + feed_epoch_definition_path=args.definition, + observation_campaign_path=args.observation_campaign, + market_context_corpus_path=args.market_context_corpus, + cftc_positioning_corpus_path=args.cftc_positioning_corpus, + profile=ReverseDegradationCorpusProfileV1( + split_periods={ + "calibration": args.calibration_period, + "validation": args.validation_period, + "final_holdout": args.final_holdout_period, + }, + synchronized_windows_per_split=args.windows_per_split, + window_duration_seconds=args.window_duration_seconds, + minimum_events_per_symbol=args.minimum_events_per_symbol, + max_events_per_symbol=args.max_events_per_symbol, + neighbor_guard_seconds=args.neighbor_guard_seconds, + max_source_bytes=args.max_source_bytes, + max_runtime_seconds=args.max_runtime_seconds, + max_peak_memory_bytes=args.max_peak_memory_bytes, + max_artifact_bytes=args.max_artifact_bytes, + ), + gate_policy_commit=args.gate_policy_commit, + ) + benchmark_campaign, motif_index = ( + run_reverse_degradation_benchmark_campaign( + benchmark_corpus, args.source_root + ) + ) + benchmark_artifacts = write_reverse_degradation_benchmark_artifacts( + benchmark_corpus, + benchmark_campaign, + motif_index, + args.artifact_dir, + ) + except (OSError, RuntimeError, ValueError) as exc: + print( + f"reverse-degradation benchmark error: {exc}", + file=sys.stderr, + ) # noqa:T201 + return 1 + benchmark_payload = { + "schema_version": benchmark_campaign.schema_version, + "corpus_id": benchmark_corpus.corpus_id, + "campaign_id": benchmark_campaign.campaign_id, + "motif_index_id": motif_index.index_id, + "source_count": len(benchmark_corpus.sources), + "window_count": len(benchmark_corpus.windows), + "runtime_seconds": benchmark_campaign.runtime_seconds, + "peak_memory_bytes": benchmark_campaign.peak_memory_bytes, + "campaign_promotion_eligible": ( + benchmark_campaign.campaign_gate_decision.promotion_eligible + ), + "candidate_decisions": { + item.method_name: { + "promotion_eligible": item.gate_decision.promotion_eligible, + "provisional": item.provisional, + "decision_id": item.gate_decision.decision_id, + } + for item in benchmark_campaign.candidate_reports + }, + "artifacts": { + name: artifact.to_dict() + for name, artifact in sorted(benchmark_artifacts.items()) + }, + } + if args.json: + print( + json.dumps(benchmark_payload, indent=2, sort_keys=True) + ) # noqa:T201 + else: + print( # noqa:T201 + "Real reverse-degradation benchmark\n" + f"sources: {len(benchmark_corpus.sources)}\n" + f"synchronized windows: {len(benchmark_corpus.windows)}\n" + f"campaign gate: " + f"{'pass' if benchmark_campaign.campaign_gate_decision.promotion_eligible else 'fail'}\n" + f"manifest: {benchmark_artifacts['manifest'].path}\n" + f"scorecard: {benchmark_artifacts['scorecard'].path}" + ) + return ( + 0 + if benchmark_campaign.campaign_gate_decision.promotion_eligible + else 2 + ) + if args.analytics_command == "cftc-positioning-corpus": + try: + positioning_build = build_live_cftc_positioning_corpus( + CftcPositioningFetchProfileV1( + start_date=args.start_date, + end_date=args.end_date, + page_size=args.page_size, + max_pages_per_dataset=args.max_pages_per_dataset, + timeout_seconds=args.timeout_seconds, + max_response_bytes=args.max_response_bytes, + max_total_source_bytes=args.max_total_source_bytes, + max_rows=args.max_rows, + max_runtime_seconds=args.max_runtime_seconds, + max_peak_memory_bytes=args.max_peak_memory_bytes, + max_staleness_days=args.max_staleness_days, + ) + ) + previous = ( + read_cftc_positioning_corpus(args.previous_corpus) + if args.previous_corpus + else None + ) + positioning_artifacts = write_cftc_positioning_corpus( + positioning_build, + args.artifact_dir, + previous_corpus=previous, + ) + except (OSError, RequestException, ValueError) as exc: + print( + f"CFTC positioning corpus error: {exc}", file=sys.stderr + ) # noqa:T201 + return 1 + positioning_corpus = positioning_build.corpus + positioning_payload = { + "schema_version": positioning_corpus.schema_version, + "corpus_id": positioning_corpus.corpus_id, + "snapshot_count": len(positioning_corpus.snapshots), + "source_count": len(positioning_corpus.sources), + "source_bytes": sum( + cast(int, item["size_bytes"]) + for item in positioning_corpus.sources + ), + "duplicate_key_count": positioning_corpus.duplicate_key_count, + "runtime_seconds": positioning_corpus.runtime_seconds, + "peak_memory_bytes": positioning_corpus.peak_memory_bytes, + "coverage_slice_count": len(positioning_corpus.coverage), + "archive_consistency": [ + item.to_dict() + for item in positioning_corpus.archive_consistency + ], + "artifacts": { + name: artifact.to_dict() + for name, artifact in sorted(positioning_artifacts.items()) + }, + } + if args.json: + print( + json.dumps(positioning_payload, indent=2, sort_keys=True) + ) # noqa:T201 + else: + print( # noqa:T201 + "CFTC positioning corpus\n" + f"sources: {len(positioning_corpus.sources)}\n" + f"source bytes: {positioning_payload['source_bytes']}\n" + f"snapshots: {len(positioning_corpus.snapshots)}\n" + f"corpus: {positioning_artifacts['corpus'].path}" + ) + return 0 + if args.analytics_command == "market-context-corpus": + try: + build = build_live_market_context_corpus( + MarketContextFetchProfileV1( + start_date=args.start_date, + end_date=args.end_date, + sources=tuple(args.sources), + ons_queries=tuple(args.ons_queries), + timeout_seconds=args.timeout_seconds, + max_response_bytes=args.max_response_bytes, + max_total_source_bytes=args.max_total_source_bytes, + max_ons_pages_per_query=args.max_ons_pages_per_query, + max_events=args.max_events, + max_runtime_seconds=args.max_runtime_seconds, + ), + operator_catalog_path=args.operator_catalog or None, + ) + artifacts = write_market_context_corpus(build, args.artifact_dir) + except (OSError, ValueError) as exc: + print( + f"market-context corpus error: {exc}", file=sys.stderr + ) # noqa:T201 + return 1 + corpus = build.corpus + context_payload = { + "schema_version": corpus.schema_version, + "corpus_id": corpus.corpus_id, + "timeline_id": corpus.timeline.timeline_id, + "event_count": len(corpus.timeline.events), + "source_count": len(corpus.sources), + "source_bytes": sum(item.size_bytes for item in corpus.sources), + "duplicate_event_count": corpus.duplicate_event_count, + "runtime_seconds": corpus.runtime_seconds, + "peak_memory_bytes": corpus.peak_memory_bytes, + "coverage": [item.to_dict() for item in corpus.coverage], + "artifacts": { + name: artifact.to_dict() + for name, artifact in sorted(artifacts.items()) + }, + } + if args.json: + print( + json.dumps(context_payload, indent=2, sort_keys=True) + ) # noqa:T201 + else: + print( # noqa:T201 + "Point-in-time market-context corpus\n" + f"sources: {len(corpus.sources)}\n" + f"source bytes: {context_payload['source_bytes']}\n" + f"events: {len(corpus.timeline.events)}\n" + f"duplicates removed: {corpus.duplicate_event_count}\n" + f"timeline: {artifacts['timeline'].path}\n" + f"corpus: {artifacts['corpus'].path}" + ) + return 0 + if args.analytics_command == "observation-calibrate-v2": + definition = read_active_time_feed_epoch_definition(args.definition) + evidence = read_feed_epoch_evidence_v2(args.evidence) + split_periods = { + "calibration": args.calibration_period, + "validation": args.validation_period, + "final_holdout": args.final_holdout_period, + } + configured_splits = { + key: value for key, value in split_periods.items() if value + } + calibration_campaign = calibrate_historical_observation_operators( + evidence, + epoch_definition=definition, + profile=ObservationCalibrationProfileV2( + split_periods=configured_splits, + sessions=tuple(args.sessions), + max_events_per_window=args.max_events_per_window, + minimum_events_per_window=args.minimum_events_per_window, + max_source_bytes=args.max_source_bytes, + max_runtime_seconds=args.max_runtime_seconds, + max_peak_memory_bytes=args.max_peak_memory_bytes, + ), + ) + artifacts = write_observation_calibration_campaign( + calibration_campaign, args.artifact_dir + ) + calibration_payload = { + "schema_version": calibration_campaign.schema_version, + "campaign_id": calibration_campaign.campaign_id, + "operator_id": calibration_campaign.operator.operator_id, + "readiness_status": calibration_campaign.readiness_status, + "readiness_reasons": list(calibration_campaign.readiness_reasons), + "window_count": len(calibration_campaign.windows), + "runtime_seconds": calibration_campaign.runtime_seconds, + "peak_memory_bytes": calibration_campaign.peak_memory_bytes, + "artifacts": { + name: artifact.to_dict() + for name, artifact in sorted(artifacts.items()) + }, + } + if args.json: + print( # noqa:T201 + json.dumps(calibration_payload, indent=2, sort_keys=True) + ) + else: + print( # noqa:T201 + "Observation calibration v2\n" + f"operator: {calibration_campaign.operator.operator_id}\n" + f"targets: {len(calibration_campaign.targets)}\n" + f"windows: {len(calibration_campaign.windows)}\n" + f"readiness: {calibration_campaign.readiness_status}\n" + f"campaign: {artifacts['campaign'].path}" + ) + return 0 if calibration_campaign.valid_for_application else 2 + if args.analytics_command == "feed-epochs-v2": + epoch_campaign = analyze_active_time_feed_epochs( + args.paths, + config=_fit_v2_config_from_args(args), + ) + artifacts = write_active_time_feed_epoch_campaign( + epoch_campaign, args.artifact_dir + ) + epoch_payload = epoch_campaign.to_dict(include_evidence=False) + epoch_payload["artifacts"] = { + name: artifact.to_dict() + for name, artifact in sorted(artifacts.items()) + } + if args.json: + print( + json.dumps(epoch_payload, indent=2, sort_keys=True) + ) # noqa:T201 + else: + definition = epoch_campaign.definition + print( # noqa:T201 + "Active-time feed epoch fit\n" + f"sources: {epoch_campaign.source_count}\n" + f"common periods: {definition.period_count}\n" + f"symbols: {', '.join(definition.symbols)}\n" + f"boundaries: {len(definition.boundaries)}\n" + f"stability: {definition.stability.status}\n" + f"definition: {artifacts['definition'].path}" + ) + return 0 if args.analytics_command != "feed-regimes": parser.error(f"unsupported analytics command: {args.analytics_command}") + fit_config = _fit_config_from_args(args) report = analyze_feed_regimes( args.paths, bucket=args.bucket, quiet_gap_ms=args.quiet_gap_ms, + fit_config=fit_config, ) artifact = ( write_feed_regime_report(report, args.report) if args.report else None ) + epoch_artifact = ( + write_feed_epoch_definition( + report.epoch_definition, + args.epoch_artifact, + ) + if args.epoch_artifact and report.epoch_definition is not None + else None + ) if args.json: - payload = report.to_dict() + report_payload = report.to_dict() if artifact is not None: - payload["report_artifact"] = artifact.to_dict() - print(json.dumps(payload, indent=2, sort_keys=True)) # noqa:T201 + report_payload["report_artifact"] = artifact.to_dict() + if epoch_artifact is not None: + report_payload["epoch_artifact"] = epoch_artifact.to_dict() + print(json.dumps(report_payload, indent=2, sort_keys=True)) # noqa:T201 else: - print( # noqa:T201 - format_feed_regime_console_summary( - report, - artifact=artifact, - ) - ) + summary = format_feed_regime_console_summary(report, artifact=artifact) + if epoch_artifact is not None: + summary += f"\nepoch artifact: {epoch_artifact.path}" + print(summary) # noqa:T201 return 0 + + +def _fit_config_from_args( + args: argparse.Namespace, +) -> FeedEpochFitConfigV1 | None: + """Return an explicit fit policy only when the operator configured one.""" + values = { + "feature_names": tuple(args.features) if args.features else None, + "min_evidence_periods": args.min_evidence_periods, + "min_segment_periods": args.min_segment_periods, + "min_feature_coverage": args.min_feature_coverage, + "min_change_score": args.min_change_score, + "min_boundary_support": args.min_boundary_support, + "boundary_match_tolerance_periods": ( + args.boundary_match_tolerance_periods + ), + "max_evidence": args.max_evidence, + "max_sensitivity_runs": args.max_sensitivity_runs, + } + configured = { + name: value for name, value in values.items() if value is not None + } + return FeedEpochFitConfigV1(**configured) if configured else None + + +def _fit_v2_config_from_args( + args: argparse.Namespace, +) -> FeedEpochFitConfigV2 | None: + """Return an explicit v2 policy only when the operator configured one.""" + values = { + "feature_names": tuple(args.features) if args.features else None, + "min_evidence_periods": args.min_evidence_periods, + "min_segment_periods": args.min_segment_periods, + "min_feature_coverage": args.min_feature_coverage, + "min_symbol_count": args.min_symbol_count, + "penalty_multiplier": args.penalty_multiplier, + "robust_clip": args.robust_clip, + "min_boundary_support": args.min_boundary_support, + "boundary_match_tolerance_periods": ( + args.boundary_match_tolerance_periods + ), + "active_gap_cap_ms": args.active_gap_cap_ms, + "burst_interval_ms": args.burst_interval_ms, + "activity_bin_ms": args.activity_bin_ms, + "max_evidence": args.max_evidence, + "max_sensitivity_runs": args.max_sensitivity_runs, + } + configured = { + name: value for name, value in values.items() if value is not None + } + return FeedEpochFitConfigV2(**configured) if configured else None diff --git a/src/histdatacom/data_analytics/feed_epochs.py b/src/histdatacom/data_analytics/feed_epochs.py new file mode 100644 index 00000000..0bd84838 --- /dev/null +++ b/src/histdatacom/data_analytics/feed_epochs.py @@ -0,0 +1,2255 @@ +"""Deterministic technological feed-epoch contracts and fitting. + +This module consumes bounded canonical time-series fingerprints. It never +discovers or scans raw tick files. The resulting artifact describes changes +in the technical observation process, including uncertainty and deterministic +stability evidence, without claiming that calendar years are regimes. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from statistics import median +from typing import Any + +from histdatacom.data_quality.fingerprints import ( + TIME_SERIES_FINGERPRINT_SCHEMA_VERSION, +) +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.contracts import canonical_contract_json + +FEED_EPOCH_EVIDENCE_SCHEMA_VERSION = "histdatacom.feed-epoch-evidence.v1" +FEED_EPOCH_FIT_CONFIG_SCHEMA_VERSION = "histdatacom.feed-epoch-fit-config.v1" +FEED_EPOCH_BOUNDARY_SCHEMA_VERSION = "histdatacom.feed-epoch-boundary.v1" +FEED_EPOCH_INTERVAL_SCHEMA_VERSION = "histdatacom.feed-epoch-interval.v1" +FEED_EPOCH_STABILITY_SCHEMA_VERSION = "histdatacom.feed-epoch-stability.v1" +FEED_EPOCH_DEFINITION_SCHEMA_VERSION = "histdatacom.feed-epoch-definition.v1" +FEED_EPOCH_ASSIGNMENT_SCHEMA_VERSION = "histdatacom.feed-epoch-assignment.v1" + +MAX_FEED_EPOCH_EVIDENCE = 4096 +MAX_FEED_EPOCH_FEATURES = 64 +MAX_FEED_EPOCH_SENSITIVITY_RUNS = 256 +_PERIOD_RE = re.compile(r"^\d{4}(?:\d{2})?$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_STABILITY_ANALYSES = ("sampling", "missing_periods", "feature_removal") +_SOURCE_HASH_BASES = { + "canonical_fingerprint_id", + "canonical_fingerprint_aggregate_id", + "persisted_fingerprint_artifact_sha256", +} + +DEFAULT_FEED_EPOCH_FEATURES = ( + "log_tick_rate_per_hour", + "log_median_interarrival_ms", + "log_p95_interarrival_ms", + "minimum_observed_interval_ms", + "price_precision_digits", + "spread_median", + "conditioned_spread_median", + "absolute_spread_change_median", + "stale_repeat_rate", + "burst_rate", + "duplicate_timestamp_rate", + "suspicious_gap_rate", + "source_quality_penalty", +) + +_FEATURE_PROVENANCE = { + "log_tick_rate_per_hour": ("coverage.row_count", "coverage.duration_ms"), + "log_median_interarrival_ms": ( + "microstructure_dynamics.interarrival_ms.median", + ), + "log_p95_interarrival_ms": ( + "microstructure_dynamics.interarrival_ms.quantiles.0.95", + ), + "minimum_observed_interval_ms": ("temporal_topology.min_interval_ms",), + "price_precision_digits": ( + "tick_distribution.precision.column_decimal_place_counts", + ), + "spread_median": ("tick_distribution.spread.median",), + "conditioned_spread_median": ( + "conditional_distributions.by_active_session.*.spread.median", + ), + "absolute_spread_change_median": ( + "microstructure_dynamics.absolute_spread_change.median", + ), + "stale_repeat_rate": ("microstructure_dynamics.stale_quote.repeat_rate",), + "burst_rate": ("microstructure_dynamics.burst.burst_rate",), + "duplicate_timestamp_rate": ( + "temporal_topology.duplicate_timestamp_count", + "temporal_topology.parsed_row_count", + ), + "suspicious_gap_rate": ( + "temporal_topology.suspicious_gap_count", + "temporal_topology.interval_count", + ), + "source_quality_penalty": ( + "fingerprint_audit.section_statuses", + "fingerprint_audit.source_status", + ), +} + + +@dataclass(frozen=True, slots=True) +class FeedEpochEvidenceV1: + """One bounded canonical fingerprint observation for epoch fitting.""" + + symbol: str + period: str + start_timestamp_utc_ms: int + end_timestamp_utc_ms: int + fingerprint_id: str + source_artifact_sha256: str + source_hash_basis: str + source_kind: str + feature_values: Mapping[str, float] + feature_provenance: Mapping[str, tuple[str, ...]] + conditioning: Mapping[str, JSONValue] + quality: Mapping[str, JSONValue] + profile: Mapping[str, JSONValue] + evidence_id: str = "" + schema_version: str = FEED_EPOCH_EVIDENCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_EVIDENCE_SCHEMA_VERSION: + raise ValueError("unsupported feed epoch evidence schema") + symbol = _required_text(self.symbol, "symbol").upper() + period = _required_text(self.period, "period") + if not _valid_period(period): + raise ValueError("period must use YYYY or YYYYMM") + start = _strict_int( + self.start_timestamp_utc_ms, "start_timestamp_utc_ms" + ) + end = _strict_int(self.end_timestamp_utc_ms, "end_timestamp_utc_ms") + if end < start: + raise ValueError("evidence end timestamp precedes start timestamp") + fingerprint_id = _required_sha256_id( + self.fingerprint_id, "fingerprint_id" + ) + source_hash = _required_sha256_id( + self.source_artifact_sha256, + "source_artifact_sha256", + ) + source_kind = _required_text(self.source_kind, "source_kind") + source_hash_basis = _required_text( + self.source_hash_basis, + "source_hash_basis", + ) + if source_hash_basis not in _SOURCE_HASH_BASES: + raise ValueError("unsupported source hash basis") + feature_values = { + _required_text(name, "feature name"): _finite_float(value, name) + for name, value in sorted(self.feature_values.items()) + } + if not feature_values: + raise ValueError("feed epoch evidence requires feature values") + if len(feature_values) > MAX_FEED_EPOCH_FEATURES: + raise ValueError("feed epoch evidence exceeds feature limit") + feature_provenance = { + name: tuple( + _required_text(path, "feature provenance") + for path in self.feature_provenance.get(name, ()) + ) + for name in feature_values + } + if any(not paths for paths in feature_provenance.values()): + raise ValueError("every feed epoch feature requires provenance") + object.__setattr__(self, "symbol", symbol) + object.__setattr__(self, "period", period) + object.__setattr__(self, "start_timestamp_utc_ms", start) + object.__setattr__(self, "end_timestamp_utc_ms", end) + object.__setattr__(self, "fingerprint_id", fingerprint_id) + object.__setattr__(self, "source_artifact_sha256", source_hash) + object.__setattr__(self, "source_hash_basis", source_hash_basis) + object.__setattr__(self, "source_kind", source_kind) + object.__setattr__(self, "feature_values", feature_values) + object.__setattr__(self, "feature_provenance", feature_provenance) + object.__setattr__(self, "conditioning", dict(self.conditioning)) + object.__setattr__(self, "quality", dict(self.quality)) + object.__setattr__(self, "profile", dict(self.profile)) + expected = _stable_id("feed-epoch-evidence", self.identity_payload()) + supplied = str(self.evidence_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "evidence_id does not match deterministic identity" + ) + object.__setattr__(self, "evidence_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the complete deterministic evidence identity payload.""" + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "period": self.period, + "start_timestamp_utc_ms": self.start_timestamp_utc_ms, + "end_timestamp_utc_ms": self.end_timestamp_utc_ms, + "fingerprint_id": self.fingerprint_id, + "source_artifact_sha256": self.source_artifact_sha256, + "source_hash_basis": self.source_hash_basis, + "source_kind": self.source_kind, + "feature_values": dict(self.feature_values), + "feature_provenance": { + name: list(paths) + for name, paths in sorted(self.feature_provenance.items()) + }, + "conditioning": dict(self.conditioning), + "quality": dict(self.quality), + "profile": dict(self.profile), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible evidence.""" + return {**self.identity_payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochEvidenceV1": + """Read strict version-one evidence.""" + provenance = _mapping(data.get("feature_provenance")) + return cls( + schema_version=str(data.get("schema_version", "")), + symbol=str(data.get("symbol", "")), + period=str(data.get("period", "")), + start_timestamp_utc_ms=_strict_int( + data.get("start_timestamp_utc_ms"), + "start_timestamp_utc_ms", + ), + end_timestamp_utc_ms=_strict_int( + data.get("end_timestamp_utc_ms"), + "end_timestamp_utc_ms", + ), + fingerprint_id=str(data.get("fingerprint_id", "")), + source_artifact_sha256=str(data.get("source_artifact_sha256", "")), + source_hash_basis=str(data.get("source_hash_basis", "")), + source_kind=str(data.get("source_kind", "")), + feature_values={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping(data.get("feature_values")).items() + }, + feature_provenance={ + str(name): tuple(str(value) for value in _sequence(paths)) + for name, paths in provenance.items() + }, + conditioning=_mapping(data.get("conditioning")), + quality=_mapping(data.get("quality")), + profile=_mapping(data.get("profile")), + evidence_id=str(data.get("evidence_id", "")), + ) + + @classmethod + def from_fingerprint( + cls, + payload: Mapping[str, Any], + *, + source_artifact_sha256: str | None = None, + ) -> "FeedEpochEvidenceV1": + """Build bounded epoch evidence from one canonical fingerprint.""" + if ( + payload.get("schema_version") + != TIME_SERIES_FINGERPRINT_SCHEMA_VERSION + ): + raise ValueError( + "feed epoch evidence requires a canonical v1 fingerprint" + ) + axis = _mapping(payload.get("target_axis")) + if str(axis.get("data_format", "")).lower() != "ascii": + raise ValueError("feed epoch evidence requires ASCII fingerprints") + if str(axis.get("timeframe", "")).upper() != "T": + raise ValueError("feed epoch evidence requires tick fingerprints") + coverage = _mapping(payload.get("coverage")) + start = _strict_int( + coverage.get("start_timestamp_utc_ms"), + "coverage.start_timestamp_utc_ms", + ) + end = _strict_int( + coverage.get("end_timestamp_utc_ms"), + "coverage.end_timestamp_utc_ms", + ) + fingerprint_id = _required_sha256_id( + payload.get("fingerprint_id"), + "fingerprint_id", + ) + feature_values = _fingerprint_feature_values(payload) + source = _mapping(payload.get("source")) + return cls( + symbol=str(axis.get("symbol", "")), + period=str(axis.get("period", "")), + start_timestamp_utc_ms=start, + end_timestamp_utc_ms=end, + fingerprint_id=fingerprint_id, + source_artifact_sha256=(source_artifact_sha256 or fingerprint_id), + source_hash_basis=( + "persisted_fingerprint_artifact_sha256" + if source_artifact_sha256 is not None + else "canonical_fingerprint_id" + ), + source_kind=str(source.get("kind", "unknown")), + feature_values=feature_values, + feature_provenance={ + name: _FEATURE_PROVENANCE[name] for name in feature_values + }, + conditioning=_fingerprint_conditioning(payload), + quality=_fingerprint_quality(payload), + profile=_fingerprint_profile(payload, feature_values), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochFitConfigV1: + """Deterministic and bounded feed-epoch fitting policy.""" + + feature_names: tuple[str, ...] = DEFAULT_FEED_EPOCH_FEATURES + min_evidence_periods: int = 6 + min_segment_periods: int = 2 + min_feature_coverage: float = 0.75 + min_change_score: float = 0.8 + min_boundary_support: float = 0.6 + boundary_match_tolerance_periods: int = 1 + max_evidence: int = MAX_FEED_EPOCH_EVIDENCE + max_sensitivity_runs: int = 128 + rounding_digits: int = 8 + config_id: str = "" + schema_version: str = FEED_EPOCH_FIT_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_FIT_CONFIG_SCHEMA_VERSION: + raise ValueError("unsupported feed epoch fit config schema") + names = tuple( + dict.fromkeys( + _required_text(name, "feature") for name in self.feature_names + ) + ) + if not names or len(names) > MAX_FEED_EPOCH_FEATURES: + raise ValueError("feature_names must be non-empty and bounded") + unknown = sorted(set(names).difference(DEFAULT_FEED_EPOCH_FEATURES)) + if unknown: + raise ValueError( + "unsupported feed epoch feature(s): " + ", ".join(unknown) + ) + min_periods = _bounded_int( + self.min_evidence_periods, + "min_evidence_periods", + minimum=2, + maximum=MAX_FEED_EPOCH_EVIDENCE, + ) + min_segment = _bounded_int( + self.min_segment_periods, + "min_segment_periods", + minimum=1, + maximum=MAX_FEED_EPOCH_EVIDENCE // 2, + ) + if min_segment * 2 > min_periods: + raise ValueError( + "min_segment_periods cannot exceed half min_evidence_periods" + ) + object.__setattr__(self, "feature_names", names) + object.__setattr__(self, "min_evidence_periods", min_periods) + object.__setattr__(self, "min_segment_periods", min_segment) + object.__setattr__( + self, + "min_feature_coverage", + _bounded_float( + self.min_feature_coverage, "min_feature_coverage", 0.0, 1.0 + ), + ) + object.__setattr__( + self, + "min_change_score", + _bounded_float( + self.min_change_score, "min_change_score", 0.0, 1_000_000.0 + ), + ) + object.__setattr__( + self, + "min_boundary_support", + _bounded_float( + self.min_boundary_support, "min_boundary_support", 0.0, 1.0 + ), + ) + object.__setattr__( + self, + "boundary_match_tolerance_periods", + _bounded_int( + self.boundary_match_tolerance_periods, + "boundary_match_tolerance_periods", + minimum=0, + maximum=24, + ), + ) + object.__setattr__( + self, + "max_evidence", + _bounded_int( + self.max_evidence, + "max_evidence", + minimum=min_periods, + maximum=MAX_FEED_EPOCH_EVIDENCE, + ), + ) + object.__setattr__( + self, + "max_sensitivity_runs", + _bounded_int( + self.max_sensitivity_runs, + "max_sensitivity_runs", + minimum=3, + maximum=MAX_FEED_EPOCH_SENSITIVITY_RUNS, + ), + ) + object.__setattr__( + self, + "rounding_digits", + _bounded_int( + self.rounding_digits, "rounding_digits", minimum=0, maximum=12 + ), + ) + expected = _stable_id("feed-epoch-config", self.identity_payload()) + supplied = str(self.config_id or "").strip() + if supplied and supplied != expected: + raise ValueError("config_id does not match deterministic identity") + object.__setattr__(self, "config_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return config fields that affect fitting semantics.""" + return { + "schema_version": self.schema_version, + "feature_names": list(self.feature_names), + "min_evidence_periods": self.min_evidence_periods, + "min_segment_periods": self.min_segment_periods, + "min_feature_coverage": self.min_feature_coverage, + "min_change_score": self.min_change_score, + "min_boundary_support": self.min_boundary_support, + "boundary_match_tolerance_periods": self.boundary_match_tolerance_periods, + "max_evidence": self.max_evidence, + "max_sensitivity_runs": self.max_sensitivity_runs, + "rounding_digits": self.rounding_digits, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible configuration.""" + return {**self.identity_payload(), "config_id": self.config_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochFitConfigV1": + """Read a strict fit configuration.""" + return cls( + schema_version=str(data.get("schema_version", "")), + feature_names=tuple( + str(value) for value in _sequence(data.get("feature_names")) + ), + min_evidence_periods=_strict_int( + data.get("min_evidence_periods"), "min_evidence_periods" + ), + min_segment_periods=_strict_int( + data.get("min_segment_periods"), "min_segment_periods" + ), + min_feature_coverage=_finite_float( + data.get("min_feature_coverage"), "min_feature_coverage" + ), + min_change_score=_finite_float( + data.get("min_change_score"), "min_change_score" + ), + min_boundary_support=_finite_float( + data.get("min_boundary_support"), "min_boundary_support" + ), + boundary_match_tolerance_periods=_strict_int( + data.get("boundary_match_tolerance_periods"), + "boundary_match_tolerance_periods", + ), + max_evidence=_strict_int(data.get("max_evidence"), "max_evidence"), + max_sensitivity_runs=_strict_int( + data.get("max_sensitivity_runs"), + "max_sensitivity_runs", + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + config_id=str(data.get("config_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochBoundaryV1: + """One uncertain and stability-scored technological boundary.""" + + boundary_id: str + left_period: str + right_period: str + central_timestamp_utc_ms: int + uncertainty_start_utc_ms: int + uncertainty_end_utc_ms: int + change_score: float + confidence: float + support: float + support_by_analysis: Mapping[str, float] + contributing_features: tuple[str, ...] + transition_label: str + schema_version: str = FEED_EPOCH_BOUNDARY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_BOUNDARY_SCHEMA_VERSION: + raise ValueError("unsupported feed epoch boundary schema") + start = _strict_int( + self.uncertainty_start_utc_ms, "uncertainty_start_utc_ms" + ) + central = _strict_int( + self.central_timestamp_utc_ms, "central_timestamp_utc_ms" + ) + end = _strict_int(self.uncertainty_end_utc_ms, "uncertainty_end_utc_ms") + if not start <= central <= end: + raise ValueError( + "boundary central timestamp must lie inside uncertainty interval" + ) + object.__setattr__(self, "uncertainty_start_utc_ms", start) + object.__setattr__(self, "central_timestamp_utc_ms", central) + object.__setattr__(self, "uncertainty_end_utc_ms", end) + for name in ("left_period", "right_period"): + if not _valid_period(str(getattr(self, name))): + raise ValueError(f"{name} must use YYYY or YYYYMM") + object.__setattr__( + self, "boundary_id", _required_text(self.boundary_id, "boundary_id") + ) + object.__setattr__( + self, + "change_score", + _finite_float(self.change_score, "change_score"), + ) + object.__setattr__( + self, + "confidence", + _bounded_float(self.confidence, "confidence", 0.0, 1.0), + ) + object.__setattr__( + self, "support", _bounded_float(self.support, "support", 0.0, 1.0) + ) + object.__setattr__( + self, + "support_by_analysis", + { + str(name): _bounded_float(value, str(name), 0.0, 1.0) + for name, value in sorted(self.support_by_analysis.items()) + }, + ) + object.__setattr__( + self, "contributing_features", tuple(self.contributing_features) + ) + object.__setattr__( + self, + "transition_label", + _required_text(self.transition_label, "transition_label"), + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic boundary metadata.""" + return { + "schema_version": self.schema_version, + "boundary_id": self.boundary_id, + "left_period": self.left_period, + "right_period": self.right_period, + "central_timestamp_utc_ms": self.central_timestamp_utc_ms, + "uncertainty_start_utc_ms": self.uncertainty_start_utc_ms, + "uncertainty_end_utc_ms": self.uncertainty_end_utc_ms, + "change_score": self.change_score, + "confidence": self.confidence, + "support": self.support, + "support_by_analysis": dict(self.support_by_analysis), + "contributing_features": list(self.contributing_features), + "transition_label": self.transition_label, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochBoundaryV1": + """Read a strict boundary contract.""" + return cls( + schema_version=str(data.get("schema_version", "")), + boundary_id=str(data.get("boundary_id", "")), + left_period=str(data.get("left_period", "")), + right_period=str(data.get("right_period", "")), + central_timestamp_utc_ms=_strict_int( + data.get("central_timestamp_utc_ms"), "central_timestamp_utc_ms" + ), + uncertainty_start_utc_ms=_strict_int( + data.get("uncertainty_start_utc_ms"), "uncertainty_start_utc_ms" + ), + uncertainty_end_utc_ms=_strict_int( + data.get("uncertainty_end_utc_ms"), "uncertainty_end_utc_ms" + ), + change_score=_finite_float( + data.get("change_score"), "change_score" + ), + confidence=_finite_float(data.get("confidence"), "confidence"), + support=_finite_float(data.get("support"), "support"), + support_by_analysis={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping( + data.get("support_by_analysis") + ).items() + }, + contributing_features=tuple( + str(value) + for value in _sequence(data.get("contributing_features")) + ), + transition_label=str(data.get("transition_label", "")), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochIntervalV1: + """One central technological epoch between uncertain transitions.""" + + epoch_id: str + label: str + period_start: str + period_end: str + start_timestamp_utc_ms: int + end_timestamp_utc_ms: int + evidence_count: int + schema_version: str = FEED_EPOCH_INTERVAL_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_INTERVAL_SCHEMA_VERSION: + raise ValueError("unsupported feed epoch interval schema") + start = _strict_int( + self.start_timestamp_utc_ms, "start_timestamp_utc_ms" + ) + end = _strict_int(self.end_timestamp_utc_ms, "end_timestamp_utc_ms") + if end < start: + raise ValueError("epoch end timestamp precedes start timestamp") + for name in ("period_start", "period_end"): + if not _valid_period(str(getattr(self, name))): + raise ValueError(f"{name} must use YYYY or YYYYMM") + if self.period_end < self.period_start: + raise ValueError("epoch period_end precedes period_start") + object.__setattr__(self, "start_timestamp_utc_ms", start) + object.__setattr__(self, "end_timestamp_utc_ms", end) + object.__setattr__( + self, "epoch_id", _required_text(self.epoch_id, "epoch_id") + ) + object.__setattr__(self, "label", _required_text(self.label, "label")) + object.__setattr__( + self, + "evidence_count", + _bounded_int( + self.evidence_count, + "evidence_count", + 1, + MAX_FEED_EPOCH_EVIDENCE, + ), + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic interval metadata.""" + return { + "schema_version": self.schema_version, + "epoch_id": self.epoch_id, + "label": self.label, + "period_start": self.period_start, + "period_end": self.period_end, + "start_timestamp_utc_ms": self.start_timestamp_utc_ms, + "end_timestamp_utc_ms": self.end_timestamp_utc_ms, + "evidence_count": self.evidence_count, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochIntervalV1": + """Read a strict interval contract.""" + return cls( + schema_version=str(data.get("schema_version", "")), + epoch_id=str(data.get("epoch_id", "")), + label=str(data.get("label", "")), + period_start=str(data.get("period_start", "")), + period_end=str(data.get("period_end", "")), + start_timestamp_utc_ms=_strict_int( + data.get("start_timestamp_utc_ms"), "start_timestamp_utc_ms" + ), + end_timestamp_utc_ms=_strict_int( + data.get("end_timestamp_utc_ms"), "end_timestamp_utc_ms" + ), + evidence_count=_strict_int( + data.get("evidence_count"), "evidence_count" + ), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochStabilityV1: + """Deterministic sampling, missingness, and feature sensitivity evidence.""" + + status: str + run_counts: Mapping[str, int] + usable_run_counts: Mapping[str, int] + unstable_boundary_ids: tuple[str, ...] = () + limitations: tuple[str, ...] = () + schema_version: str = FEED_EPOCH_STABILITY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_STABILITY_SCHEMA_VERSION: + raise ValueError("unsupported feed epoch stability schema") + if self.status not in {"pass", "limited", "fail"}: + raise ValueError("unsupported feed epoch stability status") + run_counts = _count_mapping(self.run_counts) + usable_run_counts = _count_mapping(self.usable_run_counts) + required = set(_STABILITY_ANALYSES) + if set(run_counts) != required or set(usable_run_counts) != required: + raise ValueError( + "stability counts must cover every required analysis" + ) + if any(usable_run_counts[name] > run_counts[name] for name in required): + raise ValueError("usable stability runs cannot exceed planned runs") + unstable = tuple( + _required_text(value, "unstable_boundary_id") + for value in self.unstable_boundary_ids + ) + if len(set(unstable)) != len(unstable): + raise ValueError("unstable boundary IDs must be unique") + missing_analysis = any( + usable_run_counts[name] == 0 for name in required + ) + expected_status = ( + "fail" if unstable else ("limited" if missing_analysis else "pass") + ) + if self.status != expected_status: + raise ValueError( + "stability status does not reconcile with its evidence" + ) + object.__setattr__(self, "run_counts", run_counts) + object.__setattr__(self, "usable_run_counts", usable_run_counts) + object.__setattr__(self, "unstable_boundary_ids", unstable) + object.__setattr__(self, "limitations", tuple(self.limitations)) + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic stability metadata.""" + return { + "schema_version": self.schema_version, + "status": self.status, + "run_counts": dict(self.run_counts), + "usable_run_counts": dict(self.usable_run_counts), + "unstable_boundary_ids": list(self.unstable_boundary_ids), + "limitations": list(self.limitations), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochStabilityV1": + """Read a strict stability contract.""" + return cls( + schema_version=str(data.get("schema_version", "")), + status=str(data.get("status", "")), + run_counts={ + str(name): _strict_int(value, str(name)) + for name, value in _mapping(data.get("run_counts")).items() + }, + usable_run_counts={ + str(name): _strict_int(value, str(name)) + for name, value in _mapping( + data.get("usable_run_counts") + ).items() + }, + unstable_boundary_ids=tuple( + str(value) + for value in _sequence(data.get("unstable_boundary_ids")) + ), + limitations=tuple( + str(value) for value in _sequence(data.get("limitations")) + ), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochAssignmentV1: + """Stable epoch or transition assignment for one event timestamp.""" + + definition_id: str + symbol: str + timestamp_utc_ms: int + assignment_kind: str + label: str + epoch_id: str | None = None + boundary_id: str | None = None + schema_version: str = FEED_EPOCH_ASSIGNMENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_ASSIGNMENT_SCHEMA_VERSION: + raise ValueError("unsupported feed epoch assignment schema") + kind = str(self.assignment_kind or "").strip() + if kind not in {"epoch", "transition", "out_of_scope"}: + raise ValueError("unsupported feed epoch assignment kind") + object.__setattr__( + self, + "definition_id", + _required_text(self.definition_id, "definition_id"), + ) + object.__setattr__( + self, "symbol", _required_text(self.symbol, "symbol").upper() + ) + object.__setattr__( + self, + "timestamp_utc_ms", + _strict_int(self.timestamp_utc_ms, "timestamp_utc_ms"), + ) + object.__setattr__(self, "assignment_kind", kind) + object.__setattr__(self, "label", _required_text(self.label, "label")) + if kind == "epoch" and (not self.epoch_id or self.boundary_id): + raise ValueError("epoch assignments require only epoch_id") + if kind == "transition" and (not self.boundary_id or self.epoch_id): + raise ValueError("transition assignments require only boundary_id") + if kind == "out_of_scope" and (self.epoch_id or self.boundary_id): + raise ValueError( + "out-of-scope assignments cannot reference an epoch or boundary" + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic assignment metadata.""" + return { + "schema_version": self.schema_version, + "definition_id": self.definition_id, + "symbol": self.symbol, + "timestamp_utc_ms": self.timestamp_utc_ms, + "assignment_kind": self.assignment_kind, + "label": self.label, + "epoch_id": self.epoch_id, + "boundary_id": self.boundary_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochAssignmentV1": + """Read a strict assignment contract.""" + return cls( + schema_version=str(data.get("schema_version", "")), + definition_id=str(data.get("definition_id", "")), + symbol=str(data.get("symbol", "")), + timestamp_utc_ms=_strict_int( + data.get("timestamp_utc_ms"), + "timestamp_utc_ms", + ), + assignment_kind=str(data.get("assignment_kind", "")), + label=str(data.get("label", "")), + epoch_id=( + str(data["epoch_id"]) + if data.get("epoch_id") is not None + else None + ), + boundary_id=( + str(data["boundary_id"]) + if data.get("boundary_id") is not None + else None + ), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochDefinitionV1: + """Versioned, lineage-complete technological feed-epoch artifact.""" + + config: FeedEpochFitConfigV1 + symbols: tuple[str, ...] + coverage_start_utc_ms: int + coverage_end_utc_ms: int + evidence_count: int + period_count: int + feature_names: tuple[str, ...] + boundaries: tuple[FeedEpochBoundaryV1, ...] + epochs: tuple[FeedEpochIntervalV1, ...] + stability: FeedEpochStabilityV1 + lineage: Mapping[str, JSONValue] + definition_id: str = "" + schema_version: str = FEED_EPOCH_DEFINITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_DEFINITION_SCHEMA_VERSION: + raise ValueError("unsupported feed epoch definition schema") + symbols = tuple( + sorted( + { + _required_text(symbol, "symbol").upper() + for symbol in self.symbols + } + ) + ) + if not symbols: + raise ValueError("feed epoch definition requires symbols") + coverage_start = _strict_int( + self.coverage_start_utc_ms, + "coverage_start_utc_ms", + ) + coverage_end = _strict_int( + self.coverage_end_utc_ms, + "coverage_end_utc_ms", + ) + if coverage_end < coverage_start: + raise ValueError("definition coverage end precedes start") + if not self.epochs: + raise ValueError( + "feed epoch definition requires at least one epoch" + ) + feature_names = tuple(self.feature_names) + if not feature_names or not set(feature_names).issubset( + self.config.feature_names + ): + raise ValueError( + "definition feature_names must come from its fit config" + ) + if len(self.epochs) != len(self.boundaries) + 1: + raise ValueError( + "feed epoch definitions require one more epoch than boundary" + ) + if len({item.boundary_id for item in self.boundaries}) != len( + self.boundaries + ): + raise ValueError("feed epoch boundary IDs must be unique") + if len({item.epoch_id for item in self.epochs}) != len(self.epochs): + raise ValueError("feed epoch IDs must be unique") + if ( + tuple( + sorted( + self.boundaries, + key=lambda item: item.central_timestamp_utc_ms, + ) + ) + != self.boundaries + ): + raise ValueError("feed epoch boundaries must be time ordered") + if ( + tuple( + sorted( + self.epochs, key=lambda item: item.start_timestamp_utc_ms + ) + ) + != self.epochs + ): + raise ValueError("feed epochs must be time ordered") + for index, boundary in enumerate(self.boundaries): + left_epoch = self.epochs[index] + right_epoch = self.epochs[index + 1] + if ( + boundary.left_period != left_epoch.period_end + or boundary.right_period != right_epoch.period_start + ): + raise ValueError( + "feed epoch boundaries must align with adjacent epochs" + ) + if ( + left_epoch.end_timestamp_utc_ms + >= right_epoch.start_timestamp_utc_ms + ): + raise ValueError("adjacent feed epochs must not overlap") + if self.epochs[0].start_timestamp_utc_ms != coverage_start: + raise ValueError("first epoch must start at definition coverage") + if self.epochs[-1].end_timestamp_utc_ms != coverage_end: + raise ValueError("last epoch must end at definition coverage") + lineage = dict(self.lineage) + if lineage.get("config_id") != self.config.config_id: + raise ValueError("definition lineage config_id mismatch") + if ( + _strict_int(lineage.get("source_count"), "lineage.source_count") + != self.evidence_count + ): + raise ValueError("definition lineage source_count mismatch") + sources = _sequence(lineage.get("sources")) + if len(sources) != self.evidence_count: + raise ValueError("definition lineage must include every source") + source_rows = tuple(_mapping(source) for source in sources) + if any(not source for source in source_rows): + raise ValueError( + "definition lineage source entries must be objects" + ) + evidence_ids = [ + _required_text(source.get("evidence_id"), "lineage evidence_id") + for source in source_rows + ] + if len(set(evidence_ids)) != len(evidence_ids): + raise ValueError("definition lineage evidence IDs must be unique") + for source in source_rows: + _required_sha256_id( + source.get("fingerprint_id"), + "lineage fingerprint_id", + ) + _required_sha256_id( + source.get("source_artifact_sha256"), + "lineage source_artifact_sha256", + ) + _required_text(source.get("symbol"), "lineage symbol") + if not _valid_period(str(source.get("period", ""))): + raise ValueError("lineage period must use YYYY or YYYYMM") + _required_text(source.get("source_kind"), "lineage source_kind") + source_hash_basis = _required_text( + source.get("source_hash_basis"), + "lineage source_hash_basis", + ) + if source_hash_basis not in _SOURCE_HASH_BASES: + raise ValueError("unsupported lineage source hash basis") + source_kind_counts = Counter( + str(source["source_kind"]) for source in source_rows + ) + if _count_mapping_unbounded(lineage.get("source_kind_counts")) != dict( + sorted(source_kind_counts.items()) + ): + raise ValueError("definition lineage source-kind counts mismatch") + canonical_sources = tuple( + _mapping(source) + for source in _sequence(lineage.get("canonical_sources")) + ) + if any(not source for source in canonical_sources): + raise ValueError("canonical lineage source entries must be objects") + canonical_source_count = _strict_int( + lineage.get("canonical_source_count"), + "lineage.canonical_source_count", + ) + if canonical_source_count != len(canonical_sources): + raise ValueError("canonical lineage source count mismatch") + canonical_axes: list[tuple[str, str, str]] = [] + for source in canonical_sources: + fingerprint_id = _required_sha256_id( + source.get("fingerprint_id"), + "canonical lineage fingerprint_id", + ) + _required_sha256_id( + source.get("source_artifact_sha256"), + "canonical lineage source_artifact_sha256", + ) + symbol = _required_text( + source.get("symbol"), "canonical lineage symbol" + ) + period = str(source.get("period", "")) + if not _valid_period(period): + raise ValueError( + "canonical lineage period must use YYYY or YYYYMM" + ) + source_hash_basis = _required_text( + source.get("source_hash_basis"), + "canonical lineage source_hash_basis", + ) + if source_hash_basis not in _SOURCE_HASH_BASES: + raise ValueError( + "unsupported canonical lineage source hash basis" + ) + _required_text( + source.get("source_kind"), "canonical lineage source_kind" + ) + canonical_axes.append((symbol, period, fingerprint_id)) + if len(set(canonical_axes)) != len(canonical_axes): + raise ValueError("canonical lineage sources must be unique") + unstable_ids = set(self.stability.unstable_boundary_ids) + boundary_ids = {item.boundary_id for item in self.boundaries} + if not unstable_ids.issubset(boundary_ids): + raise ValueError("stability references an unknown boundary") + if ( + sum(self.stability.run_counts.values()) + > self.config.max_sensitivity_runs + ): + raise ValueError( + "definition stability exceeds configured run limit" + ) + supplied_lineage_hash = _required_sha256_id( + lineage.get("lineage_sha256"), + "lineage_sha256", + ) + expected_lineage_hash = _payload_sha256(_lineage_hash_material(lineage)) + if supplied_lineage_hash != expected_lineage_hash: + raise ValueError("definition lineage hash mismatch") + object.__setattr__(self, "symbols", symbols) + object.__setattr__(self, "coverage_start_utc_ms", coverage_start) + object.__setattr__(self, "coverage_end_utc_ms", coverage_end) + object.__setattr__( + self, + "evidence_count", + _bounded_int( + self.evidence_count, + "evidence_count", + 1, + self.config.max_evidence, + ), + ) + object.__setattr__( + self, + "period_count", + _bounded_int( + self.period_count, "period_count", 1, self.evidence_count + ), + ) + object.__setattr__(self, "feature_names", feature_names) + object.__setattr__(self, "lineage", lineage) + expected = _stable_id("feed-epoch-definition", self.identity_payload()) + supplied = str(self.definition_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "definition_id does not match deterministic identity" + ) + object.__setattr__(self, "definition_id", expected) + + @property + def valid_for_observation_models(self) -> bool: + """Return whether downstream observation models may consume this artifact.""" + return self.stability.status == "pass" + + def identity_payload(self) -> dict[str, JSONValue]: + """Return all definition fields that determine semantic identity.""" + return { + "schema_version": self.schema_version, + "config": self.config.to_dict(), + "symbols": list(self.symbols), + "coverage_start_utc_ms": self.coverage_start_utc_ms, + "coverage_end_utc_ms": self.coverage_end_utc_ms, + "evidence_count": self.evidence_count, + "period_count": self.period_count, + "feature_names": list(self.feature_names), + "boundaries": [boundary.to_dict() for boundary in self.boundaries], + "epochs": [epoch.to_dict() for epoch in self.epochs], + "stability": self.stability.to_dict(), + "lineage": dict(self.lineage), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible artifact metadata.""" + return { + **self.identity_payload(), + "definition_id": self.definition_id, + "valid_for_observation_models": self.valid_for_observation_models, + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochDefinitionV1": + """Read a strict definition and verify its identity.""" + return cls( + schema_version=str(data.get("schema_version", "")), + config=FeedEpochFitConfigV1.from_dict(_mapping(data.get("config"))), + symbols=tuple( + str(value) for value in _sequence(data.get("symbols")) + ), + coverage_start_utc_ms=_strict_int( + data.get("coverage_start_utc_ms"), "coverage_start_utc_ms" + ), + coverage_end_utc_ms=_strict_int( + data.get("coverage_end_utc_ms"), "coverage_end_utc_ms" + ), + evidence_count=_strict_int( + data.get("evidence_count"), "evidence_count" + ), + period_count=_strict_int(data.get("period_count"), "period_count"), + feature_names=tuple( + str(value) for value in _sequence(data.get("feature_names")) + ), + boundaries=tuple( + FeedEpochBoundaryV1.from_dict(_mapping(value)) + for value in _sequence(data.get("boundaries")) + ), + epochs=tuple( + FeedEpochIntervalV1.from_dict(_mapping(value)) + for value in _sequence(data.get("epochs")) + ), + stability=FeedEpochStabilityV1.from_dict( + _mapping(data.get("stability")) + ), + lineage=_mapping(data.get("lineage")), + definition_id=str(data.get("definition_id", "")), + ) + + @classmethod + def from_json(cls, value: str) -> "FeedEpochDefinitionV1": + """Read a strict definition from JSON.""" + data = json.loads(value) + if not isinstance(data, Mapping): + raise ValueError("feed epoch definition JSON must be an object") + return cls.from_dict(data) + + def assign( + self, + *, + symbol: str, + timestamp_utc_ms: int, + require_stable: bool = True, + ) -> FeedEpochAssignmentV1: + """Assign an event to an epoch or explicit transition interval.""" + normalized_symbol = _required_text(symbol, "symbol").upper() + timestamp = _strict_int(timestamp_utc_ms, "timestamp_utc_ms") + if require_stable and not self.valid_for_observation_models: + raise ValueError( + "feed epoch definition has not passed stability checks" + ) + if normalized_symbol not in self.symbols: + return FeedEpochAssignmentV1( + definition_id=self.definition_id, + symbol=normalized_symbol, + timestamp_utc_ms=timestamp, + assignment_kind="out_of_scope", + label="symbol_out_of_scope", + ) + if ( + not self.coverage_start_utc_ms + <= timestamp + <= self.coverage_end_utc_ms + ): + return FeedEpochAssignmentV1( + definition_id=self.definition_id, + symbol=normalized_symbol, + timestamp_utc_ms=timestamp, + assignment_kind="out_of_scope", + label="timestamp_out_of_scope", + ) + for boundary in self.boundaries: + if ( + boundary.uncertainty_start_utc_ms + <= timestamp + <= boundary.uncertainty_end_utc_ms + ): + return FeedEpochAssignmentV1( + definition_id=self.definition_id, + symbol=normalized_symbol, + timestamp_utc_ms=timestamp, + assignment_kind="transition", + label=boundary.transition_label, + boundary_id=boundary.boundary_id, + ) + for epoch in self.epochs: + if ( + epoch.start_timestamp_utc_ms + <= timestamp + <= epoch.end_timestamp_utc_ms + ): + return FeedEpochAssignmentV1( + definition_id=self.definition_id, + symbol=normalized_symbol, + timestamp_utc_ms=timestamp, + assignment_kind="epoch", + label=epoch.label, + epoch_id=epoch.epoch_id, + ) + return FeedEpochAssignmentV1( + definition_id=self.definition_id, + symbol=normalized_symbol, + timestamp_utc_ms=timestamp, + assignment_kind="out_of_scope", + label="unassigned_gap", + ) + + +@dataclass(frozen=True, slots=True) +class _PeriodPoint: + """One deterministic panel aggregate used internally by the fitter.""" + + period: str + start_utc_ms: int + end_utc_ms: int + evidence_count: int + values: Mapping[str, float] + + +@dataclass(frozen=True, slots=True) +class _Candidate: + """One boundary candidate before stability enrichment.""" + + left_index: int + right_index: int + left_period: str + right_period: str + left_end_utc_ms: int + right_start_utc_ms: int + score: float + feature_differences: Mapping[str, float] + + +@dataclass(frozen=True, slots=True) +class _SensitivityVariant: + """One deterministic perturbation of period points or features.""" + + analysis: str + points: tuple[_PeriodPoint, ...] + features: tuple[str, ...] + + +def fit_feed_epochs( + evidence: Iterable[FeedEpochEvidenceV1], + *, + config: FeedEpochFitConfigV1 | None = None, +) -> FeedEpochDefinitionV1: + """Fit uncertainty-aware technological epochs from canonical evidence.""" + policy = config or FeedEpochFitConfigV1() + ordered = tuple( + sorted( + evidence, + key=lambda item: ( + item.start_timestamp_utc_ms, + item.period, + item.symbol, + item.evidence_id, + ), + ) + ) + if not ordered: + raise ValueError("feed epoch fitting requires canonical evidence") + if len(ordered) > policy.max_evidence: + raise ValueError("feed epoch evidence exceeds configured maximum") + evidence_ids = [item.evidence_id for item in ordered] + if len(set(evidence_ids)) != len(evidence_ids): + raise ValueError("duplicate feed epoch evidence_id") + source_axes = [(item.symbol, item.period) for item in ordered] + if len(set(source_axes)) != len(source_axes): + raise ValueError("feed epoch evidence duplicates a symbol-period axis") + + points = _aggregate_period_points(ordered, policy) + if len(points) < policy.min_evidence_periods: + raise ValueError( + "feed epoch fitting requires at least " + f"{policy.min_evidence_periods} distinct periods" + ) + feature_names = _eligible_features(points, policy) + if len(feature_names) < 2: + raise ValueError( + "feed epoch fitting requires at least two covered features" + ) + candidates = _detect_boundaries(points, feature_names, policy) + variants = _sensitivity_variants(points, feature_names, policy) + boundaries, stability = _stabilize_boundaries( + points, + feature_names, + candidates, + variants, + policy, + ) + epochs = _epoch_intervals(points, boundaries) + lineage = _definition_lineage(ordered, feature_names, policy) + return FeedEpochDefinitionV1( + config=policy, + symbols=tuple(sorted({item.symbol for item in ordered})), + coverage_start_utc_ms=min( + item.start_timestamp_utc_ms for item in ordered + ), + coverage_end_utc_ms=max(item.end_timestamp_utc_ms for item in ordered), + evidence_count=len(ordered), + period_count=len(points), + feature_names=feature_names, + boundaries=boundaries, + epochs=epochs, + stability=stability, + lineage=lineage, + ) + + +def feed_epoch_definition_to_json(definition: FeedEpochDefinitionV1) -> str: + """Return deterministic formatted artifact JSON.""" + return json.dumps(definition.to_dict(), indent=2, sort_keys=True) + + +def write_feed_epoch_definition( + definition: FeedEpochDefinitionV1, + path: str | Path, +) -> ArtifactRef: + """Write a definition artifact and return a compact reference.""" + output = Path(path).expanduser() + output.parent.mkdir(parents=True, exist_ok=True) + encoded = f"{feed_epoch_definition_to_json(definition)}\n".encode("utf-8") + output.write_bytes(encoded) + return ArtifactRef( + kind="feed-epoch-definition", + path=str(output.resolve()), + size_bytes=len(encoded), + sha256=hashlib.sha256(encoded).hexdigest(), + metadata={ + "schema_version": definition.schema_version, + "definition_id": definition.definition_id, + "stability_status": definition.stability.status, + "valid_for_observation_models": ( + definition.valid_for_observation_models + ), + "symbol_count": len(definition.symbols), + "period_count": definition.period_count, + "boundary_count": len(definition.boundaries), + }, + ) + + +def _fingerprint_feature_values(payload: Mapping[str, Any]) -> dict[str, float]: + coverage = _mapping(payload.get("coverage")) + topology = _mapping(payload.get("temporal_topology")) + distribution = _mapping(payload.get("tick_distribution")) + dynamics = _mapping(payload.get("microstructure_dynamics")) + audit = _mapping(payload.get("fingerprint_audit")) + row_count = _nonnegative_int(coverage.get("row_count")) + duration_ms = _optional_finite_float(coverage.get("duration_ms")) + values: dict[str, float] = {} + if duration_ms is not None and duration_ms > 0.0: + values["log_tick_rate_per_hour"] = math.log1p( + row_count * 3_600_000.0 / duration_ms + ) + interarrival = _mapping(dynamics.get("interarrival_ms")) + median_interval = _optional_finite_float(interarrival.get("median")) + if median_interval is not None and median_interval >= 0.0: + values["log_median_interarrival_ms"] = math.log1p(median_interval) + p95_interval = _quantile_value(interarrival, 0.95) + if p95_interval is not None and p95_interval >= 0.0: + values["log_p95_interarrival_ms"] = math.log1p(p95_interval) + min_interval = _optional_finite_float(topology.get("min_interval_ms")) + if min_interval is not None and min_interval >= 0.0: + values["minimum_observed_interval_ms"] = min_interval + precision = _price_precision_digits(distribution) + if precision is not None: + values["price_precision_digits"] = precision + spread = _mapping(distribution.get("spread")) + spread_median = _optional_finite_float(spread.get("median")) + if spread_median is not None: + values["spread_median"] = spread_median + conditioned_spread = _conditioned_spread_median(payload) + if conditioned_spread is not None: + values["conditioned_spread_median"] = conditioned_spread + absolute_spread_change = _mapping(dynamics.get("absolute_spread_change")) + spread_change_median = _optional_finite_float( + absolute_spread_change.get("median") + ) + if spread_change_median is not None: + values["absolute_spread_change_median"] = spread_change_median + stale = _mapping(dynamics.get("stale_quote")) + stale_rate = _optional_finite_float(stale.get("repeat_rate")) + if stale_rate is not None: + values["stale_repeat_rate"] = stale_rate + burst = _mapping(dynamics.get("burst")) + burst_rate = _optional_finite_float(burst.get("burst_rate")) + if burst_rate is not None: + values["burst_rate"] = burst_rate + duplicate_count = _nonnegative_int( + topology.get("duplicate_timestamp_count") + ) + parsed_count = _nonnegative_int(topology.get("parsed_row_count")) + if parsed_count: + values["duplicate_timestamp_rate"] = duplicate_count / parsed_count + suspicious_count = _nonnegative_int(topology.get("suspicious_gap_count")) + interval_count = _nonnegative_int(topology.get("interval_count")) + if interval_count: + values["suspicious_gap_rate"] = suspicious_count / interval_count + statuses = _mapping(audit.get("section_statuses")) + if statuses: + limited = sum( + 1 + for status in statuses.values() + if str(status) in {"limited", "skipped", "unavailable"} + ) + values["source_quality_penalty"] = limited / len(statuses) + return values + + +def _fingerprint_conditioning( + payload: Mapping[str, Any], +) -> dict[str, JSONValue]: + calendar = _mapping(payload.get("calendar_regimes")) + conditional = _mapping(payload.get("conditional_distributions")) + return { + "calendar_schema_version": calendar.get("schema_version"), + "calendar_status": calendar.get("status", "unavailable"), + "calendar_profile_name": calendar.get("calendar_profile_name"), + "calendar_profile_version": calendar.get("calendar_profile_version"), + "calendar_profile_complete": calendar.get("calendar_profile_complete"), + "session_state_counts": dict( + _mapping(calendar.get("session_state_counts")) + ), + "active_session_counts": dict( + _mapping(calendar.get("active_session_counts")) + ), + "special_tag_counts": dict( + _mapping(calendar.get("special_tag_counts")) + ), + "holiday_tag_counts": dict( + _mapping(calendar.get("holiday_tag_counts")) + ), + "event_tag_counts": dict(_mapping(calendar.get("event_tag_counts"))), + "conditional_schema_version": conditional.get("schema_version"), + "conditional_status": conditional.get("status", "unavailable"), + "conditioning_payload_sha256": _payload_sha256( + {"calendar": dict(calendar), "conditional": dict(conditional)} + ), + } + + +def _fingerprint_quality(payload: Mapping[str, Any]) -> dict[str, JSONValue]: + audit = _mapping(payload.get("fingerprint_audit")) + topology = _mapping(payload.get("temporal_topology")) + dynamics = _mapping(payload.get("microstructure_dynamics")) + source = _mapping(payload.get("source")) + return { + "audit_schema_version": audit.get("schema_version"), + "section_statuses": dict(_mapping(audit.get("section_statuses"))), + "sections_skipped": dict(_mapping(audit.get("sections_skipped"))), + "source_status": dict(_mapping(audit.get("source_status"))), + "source_kind": source.get("kind", "unknown"), + "calculation_basis": dynamics.get( + "basis", topology.get("sampling_basis") + ), + "computed_from": dynamics.get( + "computed_from", topology.get("computed_from") + ), + "cache_source": dynamics.get( + "cache_source", topology.get("cache_source") + ), + "sequence_status": dynamics.get("sequence_status", "unavailable"), + "limitations": list(_sequence(dynamics.get("limitations"))), + } + + +def _fingerprint_profile( + payload: Mapping[str, Any], + feature_values: Mapping[str, float], +) -> dict[str, JSONValue]: + coverage = _mapping(payload.get("coverage")) + topology = _mapping(payload.get("temporal_topology")) + distribution = _mapping(payload.get("tick_distribution")) + dynamics = _mapping(payload.get("microstructure_dynamics")) + calendar = _mapping(payload.get("calendar_regimes")) + interarrival = _mapping(dynamics.get("interarrival_ms")) + stale = _mapping(dynamics.get("stale_quote")) + spread = _mapping(distribution.get("spread")) + row_count = _nonnegative_int(coverage.get("row_count")) + repeat_count = _nonnegative_int(stale.get("repeat_count")) + interval_count = max(0, row_count - 1) + tick_rate_log = feature_values.get("log_tick_rate_per_hour") + return { + "row_count": row_count, + "tick_rate_per_hour": ( + math.expm1(tick_rate_log) if tick_rate_log is not None else 0.0 + ), + "median_interarrival_ms": _optional_finite_float( + interarrival.get("median") + ) + or 0.0, + "p95_interarrival_ms": _quantile_value(interarrival, 0.95) or 0.0, + "max_interarrival_ms": _optional_finite_float(interarrival.get("max")) + or 0.0, + "quiet_gap_count": _nonnegative_int( + topology.get("suspicious_gap_count") + ), + "quote_update_count": max(0, interval_count - repeat_count), + "quote_update_ratio": ( + max(0, interval_count - repeat_count) / interval_count + if interval_count + else 0.0 + ), + "zero_change_run_count": _nonnegative_int(stale.get("run_count")), + "zero_change_tick_count": _nonnegative_int( + stale.get("affected_row_count") + ), + "spread_min": _optional_finite_float(spread.get("min")) or 0.0, + "spread_median": _optional_finite_float(spread.get("median")) or 0.0, + "spread_mean": _optional_finite_float(spread.get("mean")) or 0.0, + "spread_max": _optional_finite_float(spread.get("max")) or 0.0, + "session_counts": dict(_mapping(calendar.get("active_session_counts"))), + } + + +def _aggregate_period_points( + evidence: Sequence[FeedEpochEvidenceV1], + config: FeedEpochFitConfigV1, +) -> tuple[_PeriodPoint, ...]: + normalized_by_evidence = _symbol_normalized_evidence_values( + evidence, + config.feature_names, + ) + grouped: dict[str, list[FeedEpochEvidenceV1]] = defaultdict(list) + for item in evidence: + grouped[item.period].append(item) + points: list[_PeriodPoint] = [] + for period, items in grouped.items(): + values: dict[str, float] = {} + for feature in config.feature_names: + observed = [ + normalized_by_evidence[item.evidence_id][feature] + for item in items + if feature in normalized_by_evidence[item.evidence_id] + ] + if observed: + values[feature] = float(median(observed)) + points.append( + _PeriodPoint( + period=period, + start_utc_ms=min(item.start_timestamp_utc_ms for item in items), + end_utc_ms=max(item.end_timestamp_utc_ms for item in items), + evidence_count=len(items), + values=values, + ) + ) + return tuple( + sorted(points, key=lambda item: (item.start_utc_ms, item.period)) + ) + + +def _symbol_normalized_evidence_values( + evidence: Sequence[FeedEpochEvidenceV1], + features: Sequence[str], +) -> dict[str, dict[str, float]]: + """Normalize within symbols so panel membership cannot create epochs.""" + grouped: dict[str, list[FeedEpochEvidenceV1]] = defaultdict(list) + for item in evidence: + grouped[item.symbol].append(item) + normalized: dict[str, dict[str, float]] = { + item.evidence_id: {} for item in evidence + } + for items in grouped.values(): + for feature in features: + observed = [ + float(item.feature_values[feature]) + for item in items + if feature in item.feature_values + ] + if not observed: + continue + center = float(median(observed)) + deviations = [abs(value - center) for value in observed] + scale = float(median(deviations)) * 1.4826 + if scale <= 0.0: + scale = max(observed) - min(observed) + for item in items: + if feature not in item.feature_values: + continue + value = float(item.feature_values[feature]) + normalized[item.evidence_id][feature] = ( + (value - center) / scale if scale > 0.0 else 0.0 + ) + return normalized + + +def _eligible_features( + points: Sequence[_PeriodPoint], + config: FeedEpochFitConfigV1, +) -> tuple[str, ...]: + minimum = max(2, math.ceil(len(points) * config.min_feature_coverage)) + return tuple( + feature + for feature in config.feature_names + if sum(feature in point.values for point in points) >= minimum + ) + + +def _detect_boundaries( + points: Sequence[_PeriodPoint], + features: Sequence[str], + config: FeedEpochFitConfigV1, +) -> tuple[_Candidate, ...]: + if len(points) < max(2, config.min_segment_periods * 2): + return () + normalized = _normalized_feature_values(points, features) + candidates: list[_Candidate] = [] + for right_index in range(1, len(points)): + if right_index < config.min_segment_periods: + continue + if len(points) - right_index < config.min_segment_periods: + continue + differences = { + feature: abs( + normalized[feature][right_index] + - normalized[feature][right_index - 1] + ) + for feature in normalized + if right_index < len(normalized[feature]) + } + if not differences: + continue + score = sum(differences.values()) / len(differences) + if score < config.min_change_score: + continue + left = points[right_index - 1] + right = points[right_index] + candidates.append( + _Candidate( + left_index=right_index - 1, + right_index=right_index, + left_period=left.period, + right_period=right.period, + left_end_utc_ms=left.end_utc_ms, + right_start_utc_ms=right.start_utc_ms, + score=score, + feature_differences=differences, + ) + ) + return _select_separated_candidates(candidates, len(points), config) + + +def _normalized_feature_values( + points: Sequence[_PeriodPoint], + features: Sequence[str], +) -> dict[str, list[float]]: + result: dict[str, list[float]] = {} + for feature in features: + observed = [point.values.get(feature) for point in points] + finite = [float(value) for value in observed if value is not None] + if len(finite) < 2: + continue + center = float(median(finite)) + deviations = [abs(value - center) for value in finite] + scale = float(median(deviations)) * 1.4826 + if scale <= 0.0: + scale = max(finite) - min(finite) + if scale <= 0.0: + continue + result[feature] = [ + ((float(value) - center) / scale if value is not None else 0.0) + for value in observed + ] + return result + + +def _select_separated_candidates( + candidates: Sequence[_Candidate], + point_count: int, + config: FeedEpochFitConfigV1, +) -> tuple[_Candidate, ...]: + selected: list[_Candidate] = [] + for candidate in sorted( + candidates, key=lambda item: (-item.score, item.right_index) + ): + if candidate.right_index < config.min_segment_periods: + continue + if point_count - candidate.right_index < config.min_segment_periods: + continue + if any( + abs(candidate.right_index - existing.right_index) + < config.min_segment_periods + for existing in selected + ): + continue + selected.append(candidate) + return tuple(sorted(selected, key=lambda item: item.right_index)) + + +def _sensitivity_variants( + points: tuple[_PeriodPoint, ...], + features: tuple[str, ...], + config: FeedEpochFitConfigV1, +) -> tuple[_SensitivityVariant, ...]: + variants: list[_SensitivityVariant] = [] + for offset in (0, 1): + sampled = points[offset::2] + if len(sampled) >= config.min_evidence_periods: + variants.append(_SensitivityVariant("sampling", sampled, features)) + for index in range(1, len(points) - 1): + reduced = points[:index] + points[index + 1 :] + if len(reduced) >= config.min_evidence_periods: + variants.append( + _SensitivityVariant("missing_periods", reduced, features) + ) + for feature in features: + reduced_features = tuple(name for name in features if name != feature) + if len(reduced_features) >= 2: + variants.append( + _SensitivityVariant("feature_removal", points, reduced_features) + ) + by_analysis: dict[str, list[_SensitivityVariant]] = defaultdict(list) + for variant in variants: + by_analysis[variant.analysis].append(variant) + for analysis in by_analysis: + by_analysis[analysis].sort( + key=lambda item: ( + tuple(point.period for point in item.points), + item.features, + ) + ) + selected: list[_SensitivityVariant] = [] + for analysis in _STABILITY_ANALYSES: + if by_analysis[analysis]: + selected.append(by_analysis[analysis].pop(0)) + remaining = config.max_sensitivity_runs - len(selected) + for analysis in ("sampling", "feature_removal"): + if remaining <= 0: + break + additions = by_analysis[analysis][:remaining] + selected.extend(additions) + remaining -= len(additions) + if remaining > 0: + selected.extend( + _evenly_spaced_variants(by_analysis["missing_periods"], remaining) + ) + return tuple(selected) + + +def _evenly_spaced_variants( + variants: Sequence[_SensitivityVariant], + limit: int, +) -> tuple[_SensitivityVariant, ...]: + """Select a deterministic whole-history sample under a bounded budget.""" + if limit <= 0 or not variants: + return () + if len(variants) <= limit: + return tuple(variants) + if limit == 1: + return (variants[len(variants) // 2],) + indices = { + round(offset * (len(variants) - 1) / (limit - 1)) + for offset in range(limit) + } + return tuple(variants[index] for index in sorted(indices)) + + +def _stabilize_boundaries( + points: tuple[_PeriodPoint, ...], + features: tuple[str, ...], + candidates: tuple[_Candidate, ...], + variants: tuple[_SensitivityVariant, ...], + config: FeedEpochFitConfigV1, +) -> tuple[tuple[FeedEpochBoundaryV1, ...], FeedEpochStabilityV1]: + run_counts: Counter[str] = Counter() + usable_counts: Counter[str] = Counter() + variant_candidates: list[tuple[str, tuple[_Candidate, ...]]] = [] + for variant in variants: + run_counts[variant.analysis] += 1 + detected = _detect_boundaries(variant.points, variant.features, config) + usable_counts[variant.analysis] += 1 + variant_candidates.append((variant.analysis, detected)) + for analysis in _STABILITY_ANALYSES: + run_counts.setdefault(analysis, 0) + usable_counts.setdefault(analysis, 0) + + positions = {point.period: index for index, point in enumerate(points)} + boundaries: list[FeedEpochBoundaryV1] = [] + unstable_ids: list[str] = [] + limitations: list[str] = [] + for index, candidate in enumerate(candidates, start=1): + boundary_id = f"boundary-{index:03d}" + matches_by_analysis: Counter[str] = Counter() + matched_intervals = [ + (candidate.left_end_utc_ms, candidate.right_start_utc_ms) + ] + for analysis, detected in variant_candidates: + match = _matching_candidate(candidate, detected, positions, config) + if match is None: + continue + matches_by_analysis[analysis] += 1 + matched_intervals.append( + (match.left_end_utc_ms, match.right_start_utc_ms) + ) + support_by_analysis = { + analysis: ( + matches_by_analysis[analysis] / usable_counts[analysis] + if usable_counts[analysis] + else 0.0 + ) + for analysis in sorted(run_counts) + } + total_runs = sum(usable_counts.values()) + support = ( + sum(matches_by_analysis.values()) / total_runs + if total_runs + else 0.0 + ) + available_support = [ + support_by_analysis[name] + for name in support_by_analysis + if usable_counts[name] + ] + gate_support = min(available_support) if available_support else 0.0 + if gate_support < config.min_boundary_support: + unstable_ids.append(boundary_id) + uncertainty_values = [ + value for interval in matched_intervals for value in interval + ] + uncertainty_start = min(uncertainty_values) + uncertainty_end = max(uncertainty_values) + if uncertainty_end < uncertainty_start: + uncertainty_start, uncertainty_end = ( + uncertainty_end, + uncertainty_start, + ) + central = ( + candidate.left_end_utc_ms + + (candidate.right_start_utc_ms - candidate.left_end_utc_ms) // 2 + ) + confidence = ( + min( + 1.0, + (candidate.score / max(config.min_change_score, 1e-12)) / 2.0, + ) + * support + ) + contributing = tuple( + name + for name, _ in sorted( + candidate.feature_differences.items(), + key=lambda item: (-item[1], item[0]), + )[:8] + ) + boundaries.append( + FeedEpochBoundaryV1( + boundary_id=boundary_id, + left_period=candidate.left_period, + right_period=candidate.right_period, + central_timestamp_utc_ms=central, + uncertainty_start_utc_ms=uncertainty_start, + uncertainty_end_utc_ms=uncertainty_end, + change_score=_rounded(candidate.score, config.rounding_digits), + confidence=_rounded(confidence, config.rounding_digits), + support=_rounded(support, config.rounding_digits), + support_by_analysis={ + name: _rounded(value, config.rounding_digits) + for name, value in support_by_analysis.items() + }, + contributing_features=contributing, + transition_label=( + f"transition:epoch-{index:03d}:epoch-{index + 1:03d}" + ), + ) + ) + + missing_analyses = [ + analysis for analysis, count in usable_counts.items() if count == 0 + ] + limitations.extend( + f"{analysis}_sensitivity_unavailable" for analysis in missing_analyses + ) + if unstable_ids: + limitations.append("unstable_boundaries") + extra_boundary_run_count = sum( + 1 + for _, detected in variant_candidates + if len(detected) > len(candidates) + ) + if extra_boundary_run_count: + limitations.append("perturbations_produced_extra_boundaries") + if unstable_ids: + status = "fail" + elif missing_analyses: + status = "limited" + else: + status = "pass" + return ( + tuple(boundaries), + FeedEpochStabilityV1( + status=status, + run_counts=dict(run_counts), + usable_run_counts=dict(usable_counts), + unstable_boundary_ids=tuple(unstable_ids), + limitations=tuple(sorted(set(limitations))), + ), + ) + + +def _matching_candidate( + baseline: _Candidate, + detected: Sequence[_Candidate], + positions: Mapping[str, int], + config: FeedEpochFitConfigV1, +) -> _Candidate | None: + baseline_position = positions[baseline.right_period] + ranked = sorted( + detected, + key=lambda candidate: ( + abs( + positions.get(candidate.right_period, -10_000) + - baseline_position + ), + -candidate.score, + candidate.right_period, + ), + ) + if not ranked: + return None + candidate = ranked[0] + position = positions.get(candidate.right_period) + if position is None: + return None + if ( + abs(position - baseline_position) + > config.boundary_match_tolerance_periods + ): + return None + return candidate + + +def _epoch_intervals( + points: tuple[_PeriodPoint, ...], + boundaries: tuple[FeedEpochBoundaryV1, ...], +) -> tuple[FeedEpochIntervalV1, ...]: + boundary_periods = [boundary.right_period for boundary in boundaries] + period_positions = { + point.period: index for index, point in enumerate(points) + } + split_positions = [period_positions[period] for period in boundary_periods] + starts = [0, *split_positions] + ends = [position - 1 for position in split_positions] + [len(points) - 1] + epochs: list[FeedEpochIntervalV1] = [] + for index, (start_index, end_index) in enumerate( + zip(starts, ends, strict=True), start=1 + ): + segment = points[start_index : end_index + 1] + epoch_id = f"epoch-{index:03d}" + epochs.append( + FeedEpochIntervalV1( + epoch_id=epoch_id, + label=epoch_id, + period_start=segment[0].period, + period_end=segment[-1].period, + start_timestamp_utc_ms=segment[0].start_utc_ms, + end_timestamp_utc_ms=segment[-1].end_utc_ms, + evidence_count=sum(point.evidence_count for point in segment), + ) + ) + return tuple(epochs) + + +def _definition_lineage( + evidence: Sequence[FeedEpochEvidenceV1], + features: Sequence[str], + config: FeedEpochFitConfigV1, +) -> dict[str, JSONValue]: + sources: list[JSONValue] = [ + { + "evidence_id": item.evidence_id, + "fingerprint_id": item.fingerprint_id, + "source_artifact_sha256": item.source_artifact_sha256, + "source_hash_basis": item.source_hash_basis, + "symbol": item.symbol, + "period": item.period, + "source_kind": item.source_kind, + } + for item in evidence + ] + canonical_sources: list[JSONValue] = [] + for item in evidence: + components = _sequence(item.quality.get("component_sources")) + if components: + canonical_sources.extend( + dict(_mapping(component)) for component in components + ) + else: + canonical_sources.append( + { + "fingerprint_id": item.fingerprint_id, + "source_artifact_sha256": item.source_artifact_sha256, + "source_hash_basis": item.source_hash_basis, + "symbol": item.symbol, + "period": item.period, + "source_kind": item.source_kind, + } + ) + canonical_sources.sort( + key=lambda source: ( + str(_mapping(source).get("symbol", "")), + str(_mapping(source).get("period", "")), + str(_mapping(source).get("fingerprint_id", "")), + ) + ) + feature_sources: dict[str, JSONValue] = {} + for feature in features: + paths: list[JSONValue] = [] + paths.extend( + sorted( + { + path + for item in evidence + for path in item.feature_provenance.get(feature, ()) + } + ) + ) + feature_sources[feature] = paths + conditioning_status_counts = Counter( + str(item.conditioning.get("calendar_status", "unavailable")) + for item in evidence + ) + source_kind_counts = Counter(item.source_kind for item in evidence) + result: dict[str, JSONValue] = { + "config_id": config.config_id, + "source_count": len(evidence), + "sources": sources, + "canonical_source_count": len(canonical_sources), + "canonical_sources": canonical_sources, + "source_kind_counts": dict(sorted(source_kind_counts.items())), + "conditioning_status_counts": dict( + sorted(conditioning_status_counts.items()) + ), + "fingerprint_schema_versions": [TIME_SERIES_FINGERPRINT_SCHEMA_VERSION], + "feature_provenance": feature_sources, + "panel_normalization": "within_symbol_robust_then_period_median", + } + result["lineage_sha256"] = _payload_sha256(_lineage_hash_material(result)) + return result + + +def _lineage_hash_material( + lineage: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + """Return every lineage field except its self-referential digest.""" + return { + str(name): value + for name, value in sorted(lineage.items()) + if name != "lineage_sha256" + } + + +def _price_precision_digits(distribution: Mapping[str, Any]) -> float | None: + precision = _mapping(distribution.get("precision")) + columns = _mapping(precision.get("column_decimal_place_counts")) + counts: Counter[int] = Counter() + for column in ("bid", "ask"): + for digits, count in _mapping(columns.get(column)).items(): + try: + counts[int(str(digits))] += _nonnegative_int(count) + except ValueError: + continue + if not counts: + return None + return float( + sorted(counts.items(), key=lambda item: (-item[1], item[0]))[0][0] + ) + + +def _conditioned_spread_median(payload: Mapping[str, Any]) -> float | None: + conditional = _mapping(payload.get("conditional_distributions")) + by_session = _mapping(conditional.get("by_active_session")) + values: list[float] = [] + for row in by_session.values(): + spread = _mapping(_mapping(row).get("spread")) + value = _optional_finite_float(spread.get("median")) + if value is not None: + values.append(value) + return float(median(values)) if values else None + + +def _quantile_value( + summary: Mapping[str, Any], quantile: float +) -> float | None: + quantiles = _mapping(summary.get("quantiles")) + candidates = ( + str(quantile), + f"{quantile:.2f}", + f"p{int(round(quantile * 100))}", + ) + for key in candidates: + if key in quantiles: + return _optional_finite_float(quantiles[key]) + return None + + +def _payload_sha256(value: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(value).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _stable_id(prefix: str, value: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(value).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _required_sha256_id(value: Any, name: str) -> str: + text = _required_text(value, name) + digest = text.removeprefix("sha256:") + if not _SHA256_RE.fullmatch(digest): + raise ValueError(f"{name} must be a sha256 identifier") + return "sha256:" + digest + + +def _required_text(value: Any, name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"{name} must not be empty") + return text + + +def _valid_period(value: str) -> bool: + if not _PERIOD_RE.fullmatch(value): + return False + return len(value) == 4 or 1 <= int(value[4:]) <= 12 + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool): + raise ValueError(f"{name} must be an integer") + try: + result = int(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must be an integer") from exc + if isinstance(value, float) and not value.is_integer(): + raise ValueError(f"{name} must be an integer") + return result + + +def _nonnegative_int(value: Any) -> int: + try: + return max(0, _strict_int(value, "value")) + except ValueError: + return 0 + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be finite") + try: + result = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must be finite") from exc + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _optional_finite_float(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + try: + result = float(value) + except (TypeError, ValueError, OverflowError): + return None + return result if math.isfinite(result) else None + + +def _bounded_int( + value: Any, + name: str, + minimum: int, + maximum: int, +) -> int: + result = _strict_int(value, name) + if not minimum <= result <= maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}") + return result + + +def _bounded_float( + value: Any, + name: str, + minimum: float, + maximum: float, +) -> float: + result = _finite_float(value, name) + if not minimum <= result <= maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}") + return result + + +def _rounded(value: float, digits: int) -> float: + result = round(float(value), digits) + return 0.0 if result == 0.0 else result + + +def _mapping(value: Any) -> dict[str, Any]: + if not isinstance(value, Mapping): + return {} + return {str(key): item for key, item in value.items()} + + +def _sequence(value: Any) -> tuple[Any, ...]: + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + return tuple(value) + return () + + +def _count_mapping(value: Mapping[str, int]) -> dict[str, int]: + return { + str(name): _bounded_int( + count, str(name), 0, MAX_FEED_EPOCH_SENSITIVITY_RUNS + ) + for name, count in sorted(value.items()) + } + + +def _count_mapping_unbounded(value: Any) -> dict[str, int]: + return { + str(name): _bounded_int( + count, + str(name), + 0, + MAX_FEED_EPOCH_EVIDENCE, + ) + for name, count in sorted(_mapping(value).items()) + } diff --git a/src/histdatacom/data_analytics/feed_epochs_v2.py b/src/histdatacom/data_analytics/feed_epochs_v2.py new file mode 100644 index 00000000..ea8a0498 --- /dev/null +++ b/src/histdatacom/data_analytics/feed_epochs_v2.py @@ -0,0 +1,2420 @@ +"""Active-time multivariate technological feed-epoch fitting. + +Version two deliberately lives beside :mod:`feed_epochs`. Version-one +artifacts retain their original adjacent-period semantics; this module fits a +panel of real ASCII tick caches with explicit time denominators, a robust +multivariate PELT objective, and bounded sensitivity evidence. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import time +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone +from functools import lru_cache +from pathlib import Path +from statistics import median +from typing import Any + +from histdatacom.data_quality.calendar import ( + FX_CLOSE_OPEN_MINUTE, + SESSION_WINDOWS, + calendar_policy_metadata, +) +from histdatacom.data_quality.contracts import QualityTargetKind +from histdatacom.data_quality.discovery import discover_quality_targets +from histdatacom.histdata_ascii import EST_NO_DST_OFFSET_MS +from histdatacom.resource_usage import peak_rss_bytes +from histdatacom.runtime_contracts import ArtifactRef, JSONValue + +FEED_EPOCH_EVIDENCE_V2_SCHEMA_VERSION = "histdatacom.feed-epoch-evidence.v2" +FEED_EPOCH_FIT_CONFIG_V2_SCHEMA_VERSION = "histdatacom.feed-epoch-fit-config.v2" +FEED_EPOCH_BOUNDARY_V2_SCHEMA_VERSION = "histdatacom.feed-epoch-boundary.v2" +FEED_EPOCH_INTERVAL_V2_SCHEMA_VERSION = "histdatacom.feed-epoch-interval.v2" +FEED_EPOCH_STABILITY_V2_SCHEMA_VERSION = "histdatacom.feed-epoch-stability.v2" +FEED_EPOCH_DEFINITION_V2_SCHEMA_VERSION = "histdatacom.feed-epoch-definition.v2" +FEED_EPOCH_ASSIGNMENT_V2_SCHEMA_VERSION = "histdatacom.feed-epoch-assignment.v2" +FEED_EPOCH_CAMPAIGN_V2_SCHEMA_VERSION = "histdatacom.feed-epoch-campaign.v2" + +CALENDAR_POLICY_VERSION = "histdatacom.fx-active-time.v1" +HOUR_MS = 3_600_000 +MINUTE_MS = 60_000 +SOURCE_OFFSET_MS = -EST_NO_DST_OFFSET_MS +MAX_FEATURES = 64 +MAX_EVIDENCE = 4096 +MAX_ACTIVITY_BINS = 800 +SESSION_ACTIVITY_KEYS = tuple(window.name for window in SESSION_WINDOWS) + ( + "off_session", +) +DAY_ACTIVITY_KEYS = tuple(str(value) for value in range(7)) + +DEFAULT_ACTIVE_TIME_FEATURES = ( + "log_calendar_tick_rate_per_hour", + "log_market_open_tick_rate_per_hour", + "log_active_window_tick_rate_per_hour", + "bid_only_rate", + "ask_only_rate", + "joint_move_rate", + "unchanged_rate", + "subwindow_count_fano", + "log_interarrival_median_ms", + "interarrival_dispersion", + "interarrival_lag1", + "timestamp_exact_second_rate", + "timestamp_last_digit_entropy", + "price_precision_digits", + "duplicate_timestamp_rate", + "burst_interval_rate", + "stale_quote_rate", + "log_stale_run_p95", + "log_stale_run_max", + "log_spread_median", + "spread_tail_ratio", + "spread_jump_rate", + "session_activity_dispersion", + "day_activity_dispersion", + *(f"session_activity_share_{name}" for name in SESSION_ACTIVITY_KEYS), + *(f"day_activity_share_{name}" for name in DAY_ACTIVITY_KEYS), + "cross_symbol_activity_correlation", + "cross_symbol_active_bin_overlap", +) + + +@dataclass(frozen=True, slots=True) +class FeedEpochEvidenceV2: + """One bounded monthly observation of a feed's delivery technology.""" + + symbol: str + period: str + source_path: str + source_artifact_sha256: str + source_size_bytes: int + start_timestamp_utc_ms: int + end_timestamp_utc_ms: int + row_count: int + denominators_ms: Mapping[str, int] + counts: Mapping[str, int] + feature_values: Mapping[str, float] + feature_provenance: Mapping[str, tuple[str, ...]] + activity_bin_counts: Mapping[str, int] + calendar_policy: Mapping[str, JSONValue] + limitations: tuple[str, ...] = () + evidence_id: str = "" + schema_version: str = FEED_EPOCH_EVIDENCE_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_EVIDENCE_V2_SCHEMA_VERSION: + raise ValueError("unsupported v2 feed epoch evidence schema") + symbol = _required_text(self.symbol, "symbol").upper() + if not _valid_month(self.period): + raise ValueError("v2 feed epoch evidence requires YYYYMM periods") + source_hash = _sha256_id( + self.source_artifact_sha256, "source_artifact_sha256" + ) + row_count = _bounded_int(self.row_count, "row_count", 2, 2**63 - 1) + start = _strict_int(self.start_timestamp_utc_ms, "start timestamp") + end = _strict_int(self.end_timestamp_utc_ms, "end timestamp") + if end < start: + raise ValueError("evidence end precedes start") + denominators = { + _required_text(name, "denominator"): _bounded_int( + value, f"denominator {name}", 1, 2**63 - 1 + ) + for name, value in sorted(self.denominators_ms.items()) + } + required_denominators = { + "calendar_duration_ms", + "market_open_duration_ms", + "active_window_duration_ms", + } + if not required_denominators.issubset(denominators): + raise ValueError( + "v2 evidence is missing an active-time denominator" + ) + counts = { + _required_text(name, "count"): _bounded_int( + value, f"count {name}", 0, 2**63 - 1 + ) + for name, value in sorted(self.counts.items()) + } + features = { + _required_text(name, "feature"): _finite_float(value, name) + for name, value in sorted(self.feature_values.items()) + } + if not features or len(features) > MAX_FEATURES: + raise ValueError( + "v2 evidence features must be non-empty and bounded" + ) + required_numerators = { + "log_market_open_tick_rate_per_hour": "market_open_row_count", + "log_active_window_tick_rate_per_hour": ( + "active_window_interval_count" + ), + } + for feature, count in required_numerators.items(): + if feature in features and count not in counts: + raise ValueError( + f"v2 evidence is missing rate numerator {count}" + ) + if counts.get("market_open_row_count", 0) > row_count: + raise ValueError("market-open row count exceeds evidence row count") + if counts.get("active_window_interval_count", 0) > row_count - 1: + raise ValueError( + "active-window interval count exceeds evidence transitions" + ) + provenance = { + name: tuple( + _required_text(path, "feature provenance") + for path in self.feature_provenance.get(name, ()) + ) + for name in features + } + if any(not paths for paths in provenance.values()): + raise ValueError("every v2 feature requires provenance") + bins = { + str(_integer_text(key, "activity bin")): _bounded_int( + value, "activity bin count", 1, 2**63 - 1 + ) + for key, value in sorted( + self.activity_bin_counts.items(), key=lambda item: int(item[0]) + ) + } + if len(bins) > MAX_ACTIVITY_BINS: + raise ValueError("activity-bin evidence exceeds monthly bound") + object.__setattr__(self, "symbol", symbol) + object.__setattr__( + self, "source_path", _required_text(self.source_path, "source_path") + ) + object.__setattr__(self, "source_artifact_sha256", source_hash) + object.__setattr__( + self, + "source_size_bytes", + _bounded_int( + self.source_size_bytes, "source_size_bytes", 1, 2**63 - 1 + ), + ) + object.__setattr__(self, "row_count", row_count) + object.__setattr__(self, "start_timestamp_utc_ms", start) + object.__setattr__(self, "end_timestamp_utc_ms", end) + object.__setattr__(self, "denominators_ms", denominators) + object.__setattr__(self, "counts", counts) + object.__setattr__(self, "feature_values", features) + object.__setattr__(self, "feature_provenance", provenance) + object.__setattr__(self, "activity_bin_counts", bins) + object.__setattr__(self, "calendar_policy", dict(self.calendar_policy)) + object.__setattr__( + self, + "limitations", + tuple(dict.fromkeys(str(value) for value in self.limitations)), + ) + expected = _stable_id("feed-epoch-evidence-v2", self.identity_payload()) + if self.evidence_id and self.evidence_id != expected: + raise ValueError( + "evidence_id does not match deterministic identity" + ) + object.__setattr__(self, "evidence_id", expected) + + @property + def profile(self) -> Mapping[str, JSONValue]: + """Expose a bounded compatibility profile for observation fitting.""" + median_interval = math.expm1( + self.feature_values.get("log_interarrival_median_ms", 0.0) + ) + return { + "row_count": self.row_count, + "tick_rate_per_hour": math.expm1( + self.feature_values.get( + "log_active_window_tick_rate_per_hour", 0.0 + ) + ), + "median_interarrival_ms": median_interval, + "p95_interarrival_ms": median_interval + * self.feature_values.get("interarrival_dispersion", 1.0), + "calendar_policy_version": CALENDAR_POLICY_VERSION, + } + + @property + def source_hash_basis(self) -> str: + """Return the provenance vocabulary consumed by observation fitting.""" + return "cache_content_sha256" + + def identity_payload(self) -> dict[str, JSONValue]: + """Return all semantic fields used by the evidence ID.""" + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "period": self.period, + "source_artifact_sha256": self.source_artifact_sha256, + "source_size_bytes": self.source_size_bytes, + "start_timestamp_utc_ms": self.start_timestamp_utc_ms, + "end_timestamp_utc_ms": self.end_timestamp_utc_ms, + "row_count": self.row_count, + "denominators_ms": dict(self.denominators_ms), + "counts": dict(self.counts), + "feature_values": dict(self.feature_values), + "feature_provenance": { + name: list(paths) + for name, paths in sorted(self.feature_provenance.items()) + }, + "activity_bin_counts": dict(self.activity_bin_counts), + "calendar_policy": dict(self.calendar_policy), + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible evidence.""" + return { + **self.identity_payload(), + "source_path": self.source_path, + "evidence_id": self.evidence_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochEvidenceV2": + """Read strict version-two evidence.""" + return cls( + schema_version=str(data.get("schema_version", "")), + symbol=str(data.get("symbol", "")), + period=str(data.get("period", "")), + source_path=str(data.get("source_path", "")), + source_artifact_sha256=str(data.get("source_artifact_sha256", "")), + source_size_bytes=_strict_int( + data.get("source_size_bytes"), "source_size_bytes" + ), + start_timestamp_utc_ms=_strict_int( + data.get("start_timestamp_utc_ms"), "start timestamp" + ), + end_timestamp_utc_ms=_strict_int( + data.get("end_timestamp_utc_ms"), "end timestamp" + ), + row_count=_strict_int(data.get("row_count"), "row_count"), + denominators_ms=_int_mapping(data.get("denominators_ms")), + counts=_int_mapping(data.get("counts")), + feature_values=_float_mapping(data.get("feature_values")), + feature_provenance={ + str(name): tuple(str(value) for value in _sequence(paths)) + for name, paths in _mapping( + data.get("feature_provenance") + ).items() + }, + activity_bin_counts=_int_mapping(data.get("activity_bin_counts")), + calendar_policy=_mapping(data.get("calendar_policy")), + limitations=tuple( + str(value) for value in _sequence(data.get("limitations")) + ), + evidence_id=str(data.get("evidence_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochFitConfigV2: + """Explicit robust PELT and sensitivity policy.""" + + feature_names: tuple[str, ...] = DEFAULT_ACTIVE_TIME_FEATURES + min_evidence_periods: int = 24 + min_segment_periods: int = 6 + min_feature_coverage: float = 0.80 + min_symbol_count: int = 3 + penalty_multiplier: float = 24.0 + robust_clip: float = 6.0 + min_boundary_support: float = 0.60 + boundary_match_tolerance_periods: int = 2 + sensitivity_penalty_multipliers: tuple[float, ...] = (0.75, 1.25) + active_gap_cap_ms: int = 60_000 + burst_interval_ms: int = 100 + activity_bin_ms: int = HOUR_MS + max_evidence: int = MAX_EVIDENCE + max_sensitivity_runs: int = 128 + rounding_digits: int = 10 + config_id: str = "" + schema_version: str = FEED_EPOCH_FIT_CONFIG_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_FIT_CONFIG_V2_SCHEMA_VERSION: + raise ValueError("unsupported v2 feed epoch fit config schema") + names = tuple(dict.fromkeys(str(name) for name in self.feature_names)) + unknown = sorted(set(names).difference(DEFAULT_ACTIVE_TIME_FEATURES)) + if not names or len(names) > MAX_FEATURES or unknown: + raise ValueError( + "unsupported v2 feature selection" + + (": " + ", ".join(unknown) if unknown else "") + ) + min_periods = _bounded_int( + self.min_evidence_periods, "min_evidence_periods", 4, MAX_EVIDENCE + ) + min_segment = _bounded_int( + self.min_segment_periods, "min_segment_periods", 2, MAX_EVIDENCE + ) + if min_segment * 2 > min_periods: + raise ValueError("minimum segment exceeds half minimum evidence") + object.__setattr__(self, "feature_names", names) + object.__setattr__(self, "min_evidence_periods", min_periods) + object.__setattr__(self, "min_segment_periods", min_segment) + object.__setattr__( + self, + "min_feature_coverage", + _bounded_float(self.min_feature_coverage, "coverage", 0.0, 1.0), + ) + object.__setattr__( + self, + "min_boundary_support", + _bounded_float(self.min_boundary_support, "support", 0.0, 1.0), + ) + _bounded_float(self.penalty_multiplier, "penalty", 0.0001, 1_000.0) + _bounded_float(self.robust_clip, "robust_clip", 0.1, 100.0) + penalties = tuple( + _bounded_float(value, "sensitivity penalty", 0.01, 100.0) + for value in self.sensitivity_penalty_multipliers + ) + object.__setattr__(self, "sensitivity_penalty_multipliers", penalties) + _bounded_int(self.min_symbol_count, "min_symbol_count", 2, 32) + _bounded_int( + self.boundary_match_tolerance_periods, + "boundary_match_tolerance_periods", + 0, + 24, + ) + _bounded_int(self.active_gap_cap_ms, "active_gap_cap_ms", 1, 86_400_000) + _bounded_int(self.burst_interval_ms, "burst_interval_ms", 1, 60_000) + _bounded_int( + self.activity_bin_ms, "activity_bin_ms", MINUTE_MS, 86_400_000 + ) + max_evidence = _bounded_int( + self.max_evidence, "max_evidence", min_periods, MAX_EVIDENCE + ) + if max_evidence < min_periods * self.min_symbol_count: + raise ValueError("max_evidence cannot hold minimum panel support") + _bounded_int(self.max_sensitivity_runs, "max_sensitivity_runs", 1, 4096) + _bounded_int(self.rounding_digits, "rounding_digits", 1, 15) + payload = self.identity_payload(include_id=False) + expected = _stable_id("feed-epoch-fit-config-v2", payload) + if self.config_id and self.config_id != expected: + raise ValueError("config_id does not match deterministic identity") + object.__setattr__(self, "config_id", expected) + + def identity_payload( + self, *, include_id: bool = True + ) -> dict[str, JSONValue]: + """Return deterministic config metadata.""" + payload: dict[str, JSONValue] = { + "schema_version": self.schema_version, + "feature_names": list(self.feature_names), + "min_evidence_periods": self.min_evidence_periods, + "min_segment_periods": self.min_segment_periods, + "min_feature_coverage": self.min_feature_coverage, + "min_symbol_count": self.min_symbol_count, + "penalty_multiplier": self.penalty_multiplier, + "robust_clip": self.robust_clip, + "min_boundary_support": self.min_boundary_support, + "boundary_match_tolerance_periods": ( + self.boundary_match_tolerance_periods + ), + "sensitivity_penalty_multipliers": list( + self.sensitivity_penalty_multipliers + ), + "active_gap_cap_ms": self.active_gap_cap_ms, + "burst_interval_ms": self.burst_interval_ms, + "activity_bin_ms": self.activity_bin_ms, + "max_evidence": self.max_evidence, + "max_sensitivity_runs": self.max_sensitivity_runs, + "rounding_digits": self.rounding_digits, + } + if include_id: + payload["config_id"] = self.config_id + return payload + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic config metadata.""" + return self.identity_payload() + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochFitConfigV2": + """Read a strict version-two fit config.""" + return cls( + schema_version=str(data.get("schema_version", "")), + feature_names=tuple( + str(value) for value in _sequence(data.get("feature_names")) + ), + min_evidence_periods=_strict_int( + data.get("min_evidence_periods"), "min_evidence_periods" + ), + min_segment_periods=_strict_int( + data.get("min_segment_periods"), "min_segment_periods" + ), + min_feature_coverage=_finite_float( + data.get("min_feature_coverage"), "min_feature_coverage" + ), + min_symbol_count=_strict_int( + data.get("min_symbol_count"), "min_symbol_count" + ), + penalty_multiplier=_finite_float( + data.get("penalty_multiplier"), "penalty_multiplier" + ), + robust_clip=_finite_float(data.get("robust_clip"), "robust_clip"), + min_boundary_support=_finite_float( + data.get("min_boundary_support"), "min_boundary_support" + ), + boundary_match_tolerance_periods=_strict_int( + data.get("boundary_match_tolerance_periods"), + "boundary_match_tolerance_periods", + ), + sensitivity_penalty_multipliers=tuple( + _finite_float(value, "sensitivity penalty") + for value in _sequence( + data.get("sensitivity_penalty_multipliers") + ) + ), + active_gap_cap_ms=_strict_int( + data.get("active_gap_cap_ms"), "active_gap_cap_ms" + ), + burst_interval_ms=_strict_int( + data.get("burst_interval_ms"), "burst_interval_ms" + ), + activity_bin_ms=_strict_int( + data.get("activity_bin_ms"), "activity_bin_ms" + ), + max_evidence=_strict_int(data.get("max_evidence"), "max_evidence"), + max_sensitivity_runs=_strict_int( + data.get("max_sensitivity_runs"), "max_sensitivity_runs" + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + config_id=str(data.get("config_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochBoundaryV2: + """A shared panel boundary with sensitivity-derived uncertainty.""" + + left_period: str + right_period: str + central_timestamp_utc_ms: int + support: float + uncertainty_start_period: str + uncertainty_end_period: str + objective_gain: float + supporting_features: tuple[str, ...] + boundary_id: str = "" + schema_version: str = FEED_EPOCH_BOUNDARY_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_BOUNDARY_V2_SCHEMA_VERSION: + raise ValueError("unsupported v2 feed epoch boundary schema") + for value in ( + self.left_period, + self.right_period, + self.uncertainty_start_period, + self.uncertainty_end_period, + ): + if not _valid_month(value): + raise ValueError("v2 boundary periods must use YYYYMM") + _bounded_float(self.support, "boundary support", 0.0, 1.0) + _bounded_float(self.objective_gain, "objective gain", 0.0, math.inf) + expected = _stable_id("feed-epoch-boundary-v2", self.identity_payload()) + if self.boundary_id and self.boundary_id != expected: + raise ValueError( + "boundary_id does not match deterministic identity" + ) + object.__setattr__(self, "boundary_id", expected) + + @property + def transition_label(self) -> str: + """Return the stable transition label used by observation models.""" + return f"transition:{self.left_period}-{self.right_period}" + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic boundary metadata.""" + return { + **self.identity_payload(), + "transition_label": self.transition_label, + "boundary_id": self.boundary_id, + } + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic boundary fields without the derived ID.""" + return { + "schema_version": self.schema_version, + "left_period": self.left_period, + "right_period": self.right_period, + "central_timestamp_utc_ms": self.central_timestamp_utc_ms, + "support": self.support, + "uncertainty_start_period": self.uncertainty_start_period, + "uncertainty_end_period": self.uncertainty_end_period, + "objective_gain": self.objective_gain, + "supporting_features": list(self.supporting_features), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochBoundaryV2": + """Read a strict version-two boundary.""" + return cls( + schema_version=str(data.get("schema_version", "")), + left_period=str(data.get("left_period", "")), + right_period=str(data.get("right_period", "")), + central_timestamp_utc_ms=_strict_int( + data.get("central_timestamp_utc_ms"), "boundary timestamp" + ), + support=_finite_float(data.get("support"), "boundary support"), + uncertainty_start_period=str( + data.get("uncertainty_start_period", "") + ), + uncertainty_end_period=str(data.get("uncertainty_end_period", "")), + objective_gain=_finite_float( + data.get("objective_gain"), "objective gain" + ), + supporting_features=tuple( + str(value) + for value in _sequence(data.get("supporting_features")) + ), + boundary_id=str(data.get("boundary_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochIntervalV2: + """One stable shared technology epoch.""" + + label: str + period_start: str + period_end: str + start_timestamp_utc_ms: int + end_timestamp_utc_ms: int + evidence_count: int + feature_medians: Mapping[str, float] + epoch_id: str = "" + schema_version: str = FEED_EPOCH_INTERVAL_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_INTERVAL_V2_SCHEMA_VERSION: + raise ValueError("unsupported v2 feed epoch interval schema") + if not _valid_month(self.period_start) or not _valid_month( + self.period_end + ): + raise ValueError("v2 interval periods must use YYYYMM") + expected = _stable_id("feed-epoch-interval-v2", self.identity_payload()) + if self.epoch_id and self.epoch_id != expected: + raise ValueError("epoch_id does not match deterministic identity") + object.__setattr__(self, "epoch_id", expected) + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic interval metadata.""" + return {**self.identity_payload(), "epoch_id": self.epoch_id} + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic interval fields without the derived ID.""" + return { + "schema_version": self.schema_version, + "label": self.label, + "period_start": self.period_start, + "period_end": self.period_end, + "start_timestamp_utc_ms": self.start_timestamp_utc_ms, + "end_timestamp_utc_ms": self.end_timestamp_utc_ms, + "evidence_count": self.evidence_count, + "feature_medians": dict(self.feature_medians), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochIntervalV2": + """Read a strict version-two interval.""" + return cls( + schema_version=str(data.get("schema_version", "")), + label=str(data.get("label", "")), + period_start=str(data.get("period_start", "")), + period_end=str(data.get("period_end", "")), + start_timestamp_utc_ms=_strict_int( + data.get("start_timestamp_utc_ms"), "interval start" + ), + end_timestamp_utc_ms=_strict_int( + data.get("end_timestamp_utc_ms"), "interval end" + ), + evidence_count=_strict_int( + data.get("evidence_count"), "interval evidence_count" + ), + feature_medians=_float_mapping(data.get("feature_medians")), + epoch_id=str(data.get("epoch_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochSymbolDeviationV2: + """A symbol-specific boundary not supported by the shared panel.""" + + symbol: str + left_period: str + right_period: str + nearest_global_distance_periods: int | None + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic deviation metadata.""" + return { + "symbol": self.symbol, + "left_period": self.left_period, + "right_period": self.right_period, + "nearest_global_distance_periods": ( + self.nearest_global_distance_periods + ), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochSymbolDeviationV2": + """Read one symbol-specific deviation.""" + distance = data.get("nearest_global_distance_periods") + return cls( + symbol=str(data.get("symbol", "")), + left_period=str(data.get("left_period", "")), + right_period=str(data.get("right_period", "")), + nearest_global_distance_periods=( + _strict_int(distance, "nearest boundary distance") + if distance is not None + else None + ), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochStabilityV2: + """Bounded sensitivity and support result.""" + + status: str + reasons: tuple[str, ...] + run_count: int + run_counts: Mapping[str, int] + boundary_support: Mapping[str, float] + boundary_support_by_family: Mapping[str, Mapping[str, float]] + rejected_candidates: Mapping[str, Mapping[str, JSONValue]] + feature_coverage: Mapping[str, float] + common_period_count: int + symbol_count: int + schema_version: str = FEED_EPOCH_STABILITY_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_STABILITY_V2_SCHEMA_VERSION: + raise ValueError("unsupported v2 feed epoch stability schema") + if self.status not in {"pass", "fail"}: + raise ValueError("unsupported v2 feed epoch stability status") + if sum(self.run_counts.values()) != self.run_count: + raise ValueError("v2 stability run counts do not reconcile") + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic stability metadata.""" + return { + "schema_version": self.schema_version, + "status": self.status, + "reasons": list(self.reasons), + "run_count": self.run_count, + "run_counts": dict(self.run_counts), + "boundary_support": dict(self.boundary_support), + "boundary_support_by_family": { + period: dict(values) + for period, values in sorted( + self.boundary_support_by_family.items() + ) + }, + "rejected_candidates": { + period: dict(values) + for period, values in sorted(self.rejected_candidates.items()) + }, + "feature_coverage": dict(self.feature_coverage), + "common_period_count": self.common_period_count, + "symbol_count": self.symbol_count, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochStabilityV2": + """Read strict version-two stability evidence.""" + family = _mapping(data.get("boundary_support_by_family")) + return cls( + schema_version=str(data.get("schema_version", "")), + status=str(data.get("status", "")), + reasons=tuple( + str(value) for value in _sequence(data.get("reasons")) + ), + run_count=_strict_int(data.get("run_count"), "run_count"), + run_counts=_int_mapping(data.get("run_counts")), + boundary_support=_float_mapping(data.get("boundary_support")), + boundary_support_by_family={ + str(period): _float_mapping(values) + for period, values in family.items() + }, + rejected_candidates={ + str(period): _mapping(values) + for period, values in _mapping( + data.get("rejected_candidates") + ).items() + }, + feature_coverage=_float_mapping(data.get("feature_coverage")), + common_period_count=_strict_int( + data.get("common_period_count"), "common_period_count" + ), + symbol_count=_strict_int(data.get("symbol_count"), "symbol_count"), + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochAssignmentV2: + """Assignment compatible with the downstream observation hierarchy.""" + + definition_id: str + symbol: str + timestamp_utc_ms: int + assignment_kind: str + label: str + epoch_id: str | None = None + boundary_id: str | None = None + schema_version: str = FEED_EPOCH_ASSIGNMENT_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_ASSIGNMENT_V2_SCHEMA_VERSION: + raise ValueError("unsupported v2 feed epoch assignment schema") + if self.assignment_kind not in {"epoch", "transition", "out_of_scope"}: + raise ValueError("unsupported v2 feed epoch assignment kind") + + +@dataclass(frozen=True, slots=True) +class FeedEpochDefinitionV2: + """Versioned shared-epoch definition fitted from a tick panel.""" + + config: FeedEpochFitConfigV2 + symbols: tuple[str, ...] + coverage_start_utc_ms: int + coverage_end_utc_ms: int + evidence_count: int + period_count: int + feature_names: tuple[str, ...] + boundaries: tuple[FeedEpochBoundaryV2, ...] + epochs: tuple[FeedEpochIntervalV2, ...] + symbol_deviations: tuple[FeedEpochSymbolDeviationV2, ...] + stability: FeedEpochStabilityV2 + lineage: Mapping[str, JSONValue] + definition_id: str = "" + schema_version: str = FEED_EPOCH_DEFINITION_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != FEED_EPOCH_DEFINITION_V2_SCHEMA_VERSION: + raise ValueError("unsupported v2 feed epoch definition schema") + if len(self.epochs) != len(self.boundaries) + 1: + raise ValueError( + "v2 definitions require one more epoch than boundary" + ) + expected = _stable_id( + "feed-epoch-definition-v2", self.identity_payload() + ) + if self.definition_id and self.definition_id != expected: + raise ValueError( + "definition_id does not match deterministic identity" + ) + object.__setattr__(self, "definition_id", expected) + + @property + def valid_for_observation_models(self) -> bool: + """Return whether observation operators may consume this definition.""" + return self.stability.status == "pass" + + def identity_payload(self) -> dict[str, JSONValue]: + """Return all semantic definition fields.""" + return { + "schema_version": self.schema_version, + "config": self.config.to_dict(), + "symbols": list(self.symbols), + "coverage_start_utc_ms": self.coverage_start_utc_ms, + "coverage_end_utc_ms": self.coverage_end_utc_ms, + "evidence_count": self.evidence_count, + "period_count": self.period_count, + "feature_names": list(self.feature_names), + "boundaries": [item.to_dict() for item in self.boundaries], + "epochs": [item.to_dict() for item in self.epochs], + "symbol_deviations": [ + item.to_dict() for item in self.symbol_deviations + ], + "stability": self.stability.to_dict(), + "lineage": dict(self.lineage), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible definition metadata.""" + return { + **self.identity_payload(), + "definition_id": self.definition_id, + "valid_for_observation_models": self.valid_for_observation_models, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "FeedEpochDefinitionV2": + """Read a strict version-two definition while leaving v1 untouched.""" + return cls( + schema_version=str(data.get("schema_version", "")), + config=FeedEpochFitConfigV2.from_dict(_mapping(data.get("config"))), + symbols=tuple( + str(value) for value in _sequence(data.get("symbols")) + ), + coverage_start_utc_ms=_strict_int( + data.get("coverage_start_utc_ms"), "coverage start" + ), + coverage_end_utc_ms=_strict_int( + data.get("coverage_end_utc_ms"), "coverage end" + ), + evidence_count=_strict_int( + data.get("evidence_count"), "evidence_count" + ), + period_count=_strict_int(data.get("period_count"), "period_count"), + feature_names=tuple( + str(value) for value in _sequence(data.get("feature_names")) + ), + boundaries=tuple( + FeedEpochBoundaryV2.from_dict(_mapping(value)) + for value in _sequence(data.get("boundaries")) + ), + epochs=tuple( + FeedEpochIntervalV2.from_dict(_mapping(value)) + for value in _sequence(data.get("epochs")) + ), + symbol_deviations=tuple( + FeedEpochSymbolDeviationV2.from_dict(_mapping(value)) + for value in _sequence(data.get("symbol_deviations")) + ), + stability=FeedEpochStabilityV2.from_dict( + _mapping(data.get("stability")) + ), + lineage=_mapping(data.get("lineage")), + definition_id=str(data.get("definition_id", "")), + ) + + def assign( + self, *, symbol: str, timestamp_utc_ms: int + ) -> FeedEpochAssignmentV2: + """Assign an event without collapsing uncertainty windows.""" + normalized = symbol.upper() + timestamp = int(timestamp_utc_ms) + if normalized not in self.symbols or not ( + self.coverage_start_utc_ms <= timestamp <= self.coverage_end_utc_ms + ): + return FeedEpochAssignmentV2( + definition_id=self.definition_id, + symbol=normalized, + timestamp_utc_ms=timestamp, + assignment_kind="out_of_scope", + label="out_of_scope", + ) + for boundary in self.boundaries: + start = _period_start_ms(boundary.uncertainty_start_period) + end = _period_end_ms(boundary.uncertainty_end_period) + if start <= timestamp <= end: + return FeedEpochAssignmentV2( + definition_id=self.definition_id, + symbol=normalized, + timestamp_utc_ms=timestamp, + assignment_kind="transition", + label=boundary.transition_label, + boundary_id=boundary.boundary_id, + ) + for epoch in self.epochs: + if ( + epoch.start_timestamp_utc_ms + <= timestamp + <= epoch.end_timestamp_utc_ms + ): + return FeedEpochAssignmentV2( + definition_id=self.definition_id, + symbol=normalized, + timestamp_utc_ms=timestamp, + assignment_kind="epoch", + label=epoch.label, + epoch_id=epoch.epoch_id, + ) + return FeedEpochAssignmentV2( + definition_id=self.definition_id, + symbol=normalized, + timestamp_utc_ms=timestamp, + assignment_kind="out_of_scope", + label="out_of_scope", + ) + + +@dataclass(frozen=True, slots=True) +class FeedEpochCampaignV2: + """Complete bounded output from discovery, scan, fit, and sensitivity.""" + + definition: FeedEpochDefinitionV2 + evidence: tuple[FeedEpochEvidenceV2, ...] + source_count: int + source_bytes: int + runtime_seconds: float + peak_memory_bytes: int + skipped_sources: tuple[Mapping[str, JSONValue], ...] = () + schema_version: str = FEED_EPOCH_CAMPAIGN_V2_SCHEMA_VERSION + + def to_dict(self, *, include_evidence: bool = True) -> dict[str, JSONValue]: + """Return campaign evidence with an optional compact source table.""" + payload: dict[str, JSONValue] = { + "schema_version": self.schema_version, + "definition": self.definition.to_dict(), + "source_count": self.source_count, + "source_bytes": self.source_bytes, + "runtime_seconds": self.runtime_seconds, + "peak_memory_bytes": self.peak_memory_bytes, + "skipped_sources": [dict(value) for value in self.skipped_sources], + "limitations": [ + "epochs describe delivery technology, not latent market state", + ( + "hourly synchronization is an activity-count proxy, " + "not tick identity" + ), + ( + "the static HistData source calendar is advisory for " + "holidays/events" + ), + ], + } + if include_evidence: + payload["evidence"] = [item.to_dict() for item in self.evidence] + return payload + + +def scan_active_time_evidence( + path: str | Path, + *, + symbol: str, + period: str, + config: FeedEpochFitConfigV2 | None = None, +) -> FeedEpochEvidenceV2: + """Scan one Arrow cache into bounded, denominator-explicit evidence.""" + import polars as pl # pylint: disable=import-outside-toplevel + + selected = config or FeedEpochFitConfigV2() + source = Path(path).resolve() + lazy = pl.scan_ipc(source) + schema = set(lazy.collect_schema().names()) + if not {"datetime", "bid", "ask"}.issubset(schema): + raise ValueError("tick cache requires datetime, bid, and ask columns") + timestamp = pl.col("datetime").cast(pl.Int64) + bid = pl.col("bid").cast(pl.Float64) + ask = pl.col("ask").cast(pl.Float64) + interval = timestamp.diff() + bid_change = bid.diff() + ask_change = ask.diff() + spread = ask - bid + transition_count = pl.len() - 1 + source_minutes = (timestamp + SOURCE_OFFSET_MS) // MINUTE_MS + source_weekday = (source_minutes // (24 * 60) + 3) % 7 + source_minute = source_minutes % (24 * 60) + market_open = ~( + (source_weekday == 5) + | ((source_weekday == 4) & (source_minute >= FX_CLOSE_OPEN_MINUTE)) + | ((source_weekday == 6) & (source_minute < FX_CLOSE_OPEN_MINUTE)) + ) + active_interval = ( + (interval > 0) & (interval <= selected.active_gap_cap_ms) & market_open + ) + expressions: list[Any] = [ + pl.len().alias("row_count"), + timestamp.min().alias("start"), + timestamp.max().alias("end"), + interval.filter(interval > 0).median().alias("interval_median"), + interval.filter(interval > 0).quantile(0.9).alias("interval_p90"), + pl.corr( + interval.cast(pl.Float64), interval.shift(1).cast(pl.Float64) + ).alias("interval_lag1"), + ((bid_change != 0) & (ask_change == 0)).sum().alias("bid_only"), + ((bid_change == 0) & (ask_change != 0)).sum().alias("ask_only"), + ((bid_change != 0) & (ask_change != 0)).sum().alias("joint"), + ((bid_change == 0) & (ask_change == 0)).sum().alias("unchanged"), + (interval == 0).sum().alias("duplicate_timestamp"), + (interval < 0).sum().alias("non_monotonic_timestamp"), + interval.filter( + (interval > 0) & (interval <= selected.burst_interval_ms) + ) + .count() + .alias("burst_interval"), + interval.filter(active_interval).sum().alias("active_window_duration"), + active_interval.sum().alias("active_window_interval_count"), + spread.median().alias("spread_median"), + spread.quantile(0.99).alias("spread_p99"), + (spread.diff().abs() > spread.median() * 5).sum().alias("spread_jump"), + ((timestamp % 1000) == 0).mean().alias("exact_second_rate"), + bid_change.abs() + .filter(bid_change.abs() > 0) + .quantile(0.01) + .alias("bid_step"), + ask_change.abs() + .filter(ask_change.abs() > 0) + .quantile(0.01) + .alias("ask_step"), + market_open.sum().alias("market_open_row_count"), + transition_count.alias("transition_count"), + ] + expressions.extend( + ((timestamp % 10) == digit).sum().alias(f"digit_{digit}") + for digit in range(10) + ) + summary = ( + lazy.select(expressions).collect(engine="streaming").row(named=True) + ) + row_count = int(summary["row_count"]) + if row_count < 2: + raise ValueError("tick cache needs at least two rows") + activity = ( + lazy.group_by((timestamp // selected.activity_bin_ms).alias("bucket")) + .len() + .sort("bucket") + .collect(engine="streaming") + ) + activity_bins = { + str(int(bucket)): int(count) for bucket, count in activity.iter_rows() + } + if len(activity_bins) > MAX_ACTIVITY_BINS: + raise ValueError( + "configured activity bins exceed monthly evidence bound" + ) + session_counts, day_counts = _activity_conditioning_counts( + lazy, selected.activity_bin_ms + ) + stale_runs = _stale_run_summary(source) + calendar_duration, market_open_duration = _period_denominators(period) + active_duration = max(1, int(summary["active_window_duration"] or 0)) + active_interval_count = int(summary["active_window_interval_count"] or 0) + market_open_row_count = int(summary["market_open_row_count"] or 0) + transitions = max(1, int(summary["transition_count"] or row_count - 1)) + activity_values = [float(value) for value in activity_bins.values()] + spread_median = _optional_float(summary.get("spread_median")) or 0.0 + spread_p99 = _optional_float(summary.get("spread_p99")) or spread_median + median_interval = _optional_float(summary.get("interval_median")) or 0.0 + p90_interval = ( + _optional_float(summary.get("interval_p90")) or median_interval + ) + price_steps = [ + value + for value in ( + _positive_float(summary.get("bid_step")), + _positive_float(summary.get("ask_step")), + ) + if value is not None + ] + price_step = min(price_steps) if price_steps else 1e-8 + precision = max(0.0, min(12.0, round(-math.log10(price_step)))) + digit_counts = [int(summary[f"digit_{value}"] or 0) for value in range(10)] + features = { + "log_calendar_tick_rate_per_hour": math.log1p( + row_count * HOUR_MS / calendar_duration + ), + "log_market_open_tick_rate_per_hour": math.log1p( + market_open_row_count * HOUR_MS / market_open_duration + ), + "log_active_window_tick_rate_per_hour": math.log1p( + active_interval_count * HOUR_MS / active_duration + ), + "bid_only_rate": int(summary["bid_only"] or 0) / transitions, + "ask_only_rate": int(summary["ask_only"] or 0) / transitions, + "joint_move_rate": int(summary["joint"] or 0) / transitions, + "unchanged_rate": int(summary["unchanged"] or 0) / transitions, + "subwindow_count_fano": _fano(activity_values), + "log_interarrival_median_ms": math.log1p(median_interval), + "interarrival_dispersion": p90_interval / max(1.0, median_interval), + "interarrival_lag1": _optional_float(summary.get("interval_lag1")) + or 0.0, + "timestamp_exact_second_rate": _optional_float( + summary.get("exact_second_rate") + ) + or 0.0, + "timestamp_last_digit_entropy": _normalized_entropy(digit_counts), + "price_precision_digits": precision, + "duplicate_timestamp_rate": int(summary["duplicate_timestamp"] or 0) + / transitions, + "burst_interval_rate": int(summary["burst_interval"] or 0) + / transitions, + "stale_quote_rate": int(summary["unchanged"] or 0) / transitions, + "log_stale_run_p95": math.log1p(stale_runs["p95"]), + "log_stale_run_max": math.log1p(stale_runs["max"]), + "log_spread_median": math.log1p(max(0.0, spread_median) * 1e6), + "spread_tail_ratio": spread_p99 / max(1e-12, spread_median), + "spread_jump_rate": int(summary["spread_jump"] or 0) / transitions, + "session_activity_dispersion": _count_dispersion(session_counts), + "day_activity_dispersion": _count_dispersion(day_counts), + } + features.update( + { + f"session_activity_share_{name}": value + for name, value in _normalized_activity( + session_counts, SESSION_ACTIVITY_KEYS + ).items() + } + ) + features.update( + { + f"day_activity_share_{name}": value + for name, value in _normalized_activity( + day_counts, DAY_ACTIVITY_KEYS + ).items() + } + ) + rounded = { + name: round(value, selected.rounding_digits) + for name, value in features.items() + if math.isfinite(value) + } + provenance = {name: _feature_provenance(name) for name in rounded} + counts = { + "transition_count": transitions, + "market_open_row_count": market_open_row_count, + "active_window_interval_count": active_interval_count, + "bid_only_count": int(summary["bid_only"] or 0), + "ask_only_count": int(summary["ask_only"] or 0), + "joint_move_count": int(summary["joint"] or 0), + "unchanged_count": int(summary["unchanged"] or 0), + "duplicate_timestamp_count": int(summary["duplicate_timestamp"] or 0), + "non_monotonic_timestamp_count": int( + summary["non_monotonic_timestamp"] or 0 + ), + "burst_interval_count": int(summary["burst_interval"] or 0), + "stale_run_count": stale_runs["count"], + "stale_run_p95": stale_runs["p95"], + "stale_run_max": stale_runs["max"], + "spread_jump_count": int(summary["spread_jump"] or 0), + } + counts.update( + { + f"timestamp_last_digit_{digit}_count": count + for digit, count in enumerate(digit_counts) + } + ) + policy = calendar_policy_metadata() + policy.update( + { + "active_time_policy_version": CALENDAR_POLICY_VERSION, + "market_open_denominator_states": [ + "market_open", + "friday_close", + "sunday_open", + ], + "active_window_basis": "sum_positive_interarrival_at_or_below_cap", + "active_gap_cap_ms": selected.active_gap_cap_ms, + "activity_bin_ms": selected.activity_bin_ms, + "conditioning_activity_basis": ( + "mean_tick_count_per_observed_activity_bin" + ), + } + ) + return FeedEpochEvidenceV2( + symbol=symbol, + period=period, + source_path=str(source), + source_artifact_sha256="sha256:" + _file_sha256(source), + source_size_bytes=source.stat().st_size, + start_timestamp_utc_ms=int(summary["start"]), + end_timestamp_utc_ms=int(summary["end"]), + row_count=row_count, + denominators_ms={ + "calendar_duration_ms": calendar_duration, + "market_open_duration_ms": market_open_duration, + "active_window_duration_ms": active_duration, + }, + counts=counts, + feature_values=rounded, + feature_provenance=provenance, + activity_bin_counts=activity_bins, + calendar_policy=policy, + limitations=( + "hourly_activity_bins_are_synchronization_proxies", + "static_holiday_and_event_calendar_is_advisory", + ), + ) + + +def fit_active_time_feed_epochs( + evidence: Sequence[FeedEpochEvidenceV2], + *, + config: FeedEpochFitConfigV2 | None = None, +) -> FeedEpochDefinitionV2: + """Fit shared robust PELT epochs and symbol-specific deviations.""" + selected = config or FeedEpochFitConfigV2() + prepared = _prepare_evidence(evidence, selected) + prepared = _augment_cross_symbol_evidence(prepared, selected) + symbols = tuple(sorted({item.symbol for item in prepared})) + common_periods = _common_periods(prepared, symbols) + if not common_periods: + raise ValueError("v2 panel has no common symbol periods") + feature_coverage = _feature_coverage(prepared, common_periods, symbols) + features = tuple( + name + for name in selected.feature_names + if feature_coverage.get(name, 0.0) >= selected.min_feature_coverage + ) + if not features: + raise ValueError( + "no v2 features pass the configured coverage threshold" + ) + global_rows = _normalized_panel_rows( + prepared, + periods=common_periods, + symbols=symbols, + features=features, + robust_clip=selected.robust_clip, + ) + candidate_indexes, _ = _pelt_boundaries( + global_rows, + min_segment=selected.min_segment_periods, + penalty=_pelt_penalty(len(common_periods), len(features), selected), + ) + sensitivity = _sensitivity_runs( + prepared, + common_periods, + symbols, + features, + selected, + ) + candidate_supports, candidate_family_supports, uncertainty = ( + _boundary_support( + candidate_indexes, + sensitivity, + tolerance=selected.boundary_match_tolerance_periods, + ) + ) + boundary_indexes = tuple( + index + for index in candidate_indexes + if candidate_supports.get(index, 0.0) >= selected.min_boundary_support + and all( + support >= selected.min_boundary_support + for support in candidate_family_supports.get(index, {}).values() + ) + ) + rejected_indexes = tuple( + index for index in candidate_indexes if index not in boundary_indexes + ) + boundary_supports = { + index: candidate_supports[index] for index in boundary_indexes + } + family_supports = { + index: candidate_family_supports[index] for index in boundary_indexes + } + boundaries: list[FeedEpochBoundaryV2] = [] + pelt_penalty = _pelt_penalty(len(common_periods), len(features), selected) + for position, index in enumerate(boundary_indexes): + segment_start = boundary_indexes[position - 1] if position else 0 + segment_end = ( + boundary_indexes[position + 1] + if position + 1 < len(boundary_indexes) + else len(common_periods) + ) + left_period = common_periods[index - 1] + right_period = common_periods[index] + matched = uncertainty.get(index, (index,)) + item = FeedEpochBoundaryV2( + left_period=left_period, + right_period=right_period, + central_timestamp_utc_ms=_period_start_ms(right_period), + support=round(boundary_supports.get(index, 0.0), 8), + uncertainty_start_period=common_periods[max(0, min(matched))], + uncertainty_end_period=common_periods[ + min(len(common_periods) - 1, max(matched)) + ], + objective_gain=round( + _local_boundary_gain( + global_rows, + segment_start, + index, + segment_end, + pelt_penalty, + ), + 8, + ), + supporting_features=_boundary_supporting_features( + global_rows, + segment_start, + index, + segment_end, + features, + ), + ) + boundaries.append(item) + epochs = _epoch_intervals( + prepared, common_periods, boundary_indexes, features + ) + deviations = _symbol_deviations( + prepared, + periods=common_periods, + symbols=symbols, + features=features, + global_boundaries=boundary_indexes, + global_candidates=candidate_indexes, + config=selected, + ) + run_counts: dict[str, int] = defaultdict(int) + for label, _ in sensitivity: + run_counts[label.split(":", 1)[0]] += 1 + reasons: list[str] = [] + if len(symbols) < selected.min_symbol_count: + reasons.append("insufficient_symbol_support") + if len(common_periods) < selected.min_evidence_periods: + reasons.append("insufficient_common_period_support") + if not boundary_indexes: + reasons.append("no_shared_technology_boundary_detected") + stability = FeedEpochStabilityV2( + status="pass" if not reasons else "fail", + reasons=tuple(reasons), + run_count=len(sensitivity), + run_counts=dict(sorted(run_counts.items())), + boundary_support={ + common_periods[index]: round(value, 8) + for index, value in sorted(boundary_supports.items()) + }, + boundary_support_by_family={ + common_periods[index]: { + family: round(value, 8) + for family, value in sorted(values.items()) + } + for index, values in sorted(family_supports.items()) + }, + rejected_candidates={ + common_periods[index]: { + "overall_support": round(candidate_supports[index], 8), + "support_by_family": { + family: round(value, 8) + for family, value in sorted( + candidate_family_supports[index].items() + ) + }, + "reason": "sensitivity_support_below_threshold", + } + for index in rejected_indexes + }, + feature_coverage={ + name: round(feature_coverage[name], 8) for name in features + }, + common_period_count=len(common_periods), + symbol_count=len(symbols), + ) + lineage = _lineage(prepared, common_periods, sensitivity) + return FeedEpochDefinitionV2( + config=selected, + symbols=symbols, + coverage_start_utc_ms=min( + item.start_timestamp_utc_ms for item in prepared + ), + coverage_end_utc_ms=max(item.end_timestamp_utc_ms for item in prepared), + evidence_count=len(prepared), + period_count=len(common_periods), + feature_names=features, + boundaries=tuple(boundaries), + epochs=epochs, + symbol_deviations=deviations, + stability=stability, + lineage=lineage, + ) + + +def analyze_active_time_feed_epochs( + paths: Iterable[str | Path], + *, + config: FeedEpochFitConfigV2 | None = None, +) -> FeedEpochCampaignV2: + """Discover and fit every supported monthly ASCII tick cache.""" + selected = config or FeedEpochFitConfigV2() + started = time.perf_counter() + discovered = discover_quality_targets(paths) + candidates = [ + target + for target in discovered.targets + if target.kind is QualityTargetKind.CACHE + and target.data_format.lower() == "ascii" + and target.timeframe.upper() == "T" + and _valid_month(target.period) + ] + if not candidates: + raise ValueError("no monthly ASCII tick caches were discovered") + if len(candidates) > selected.max_evidence: + raise ValueError("discovered evidence exceeds configured maximum") + evidence: list[FeedEpochEvidenceV2] = [] + skipped: list[Mapping[str, JSONValue]] = [] + for target in candidates: + try: + evidence.append( + scan_active_time_evidence( + target.path, + symbol=target.symbol, + period=target.period, + config=selected, + ) + ) + except (OSError, TypeError, ValueError) as exc: + skipped.append( + { + "path": target.path, + "symbol": target.symbol, + "period": target.period, + "reason": str(exc)[:512], + } + ) + fitted_evidence = _augment_cross_symbol_evidence(evidence, selected) + definition = fit_active_time_feed_epochs(fitted_evidence, config=selected) + return FeedEpochCampaignV2( + definition=definition, + evidence=fitted_evidence, + source_count=len(evidence), + source_bytes=sum(item.source_size_bytes for item in evidence), + runtime_seconds=round(time.perf_counter() - started, 6), + peak_memory_bytes=peak_rss_bytes(), + skipped_sources=tuple(skipped), + ) + + +def write_active_time_feed_epoch_campaign( + campaign: FeedEpochCampaignV2, + directory: str | Path, +) -> Mapping[str, ArtifactRef]: + """Write separate compact definition, evidence, and campaign artifacts.""" + root = Path(directory).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + payloads: dict[str, Mapping[str, JSONValue]] = { + "definition": campaign.definition.to_dict(), + "evidence": { + "schema_version": FEED_EPOCH_EVIDENCE_V2_SCHEMA_VERSION, + "evidence_count": len(campaign.evidence), + "evidence": [item.to_dict() for item in campaign.evidence], + }, + "campaign": campaign.to_dict(include_evidence=False), + } + artifacts: dict[str, ArtifactRef] = {} + for name, payload in payloads.items(): + encoded = _canonical_json(payload).encode("utf-8") + b"\n" + path = root / f"feed-epochs-v2-{name}.json" + path.write_bytes(encoded) + artifacts[name] = ArtifactRef( + path=str(path), + sha256=hashlib.sha256(encoded).hexdigest(), + size_bytes=len(encoded), + kind=f"feed_epochs_v2_{name}", + ) + return artifacts + + +def read_active_time_feed_epoch_definition( + path: str | Path, +) -> FeedEpochDefinitionV2: + """Read and validate a persisted version-two definition artifact.""" + source = Path(path).expanduser().resolve() + if source.stat().st_size > 64 * 1024 * 1024: + raise ValueError("v2 definition artifact exceeds its size bound") + payload = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise ValueError("v2 definition artifact must be a JSON object") + return FeedEpochDefinitionV2.from_dict(payload) + + +def _activity_conditioning_counts( + lazy: Any, + activity_bin_ms: int, +) -> tuple[dict[str, float], dict[str, float]]: + import polars as pl # pylint: disable=import-outside-toplevel + + timestamp = pl.col("datetime").cast(pl.Int64) + utc_minutes = timestamp // MINUTE_MS + utc_minute = utc_minutes % (24 * 60) + source_minutes = (timestamp + SOURCE_OFFSET_MS) // MINUTE_MS + source_weekday = (source_minutes // (24 * 60) + 3) % 7 + bucket = (timestamp // activity_bin_ms).alias("bucket") + conditions: dict[str, Any] = {} + any_session = pl.lit(False) + for window in SESSION_WINDOWS: + condition = utc_minute.is_between( + window.start_minute_utc, window.end_minute_utc - 1 + ) + conditions[window.name] = condition + any_session = any_session | condition + conditions["off_session"] = ~any_session + session_frame = ( + lazy.select( + bucket, + *( + condition.cast(pl.UInt64).alias(name) + for name, condition in conditions.items() + ), + ) + .group_by("bucket") + .agg(*(pl.col(name).sum() for name in conditions)) + .select(*(pl.col(name).mean().alias(name) for name in conditions)) + .collect(engine="streaming") + ) + day_frame = ( + lazy.select(bucket, source_weekday.alias("weekday")) + .group_by("bucket", "weekday") + .len() + .group_by("weekday") + .agg(pl.col("len").mean().alias("mean_count")) + .collect(engine="streaming") + ) + sessions = { + name: float(session_frame[name][0] or 0.0) for name in conditions + } + days = {str(int(key)): float(value) for key, value in day_frame.iter_rows()} + return sessions, days + + +def _stale_run_summary(path: Path) -> dict[str, int]: + """Return exact bounded summaries of unchanged bid/ask transition runs.""" + import numpy as np # pylint: disable=import-outside-toplevel + import pyarrow as pa # pylint: disable=import-outside-toplevel + import pyarrow.ipc as ipc # pylint: disable=import-outside-toplevel + + histogram: Counter[int] = Counter() + current_run = 0 + previous_bid: float | None = None + previous_ask: float | None = None + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_file(source) + bid_index = reader.schema.get_field_index("bid") + ask_index = reader.schema.get_field_index("ask") + if bid_index < 0 or ask_index < 0: + raise ValueError("tick cache requires bid and ask columns") + for index in range(reader.num_record_batches): + batch = reader.get_batch(index) + bids = batch.column(bid_index).to_numpy(zero_copy_only=False) + asks = batch.column(ask_index).to_numpy(zero_copy_only=False) + if len(bids) == 0: + continue + unchanged = np.empty(len(bids), dtype=np.bool_) + unchanged[0] = ( + previous_bid is not None + and previous_ask is not None + and bids[0] == previous_bid + and asks[0] == previous_ask + ) + unchanged[1:] = (bids[1:] == bids[:-1]) & (asks[1:] == asks[:-1]) + breaks = np.flatnonzero(~unchanged) + if len(breaks) == 0: + current_run += len(unchanged) + else: + current_run += int(breaks[0]) + if current_run: + histogram[current_run] += 1 + between = np.diff(breaks) - 1 + positive = between[between > 0] + if len(positive): + lengths, frequencies = np.unique( + positive, return_counts=True + ) + histogram.update( + { + int(length): int(frequency) + for length, frequency in zip( + lengths, frequencies, strict=True + ) + } + ) + current_run = len(unchanged) - int(breaks[-1]) - 1 + previous_bid = float(bids[-1]) + previous_ask = float(asks[-1]) + if current_run: + histogram[current_run] += 1 + run_count = sum(histogram.values()) + if not run_count: + return {"count": 0, "p95": 0, "max": 0} + target = math.ceil(run_count * 0.95) + cumulative = 0 + p95 = 0 + for length, frequency in sorted(histogram.items()): + cumulative += frequency + if cumulative >= target: + p95 = length + break + return {"count": run_count, "p95": p95, "max": max(histogram)} + + +@lru_cache(maxsize=512) +def _period_denominators(period: str) -> tuple[int, int]: + start = datetime(int(period[:4]), int(period[4:]), 1, tzinfo=timezone.utc) + if start.month == 12: + end = datetime(start.year + 1, 1, 1, tzinfo=timezone.utc) + else: + end = datetime(start.year, start.month + 1, 1, tzinfo=timezone.utc) + calendar_duration = int((end - start).total_seconds() * 1000) + open_minutes = 0 + current = start + while current < end: + source = current + timedelta(hours=-5) + weekday = source.weekday() + minute = source.hour * 60 + source.minute + closed = ( + weekday == 5 + or (weekday == 4 and minute >= FX_CLOSE_OPEN_MINUTE) + or (weekday == 6 and minute < FX_CLOSE_OPEN_MINUTE) + ) + open_minutes += int(not closed) + current += timedelta(minutes=1) + return calendar_duration, open_minutes * MINUTE_MS + + +def _augment_cross_symbol_evidence( + evidence: Sequence[FeedEpochEvidenceV2], + config: FeedEpochFitConfigV2, +) -> tuple[FeedEpochEvidenceV2, ...]: + by_period: dict[str, list[FeedEpochEvidenceV2]] = defaultdict(list) + for item in evidence: + by_period[item.period].append(item) + result: list[FeedEpochEvidenceV2] = [] + for item in evidence: + peers = [value for value in by_period[item.period] if value != item] + correlations: list[float] = [] + overlaps: list[float] = [] + for peer in peers: + keys = sorted( + set(item.activity_bin_counts) | set(peer.activity_bin_counts), + key=int, + ) + left = [float(item.activity_bin_counts.get(key, 0)) for key in keys] + right = [ + float(peer.activity_bin_counts.get(key, 0)) for key in keys + ] + correlation = _correlation(left, right) + if correlation is not None: + correlations.append(correlation) + active_union = sum( + 1 for lvalue, rvalue in zip(left, right) if lvalue or rvalue + ) + active_intersection = sum( + 1 for lvalue, rvalue in zip(left, right) if lvalue and rvalue + ) + if active_union: + overlaps.append(active_intersection / active_union) + features = dict(item.feature_values) + provenance = dict(item.feature_provenance) + if correlations: + features["cross_symbol_activity_correlation"] = round( + median(correlations), config.rounding_digits + ) + provenance["cross_symbol_activity_correlation"] = ( + "activity_bin_counts", + "peer.activity_bin_counts", + ) + if overlaps: + features["cross_symbol_active_bin_overlap"] = round( + median(overlaps), config.rounding_digits + ) + provenance["cross_symbol_active_bin_overlap"] = ( + "activity_bin_counts", + "peer.activity_bin_counts", + ) + result.append( + replace( + item, + feature_values=features, + feature_provenance=provenance, + evidence_id="", + ) + ) + return tuple(result) + + +def _prepare_evidence( + evidence: Sequence[FeedEpochEvidenceV2], config: FeedEpochFitConfigV2 +) -> tuple[FeedEpochEvidenceV2, ...]: + ordered = tuple( + sorted(evidence, key=lambda item: (item.period, item.symbol)) + ) + if not ordered or len(ordered) > config.max_evidence: + raise ValueError("v2 evidence is empty or exceeds configured maximum") + axes = {(item.symbol, item.period) for item in ordered} + if len(axes) != len(ordered): + raise ValueError("duplicate symbol-period v2 evidence") + return ordered + + +def _common_periods( + evidence: Sequence[FeedEpochEvidenceV2], symbols: Sequence[str] +) -> tuple[str, ...]: + by_symbol = { + symbol: {item.period for item in evidence if item.symbol == symbol} + for symbol in symbols + } + return tuple( + sorted(set.intersection(*(by_symbol[symbol] for symbol in symbols))) + ) + + +def _feature_coverage( + evidence: Sequence[FeedEpochEvidenceV2], + periods: Sequence[str], + symbols: Sequence[str], +) -> dict[str, float]: + selected = [ + item + for item in evidence + if item.period in periods and item.symbol in symbols + ] + denominator = max(1, len(selected)) + return { + name: sum(name in item.feature_values for item in selected) + / denominator + for name in DEFAULT_ACTIVE_TIME_FEATURES + } + + +def _normalized_panel_rows( + evidence: Sequence[FeedEpochEvidenceV2], + *, + periods: Sequence[str], + symbols: Sequence[str], + features: Sequence[str], + robust_clip: float, +) -> tuple[tuple[float | None, ...], ...]: + lookup = {(item.symbol, item.period): item for item in evidence} + normalized: dict[tuple[str, str, str], float] = {} + for symbol in symbols: + for feature in features: + values: list[float | None] = [ + lookup[(symbol, period)].feature_values.get(feature) + for period in periods + ] + finite = [value for value in values if value is not None] + if not finite: + continue + center = median(finite) + absolute = [abs(value - center) for value in finite] + scale = max(1e-12, median(absolute) * 1.4826) + for period, value in zip(periods, values): + if value is not None: + normalized[(symbol, period, feature)] = max( + -robust_clip, min(robust_clip, (value - center) / scale) + ) + rows: list[tuple[float | None, ...]] = [] + for period in periods: + row: list[float | None] = [] + for feature in features: + period_values: list[float] = [ + normalized[(symbol, period, feature)] + for symbol in symbols + if (symbol, period, feature) in normalized + ] + row.append(median(period_values) if period_values else None) + rows.append(tuple(row)) + return tuple(rows) + + +def _pelt_penalty( + period_count: int, feature_count: int, config: FeedEpochFitConfigV2 +) -> float: + return ( + config.penalty_multiplier + * math.log(max(2, period_count)) + * math.sqrt(max(1, feature_count)) + ) + + +def _pelt_boundaries( + rows: Sequence[Sequence[float | None]], + *, + min_segment: int, + penalty: float, +) -> tuple[tuple[int, ...], float]: + """Return exact PELT boundaries using a missing-aware squared cost.""" + count = len(rows) + if count < min_segment * 2: + return (), 0.0 + prefix = _cost_prefix(rows) + costs = [math.inf] * (count + 1) + changes: list[tuple[int, ...]] = [()] * (count + 1) + costs[0] = -penalty + candidates: list[int] = [] + for end in range(min_segment, count + 1): + new_candidate = end - min_segment + if new_candidate == 0 or costs[new_candidate] < math.inf: + candidates.append(new_candidate) + evaluated: list[tuple[float, int]] = [] + for start in candidates: + if end - start < min_segment or costs[start] == math.inf: + continue + value = costs[start] + _segment_cost(prefix, start, end) + penalty + evaluated.append((value, start)) + if not evaluated: + continue + best, best_start = min( + evaluated, key=lambda value: (value[0], value[1]) + ) + costs[end] = best + changes[end] = changes[best_start] + ( + (best_start,) if best_start else () + ) + candidates = [ + start + for value, start in evaluated + if value <= best + penalty + 1e-12 + ] + if costs[count] == math.inf: + return (), 0.0 + no_change = _segment_cost(prefix, 0, count) + objective_gain = max(0.0, no_change - costs[count]) + return tuple(index for index in changes[count] if index), objective_gain + + +def _cost_prefix( + rows: Sequence[Sequence[float | None]], +) -> tuple[tuple[list[int], list[float], list[float]], ...]: + width = len(rows[0]) if rows else 0 + result: list[tuple[list[int], list[float], list[float]]] = [] + for column in range(width): + counts = [0] + sums = [0.0] + squares = [0.0] + for row in rows: + value = row[column] + counts.append(counts[-1] + int(value is not None)) + sums.append(sums[-1] + (value or 0.0)) + squares.append(squares[-1] + ((value or 0.0) ** 2)) + result.append((counts, sums, squares)) + return tuple(result) + + +def _segment_cost( + prefix: Sequence[tuple[list[int], list[float], list[float]]], + start: int, + end: int, +) -> float: + result = 0.0 + for counts, sums, squares in prefix: + count = counts[end] - counts[start] + if count < 2: + continue + total = sums[end] - sums[start] + square_total = squares[end] - squares[start] + result += max(0.0, square_total - total * total / count) + return result + + +def _sensitivity_runs( + evidence: Sequence[FeedEpochEvidenceV2], + periods: tuple[str, ...], + symbols: tuple[str, ...], + features: tuple[str, ...], + config: FeedEpochFitConfigV2, +) -> tuple[tuple[str, tuple[int, ...]], ...]: + specs: list[tuple[str, tuple[str, ...], tuple[str, ...], float]] = [] + for multiplier in config.sensitivity_penalty_multipliers: + specs.append((f"penalty:{multiplier:g}", symbols, features, multiplier)) + for symbol in symbols: + retained = tuple(value for value in symbols if value != symbol) + specs.append((f"symbol_holdout:{symbol}", retained, features, 1.0)) + for feature in features: + retained = tuple(value for value in features if value != feature) + if retained: + specs.append((f"feature_holdout:{feature}", symbols, retained, 1.0)) + for label, excluded in ( + ("calendar_policy", "log_calendar_tick_rate_per_hour"), + ("market_open_policy", "log_market_open_tick_rate_per_hour"), + ("duplicate_policy", "duplicate_timestamp_rate"), + ): + retained = tuple(value for value in features if value != excluded) + if retained: + specs.append((f"{label}:exclude", symbols, retained, 1.0)) + specs = specs[: config.max_sensitivity_runs] + runs: list[tuple[str, tuple[int, ...]]] = [] + for label, selected_symbols, selected_features, multiplier in specs: + if not selected_symbols or not selected_features: + continue + rows = _normalized_panel_rows( + evidence, + periods=periods, + symbols=selected_symbols, + features=selected_features, + robust_clip=config.robust_clip, + ) + indexes, _ = _pelt_boundaries( + rows, + min_segment=config.min_segment_periods, + penalty=_pelt_penalty(len(periods), len(selected_features), config) + * multiplier, + ) + runs.append((label, indexes)) + return tuple(runs) + + +def _boundary_support( + boundaries: Sequence[int], + runs: Sequence[tuple[str, tuple[int, ...]]], + *, + tolerance: int, +) -> tuple[ + dict[int, float], + dict[int, dict[str, float]], + dict[int, tuple[int, ...]], +]: + supports: dict[int, float] = {} + family_supports: dict[int, dict[str, float]] = {} + uncertainty: dict[int, tuple[int, ...]] = {} + denominator = max(1, len(runs)) + for boundary in boundaries: + matches = [ + candidate + for _, indexes in runs + for candidate in indexes + if abs(candidate - boundary) <= tolerance + ] + supported_runs = sum( + any(abs(candidate - boundary) <= tolerance for candidate in indexes) + for _, indexes in runs + ) + supports[boundary] = supported_runs / denominator + families = sorted({label.split(":", 1)[0] for label, _ in runs}) + family_supports[boundary] = {} + for family in families: + family_runs = [ + indexes + for label, indexes in runs + if label.split(":", 1)[0] == family + ] + family_supports[boundary][family] = sum( + any( + abs(candidate - boundary) <= tolerance + for candidate in indexes + ) + for indexes in family_runs + ) / max(1, len(family_runs)) + uncertainty[boundary] = tuple(matches or (boundary,)) + return supports, family_supports, uncertainty + + +def _boundary_supporting_features( + rows: Sequence[Sequence[float | None]], + segment_start: int, + index: int, + segment_end: int, + features: Sequence[str], +) -> tuple[str, ...]: + result: list[tuple[float, str]] = [] + for column, feature in enumerate(features): + left: list[float] = [] + right: list[float] = [] + for row in rows[segment_start:index]: + value = row[column] + if value is not None: + left.append(value) + for row in rows[index:segment_end]: + value = row[column] + if value is not None: + right.append(value) + if left and right: + result.append((abs(median(right) - median(left)), feature)) + return tuple(value[1] for value in sorted(result, reverse=True)[:8]) + + +def _local_boundary_gain( + rows: Sequence[Sequence[float | None]], + segment_start: int, + index: int, + segment_end: int, + penalty: float, +) -> float: + prefix = _cost_prefix(rows) + unsplit = _segment_cost(prefix, segment_start, segment_end) + split = _segment_cost(prefix, segment_start, index) + _segment_cost( + prefix, index, segment_end + ) + return max(0.0, unsplit - split - penalty) + + +def _epoch_intervals( + evidence: Sequence[FeedEpochEvidenceV2], + periods: tuple[str, ...], + boundaries: tuple[int, ...], + features: tuple[str, ...], +) -> tuple[FeedEpochIntervalV2, ...]: + result: list[FeedEpochIntervalV2] = [] + indexes = (0, *boundaries, len(periods)) + for number, (start, end) in enumerate(zip(indexes, indexes[1:]), 1): + selected_periods = set(periods[start:end]) + selected = [ + item for item in evidence if item.period in selected_periods + ] + medians = { + feature: round( + median( + item.feature_values[feature] + for item in selected + if feature in item.feature_values + ), + 10, + ) + for feature in features + if any(feature in item.feature_values for item in selected) + } + item = FeedEpochIntervalV2( + label=f"technology_epoch_{number:02d}", + period_start=periods[start], + period_end=periods[end - 1], + start_timestamp_utc_ms=_period_start_ms(periods[start]), + end_timestamp_utc_ms=_period_end_ms(periods[end - 1]), + evidence_count=len(selected), + feature_medians=medians, + ) + result.append(item) + return tuple(result) + + +def _symbol_deviations( + evidence: Sequence[FeedEpochEvidenceV2], + *, + periods: tuple[str, ...], + symbols: tuple[str, ...], + features: tuple[str, ...], + global_boundaries: tuple[int, ...], + global_candidates: tuple[int, ...], + config: FeedEpochFitConfigV2, +) -> tuple[FeedEpochSymbolDeviationV2, ...]: + result: list[FeedEpochSymbolDeviationV2] = [] + for symbol in symbols: + rows = _normalized_panel_rows( + evidence, + periods=periods, + symbols=(symbol,), + features=features, + robust_clip=config.robust_clip, + ) + indexes, _ = _pelt_boundaries( + rows, + min_segment=config.min_segment_periods, + penalty=_pelt_penalty(len(periods), len(features), config), + ) + for index in indexes: + if any( + abs(index - candidate) + <= config.boundary_match_tolerance_periods + for candidate in global_candidates + ): + continue + distance = ( + min(abs(index - value) for value in global_boundaries) + if global_boundaries + else None + ) + if ( + distance is None + or distance > config.boundary_match_tolerance_periods + ): + result.append( + FeedEpochSymbolDeviationV2( + symbol=symbol, + left_period=periods[index - 1], + right_period=periods[index], + nearest_global_distance_periods=distance, + ) + ) + return tuple(result) + + +def _lineage( + evidence: Sequence[FeedEpochEvidenceV2], + periods: Sequence[str], + sensitivity: Sequence[tuple[str, tuple[int, ...]]], +) -> Mapping[str, JSONValue]: + sources: list[JSONValue] = [ + { + "symbol": item.symbol, + "period": item.period, + "source_artifact_sha256": item.source_artifact_sha256, + "evidence_id": item.evidence_id, + } + for item in evidence + ] + payload: dict[str, JSONValue] = { + "fitting_basis": "real_ascii_tick_arrow_caches", + "algorithm": "robust_multivariate_pelt", + "cost": "within_segment_winsorized_squared_error", + "global_aggregation": "within_symbol_robust_scale_then_symbol_median", + "calendar_policy_version": CALENDAR_POLICY_VERSION, + "common_period_start": periods[0], + "common_period_end": periods[-1], + "sources": sources, + "sensitivity_runs": [ + {"label": label, "boundary_indexes": list(indexes)} + for label, indexes in sensitivity + ], + } + payload["lineage_sha256"] = _payload_sha256(payload) + return payload + + +def _feature_provenance(name: str) -> tuple[str, ...]: + if name.startswith("log_calendar"): + return ("row_count", "denominators_ms.calendar_duration_ms") + if name.startswith("log_market_open"): + return ( + "counts.market_open_row_count", + "denominators_ms.market_open_duration_ms", + ) + if name.startswith("log_active_window"): + return ( + "counts.active_window_interval_count", + "denominators_ms.active_window_duration_ms", + ) + if name.startswith("cross_symbol"): + return ("activity_bin_counts", "peer.activity_bin_counts") + if name in { + "bid_only_rate", + "ask_only_rate", + "joint_move_rate", + "unchanged_rate", + "stale_quote_rate", + }: + return ( + "cache.bid.observed_sequence_diff", + "cache.ask.observed_sequence_diff", + "counts.transition_count", + ) + if name == "subwindow_count_fano": + return ("activity_bin_counts", "variance_over_mean") + if name in { + "log_interarrival_median_ms", + "interarrival_dispersion", + "interarrival_lag1", + "burst_interval_rate", + }: + return ("cache.datetime.observed_sequence_diff",) + if name in { + "timestamp_exact_second_rate", + "timestamp_last_digit_entropy", + "duplicate_timestamp_rate", + }: + return ("cache.datetime", "cache.datetime.observed_sequence_diff") + if name == "price_precision_digits": + return ( + "cache.bid.nonzero_diff_p01", + "cache.ask.nonzero_diff_p01", + "negative_log10_rounded", + ) + if name in {"log_spread_median", "spread_tail_ratio"}: + return ("cache.ask_minus_bid",) + if name == "spread_jump_rate": + return ( + "cache.ask_minus_bid.observed_sequence_diff", + "median_spread_x5_threshold", + ) + if name.startswith("log_stale_run"): + return ( + "cache.bid.observed_sequence_diff", + "cache.ask.observed_sequence_diff", + "exact_run_length_histogram", + ) + if name.startswith("session_activity_share_"): + return ( + "cache.datetime", + "calendar_policy.session_windows", + "mean_tick_count_per_observed_activity_bin", + ) + if name.startswith("day_activity_share_"): + return ( + "cache.datetime", + "calendar_policy.source_timezone", + "mean_tick_count_per_observed_activity_bin", + ) + if name in {"session_activity_dispersion", "day_activity_dispersion"}: + return ( + "cache.datetime", + "mean_tick_count_per_observed_activity_bin", + "coefficient_of_variation", + ) + return ("cache.datetime", "cache.bid", "cache.ask") + + +def _fano(values: Sequence[float]) -> float: + if not values: + return 0.0 + mean = sum(values) / len(values) + if mean <= 0: + return 0.0 + variance = sum((value - mean) ** 2 for value in values) / len(values) + return variance / mean + + +def _normalized_entropy(counts: Sequence[int]) -> float: + total = sum(counts) + if total <= 0: + return 0.0 + entropy = -sum( + (value / total) * math.log(value / total) + for value in counts + if value > 0 + ) + return entropy / math.log(max(2, len(counts))) + + +def _count_dispersion(counts: Mapping[str, int | float]) -> float: + values = [float(value) for value in counts.values()] + if not values: + return 0.0 + center = sum(values) / len(values) + if center <= 0: + return 0.0 + return ( + math.sqrt(sum((value - center) ** 2 for value in values) / len(values)) + / center + ) + + +def _normalized_activity( + counts: Mapping[str, int | float], keys: Sequence[str] +) -> dict[str, float]: + values = {key: float(counts.get(key, 0.0)) for key in keys} + total = sum(values.values()) + if total <= 0: + return {key: 0.0 for key in keys} + return {key: value / total for key, value in values.items()} + + +def _correlation(left: Sequence[float], right: Sequence[float]) -> float | None: + if len(left) != len(right) or len(left) < 2: + return None + left_mean = sum(left) / len(left) + right_mean = sum(right) / len(right) + numerator = sum( + (lvalue - left_mean) * (rvalue - right_mean) + for lvalue, rvalue in zip(left, right) + ) + left_square = sum((value - left_mean) ** 2 for value in left) + right_square = sum((value - right_mean) ** 2 for value in right) + denominator = math.sqrt(left_square * right_square) + return numerator / denominator if denominator else None + + +def _period_start_ms(period: str) -> int: + value = datetime(int(period[:4]), int(period[4:]), 1, tzinfo=timezone.utc) + return int(value.timestamp() * 1000) + + +def _period_end_ms(period: str) -> int: + start = datetime(int(period[:4]), int(period[4:]), 1, tzinfo=timezone.utc) + if start.month == 12: + end = datetime(start.year + 1, 1, 1, tzinfo=timezone.utc) + else: + end = datetime(start.year, start.month + 1, 1, tzinfo=timezone.utc) + return int(end.timestamp() * 1000) - 1 + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _payload_sha256(payload: Mapping[str, JSONValue]) -> str: + return ( + "sha256:" + + hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() + ) + + +def _sha256_id(value: Any, name: str) -> str: + text = str(value or "").strip().lower() + digest = text.removeprefix("sha256:") + if len(digest) != 64 or any( + character not in "0123456789abcdef" for character in digest + ): + raise ValueError(f"{name} must be a sha256 identifier") + return "sha256:" + digest + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = _canonical_json(payload).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _canonical_json(payload: Mapping[str, JSONValue]) -> str: + return json.dumps( + payload, sort_keys=True, separators=(",", ":"), allow_nan=False + ) + + +def _valid_month(value: str) -> bool: + return len(value) == 6 and value.isdigit() and 1 <= int(value[4:]) <= 12 + + +def _required_text(value: Any, name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"{name} is required") + return text + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _integer_text(value: Any, name: str) -> int: + try: + result = int(str(value)) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be an integer") from exc + return result + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + result = _strict_int(value, name) + if not minimum <= result <= maximum: + raise ValueError(f"{name} is outside its supported range") + return result + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be finite") + try: + result = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be finite") from exc + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _bounded_float( + value: Any, name: str, minimum: float, maximum: float +) -> float: + result = _finite_float(value, name) + if not minimum <= result <= maximum: + raise ValueError(f"{name} is outside its supported range") + return result + + +def _optional_float(value: Any) -> float | None: + try: + result = float(value) + except (TypeError, ValueError): + return None + return result if math.isfinite(result) else None + + +def _positive_float(value: Any) -> float | None: + result = _optional_float(value) + return result if result is not None and result > 0 else None + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + return value + return () + + +def _int_mapping(value: Any) -> dict[str, int]: + return { + str(name): _strict_int(item, str(name)) + for name, item in _mapping(value).items() + } + + +def _float_mapping(value: Any) -> dict[str, float]: + return { + str(name): _finite_float(item, str(name)) + for name, item in _mapping(value).items() + } diff --git a/src/histdatacom/data_analytics/feed_regimes.py b/src/histdatacom/data_analytics/feed_regimes.py index 3382eac4..2e262c42 100644 --- a/src/histdatacom/data_analytics/feed_regimes.py +++ b/src/histdatacom/data_analytics/feed_regimes.py @@ -1,49 +1,44 @@ -"""Feed-regime analytics for HistData ASCII tick artifacts.""" +"""Canonical feed-regime compatibility and feed-epoch analytics surface.""" from __future__ import annotations -from collections import Counter, defaultdict -from collections.abc import Iterable, Sequence -from dataclasses import dataclass, field -from datetime import datetime, timedelta, timezone import hashlib import json -from math import ceil +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field from pathlib import Path -import re - -from histdatacom.histdata_ascii import ( - CACHE_FILENAME, - EST_NO_DST_OFFSET_MS, - TICK, - read_ascii_file, - read_polars_cache, +from statistics import median +from typing import Any + +from histdatacom.data_analytics.feed_epochs import ( + FeedEpochDefinitionV1, + FeedEpochEvidenceV1, + FeedEpochFitConfigV1, + fit_feed_epochs, +) +from histdatacom.data_quality.discovery import ( + discover_quality_targets, + quality_target_from_path, ) +from histdatacom.data_quality.engine import run_quality_assessment +from histdatacom.data_quality.fingerprints import ( + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataFingerprintProfile, + fingerprint_quality_rules, +) +from histdatacom.histdata_ascii import TICK from histdatacom.runtime_contracts import ArtifactRef, JSONValue ANALYTICS_REPORT_SCHEMA_VERSION = "histdatacom.feed-regime-report.v1" FEED_REGIME_OPERATION = "feed-regime-detection" DEFAULT_QUIET_GAP_MS = 60_000 _ASCII_FORMAT = "ascii" -_CSV_SUFFIX = ".csv" -_ZIP_SUFFIX = ".zip" - -_HISTDATA_DATA_FILENAME_RE = re.compile( - r"^DAT_(?PASCII)_(?P[A-Z0-9]+)_" - r"(?P[A-Z0-9]+)_(?P\d{4}(?:\d{2})?)" - r"(?:_[A-Z0-9_]+)?(?:\.csv)?$", - re.IGNORECASE, -) -_HISTDATA_ARCHIVE_FILENAME_RE = re.compile( - r"^HISTDATA_COM_(?PASCII)_(?P[A-Z0-9]+)_" - r"(?P[A-Z0-9]+)(?P\d{4}(?:\d{2})?)$", - re.IGNORECASE, -) @dataclass(frozen=True, slots=True) class AnalyticsTarget: - """One local artifact considered by feed-regime analytics.""" + """Compatibility projection of one canonical quality target.""" path: str kind: str @@ -55,7 +50,7 @@ class AnalyticsTarget: @property def is_supported_tick_target(self) -> bool: - """Return whether the target can feed tick-regime analytics.""" + """Return whether the target can feed technological epoch fitting.""" return self.data_format == _ASCII_FORMAT and self.timeframe == TICK def to_dict(self) -> dict[str, JSONValue]: @@ -73,7 +68,7 @@ def to_dict(self) -> dict[str, JSONValue]: @dataclass(frozen=True, slots=True) class AnalyticsDiscoveryResult: - """Result of discovering local analytics targets.""" + """Canonical discovery result projected for analytics callers.""" roots: tuple[str, ...] = () targets: tuple[AnalyticsTarget, ...] = () @@ -96,7 +91,7 @@ def to_dict(self) -> dict[str, JSONValue]: @dataclass(frozen=True, slots=True) class FeedPeriodProfile: - """Tick-feed summary statistics for one symbol/time bucket.""" + """Canonical fingerprint summary for one symbol-period.""" symbol: str period: str @@ -149,7 +144,7 @@ def to_dict(self) -> dict[str, JSONValue]: @dataclass(frozen=True, slots=True) class FeedRegimeEra: - """A contiguous run of similar feed behavior for one symbol.""" + """Compatibility projection of one versioned technological epoch.""" symbol: str label: str @@ -190,11 +185,12 @@ def to_dict(self) -> dict[str, JSONValue]: @dataclass(frozen=True, slots=True) class FeedRegimeReport: - """Machine-readable feed-regime analytics output.""" + """Machine-readable compatibility report containing the epoch artifact.""" discovery: AnalyticsDiscoveryResult period_profiles: tuple[FeedPeriodProfile, ...] = () regimes: tuple[FeedRegimeEra, ...] = () + epoch_definition: FeedEpochDefinitionV1 | None = None metadata: dict[str, JSONValue] = field(default_factory=dict) def summary(self) -> dict[str, JSONValue]: @@ -207,13 +203,22 @@ def summary(self) -> dict[str, JSONValue]: "operation": FEED_REGIME_OPERATION, "target_count": self.discovery.target_count, "supported_target_count": sum( - 1 + target.is_supported_tick_target for target in self.discovery.targets - if target.is_supported_tick_target ), "profile_count": len(self.period_profiles), "regime_count": len(self.regimes), "symbols": symbols, + "epoch_definition_id": ( + self.epoch_definition.definition_id + if self.epoch_definition is not None + else None + ), + "stability_status": ( + self.epoch_definition.stability.status + if self.epoch_definition is not None + else "unavailable" + ), } def to_dict(self) -> dict[str, JSONValue]: @@ -227,80 +232,33 @@ def to_dict(self) -> dict[str, JSONValue]: profile.to_dict() for profile in self.period_profiles ], "regimes": [regime.to_dict() for regime in self.regimes], + "epoch_definition": ( + self.epoch_definition.to_dict() + if self.epoch_definition is not None + else None + ), "metadata": dict(self.metadata), } -@dataclass(frozen=True, slots=True) -class _TickObservation: - utc_ms: int - bid: float - ask: float - symbol: str - target_path: str - - def discover_analytics_targets( paths: Iterable[str | Path], ) -> AnalyticsDiscoveryResult: - """Discover local files usable by data-analytics operations.""" - roots = _normalize_roots(paths) - seen_paths: set[str] = set() - targets: list[AnalyticsTarget] = [] - for root in roots: - if not root.exists(): - raise ValueError(f"analytics target path does not exist: {root}") - candidates = ( - sorted(path for path in root.rglob("*") if path.is_file()) - if root.is_dir() - else [root] - ) - for candidate in candidates: - target = analytics_target_from_path(candidate) - if target is None: - if root == candidate: - raise ValueError( - f"unsupported analytics target file type: {candidate}" - ) - continue - resolved = str(candidate.resolve()) - if resolved in seen_paths: - continue - seen_paths.add(resolved) - targets.append(target) - + """Project canonical quality discovery without an independent scanner.""" + discovery = discover_quality_targets(paths) return AnalyticsDiscoveryResult( - roots=tuple(str(root) for root in roots), - targets=tuple(sorted(targets, key=lambda target: target.path)), - metadata={ - "operation": FEED_REGIME_OPERATION, - "supported_timeframe": TICK, - "quality_semantics": "analytics-only; no pass/fail status", - }, + roots=discovery.roots, + targets=tuple( + _analytics_target(target) for target in discovery.targets + ), + metadata=_discovery_metadata(), ) def analytics_target_from_path(path: str | Path) -> AnalyticsTarget | None: - """Return a local analytics target for supported artifact paths.""" - source = Path(path) - kind = _target_kind(source) - if kind is None: - return None - metadata = _metadata_from_filename(source) - metadata["filename"] = source.name - metadata["supported_for_feed_regimes"] = ( - metadata.get("data_format") == _ASCII_FORMAT - and metadata.get("timeframe") == TICK - ) - return AnalyticsTarget( - path=str(source.resolve()), - kind=kind, - data_format=str(metadata.get("data_format", "") or ""), - timeframe=str(metadata.get("timeframe", "") or ""), - symbol=str(metadata.get("symbol", "") or ""), - period=str(metadata.get("period", "") or ""), - metadata=metadata, - ) + """Project one canonical quality target into the analytics type.""" + target = quality_target_from_path(path) + return _analytics_target(target) if target is not None else None def analyze_feed_regimes( @@ -308,39 +266,76 @@ def analyze_feed_regimes( *, bucket: str = "month", quiet_gap_ms: int = DEFAULT_QUIET_GAP_MS, + fit_config: FeedEpochFitConfigV1 | None = None, + fingerprint_profile: HistDataFingerprintProfile | None = None, ) -> FeedRegimeReport: - """Analyze tick-rate and quote-update regimes across local targets.""" + """Fit feed epochs through canonical discovery and fingerprint evidence.""" normalized_bucket = _normalize_bucket(bucket) - discovery = discover_analytics_targets(paths) - observations_by_bucket: dict[tuple[str, str], list[_TickObservation]] - observations_by_bucket = defaultdict(list) - for target in discovery.targets: - for observation in _target_tick_observations(target): - period = _period_for_observation(observation, normalized_bucket) - observations_by_bucket[(observation.symbol, period)].append( - observation - ) - - profiles = tuple( - _profile_observations( - symbol=symbol, - period=period, - bucket=normalized_bucket, - observations=tuple(observations), - quiet_gap_ms=quiet_gap_ms, + if quiet_gap_ms <= 0: + raise ValueError("quiet_gap_ms must be positive") + quality_discovery = discover_quality_targets(tuple(paths)) + discovery = AnalyticsDiscoveryResult( + roots=quality_discovery.roots, + targets=tuple( + _analytics_target(target) for target in quality_discovery.targets + ), + metadata=_discovery_metadata(), + ) + quality_report = run_quality_assessment( + quality_discovery.targets, + fingerprint_quality_rules(fingerprint_profile), + metadata={ + "operation": FEED_REGIME_OPERATION, + "consumer": "feed_epoch_fitting", + }, + ) + raw_evidence = tuple( + FeedEpochEvidenceV1.from_fingerprint(payload) + for payload in ( + finding.metadata.get(TIME_SERIES_FINGERPRINT_METADATA_KEY) + for finding in quality_report.findings ) - for (symbol, period), observations in sorted( - observations_by_bucket.items() + if isinstance(payload, Mapping) and _fingerprint_is_usable_tick(payload) + ) + evidence, evidence_preparation = _prepare_canonical_evidence( + raw_evidence, + requested_bucket=normalized_bucket, + ) + effective_bucket = str(evidence_preparation["effective_bucket"]) + effective_config = fit_config + if effective_config is None and len({item.period for item in evidence}) < 6: + effective_config = FeedEpochFitConfigV1( + min_evidence_periods=2, + min_segment_periods=1, ) + definition = fit_feed_epochs(evidence, config=effective_config) + profiles = tuple( + _period_profile_from_evidence(item, bucket=effective_bucket) + for item in evidence + ) + regimes = _regimes_from_definition( + definition, + bucket=effective_bucket, + profiles=profiles, ) - regimes = _segment_regimes(profiles) return FeedRegimeReport( discovery=discovery, period_profiles=profiles, regimes=regimes, + epoch_definition=definition, metadata={ "quiet_gap_ms": quiet_gap_ms, - "bucket": normalized_bucket, + "bucket": effective_bucket, + "requested_bucket": normalized_bucket, + "evidence_preparation": evidence_preparation, + "fitting_basis": "canonical_time_series_fingerprint", + "fingerprint_target_count": len(quality_discovery.targets), + "raw_fingerprint_evidence_count": len(raw_evidence), + "evidence_count": len(evidence), + "config_id": definition.config.config_id, + "short_history_compatibility_fit": ( + fit_config is None and definition.period_count < 6 + ), "quality_semantics": ( "Feed-regime analytics are descriptive feature-engineering " "signals and do not imply data-quality pass/fail status." @@ -358,23 +353,31 @@ def write_feed_regime_report( report: FeedRegimeReport, path: str | Path, ) -> ArtifactRef: - """Write a feed-regime report and return its artifact reference.""" + """Write a report containing its versioned epoch artifact.""" output = Path(path).expanduser() output.parent.mkdir(parents=True, exist_ok=True) encoded = f"{feed_regime_report_to_json(report)}\n".encode("utf-8") output.write_bytes(encoded) - digest = hashlib.sha256(encoded).hexdigest() + definition = report.epoch_definition return ArtifactRef( kind="feed-regime-report", path=str(output.resolve()), size_bytes=len(encoded), - sha256=digest, + sha256=hashlib.sha256(encoded).hexdigest(), metadata={ "schema_version": ANALYTICS_REPORT_SCHEMA_VERSION, "operation": FEED_REGIME_OPERATION, "target_count": report.discovery.target_count, "profile_count": len(report.period_profiles), "regime_count": len(report.regimes), + "epoch_definition_id": ( + definition.definition_id if definition is not None else None + ), + "stability_status": ( + definition.stability.status + if definition is not None + else "unavailable" + ), }, ) @@ -384,7 +387,7 @@ def format_feed_regime_console_summary( *, artifact: ArtifactRef | None = None, ) -> str: - """Return a compact human-readable feed-regime summary.""" + """Return a compact human-readable technological epoch summary.""" summary = report.summary() lines = [ "Feed regime analytics", @@ -392,14 +395,14 @@ def format_feed_regime_console_summary( f"supported tick targets: {summary['supported_target_count']}", f"profiles: {summary['profile_count']}", f"regimes: {summary['regime_count']}", + f"stability: {summary['stability_status']}", ] if artifact is not None: lines.append(f"report: {artifact.path}") if not report.period_profiles: - lines.append("No supported ASCII tick targets discovered.") + lines.append("No supported ASCII tick fingerprint evidence discovered.") return "\n".join(lines) - - lines.extend(("", "Regime eras")) + lines.extend(("", "Technological epochs")) for regime in report.regimes: lines.append( "- " @@ -408,324 +411,458 @@ def format_feed_regime_console_summary( f"rate={_round_float(regime.mean_tick_rate_per_hour)}/hour " f"median_gap={_round_float(regime.median_interarrival_ms)}ms" ) + definition = report.epoch_definition + if definition is not None and definition.boundaries: + lines.extend(("", "Uncertain transitions")) + for boundary in definition.boundaries: + lines.append( + "- " + f"{boundary.left_period}->{boundary.right_period} " + f"support={boundary.support} confidence={boundary.confidence} " + f"interval={boundary.uncertainty_start_utc_ms}:" + f"{boundary.uncertainty_end_utc_ms}" + ) return "\n".join(lines) -def _normalize_roots(paths: Iterable[str | Path]) -> tuple[Path, ...]: - roots = tuple(Path(path).expanduser() for path in paths) - if not roots: - raise ValueError("at least one analytics target path is required") - return roots - - -def _target_kind(path: Path) -> str | None: - if path.name == CACHE_FILENAME: - return "cache" - suffix = path.suffix.lower() - if suffix == _CSV_SUFFIX: - return "csv" - if suffix == _ZIP_SUFFIX: - return "zip" - return None - - -def _metadata_from_filename(path: Path) -> dict[str, JSONValue]: - if path.name == CACHE_FILENAME: - return { - "data_format": "", - "timeframe": "", - "symbol": "", - "period": "", - "cache_metadata_required": True, - } - stem = path.stem if path.suffix.lower() == _ZIP_SUFFIX else path.name - match = _HISTDATA_DATA_FILENAME_RE.match(path.name) - if match is None: - match = _HISTDATA_ARCHIVE_FILENAME_RE.match(stem) - if match is None: - return { - "data_format": "", - "timeframe": "", - "symbol": "", - "period": "", - } - groups = match.groupdict() +def _discovery_metadata() -> dict[str, JSONValue]: return { - "data_format": str(groups.get("format", "")).lower(), - "timeframe": str(groups.get("timeframe", "")).upper(), - "symbol": str(groups.get("symbol", "")).upper(), - "period": str(groups.get("period", "")), + "operation": FEED_REGIME_OPERATION, + "supported_timeframe": TICK, + "quality_semantics": "analytics-only; no pass/fail status", + "discovery_basis": "canonical_quality_discovery", } -def _target_tick_observations( - target: AnalyticsTarget, -) -> tuple[_TickObservation, ...]: - if not target.is_supported_tick_target: - return () - if target.kind == "cache": - return _cache_tick_observations(target) - batch = read_ascii_file(Path(target.path), target.timeframe) - return tuple( - _TickObservation( - utc_ms=int(row[0]), - bid=float(row[1]), - ask=float(row[2]), - symbol=target.symbol or "UNKNOWN", - target_path=target.path, - ) - for row in batch.rows +def _analytics_target(target: Any) -> AnalyticsTarget: + metadata = dict(target.metadata) + metadata["supported_for_feed_regimes"] = ( + target.data_format == _ASCII_FORMAT and target.timeframe == TICK + ) + metadata["discovery_basis"] = "canonical_quality_discovery" + return AnalyticsTarget( + path=target.path, + kind=target.kind.value, + data_format=target.data_format, + timeframe=target.timeframe, + symbol=target.symbol, + period=target.period, + metadata=metadata, ) -def _cache_tick_observations( - target: AnalyticsTarget, -) -> tuple[_TickObservation, ...]: - frame = read_polars_cache(Path(target.path)) - required = {"datetime", "bid", "ask"} - if not required.issubset(set(frame.columns)): - return () - symbol = target.symbol or "UNKNOWN" - return tuple( - _TickObservation( - utc_ms=int(row["datetime"]), - bid=float(row["bid"]), - ask=float(row["ask"]), - symbol=symbol, - target_path=target.path, - ) - for row in frame.select(["datetime", "bid", "ask"]).iter_rows( - named=True - ) +def _fingerprint_is_usable_tick(payload: Mapping[str, Any]) -> bool: + axis = payload.get("target_axis") + coverage = payload.get("coverage") + source = payload.get("source") + if not isinstance(axis, Mapping) or not isinstance(coverage, Mapping): + return False + if not isinstance(source, Mapping): + return False + return ( + str(axis.get("data_format", "")).lower() == _ASCII_FORMAT + and str(axis.get("timeframe", "")).upper() == TICK + and source.get("kind") != "unavailable" + and int(coverage.get("parsed_row_count", 0) or 0) > 0 ) -def _normalize_bucket(bucket: str) -> str: - normalized = str(bucket or "month").strip().lower() - if normalized not in {"month", "year"}: - raise ValueError("feed-regime bucket must be 'month' or 'year'") - return normalized +def _prepare_canonical_evidence( + evidence: Sequence[FeedEpochEvidenceV1], + *, + requested_bucket: str, +) -> tuple[tuple[FeedEpochEvidenceV1, ...], dict[str, JSONValue]]: + """Select one canonical axis and coarsen safely to a common period grid.""" + by_axis: dict[tuple[str, str], list[FeedEpochEvidenceV1]] = defaultdict( + list + ) + for item in evidence: + by_axis[(item.symbol, item.period)].append(item) + selected = tuple( + min(items, key=_canonical_evidence_rank) + for _, items in sorted(by_axis.items()) + ) + duplicate_axis_count = len(evidence) - len(selected) + has_annual_evidence = any(len(item.period) == 4 for item in selected) + effective_bucket = ( + "year" if requested_bucket == "year" or has_annual_evidence else "month" + ) + annual_overlap_skip_count = 0 + if effective_bucket == "year": + by_year: dict[tuple[str, str], list[FeedEpochEvidenceV1]] = defaultdict( + list + ) + for item in selected: + by_year[(item.symbol, item.period[:4])].append(item) + prepared: list[FeedEpochEvidenceV1] = [] + for (_, year), items in sorted(by_year.items()): + annual = [item for item in items if len(item.period) == 4] + if annual: + chosen = min(annual, key=_canonical_evidence_rank) + prepared.append(chosen) + annual_overlap_skip_count += len(items) - 1 + else: + prepared.append(_aggregate_annual_evidence(items, year=year)) + result = tuple(prepared) + else: + result = selected + reason = "requested_bucket" + if requested_bucket == "month" and has_annual_evidence: + reason = "annual_evidence_cannot_be_safely_disaggregated" + return result, { + "requested_bucket": requested_bucket, + "effective_bucket": effective_bucket, + "effective_bucket_reason": reason, + "raw_evidence_count": len(evidence), + "selected_axis_count": len(selected), + "fitted_evidence_count": len(result), + "duplicate_axis_skip_count": duplicate_axis_count, + "annual_overlap_skip_count": annual_overlap_skip_count, + "duplicate_axis_policy": "prefer_direct_cache_then_sibling_cache_then_text", + "mixed_granularity_policy": "coarsen_to_year_never_disaggregate", + "annual_overlap_policy": "prefer_canonical_annual_evidence", + } -def _period_for_observation( - observation: _TickObservation, - bucket: str, -) -> str: - source = _source_datetime(observation.utc_ms) - if bucket == "year": - return f"{source.year:04d}" - return f"{source.year:04d}{source.month:02d}" +def _canonical_evidence_rank(item: FeedEpochEvidenceV1) -> tuple[int, str]: + cache_source = str(item.quality.get("cache_source", "")) + if item.source_kind == "cache" and cache_source == "direct": + rank = 0 + elif item.source_kind == "cache": + rank = 1 + elif item.source_kind == "csv_text": + rank = 2 + elif item.source_kind == "zip_member": + rank = 3 + else: + rank = 4 + return rank, item.evidence_id -def _profile_observations( +def _aggregate_annual_evidence( + items: Sequence[FeedEpochEvidenceV1], *, - symbol: str, - period: str, - bucket: str, - observations: Sequence[_TickObservation], - quiet_gap_ms: int, -) -> FeedPeriodProfile: - ordered = tuple(sorted(observations, key=lambda item: item.utc_ms)) - timestamps = [item.utc_ms for item in ordered] - deltas = [ - max(0, current - previous) - for previous, current in zip(timestamps, timestamps[1:], strict=False) + year: str, +) -> FeedEpochEvidenceV1: + ordered = tuple( + sorted(items, key=lambda item: (item.period, item.evidence_id)) + ) + symbol = ordered[0].symbol + component_sources: list[JSONValue] = [ + { + "fingerprint_id": item.fingerprint_id, + "source_artifact_sha256": item.source_artifact_sha256, + "source_hash_basis": item.source_hash_basis, + "symbol": item.symbol, + "period": item.period, + "source_kind": item.source_kind, + } + for item in ordered ] - spreads = [item.ask - item.bid for item in ordered] - quote_update_count = _quote_update_count(ordered) - zero_runs, zero_ticks = _zero_change_runs(ordered) - duration_ms = max(0, timestamps[-1] - timestamps[0]) if timestamps else 0 - rate = len(ordered) * 3_600_000 / duration_ms if duration_ms > 0 else 0.0 - return FeedPeriodProfile( + feature_names = tuple( + sorted({name for item in ordered for name in item.feature_values}) + ) + feature_values = { + name: float( + median( + item.feature_values[name] + for item in ordered + if name in item.feature_values + ) + ) + for name in feature_names + } + feature_provenance = { + name: tuple( + sorted( + { + path + for item in ordered + for path in item.feature_provenance.get(name, ()) + } + ) + ) + for name in feature_names + } + fingerprint_id = _semantic_sha256( + { + "kind": "canonical_fingerprint_annual_aggregate", + "symbol": symbol, + "period": year, + "fingerprint_ids": [item.fingerprint_id for item in ordered], + } + ) + source_hash = _semantic_sha256( + { + "kind": "canonical_fingerprint_source_aggregate", + "source_hashes": [item.source_artifact_sha256 for item in ordered], + } + ) + return FeedEpochEvidenceV1( symbol=symbol, - period=period, - bucket=bucket, - row_count=len(ordered), - start_utc_ms=timestamps[0] if timestamps else 0, - end_utc_ms=timestamps[-1] if timestamps else 0, - tick_rate_per_hour=rate, - median_interarrival_ms=_median(deltas), - p95_interarrival_ms=_percentile(deltas, 95), - max_interarrival_ms=max(deltas) if deltas else 0, - quiet_gap_count=sum(1 for delta in deltas if delta >= quiet_gap_ms), - quote_update_count=quote_update_count, - quote_update_ratio=( - quote_update_count / max(1, len(ordered) - 1) if ordered else 0.0 + period=year, + start_timestamp_utc_ms=min( + item.start_timestamp_utc_ms for item in ordered ), - zero_change_run_count=zero_runs, - zero_change_tick_count=zero_ticks, - spread_min=min(spreads) if spreads else 0.0, - spread_median=_median(spreads), - spread_mean=(sum(spreads) / len(spreads) if spreads else 0.0), - spread_max=max(spreads) if spreads else 0.0, - session_counts=_session_counts(ordered), - target_paths=tuple(sorted({item.target_path for item in ordered})), + end_timestamp_utc_ms=max(item.end_timestamp_utc_ms for item in ordered), + fingerprint_id=fingerprint_id, + source_artifact_sha256=source_hash, + source_hash_basis="canonical_fingerprint_aggregate_id", + source_kind="canonical_fingerprint_aggregate", + feature_values=feature_values, + feature_provenance=feature_provenance, + conditioning=_aggregate_conditioning(ordered), + quality={ + "aggregation": "median_monthly_canonical_fingerprints", + "component_sources": component_sources, + "component_count": len(ordered), + "cache_source": "mixed", + "sequence_status": "derived", + "limitations": ["annual_values_are_monthly_fingerprint_aggregates"], + }, + profile=_aggregate_profile(ordered), ) -def _segment_regimes( - profiles: Sequence[FeedPeriodProfile], -) -> tuple[FeedRegimeEra, ...]: - grouped: dict[str, list[FeedPeriodProfile]] = defaultdict(list) - for profile in profiles: - grouped[profile.symbol].append(profile) - - regimes: list[FeedRegimeEra] = [] - for symbol, symbol_profiles in sorted(grouped.items()): - ordered = sorted(symbol_profiles, key=lambda item: item.period) - labels = _regime_labels(ordered) - current: list[FeedPeriodProfile] = [] - current_label = "" - for profile, label in zip(ordered, labels, strict=True): - if current and label != current_label: - regimes.append(_era(symbol, current_label, tuple(current))) - current = [] - current_label = label - current.append(profile) - if current: - regimes.append(_era(symbol, current_label, tuple(current))) - return tuple(regimes) - - -def _regime_labels(profiles: Sequence[FeedPeriodProfile]) -> tuple[str, ...]: - if len(profiles) <= 1: - return tuple("stable" for _ in profiles) - rates = [profile.tick_rate_per_hour for profile in profiles] - max_rate = max(rates) - min_positive = min((rate for rate in rates if rate > 0), default=0.0) - if max_rate <= 0 or (min_positive and max_rate / min_positive < 1.5): - return tuple("stable" for _ in profiles) - labels: list[str] = [] - for rate in rates: - ratio = rate / max_rate if max_rate else 0.0 - if ratio < 0.25: - labels.append("sparse") - elif ratio < 0.70: - labels.append("transitional") - else: - labels.append("dense") - return tuple(labels) - - -def _era( - symbol: str, - label: str, - profiles: Sequence[FeedPeriodProfile], -) -> FeedRegimeEra: - rates = [profile.tick_rate_per_hour for profile in profiles] - gaps = [profile.median_interarrival_ms for profile in profiles] - updates = [profile.quote_update_ratio for profile in profiles] - return FeedRegimeEra( - symbol=symbol, - label=label, - bucket=profiles[0].bucket, - period_start=profiles[0].period, - period_end=profiles[-1].period, - start_utc_ms=min(profile.start_utc_ms for profile in profiles), - end_utc_ms=max(profile.end_utc_ms for profile in profiles), - profile_count=len(profiles), - row_count=sum(profile.row_count for profile in profiles), - mean_tick_rate_per_hour=sum(rates) / len(rates), - median_interarrival_ms=_median(gaps), - quote_update_ratio=sum(updates) / len(updates), - quiet_gap_count=sum(profile.quiet_gap_count for profile in profiles), - metadata={ - "periods": [profile.period for profile in profiles], - "rate_min": _round_float(min(rates) if rates else 0.0), - "rate_max": _round_float(max(rates) if rates else 0.0), - }, +def _aggregate_conditioning( + items: Sequence[FeedEpochEvidenceV1], +) -> dict[str, JSONValue]: + count_fields = ( + "session_state_counts", + "active_session_counts", + "special_tag_counts", + "holiday_tag_counts", + "event_tag_counts", + ) + result: dict[str, JSONValue] = { + name: _sum_count_maps(item.conditioning.get(name) for item in items) + for name in count_fields + } + statuses = sorted( + { + str(item.conditioning.get("calendar_status", "unavailable")) + for item in items + } + ) + status_values: list[JSONValue] = [] + status_values.extend(statuses) + result.update( + { + "calendar_status": statuses[0] if len(statuses) == 1 else "mixed", + "calendar_statuses": status_values, + "aggregation": "sum_counts_across_canonical_fingerprints", + "conditioning_payload_sha256": _semantic_sha256( + { + "conditioning_hashes": [ + item.conditioning.get("conditioning_payload_sha256") + for item in items + ] + } + ), + } ) + return result -def _quote_update_count(observations: Sequence[_TickObservation]) -> int: - return sum( - 1 - for previous, current in zip( - observations, - observations[1:], - strict=False, - ) - if previous.bid != current.bid or previous.ask != current.ask +def _aggregate_profile( + items: Sequence[FeedEpochEvidenceV1], +) -> dict[str, JSONValue]: + row_count = sum(_profile_int(item, "row_count") for item in items) + quote_update_count = sum( + _profile_int(item, "quote_update_count") for item in items + ) + interval_count = sum( + max(0, _profile_int(item, "row_count") - 1) for item in items + ) + durations = sum( + max(0, item.end_timestamp_utc_ms - item.start_timestamp_utc_ms) + for item in items ) + weighted_spread_total = sum( + _profile_float(item, "spread_mean") * _profile_int(item, "row_count") + for item in items + ) + return { + "row_count": row_count, + "tick_rate_per_hour": ( + row_count * 3_600_000.0 / durations if durations else 0.0 + ), + "median_interarrival_ms": _median_profile( + items, "median_interarrival_ms" + ), + "p95_interarrival_ms": _median_profile(items, "p95_interarrival_ms"), + "max_interarrival_ms": max( + (_profile_int(item, "max_interarrival_ms") for item in items), + default=0, + ), + "quiet_gap_count": sum( + _profile_int(item, "quiet_gap_count") for item in items + ), + "quote_update_count": quote_update_count, + "quote_update_ratio": ( + quote_update_count / interval_count if interval_count else 0.0 + ), + "zero_change_run_count": sum( + _profile_int(item, "zero_change_run_count") for item in items + ), + "zero_change_tick_count": sum( + _profile_int(item, "zero_change_tick_count") for item in items + ), + "spread_min": min( + (_profile_float(item, "spread_min") for item in items), + default=0.0, + ), + "spread_median": _median_profile(items, "spread_median"), + "spread_mean": weighted_spread_total / row_count if row_count else 0.0, + "spread_max": max( + (_profile_float(item, "spread_max") for item in items), + default=0.0, + ), + "session_counts": _sum_count_maps( + item.profile.get("session_counts") for item in items + ), + } -def _zero_change_runs( - observations: Sequence[_TickObservation], -) -> tuple[int, int]: - runs = 0 - ticks = 0 - current_run = 1 - for previous, current in zip( - observations, - observations[1:], - strict=False, - ): - if previous.bid == current.bid and previous.ask == current.ask: - current_run += 1 - continue - if current_run > 1: - runs += 1 - ticks += current_run - current_run = 1 - if current_run > 1: - runs += 1 - ticks += current_run - return runs, ticks - - -def _session_counts( - observations: Sequence[_TickObservation], -) -> dict[str, int]: +def _sum_count_maps(values: Iterable[Any]) -> dict[str, JSONValue]: counts: Counter[str] = Counter() - for observation in observations: - sessions = _sessions_for_utc(_utc_datetime(observation.utc_ms)) - counts.update(sessions or ("market_closed",)) - return dict(counts) + for value in values: + if not isinstance(value, Mapping): + continue + for name, count in value.items(): + try: + counts[str(name)] += max(0, int(count)) + except (TypeError, ValueError, OverflowError): + continue + result: dict[str, JSONValue] = {} + for name, count in sorted(counts.items()): + result[name] = int(count) + return result -def _sessions_for_utc(value: datetime) -> tuple[str, ...]: - minute = value.hour * 60 + value.minute - sessions: list[str] = [] - if _minute_in_window(minute, 0, 8 * 60): - sessions.append("asia") - if _minute_in_window(minute, 7 * 60, 16 * 60): - sessions.append("london") - if _minute_in_window(minute, 13 * 60, 22 * 60): - sessions.append("new_york") - return tuple(sessions) +def _profile_int(item: FeedEpochEvidenceV1, name: str) -> int: + return max(0, int(_json_float(item.profile.get(name), default=0.0))) -def _minute_in_window(minute: int, start: int, end: int) -> bool: - return start <= minute < end +def _profile_float(item: FeedEpochEvidenceV1, name: str) -> float: + return _json_float(item.profile.get(name), default=0.0) -def _source_datetime(utc_ms: int) -> datetime: - return _utc_datetime(utc_ms) - timedelta(milliseconds=EST_NO_DST_OFFSET_MS) +def _json_float(value: JSONValue, *, default: float) -> float: + if isinstance(value, bool) or not isinstance(value, (str, int, float)): + return default + try: + return float(value) + except (TypeError, ValueError, OverflowError): + return default -def _utc_datetime(utc_ms: int) -> datetime: - return datetime.fromtimestamp(utc_ms / 1000, tz=timezone.utc) +def _median_profile(items: Sequence[FeedEpochEvidenceV1], name: str) -> float: + return float(median(_profile_float(item, name) for item in items)) -def _median(values: Sequence[float | int]) -> float: - if not values: - return 0.0 - ordered = sorted(float(value) for value in values) - midpoint = len(ordered) // 2 - if len(ordered) % 2: - return ordered[midpoint] - return (ordered[midpoint - 1] + ordered[midpoint]) / 2 +def _semantic_sha256(value: Mapping[str, Any]) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + return "sha256:" + hashlib.sha256(encoded).hexdigest() -def _percentile(values: Sequence[float | int], percentile: int) -> float: - if not values: - return 0.0 - ordered = sorted(float(value) for value in values) - index = max( - 0, min(len(ordered) - 1, ceil(percentile / 100 * len(ordered)) - 1) +def _period_profile_from_evidence( + evidence: FeedEpochEvidenceV1, + *, + bucket: str, +) -> FeedPeriodProfile: + profile = evidence.profile + session_counts = profile.get("session_counts", {}) + if not isinstance(session_counts, Mapping): + session_counts = {} + return FeedPeriodProfile( + symbol=evidence.symbol, + period=evidence.period, + bucket=bucket, + row_count=_profile_int(evidence, "row_count"), + start_utc_ms=evidence.start_timestamp_utc_ms, + end_utc_ms=evidence.end_timestamp_utc_ms, + tick_rate_per_hour=_profile_float(evidence, "tick_rate_per_hour"), + median_interarrival_ms=_profile_float( + evidence, "median_interarrival_ms" + ), + p95_interarrival_ms=_profile_float(evidence, "p95_interarrival_ms"), + max_interarrival_ms=int( + _profile_float(evidence, "max_interarrival_ms") + ), + quiet_gap_count=_profile_int(evidence, "quiet_gap_count"), + quote_update_count=_profile_int(evidence, "quote_update_count"), + quote_update_ratio=_profile_float(evidence, "quote_update_ratio"), + zero_change_run_count=_profile_int(evidence, "zero_change_run_count"), + zero_change_tick_count=_profile_int(evidence, "zero_change_tick_count"), + spread_min=_profile_float(evidence, "spread_min"), + spread_median=_profile_float(evidence, "spread_median"), + spread_mean=_profile_float(evidence, "spread_mean"), + spread_max=_profile_float(evidence, "spread_max"), + session_counts={ + str(name): max(0, int(_json_float(value, default=0.0))) + for name, value in session_counts.items() + }, ) - return ordered[index] -def _round_float(value: float) -> float: - return round(float(value), 6) +def _regimes_from_definition( + definition: FeedEpochDefinitionV1, + *, + bucket: str, + profiles: Sequence[FeedPeriodProfile], +) -> tuple[FeedRegimeEra, ...]: + result: list[FeedRegimeEra] = [] + for epoch in definition.epochs: + members = tuple( + profile + for profile in profiles + if epoch.period_start <= profile.period <= epoch.period_end + ) + rates = [profile.tick_rate_per_hour for profile in members] + gaps = [profile.median_interarrival_ms for profile in members] + updates = [profile.quote_update_ratio for profile in members] + result.append( + FeedRegimeEra( + symbol="+".join(definition.symbols), + label=epoch.label, + bucket=bucket, + period_start=epoch.period_start, + period_end=epoch.period_end, + start_utc_ms=epoch.start_timestamp_utc_ms, + end_utc_ms=epoch.end_timestamp_utc_ms, + profile_count=len(members), + row_count=sum(profile.row_count for profile in members), + mean_tick_rate_per_hour=( + sum(rates) / len(rates) if rates else 0.0 + ), + median_interarrival_ms=(float(median(gaps)) if gaps else 0.0), + quote_update_ratio=( + sum(updates) / len(updates) if updates else 0.0 + ), + quiet_gap_count=sum( + profile.quiet_gap_count for profile in members + ), + metadata={ + "epoch_id": epoch.epoch_id, + "definition_id": definition.definition_id, + "stability_status": definition.stability.status, + "transition_intervals_are_separate": True, + }, + ) + ) + return tuple(result) + + +def _normalize_bucket(bucket: str) -> str: + normalized = str(bucket or "").strip().lower() + if normalized not in {"month", "year"}: + raise ValueError("feed-regime bucket must be 'month' or 'year'") + return normalized + + +def _round_float(value: float, digits: int = 6) -> float: + rounded = round(float(value), digits) + return 0.0 if rounded == 0.0 else rounded diff --git a/src/histdatacom/data_quality/__init__.py b/src/histdatacom/data_quality/__init__.py index f27151e6..59e6f4dd 100644 --- a/src/histdatacom/data_quality/__init__.py +++ b/src/histdatacom/data_quality/__init__.py @@ -20,12 +20,98 @@ CALENDAR_PROFILE_SCHEMA_VERSION, DEFAULT_CALENDAR_PROFILE_NAME, DEFAULT_CALENDAR_PROFILE_SOURCE, + DEFAULT_EXPECTED_SESSION_CLOSURE_POLICY, + DEFAULT_WEEKEND_ACTIVITY_POLICY, + EXPECTED_SESSION_CLOSURE_POLICIES, + WEEKEND_ACTIVITY_POLICIES, HistDataCalendarDateTag, HistDataCalendarProfile, HistDataCalendarWindowTag, calendar_profile_from_mapping, default_calendar_profile, ) +from histdatacom.data_quality.classical_baselines import ( + BASELINE_DEFERRED_MODEL_FAMILIES, + BASELINE_REQUIRED_IDENTITY_COLUMNS, + BASELINE_REQUIRED_INPUT_COLUMNS, + CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY, + CLASSICAL_BASELINE_SCHEMA_VERSION, + CLASSICAL_BASELINE_SUMMARY_METADATA_KEY, + CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION, + CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION, + DEFAULT_BASELINE_EVALUATION_FRACTION, + DEFAULT_BASELINE_MINIMUM_EVALUATION_ROWS, + DEFAULT_BASELINE_MINIMUM_TRAINING_ROWS, + DEFAULT_BASELINE_ROLLING_WINDOWS, + DEFAULT_BASELINE_ROUNDING_DIGITS, + DEFAULT_BASELINE_SUMMARY_TARGET_LIMIT, + ClassicalBaselineProfile, + classical_baseline_diagnostics_from_training_frame, + classical_baseline_summary, + classical_baseline_training_projection, + format_classical_baseline_summary_lines, + project_classical_baseline_onto_training_frame, +) +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION, + CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION, + CLASSICAL_MODEL_FOLD_SCHEMA_VERSION, + CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY, + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY, + CLASSICAL_MODEL_INPUT_SUMMARY_SCHEMA_VERSION, + CLASSICAL_MODEL_TRAINING_PROJECTION_SCHEMA_VERSION, + ClassicalModelInputProfile, + ClassicalModelInputResult, + ClassicalModelResourcePolicy, + build_classical_model_input, + classical_model_dependency_status, + classical_model_evaluation_result, + classical_model_fit_result, + classical_model_input_contract_from_training_frame, + classical_model_input_summary, + format_classical_model_input_summary_lines, + project_classical_model_input_onto_training_frame, +) +from histdatacom.data_quality.classical_model_comparison import ( + CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY, + CLASSICAL_MODEL_COMPARISON_ELIGIBILITY_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY, + CLASSICAL_MODEL_COMPARISON_SUMMARY_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION, + CLASSICAL_MODEL_FIT_ACCOUNTING_SCHEMA_VERSION, + CLASSICAL_MODEL_SKILL_SCHEMA_VERSION, + CLASSICAL_MODEL_STABILITY_SCHEMA_VERSION, + ClassicalModelComparisonProfile, + ClassicalModelComparisonResult, + classical_model_comparison_from_saved_results, + classical_model_comparison_summary, + format_classical_model_comparison_summary_lines, + project_classical_model_comparison_onto_training_frame, +) +from histdatacom.data_quality.autoregressive import ( + AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY, + AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION, + AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION, + AUTOREGRESSIVE_FAMILIES, + AUTOREGRESSIVE_FAMILY_CODES, + AUTOREGRESSIVE_FIT_SCHEMA_VERSION, + AUTOREGRESSIVE_FORECAST_SCHEMA_VERSION, + AUTOREGRESSIVE_SCHEMA_VERSION, + AUTOREGRESSIVE_SUMMARY_METADATA_KEY, + AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION, + AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION, + AutoregressiveProfile, + AutoregressiveResult, + AutoregressiveSpecification, + autoregressive_diagnostics_from_training_frame, + autoregressive_from_model_input, + autoregressive_from_training_frame, + autoregressive_summary, + format_autoregressive_summary_lines, + project_autoregressive_onto_training_frame, +) from histdatacom.data_quality.contracts import ( QualityFinding, QualityLocation, @@ -35,6 +121,7 @@ QualityRunRule, QualityRunSummary, QualitySeverity, + QualitySkipEvent, QualityStatus, QualityTarget, QualityTargetKind, @@ -57,9 +144,40 @@ quality_target_from_path, ) from histdatacom.data_quality.engine import ( + DEFAULT_QUALITY_SKIP_COUNT_LIMIT, + DEFAULT_QUALITY_SKIP_EVENT_LIMIT, + QUALITY_ENGINE_METADATA_KEY, + QUALITY_ENGINE_SCHEMA_VERSION, + QUALITY_SKIP_EVENTS_SCHEMA_VERSION, + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV, evaluate_quality_rule, run_quality_assessment, ) +from histdatacom.data_quality.exponential_smoothing import ( + DEFAULT_EXPONENTIAL_SMOOTHING_ROLLING_WINDOWS, + DEFAULT_EXPONENTIAL_SMOOTHING_ROUNDING_DIGITS, + DEFAULT_EXPONENTIAL_SMOOTHING_SUMMARY_TARGET_LIMIT, + EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY, + EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_FAMILIES, + EXPONENTIAL_SMOOTHING_FAMILY_CODES, + EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_FORECAST_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY, + EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION, + ExponentialSmoothingProfile, + ExponentialSmoothingResult, + ExponentialSmoothingSpecification, + exponential_smoothing_diagnostics_from_training_frame, + exponential_smoothing_from_model_input, + exponential_smoothing_from_training_frame, + exponential_smoothing_summary, + format_exponential_smoothing_summary_lines, + project_exponential_smoothing_onto_training_frame, +) from histdatacom.data_quality.format_support import ( HISTDATA_FORMAT_SUPPORT_RULE_ID, HistDataFormatSupport, @@ -72,13 +190,18 @@ quality_support_from_metadata, ) from histdatacom.data_quality.fingerprints import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, CROSS_SERIES_FINGERPRINT_RULE_ID, + CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, DEFAULT_FINGERPRINT_HISTOGRAM_BINS, DEFAULT_FINGERPRINT_LAGS, DEFAULT_FINGERPRINT_MAX_ROWS, + DEFAULT_FINGERPRINT_PARITY_MISMATCH_LIMIT, + DEFAULT_FINGERPRINT_PARITY_SUMMARY_LIMIT, DEFAULT_FINGERPRINT_QUANTILES, DEFAULT_FINGERPRINT_ROLLING_WINDOWS, DEFAULT_FINGERPRINT_ROUNDING_DIGITS, + DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT, TIME_SERIES_FINGERPRINT_DISTRIBUTION_ATTENTION_METADATA_KEY, TIME_SERIES_FINGERPRINT_DISTRIBUTION_ATTENTION_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_METADATA_KEY, @@ -94,9 +217,14 @@ TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY, TIME_SERIES_FINGERPRINT_COVERAGE_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_CONDITIONAL_DISTRIBUTIONS_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DEPENDENCE_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DYNAMICS_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_STATIONARITY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_TOPOLOGY_ATTENTION_METADATA_KEY, @@ -104,12 +232,17 @@ TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_SCHEMA_VERSION, HistDataFingerprintDistributionAttentionProfile, + HistDataFingerprintParityProfile, HistDataFingerprintProfile, + HistDataCrossSeriesFingerprintRule, HistDataSeriesFingerprintRule, fingerprint_quality_rules, + decomposition_training_projection, + project_decomposition_onto_training_frame, series_fingerprint_coverage_summary, series_fingerprint_distribution_attention_summary, series_fingerprint_distribution_summary, + series_fingerprint_parity_summary, series_fingerprint_readiness_summary, series_fingerprint_readiness_risk_summary, series_fingerprint_regime_summary, @@ -126,6 +259,39 @@ format_fingerprint_contract_audit, format_fingerprint_schema_discovery, ) +from histdatacom.data_quality.fingerprint_next_work import ( + DEFAULT_FINGERPRINT_NEXT_WORK_ALTERNATE_LIMIT, + DEFAULT_FINGERPRINT_NEXT_WORK_TARGET_AXIS_LIMIT, + FINGERPRINT_NEXT_WORK_SCHEMA_VERSION, + fingerprint_next_work_recommendation, + format_fingerprint_next_work, +) +from histdatacom.data_quality.training_features import ( + AUTOREGRESSIVE_COLUMNS, + AUTOREGRESSIVE_FAMILY_COLUMN_SUFFIXES, + CLASSICAL_MODEL_CONTRACT_COLUMNS, + CLASSICAL_MODEL_COMPARISON_COLUMNS, + DEFAULT_SUSPICIOUS_TICK_GAP_MS, + EXPONENTIAL_SMOOTHING_COLUMNS, + HARD_ISSUE_COLUMNS, + IDENTITY_COLUMNS, + KALMAN_COLUMNS, + QUALITY_ISSUE_COLUMNS, + SEASONAL_EXOGENOUS_COLUMNS, + SEASONAL_EXOGENOUS_FAMILY_COLUMN_SUFFIXES, + STATE_SPACE_COLUMNS, + SYNTHETIC_PLACEHOLDER_COLUMNS, + TRAINING_FEATURE_REPORT_RULE_ID, + TRAINING_FEATURE_REPORT_SCHEMA_VERSION, + TRAINING_REQUIRED_COLUMNS, + TRAINING_SCHEMA_VERSION, + TrainingFeatureDefinition, + ensure_tick_training_features, + enrich_tick_cache_with_training_features, + quality_report_from_training_features, + required_training_feature_columns, + training_feature_definitions, +) from histdatacom.data_quality.bounded_payload_contracts import ( BOUNDED_PAYLOAD_CONTRACT_AUDIT_SCHEMA_VERSION, bounded_payload_contract_audit, @@ -173,7 +339,11 @@ provenance_quality_run_rules, ) from histdatacom.data_quality.remediation import ( + CALENDAR_POLICY_REMEDIATION_CONTEXT_SCHEMA_VERSION, QualityRemediationHint, + RemediationActionability, + RemediationActionabilityDecision, + classify_remediation_actionability, remediation_hint_payloads_for_finding, remediation_hint_payloads_for_flags, remediation_hints_for_finding_code, @@ -185,6 +355,7 @@ DEFAULT_REMEDIATION_CATALOG_AUDIT_SOURCE_LIMIT, DEFAULT_REMEDIATION_CATALOG_AUDIT_TARGET_AXIS_LIMIT, QUALITY_REMEDIATION_CATALOG_AUDIT_SCHEMA_VERSION, + QUALITY_REMEDIATION_PLAN_SCHEMA_VERSION, KnownQualityFindingCode, audit_remediation_catalog, audit_remediation_catalog_report_paths, @@ -194,6 +365,106 @@ remediation_catalog_audit_has_warning_error_gaps, remediation_catalog_audit_to_json, ) +from histdatacom.data_quality.repair_plan import ( + DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + DEFAULT_QUALITY_REPAIR_PLAN_ITEM_LIMIT, + MAXIMUM_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + MAXIMUM_QUALITY_REPAIR_PLAN_ITEM_LIMIT, + QUALITY_REPAIR_PLAN_SCHEMA_VERSION, + format_quality_repair_plan, + quality_repair_plan, + quality_repair_plan_to_json, +) +from histdatacom.data_quality.seasonal_exogenous import ( + CALENDAR_REGRESSOR_NAMES, + DEFAULT_SEASONAL_EXOGENOUS_ROLLING_WINDOWS, + DEFAULT_SEASONAL_EXOGENOUS_ROUNDING_DIGITS, + DEFAULT_SEASONAL_EXOGENOUS_SUMMARY_TARGET_LIMIT, + SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY, + SEASONAL_EXOGENOUS_CONFIGURATION_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_FAMILIES, + SEASONAL_EXOGENOUS_FAMILY_CODES, + SEASONAL_EXOGENOUS_FIT_STATUS_CODES, + SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_FORECAST_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_REGRESSOR_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_REASON_CODES, + SEASONAL_EXOGENOUS_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY, + SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION, + CalendarRegressorProfile, + SeasonalExogenousProfile, + SeasonalExogenousResult, + SeasonalExogenousSpecification, + format_seasonal_exogenous_summary_lines, + project_seasonal_exogenous_onto_training_frame, + seasonal_exogenous_diagnostics_from_training_frame, + seasonal_exogenous_from_model_input, + seasonal_exogenous_from_training_frame, + seasonal_exogenous_summary, +) +from histdatacom.data_quality.state_space import ( + DEFAULT_STATE_SPACE_ROLLING_WINDOWS, + DEFAULT_STATE_SPACE_ROUNDING_DIGITS, + DEFAULT_STATE_SPACE_SUMMARY_TARGET_LIMIT, + KALMAN_FILTERED_CALCULATION_BASIS_CODE, + KALMAN_SMOOTHED_CALCULATION_BASIS_CODE, + STATE_SPACE_BOUNDED_PAYLOAD_KEY, + STATE_SPACE_CONFIGURATION_SCHEMA_VERSION, + STATE_SPACE_EVALUATION_SCHEMA_VERSION, + STATE_SPACE_FAMILIES, + STATE_SPACE_FAMILY_CODES, + STATE_SPACE_FIT_SCHEMA_VERSION, + STATE_SPACE_FIT_STATUS_CODES, + STATE_SPACE_FORECAST_SCHEMA_VERSION, + STATE_SPACE_REASON_CODES, + STATE_SPACE_SCHEMA_VERSION, + STATE_SPACE_STATE_RESULT_SCHEMA_VERSION, + STATE_SPACE_SUMMARY_METADATA_KEY, + STATE_SPACE_SUMMARY_SCHEMA_VERSION, + STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION, + StateSpaceProfile, + StateSpaceResult, + StateSpaceSpecification, + format_state_space_summary_lines, + project_state_space_onto_training_frame, + state_space_diagnostics_from_training_frame, + state_space_from_model_input, + state_space_from_training_frame, + state_space_summary, +) +from histdatacom.data_quality.volatility import ( + ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY, + DEFAULT_VOLATILITY_ROUNDING_DIGITS, + DEFAULT_VOLATILITY_SUMMARY_TARGET_LIMIT, + VOLATILITY_BOUNDED_PAYLOAD_KEY, + VOLATILITY_CONFIGURATION_SCHEMA_VERSION, + VOLATILITY_DISTRIBUTIONS, + VOLATILITY_EVALUATION_SCHEMA_VERSION, + VOLATILITY_FAMILIES, + VOLATILITY_FAMILY_CODES, + VOLATILITY_FIT_SCHEMA_VERSION, + VOLATILITY_FIT_STATUS_CODES, + VOLATILITY_FORECAST_SCHEMA_VERSION, + VOLATILITY_INPUT_DEFINITIONS, + VOLATILITY_MEAN_MODELS, + VOLATILITY_REASON_CODES, + VOLATILITY_SCHEMA_VERSION, + VOLATILITY_SUMMARY_METADATA_KEY, + VOLATILITY_SUMMARY_SCHEMA_VERSION, + VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION, + VolatilityProfile, + VolatilityResult, + VolatilitySpecification, + format_volatility_summary_lines, + project_volatility_onto_training_frame, + volatility_diagnostics_from_training_frame, + volatility_from_model_input, + volatility_from_training_frame, + volatility_summary, +) from histdatacom.data_quality.reporting import ( QUALITY_EXIT_TRIGGERS, QUALITY_NEXT_ACTIONS_METADATA_KEY, @@ -206,14 +477,17 @@ QualityExitPolicy, QualityExitTrigger, bounded_quality_payload, + format_cross_series_fingerprint_lines, format_fingerprint_distribution_attention_lines, format_fingerprint_distribution_summary_lines, + format_fingerprint_parity_summary_lines, format_fingerprint_readiness_risk_lines, format_fingerprint_readiness_summary_lines, fingerprint_readiness_risk_summary, format_fingerprint_topology_attention_lines, format_fingerprint_topology_summary_lines, format_quality_console_summary, + format_quality_engine_skip_lines, format_quality_next_action_lines, format_quality_remediation_coverage_lines, format_quality_remediation_catalog_audit_lines, @@ -233,19 +507,27 @@ DEFAULT_QUALITY_PROFILE_SOURCE, OPERATOR_QUALITY_PROFILE_SOURCE, QUALITY_PROFILE_METADATA_KEY, + QUALITY_PROFILE_RESOLUTION_SCHEMA_VERSION, QUALITY_PROFILE_SCHEMA_VERSION, QUALITY_REPORTING_METADATA_KEY, CONFIGURABLE_QUALITY_RULE_IDS, HistDataRowCountProfile, QualityProfile, QualityProfileError, + QualityProfileResolution, + QualityProfileValueSource, QualityRemediationCatalogAuditProfile, QualityReportingProfile, + apply_quality_profile_overrides, default_quality_profile, load_quality_profile_file, + load_quality_profile_file_resolution, quality_profile_from_mapping, quality_profile_from_value, quality_profile_metadata, + quality_profile_resolution_from_value, + quality_profile_source_kind, + resolve_quality_profile, validate_quality_profile, ) from histdatacom.data_quality.rules import ( @@ -260,6 +542,8 @@ ASSET_CLASS_OIL, ASSET_CLASS_UNKNOWN, CROSS_INSTRUMENT_METADATA_KEY, + CrossInstrumentPointInput, + CrossInstrumentSeriesInput, DOMAIN_CROSS_INSTRUMENT_RULE_ID, DOMAIN_SYMBOL_METADATA_RULE_ID, FX_JPY_PRECISION_RULE, @@ -277,11 +561,53 @@ normalize_histdata_symbol, symbol_metadata_for, ) +from histdatacom.data_quality.synthetic_constraints import ( + DEFAULT_SYNTHETIC_CONSTRAINT_CATEGORY_LIMIT, + DEFAULT_SYNTHETIC_CONSTRAINT_HINT_LIMIT, + DEFAULT_SYNTHETIC_CONSTRAINT_SUMMARY_LIMIT, + DEFAULT_SYNTHETIC_VALIDATION_MISMATCH_LIMIT, + DEFAULT_SYNTHETIC_VALIDATION_TARGET_LIMIT, + SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY, + SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY, + SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION, + SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION, + SYNTHETIC_VALIDATION_SCHEMA_VERSION, + format_synthetic_constraint_summary_lines, + format_synthetic_validation, + synthetic_constraint_summary, + synthetic_constraints_from_fingerprint, + synthetic_constraints_from_training_frame, + validate_synthetic_constraint_reports, +) +from histdatacom.data_quality.synthetic_generation import ( + DEFAULT_SYNTHETIC_TICK_BLOCK_SIZE, + DEFAULT_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT, + DEFAULT_SYNTHETIC_TICK_MAX_ABS_LOG_RETURN, + DEFAULT_SYNTHETIC_TICK_MAX_GENERATED_ROWS, + DEFAULT_SYNTHETIC_TICK_MAX_REFERENCE_ROWS, + DEFAULT_SYNTHETIC_TICK_MINIMUM_REFERENCE_ROWS, + DEFAULT_SYNTHETIC_TICK_ROUNDING_DIGITS, + DEFAULT_SYNTHETIC_TICK_SEED, + SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION, + SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION, + SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION, + SYNTHETIC_TICK_METHOD_CODES, + SYNTHETIC_TICK_REASON_CODES, + SYNTHETIC_TICK_STATUS_CODES, + SyntheticTickGenerationProfile, + SyntheticTickGenerationResult, + format_synthetic_tick_generation, + generate_synthetic_ticks_from_reference, + reference_fingerprint_from_report, + validate_synthetic_tick_cache, + validate_synthetic_tick_frame, +) from histdatacom.data_quality.time import ( ASCII_EST_NO_DST_TIME_RULE_ID, ASCII_TIMESTAMP_CONTINUITY_RULE_ID, ASCII_TIMESTAMP_GAP_RULE_ID, ASCII_TIMESTAMP_SEQUENCE_RULE_ID, + TIMESTAMP_TOPOLOGY_INSPECTION_SCHEMA_VERSION, TIMESTAMP_CONTINUITY_METADATA_KEY, HistDataAsciiTimestampContinuityRule, HistDataAsciiTimestampGapRule, @@ -321,19 +647,178 @@ "ASSET_CLASS_METAL", "ASSET_CLASS_OIL", "ASSET_CLASS_UNKNOWN", + "BASELINE_DEFERRED_MODEL_FAMILIES", + "BASELINE_REQUIRED_IDENTITY_COLUMNS", + "BASELINE_REQUIRED_INPUT_COLUMNS", + "CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY", + "CLASSICAL_BASELINE_SCHEMA_VERSION", + "CLASSICAL_BASELINE_SUMMARY_METADATA_KEY", + "CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION", + "CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION", + "CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION", + "CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION", + "CLASSICAL_MODEL_FOLD_SCHEMA_VERSION", + "CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY", + "CLASSICAL_MODEL_INPUT_SCHEMA_VERSION", + "CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY", + "CLASSICAL_MODEL_INPUT_SUMMARY_SCHEMA_VERSION", + "CLASSICAL_MODEL_TRAINING_PROJECTION_SCHEMA_VERSION", + "CLASSICAL_MODEL_CONTRACT_COLUMNS", + "CLASSICAL_MODEL_COMPARISON_COLUMNS", + "CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY", + "CLASSICAL_MODEL_COMPARISON_ELIGIBILITY_SCHEMA_VERSION", + "CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION", + "CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY", + "CLASSICAL_MODEL_COMPARISON_SUMMARY_SCHEMA_VERSION", + "CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION", + "CLASSICAL_MODEL_FIT_ACCOUNTING_SCHEMA_VERSION", + "CLASSICAL_MODEL_SKILL_SCHEMA_VERSION", + "CLASSICAL_MODEL_STABILITY_SCHEMA_VERSION", + "ClassicalModelComparisonProfile", + "ClassicalModelComparisonResult", + "KALMAN_COLUMNS", + "AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY", + "AUTOREGRESSIVE_COLUMNS", + "AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION", + "AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION", + "AUTOREGRESSIVE_FAMILIES", + "AUTOREGRESSIVE_FAMILY_CODES", + "AUTOREGRESSIVE_FAMILY_COLUMN_SUFFIXES", + "AUTOREGRESSIVE_FIT_SCHEMA_VERSION", + "AUTOREGRESSIVE_FORECAST_SCHEMA_VERSION", + "AUTOREGRESSIVE_SCHEMA_VERSION", + "AUTOREGRESSIVE_SUMMARY_METADATA_KEY", + "AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION", + "AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION", + "AutoregressiveProfile", + "AutoregressiveResult", + "AutoregressiveSpecification", + "CALENDAR_REGRESSOR_NAMES", + "CalendarRegressorProfile", + "DEFAULT_EXPONENTIAL_SMOOTHING_ROLLING_WINDOWS", + "DEFAULT_EXPONENTIAL_SMOOTHING_ROUNDING_DIGITS", + "DEFAULT_EXPONENTIAL_SMOOTHING_SUMMARY_TARGET_LIMIT", + "EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY", + "EXPONENTIAL_SMOOTHING_COLUMNS", + "EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION", + "EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION", + "EXPONENTIAL_SMOOTHING_FAMILIES", + "EXPONENTIAL_SMOOTHING_FAMILY_CODES", + "EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION", + "EXPONENTIAL_SMOOTHING_FORECAST_SCHEMA_VERSION", + "EXPONENTIAL_SMOOTHING_SCHEMA_VERSION", + "EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY", + "EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION", + "EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION", + "DEFAULT_SEASONAL_EXOGENOUS_ROLLING_WINDOWS", + "DEFAULT_SEASONAL_EXOGENOUS_ROUNDING_DIGITS", + "DEFAULT_SEASONAL_EXOGENOUS_SUMMARY_TARGET_LIMIT", + "SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY", + "SEASONAL_EXOGENOUS_COLUMNS", + "STATE_SPACE_COLUMNS", + "SEASONAL_EXOGENOUS_CONFIGURATION_SCHEMA_VERSION", + "SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION", + "SEASONAL_EXOGENOUS_FAMILIES", + "SEASONAL_EXOGENOUS_FAMILY_CODES", + "SEASONAL_EXOGENOUS_FAMILY_COLUMN_SUFFIXES", + "SEASONAL_EXOGENOUS_FIT_STATUS_CODES", + "SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION", + "SEASONAL_EXOGENOUS_FORECAST_SCHEMA_VERSION", + "SEASONAL_EXOGENOUS_REGRESSOR_SCHEMA_VERSION", + "SEASONAL_EXOGENOUS_REASON_CODES", + "SEASONAL_EXOGENOUS_SCHEMA_VERSION", + "SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY", + "SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION", + "SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION", + "SeasonalExogenousProfile", + "SeasonalExogenousResult", + "SeasonalExogenousSpecification", + "DEFAULT_STATE_SPACE_ROLLING_WINDOWS", + "DEFAULT_STATE_SPACE_ROUNDING_DIGITS", + "DEFAULT_STATE_SPACE_SUMMARY_TARGET_LIMIT", + "KALMAN_FILTERED_CALCULATION_BASIS_CODE", + "KALMAN_SMOOTHED_CALCULATION_BASIS_CODE", + "STATE_SPACE_BOUNDED_PAYLOAD_KEY", + "STATE_SPACE_CONFIGURATION_SCHEMA_VERSION", + "STATE_SPACE_EVALUATION_SCHEMA_VERSION", + "STATE_SPACE_FAMILIES", + "STATE_SPACE_FAMILY_CODES", + "STATE_SPACE_FIT_SCHEMA_VERSION", + "STATE_SPACE_FIT_STATUS_CODES", + "STATE_SPACE_FORECAST_SCHEMA_VERSION", + "STATE_SPACE_REASON_CODES", + "STATE_SPACE_SCHEMA_VERSION", + "STATE_SPACE_STATE_RESULT_SCHEMA_VERSION", + "STATE_SPACE_SUMMARY_METADATA_KEY", + "STATE_SPACE_SUMMARY_SCHEMA_VERSION", + "STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION", + "StateSpaceProfile", + "StateSpaceResult", + "StateSpaceSpecification", + "ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY", + "DEFAULT_VOLATILITY_ROUNDING_DIGITS", + "DEFAULT_VOLATILITY_SUMMARY_TARGET_LIMIT", + "VOLATILITY_BOUNDED_PAYLOAD_KEY", + "VOLATILITY_CONFIGURATION_SCHEMA_VERSION", + "VOLATILITY_DISTRIBUTIONS", + "VOLATILITY_EVALUATION_SCHEMA_VERSION", + "VOLATILITY_FAMILIES", + "VOLATILITY_FAMILY_CODES", + "VOLATILITY_FIT_SCHEMA_VERSION", + "VOLATILITY_FIT_STATUS_CODES", + "VOLATILITY_FORECAST_SCHEMA_VERSION", + "VOLATILITY_INPUT_DEFINITIONS", + "VOLATILITY_MEAN_MODELS", + "VOLATILITY_REASON_CODES", + "VOLATILITY_SCHEMA_VERSION", + "VOLATILITY_SUMMARY_METADATA_KEY", + "VOLATILITY_SUMMARY_SCHEMA_VERSION", + "VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION", + "VolatilityProfile", + "VolatilityResult", + "VolatilitySpecification", "COVERAGE_MANIFEST_SCHEMA_VERSION", "CAMPAIGN_REPORT_SCHEMA_VERSION", "CAMPAIGN_PLAN_SCHEMA_VERSION", "CALENDAR_PROFILE_SCHEMA_VERSION", + "CALENDAR_REGRESSOR_NAMES", "CONFIGURABLE_QUALITY_RULE_IDS", "CROSS_INSTRUMENT_METADATA_KEY", + "CROSS_SERIES_FINGERPRINT_METADATA_KEY", "CROSS_SERIES_FINGERPRINT_RULE_ID", + "CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION", "DEFAULT_FINGERPRINT_HISTOGRAM_BINS", "DEFAULT_FINGERPRINT_LAGS", "DEFAULT_FINGERPRINT_MAX_ROWS", + "DEFAULT_FINGERPRINT_PARITY_MISMATCH_LIMIT", + "DEFAULT_FINGERPRINT_PARITY_SUMMARY_LIMIT", "DEFAULT_FINGERPRINT_QUANTILES", "DEFAULT_FINGERPRINT_ROLLING_WINDOWS", "DEFAULT_FINGERPRINT_ROUNDING_DIGITS", + "DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT", + "DEFAULT_BASELINE_EVALUATION_FRACTION", + "DEFAULT_BASELINE_MINIMUM_EVALUATION_ROWS", + "DEFAULT_BASELINE_MINIMUM_TRAINING_ROWS", + "DEFAULT_BASELINE_ROLLING_WINDOWS", + "DEFAULT_BASELINE_ROUNDING_DIGITS", + "DEFAULT_BASELINE_SUMMARY_TARGET_LIMIT", + "DEFAULT_SYNTHETIC_CONSTRAINT_CATEGORY_LIMIT", + "DEFAULT_SYNTHETIC_CONSTRAINT_HINT_LIMIT", + "DEFAULT_SYNTHETIC_CONSTRAINT_SUMMARY_LIMIT", + "DEFAULT_SYNTHETIC_VALIDATION_MISMATCH_LIMIT", + "DEFAULT_SYNTHETIC_VALIDATION_TARGET_LIMIT", + "DEFAULT_SYNTHETIC_TICK_BLOCK_SIZE", + "DEFAULT_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT", + "DEFAULT_SYNTHETIC_TICK_MAX_ABS_LOG_RETURN", + "DEFAULT_SYNTHETIC_TICK_MAX_GENERATED_ROWS", + "DEFAULT_SYNTHETIC_TICK_MAX_REFERENCE_ROWS", + "DEFAULT_SYNTHETIC_TICK_MINIMUM_REFERENCE_ROWS", + "DEFAULT_SYNTHETIC_TICK_ROUNDING_DIGITS", + "DEFAULT_SYNTHETIC_TICK_SEED", + "DEFAULT_QUALITY_SKIP_COUNT_LIMIT", + "DEFAULT_QUALITY_SKIP_EVENT_LIMIT", + "DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT", + "DEFAULT_QUALITY_REPAIR_PLAN_ITEM_LIMIT", "DEFAULT_REMEDIATION_CATALOG_AUDIT_CODE_LIMIT", "DEFAULT_REMEDIATION_CATALOG_AUDIT_RULE_LIMIT", "DEFAULT_REMEDIATION_CATALOG_AUDIT_SOURCE_LIMIT", @@ -350,6 +835,8 @@ "INDEX_PRECISION_RULE", "HISTDATA_FORMAT_SUPPORT_RULE_ID", "METAL_PRECISION_RULE", + "MAXIMUM_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT", + "MAXIMUM_QUALITY_REPAIR_PLAN_ITEM_LIMIT", "MODELING_READINESS_METADATA_KEY", "MODELING_READINESS_RULE_ID", "OIL_PRECISION_RULE", @@ -357,11 +844,31 @@ "PROVENANCE_MANIFEST_RULE_ID", "PROVENANCE_MANIFEST_SCHEMA_VERSION", "PROVENANCE_METADATA_KEY", + "QUALITY_ENGINE_METADATA_KEY", + "QUALITY_ENGINE_SCHEMA_VERSION", "QUALITY_EXIT_TRIGGERS", "QUALITY_PROFILE_METADATA_KEY", "QUALITY_PROFILE_SCHEMA_VERSION", + "CALENDAR_POLICY_REMEDIATION_CONTEXT_SCHEMA_VERSION", + "DEFAULT_EXPECTED_SESSION_CLOSURE_POLICY", + "DEFAULT_WEEKEND_ACTIVITY_POLICY", + "EXPECTED_SESSION_CLOSURE_POLICIES", + "QUALITY_REPAIR_PLAN_SCHEMA_VERSION", "QUALITY_REPORT_SCHEMA_VERSION", "SERIES_FINGERPRINT_RULE_ID", + "SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY", + "SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY", + "SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION", + "SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION", + "SYNTHETIC_VALIDATION_SCHEMA_VERSION", + "SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION", + "SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION", + "SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION", + "SYNTHETIC_TICK_METHOD_CODES", + "SYNTHETIC_TICK_REASON_CODES", + "SYNTHETIC_TICK_STATUS_CODES", + "SyntheticTickGenerationProfile", + "SyntheticTickGenerationResult", "TIMESTAMP_CONTINUITY_METADATA_KEY", "TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY", "TIME_SERIES_FINGERPRINT_COVERAGE_SCHEMA_VERSION", @@ -370,6 +877,9 @@ "TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_METADATA_KEY", "TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_METADATA_KEY", + "TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION", + "TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY", + "TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY", "TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY", @@ -382,6 +892,8 @@ "TIME_SERIES_FINGERPRINT_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_CALENDAR_REGIMES_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_CONDITIONAL_DISTRIBUTIONS_SCHEMA_VERSION", + "TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION", + "TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_DEPENDENCE_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_DYNAMICS_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_STATIONARITY_SCHEMA_VERSION", @@ -390,8 +902,12 @@ "TIME_SERIES_FINGERPRINT_TOPOLOGY_ATTENTION_SCHEMA_VERSION", "TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY", "TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_SCHEMA_VERSION", + "TIMESTAMP_TOPOLOGY_INSPECTION_SCHEMA_VERSION", + "WEEKEND_ACTIVITY_POLICIES", "DEEP_QUALITY_DIMENSIONS", "CoverageDimension", + "CrossInstrumentPointInput", + "CrossInstrumentSeriesInput", "HistDataAsciiTimestampContinuityRule", "HistDataAsciiTimestampGapRule", "HistDataAsciiEstNoDstTimeRule", @@ -411,9 +927,11 @@ "HistDataCoverageManifestRule", "HistDataCrossInstrumentConsistencyRule", "HistDataCrossInstrumentTolerance", + "HistDataCrossSeriesFingerprintRule", "HistDataFormatSupport", "HistDataFormatSupportRule", "HistDataFingerprintDistributionAttentionProfile", + "HistDataFingerprintParityProfile", "HistDataFingerprintProfile", "HistDataGapTolerance", "HistDataModelingReadinessRule", @@ -438,6 +956,8 @@ "QualityLocation", "QualityProfile", "QualityProfileError", + "QualityProfileResolution", + "QualityProfileValueSource", "QualityReport", "QualityRemediationCatalogAuditProfile", "QualityRemediationHint", @@ -447,18 +967,43 @@ "QualityRunRule", "QualityRunSummary", "QualitySeverity", + "QualitySkipEvent", "QualityStatus", "QualityTarget", "QualityTargetKind", "QualityTargetSummary", "QUALITY_CHECK_GROUPS", + "QUALITY_ISSUE_COLUMNS", "QUALITY_NEXT_ACTIONS_METADATA_KEY", "QUALITY_NEXT_ACTIONS_SCHEMA_VERSION", + "QUALITY_PROFILE_RESOLUTION_SCHEMA_VERSION", "QUALITY_REMEDIATION_CATALOG_AUDIT_METADATA_KEY", "QUALITY_REMEDIATION_CATALOG_AUDIT_SCHEMA_VERSION", + "QUALITY_REMEDIATION_PLAN_SCHEMA_VERSION", "QUALITY_REMEDIATION_COVERAGE_METADATA_KEY", + "QUALITY_SKIP_EVENTS_SCHEMA_VERSION", + "QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV", "QUALITY_REMEDIATION_COVERAGE_SCHEMA_VERSION", "QUALITY_REPORTING_METADATA_KEY", + "DEFAULT_SUSPICIOUS_TICK_GAP_MS", + "HARD_ISSUE_COLUMNS", + "IDENTITY_COLUMNS", + "SYNTHETIC_PLACEHOLDER_COLUMNS", + "TRAINING_FEATURE_REPORT_RULE_ID", + "TRAINING_FEATURE_REPORT_SCHEMA_VERSION", + "TRAINING_REQUIRED_COLUMNS", + "TRAINING_SCHEMA_VERSION", + "DEFAULT_FINGERPRINT_NEXT_WORK_ALTERNATE_LIMIT", + "DEFAULT_FINGERPRINT_NEXT_WORK_TARGET_AXIS_LIMIT", + "FINGERPRINT_NEXT_WORK_SCHEMA_VERSION", + "ClassicalBaselineProfile", + "ClassicalModelInputProfile", + "ClassicalModelInputResult", + "ClassicalModelResourcePolicy", + "ExponentialSmoothingProfile", + "ExponentialSmoothingResult", + "ExponentialSmoothingSpecification", + "TrainingFeatureDefinition", "BoundedReportLimit", "BOUNDED_PAYLOAD_CONTRACT_AUDIT_SCHEMA_VERSION", "bounded_payload_contract_audit", @@ -470,32 +1015,90 @@ "calendar_profile_from_mapping", "calendar_quality_rules", "calendar_regime_payload_for_target", + "classical_baseline_diagnostics_from_training_frame", + "classical_baseline_summary", + "classical_baseline_training_projection", + "build_classical_model_input", + "classical_model_dependency_status", + "classical_model_evaluation_result", + "classical_model_fit_result", + "classical_model_input_contract_from_training_frame", + "classical_model_input_summary", + "autoregressive_diagnostics_from_training_frame", + "autoregressive_from_model_input", + "autoregressive_from_training_frame", + "autoregressive_summary", + "seasonal_exogenous_diagnostics_from_training_frame", + "seasonal_exogenous_from_model_input", + "seasonal_exogenous_from_training_frame", + "seasonal_exogenous_summary", + "state_space_diagnostics_from_training_frame", + "state_space_from_model_input", + "state_space_from_training_frame", + "state_space_summary", + "volatility_diagnostics_from_training_frame", + "volatility_from_model_input", + "volatility_from_training_frame", + "volatility_summary", + "exponential_smoothing_diagnostics_from_training_frame", + "exponential_smoothing_from_model_input", + "exponential_smoothing_from_training_frame", + "exponential_smoothing_summary", "classify_histdata_source_timestamp", "classify_histdata_timestamp", "coverage_manifest_metadata", "data_format_from_code", + "apply_quality_profile_overrides", "default_quality_profile", "default_calendar_profile", "domain_quality_rules", "domain_quality_run_rules", "discover_quality_targets", + "ensure_tick_training_features", + "enrich_tick_cache_with_training_features", "evaluate_quality_rule", "fingerprint_contract_audit", "fingerprint_quality_rules", + "decomposition_training_projection", + "project_decomposition_onto_training_frame", + "project_classical_baseline_onto_training_frame", + "project_classical_model_input_onto_training_frame", + "project_exponential_smoothing_onto_training_frame", + "project_autoregressive_onto_training_frame", + "project_seasonal_exogenous_onto_training_frame", + "project_state_space_onto_training_frame", + "project_volatility_onto_training_frame", "fingerprint_report_surface_evidence", "fingerprint_schema_discovery", + "fingerprint_next_work_recommendation", "format_bounded_payload_contract_audit", + "format_classical_baseline_summary_lines", + "format_classical_model_input_summary_lines", + "format_exponential_smoothing_summary_lines", + "format_autoregressive_summary_lines", + "format_seasonal_exogenous_summary_lines", + "format_state_space_summary_lines", + "format_volatility_summary_lines", + "format_classical_model_comparison_summary_lines", "format_fingerprint_contract_audit", + "format_cross_series_fingerprint_lines", "format_fingerprint_schema_discovery", + "format_fingerprint_next_work", "format_fingerprint_topology_attention_lines", "format_fingerprint_topology_summary_lines", "format_quality_console_summary", + "format_quality_engine_skip_lines", "format_fingerprint_distribution_attention_lines", "format_fingerprint_distribution_summary_lines", + "format_fingerprint_parity_summary_lines", "format_fingerprint_readiness_risk_lines", "format_fingerprint_readiness_summary_lines", + "format_synthetic_constraint_summary_lines", + "format_synthetic_validation", + "format_synthetic_tick_generation", "fingerprint_readiness_risk_summary", "format_quality_remediation_catalog_audit_lines", + "format_quality_repair_plan", "format_quality_next_action_lines", "format_quality_remediation_coverage_lines", "representative_quality_report", @@ -506,6 +1109,7 @@ "known_histdata_timeframes", "KnownQualityFindingCode", "load_quality_profile_file", + "load_quality_profile_file_resolution", "load_quality_report", "manifest_quality_run_rules", "modeling_quality_rules", @@ -522,19 +1126,26 @@ "audit_remediation_catalog_report_paths", "quality_remediation_catalog_audit_summary", "quality_remediation_coverage_summary", + "quality_repair_plan", + "quality_repair_plan_to_json", "quality_report_payload", "quality_report_to_json", "representative_bounded_quality_payload", "quality_profile_from_mapping", "quality_profile_from_value", "quality_profile_metadata", + "quality_profile_resolution_from_value", "quality_profile_report_metadata", + "quality_profile_source_kind", + "quality_report_from_training_features", "quality_rules_for_groups", "quality_run_rules_for_groups", "quality_support_for_target", "quality_support_from_metadata", "quality_target_from_path", + "resolve_quality_profile", "discover_known_quality_findings", + "classify_remediation_actionability", "format_remediation_catalog_audit", "remediation_hint_payloads_for_finding", "remediation_hint_payloads_for_flags", @@ -542,20 +1153,36 @@ "remediation_hints_for_flags", "remediation_catalog_audit_has_warning_error_gaps", "remediation_catalog_audit_to_json", + "RemediationActionability", + "RemediationActionabilityDecision", "run_quality_assessment", + "classical_model_comparison_from_saved_results", + "classical_model_comparison_summary", + "project_classical_model_comparison_onto_training_frame", "series_fingerprint_coverage_summary", "series_fingerprint_distribution_attention_summary", "series_fingerprint_distribution_summary", + "series_fingerprint_parity_summary", "series_fingerprint_readiness_summary", "series_fingerprint_readiness_risk_summary", "series_fingerprint_regime_summary", "series_fingerprint_topology_attention_summary", "series_fingerprint_topology_summary", + "synthetic_constraint_summary", + "synthetic_constraints_from_fingerprint", + "synthetic_constraints_from_training_frame", + "generate_synthetic_ticks_from_reference", + "reference_fingerprint_from_report", "symbol_metadata_for", "time_quality_run_rules", "time_quality_rules", "timestamp_topology_payload_for_target", "ticks_quality_rules", + "required_training_feature_columns", + "training_feature_definitions", "validate_quality_profile", + "validate_synthetic_constraint_reports", + "validate_synthetic_tick_cache", + "validate_synthetic_tick_frame", "write_quality_report", ] diff --git a/src/histdatacom/data_quality/autoregressive.py b/src/histdatacom/data_quality/autoregressive.py new file mode 100644 index 00000000..869aafca --- /dev/null +++ b/src/histdatacom/data_quality/autoregressive.py @@ -0,0 +1,1821 @@ +"""Optional explicit-order AR, ARMA, and ARIMA model diagnostics. + +The family consumes the regular grid and rolling-origin folds established by +the classical-model input contract. Statsmodels is imported lazily, all model +specifications are explicit, and every failure remains bounded and advisory. +""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.metadata +import json +import math +import re +import time +import warnings +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from contextlib import nullcontext +from dataclasses import dataclass, field +from statistics import median +from typing import Any, cast + +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + ClassicalModelInputProfile, + ClassicalModelInputResult, + build_classical_model_input, +) +from histdatacom.data_quality.contracts import QualityFinding +from histdatacom.data_quality.exponential_smoothing import ( + _inverse_forecasts, + _reference_baseline_payloads, + _trailing_contiguous_indexes, +) +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.training_features import ( + AUTOREGRESSIVE_COLUMNS, + AUTOREGRESSIVE_FAMILY_COLUMN_SUFFIXES, + ensure_tick_training_features, +) +from histdatacom.runtime_contracts import JSONValue + +AUTOREGRESSIVE_SCHEMA_VERSION = "histdatacom.autoregressive.v1" +AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION = ( + "histdatacom.autoregressive-configuration.v1" +) +AUTOREGRESSIVE_FIT_SCHEMA_VERSION = "histdatacom.autoregressive-fit-result.v1" +AUTOREGRESSIVE_FORECAST_SCHEMA_VERSION = ( + "histdatacom.autoregressive-forecast.v1" +) +AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION = ( + "histdatacom.autoregressive-evaluation.v1" +) +AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.autoregressive-training-projection.v1" +) +AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION = "histdatacom.autoregressive-summary.v1" +AUTOREGRESSIVE_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_autoregressive_summary" +) +AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY = "fingerprint_autoregressive" + +DEFAULT_AUTOREGRESSIVE_SUMMARY_TARGET_LIMIT = 16 +DEFAULT_AUTOREGRESSIVE_ROLLING_WINDOWS = (5, 20) +DEFAULT_AUTOREGRESSIVE_ROUNDING_DIGITS = 12 +MAX_AUTOREGRESSIVE_SPECIFICATIONS = 32 +MAX_AUTOREGRESSIVE_ORDER = 64 +MAX_AUTOREGRESSIVE_FIXED_PARAMETERS = 32 + +AUTOREGRESSIVE_FAMILIES = ("ar", "arma", "arima") +AUTOREGRESSIVE_FAMILY_CODES = {"ar": 1, "arma": 2, "arima": 3} +AUTOREGRESSIVE_TREND_CODES = {"n": 0, "c": 1, "t": 2, "ct": 3} +AUTOREGRESSIVE_INITIALIZATION_CODES = { + "default": 1, + "stationary": 2, + "approximate_diffuse": 3, +} +AUTOREGRESSIVE_FIT_STATUS_CODES = { + "unavailable": 1, + "skipped": 2, + "failed": 3, + "limited": 4, + "fitted": 5, + "converged": 6, +} +AUTOREGRESSIVE_REASON_CODES = { + "": 0, + "dependency_unavailable": 1, + "input_contract_unavailable": 2, + "insufficient_folds": 3, + "insufficient_lags": 4, + "invalid_order": 5, + "invalid_configuration": 6, + "nonstationary_configuration": 7, + "noninvertible_configuration": 8, + "zero_variance": 9, + "optimizer_failure": 10, + "singularity": 11, + "numerical_overflow": 12, + "numerical_failure": 13, + "resource_limit": 14, + "timeout": 15, + "backend_failure": 16, + "target_unavailable": 17, + "inverse_transform_unavailable": 18, +} +AUTOREGRESSIVE_WARNING_CODES = { + "ConvergenceWarning": "convergence_warning", + "RuntimeWarning": "runtime_warning", + "ValueWarning": "value_warning", + "UserWarning": "user_warning", +} + +_SPECIFICATION_ID = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") +_PARAMETER_NAME = re.compile(r"^[A-Za-z0-9_.()\[\]-]{1,96}$") +_ESTIMATION_METHODS = { + "statespace", + "innovations_mle", + "hannan_rissanen", + "burg", + "yule_walker", +} + + +@dataclass(frozen=True, slots=True) +class AutoregressiveSpecification: + """One explicit AR, ARMA, or ARIMA specification.""" + + specification_id: str + family: str + p: int + d: int = 0 + q: int = 0 + trend: str = "n" + initialization_method: str = "default" + estimation_method: str = "statespace" + enforce_stationarity: bool = True + enforce_invertibility: bool = True + concentrate_scale: bool = False + fixed_parameters: tuple[tuple[str, float], ...] = () + max_iterations: int = 200 + + def __post_init__(self) -> None: + if not _SPECIFICATION_ID.fullmatch(self.specification_id): + raise ValueError("invalid autoregressive specification_id") + if self.family not in AUTOREGRESSIVE_FAMILIES: + raise ValueError("unsupported autoregressive family") + for name, value in (("p", self.p), ("d", self.d), ("q", self.q)): + if value < 0 or value > MAX_AUTOREGRESSIVE_ORDER: + raise ValueError(f"{name} must be between 0 and 64") + if self.family == "ar" and not ( + self.p >= 1 and self.d == 0 and self.q == 0 + ): + raise ValueError("AR requires p >= 1 with d = q = 0") + if self.family == "arma" and not ( + self.p >= 1 and self.d == 0 and self.q >= 1 + ): + raise ValueError("ARMA requires p >= 1, q >= 1, and d = 0") + if self.family == "arima" and self.d < 1: + raise ValueError("ARIMA requires d >= 1; use AR or ARMA when d = 0") + if self.trend not in AUTOREGRESSIVE_TREND_CODES: + raise ValueError("trend must be n, c, t, or ct") + if self.d > 0 and "c" in self.trend: + raise ValueError( + "integrated ARIMA cannot include a constant trend term" + ) + if ( + self.initialization_method + not in AUTOREGRESSIVE_INITIALIZATION_CODES + ): + raise ValueError("unsupported autoregressive initialization_method") + if self.initialization_method == "stationary" and self.d > 0: + raise ValueError("stationary initialization requires d = 0") + if self.estimation_method not in _ESTIMATION_METHODS: + raise ValueError("unsupported autoregressive estimation_method") + if self.family == "arma" and self.estimation_method in { + "burg", + "yule_walker", + }: + raise ValueError("AR-only estimation method cannot fit ARMA") + if self.family == "arima" and self.estimation_method not in { + "statespace", + "innovations_mle", + }: + raise ValueError( + "integrated ARIMA requires statespace or innovations_mle" + ) + if self.fixed_parameters and self.estimation_method != "statespace": + raise ValueError("fixed parameters require statespace estimation") + if len(self.fixed_parameters) > MAX_AUTOREGRESSIVE_FIXED_PARAMETERS: + raise ValueError("too many fixed autoregressive parameters") + seen: set[str] = set() + for name, fixed_value in self.fixed_parameters: + if ( + not _PARAMETER_NAME.fullmatch(name) + or name in seen + or not math.isfinite(fixed_value) + ): + raise ValueError( + "fixed parameters must be unique finite scalars" + ) + seen.add(name) + if self.max_iterations < 1: + raise ValueError("max_iterations must be positive") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return a stable serializable model specification.""" + return { + "schema_version": AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION, + "specification_id": self.specification_id, + "family": self.family, + "family_code": AUTOREGRESSIVE_FAMILY_CODES[self.family], + "order": [self.p, self.d, self.q], + "p": self.p, + "d": self.d, + "q": self.q, + "trend": self.trend, + "trend_code": AUTOREGRESSIVE_TREND_CODES[self.trend], + "initialization_method": self.initialization_method, + "estimation_method": self.estimation_method, + "enforce_stationarity": self.enforce_stationarity, + "enforce_invertibility": self.enforce_invertibility, + "concentrate_scale": self.concentrate_scale, + "fixed_parameters": cast( + JSONValue, + [ + {"parameter": name, "value": value} + for name, value in self.fixed_parameters + ], + ), + "max_iterations": self.max_iterations, + "automatic_order_selection": False, + } + + +def _default_specifications() -> tuple[AutoregressiveSpecification, ...]: + return ( + AutoregressiveSpecification("ar-1", "ar", 1, trend="c"), + AutoregressiveSpecification("arma-1-1", "arma", 1, q=1, trend="c"), + AutoregressiveSpecification("arima-1-1-1", "arima", 1, d=1, q=1), + ) + + +@dataclass(frozen=True, slots=True) +class AutoregressiveProfile: + """Explicit fitted autoregressive controls; disabled by default.""" + + enabled: bool = False + specifications: tuple[AutoregressiveSpecification, ...] = field( + default_factory=_default_specifications + ) + projection_specification_ids: tuple[str, ...] = ( + "ar-1", + "arma-1-1", + "arima-1-1-1", + ) + projection_horizon: int = 1 + baseline_rolling_windows: tuple[int, ...] = ( + DEFAULT_AUTOREGRESSIVE_ROLLING_WINDOWS + ) + compare_exponential_smoothing: bool = True + rounding_digits: int = DEFAULT_AUTOREGRESSIVE_ROUNDING_DIGITS + + def __post_init__(self) -> None: + if not self.specifications: + raise ValueError( + "at least one autoregressive specification is required" + ) + if len(self.specifications) > MAX_AUTOREGRESSIVE_SPECIFICATIONS: + raise ValueError("too many autoregressive specifications") + identifiers = tuple( + item.specification_id for item in self.specifications + ) + if len(set(identifiers)) != len(identifiers): + raise ValueError("autoregressive specification IDs must be unique") + selected = set(self.projection_specification_ids) + if not selected or not selected.issubset(identifiers): + raise ValueError( + "projection specification IDs must select configured models" + ) + selected_families = { + item.family + for item in self.specifications + if item.specification_id in selected + } + if len(selected_families) != len(selected): + raise ValueError( + "select at most one projection per autoregressive family" + ) + if self.projection_horizon < 1: + raise ValueError("projection_horizon must be positive") + if ( + not self.baseline_rolling_windows + or any(value < 1 for value in self.baseline_rolling_windows) + or tuple(sorted(set(self.baseline_rolling_windows))) + != self.baseline_rolling_windows + ): + raise ValueError( + "baseline_rolling_windows must be sorted unique positives" + ) + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between 0 and 16") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return stable profile metadata.""" + return { + "enabled": self.enabled, + "specifications": cast( + JSONValue, [item.to_metadata() for item in self.specifications] + ), + "projection_specification_ids": list( + self.projection_specification_ids + ), + "projection_horizon": self.projection_horizon, + "baseline_rolling_windows": list(self.baseline_rolling_windows), + "compare_exponential_smoothing": self.compare_exponential_smoothing, + "rounding_digits": self.rounding_digits, + "automatic_order_selection": False, + "automatic_winner": False, + } + + +@dataclass(frozen=True, slots=True) +class AutoregressiveResult: + """Bounded diagnostics plus durable row-key annotations.""" + + diagnostics: Mapping[str, JSONValue] + annotations: tuple[Mapping[str, Any], ...] + input_result: ClassicalModelInputResult + + +@dataclass(frozen=True, slots=True) +class _Backend: + version: str + arima: Any + + +@dataclass(frozen=True, slots=True) +class _FitOutcome: + status: str + reason: str + forecasts: tuple[float, ...] + parameters: Mapping[str, float] + warning_codes: tuple[str, ...] + converged: bool + stationary: bool | None + invertible: bool | None + ar_root_min_modulus: float | None + ma_root_min_modulus: float | None + covariance_condition_number: float | None + effective_observation_count: int + + +def autoregressive_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: AutoregressiveProfile | None = None, + exponential_smoothing: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> AutoregressiveResult: + """Regularize an enriched tick frame and evaluate configured models.""" + selected_input = input_profile or ClassicalModelInputProfile(enabled=True) + input_result = build_classical_model_input( + frame, + fingerprint, + profile=selected_input, + target=target, + ) + return autoregressive_from_model_input( + frame, + input_result, + fingerprint, + input_profile=selected_input, + profile=profile, + exponential_smoothing=exponential_smoothing, + target=target, + ) + + +def autoregressive_from_model_input( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile, + profile: AutoregressiveProfile | None = None, + exponential_smoothing: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> AutoregressiveResult: + """Fit explicit AR, ARMA, and ARIMA specifications over #421 folds.""" + selected = profile or AutoregressiveProfile(enabled=True) + base = _base_payload(input_result, fingerprint, selected) + if selected.projection_horizon not in input_profile.horizons: + return _unavailable_result(input_result, base, "invalid_configuration") + if input_result.contract.get("status") == "unavailable": + return _unavailable_result( + input_result, base, "input_contract_unavailable" + ) + if not input_result.folds: + return _unavailable_result( + input_result, base, "insufficient_folds", status="limited" + ) + backend = _load_backend() + if backend is None: + return _unavailable_result(input_result, base, "dependency_unavailable") + return _evaluate_models( + frame, + input_result, + fingerprint, + input_profile, + selected, + backend, + exponential_smoothing=exponential_smoothing, + target=target, + ) + + +def autoregressive_diagnostics_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: AutoregressiveProfile | None = None, + exponential_smoothing: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> dict[str, JSONValue]: + """Return only serializable fitted-family diagnostics.""" + return dict( + autoregressive_from_training_frame( + frame, + fingerprint, + input_profile=input_profile, + profile=profile, + exponential_smoothing=exponential_smoothing, + target=target, + ).diagnostics + ) + + +def project_autoregressive_onto_training_frame( + frame: Any, + result: AutoregressiveResult, + *, + target: Any | None = None, +) -> Any: + """Join bounded autoregressive annotations by durable row identity.""" + import polars as pl + + columns = set(getattr(frame, "columns", ())) + if not {"series_id", "period", "row_id"}.issubset(columns): + enriched = ensure_tick_training_features(frame, target=target) + else: + enriched = frame + left = enriched.drop( + [name for name in AUTOREGRESSIVE_COLUMNS if name in enriched.columns] + ).with_row_index("__cm_autoregressive_original_order") + if result.annotations: + right = pl.DataFrame( + [dict(row) for row in result.annotations], infer_schema_length=None + ) + projected = left.join( + right, + on=["series_id", "period", "row_id"], + how="left", + validate="m:1", + ) + else: + projected = left + projected = _ensure_projection_columns(projected) + return projected.sort("__cm_autoregressive_original_order").drop( + "__cm_autoregressive_original_order" + ) + + +def autoregressive_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = DEFAULT_AUTOREGRESSIVE_SUMMARY_TARGET_LIMIT, +) -> dict[str, JSONValue] | None: + """Return bounded report metadata for autoregressive results.""" + targets: list[dict[str, JSONValue]] = [] + statuses: Counter[str] = Counter() + for finding in findings: + fingerprint = _mapping(finding.metadata.get("time_series_fingerprint")) + payload = _mapping(fingerprint.get("autoregressive")) + if not payload: + continue + status = _text(payload.get("status")) or "unavailable" + evaluation = _mapping(payload.get("evaluation")) + fit_summary = _mapping(payload.get("fit_summary")) + statuses[status] += 1 + targets.append( + { + "target_axis": dict(_mapping(payload.get("target_axis"))), + "status": status, + "reason": payload.get("reason"), + "model_count": _int(evaluation.get("model_count")), + "fit_attempt_count": _int(fit_summary.get("fit_attempt_count")), + "evaluated_fold_count": _int( + evaluation.get("evaluated_fold_count") + ), + "failed_fit_count": _int(fit_summary.get("failed_fit_count")), + } + ) + if not targets: + return None + targets.sort(key=_target_sort_key) + limit = bounded_report_limit( + target_limit, + default_limit=DEFAULT_AUTOREGRESSIVE_SUMMARY_TARGET_LIMIT, + allow_unbounded=True, + ) + included = limit.slice(targets) + omitted = len(targets) - len(included) + return { + "schema_version": AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION, + "advisory": True, + "target_count": len(targets), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "status_counts": dict(sorted(statuses.items())), + "target_summaries": cast(JSONValue, included), + "limit_metadata": {"targets": limit.limit_payload()}, + } + + +def format_autoregressive_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> tuple[str, ...]: + """Return concise human-readable fitted-family lines.""" + if not summary: + return () + statuses = _mapping(summary.get("status_counts")) + lines = [ + "", + "Autoregressive models", + ( + f"targets: {_int(summary.get('target_count'))} " + f"ready: {_int(statuses.get('ready'))} " + f"limited: {_int(statuses.get('limited'))} " + f"unavailable: {_int(statuses.get('unavailable'))}" + ), + ] + for target in _mapping_rows(summary.get("target_summaries")): + axis = _mapping(target.get("target_axis")) + label = "/".join( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + lines.append( + f"- {label}: {_text(target.get('status'))} " + f"models={_int(target.get('model_count'))} " + f"fits={_int(target.get('fit_attempt_count'))} " + f"folds={_int(target.get('evaluated_fold_count'))}" + ) + if summary.get("truncated") is True: + lines.append( + f"- {_int(summary.get('omitted_target_count'))} targets omitted" + ) + return tuple(lines) + + +def _evaluate_models( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + input_profile: ClassicalModelInputProfile, + profile: AutoregressiveProfile, + backend: _Backend, + *, + exponential_smoothing: Mapping[str, JSONValue] | None, + target: Any | None, +) -> AutoregressiveResult: + rows = cast(list[dict[str, Any]], input_result.regularized_frame.to_dicts()) + folds = [dict(fold) for fold in input_result.folds] + origins = _folds_by_origin(folds) + resources = input_profile.resources + specifications = profile.specifications[: resources.max_candidate_orders] + limitations: list[str] = [] + estimated_memory = _estimated_working_memory_bytes( + len(rows), len(folds), len(specifications) + ) + if len(specifications) < len(profile.specifications): + limitations.append("resource_limit") + if estimated_memory > resources.max_memory_bytes: + limitations.append("resource_limit") + specifications = () + started = time.monotonic() + fit_attempt_count = 0 + all_evaluations: list[dict[str, Any]] = [] + model_payloads: list[dict[str, JSONValue]] = [] + all_fit_samples: list[dict[str, JSONValue]] = [] + fit_statuses: Counter[str] = Counter() + fit_reasons: Counter[str] = Counter() + warning_counts: Counter[str] = Counter() + + for specification_code, specification in enumerate(specifications, start=1): + model_id = _model_id( + specification, + _text(input_result.contract.get("derivation_id")), + backend.version, + ) + model_evaluations: list[dict[str, Any]] = [] + model_fit_samples: list[dict[str, JSONValue]] = [] + for _, origin_folds in origins: + if fit_attempt_count >= resources.max_fit_attempts: + limitations.append("resource_limit") + break + if time.monotonic() - started >= resources.max_wall_time_seconds: + limitations.append("timeout") + break + fit_attempt_count += 1 + origin = origin_folds[0] + training_start = _int(origin.get("training_start_index")) + training_end = _int(origin.get("training_end_index")) + indexes = _trailing_contiguous_indexes( + rows, + training_start, + training_end, + _text(origin.get("series_id")), + _text(origin.get("period")), + ) + values = tuple( + _float(rows[index].get("cm_input_value")) for index in indexes + ) + max_horizon = max( + _int(fold.get("horizon")) for fold in origin_folds + ) + outcome = _fit_specification( + specification, + values, + max_horizon, + backend, + ) + fit_statuses[outcome.status] += 1 + if outcome.reason: + fit_reasons[outcome.reason] += 1 + warning_counts.update(outcome.warning_codes) + fit_sample = _fit_sample( + outcome, + specification, + model_id, + origin, + indexes, + len(values), + ) + model_fit_samples.append(fit_sample) + all_fit_samples.append(fit_sample) + original_forecasts = _inverse_forecasts( + rows, + training_end, + outcome.forecasts, + input_profile, + profile.rounding_digits, + ) + for fold in origin_folds: + evaluation = _fold_evaluation( + rows, + fold, + specification, + specification_code, + model_id, + outcome, + original_forecasts, + profile.rounding_digits, + ) + model_evaluations.append(evaluation) + all_evaluations.append(evaluation) + model_payloads.append( + _model_evaluation_payload( + specification, + specification_code, + model_id, + model_evaluations, + model_fit_samples, + resources.max_retained_diagnostics, + profile.rounding_digits, + ) + ) + if limitations and limitations[-1] in {"resource_limit", "timeout"}: + break + + baselines = _reference_baseline_payloads( + rows, + folds, + profile.baseline_rolling_windows, + profile.rounding_digits, + ) + ets_references = _exponential_smoothing_references( + ( + exponential_smoothing + if profile.compare_exponential_smoothing + else None + ), + enabled=profile.compare_exponential_smoothing, + ) + annotations, collisions = _build_annotations( + frame, all_evaluations, profile, input_result, target=target + ) + evaluated_count = sum( + item.get("status") == "evaluated" for item in all_evaluations + ) + failed_count = sum( + status in {"failed", "unavailable"} + for status in fit_statuses.elements() + ) + limited_count = fit_statuses["limited"] + fit_statuses["skipped"] + if not evaluated_count: + limitations.append("insufficient_lags") + limitations = list(dict.fromkeys(limitations)) + status = ( + "ready" + if not limitations and not failed_count and not limited_count + else "limited" + ) + reason = ( + limitations[0] + if limitations + else sorted(fit_reasons)[0] if fit_reasons else None + ) + diagnostics: dict[str, JSONValue] = { + **_base_payload(input_result, fingerprint, profile), + "status": status, + "reason": reason, + "limitations": cast(JSONValue, limitations), + "backend": { + "provider": "statsmodels", + "version": backend.version, + "available": True, + "import_basis": "optional_models_extra", + }, + "fit_summary": { + "schema_version": AUTOREGRESSIVE_FIT_SCHEMA_VERSION, + "fit_attempt_count": fit_attempt_count, + "status_counts": dict(sorted(fit_statuses.items())), + "reason_counts": dict(sorted(fit_reasons.items())), + "warning_counts": dict(sorted(warning_counts.items())), + "failed_fit_count": failed_count, + "limited_fit_count": limited_count, + "convergence_rate": _rate( + fit_statuses["converged"], + fit_attempt_count, + profile.rounding_digits, + ), + "failure_rate": _rate( + failed_count, fit_attempt_count, profile.rounding_digits + ), + "fit_samples": cast( + JSONValue, all_fit_samples[: resources.max_retained_diagnostics] + ), + "fit_samples_truncated": ( + len(all_fit_samples) > resources.max_retained_diagnostics + ), + }, + "resource_usage": { + "limits": resources.to_metadata(), + "estimated_working_memory_bytes": estimated_memory, + "memory_limit_exceeded": estimated_memory + > resources.max_memory_bytes, + "fit_attempt_count": fit_attempt_count, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "evaluation": { + "schema_version": AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin", + "original_scale": True, + "model_count": len(model_payloads), + "fold_count": len(folds), + "evaluated_fold_count": evaluated_count, + "skipped_evaluation_count": len(all_evaluations) - evaluated_count, + "forecast_coverage_rate": _rate( + evaluated_count, len(all_evaluations), profile.rounding_digits + ), + "models": cast(JSONValue, model_payloads), + "reference_baselines": cast(JSONValue, baselines), + "reference_exponential_smoothing": cast(JSONValue, ets_references), + "comparison_semantics": "descriptive_shared_folds_only", + "automatic_winner": False, + }, + "training_projection": _training_projection_metadata( + profile, input_result, len(annotations), collisions + ), + "fit_duration_included": False, + } + return AutoregressiveResult(diagnostics, annotations, input_result) + + +def _fit_specification( + specification: AutoregressiveSpecification, + values: Sequence[float], + horizon: int, + backend: _Backend, +) -> _FitOutcome: + effective_count = max( + 0, len(values) - specification.d - max(specification.p, specification.q) + ) + minimum = max(8, specification.p + specification.q + specification.d + 3) + if len(values) < minimum or effective_count < 3: + return _empty_fit("skipped", "insufficient_lags", effective_count) + if max(values) - min(values) <= 1e-15: + return _empty_fit("skipped", "zero_variance", effective_count) + try: + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + fitted = _statsmodels_fit(specification, values, backend) + raw_forecasts = fitted.forecast(horizon) + stationary, ar_min = _root_status(getattr(fitted, "arroots", ())) + invertible, ma_min = _root_status(getattr(fitted, "maroots", ())) + condition_number = _covariance_condition_number(fitted) + forecasts = tuple(float(value) for value in raw_forecasts) + warning_codes = _warning_codes(captured) + if not forecasts or any( + not math.isfinite(value) for value in forecasts + ): + return _empty_fit( + "failed", + "numerical_failure", + effective_count, + warning_codes=warning_codes, + ) + converged = _fit_converged(fitted, warning_codes) + status = ( + "limited" + if "convergence_warning" in warning_codes or not converged + else "converged" + ) + reason = "optimizer_failure" if status == "limited" else "" + return _FitOutcome( + status=status, + reason=reason, + forecasts=forecasts, + parameters=_fitted_parameters(fitted), + warning_codes=warning_codes, + converged=converged, + stationary=stationary, + invertible=invertible, + ar_root_min_modulus=ar_min, + ma_root_min_modulus=ma_min, + covariance_condition_number=condition_number, + effective_observation_count=effective_count, + ) + except (ArithmeticError, FloatingPointError, OverflowError): + return _empty_fit("failed", "numerical_overflow", effective_count) + except Exception as exc: # backend-specific safety boundary + return _empty_fit( + "failed", _backend_failure_reason(exc), effective_count + ) + + +def _statsmodels_fit( + specification: AutoregressiveSpecification, + values: Sequence[float], + backend: _Backend, +) -> Any: + model = backend.arima( + values, + order=(specification.p, specification.d, specification.q), + seasonal_order=(0, 0, 0, 0), + trend=specification.trend, + enforce_stationarity=specification.enforce_stationarity, + enforce_invertibility=specification.enforce_invertibility, + concentrate_scale=specification.concentrate_scale, + missing="raise", + validate_specification=True, + ) + if specification.initialization_method == "stationary": + model.initialize_stationary() + elif specification.initialization_method == "approximate_diffuse": + model.initialize_approximate_diffuse() + fixed = dict(specification.fixed_parameters) + unknown = sorted(set(fixed) - set(model.param_names)) + if unknown: + raise ValueError( + "fixed parameter is not present in model specification" + ) + method_kwargs: dict[str, Any] | None = None + if specification.estimation_method == "statespace": + method_kwargs = {"maxiter": specification.max_iterations, "disp": 0} + elif specification.estimation_method == "innovations_mle": + method_kwargs = { + "minimize_kwargs": { + "options": {"maxiter": specification.max_iterations} + } + } + context = model.fix_params(fixed) if fixed else nullcontext() + with context: + return model.fit( + method=specification.estimation_method, + method_kwargs=method_kwargs, + low_memory=False, + ) + + +def _empty_fit( + status: str, + reason: str, + effective_count: int, + *, + warning_codes: tuple[str, ...] = (), +) -> _FitOutcome: + return _FitOutcome( + status, + reason, + (), + {}, + warning_codes, + False, + None, + None, + None, + None, + None, + effective_count, + ) + + +def _fit_sample( + outcome: _FitOutcome, + specification: AutoregressiveSpecification, + model_id: str, + fold: Mapping[str, Any], + indexes: Sequence[int], + observation_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": AUTOREGRESSIVE_FIT_SCHEMA_VERSION, + "model_id": model_id, + "specification_id": specification.specification_id, + "family": specification.family, + "order": [specification.p, specification.d, specification.q], + "status": outcome.status, + "reason": outcome.reason or None, + "converged": outcome.converged, + "stationary": outcome.stationary, + "invertible": outcome.invertible, + "ar_root_min_modulus": _rounded(outcome.ar_root_min_modulus, 12), + "ma_root_min_modulus": _rounded(outcome.ma_root_min_modulus, 12), + "covariance_condition_number": _rounded( + outcome.covariance_condition_number, 12 + ), + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "observation_count": observation_count, + "effective_observation_count": outcome.effective_observation_count, + "segment_start_index": indexes[0] if indexes else None, + "segment_end_index": indexes[-1] if indexes else None, + "training_segment_policy": "trailing_contiguous_after_missing", + "parameters": dict(outcome.parameters), + "parameter_count": len(outcome.parameters), + "warning_codes": cast(JSONValue, list(outcome.warning_codes)), + "fit_duration_included": False, + "backend_exception_text_included": False, + } + + +def _fold_evaluation( + rows: Sequence[Mapping[str, Any]], + fold: Mapping[str, Any], + specification: AutoregressiveSpecification, + specification_code: int, + model_id: str, + outcome: _FitOutcome, + original_forecasts: Sequence[float | None], + rounding_digits: int, +) -> dict[str, Any]: + horizon = _int(fold.get("horizon")) + forecast = ( + original_forecasts[horizon - 1] + if 0 < horizon <= len(original_forecasts) + else None + ) + transformed = ( + outcome.forecasts[horizon - 1] + if 0 < horizon <= len(outcome.forecasts) + else None + ) + target_index = _int(fold.get("target_index")) + actual = ( + _optional_float(rows[target_index].get("cm_input_observed_value")) + if 0 <= target_index < len(rows) + else None + ) + if outcome.status in {"failed", "skipped", "unavailable"}: + status, reason = "not_evaluated", outcome.reason + elif fold.get("status") != "valid" or actual is None: + status, reason = "skipped", "target_unavailable" + elif forecast is None: + status, reason = "skipped", "inverse_transform_unavailable" + else: + status, reason = "evaluated", "" + error = ( + forecast - actual + if status == "evaluated" and forecast is not None and actual is not None + else None + ) + return { + "schema_version": AUTOREGRESSIVE_FORECAST_SCHEMA_VERSION, + "status": status, + "reason": reason or None, + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "family_code": AUTOREGRESSIVE_FAMILY_CODES[specification.family], + "p": specification.p, + "d": specification.d, + "q": specification.q, + "trend": specification.trend, + "trend_code": AUTOREGRESSIVE_TREND_CODES[specification.trend], + "initialization_method": specification.initialization_method, + "estimation_method": specification.estimation_method, + "fit_status": outcome.status, + "fit_reason": outcome.reason or None, + "converged": outcome.converged, + "stationary": outcome.stationary, + "invertible": outcome.invertible, + "ar_root_min_modulus": _rounded( + outcome.ar_root_min_modulus, rounding_digits + ), + "ma_root_min_modulus": _rounded( + outcome.ma_root_min_modulus, rounding_digits + ), + "covariance_condition_number": _rounded( + outcome.covariance_condition_number, rounding_digits + ), + "effective_observation_count": outcome.effective_observation_count, + "fold_id": _int(fold.get("fold_id")), + "origin_row_id": fold.get("origin_row_id"), + "target_row_id": fold.get("target_row_id"), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "target_bin_end_utc_ms": _int(fold.get("target_bin_end_utc_ms")), + "horizon": horizon, + "transformed_forecast": _rounded(transformed, rounding_digits), + "forecast": _rounded(forecast, rounding_digits), + "actual": _rounded(actual, rounding_digits), + "error": _rounded(error, rounding_digits), + "absolute_error": _rounded( + abs(error) if error is not None else None, rounding_digits + ), + "squared_error": _rounded( + error * error if error is not None else None, rounding_digits + ), + "original_scale": True, + "future_values_visible": False, + "residual_state_reused_across_origins": False, + "automatic_winner": False, + } + + +def _model_evaluation_payload( + specification: AutoregressiveSpecification, + specification_code: int, + model_id: str, + evaluations: Sequence[Mapping[str, Any]], + fit_samples: Sequence[Mapping[str, JSONValue]], + retained_limit: int, + rounding_digits: int, +) -> dict[str, JSONValue]: + statuses = Counter(_text(row.get("status")) for row in evaluations) + fit_statuses = Counter(_text(row.get("status")) for row in fit_samples) + reasons = Counter( + _text(row.get("reason")) + for row in evaluations + if _text(row.get("reason")) + ) + horizons = sorted({_int(row.get("horizon")) for row in evaluations}) + return { + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "configuration": specification.to_metadata(), + "status": "ready" if statuses["evaluated"] else "limited", + "fit_status_counts": dict(sorted(fit_statuses.items())), + "evaluation_status_counts": dict(sorted(statuses.items())), + "reason_counts": dict(sorted(reasons.items())), + "forecast_coverage_rate": _rate( + statuses["evaluated"], len(evaluations), rounding_digits + ), + "convergence_rate": _rate( + fit_statuses["converged"], len(fit_samples), rounding_digits + ), + "horizon_metrics": cast( + JSONValue, + [ + _metrics_for_horizon(evaluations, horizon, rounding_digits) + for horizon in horizons + ], + ), + "parameter_stability": _parameter_stability( + fit_samples, rounding_digits + ), + "fold_results": cast( + JSONValue, [dict(row) for row in evaluations[:retained_limit]] + ), + "fold_results_truncated": len(evaluations) > retained_limit, + "fit_samples": cast( + JSONValue, [dict(row) for row in fit_samples[:retained_limit]] + ), + "fit_samples_truncated": len(fit_samples) > retained_limit, + "automatic_winner": False, + } + + +def _metrics_for_horizon( + evaluations: Sequence[Mapping[str, Any]], + horizon: int, + rounding_digits: int, +) -> dict[str, JSONValue]: + selected = [ + row + for row in evaluations + if _int(row.get("horizon")) == horizon + and row.get("status") == "evaluated" + ] + errors = [_float(row.get("error")) for row in selected] + absolute = [abs(value) for value in errors] + squared = [value * value for value in errors] + return { + "horizon": horizon, + "evaluation_count": len(selected), + "mae": _rounded( + sum(absolute) / len(absolute) if absolute else None, rounding_digits + ), + "rmse": _rounded( + math.sqrt(sum(squared) / len(squared)) if squared else None, + rounding_digits, + ), + "bias": _rounded( + sum(errors) / len(errors) if errors else None, rounding_digits + ), + "original_scale": True, + } + + +def _parameter_stability( + fit_samples: Sequence[Mapping[str, JSONValue]], rounding_digits: int +) -> dict[str, JSONValue]: + parameters: dict[str, list[float]] = {} + for sample in fit_samples: + for name, raw in _mapping(sample.get("parameters")).items(): + value = _optional_float(raw) + if value is not None: + parameters.setdefault(name, []).append(value) + summaries: dict[str, JSONValue] = {} + for name, values in sorted(parameters.items()): + center = median(values) + summaries[name] = { + "count": len(values), + "min": _rounded(min(values), rounding_digits), + "max": _rounded(max(values), rounding_digits), + "median": _rounded(center, rounding_digits), + "mad": _rounded( + median(abs(value - center) for value in values), rounding_digits + ), + } + return { + "parameter_count": len(summaries), + "parameters": summaries, + "bounded": True, + } + + +def _exponential_smoothing_references( + payload: Mapping[str, JSONValue] | None, + *, + enabled: bool, +) -> list[dict[str, JSONValue]]: + if not enabled: + return [{"status": "not_requested", "automatic_winner": False}] + selected = _mapping(payload) + models = _mapping_rows(_mapping(selected.get("evaluation")).get("models")) + if not models: + return [ + { + "status": "unavailable", + "reason": "exponential_smoothing_not_enabled", + "automatic_winner": False, + } + ] + return [ + { + "status": model.get("status"), + "family": model.get("family"), + "specification_id": model.get("specification_id"), + "model_id": model.get("model_id"), + "horizon_metrics": model.get("horizon_metrics"), + "calculation_basis": "shared_regular_grid_folds", + "automatic_winner": False, + } + for model in models[:MAX_AUTOREGRESSIVE_SPECIFICATIONS] + ] + + +def _build_annotations( + frame: Any | None, + evaluations: Sequence[Mapping[str, Any]], + profile: AutoregressiveProfile, + input_result: ClassicalModelInputResult, + *, + target: Any | None, +) -> tuple[tuple[Mapping[str, Any], ...], int]: + if frame is None: + return (), 0 + try: + enriched = ensure_tick_training_features(frame, target=target) + except (AttributeError, TypeError, ValueError): + return (), 0 + source_rows = cast(list[dict[str, Any]], enriched.to_dicts()) + availability: dict[tuple[str, str], list[tuple[int, int]]] = {} + for row in source_rows: + timestamp = _optional_int(row.get("timestamp_utc_ms")) + row_id = _optional_int(row.get("row_id")) + if timestamp is not None and row_id is not None: + availability.setdefault( + (_text(row.get("series_id")), _text(row.get("period"))), [] + ).append((timestamp, row_id)) + for values in availability.values(): + values.sort() + + merged: dict[tuple[str, str, int], dict[str, Any]] = {} + family_folds: dict[tuple[str, str, int, str], int] = {} + collisions = 0 + selected = [ + row + for row in evaluations + if _text(row.get("specification_id")) + in profile.projection_specification_ids + and _int(row.get("horizon")) == profile.projection_horizon + ] + selected.sort( + key=lambda row: ( + _text(row.get("series_id")), + _text(row.get("period")), + _int(row.get("origin_bin_end_utc_ms")), + _int(row.get("fold_id")), + _text(row.get("family")), + ) + ) + for evaluation in selected: + group = ( + _text(evaluation.get("series_id")), + _text(evaluation.get("period")), + ) + family = _text(evaluation.get("family")) + for diagnostic, value_key, time_key in ( + (False, "forecast", "origin_bin_end_utc_ms"), + (True, "error", "target_bin_end_utc_ms"), + ): + if ( + diagnostic + and _optional_float(evaluation.get(value_key)) is None + ): + continue + row_id = _first_available_row_id( + availability.get(group, ()), _int(evaluation.get(time_key)) + ) + if row_id is None: + continue + key = (*group, row_id) + family_key = (*key, family) + annotation = _annotation_row( + evaluation, input_result, row_id, diagnostic=diagnostic + ) + existing = merged.setdefault( + key, + {"series_id": group[0], "period": group[1], "row_id": row_id}, + ) + existing_fold = family_folds.get(family_key) + current_fold = _int(evaluation.get("fold_id")) + if existing_fold is not None and existing_fold != current_fold: + collisions += 1 + if diagnostic and existing_fold == current_fold: + prefix = f"cm_{family}_" + for suffix in ( + "actual", + "error", + "diagnostic_available", + "diagnostic_available_at_utc_ms", + ): + existing[prefix + suffix] = annotation.get(prefix + suffix) + else: + existing.update(annotation) + family_folds[family_key] = current_fold + return tuple(merged[key] for key in sorted(merged)), collisions + + +def _annotation_row( + evaluation: Mapping[str, Any], + input_result: ClassicalModelInputResult, + row_id: int, + *, + diagnostic: bool, +) -> dict[str, Any]: + family = _text(evaluation.get("family")) + prefix = f"cm_{family}_" + fit_status = _text(evaluation.get("fit_status")) or "unavailable" + forecast_available = ( + not diagnostic + and _optional_float(evaluation.get("forecast")) is not None + ) + return { + "series_id": _text(evaluation.get("series_id")), + "period": _text(evaluation.get("period")), + "row_id": row_id, + prefix + + "schema_version": AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION, + prefix + + "input_derivation_id": _text( + input_result.contract.get("derivation_id") + ), + prefix + "model_id": _text(evaluation.get("model_id")), + prefix + "family_code": _int(evaluation.get("family_code")), + prefix + + "specification_code": _int(evaluation.get("specification_code")), + prefix + "p": _int(evaluation.get("p")), + prefix + "d": _int(evaluation.get("d")), + prefix + "q": _int(evaluation.get("q")), + prefix + "trend_code": _int(evaluation.get("trend_code")), + prefix + "calculation_basis_code": 1, + prefix + + "fit_status_code": AUTOREGRESSIVE_FIT_STATUS_CODES.get(fit_status, 1), + prefix + + "failure_reason_code": AUTOREGRESSIVE_REASON_CODES.get( + _text(evaluation.get("fit_reason")), 0 + ), + prefix + "converged": bool(evaluation.get("converged", False)), + prefix + "stationary": evaluation.get("stationary"), + prefix + "invertible": evaluation.get("invertible"), + prefix + + "ar_root_min_modulus": _optional_float( + evaluation.get("ar_root_min_modulus") + ), + prefix + + "ma_root_min_modulus": _optional_float( + evaluation.get("ma_root_min_modulus") + ), + prefix + + "covariance_condition_number": _optional_float( + evaluation.get("covariance_condition_number") + ), + prefix + + "effective_observation_count": _int( + evaluation.get("effective_observation_count") + ), + prefix + "fold_id": _int(evaluation.get("fold_id")), + prefix + "origin_row_id": evaluation.get("origin_row_id"), + prefix + "target_row_id": evaluation.get("target_row_id"), + prefix + "horizon": _int(evaluation.get("horizon")), + prefix + "forecast": _optional_float(evaluation.get("forecast")), + prefix + "forecast_available": forecast_available, + prefix + + "forecast_available_at_utc_ms": _int( + evaluation.get("origin_bin_end_utc_ms") + ), + prefix + + "actual": ( + _optional_float(evaluation.get("actual")) if diagnostic else None + ), + prefix + + "error": ( + _optional_float(evaluation.get("error")) if diagnostic else None + ), + prefix + "diagnostic_available": diagnostic, + prefix + + "diagnostic_available_at_utc_ms": ( + _int(evaluation.get("target_bin_end_utc_ms")) + if diagnostic + else None + ), + prefix + "diagnostic_only": diagnostic, + prefix + "original_scale": True, + prefix + "training_eligible": forecast_available, + } + + +def _training_projection_metadata( + profile: AutoregressiveProfile, + input_result: ClassicalModelInputResult, + annotation_count: int, + collision_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "row", + "identity_fields": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "mapping_policy": "first_source_row_at_or_after_availability", + "collision_policy": "latest_origin_wins_per_family", + "collision_count": collision_count, + "annotation_count": annotation_count, + "projection_specification_ids": list( + profile.projection_specification_ids + ), + "projection_horizon": profile.projection_horizon, + "input_derivation_id": input_result.contract.get("derivation_id"), + "column_names": list(AUTOREGRESSIVE_COLUMNS), + "family_column_prefixes": ["cm_ar_", "cm_arma_", "cm_arima_"], + "forecast_time_values_use_future": False, + "diagnostics_marked_post_observation": True, + "observed_columns_overwritten": False, + } + + +def _base_payload( + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + profile: AutoregressiveProfile, +) -> dict[str, JSONValue]: + contract = input_result.contract + regularization = _mapping(contract.get("regularization")) + audit = _mapping(fingerprint.get("fingerprint_audit")) + statuses = _mapping(audit.get("section_statuses")) + stationarity = _mapping(fingerprint.get("stationarity_diagnostics")) + return { + "schema_version": AUTOREGRESSIVE_SCHEMA_VERSION, + "advisory": True, + "target_axis": dict(_mapping(contract.get("target_axis"))), + "reference_fingerprint_id": contract.get("reference_fingerprint_id") + or fingerprint.get("fingerprint_id"), + "input_schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "input_derivation_id": contract.get("derivation_id"), + "input_status": contract.get("status"), + "calculation_basis": "regular_grid_rolling_origin", + "configuration": profile.to_metadata(), + "input_transform_policy": dict( + _mapping(contract.get("transform_policy")) + ), + "input_missingness_policy": { + "expected_closure_policy": regularization.get( + "expected_closure_policy" + ), + "unexpected_missing_policy": regularization.get( + "unexpected_missing_policy" + ), + "expected_closure_count": regularization.get( + "expected_closure_count" + ), + "unexpected_missing_count": regularization.get( + "unexpected_missing_count" + ), + "forward_fill_policy": regularization.get("forward_fill_policy"), + }, + "prerequisite_readiness": { + "dependence_status": statuses.get("dependence") or "unknown", + "stationarity_status": statuses.get("stationarity_diagnostics") + or "unknown", + "stationarity_recommendations": stationarity.get( + "recommended_transforms" + ) + or [], + "recommendations_applied_automatically": False, + "advisory_only": True, + }, + "model_differencing_is_separate_from_input_differencing": True, + "missing_observation_policy": "reset_to_trailing_contiguous_segment", + "forward_fill_policy": "never", + "original_scale_forecasts": True, + "automatic_order_selection": False, + "automatic_winner": False, + "hard_fail_quality_gate": False, + "fitted_objects_included": False, + "backend_exception_text_included": False, + } + + +def _unavailable_result( + input_result: ClassicalModelInputResult, + base: Mapping[str, JSONValue], + reason: str, + *, + status: str = "unavailable", +) -> AutoregressiveResult: + configuration = _mapping(base.get("configuration")) + diagnostics: dict[str, JSONValue] = { + **dict(base), + "status": status, + "reason": reason, + "limitations": [reason], + "backend": { + "provider": "statsmodels", + "version": None, + "available": False, + "import_basis": "optional_models_extra", + }, + "fit_summary": { + "schema_version": AUTOREGRESSIVE_FIT_SCHEMA_VERSION, + "fit_attempt_count": 0, + "status_counts": {}, + "reason_counts": {reason: 1}, + "warning_counts": {}, + "failed_fit_count": 0, + "limited_fit_count": 0, + "convergence_rate": None, + "failure_rate": None, + "fit_samples": [], + "fit_samples_truncated": False, + }, + "evaluation": { + "schema_version": AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin", + "original_scale": True, + "model_count": len( + _mapping_rows(configuration.get("specifications")) + ), + "fold_count": len(input_result.folds), + "evaluated_fold_count": 0, + "skipped_evaluation_count": 0, + "forecast_coverage_rate": None, + "models": [], + "reference_baselines": [], + "reference_exponential_smoothing": [], + "comparison_semantics": "descriptive_shared_folds_only", + "automatic_winner": False, + }, + "resource_usage": { + "limits": dict( + _mapping( + _mapping(input_result.contract.get("configuration")).get( + "resources" + ) + ) + ), + "estimated_working_memory_bytes": 0, + "memory_limit_exceeded": False, + "fit_attempt_count": 0, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "training_projection": { + "schema_version": AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "row", + "identity_fields": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "mapping_policy": "first_source_row_at_or_after_availability", + "collision_policy": "latest_origin_wins_per_family", + "collision_count": 0, + "annotation_count": 0, + "projection_specification_ids": configuration.get( + "projection_specification_ids" + ), + "projection_horizon": configuration.get("projection_horizon"), + "input_derivation_id": input_result.contract.get("derivation_id"), + "column_names": list(AUTOREGRESSIVE_COLUMNS), + "family_column_prefixes": ["cm_ar_", "cm_arma_", "cm_arima_"], + "forecast_time_values_use_future": False, + "diagnostics_marked_post_observation": True, + "observed_columns_overwritten": False, + }, + "fit_duration_included": False, + } + return AutoregressiveResult(diagnostics, (), input_result) + + +def _load_backend() -> _Backend | None: + try: + statsmodels = importlib.import_module("statsmodels") + module = importlib.import_module("statsmodels.tsa.arima.model") + version = getattr(statsmodels, "__version__", "") or ( + importlib.metadata.version("statsmodels") + ) + return _Backend(version=str(version), arima=module.ARIMA) + except ( + ImportError, + ModuleNotFoundError, + importlib.metadata.PackageNotFoundError, + ): + return None + + +def _warning_codes( + captured: Sequence[warnings.WarningMessage], +) -> tuple[str, ...]: + return tuple( + sorted( + { + AUTOREGRESSIVE_WARNING_CODES.get( + item.category.__name__, "backend_warning" + ) + for item in captured + } + ) + ) + + +def _fit_converged(fitted: Any, warning_codes: Sequence[str]) -> bool: + if "convergence_warning" in warning_codes: + return False + result = getattr(fitted, "mle_retvals", None) + if result is None: + return True + if isinstance(result, Mapping): + value = result.get("converged", result.get("success")) + else: + value = getattr(result, "success", getattr(result, "converged", None)) + return True if value is None else bool(value) + + +def _root_status(raw: Any) -> tuple[bool | None, float | None]: + try: + moduli = [abs(complex(value)) for value in raw] + except (TypeError, ValueError): + return None, None + if not moduli: + return None, None + minimum = min(moduli) + return all(value > 1.0 for value in moduli), float(minimum) + + +def _fitted_parameters(fitted: Any) -> dict[str, float]: + names = [str(name) for name in getattr(fitted, "param_names", ())] + raw = getattr(fitted, "params", ()) + values: dict[str, float] = {} + for name, value in zip(names, raw, strict=False): + scalar = _optional_float(value) + if scalar is not None: + values[name.removesuffix(" (fixed)")] = scalar + return dict(sorted(values.items())) + + +def _covariance_condition_number(fitted: Any) -> float | None: + try: + numpy = importlib.import_module("numpy") + value = float(numpy.linalg.cond(fitted.cov_params())) + return value if math.isfinite(value) else None + except Exception: + return None + + +def _backend_failure_reason(exc: Exception) -> str: + message = str(exc).lower() + if "not stationary" in message or "non-stationary" in message: + return "nonstationary_configuration" + if "not invertible" in message or "non-invertible" in message: + return "noninvertible_configuration" + if "singular" in message or "linalg" in message: + return "singularity" + if "order" in message or "lag" in message: + return "invalid_order" + if "trend" in message or "parameter" in message or "initial" in message: + return "invalid_configuration" + if "overflow" in message or "finite" in message: + return "numerical_overflow" + return "backend_failure" + + +def _folds_by_origin( + folds: Sequence[Mapping[str, Any]], +) -> list[tuple[tuple[str, str, int], list[dict[str, Any]]]]: + grouped: dict[tuple[str, str, int], list[dict[str, Any]]] = {} + for fold in folds: + key = ( + _text(fold.get("series_id")), + _text(fold.get("period")), + _int(fold.get("origin_bin_end_utc_ms")), + ) + grouped.setdefault(key, []).append(dict(fold)) + return [ + (key, sorted(values, key=lambda item: _int(item.get("horizon")))) + for key, values in sorted(grouped.items()) + ] + + +def _estimated_working_memory_bytes( + row_count: int, fold_count: int, specification_count: int +) -> int: + return ( + row_count * 24 * 8 + + fold_count * 96 * 8 + + specification_count * 1024 * 8 + ) + + +def _model_id( + specification: AutoregressiveSpecification, + input_derivation_id: str, + backend_version: str, +) -> str: + payload = { + "schema_version": AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION, + "input_derivation_id": input_derivation_id, + "backend": "statsmodels", + "backend_version": backend_version, + "specification": specification.to_metadata(), + } + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("ascii") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _first_available_row_id( + values: Sequence[tuple[int, int]], threshold: int +) -> int | None: + return next( + (row_id for timestamp, row_id in values if timestamp >= threshold), None + ) + + +def _ensure_projection_columns(frame: Any) -> Any: + import polars as pl + + expressions = [] + for name, dtype in _projection_dtypes().items(): + if name in frame.columns: + expressions.append(pl.col(name).cast(dtype).alias(name)) + else: + expressions.append(pl.lit(None).cast(dtype).alias(name)) + return frame.with_columns(expressions) + + +def _projection_dtypes() -> dict[str, Any]: + import polars as pl + + strings = {"schema_version", "input_derivation_id", "model_id"} + booleans = { + "converged", + "stationary", + "invertible", + "forecast_available", + "diagnostic_available", + "diagnostic_only", + "original_scale", + "training_eligible", + } + floats = { + "forecast", + "actual", + "error", + "ar_root_min_modulus", + "ma_root_min_modulus", + "covariance_condition_number", + } + return { + f"cm_{family}_{suffix}": ( + pl.Utf8 + if suffix in strings + else ( + pl.Boolean + if suffix in booleans + else pl.Float64 if suffix in floats else pl.Int64 + ) + ) + for family in AUTOREGRESSIVE_FAMILIES + for suffix in AUTOREGRESSIVE_FAMILY_COLUMN_SUFFIXES + } + + +def _rate(numerator: int, denominator: int, digits: int) -> float | None: + return _rounded(numerator / denominator if denominator else None, digits) + + +def _target_sort_key(value: Mapping[str, JSONValue]) -> tuple[str, ...]: + axis = _mapping(value.get("target_axis")) + return tuple( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + + +def _mapping(value: object) -> Mapping[str, JSONValue]: + return ( + cast(Mapping[str, JSONValue], value) + if isinstance(value, Mapping) + else {} + ) + + +def _mapping_rows(value: object) -> list[Mapping[str, JSONValue]]: + if not isinstance(value, list): + return [] + return [ + cast(Mapping[str, JSONValue], row) + for row in value + if isinstance(row, Mapping) + ] + + +def _text(value: object) -> str: + return str(value or "") + + +def _int(value: object) -> int: + try: + return int(cast(Any, value)) + except (TypeError, ValueError, OverflowError): + return 0 + + +def _optional_int(value: object) -> int | None: + try: + return int(cast(Any, value)) if value is not None else None + except (TypeError, ValueError, OverflowError): + return None + + +def _float(value: object) -> float: + return float(cast(Any, value)) + + +def _optional_float(value: object) -> float | None: + try: + candidate = float(cast(Any, value)) + except (TypeError, ValueError, OverflowError): + return None + return candidate if math.isfinite(candidate) else None + + +def _rounded(value: float | None, digits: int) -> float | None: + if value is None or not math.isfinite(value): + return None + return round(value, digits) diff --git a/src/histdatacom/data_quality/bounded_payload_contracts.py b/src/histdatacom/data_quality/bounded_payload_contracts.py index 7a19f957..c8864a61 100644 --- a/src/histdatacom/data_quality/bounded_payload_contracts.py +++ b/src/histdatacom/data_quality/bounded_payload_contracts.py @@ -6,6 +6,9 @@ from dataclasses import dataclass from typing import cast +from histdatacom.data_quality.autoregressive import ( + AUTOREGRESSIVE_SCHEMA_VERSION, +) from histdatacom.data_quality.contracts import ( QualityFinding, QualityLocation, @@ -15,10 +18,42 @@ QualityTarget, QualityTargetKind, ) +from histdatacom.data_quality.classical_baselines import ( + CLASSICAL_BASELINE_SCHEMA_VERSION, +) +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, +) +from histdatacom.data_quality.classical_model_comparison import ( + CLASSICAL_MODEL_COMPARISON_ELIGIBILITY_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION, + CLASSICAL_MODEL_FIT_ACCOUNTING_SCHEMA_VERSION, +) +from histdatacom.data_quality.engine import ( + QUALITY_ENGINE_METADATA_KEY, + run_quality_assessment, +) +from histdatacom.data_quality.exponential_smoothing import ( + EXPONENTIAL_SMOOTHING_SCHEMA_VERSION, +) +from histdatacom.data_quality.seasonal_exogenous import ( + SEASONAL_EXOGENOUS_SCHEMA_VERSION, +) +from histdatacom.data_quality.state_space import STATE_SPACE_SCHEMA_VERSION +from histdatacom.data_quality.volatility import VOLATILITY_SCHEMA_VERSION from histdatacom.data_quality.fingerprints import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, + CROSS_SERIES_FINGERPRINT_RULE_ID, + CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, SERIES_FINGERPRINT_RULE_ID, + TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION, ) +from histdatacom.data_quality.limits import bounded_report_limit from histdatacom.data_quality.profiles import QUALITY_REPORTING_METADATA_KEY +from histdatacom.data_quality.synthetic_constraints import ( + synthetic_constraints_from_fingerprint, +) from histdatacom.data_quality.reporting import ( QUALITY_REMEDIATION_CATALOG_AUDIT_METADATA_KEY, QualityExitPolicy, @@ -133,6 +168,159 @@ class _BoundedSequenceContract: "truncated", ), ), + _BoundedSequenceContract( + "remediation_plan", + ( + "remediation_catalog_audit", + "payload_limits", + "remediation_plan", + ), + ("remediation_catalog_audit", "remediation_plan", "items"), + ( + "remediation_catalog_audit", + "payload_limits", + "remediation_plan", + "total_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "remediation_plan", + "included_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "remediation_plan", + "omitted_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "remediation_plan", + "truncated", + ), + ), + _BoundedSequenceContract( + "remediation_attribution_reasons", + ( + "remediation_catalog_audit", + "payload_limits", + "attribution_reason_counts", + ), + ( + "remediation_catalog_audit", + "known_code_counts", + "attribution_reason_counts", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "attribution_reason_counts", + "total_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "attribution_reason_counts", + "included_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "attribution_reason_counts", + "omitted_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "attribution_reason_counts", + "truncated", + ), + ), + _BoundedSequenceContract( + "remediation_unresolved_helpers", + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_source_helper_counts", + ), + ( + "remediation_catalog_audit", + "known_code_counts", + "unresolved_source_helper_counts", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_source_helper_counts", + "total_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_source_helper_counts", + "included_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_source_helper_counts", + "omitted_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_source_helper_counts", + "truncated", + ), + ), + _BoundedSequenceContract( + "remediation_unresolved_prefixes", + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_finding_code_prefix_counts", + ), + ( + "remediation_catalog_audit", + "known_code_counts", + "unresolved_finding_code_prefix_counts", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_finding_code_prefix_counts", + "total_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_finding_code_prefix_counts", + "included_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_finding_code_prefix_counts", + "omitted_count", + ), + ( + "remediation_catalog_audit", + "payload_limits", + "unresolved_finding_code_prefix_counts", + "truncated", + ), + ), + _BoundedSequenceContract( + "quality_engine_skip_events", + ("quality_engine", "skip_events", "limit_metadata", "events"), + ("quality_engine", "skip_events", "events"), + ("quality_engine", "skip_events", "event_count"), + ("quality_engine", "skip_events", "included_event_count"), + ("quality_engine", "skip_events", "omitted_event_count"), + ("quality_engine", "skip_events", "truncated"), + ), _BoundedSequenceContract( "fingerprint_distribution", ("fingerprint_distribution", "limit_metadata", "targets"), @@ -166,6 +354,19 @@ class _BoundedSequenceContract: ("fingerprint_topology", "omitted_target_count"), ("fingerprint_topology", "truncated"), ), + _BoundedSequenceContract( + "fingerprint_cross_series", + ( + "fingerprint_cross_series", + "limit_metadata", + "groups", + ), + ("fingerprint_cross_series", "groups"), + ("fingerprint_cross_series", "group_count"), + ("fingerprint_cross_series", "included_group_count"), + ("fingerprint_cross_series", "omitted_group_count"), + ("fingerprint_cross_series", "truncated"), + ), _BoundedSequenceContract( "fingerprint_topology_attention", ("fingerprint_topology_attention", "limit_metadata", "targets"), @@ -710,6 +911,15 @@ def _iter_limit_payloads( return tuple(results) +class _RepresentativeQualitySkipRule: + rule_id = "time.ascii.gaps" + description = "semantic scans prefer extracted CSVs" + + def evaluate(self, target: QualityTarget) -> tuple[QualityFinding, ...]: + del target + return () + + def _representative_quality_report() -> QualityReport: valid_tick = _target( "data/EURUSD-T-valid.csv", symbol="EURUSD", timeframe="T" @@ -735,6 +945,14 @@ def _representative_quality_report() -> QualityReport: data_format="ascii", timeframe="T", ) + duplicate_archive = QualityTarget( + path="data/EURUSD-T-valid.zip", + kind=QualityTargetKind.ZIP, + data_format=valid_tick.data_format, + timeframe=valid_tick.timeframe, + symbol=valid_tick.symbol, + period=valid_tick.period, + ) fingerprint_findings = ( _fingerprint_finding(valid_tick, _valid_tick_fingerprint_payload()), @@ -788,6 +1006,7 @@ def _representative_quality_report() -> QualityReport: }, ) targets = ( + duplicate_archive, valid_tick, limited_tick, tick, @@ -796,9 +1015,15 @@ def _representative_quality_report() -> QualityReport: negative_spread, directory, ) + skip_report = run_quality_assessment( + targets=targets, + rules=(_RepresentativeQualitySkipRule(),), + ) + quality_engine = skip_report.metadata[QUALITY_ENGINE_METADATA_KEY] return QualityReport( targets=targets, rule_results=( + *skip_report.rule_results, *( QualityRuleResult( rule_id=SERIES_FINGERPRINT_RULE_ID, @@ -829,15 +1054,116 @@ def _representative_quality_report() -> QualityReport: ), ), metadata={ + QUALITY_ENGINE_METADATA_KEY: quality_engine, + CROSS_SERIES_FINGERPRINT_METADATA_KEY: ( + _representative_cross_series_fingerprint() + ), QUALITY_REPORTING_METADATA_KEY: { QUALITY_REMEDIATION_CATALOG_AUDIT_METADATA_KEY: { "enabled": True, } - } + }, }, ) +def _representative_cross_series_fingerprint() -> dict[str, JSONValue]: + group_limit = bounded_report_limit(32, default_limit=32) + correlation_limit = bounded_report_limit(32, default_limit=32) + return { + "schema_version": CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, + "rule_id": CROSS_SERIES_FINGERPRINT_RULE_ID, + "status": "valid", + "fx_series_count": 3, + "group_count": 1, + "incomplete_group_count": 0, + "included_group_count": 1, + "omitted_group_count": 0, + "truncated": False, + "limit_metadata": { + "groups": group_limit.limit_payload(), + "correlations_per_group": correlation_limit.limit_payload(), + }, + "triangular_consistency": { + "candidate_count": 1, + "warning_count": 0, + "error_count": 0, + }, + "inverse_consistency": { + "candidate_count": 0, + "warning_count": 0, + "error_count": 0, + }, + "stale_join_risk": {"risk_count": 0, "samples": []}, + "panel_coverage": [ + { + "timeframe": "T", + "symbols": ["EURGBP", "EURUSD", "GBPUSD"], + "union_period_count": 1, + "common_period_count": 1, + "common_first_period": "201202", + "common_last_period": "201202", + "unequal_period_ranges": False, + "limiting_start_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD", + ], + "limiting_end_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD", + ], + "first_period_by_symbol": { + "EURGBP": "201202", + "EURUSD": "201202", + "GBPUSD": "201202", + }, + "last_period_by_symbol": { + "EURGBP": "201202", + "EURUSD": "201202", + "GBPUSD": "201202", + }, + "missing_period_count_by_symbol": { + "EURGBP": 0, + "EURUSD": 0, + "GBPUSD": 0, + }, + } + ], + "groups": [ + { + "group_id": "ascii:T:201202", + "symbols": ["EURGBP", "EURUSD", "GBPUSD"], + "expected_symbols": ["EURGBP", "EURUSD", "GBPUSD"], + "missing_symbols": [], + "complete": True, + "timestamp_grid": { + "common_timestamp_count": 3, + "union_timestamp_count": 3, + "common_timestamp_ratio": 1.0, + "missing_by_symbol": { + "EURGBP": 0, + "EURUSD": 0, + "GBPUSD": 0, + }, + }, + "coverage_ranges": {"unequal_ranges": False}, + "return_correlation": { + "pair_count": 3, + "included_pair_count": 3, + "omitted_pair_count": 0, + "truncated": False, + "limit_metadata": { + "pairs": correlation_limit.limit_payload() + }, + "pairs": [], + }, + } + ], + } + + def _target(path: str, *, symbol: str, timeframe: str) -> QualityTarget: return QualityTarget( path=path, @@ -916,13 +1242,36 @@ def _limited_tick_fingerprint_payload() -> dict[str, JSONValue]: tick_spread_eligible=True, tick_spread_emitted=True, ) + payload["synthetic_constraints"] = synthetic_constraints_from_fingerprint( + payload + ) + payload["classical_baselines"] = _classical_baseline_payload( + "GBPUSD", status="limited" + ) + payload["classical_model_input"] = _classical_model_input_payload( + "GBPUSD", status="limited" + ) + payload["exponential_smoothing"] = _exponential_smoothing_payload( + "GBPUSD", status="limited" + ) + payload["autoregressive"] = _autoregressive_payload( + "GBPUSD", status="limited" + ) + payload["seasonal_exogenous"] = _seasonal_exogenous_payload( + "GBPUSD", status="limited" + ) + payload["state_space"] = _state_space_payload("GBPUSD", status="limited") + payload["volatility"] = _volatility_payload("GBPUSD", status="limited") + payload["classical_model_comparison"] = _classical_model_comparison_payload( + "GBPUSD", status="limited" + ) return payload def _tick_fingerprint_payload( *, symbol: str = "EURUSD" ) -> dict[str, JSONValue]: - return { + payload: dict[str, JSONValue] = { "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), "coverage": {"row_count": 5, "parsed_row_count": 5}, "temporal_topology": _topology_payload(computed_from="text_scan"), @@ -952,6 +1301,7 @@ def _tick_fingerprint_payload( } }, }, + "cache_source_parity": _cache_source_parity_payload(symbol), "microstructure_dynamics": _microstructure_dynamics_payload(), "dependence": _dependence_payload(status="ok"), "fingerprint_audit": _audit_payload( @@ -979,6 +1329,361 @@ def _tick_fingerprint_payload( ), "source": {"kind": "csv_text"}, } + payload["synthetic_constraints"] = synthetic_constraints_from_fingerprint( + payload + ) + payload["classical_baselines"] = _classical_baseline_payload(symbol) + payload["classical_model_input"] = _classical_model_input_payload(symbol) + payload["exponential_smoothing"] = _exponential_smoothing_payload(symbol) + payload["autoregressive"] = _autoregressive_payload(symbol) + payload["seasonal_exogenous"] = _seasonal_exogenous_payload(symbol) + payload["state_space"] = _state_space_payload(symbol) + payload["volatility"] = _volatility_payload(symbol) + payload["classical_model_comparison"] = _classical_model_comparison_payload( + symbol + ) + return payload + + +def _classical_baseline_payload( + symbol: str, + *, + status: str = "ready", +) -> dict[str, JSONValue]: + return { + "schema_version": CLASSICAL_BASELINE_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": "stationarity_limited" if status == "limited" else None, + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "split_policy": { + "training_row_count": 80, + "evaluation_row_count": 20, + }, + "evaluation": { + "guard_codes": ( + ["stationarity_limited"] if status == "limited" else [] + ), + "best_model": { + "model": "naive_random_walk", + "model_code": 1, + "mae": 0.0001, + "rmse": 0.0002, + }, + }, + } + + +def _classical_model_input_payload( + symbol: str, + *, + status: str = "ready", +) -> dict[str, JSONValue]: + return { + "schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": "insufficient_folds" if status == "limited" else None, + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "regularization": { + "regularized_observation_count": 100, + "observed_bin_count": 96, + "expected_closure_count": 3, + "unexpected_missing_count": 1, + }, + "fold_policy": { + "schema_version": "histdatacom.classical-model-fold.v1", + "fold_count": 8, + "valid_fold_count": 8, + }, + } + + +def _exponential_smoothing_payload( + symbol: str, + *, + status: str = "ready", +) -> dict[str, JSONValue]: + return { + "schema_version": EXPONENTIAL_SMOOTHING_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": "insufficient_data" if status == "limited" else None, + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "fit_summary": { + "fit_attempt_count": 4, + "failed_fit_count": 1 if status == "limited" else 0, + }, + "evaluation": { + "model_count": 1, + "fold_count": 4, + "evaluated_fold_count": 3 if status == "limited" else 4, + "automatic_winner": False, + }, + } + + +def _autoregressive_payload( + symbol: str, + *, + status: str = "ready", +) -> dict[str, JSONValue]: + return { + "schema_version": AUTOREGRESSIVE_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": "insufficient_folds" if status == "limited" else None, + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "fit_summary": { + "fit_attempt_count": 12, + "failed_fit_count": 1 if status == "limited" else 0, + "converged_fit_count": 11 if status == "limited" else 12, + }, + "evaluation": { + "model_count": 3, + "families": ["ar", "arma", "arima"], + "fold_count": 4, + "evaluated_fold_count": 3 if status == "limited" else 4, + "automatic_order_selection": False, + "automatic_winner": False, + }, + } + + +def _seasonal_exogenous_payload( + symbol: str, + *, + status: str = "ready", +) -> dict[str, JSONValue]: + return { + "schema_version": SEASONAL_EXOGENOUS_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": "insufficient_folds" if status == "limited" else None, + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "regressors": { + "column_order": ["source_hour_sin", "source_hour_cos"], + "known_in_advance": True, + "future_market_values_used": False, + }, + "fit_summary": { + "fit_attempt_count": 12, + "failed_fit_count": 1 if status == "limited" else 0, + "converged_fit_count": 11 if status == "limited" else 12, + }, + "evaluation": { + "model_count": 3, + "families": ["sarima", "arimax", "sarimax"], + "fold_count": 4, + "evaluated_fold_count": 3 if status == "limited" else 4, + "automatic_order_selection": False, + "automatic_winner": False, + }, + } + + +def _state_space_payload( + symbol: str, + *, + status: str = "ready", +) -> dict[str, JSONValue]: + return { + "schema_version": STATE_SPACE_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": "insufficient_folds" if status == "limited" else None, + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "missing_observation_policy": { + "regular_time_basis": True, + "fill_policy": "none", + "transition_policy": "prediction_only", + }, + "fit_summary": { + "fit_attempt_count": 12, + "failed_fit_count": 1 if status == "limited" else 0, + "converged_fit_count": 11 if status == "limited" else 12, + }, + "evaluation": { + "model_count": 3, + "families": ["local_level", "local_linear_trend", "structural"], + "fold_count": 4, + "evaluated_fold_count": 3 if status == "limited" else 4, + "filtered_state_policy": "forecast_origin_information_only", + "smoothed_state_policy": "retrospective_diagnostic_only", + "automatic_component_selection": False, + "automatic_winner": False, + }, + } + + +def _volatility_payload( + symbol: str, + *, + status: str = "ready", +) -> dict[str, JSONValue]: + return { + "schema_version": VOLATILITY_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": "insufficient_folds" if status == "limited" else None, + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "variance_contract": { + "conditional_variance": True, + "realized_proxy": "squared_return", + "mean_metrics_separate": True, + }, + "fit_summary": { + "fit_attempt_count": 8, + "failed_fit_count": 1 if status == "limited" else 0, + }, + "evaluation": { + "model_count": 2, + "families": ["arch", "garch"], + "fold_count": 4, + "evaluated_fold_count": 3 if status == "limited" else 4, + "automatic_order_selection": False, + "automatic_winner": False, + }, + } + + +def _classical_model_comparison_payload( + symbol: str, + *, + status: str = "ready", +) -> dict[str, JSONValue]: + eligible = 2 if status == "ready" else 0 + return { + "schema_version": CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": ( + "reference_baseline_unavailable" if status == "limited" else None + ), + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "comparison_id": f"classical-model-comparison:{symbol.lower()}", + "comparison_count": 2, + "eligible_comparison_count": eligible, + "ineligible_comparison_count": 2 - eligible, + "model_count": 2, + "horizons": [1, 2], + "comparison_records": [ + { + "schema_version": ( + CLASSICAL_MODEL_COMPARISON_ELIGIBILITY_SCHEMA_VERSION + ), + "comparison_id": "comparison:representative", + "family": "ar", + "model_id": "representative-ar", + "specification_id": "ar-1", + "target_metric": "mid_level", + "scale": "original_mid", + "horizon": 1, + "metric": "mae", + "metric_value": 0.0001, + "reference_baseline": "naive_random_walk", + "reference_metric_value": 0.0002, + "eligible": status == "ready", + "descriptive_only": True, + } + ], + "fit_accounting": { + "schema_version": CLASSICAL_MODEL_FIT_ACCOUNTING_SCHEMA_VERSION, + "totals": {"attempted": 8, "converged": 7, "failed": 1}, + "failed_models_preserved_in_denominator": True, + "resource_terminations_separate": True, + }, + "selection_policy": "none", + "descriptive_only": True, + "training_projection": { + "schema_version": ( + CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION + ), + "column_prefixes": [ + "cm_comparison_", + "cm_skill_", + "cm_stability_", + ], + "annotation_count": 1, + "training_eligible": False, + }, + } + + +def _cache_source_parity_payload(symbol: str) -> dict[str, JSONValue]: + return { + "schema_version": TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION, + "status": "match", + "advisory": True, + "target_axis": _axis(symbol=symbol, timeframe="T", kind="csv"), + "base_grain": {"data_format": "ascii", "timeframe": "T"}, + "compared_section_count": 9, + "matching_section_count": 9, + "mismatched_section_count": 0, + "skipped_section_count": 0, + "mismatch_code_count": 0, + "included_mismatch_code_count": 0, + "omitted_mismatch_code_count": 0, + "truncated": False, + "limit_metadata": { + "mismatches": { + "requested_limit": 16, + "default_limit": 16, + "effective_limit": 16, + "unbounded": False, + } + }, + "mismatch_codes": [], + "skipped_reasons": [], + "bases": { + "raw_source": { + "status": "available", + "kind": "csv_text", + "path": f"DAT_ASCII_{symbol}_T_201202.csv", + "member": None, + "row_count": 3, + }, + "raw_cache": { + "status": "available", + "path": ".data", + "cache_source": "sibling", + "fresh": True, + "freshness": "fresh", + "row_count": 3, + }, + "enriched_cache": { + "status": "available", + "training_schema_version": ( + "histdatacom.ascii-tick-training-features.v1" + ), + "cache_was_enriched": True, + "legacy_cache_enriched_on_read": False, + }, + "quality_report": { + "status": "available", + "projection_kind": "audit_from_enriched_rows", + }, + "influx_projection": { + "status": "available", + "projection_kind": "same_point_enriched_fields", + "missing_required_field_count": 0, + }, + }, + "comparisons": [ + {"section": section, "status": "match"} + for section in ( + "coverage", + "temporal_topology", + "calendar_regimes", + "conditional_distributions", + "training_columns", + "row_identity", + "duplicate_timestamps", + "quality_report_projection", + "influx_projection", + ) + ], + } def _axis(*, symbol: str, timeframe: str, kind: str) -> dict[str, JSONValue]: @@ -999,7 +1704,7 @@ def _topology_payload( non_monotonic_count: int = 0, suspicious_gap_count: int = 0, ) -> dict[str, JSONValue]: - return { + payload: dict[str, JSONValue] = { "row_count": 4, "parsed_row_count": 4 - invalid_timestamp_count, "invalid_timestamp_count": invalid_timestamp_count, @@ -1014,6 +1719,38 @@ def _topology_payload( "computed_from": computed_from, "cache_source": None, } + if duplicate_timestamp_count: + payload["inspection_context"] = { + "schema_version": "histdatacom.timestamp-topology-inspection.v1", + "duplicate_timestamps": { + "total_count": 1, + "included_count": 1, + "omitted_count": 0, + "truncated": False, + "limit_metadata": { + "samples": bounded_report_limit( + 1, + default_limit=5, + minimum_limit=0, + maximum_limit=5, + allow_unbounded=False, + ).count_payload(1) + }, + "duplicate_row_count": duplicate_timestamp_count, + "samples": [ + { + "row_number": 2, + "timestamp_source": "20120201 000000000", + "timestamp_source_truncated": False, + "timestamp_utc_ms": 1328072400000, + "utc_timestamp": "2012-02-01T05:00:00Z", + "occurrence_count": duplicate_timestamp_count + 1, + "exact_row_group_count": 1, + } + ], + }, + } + return payload def _calendar_regimes_payload() -> dict[str, JSONValue]: diff --git a/src/histdatacom/data_quality/calendar.py b/src/histdatacom/data_quality/calendar.py index 916fc474..682b1576 100644 --- a/src/histdatacom/data_quality/calendar.py +++ b/src/histdatacom/data_quality/calendar.py @@ -444,6 +444,10 @@ def calendar_policy_metadata( "holiday_calendar_complete": profile.complete, "holiday_calendar_static_advisory": profile.static_advisory, "holiday_calendar_limitations": "; ".join(profile.limitations), + "weekend_activity_policy": profile.weekend_activity_policy, + "expected_session_closure_policy": ( + profile.expected_session_closure_policy + ), "static_major_holidays": [ holiday.to_metadata() for holiday in STATIC_MAJOR_HOLIDAYS ], @@ -458,6 +462,7 @@ def calendar_regime_payload_for_target( target: QualityTarget, *, calendar_profile: HistDataCalendarProfile | None = None, + prefer_cache: bool = True, ) -> dict[str, JSONValue]: """Return deterministic calendar/session fingerprint metadata.""" profile = calendar_profile or default_calendar_profile() @@ -474,6 +479,7 @@ def calendar_regime_payload_for_target( target, calendar_profile=profile, asset_class=_target_asset_class(target), + prefer_cache=prefer_cache, ) ) except ( @@ -520,8 +526,12 @@ def _calendar_scan_for_target( *, calendar_profile: HistDataCalendarProfile, asset_class: str, + prefer_cache: bool = True, ) -> tuple[_CalendarScan, str, str, str | None]: - timestamp_scan = _timestamp_scan_for_target(target) + timestamp_scan = _timestamp_scan_for_target( + target, + prefer_cache=prefer_cache, + ) if _can_use_timestamp_scan_for_calendar(timestamp_scan): source_member = ( timestamp_scan.valid_rows[0].source_member diff --git a/src/histdatacom/data_quality/calendar_profiles.py b/src/histdatacom/data_quality/calendar_profiles.py index 8cb2ab62..d24cc034 100644 --- a/src/histdatacom/data_quality/calendar_profiles.py +++ b/src/histdatacom/data_quality/calendar_profiles.py @@ -12,6 +12,10 @@ CALENDAR_PROFILE_SCHEMA_VERSION = "histdatacom.calendar-profile.v1" DEFAULT_CALENDAR_PROFILE_NAME = "static-major-holidays" DEFAULT_CALENDAR_PROFILE_SOURCE = "static_month_day_major_holidays" +DEFAULT_WEEKEND_ACTIVITY_POLICY = "advisory" +DEFAULT_EXPECTED_SESSION_CLOSURE_POLICY = "expected" +WEEKEND_ACTIVITY_POLICIES = frozenset({"strict", "advisory", "allowed"}) +EXPECTED_SESSION_CLOSURE_POLICIES = frozenset({"expected", "unexpected"}) _UNIX_EPOCH_DATE = date(1970, 1, 1) @@ -258,6 +262,10 @@ class HistDataCalendarProfile: schema_version: str = CALENDAR_PROFILE_SCHEMA_VERSION complete: bool = False static_advisory: bool = True + weekend_activity_policy: str = DEFAULT_WEEKEND_ACTIVITY_POLICY + expected_session_closure_policy: str = ( + DEFAULT_EXPECTED_SESSION_CLOSURE_POLICY + ) limitations: tuple[str, ...] = () date_tags: tuple[HistDataCalendarDateTag, ...] = () window_tags: tuple[HistDataCalendarWindowTag, ...] = () @@ -272,6 +280,16 @@ def __post_init__(self) -> None: raise ValueError(msg) _validate_name(self.name, path="calendar_profile.name") _validate_name(self.source, path="calendar_profile.source") + _validate_policy( + self.weekend_activity_policy, + allowed=WEEKEND_ACTIVITY_POLICIES, + path="calendar_profile.weekend_activity_policy", + ) + _validate_policy( + self.expected_session_closure_policy, + allowed=EXPECTED_SESSION_CLOSURE_POLICIES, + path="calendar_profile.expected_session_closure_policy", + ) def holiday_tags_for( self, @@ -372,6 +390,10 @@ def to_metadata(self) -> dict[str, JSONValue]: "version": self.version, "complete": self.complete, "static_advisory": self.static_advisory, + "weekend_activity_policy": self.weekend_activity_policy, + "expected_session_closure_policy": ( + self.expected_session_closure_policy + ), "limitations": list(self.limitations), "date_tags": [tag.to_metadata() for tag in self.date_tags], "window_tags": [tag.to_metadata() for tag in self.window_tags], @@ -419,6 +441,8 @@ def calendar_profile_from_mapping( "version", "complete", "static_advisory", + "weekend_activity_policy", + "expected_session_closure_policy", "limitations", "date_tags", "fixed_holidays", @@ -456,6 +480,18 @@ def calendar_profile_from_mapping( static_advisory=_bool_value( payload.get("static_advisory"), default=False ), + weekend_activity_policy=_policy_value( + payload.get("weekend_activity_policy"), + default=DEFAULT_WEEKEND_ACTIVITY_POLICY, + allowed=WEEKEND_ACTIVITY_POLICIES, + path="calendar_profile.weekend_activity_policy", + ), + expected_session_closure_policy=_policy_value( + payload.get("expected_session_closure_policy"), + default=DEFAULT_EXPECTED_SESSION_CLOSURE_POLICY, + allowed=EXPECTED_SESSION_CLOSURE_POLICIES, + path="calendar_profile.expected_session_closure_policy", + ), limitations=_string_tuple(payload.get("limitations")), date_tags=tuple(date_tags), window_tags=tuple(window_tags), @@ -673,6 +709,29 @@ def _bool_value(value: Any, *, default: bool) -> bool: raise ValueError(f"calendar profile boolean value invalid: {value!r}") +def _policy_value( + value: Any, + *, + default: str, + allowed: frozenset[str], + path: str, +) -> str: + normalized = str(value or default).strip().lower().replace("-", "_") + _validate_policy(normalized, allowed=allowed, path=path) + return normalized + + +def _validate_policy( + value: str, + *, + allowed: frozenset[str], + path: str, +) -> None: + if value not in allowed: + choices = ", ".join(sorted(allowed)) + raise ValueError(f"{path} must be one of: {choices}") + + def _int_value(value: Any, *, default: int) -> int: if value is None or value == "": return default diff --git a/src/histdatacom/data_quality/classical_baselines.py b/src/histdatacom/data_quality/classical_baselines.py new file mode 100644 index 00000000..05d6edce --- /dev/null +++ b/src/histdatacom/data_quality/classical_baselines.py @@ -0,0 +1,949 @@ +"""Optional deterministic classical baselines over enriched ASCII tick rows.""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from statistics import median +from typing import Any, TypedDict, cast + +from histdatacom.data_quality.contracts import QualityFinding +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.training_features import ( + TRAINING_SCHEMA_VERSION, + ensure_tick_training_features, +) +from histdatacom.runtime_contracts import JSONValue + +CLASSICAL_BASELINE_SCHEMA_VERSION = "histdatacom.classical-baselines.v1" +CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.classical-baseline-training-projection.v1" +) +CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.classical-baseline-summary.v1" +) +CLASSICAL_BASELINE_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_classical_baseline_summary" +) +CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY = "fingerprint_classical_baselines" + +DEFAULT_BASELINE_EVALUATION_FRACTION = 0.2 +DEFAULT_BASELINE_MINIMUM_TRAINING_ROWS = 20 +DEFAULT_BASELINE_MINIMUM_EVALUATION_ROWS = 5 +DEFAULT_BASELINE_ROLLING_WINDOWS = (5, 20) +DEFAULT_BASELINE_ROUNDING_DIGITS = 12 +DEFAULT_BASELINE_SUMMARY_TARGET_LIMIT = 16 +MAX_BASELINE_ROLLING_WINDOWS = 16 + +BASELINE_REQUIRED_IDENTITY_COLUMNS = ("series_id", "period", "row_id") +BASELINE_REQUIRED_INPUT_COLUMNS = ( + *BASELINE_REQUIRED_IDENTITY_COLUMNS, + "mid", + "training_usable", +) +BASELINE_DEFERRED_MODEL_FAMILIES = ( + "ets", + "arima", + "sarima", + "state_space", + "garch", +) +BASELINE_MODEL_CODES = { + "naive_random_walk": 1, + "rolling_mean": 2, + "rolling_median": 3, + "session_seasonal_naive": 4, +} +BASELINE_STATUS_CODES = {"unavailable": 1, "limited": 2, "ready": 3} +STATIONARITY_STATUS_CODES = { + "unavailable": 1, + "limited": 2, + "valid": 3, +} +BASELINE_REASON_CODES = { + "": 0, + "training_frame_unavailable": 1, + "missing_required_columns": 2, + "insufficient_training_rows": 3, + "insufficient_evaluation_rows": 4, + "no_usable_mid_values": 5, + "stationarity_unavailable": 6, + "stationarity_limited": 7, + "model_evaluation_unavailable": 8, +} +TRANSFORM_ADVISORY_BITS = { + "log_return": 1, + "differencing": 2, + "session_conditioning": 4, +} + + +class _BaselineRow(TypedDict): + series_id: str + period: str + row_id: int + mid: float + session_state: int + + +@dataclass(frozen=True, slots=True) +class ClassicalBaselineProfile: + """Opt-in controls for deterministic, advisory baseline evaluation.""" + + enabled: bool = False + evaluation_fraction: float = DEFAULT_BASELINE_EVALUATION_FRACTION + minimum_training_rows: int = DEFAULT_BASELINE_MINIMUM_TRAINING_ROWS + minimum_evaluation_rows: int = DEFAULT_BASELINE_MINIMUM_EVALUATION_ROWS + rolling_windows: tuple[int, ...] = DEFAULT_BASELINE_ROLLING_WINDOWS + session_seasonal_enabled: bool = True + rounding_digits: int = DEFAULT_BASELINE_ROUNDING_DIGITS + + def to_metadata(self) -> dict[str, JSONValue]: + """Return stable JSON-compatible profile metadata.""" + return { + "enabled": self.enabled, + "evaluation_fraction": self.evaluation_fraction, + "minimum_training_rows": self.minimum_training_rows, + "minimum_evaluation_rows": self.minimum_evaluation_rows, + "rolling_windows": list(self.rolling_windows), + "session_seasonal_enabled": self.session_seasonal_enabled, + "rounding_digits": self.rounding_digits, + } + + +def classical_baseline_diagnostics_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + profile: ClassicalBaselineProfile | None = None, + target: Any | None = None, +) -> dict[str, JSONValue]: + """Evaluate deterministic classical baselines over enriched tick rows. + + Metrics use a chronological holdout in durable ``row_id`` order. Actual + prior values are available to later walk-forward predictions; no shuffle, + fitted statistical model, or future value participates in a forecast. + """ + selected = profile or ClassicalBaselineProfile(enabled=True) + base = _base_payload(fingerprint, selected) + if frame is None: + return _unavailable_payload(base, "training_frame_unavailable") + + input_was_enriched = "training_schema_version" in getattr( + frame, "columns", () + ) + try: + enriched = ensure_tick_training_features(frame, target=target) + except (AttributeError, TypeError, ValueError): + return _unavailable_payload(base, "training_frame_unavailable") + + columns = set(getattr(enriched, "columns", ())) + training_substrate = dict(_mapping(base.get("training_substrate"))) + training_substrate["legacy_cache_enriched_on_read"] = not input_was_enriched + base["training_substrate"] = training_substrate + missing = sorted(set(BASELINE_REQUIRED_INPUT_COLUMNS) - columns) + if missing: + missing_payload = _unavailable_payload(base, "missing_required_columns") + missing_payload["training_substrate"] = { + **cast(dict[str, JSONValue], missing_payload["training_substrate"]), + "missing_required_columns": cast(JSONValue, missing), + } + return missing_payload + + rows = _usable_rows(enriched) + if not rows: + return _unavailable_payload(base, "no_usable_mid_values") + + split = _chronological_split(len(rows), selected) + base["split_policy"] = split + training_count = _int(split.get("training_row_count")) + evaluation_count = _int(split.get("evaluation_row_count")) + split["split_row_id"] = ( + rows[training_count]["row_id"] if evaluation_count else None + ) + if training_count < selected.minimum_training_rows: + return _unavailable_payload( + base, + "insufficient_training_rows", + split_policy=split, + ) + if evaluation_count < selected.minimum_evaluation_rows: + return _unavailable_payload( + base, + "insufficient_evaluation_rows", + split_policy=split, + ) + + values = [row["mid"] for row in rows] + session_states = [row["session_state"] for row in rows] + models = _evaluate_models( + values, + session_states, + split_index=training_count, + profile=selected, + fingerprint=fingerprint, + ) + evaluated = [ + model for model in models if model.get("status") == "evaluated" + ] + if not evaluated: + return _unavailable_payload( + base, + "model_evaluation_unavailable", + split_policy=split, + models=models, + ) + + best = min( + evaluated, + key=lambda model: ( + _float(model.get("mae"), default=math.inf), + _int(model.get("model_code")), + _int(model.get("window")), + ), + ) + prerequisite = _prerequisite_readiness(fingerprint) + guard_codes = _evaluation_guard_codes(fingerprint, prerequisite) + limitations = list(guard_codes) + if any( + model.get("status") != "evaluated" + and model.get("model") != "session_seasonal_naive" + for model in models + ): + limitations.append("some_models_skipped") + status = "ready" if not limitations else "limited" + reason = limitations[0] if limitations else None + payload: dict[str, JSONValue] = { + **base, + "status": status, + "reason": reason, + "prerequisite_readiness": prerequisite, + "split_policy": split, + "evaluation": { + "status": "evaluated", + "metric": "mid", + "calculation_basis": "observed_sequence_walk_forward", + "transforms_applied": [], + "recommended_transforms": _recommended_transforms(fingerprint), + "guard_codes": guard_codes, + "model_count": len(models), + "evaluated_model_count": len(evaluated), + "skipped_model_count": len(models) - len(evaluated), + "models": cast(JSONValue, models), + "best_model": dict(best), + }, + "limitations": limitations, + } + payload["training_projection"] = classical_baseline_training_projection( + payload + ) + payload["baseline_id"] = _baseline_id(payload) + return payload + + +def classical_baseline_training_projection( + diagnostics: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + """Return flat period-grain baseline scalars for enriched rows.""" + status = _text(diagnostics.get("status")) + reason = _text(diagnostics.get("reason")) + split = _mapping(diagnostics.get("split_policy")) + prerequisite = _mapping(diagnostics.get("prerequisite_readiness")) + evaluation = _mapping(diagnostics.get("evaluation")) + best = _mapping(evaluation.get("best_model")) + transforms = _strings(evaluation.get("recommended_transforms")) + return { + "schema_version": CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "period", + "identity_fields": list(BASELINE_REQUIRED_IDENTITY_COLUMNS), + "timestamp_required": False, + "values": { + "baseline_status_code": BASELINE_STATUS_CODES.get(status, 0), + "baseline_training_ready": status in {"limited", "ready"}, + "baseline_exclusion_reason_code": BASELINE_REASON_CODES.get( + reason, 99 if reason else 0 + ), + "baseline_training_row_count": _int( + split.get("training_row_count") + ), + "baseline_evaluation_row_count": _int( + split.get("evaluation_row_count") + ), + "baseline_split_row_id": split.get("split_row_id"), + "baseline_best_model_code": _int(best.get("model_code")), + "baseline_best_model_window": best.get("window"), + "baseline_best_mae": best.get("mae"), + "baseline_best_rmse": best.get("rmse"), + "baseline_stationarity_status_code": STATIONARITY_STATUS_CODES.get( + _text(prerequisite.get("stationarity_status")), 0 + ), + "baseline_transform_advisory_code": _transform_advisory_code( + transforms + ), + }, + } + + +def project_classical_baseline_onto_training_frame( + frame: Any, + diagnostics: Mapping[str, JSONValue], +) -> Any: + """Project period baseline scalars without using or joining on timestamp.""" + missing = sorted( + set(BASELINE_REQUIRED_IDENTITY_COLUMNS) + - set(getattr(frame, "columns", ())) + ) + if missing: + raise ValueError( + "classical baseline projection requires enriched ASCII tick " + f"identity columns: {', '.join(missing)}" + ) + import polars as pl + + projection = classical_baseline_training_projection(diagnostics) + values = _mapping(projection.get("values")) + integer_columns = { + "baseline_status_code", + "baseline_exclusion_reason_code", + "baseline_training_row_count", + "baseline_evaluation_row_count", + "baseline_split_row_id", + "baseline_best_model_code", + "baseline_best_model_window", + "baseline_stationarity_status_code", + "baseline_transform_advisory_code", + } + boolean_columns = {"baseline_training_ready"} + expressions = [] + for name, value in values.items(): + expression = pl.lit(value) + if name in integer_columns: + expression = expression.cast(pl.Int64) + elif name in boolean_columns: + expression = expression.cast(pl.Boolean) + else: + expression = expression.cast(pl.Float64) + expressions.append(expression.alias(name)) + return frame.with_columns(expressions) + + +def classical_baseline_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = DEFAULT_BASELINE_SUMMARY_TARGET_LIMIT, +) -> dict[str, JSONValue] | None: + """Return a bounded run summary for opt-in baseline diagnostics.""" + targets: list[dict[str, JSONValue]] = [] + status_counts: Counter[str] = Counter() + best_model_counts: Counter[str] = Counter() + for finding in findings: + fingerprint = _mapping(finding.metadata.get("time_series_fingerprint")) + diagnostics = _mapping(fingerprint.get("classical_baselines")) + if not diagnostics: + continue + status = _text(diagnostics.get("status")) or "unavailable" + evaluation = _mapping(diagnostics.get("evaluation")) + best = _mapping(evaluation.get("best_model")) + model = _text(best.get("model")) + status_counts[status] += 1 + if model: + best_model_counts[model] += 1 + split = _mapping(diagnostics.get("split_policy")) + targets.append( + { + "target_axis": dict(_mapping(diagnostics.get("target_axis"))), + "status": status, + "reason": diagnostics.get("reason"), + "training_row_count": _int(split.get("training_row_count")), + "evaluation_row_count": _int(split.get("evaluation_row_count")), + "best_model": model or None, + "best_model_code": _int(best.get("model_code")), + "best_mae": best.get("mae"), + "guard_codes": cast( + JSONValue, _strings(evaluation.get("guard_codes")) + ), + } + ) + if not targets: + return None + targets.sort(key=_target_sort_key) + limit = bounded_report_limit( + target_limit, + default_limit=DEFAULT_BASELINE_SUMMARY_TARGET_LIMIT, + allow_unbounded=True, + ) + included = limit.slice(targets) + omitted = max(0, len(targets) - len(included)) + return { + "schema_version": CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION, + "advisory": True, + "target_count": len(targets), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "status_counts": dict(sorted(status_counts.items())), + "best_model_counts": dict(sorted(best_model_counts.items())), + "target_summaries": cast(JSONValue, included), + "limit_metadata": {"targets": limit.limit_payload()}, + } + + +def format_classical_baseline_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> tuple[str, ...]: + """Return concise console lines for baseline diagnostics.""" + if not summary: + return () + status = _mapping(summary.get("status_counts")) + lines = [ + "", + "Classical fingerprint baselines", + ( + "targets: " + f"{_int(summary.get('target_count'))} " + f"ready: {_int(status.get('ready'))} " + f"limited: {_int(status.get('limited'))} " + f"unavailable: {_int(status.get('unavailable'))}" + ), + ] + for target in _mapping_rows(summary.get("target_summaries")): + axis = _mapping(target.get("target_axis")) + label = "/".join( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + best = _text(target.get("best_model")) or "none" + lines.append( + f"- {label}: {target.get('status', 'unavailable')} " + f"train={_int(target.get('training_row_count'))} " + f"eval={_int(target.get('evaluation_row_count'))} " + f"best={best}" + ) + if summary.get("truncated") is True: + lines.append( + f"- {_int(summary.get('omitted_target_count'))} targets omitted" + ) + return tuple(lines) + + +def _base_payload( + fingerprint: Mapping[str, JSONValue], + profile: ClassicalBaselineProfile, +) -> dict[str, JSONValue]: + return { + "schema_version": CLASSICAL_BASELINE_SCHEMA_VERSION, + "advisory": True, + "base_grain": {"data_format": "ascii", "timeframe": "T"}, + "target_axis": dict(_mapping(fingerprint.get("target_axis"))), + "reference_fingerprint_id": _reference_fingerprint_id(fingerprint), + "configuration": profile.to_metadata(), + "training_substrate": { + "schema_version": TRAINING_SCHEMA_VERSION, + "status": "available", + "required_columns": list(BASELINE_REQUIRED_INPUT_COLUMNS), + "identity_fields": list(BASELINE_REQUIRED_IDENTITY_COLUMNS), + "ordering_fields": list(BASELINE_REQUIRED_IDENTITY_COLUMNS), + "timestamp_required": False, + "metric": "mid", + "observed_bid_ask_preserved": True, + "legacy_cache_enriched_on_read": False, + }, + "split_policy": _empty_split_policy(profile), + "prerequisite_readiness": _prerequisite_readiness(fingerprint), + "evaluation": _empty_evaluation(fingerprint), + "limitations": [], + "deferred_model_families": list(BASELINE_DEFERRED_MODEL_FAMILIES), + "non_goals": [ + "forecasting_leaderboard", + "automatic_model_selection", + "synthetic_generation", + "hard_fail_quality_gate", + ], + } + + +def _unavailable_payload( + base: Mapping[str, JSONValue], + reason: str, + *, + split_policy: Mapping[str, JSONValue] | None = None, + models: Sequence[Mapping[str, JSONValue]] = (), +) -> dict[str, JSONValue]: + payload = dict(base) + payload.update( + { + "status": "unavailable", + "reason": reason, + "limitations": [reason], + } + ) + if split_policy is not None: + payload["split_policy"] = dict(split_policy) + evaluation = dict(_mapping(payload.get("evaluation"))) + if models: + evaluation.update( + { + "models": [dict(model) for model in models], + "model_count": len(models), + "evaluated_model_count": 0, + "skipped_model_count": len(models), + } + ) + payload["evaluation"] = evaluation + training = dict(_mapping(payload.get("training_substrate"))) + if reason in {"training_frame_unavailable", "missing_required_columns"}: + training["status"] = "unavailable" + payload["training_substrate"] = training + payload["training_projection"] = classical_baseline_training_projection( + payload + ) + payload["baseline_id"] = _baseline_id(payload) + return payload + + +def _usable_rows(frame: Any) -> list[_BaselineRow]: + selected = [ + *BASELINE_REQUIRED_IDENTITY_COLUMNS, + "mid", + "training_usable", + ] + has_session = "class_session_state_code" in frame.columns + if has_session: + selected.append("class_session_state_code") + rows: list[_BaselineRow] = [] + ordered = frame.select(selected).sort( + list(BASELINE_REQUIRED_IDENTITY_COLUMNS) + ) + for row in ordered.iter_rows(named=True): + value = row.get("mid") + if row.get("training_usable") is not True: + continue + if isinstance(value, bool) or not isinstance(value, (int, float)): + continue + mid = float(value) + if not math.isfinite(mid): + continue + rows.append( + { + "series_id": str(row.get("series_id") or ""), + "period": str(row.get("period") or ""), + "row_id": int(row.get("row_id") or 0), + "mid": mid, + "session_state": int(row.get("class_session_state_code") or 0), + } + ) + return rows + + +def _chronological_split( + row_count: int, + profile: ClassicalBaselineProfile, +) -> dict[str, JSONValue]: + requested_evaluation = max( + profile.minimum_evaluation_rows, + int(math.ceil(row_count * profile.evaluation_fraction)), + ) + evaluation_count = min(row_count, requested_evaluation) + training_count = max(0, row_count - evaluation_count) + return { + "kind": "chronological_holdout", + "order_by": list(BASELINE_REQUIRED_IDENTITY_COLUMNS), + "timestamp_required": False, + "shuffle": False, + "walk_forward": True, + "future_values_visible": False, + "evaluation_fraction": profile.evaluation_fraction, + "row_count": row_count, + "training_row_count": training_count, + "evaluation_row_count": evaluation_count, + "split_index": training_count, + "split_row_id": training_count + 1 if evaluation_count else None, + "metrics_emitted": ( + training_count >= profile.minimum_training_rows + and evaluation_count >= profile.minimum_evaluation_rows + ), + } + + +def _empty_split_policy( + profile: ClassicalBaselineProfile, +) -> dict[str, JSONValue]: + return { + "kind": "chronological_holdout", + "order_by": list(BASELINE_REQUIRED_IDENTITY_COLUMNS), + "timestamp_required": False, + "shuffle": False, + "walk_forward": True, + "future_values_visible": False, + "evaluation_fraction": profile.evaluation_fraction, + "row_count": 0, + "training_row_count": 0, + "evaluation_row_count": 0, + "split_index": 0, + "split_row_id": None, + "metrics_emitted": False, + } + + +def _evaluate_models( + values: Sequence[float], + session_states: Sequence[int], + *, + split_index: int, + profile: ClassicalBaselineProfile, + fingerprint: Mapping[str, JSONValue], +) -> list[dict[str, JSONValue]]: + models = [ + _evaluate_prediction_series( + "naive_random_walk", + [values[index - 1] for index in range(split_index, len(values))], + values[split_index:], + profile=profile, + ) + ] + for window in profile.rolling_windows[:MAX_BASELINE_ROLLING_WINDOWS]: + mean_predictions = [ + ( + _mean(values[max(0, index - window) : index]) + if index >= window + else None + ) + for index in range(split_index, len(values)) + ] + median_predictions = [ + median(values[index - window : index]) if index >= window else None + for index in range(split_index, len(values)) + ] + models.extend( + ( + _evaluate_prediction_series( + "rolling_mean", + mean_predictions, + values[split_index:], + profile=profile, + window=window, + ), + _evaluate_prediction_series( + "rolling_median", + median_predictions, + values[split_index:], + profile=profile, + window=window, + ), + ) + ) + if profile.session_seasonal_enabled: + if _session_seasonal_eligible(session_states, fingerprint): + predictions = _session_seasonal_predictions( + values, + session_states, + split_index=split_index, + ) + models.append( + _evaluate_prediction_series( + "session_seasonal_naive", + predictions, + values[split_index:], + profile=profile, + ) + ) + else: + models.append( + { + "model": "session_seasonal_naive", + "model_code": BASELINE_MODEL_CODES[ + "session_seasonal_naive" + ], + "status": "skipped", + "reason": "session_topology_unavailable", + "forecast_count": 0, + "evaluation_row_count": len(values) - split_index, + "coverage_rate": 0.0, + "mae": None, + "rmse": None, + "mean_error": None, + } + ) + return models + + +def _evaluate_prediction_series( + model: str, + predictions: Sequence[float | None], + actuals: Sequence[float], + *, + profile: ClassicalBaselineProfile, + window: int | None = None, +) -> dict[str, JSONValue]: + pairs = [ + (float(prediction), float(actual)) + for prediction, actual in zip(predictions, actuals, strict=True) + if prediction is not None and math.isfinite(float(prediction)) + ] + errors = [prediction - actual for prediction, actual in pairs] + result: dict[str, JSONValue] = { + "model": model, + "model_code": BASELINE_MODEL_CODES[model], + "status": "evaluated" if errors else "skipped", + "reason": None if errors else "insufficient_model_history", + "forecast_count": len(errors), + "evaluation_row_count": len(actuals), + "coverage_rate": _rounded( + len(errors) / len(actuals) if actuals else 0.0, + profile.rounding_digits, + ), + "mae": ( + _rounded( + _mean([abs(error) for error in errors]), + profile.rounding_digits, + ) + if errors + else None + ), + "rmse": ( + _rounded( + math.sqrt(_mean([error * error for error in errors])), + profile.rounding_digits, + ) + if errors + else None + ), + "mean_error": ( + _rounded(_mean(errors), profile.rounding_digits) if errors else None + ), + } + if window is not None: + result["window"] = window + return result + + +def _session_seasonal_eligible( + session_states: Sequence[int], + fingerprint: Mapping[str, JSONValue], +) -> bool: + calendar = _mapping(fingerprint.get("calendar_regimes")) + calendar_status = _text(calendar.get("status")) + return calendar_status in {"ok", "limited"} and len(set(session_states)) > 1 + + +def _session_seasonal_predictions( + values: Sequence[float], + session_states: Sequence[int], + *, + split_index: int, +) -> list[float | None]: + last_by_session: dict[int, float] = {} + predictions: list[float | None] = [] + for index, (value, session) in enumerate( + zip(values, session_states, strict=True) + ): + if index >= split_index: + predictions.append(last_by_session.get(session)) + last_by_session[session] = value + return predictions + + +def _prerequisite_readiness( + fingerprint: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + audit = _mapping(fingerprint.get("fingerprint_audit")) + statuses = _mapping(audit.get("section_statuses")) + required = ( + "coverage", + "temporal_topology", + "calendar_regimes", + "tick_distribution", + "microstructure_dynamics", + "dependence", + "stationarity_diagnostics", + "decomposition", + ) + missing = [section for section in required if section not in statuses] + limited = [ + section + for section in required + if _text(statuses.get(section)) in {"limited", "skipped"} + ] + unavailable = [ + section + for section in required + if _text(statuses.get(section)) == "unavailable" + ] + stationarity = _mapping(fingerprint.get("stationarity_diagnostics")) + raw_status = _text(stationarity.get("stationarity_status")) + stationarity_status = { + "ok": "valid", + "limited": "limited", + "unavailable": "unavailable", + }.get(raw_status, "unavailable") + if missing or unavailable or stationarity_status == "unavailable": + status = "unavailable" + elif limited or stationarity_status == "limited": + status = "limited" + else: + status = "valid" + return { + "status": status, + "required_sections": cast(JSONValue, list(required)), + "missing_sections": cast(JSONValue, missing), + "limited_sections": cast(JSONValue, limited), + "unavailable_sections": cast(JSONValue, unavailable), + "stationarity_status": stationarity_status, + "stationarity_reason": stationarity.get("reason"), + "rolling_drift_status": ( + "available" + if _mapping(stationarity.get("rolling_windows")) + else "unavailable" + ), + "distribution_shift_status": _text( + _mapping( + stationarity.get("first_middle_last_distribution_shift") + ).get("status") + ) + or "unavailable", + "computed_window_count": _int( + stationarity.get("computed_window_count") + ), + "skipped_window_count": _int(stationarity.get("skipped_window_count")), + "zero_variance_metrics": cast( + JSONValue, + _strings(stationarity.get("zero_variance_metrics")), + ), + "recommended_transforms": _recommended_transforms(fingerprint), + } + + +def _evaluation_guard_codes( + fingerprint: Mapping[str, JSONValue], + prerequisite: Mapping[str, JSONValue], +) -> list[JSONValue]: + guards: list[str] = [] + status = _text(prerequisite.get("stationarity_status")) + if status == "unavailable": + guards.append("stationarity_unavailable") + elif status == "limited": + guards.append("stationarity_limited") + if _int(prerequisite.get("skipped_window_count")) > 0: + guards.append("skipped_rolling_windows") + if _strings(prerequisite.get("zero_variance_metrics")): + guards.append("zero_variance") + distribution = _mapping( + _mapping(fingerprint.get("stationarity_diagnostics")).get( + "first_middle_last_distribution_shift" + ) + ) + if _text(distribution.get("status")) != "computed": + guards.append("distribution_shift_unavailable") + for transform in _recommended_transforms(fingerprint): + guards.append(f"transform_recommended:{transform}") + return list(dict.fromkeys(guards)) + + +def _recommended_transforms( + fingerprint: Mapping[str, JSONValue], +) -> list[JSONValue]: + stationarity = _mapping(fingerprint.get("stationarity_diagnostics")) + return cast( + list[JSONValue], + _strings(stationarity.get("recommended_transforms")), + ) + + +def _empty_evaluation( + fingerprint: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + return { + "status": "not_evaluated", + "metric": "mid", + "calculation_basis": "observed_sequence_walk_forward", + "transforms_applied": [], + "recommended_transforms": _recommended_transforms(fingerprint), + "guard_codes": [], + "model_count": 0, + "evaluated_model_count": 0, + "skipped_model_count": 0, + "models": [], + "best_model": {}, + } + + +def _baseline_id(payload: Mapping[str, JSONValue]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def _reference_fingerprint_id( + fingerprint: Mapping[str, JSONValue], +) -> str: + existing = _text(fingerprint.get("fingerprint_id")) + if existing: + return existing + basis = { + key: value + for key, value in fingerprint.items() + if key not in {"classical_baselines", "fingerprint_id"} + } + encoded = json.dumps(basis, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def _transform_advisory_code(transforms: Sequence[str]) -> int: + return sum(TRANSFORM_ADVISORY_BITS.get(value, 0) for value in transforms) + + +def _target_sort_key(item: Mapping[str, JSONValue]) -> tuple[str, ...]: + axis = _mapping(item.get("target_axis")) + return tuple( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period", "kind") + ) + + +def _mapping(value: Any) -> Mapping[str, JSONValue]: + return value if isinstance(value, Mapping) else {} + + +def _mapping_rows(value: Any) -> list[Mapping[str, JSONValue]]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, Mapping)] + + +def _strings(value: Any) -> list[str]: + if not isinstance(value, (list, tuple)): + return [] + return [str(item) for item in value if item not in (None, "")] + + +def _text(value: Any) -> str: + return str(value or "") + + +def _int(value: Any) -> int: + if isinstance(value, bool) or value is None: + return 0 + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _float(value: Any, *, default: float = 0.0) -> float: + if isinstance(value, bool) or value is None: + return default + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _mean(values: Sequence[float]) -> float: + return sum(values) / len(values) if values else 0.0 + + +def _rounded(value: float, digits: int) -> float: + return round(float(value), digits) diff --git a/src/histdatacom/data_quality/classical_model_comparison.py b/src/histdatacom/data_quality/classical_model_comparison.py new file mode 100644 index 00000000..0d471e37 --- /dev/null +++ b/src/histdatacom/data_quality/classical_model_comparison.py @@ -0,0 +1,1950 @@ +"""Family-neutral comparison of saved classical-model evaluation artifacts. + +The comparison layer never fits a model. It normalizes the bounded evaluation +artifacts emitted by the classical baseline, exponential-smoothing, +autoregressive, seasonal/exogenous, state-space, and volatility families. A +comparison is eligible only when its target, scale, regularization, transform, +missingness, fold, period, and horizon identities agree with an explicitly +configured reference baseline. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from statistics import median +from typing import Any, cast + +from histdatacom.data_quality.contracts import QualityFinding +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.training_features import ( + CLASSICAL_MODEL_COMPARISON_COLUMNS, + ensure_tick_training_features, +) +from histdatacom.runtime_contracts import JSONValue + +CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION = ( + "histdatacom.classical-model-comparison.v1" +) +CLASSICAL_MODEL_COMPARISON_ELIGIBILITY_SCHEMA_VERSION = ( + "histdatacom.classical-model-comparison-eligibility.v1" +) +CLASSICAL_MODEL_SKILL_SCHEMA_VERSION = "histdatacom.classical-model-skill.v1" +CLASSICAL_MODEL_STABILITY_SCHEMA_VERSION = ( + "histdatacom.classical-model-stability.v1" +) +CLASSICAL_MODEL_FIT_ACCOUNTING_SCHEMA_VERSION = ( + "histdatacom.classical-model-fit-accounting.v1" +) +CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.classical-model-comparison-training-projection.v1" +) +CLASSICAL_MODEL_COMPARISON_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.classical-model-comparison-summary.v1" +) +CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_classical_model_comparison_summary" +) +CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY = ( + "fingerprint_classical_model_comparison" +) + +DEFAULT_CLASSICAL_MODEL_COMPARISON_SUMMARY_TARGET_LIMIT = 25 +DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_MODELS = 64 +DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_HORIZONS = 16 +DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_COMPARISONS = 256 +DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_REASON_CODES = 24 +DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_SAMPLES = 12 +DEFAULT_CLASSICAL_MODEL_COMPARISON_ROUNDING_DIGITS = 12 +MAX_CLASSICAL_MODEL_COMPARISON_MODELS = 256 +MAX_CLASSICAL_MODEL_COMPARISON_HORIZONS = 64 +MAX_CLASSICAL_MODEL_COMPARISONS = 2048 +MAX_CLASSICAL_MODEL_COMPARISON_REASON_CODES = 128 +MAX_CLASSICAL_MODEL_COMPARISON_SAMPLES = 64 + +COMPARISON_FAMILY_CODES = { + "baseline": 1, + "ses": 10, + "holt": 11, + "damped_trend": 12, + "holt_winters": 13, + "ets": 14, + "ar": 20, + "arma": 21, + "arima": 22, + "sarima": 30, + "arimax": 31, + "sarimax": 32, + "local_level": 40, + "local_linear_trend": 41, + "structural": 42, + "arch": 50, + "garch": 51, +} +COMPARISON_TARGET_METRIC_CODES = { + "mid_level": 1, + "return_mean": 2, + "conditional_variance": 3, + "absolute_return_volatility": 4, +} +COMPARISON_SCALE_CODES = { + "original_mid": 1, + "unscaled_return": 2, + "unscaled_return_squared": 3, + "absolute_unscaled_return": 4, +} +COMPARISON_METRIC_CODES = { + "mae": 1, + "rmse": 2, + "bias": 3, + "mean_qlike": 4, +} +COMPARISON_STATUS_CODES = { + "unavailable": 1, + "limited": 2, + "ready": 3, +} +COMPARISON_ELIGIBILITY_CODES = { + "ineligible": 1, + "eligible": 2, + "reference": 3, +} +COMPARISON_REASON_CODES = { + "none": 0, + "no_model_results": 1, + "reference_baseline_unavailable": 2, + "reference_metric_unavailable": 3, + "baseline_near_zero": 4, + "frequency_mismatch": 5, + "transform_mismatch": 6, + "missingness_policy_mismatch": 7, + "fold_set_mismatch": 8, + "period_mismatch": 9, + "horizon_mismatch": 10, + "target_metric_mismatch": 11, + "scale_mismatch": 12, + "incomplete_fold_overlap": 13, + "fold_evidence_truncated": 14, + "raw_metric_unavailable": 15, + "reference_record": 16, + "regularization_contract_mismatch": 17, +} +COMPARISON_SKILL_STATUS_CODES = { + "unavailable": 1, + "reference": 2, + "available": 3, + "negative": 4, +} +COMPARISON_STABILITY_STATUS_CODES = { + "unavailable": 1, + "insufficient_folds": 2, + "stable": 3, + "structural_shift": 4, + "isolated_fit_failures": 5, + "persistent_degradation": 6, +} +COMPARISON_BASELINE_CODES = { + "": 0, + "naive_random_walk": 1, + "rolling_mean": 2, + "rolling_median": 3, + "session_seasonal_naive": 4, + "rolling_variance_5": 10, + "rolling_variance_20": 11, + "ewma_variance_0.94": 12, +} + +_MEAN_FAMILY_SECTIONS = ( + "exponential_smoothing", + "autoregressive", + "seasonal_exogenous", + "state_space", +) +_FIT_ACCOUNTING_BUCKETS = ( + "attempted", + "fitted", + "converged", + "limited", + "skipped", + "timed_out", + "numerically_invalid", + "dependency_unavailable", + "failed", + "resource_limited", +) +_RESOURCE_REASON_MARKERS = ("budget", "memory", "resource", "timeout") +_NUMERICAL_REASON_MARKERS = ( + "numerical", + "non_finite", + "singular", + "covariance", +) + + +@dataclass(frozen=True, slots=True) +class ClassicalModelComparisonProfile: + """Operator controls for bounded saved-artifact comparisons.""" + + enabled: bool = False + mean_reference_baseline: str = "naive_random_walk" + variance_reference_baseline: str = "ewma_variance_0.94" + near_zero_tolerance: float = 1e-12 + minimum_stability_folds: int = 3 + drift_tolerance: float = 0.25 + max_models: int = DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_MODELS + max_horizons: int = DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_HORIZONS + max_comparisons: int = DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_COMPARISONS + max_reason_codes: int = DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_REASON_CODES + max_samples: int = DEFAULT_CLASSICAL_MODEL_COMPARISON_MAX_SAMPLES + rounding_digits: int = DEFAULT_CLASSICAL_MODEL_COMPARISON_ROUNDING_DIGITS + + def __post_init__(self) -> None: + if not self.mean_reference_baseline: + raise ValueError("mean_reference_baseline must not be empty") + if not self.variance_reference_baseline: + raise ValueError("variance_reference_baseline must not be empty") + if self.near_zero_tolerance <= 0: + raise ValueError("near_zero_tolerance must be positive") + if self.minimum_stability_folds < 2: + raise ValueError("minimum_stability_folds must be at least 2") + if self.drift_tolerance < 0: + raise ValueError("drift_tolerance must be non-negative") + _bounded_positive( + self.max_models, + MAX_CLASSICAL_MODEL_COMPARISON_MODELS, + "max_models", + ) + _bounded_positive( + self.max_horizons, + MAX_CLASSICAL_MODEL_COMPARISON_HORIZONS, + "max_horizons", + ) + _bounded_positive( + self.max_comparisons, + MAX_CLASSICAL_MODEL_COMPARISONS, + "max_comparisons", + ) + _bounded_positive( + self.max_reason_codes, + MAX_CLASSICAL_MODEL_COMPARISON_REASON_CODES, + "max_reason_codes", + ) + _bounded_positive( + self.max_samples, + MAX_CLASSICAL_MODEL_COMPARISON_SAMPLES, + "max_samples", + ) + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between 0 and 16") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible profile metadata.""" + return { + "enabled": self.enabled, + "mean_reference_baseline": self.mean_reference_baseline, + "variance_reference_baseline": self.variance_reference_baseline, + "near_zero_tolerance": self.near_zero_tolerance, + "minimum_stability_folds": self.minimum_stability_folds, + "drift_tolerance": self.drift_tolerance, + "max_models": self.max_models, + "max_horizons": self.max_horizons, + "max_comparisons": self.max_comparisons, + "max_reason_codes": self.max_reason_codes, + "max_samples": self.max_samples, + "rounding_digits": self.rounding_digits, + "selection_policy": "none", + "automatic_search": False, + } + + +@dataclass(frozen=True, slots=True) +class ClassicalModelComparisonResult: + """Bounded diagnostics plus durable row-key annotations.""" + + diagnostics: Mapping[str, JSONValue] + annotations: tuple[Mapping[str, Any], ...] + + +def classical_model_comparison_from_saved_results( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + model_input: Mapping[str, JSONValue] | None = None, + classical_baselines: Mapping[str, JSONValue] | None = None, + exponential_smoothing: Mapping[str, JSONValue] | None = None, + autoregressive: Mapping[str, JSONValue] | None = None, + seasonal_exogenous: Mapping[str, JSONValue] | None = None, + state_space: Mapping[str, JSONValue] | None = None, + volatility: Mapping[str, JSONValue] | None = None, + profile: ClassicalModelComparisonProfile | None = None, + target: Any | None = None, +) -> ClassicalModelComparisonResult: + """Compare bounded saved family results without triggering model fits.""" + selected = profile or ClassicalModelComparisonProfile(enabled=True) + input_contract = dict(model_input or {}) + family_payloads = { + "classical_baselines": dict(classical_baselines or {}), + "exponential_smoothing": dict(exponential_smoothing or {}), + "autoregressive": dict(autoregressive or {}), + "seasonal_exogenous": dict(seasonal_exogenous or {}), + "state_space": dict(state_space or {}), + "volatility": dict(volatility or {}), + } + identity = _comparison_identity(fingerprint, input_contract) + candidates = _normalized_candidates( + family_payloads, + identity=identity, + profile=selected, + ) + candidates.sort(key=_candidate_sort_key) + model_count = len({str(row.get("model_id") or "") for row in candidates}) + model_truncated = model_count > selected.max_models + allowed_models = {str(row.get("model_id") or "") for row in candidates} + if model_truncated: + allowed_models = set(sorted(allowed_models)[: selected.max_models]) + candidates = [ + row + for row in candidates + if str(row.get("model_id") or "") in allowed_models + ] + horizon_values = sorted( + {_int(row.get("horizon")) for row in candidates if row.get("horizon")} + ) + horizon_truncated = len(horizon_values) > selected.max_horizons + allowed_horizons = set(horizon_values[: selected.max_horizons]) + candidates = [ + row + for row in candidates + if _int(row.get("horizon")) in allowed_horizons + ] + comparison_rows = _comparison_rows(candidates, selected) + comparison_truncated = len(comparison_rows) > selected.max_comparisons + comparison_rows = comparison_rows[: selected.max_comparisons] + accounting = _fit_accounting( + family_payloads, + candidates, + identity=identity, + profile=selected, + ) + status = "ready" if comparison_rows else "unavailable" + reason = None if comparison_rows else "no_model_results" + eligible_count = sum(row.get("eligible") is True for row in comparison_rows) + if comparison_rows and not eligible_count: + status, reason = "limited", "reference_baseline_unavailable" + reason_counts = Counter( + reason_code + for row in comparison_rows + for reason_code in _string_rows(row.get("eligibility_reasons")) + ) + bounded_reasons = dict( + sorted(reason_counts.items())[: selected.max_reason_codes] + ) + diagnostics: dict[str, JSONValue] = { + "schema_version": CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION, + "advisory": True, + "status": status, + "reason": reason, + "target_axis": cast( + JSONValue, dict(_mapping(fingerprint.get("target_axis"))) + ), + "comparison_identity": cast(JSONValue, identity), + "configuration": selected.to_metadata(), + "source_contracts": { + "model_input_schema_version": input_contract.get("schema_version"), + "family_schema_versions": { + name: payload.get("schema_version") + for name, payload in family_payloads.items() + if payload + }, + "saved_artifacts_only": True, + "model_fits_triggered": False, + }, + "metric_contract": { + "mean_level_target": "mid_level", + "volatility_mean_target": "return_mean", + "variance_target": "conditional_variance", + "volatility_target": "absolute_return_volatility", + "metrics_are_not_interchangeable": True, + "original_and_transformed_scales_distinct": True, + }, + "comparison_count": len(comparison_rows), + "eligible_comparison_count": eligible_count, + "ineligible_comparison_count": len(comparison_rows) - eligible_count, + "model_count": min(model_count, selected.max_models), + "horizons": cast(JSONValue, horizon_values[: selected.max_horizons]), + "comparison_records": cast(JSONValue, comparison_rows), + "fit_accounting": accounting, + "reason_counts": cast(JSONValue, bounded_reasons), + "stability_context": _stability_context(fingerprint), + "bounds": { + "models": _bound_payload( + model_count, selected.max_models, model_truncated + ), + "horizons": _bound_payload( + len(horizon_values), selected.max_horizons, horizon_truncated + ), + "comparisons": _bound_payload( + len(_comparison_rows(candidates, selected)), + selected.max_comparisons, + comparison_truncated, + ), + "reason_codes": _bound_payload( + len(reason_counts), + selected.max_reason_codes, + len(reason_counts) > selected.max_reason_codes, + ), + }, + "selection_policy": "none", + "deterministic_ordering_purpose": "stable_serialization_and_display_only", + "descriptive_only": True, + "hard_fail_quality_gate": False, + "non_goals": [ + "model_selection", + "production_recommendation", + "automatic_order_search", + "hyperparameter_search", + "champion_challenger_promotion", + ], + } + annotations, collisions = _build_annotations( + frame, + comparison_rows, + input_contract, + selected, + target=target, + ) + diagnostics["training_projection"] = { + "schema_version": ( + CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION + ), + "columns": list(CLASSICAL_MODEL_COMPARISON_COLUMNS), + "column_prefixes": ["cm_comparison_", "cm_skill_", "cm_stability_"], + "annotation_count": len(annotations), + "projection_collision_count": collisions, + "join_keys": ["series_id", "period", "row_id"], + "timestamp_is_sole_join_key": False, + "projection_record_policy": "stable_serialization_first", + "projection_record_is_model_selection": False, + "diagnostics_are_retrospective": True, + "training_eligible": False, + "legacy_cache_enriched_on_read": True, + "observed_columns_preserved": True, + "synthetic_columns_preserved": True, + } + diagnostics["comparison_id"] = _stable_id( + "classical-model-comparison", diagnostics + ) + return ClassicalModelComparisonResult(diagnostics, annotations) + + +def project_classical_model_comparison_onto_training_frame( + frame: Any, + result: ClassicalModelComparisonResult, + *, + target: Any | None = None, +) -> Any: + """Join retrospective comparison scalars by durable row identity.""" + import polars as pl + + columns = set(getattr(frame, "columns", ())) + enriched = ( + frame + if {"series_id", "period", "row_id"}.issubset(columns) + else ensure_tick_training_features(frame, target=target) + ) + left = enriched.drop( + [ + name + for name in CLASSICAL_MODEL_COMPARISON_COLUMNS + if name in enriched.columns + ] + ).with_row_index("__cm_comparison_original_order") + if result.annotations: + right = pl.DataFrame( + [dict(row) for row in result.annotations], infer_schema_length=None + ) + projected = left.join( + right, + on=["series_id", "period", "row_id"], + how="left", + validate="m:1", + ) + else: + projected = left + projected = _ensure_projection_columns(projected) + return projected.sort("__cm_comparison_original_order").drop( + "__cm_comparison_original_order" + ) + + +def classical_model_comparison_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = ( + DEFAULT_CLASSICAL_MODEL_COMPARISON_SUMMARY_TARGET_LIMIT + ), +) -> dict[str, JSONValue] | None: + """Return a bounded report summary for comparison findings.""" + targets: list[dict[str, JSONValue]] = [] + statuses: Counter[str] = Counter() + eligible_count = 0 + for finding in findings: + fingerprint = _mapping(finding.metadata.get("time_series_fingerprint")) + payload = _mapping(fingerprint.get("classical_model_comparison")) + if not payload: + continue + status = _text(payload.get("status")) or "unavailable" + statuses[status] += 1 + eligible = _int(payload.get("eligible_comparison_count")) + eligible_count += eligible + targets.append( + { + "target_axis": cast( + JSONValue, dict(_mapping(payload.get("target_axis"))) + ), + "status": status, + "reason": payload.get("reason"), + "comparison_id": payload.get("comparison_id"), + "comparison_count": _int(payload.get("comparison_count")), + "eligible_comparison_count": eligible, + "ineligible_comparison_count": _int( + payload.get("ineligible_comparison_count") + ), + "model_count": _int(payload.get("model_count")), + "horizons": cast(JSONValue, _int_rows(payload.get("horizons"))), + "selection_policy": "none", + } + ) + if not targets: + return None + targets.sort( + key=lambda row: _axis_sort_key(_mapping(row.get("target_axis"))) + ) + limit = bounded_report_limit( + target_limit, + default_limit=DEFAULT_CLASSICAL_MODEL_COMPARISON_SUMMARY_TARGET_LIMIT, + ) + included = limit.slice(targets) + return { + "schema_version": CLASSICAL_MODEL_COMPARISON_SUMMARY_SCHEMA_VERSION, + "advisory": True, + "target_count": len(targets), + "included_target_count": len(included), + "omitted_target_count": max(0, len(targets) - len(included)), + "truncated": len(included) < len(targets), + "status_counts": dict(sorted(statuses.items())), + "eligible_comparison_count": eligible_count, + "target_summaries": cast(JSONValue, included), + "limit_metadata": limit.count_payload(len(targets)), + "selection_policy": "none", + } + + +def format_classical_model_comparison_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> list[str]: + """Return concise human-readable report lines.""" + if not summary: + return [] + statuses = _mapping(summary.get("status_counts")) + status_text = ", ".join( + f"{name}={value}" for name, value in sorted(statuses.items()) + ) + lines = [ + "Classical model comparison", + ( + f"targets: {summary.get('target_count', 0)}; " + f"eligible comparisons: {summary.get('eligible_comparison_count', 0)}" + ), + f"statuses: {status_text or 'none'}", + "selection policy: none (descriptive comparisons only)", + ] + omitted = _int(summary.get("omitted_target_count")) + if omitted: + lines.append(f"additional comparison targets omitted: {omitted}") + return lines + + +def _normalized_candidates( + payloads: Mapping[str, Mapping[str, JSONValue]], + *, + identity: Mapping[str, JSONValue], + profile: ClassicalModelComparisonProfile, +) -> list[dict[str, Any]]: + candidates: list[dict[str, Any]] = [] + shared_baselines: list[Mapping[str, JSONValue]] = [] + for section in _MEAN_FAMILY_SECTIONS: + payload = payloads.get(section, {}) + family_identity = _family_identity(identity, payload) + evaluation = _mapping(payload.get("evaluation")) + for baseline in _mapping_rows(evaluation.get("reference_baselines")): + shared_baselines.append( + { + **baseline, + "__comparison_identity": cast(JSONValue, family_identity), + } + ) + for model in _mapping_rows(evaluation.get("models")): + candidates.extend( + _mean_model_candidates( + model, + section=section, + identity=family_identity, + profile=profile, + ) + ) + candidates.extend( + _mean_baseline_candidates( + shared_baselines, + identity=identity, + profile=profile, + ) + ) + volatility = payloads.get("volatility", {}) + volatility_identity = _family_identity(identity, volatility) + evaluation = _mapping(volatility.get("evaluation")) + for model in _mapping_rows(evaluation.get("models")): + candidates.extend( + _volatility_model_candidates( + model, + identity=volatility_identity, + profile=profile, + ) + ) + candidates.extend( + _variance_baseline_candidates( + _mapping_rows(evaluation.get("reference_variance_baselines")), + identity=volatility_identity, + profile=profile, + ) + ) + if not shared_baselines: + candidates.extend( + _legacy_baseline_candidates( + payloads.get("classical_baselines", {}), + identity=identity, + profile=profile, + ) + ) + return _deduplicated_candidates(candidates) + + +def _mean_model_candidates( + model: Mapping[str, JSONValue], + *, + section: str, + identity: Mapping[str, JSONValue], + profile: ClassicalModelComparisonProfile, +) -> list[dict[str, Any]]: + family = _text(model.get("family")) or section + output: list[dict[str, Any]] = [] + horizons = _bounded_horizon_rows(model.get("horizon_metrics")) + for horizon_row in horizons: + horizon = _int(horizon_row.get("horizon")) + output.append( + _candidate( + model, + identity=identity, + family=family, + target_metric="mid_level", + scale="original_mid", + horizon=horizon, + metrics=_simple_metrics(horizon_row), + fold_results=_fold_results(model, horizon, profile), + is_reference=False, + reference_name=profile.mean_reference_baseline, + profile=profile, + ) + ) + return output + + +def _mean_baseline_candidates( + baselines: Sequence[Mapping[str, JSONValue]], + *, + identity: Mapping[str, JSONValue], + profile: ClassicalModelComparisonProfile, +) -> list[dict[str, Any]]: + output: list[dict[str, Any]] = [] + seen: set[tuple[str, int | None, int]] = set() + for baseline in baselines: + baseline_identity = _mapping(baseline.get("__comparison_identity")) + name = _baseline_name(baseline) + window = _optional_int(baseline.get("window")) + for horizon_row in _bounded_horizon_rows( + baseline.get("horizon_metrics") + ): + horizon = _int(horizon_row.get("horizon")) + key = (name, window, horizon) + if key in seen: + continue + seen.add(key) + model_id = f"baseline:{name}" + ( + f":{window}" if window is not None else "" + ) + model = { + "model_id": model_id, + "specification_id": name, + "specification_code": COMPARISON_BASELINE_CODES.get(name, 0), + "family": "baseline", + "status": "ready", + } + output.append( + _candidate( + model, + identity=baseline_identity or identity, + family="baseline", + target_metric="mid_level", + scale="original_mid", + horizon=horizon, + metrics=_simple_metrics(horizon_row), + fold_results=(), + is_reference=name == profile.mean_reference_baseline, + reference_name=profile.mean_reference_baseline, + profile=profile, + baseline_name=name, + ) + ) + return output + + +def _legacy_baseline_candidates( + payload: Mapping[str, JSONValue], + *, + identity: Mapping[str, JSONValue], + profile: ClassicalModelComparisonProfile, +) -> list[dict[str, Any]]: + evaluation = _mapping(payload.get("evaluation")) + output: list[dict[str, Any]] = [] + for model in _mapping_rows(evaluation.get("models")): + name = _text(model.get("model")) + if not name: + continue + candidate = _candidate( + { + "model_id": f"legacy-baseline:{name}", + "specification_id": name, + "specification_code": model.get("model_code"), + "family": "baseline", + "status": model.get("status"), + }, + identity={ + **identity, + "fold_set_id": "legacy_chronological_holdout", + }, + family="baseline", + target_metric="mid_level", + scale="original_mid", + horizon=1, + metrics={ + "mae": model.get("mae"), + "rmse": model.get("rmse"), + "bias": model.get("mean_error"), + }, + fold_results=(), + is_reference=False, + reference_name=profile.mean_reference_baseline, + profile=profile, + baseline_name=name, + ) + candidate["eligibility_seed_reasons"] = ["fold_set_mismatch"] + output.append(candidate) + return output + + +def _volatility_model_candidates( + model: Mapping[str, JSONValue], + *, + identity: Mapping[str, JSONValue], + profile: ClassicalModelComparisonProfile, +) -> list[dict[str, Any]]: + family = _text(model.get("family")) or "volatility" + output: list[dict[str, Any]] = [] + for horizon, raw in _bounded_mapping_items( + model.get("horizon_metrics"), MAX_CLASSICAL_MODEL_COMPARISON_HORIZONS + ): + horizon_value = _int(horizon) + metrics = _mapping(raw) + groups = ( + ( + "return_mean", + "unscaled_return", + _mapping(metrics.get("mean_metrics")), + "", + ), + ( + "conditional_variance", + "unscaled_return_squared", + _mapping(metrics.get("variance_metrics")), + profile.variance_reference_baseline, + ), + ( + "absolute_return_volatility", + "absolute_unscaled_return", + _mapping(metrics.get("volatility_metrics")), + "", + ), + ) + for target_metric, scale, metric_group, reference_name in groups: + output.append( + _candidate( + model, + identity=identity, + family=family, + target_metric=target_metric, + scale=scale, + horizon=horizon_value, + metrics=_simple_metrics(metric_group), + fold_results=(), + is_reference=False, + reference_name=reference_name, + profile=profile, + ) + ) + return output + + +def _variance_baseline_candidates( + baselines: Sequence[Mapping[str, JSONValue]], + *, + identity: Mapping[str, JSONValue], + profile: ClassicalModelComparisonProfile, +) -> list[dict[str, Any]]: + output: list[dict[str, Any]] = [] + for baseline in baselines: + name = _text(baseline.get("name")) + if not name: + continue + for horizon, raw in _bounded_mapping_items( + baseline.get("horizon_metrics"), + MAX_CLASSICAL_MODEL_COMPARISON_HORIZONS, + ): + model = { + "model_id": f"baseline:{name}", + "specification_id": name, + "specification_code": COMPARISON_BASELINE_CODES.get(name, 0), + "family": "baseline", + "status": "ready", + } + output.append( + _candidate( + model, + identity=identity, + family="baseline", + target_metric="conditional_variance", + scale="unscaled_return_squared", + horizon=_int(horizon), + metrics=_simple_metrics(_mapping(raw)), + fold_results=(), + is_reference=name == profile.variance_reference_baseline, + reference_name=profile.variance_reference_baseline, + profile=profile, + baseline_name=name, + ) + ) + return output + + +def _candidate( + model: Mapping[str, Any], + *, + identity: Mapping[str, JSONValue], + family: str, + target_metric: str, + scale: str, + horizon: int, + metrics: Mapping[str, Any], + fold_results: Sequence[Mapping[str, Any]], + is_reference: bool, + reference_name: str, + profile: ClassicalModelComparisonProfile, + baseline_name: str = "", +) -> dict[str, Any]: + fit_samples = _mapping_rows(model.get("fit_samples")) + fit_status_counts = _count_mapping(model.get("fit_status_counts")) + if not fit_status_counts and fit_samples: + fit_status_counts = dict( + Counter( + status + for row in fit_samples + if (status := _text(row.get("status"))) + ) + ) + evaluation_status_counts = _count_mapping( + model.get("evaluation_status_counts") + ) + reason_counts = _count_mapping(model.get("reason_counts")) + if fit_samples: + sampled_reasons = Counter( + reason + for row in fit_samples + if (reason := _text(row.get("reason"))) + ) + if not reason_counts: + reason_counts = dict(sampled_reasons) + candidate: dict[str, Any] = { + **identity, + "family": family, + "family_code": COMPARISON_FAMILY_CODES.get(family, 0), + "model_id": _text(model.get("model_id")) + or f"{family}:{_text(model.get('specification_id'))}", + "specification_id": _text(model.get("specification_id")), + "specification_code": _int(model.get("specification_code")), + "model_status": _text(model.get("status")) or "unavailable", + "target_metric": target_metric, + "scale": scale, + "horizon": horizon, + "metrics": dict(metrics), + "is_reference_baseline": is_reference, + "baseline_name": baseline_name, + "reference_baseline_name": reference_name, + "fit_status_counts": fit_status_counts, + "evaluation_status_counts": evaluation_status_counts, + "reason_counts": reason_counts, + "fold_results": [dict(row) for row in fold_results], + "fold_results_truncated": model.get("fold_results_truncated") is True, + "parameter_stability": dict(_mapping(model.get("parameter_stability"))), + "rolling_window_stability": dict( + _mapping(model.get("rolling_window_stability")) + ), + "regime_error_summary": dict( + _mapping(model.get("regime_error_summary")) + ), + } + candidate["stability"] = _candidate_stability(candidate, profile) + return candidate + + +def _comparison_rows( + candidates: Sequence[Mapping[str, Any]], + profile: ClassicalModelComparisonProfile, +) -> list[dict[str, JSONValue]]: + references = { + ( + _text(row.get("target_metric")), + _text(row.get("scale")), + _int(row.get("horizon")), + _text(row.get("baseline_name")), + ): row + for row in candidates + if row.get("is_reference_baseline") is True + } + output: list[dict[str, JSONValue]] = [] + for candidate in candidates: + reference_name = _text(candidate.get("reference_baseline_name")) + reference = references.get( + ( + _text(candidate.get("target_metric")), + _text(candidate.get("scale")), + _int(candidate.get("horizon")), + reference_name, + ) + ) + for metric_name, metric_value in sorted( + _mapping(candidate.get("metrics")).items() + ): + if metric_name not in COMPARISON_METRIC_CODES: + continue + output.append( + _comparison_record( + candidate, + reference, + metric_name=metric_name, + metric_value=_optional_float(metric_value), + profile=profile, + ) + ) + output.sort(key=_comparison_sort_key) + return output + + +def _comparison_record( + candidate: Mapping[str, Any], + reference: Mapping[str, Any] | None, + *, + metric_name: str, + metric_value: float | None, + profile: ClassicalModelComparisonProfile, +) -> dict[str, JSONValue]: + reasons = list(_string_rows(candidate.get("eligibility_seed_reasons"))) + is_reference = candidate.get("is_reference_baseline") is True + baseline_value = ( + _optional_float(_mapping(reference.get("metrics")).get(metric_name)) + if reference is not None + else None + ) + if metric_value is None: + reasons.append("raw_metric_unavailable") + if is_reference: + reasons.append("reference_record") + elif reference is None: + reasons.append("reference_baseline_unavailable") + else: + reasons.extend(_identity_mismatch_reasons(candidate, reference)) + if baseline_value is None: + reasons.append("reference_metric_unavailable") + fold_overlap = _fold_overlap(candidate, reference) + if fold_overlap.get("complete") is False: + reasons.append("incomplete_fold_overlap") + if candidate.get("fold_results_truncated") is True: + reasons.append("fold_evidence_truncated") + reasons = list(dict.fromkeys(reasons)) + eligible = not reasons + if is_reference: + eligible = True + skill = _skill_payload( + metric_name, + metric_value, + baseline_value, + is_reference=is_reference, + reasons=reasons, + profile=profile, + support_count=_metric_support(candidate), + ) + record: dict[str, JSONValue] = { + "schema_version": CLASSICAL_MODEL_COMPARISON_ELIGIBILITY_SCHEMA_VERSION, + "comparison_id": "", + "dataset_id": candidate.get("dataset_id"), + "fingerprint_id": candidate.get("fingerprint_id"), + "regularization_contract_id": candidate.get( + "regularization_contract_id" + ), + "fold_set_id": candidate.get("fold_set_id"), + "family": candidate.get("family"), + "family_code": candidate.get("family_code"), + "model_id": candidate.get("model_id"), + "specification_id": candidate.get("specification_id"), + "specification_code": candidate.get("specification_code"), + "target_metric": candidate.get("target_metric"), + "target_metric_code": COMPARISON_TARGET_METRIC_CODES.get( + _text(candidate.get("target_metric")), 0 + ), + "scale": candidate.get("scale"), + "scale_code": COMPARISON_SCALE_CODES.get( + _text(candidate.get("scale")), 0 + ), + "frequency_ms": candidate.get("frequency_ms"), + "transform": candidate.get("transform"), + "missingness_policy": candidate.get("missingness_policy"), + "period": candidate.get("period"), + "horizon": candidate.get("horizon"), + "metric": metric_name, + "metric_code": COMPARISON_METRIC_CODES[metric_name], + "metric_value": _rounded(metric_value, profile.rounding_digits), + "reference_baseline": candidate.get("reference_baseline_name"), + "reference_baseline_code": COMPARISON_BASELINE_CODES.get( + _text(candidate.get("reference_baseline_name")), 0 + ), + "reference_metric_value": _rounded( + baseline_value, profile.rounding_digits + ), + "eligible": eligible, + "eligibility_status": ( + "reference" + if is_reference + else "eligible" if eligible else "ineligible" + ), + "eligibility_reasons": cast(JSONValue, reasons), + "fold_overlap": fold_overlap, + "skill": skill, + "stability": cast(JSONValue, candidate.get("stability")), + "fit_accounting": _candidate_accounting(candidate, profile), + "descriptive_only": True, + } + record["comparison_id"] = _stable_id("comparison", record) + return record + + +def _skill_payload( + metric_name: str, + metric_value: float | None, + baseline_value: float | None, + *, + is_reference: bool, + reasons: Sequence[str], + profile: ClassicalModelComparisonProfile, + support_count: int, +) -> dict[str, JSONValue]: + status = "unavailable" + reason: str | None = reasons[0] if reasons else None + value: float | None = None + mode = "ratio_reduction" + if is_reference: + status, reason = "reference", "reference_record" + elif ( + metric_value is not None and baseline_value is not None and not reasons + ): + if metric_name == "mean_qlike": + mode = "baseline_minus_model" + value = baseline_value - metric_value + elif abs(baseline_value) <= profile.near_zero_tolerance: + reason = "baseline_near_zero" + else: + value = 1.0 - metric_value / baseline_value + if value is not None: + status = "negative" if value < 0 else "available" + reason = None + return { + "schema_version": CLASSICAL_MODEL_SKILL_SCHEMA_VERSION, + "status": status, + "reason": reason, + "mode": mode, + "value": _rounded(value, profile.rounding_digits), + "negative": value is not None and value < 0, + "raw_metric_value": _rounded(metric_value, profile.rounding_digits), + "reference_metric_value": _rounded( + baseline_value, profile.rounding_digits + ), + "support_count": support_count, + "dispersion": { + "available": False, + "reason": "aggregate_reference_baseline", + "fold_level_skill_values_included": False, + }, + } + + +def _candidate_stability( + candidate: Mapping[str, Any], + profile: ClassicalModelComparisonProfile, +) -> dict[str, JSONValue]: + rows = [ + row + for row in _mapping_rows(candidate.get("fold_results")) + if row.get("status") == "evaluated" + ] + errors = [ + value + for row in rows + if (value := _optional_float(row.get("error"))) is not None + ] + absolute = [abs(value) for value in errors] + drift = _segment_drift(absolute, profile) + parameter = _parameter_drift( + _mapping(candidate.get("parameter_stability")), profile + ) + fit_counts = _count_mapping(candidate.get("fit_status_counts")) + attempted = sum(fit_counts.values()) + failures = sum( + count + for name, count in fit_counts.items() + if name in {"failed", "unavailable", "timed_out"} + ) + convergence_rate = ( + fit_counts.get("converged", 0) / attempted if attempted else None + ) + failure_rate = failures / attempted if attempted else None + status = "insufficient_folds" + if len(absolute) >= profile.minimum_stability_folds: + if parameter is not None and parameter > profile.drift_tolerance * 2: + status = "structural_shift" + elif drift is not None and drift > profile.drift_tolerance: + status = "persistent_degradation" + elif failures == 1: + status = "isolated_fit_failures" + else: + status = "stable" + rolling = _mapping(candidate.get("rolling_window_stability")) + if not absolute and _mapping_rows(rolling.get("segments")): + status = "stable" + return { + "schema_version": CLASSICAL_MODEL_STABILITY_SCHEMA_VERSION, + "status": status, + "fold_count": len(rows), + "error_drift": _rounded(drift, profile.rounding_digits), + "skill_drift": None, + "skill_drift_reason": "aggregate_reference_baseline", + "parameter_drift": _rounded(parameter, profile.rounding_digits), + "fit_duration_drift": None, + "fit_duration_reason": "fit_duration_unavailable_in_source_contract", + "convergence_rate": _rounded(convergence_rate, profile.rounding_digits), + "failure_rate": _rounded(failure_rate, profile.rounding_digits), + "regime_session_sensitivity": cast( + JSONValue, + _bounded_mapping( + _mapping(candidate.get("regime_error_summary")), + profile.max_samples, + ), + ), + "robust_bounded_summary": True, + } + + +def _fit_accounting( + payloads: Mapping[str, Mapping[str, JSONValue]], + candidates: Sequence[Mapping[str, Any]], + *, + identity: Mapping[str, JSONValue], + profile: ClassicalModelComparisonProfile, +) -> dict[str, JSONValue]: + by_family: list[dict[str, JSONValue]] = [] + totals = Counter({name: 0 for name in _FIT_ACCOUNTING_BUCKETS}) + for section in (*_MEAN_FAMILY_SECTIONS, "volatility"): + payload = payloads.get(section, {}) + if not payload: + continue + fit = _mapping(payload.get("fit_summary")) + counts = _normalized_fit_counts(fit) + totals.update(counts) + by_family.append( + { + "family_group": section, + "period": identity.get("period"), + "counts": cast(JSONValue, counts), + "rates": _fit_rates(counts, profile), + "reason_counts": cast( + JSONValue, + _bounded_count_mapping( + _mapping(fit.get("reason_counts")), + profile.max_reason_codes, + ), + ), + "warning_counts": cast( + JSONValue, + _bounded_count_mapping( + _mapping(fit.get("warning_counts")), + profile.max_reason_codes, + ), + ), + } + ) + by_specification: list[dict[str, JSONValue]] = [] + seen: set[tuple[str, str, int]] = set() + for candidate in candidates: + key = ( + _text(candidate.get("family")), + _text(candidate.get("specification_id")), + _int(candidate.get("horizon")), + ) + if key in seen or key[0] == "baseline": + continue + seen.add(key) + accounting = _candidate_accounting(candidate, profile) + by_specification.append( + { + "family": key[0], + "specification_id": key[1], + "horizon": key[2], + "period": identity.get("period"), + **accounting, + } + ) + by_specification.sort( + key=lambda row: ( + _text(row.get("family")), + _text(row.get("specification_id")), + _int(row.get("horizon")), + ) + ) + by_specification = by_specification[: profile.max_comparisons] + return { + "schema_version": CLASSICAL_MODEL_FIT_ACCOUNTING_SCHEMA_VERSION, + "totals": dict(totals), + "rates": _fit_rates(totals, profile), + "by_family": cast(JSONValue, by_family[: profile.max_models]), + "by_specification_horizon_period": cast(JSONValue, by_specification), + "failed_models_preserved_in_denominator": True, + "resource_terminations_separate": True, + "survivorship_filtering": False, + } + + +def _candidate_accounting( + candidate: Mapping[str, Any], + profile: ClassicalModelComparisonProfile, +) -> dict[str, JSONValue]: + counts = _normalized_model_counts(candidate) + return { + "counts": cast(JSONValue, counts), + "rates": _fit_rates(counts, profile), + "reason_counts": cast( + JSONValue, + _bounded_count_mapping( + _mapping(candidate.get("reason_counts")), + profile.max_reason_codes, + ), + ), + } + + +def _normalized_fit_counts(raw: Mapping[str, JSONValue]) -> dict[str, int]: + statuses = _count_mapping(raw.get("status_counts")) + reasons = _count_mapping(raw.get("reason_counts")) + attempted = _int(raw.get("fit_attempt_count")) or sum(statuses.values()) + counts = Counter({name: 0 for name in _FIT_ACCOUNTING_BUCKETS}) + counts["attempted"] = attempted + counts["converged"] = statuses.get("converged", 0) + counts["limited"] = statuses.get("limited", 0) + counts["skipped"] = statuses.get("skipped", 0) + counts["timed_out"] = statuses.get("timed_out", 0) + sum( + value for key, value in reasons.items() if "timeout" in key + ) + counts["dependency_unavailable"] = statuses.get("unavailable", 0) + sum( + value for key, value in reasons.items() if "dependency" in key + ) + counts["numerically_invalid"] = sum( + value + for key, value in reasons.items() + if any(marker in key for marker in _NUMERICAL_REASON_MARKERS) + ) + counts["resource_limited"] = sum( + value + for key, value in reasons.items() + if any(marker in key for marker in _RESOURCE_REASON_MARKERS) + ) + counts["failed"] = _int(raw.get("failed_fit_count")) or statuses.get( + "failed", 0 + ) + counts["fitted"] = max(0, attempted - counts["skipped"] - counts["failed"]) + return dict(counts) + + +def _normalized_model_counts(candidate: Mapping[str, Any]) -> dict[str, int]: + statuses = _count_mapping(candidate.get("fit_status_counts")) + reasons = _count_mapping(candidate.get("reason_counts")) + attempted = sum(statuses.values()) + return _normalized_fit_counts( + { + "fit_attempt_count": attempted, + "status_counts": cast(JSONValue, statuses), + "reason_counts": cast(JSONValue, reasons), + "failed_fit_count": sum( + value + for key, value in statuses.items() + if key in {"failed", "unavailable"} + ), + } + ) + + +def _fit_rates( + counts: Mapping[str, int], profile: ClassicalModelComparisonProfile +) -> dict[str, JSONValue]: + attempted = counts.get("attempted", 0) + return { + f"{name}_rate": _rounded( + counts.get(name, 0) / attempted if attempted else None, + profile.rounding_digits, + ) + for name in _FIT_ACCOUNTING_BUCKETS + if name != "attempted" + } + + +def _comparison_identity( + fingerprint: Mapping[str, JSONValue], + model_input: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + target_axis = _mapping(fingerprint.get("target_axis")) + regularization = _mapping(model_input.get("regularization")) + transform = _mapping(model_input.get("transform_policy")) + fold_policy = _mapping(model_input.get("fold_policy")) + missingness = { + "expected_closure_policy": regularization.get( + "expected_closure_policy" + ), + "unexpected_missing_policy": regularization.get( + "unexpected_missing_policy" + ), + "empty_bin_value_policy": regularization.get("empty_bin_value_policy"), + "forward_fill_policy": regularization.get("forward_fill_policy"), + } + dataset_id = _stable_id( + "dataset", + { + "target_axis": dict(target_axis), + "fingerprint_id": fingerprint.get("fingerprint_id"), + }, + ) + regularization_id = _text(model_input.get("derivation_id")) or _stable_id( + "regularization", regularization + ) + fold_set_id = _stable_id( + "fold-set", + { + "kind": fold_policy.get("kind"), + "fold_count": fold_policy.get("fold_count"), + "horizons": fold_policy.get("horizons"), + "minimum_training_observations": fold_policy.get( + "minimum_training_observations" + ), + "rolling_window_observations": fold_policy.get( + "rolling_window_observations" + ), + "embargo_observations": fold_policy.get("embargo_observations"), + }, + ) + return { + "dataset_id": dataset_id, + "fingerprint_id": fingerprint.get("fingerprint_id"), + "regularization_contract_id": regularization_id, + "fold_set_id": fold_set_id, + "frequency_ms": regularization.get("frequency_ms"), + "transform": transform.get("transform"), + "missingness_policy": _stable_id("missingness", missingness), + "period": target_axis.get("period"), + } + + +def _family_identity( + identity: Mapping[str, JSONValue], + payload: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + """Overlay identity evidence persisted by one saved family artifact.""" + if not payload: + return dict(identity) + output = dict(identity) + derivation_id = _text(payload.get("input_derivation_id")) + if derivation_id: + output["regularization_contract_id"] = derivation_id + transform = _mapping(payload.get("input_transform_policy")) + if transform.get("transform") is not None: + output["transform"] = transform.get("transform") + target_axis = _mapping(payload.get("target_axis")) + if target_axis.get("period") is not None: + output["period"] = target_axis.get("period") + configuration = _mapping(payload.get("configuration")) + if configuration.get("frequency_ms") is not None: + output["frequency_ms"] = configuration.get("frequency_ms") + return output + + +def _build_annotations( + frame: Any | None, + comparisons: Sequence[Mapping[str, JSONValue]], + model_input: Mapping[str, JSONValue], + profile: ClassicalModelComparisonProfile, + *, + target: Any | None, +) -> tuple[tuple[Mapping[str, Any], ...], int]: + if frame is None or not comparisons: + return (), 0 + try: + enriched = ensure_tick_training_features(frame, target=target) + except (AttributeError, TypeError, ValueError): + return (), 0 + availability: dict[tuple[str, str], list[tuple[int, int]]] = {} + for row in cast(list[dict[str, Any]], enriched.to_dicts()): + timestamp = _optional_int(row.get("timestamp_utc_ms")) + row_id = _optional_int(row.get("row_id")) + if timestamp is None or row_id is None: + continue + key = (_text(row.get("series_id")), _text(row.get("period"))) + availability.setdefault(key, []).append((timestamp, row_id)) + for values in availability.values(): + values.sort() + folds = _mapping_rows( + _mapping(model_input.get("fold_policy")).get("fold_samples") + ) + comparisons_by_horizon: dict[int, list[Mapping[str, JSONValue]]] = {} + for comparison in comparisons: + comparisons_by_horizon.setdefault( + _int(comparison.get("horizon")), [] + ).append(comparison) + for rows in comparisons_by_horizon.values(): + rows.sort(key=_projection_sort_key) + annotations: dict[tuple[str, str, int], dict[str, Any]] = {} + collisions = 0 + for fold in folds: + horizon = _int(fold.get("horizon")) + options = comparisons_by_horizon.get(horizon, ()) + if not options: + continue + comparison = options[0] + group = (_text(fold.get("series_id")), _text(fold.get("period"))) + target_time = _int(fold.get("target_bin_end_utc_ms")) + row_id = _first_available_row_id( + availability.get(group, ()), target_time + ) + if row_id is None: + continue + annotation_key = (*group, row_id) + annotation = _annotation_row(comparison, fold, row_id, profile) + if annotation_key in annotations: + collisions += 1 + continue + annotations[annotation_key] = annotation + return ( + tuple(annotations[item] for item in sorted(annotations)), + collisions, + ) + + +def _annotation_row( + comparison: Mapping[str, JSONValue], + fold: Mapping[str, JSONValue], + row_id: int, + profile: ClassicalModelComparisonProfile, +) -> dict[str, Any]: + skill = _mapping(comparison.get("skill")) + stability = _mapping(comparison.get("stability")) + accounting = _mapping(comparison.get("fit_accounting")) + rates = _mapping(accounting.get("rates")) + eligible = comparison.get("eligible") is True + reasons = _string_rows(comparison.get("eligibility_reasons")) + reason = reasons[0] if reasons else "none" + target_time = _int(fold.get("target_bin_end_utc_ms")) + return { + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "row_id": row_id, + "cm_comparison_schema_version": ( + CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION + ), + "cm_comparison_id": comparison.get("comparison_id"), + "cm_comparison_dataset_id": comparison.get("dataset_id"), + "cm_comparison_regularization_contract_id": comparison.get( + "regularization_contract_id" + ), + "cm_comparison_fold_set_id": comparison.get("fold_set_id"), + "cm_comparison_family_code": comparison.get("family_code"), + "cm_comparison_model_id": comparison.get("model_id"), + "cm_comparison_specification_code": comparison.get( + "specification_code" + ), + "cm_comparison_target_metric_code": comparison.get( + "target_metric_code" + ), + "cm_comparison_scale_code": comparison.get("scale_code"), + "cm_comparison_metric_code": comparison.get("metric_code"), + "cm_comparison_horizon": comparison.get("horizon"), + "cm_comparison_fold_id": fold.get("fold_id"), + "cm_comparison_origin_row_id": fold.get("origin_row_id"), + "cm_comparison_target_row_id": fold.get("target_row_id"), + "cm_comparison_eligible": eligible, + "cm_comparison_eligibility_code": COMPARISON_ELIGIBILITY_CODES[ + _text(comparison.get("eligibility_status")) or "ineligible" + ], + "cm_comparison_reason_code": COMPARISON_REASON_CODES.get(reason, 0), + "cm_comparison_metric_value": comparison.get("metric_value"), + "cm_comparison_reference_metric_value": comparison.get( + "reference_metric_value" + ), + "cm_comparison_diagnostic_available": True, + "cm_comparison_diagnostic_available_at_utc_ms": target_time, + "cm_comparison_diagnostic_only": True, + "cm_comparison_training_eligible": False, + "cm_skill_schema_version": CLASSICAL_MODEL_SKILL_SCHEMA_VERSION, + "cm_skill_reference_baseline_code": comparison.get( + "reference_baseline_code" + ), + "cm_skill_status_code": COMPARISON_SKILL_STATUS_CODES.get( + _text(skill.get("status")), 1 + ), + "cm_skill_value": skill.get("value"), + "cm_skill_negative": skill.get("negative"), + "cm_skill_support_count": skill.get("support_count"), + "cm_skill_available": skill.get("value") is not None, + "cm_skill_diagnostic_only": True, + "cm_stability_schema_version": CLASSICAL_MODEL_STABILITY_SCHEMA_VERSION, + "cm_stability_status_code": COMPARISON_STABILITY_STATUS_CODES.get( + _text(stability.get("status")), 1 + ), + "cm_stability_fold_count": stability.get("fold_count"), + "cm_stability_error_drift": stability.get("error_drift"), + "cm_stability_skill_drift": stability.get("skill_drift"), + "cm_stability_parameter_drift": stability.get("parameter_drift"), + "cm_stability_fit_duration_drift": stability.get("fit_duration_drift"), + "cm_stability_convergence_rate": rates.get("converged_rate"), + "cm_stability_failure_rate": rates.get("failed_rate"), + "cm_stability_available": stability.get("status") + not in {None, "unavailable"}, + "cm_stability_diagnostic_only": True, + } + + +def _ensure_projection_columns(frame: Any) -> Any: + import polars as pl + + string_suffixes = { + "schema_version", + "id", + "dataset_id", + "regularization_contract_id", + "fold_set_id", + "model_id", + } + boolean_suffixes = { + "eligible", + "diagnostic_available", + "diagnostic_only", + "training_eligible", + "negative", + "available", + } + float_suffixes = { + "metric_value", + "reference_metric_value", + "value", + "error_drift", + "skill_drift", + "parameter_drift", + "fit_duration_drift", + "convergence_rate", + "failure_rate", + } + expressions = [] + for name in CLASSICAL_MODEL_COMPARISON_COLUMNS: + if name in frame.columns: + continue + suffix = _comparison_column_suffix(name) + dtype = ( + pl.Utf8 + if suffix in string_suffixes + else ( + pl.Boolean + if suffix in boolean_suffixes + else pl.Float64 if suffix in float_suffixes else pl.Int64 + ) + ) + expressions.append(pl.lit(None, dtype=dtype).alias(name)) + return frame.with_columns(expressions) if expressions else frame + + +def _identity_mismatch_reasons( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> list[str]: + fields = ( + ( + "regularization_contract_id", + "regularization_contract_mismatch", + ), + ("frequency_ms", "frequency_mismatch"), + ("transform", "transform_mismatch"), + ("missingness_policy", "missingness_policy_mismatch"), + ("fold_set_id", "fold_set_mismatch"), + ("period", "period_mismatch"), + ("horizon", "horizon_mismatch"), + ("target_metric", "target_metric_mismatch"), + ("scale", "scale_mismatch"), + ) + return [ + reason + for field, reason in fields + if candidate.get(field) != reference.get(field) + ] + + +def _fold_overlap( + candidate: Mapping[str, Any], reference: Mapping[str, Any] | None +) -> dict[str, JSONValue]: + candidate_ids = { + _int(row.get("fold_id")) + for row in _mapping_rows(candidate.get("fold_results")) + if row.get("fold_id") is not None + } + reference_ids = ( + { + _int(row.get("fold_id")) + for row in _mapping_rows(reference.get("fold_results")) + if row.get("fold_id") is not None + } + if reference is not None + else set() + ) + if not candidate_ids or not reference_ids: + return { + "available": False, + "complete": None, + "candidate_fold_count": len(candidate_ids), + "reference_fold_count": len(reference_ids), + "overlap_count": 0, + "reason": "aggregate_fold_evidence", + } + overlap = candidate_ids & reference_ids + return { + "available": True, + "complete": overlap == candidate_ids == reference_ids, + "candidate_fold_count": len(candidate_ids), + "reference_fold_count": len(reference_ids), + "overlap_count": len(overlap), + "reason": None, + } + + +def _stability_context( + fingerprint: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + stationarity = _mapping(fingerprint.get("stationarity_diagnostics")) + decomposition = _mapping(fingerprint.get("decomposition")) + calendar = _mapping(fingerprint.get("calendar_regimes")) + return { + "stationarity_status": stationarity.get("status"), + "decomposition_status": decomposition.get("status"), + "calendar_regime_status": calendar.get("status"), + "relationship": "contextual_association_only", + "causal_conclusion": False, + } + + +def _segment_drift( + values: Sequence[float], profile: ClassicalModelComparisonProfile +) -> float | None: + if len(values) < profile.minimum_stability_folds: + return None + width = max(1, len(values) // 3) + first = median(values[:width]) + last = median(values[-width:]) + denominator = max(abs(first), profile.near_zero_tolerance) + return (last - first) / denominator + + +def _parameter_drift( + payload: Mapping[str, JSONValue], profile: ClassicalModelComparisonProfile +) -> float | None: + parameters = _mapping(payload.get("parameters")) + drifts: list[float] = [] + for raw in parameters.values(): + summary = _mapping(raw) + minimum = _optional_float(summary.get("min")) + maximum = _optional_float(summary.get("max")) + center = _optional_float(summary.get("median")) + if minimum is None or maximum is None or center is None: + continue + drifts.append( + (maximum - minimum) / max(abs(center), profile.near_zero_tolerance) + ) + return max(drifts) if drifts else None + + +def _fold_results( + model: Mapping[str, JSONValue], + horizon: int, + profile: ClassicalModelComparisonProfile, +) -> list[Mapping[str, Any]]: + rows = [ + row + for row in _mapping_rows(model.get("fold_results")) + if _int(row.get("horizon")) == horizon + ] + rows.sort( + key=lambda row: ( + _int(row.get("fold_id")), + _int(row.get("target_row_id")), + ) + ) + return rows[: profile.max_samples] + + +def _simple_metrics(raw: Mapping[str, Any]) -> dict[str, JSONValue]: + metrics = { + name: cast(JSONValue, raw.get(name)) + for name in COMPARISON_METRIC_CODES + if name in raw + } + if "evaluation_count" in raw: + metrics["evaluation_count"] = cast( + JSONValue, raw.get("evaluation_count") + ) + elif "count" in raw: + metrics["evaluation_count"] = cast(JSONValue, raw.get("count")) + return metrics + + +def _bounded_horizon_rows(raw: Any) -> list[Mapping[str, Any]]: + rows = _mapping_rows(raw) + rows.sort(key=lambda row: _int(row.get("horizon"))) + return rows[:MAX_CLASSICAL_MODEL_COMPARISON_HORIZONS] + + +def _bounded_mapping_items(raw: Any, limit: int) -> list[tuple[str, Any]]: + return sorted(_mapping(raw).items(), key=lambda item: _int(item[0]))[:limit] + + +def _baseline_name(payload: Mapping[str, JSONValue]) -> str: + return _text(payload.get("model")) or _text(payload.get("name")) + + +def _metric_support(candidate: Mapping[str, Any]) -> int: + metrics = _mapping(candidate.get("metrics")) + count = _optional_int(metrics.get("evaluation_count")) + if count is not None: + return count + fold_rows = _mapping_rows(candidate.get("fold_results")) + return sum(row.get("status") == "evaluated" for row in fold_rows) + + +def _deduplicated_candidates( + candidates: Sequence[dict[str, Any]], +) -> list[dict[str, Any]]: + output: dict[tuple[str, str, int, str], dict[str, Any]] = {} + for candidate in candidates: + key = ( + _text(candidate.get("model_id")), + _text(candidate.get("target_metric")), + _int(candidate.get("horizon")), + _text(candidate.get("scale")), + ) + output.setdefault(key, candidate) + return list(output.values()) + + +def _candidate_sort_key(row: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + _text(row.get("target_metric")), + _text(row.get("scale")), + _int(row.get("horizon")), + 0 if row.get("is_reference_baseline") is True else 1, + _text(row.get("family")), + _text(row.get("specification_id")), + _text(row.get("model_id")), + ) + + +def _comparison_sort_key(row: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + _text(row.get("target_metric")), + _text(row.get("scale")), + _int(row.get("horizon")), + _text(row.get("metric")), + 0 if row.get("eligibility_status") == "reference" else 1, + _text(row.get("family")), + _text(row.get("specification_id")), + _text(row.get("model_id")), + ) + + +def _projection_sort_key(row: Mapping[str, Any]) -> tuple[Any, ...]: + """Prefer a usable non-reference diagnostic for the bounded row view.""" + skill = _mapping(row.get("skill")) + return ( + 0 if row.get("eligibility_status") != "reference" else 1, + 0 if row.get("eligible") is True else 1, + 0 if skill.get("value") is not None else 1, + *_comparison_sort_key(row), + ) + + +def _comparison_column_suffix(name: str) -> str: + for prefix in ("cm_comparison_", "cm_skill_", "cm_stability_"): + if name.startswith(prefix): + return name.removeprefix(prefix) + return name + + +def _first_available_row_id( + rows: Sequence[tuple[int, int]], available_at: int +) -> int | None: + return next( + (row_id for timestamp, row_id in rows if timestamp >= available_at), + None, + ) + + +def _stable_id(prefix: str, payload: Mapping[str, Any]) -> str: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + default=str, + ).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _bound_payload( + total: int, limit: int, truncated: bool +) -> dict[str, JSONValue]: + return { + "total_count": total, + "included_count": min(total, limit), + "omitted_count": max(0, total - limit), + "limit": limit, + "truncated": truncated, + } + + +def _bounded_mapping( + payload: Mapping[str, JSONValue], limit: int +) -> dict[str, JSONValue]: + return dict(sorted(payload.items())[:limit]) + + +def _bounded_count_mapping(raw: Any, limit: int) -> dict[str, int]: + return dict(sorted(_count_mapping(raw).items())[:limit]) + + +def _count_mapping(raw: Any) -> dict[str, int]: + return { + str(key): _int(value) + for key, value in _mapping(raw).items() + if _int(value) >= 0 + } + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _mapping_rows(value: Any) -> list[Mapping[str, Any]]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + return [] + return [row for row in value if isinstance(row, Mapping)] + + +def _string_rows(value: Any) -> list[str]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + return [] + return [str(item) for item in value if str(item)] + + +def _int_rows(value: Any) -> list[int]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + return [] + return [_int(item) for item in value] + + +def _text(value: Any) -> str: + return str(value) if value is not None else "" + + +def _int(value: Any) -> int: + if isinstance(value, bool): + return int(value) + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +def _optional_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _optional_float(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + +def _rounded(value: float | None, digits: int) -> float | None: + return ( + round(value, digits) + if value is not None and math.isfinite(value) + else None + ) + + +def _axis_sort_key(axis: Mapping[str, Any]) -> tuple[str, ...]: + return tuple( + _text(axis.get(name)) + for name in ("data_format", "timeframe", "symbol", "period", "kind") + ) + + +def _bounded_positive(value: int, maximum: int, name: str) -> None: + if not 1 <= value <= maximum: + raise ValueError(f"{name} must be between 1 and {maximum}") diff --git a/src/histdatacom/data_quality/classical_model_contracts.py b/src/histdatacom/data_quality/classical_model_contracts.py new file mode 100644 index 00000000..545c8631 --- /dev/null +++ b/src/histdatacom/data_quality/classical_model_contracts.py @@ -0,0 +1,1588 @@ +"""Classical-model input, fold, fit, and evaluation contracts. + +This module deliberately stops before fitting a statistical model. It turns +the enriched ASCII tick training frame into a deterministic regular-grid view, +defines leakage-safe evaluation folds, and projects only point-in-time-safe +scalars back onto the canonical tick rows. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import math +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, TypedDict, cast + +from histdatacom.data_quality.calendar import ( + SESSION_STATE_WEEKEND_CLOSURE, + classify_histdata_timestamp, +) +from histdatacom.data_quality.contracts import QualityFinding +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.training_features import ( + CLASSICAL_MODEL_CONTRACT_COLUMNS, + TRAINING_SCHEMA_VERSION, + ensure_tick_training_features, +) +from histdatacom.runtime_contracts import JSONValue + +CLASSICAL_MODEL_INPUT_SCHEMA_VERSION = "histdatacom.classical-model-input.v1" +CLASSICAL_MODEL_FOLD_SCHEMA_VERSION = "histdatacom.classical-model-fold.v1" +CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION = ( + "histdatacom.classical-model-fit-result.v1" +) +CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION = ( + "histdatacom.classical-model-evaluation-result.v1" +) +CLASSICAL_MODEL_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.classical-model-training-projection.v1" +) +CLASSICAL_MODEL_INPUT_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.classical-model-input-summary.v1" +) +CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_classical_model_input_summary" +) +CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY = "fingerprint_classical_model_input" + +DEFAULT_MODEL_FREQUENCY_MS = 60_000 +DEFAULT_MODEL_MINIMUM_TRAINING_OBSERVATIONS = 20 +DEFAULT_MODEL_MINIMUM_EVALUATION_OBSERVATIONS = 5 +DEFAULT_MODEL_STEP_SIZE = 5 +DEFAULT_MODEL_HORIZONS = (1,) +DEFAULT_MODEL_ROUNDING_DIGITS = 12 +DEFAULT_MODEL_INPUT_SUMMARY_TARGET_LIMIT = 16 +MAX_MODEL_HORIZONS = 16 + +MODEL_INPUT_REQUIRED_COLUMNS = ( + "series_id", + "period", + "row_id", + "timestamp_utc_ms", + "mid", + "spread", + "training_usable", +) +MODEL_INPUT_STATUS_CODES = { + "unavailable": 1, + "limited": 2, + "ready": 3, +} +MODEL_INPUT_REASON_CODES = { + "": 0, + "training_frame_unavailable": 1, + "missing_required_columns": 2, + "no_usable_rows": 3, + "source_row_limit": 4, + "regularized_observation_limit": 5, + "insufficient_regularized_observations": 6, + "insufficient_folds": 7, + "invalid_transform_domain": 8, +} +MODEL_BIN_STATUS_CODES = { + "observed": 1, + "expected_closure": 2, + "unexpected_missing": 3, + "insufficient_observations": 4, +} +MODEL_TRANSFORM_CODES = { + "level": 1, + "log_level": 2, + "return": 3, + "log_return": 4, +} +MODEL_FOLD_KIND_CODES = {"expanding": 1, "rolling": 2} +MODEL_EVALUATION_STATUS_CODES = { + "unavailable": 1, + "contract_ready": 2, + "evaluated": 3, +} +MODEL_FIT_STATUSES = ( + "fitted", + "converged", + "limited", + "skipped", + "timed_out", + "numerically_invalid", + "dependency_unavailable", + "failed", +) +MODEL_FAILURE_REASON_CODES = ( + "", + "insufficient_data", + "invalid_configuration", + "optimizer_failure", + "numerical_failure", + "resource_limit", + "timeout", + "dependency_unavailable", + "backend_failure", +) +MODEL_OPTIONAL_DEPENDENCIES = ( + ("statsmodels", "ETS, ARIMA, SARIMAX, and state-space families"), + ("arch", "ARCH and GARCH volatility families"), +) + + +class _GridRow(TypedDict): + series_id: str + period: str + cm_input_bin_start_utc_ms: int + cm_input_bin_end_utc_ms: int + cm_input_available_at_utc_ms: int + cm_input_observation_count: int + cm_input_source_first_row_id: int | None + cm_input_source_last_row_id: int | None + cm_input_status_code: int + cm_input_expected_closure: bool + cm_input_unexpected_missing: bool + cm_input_mid_open: float | None + cm_input_mid_high: float | None + cm_input_mid_low: float | None + cm_input_mid_close: float | None + cm_input_mid_mean: float | None + cm_input_mid_median: float | None + cm_input_spread: float | None + cm_input_observed_value: float | None + cm_input_value: float | None + cm_input_transform_valid: bool + + +@dataclass(frozen=True, slots=True) +class ClassicalModelResourcePolicy: + """Bounded resource limits shared by future classical model families.""" + + max_source_rows: int = 1_000_000 + max_regularized_observations: int = 100_000 + max_folds: int = 64 + max_horizons: int = MAX_MODEL_HORIZONS + max_candidate_orders: int = 32 + max_fit_attempts: int = 64 + max_wall_time_seconds: int = 300 + max_memory_bytes: int = 536_870_912 + max_retained_diagnostics: int = 64 + + def __post_init__(self) -> None: + for name, value in self.to_metadata().items(): + if not isinstance(value, int) or value < 1: + raise ValueError(f"{name} must be a positive integer") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return stable JSON-compatible limit metadata.""" + return { + "max_source_rows": self.max_source_rows, + "max_regularized_observations": (self.max_regularized_observations), + "max_folds": self.max_folds, + "max_horizons": self.max_horizons, + "max_candidate_orders": self.max_candidate_orders, + "max_fit_attempts": self.max_fit_attempts, + "max_wall_time_seconds": self.max_wall_time_seconds, + "max_memory_bytes": self.max_memory_bytes, + "max_retained_diagnostics": self.max_retained_diagnostics, + } + + +@dataclass(frozen=True, slots=True) +class ClassicalModelInputProfile: + """Explicit regularization, transform, fold, and limit controls.""" + + enabled: bool = False + frequency_ms: int = DEFAULT_MODEL_FREQUENCY_MS + alignment_epoch_ms: int = 0 + closed_side: str = "left" + label_side: str = "left" + midpoint_aggregation: str = "last" + spread_aggregation: str = "last" + minimum_observations_per_bin: int = 1 + expected_closure_policy: str = "mark" + unexpected_missing_policy: str = "mark" + transform: str = "level" + differencing_order: int = 0 + seasonal_differencing_order: int = 0 + seasonal_period: int = 0 + horizons: tuple[int, ...] = DEFAULT_MODEL_HORIZONS + fold_kind: str = "expanding" + minimum_training_observations: int = ( + DEFAULT_MODEL_MINIMUM_TRAINING_OBSERVATIONS + ) + minimum_evaluation_observations: int = ( + DEFAULT_MODEL_MINIMUM_EVALUATION_OBSERVATIONS + ) + step_size: int = DEFAULT_MODEL_STEP_SIZE + rolling_window: int = 0 + embargo_observations: int = 0 + rounding_digits: int = DEFAULT_MODEL_ROUNDING_DIGITS + resources: ClassicalModelResourcePolicy = field( + default_factory=ClassicalModelResourcePolicy + ) + + def __post_init__(self) -> None: + if self.frequency_ms < 1: + raise ValueError("frequency_ms must be positive") + if self.closed_side != "left" or self.label_side != "left": + raise ValueError( + "only left-closed, left-labeled UTC bins are supported" + ) + allowed_aggregations = {"first", "last", "mean", "median"} + if self.midpoint_aggregation not in allowed_aggregations: + raise ValueError("unsupported midpoint_aggregation") + if self.spread_aggregation not in allowed_aggregations: + raise ValueError("unsupported spread_aggregation") + if self.minimum_observations_per_bin < 1: + raise ValueError("minimum_observations_per_bin must be positive") + if self.expected_closure_policy not in {"mark", "omit"}: + raise ValueError("unsupported expected_closure_policy") + if self.unexpected_missing_policy != "mark": + raise ValueError("unexpected missing bins must be marked") + if self.transform not in MODEL_TRANSFORM_CODES: + raise ValueError("unsupported transform") + if self.differencing_order not in {0, 1, 2}: + raise ValueError("differencing_order must be 0, 1, or 2") + if self.seasonal_differencing_order not in {0, 1}: + raise ValueError("seasonal_differencing_order must be 0 or 1") + if self.seasonal_differencing_order and self.seasonal_period < 2: + raise ValueError("seasonal_period must be at least 2") + if not self.horizons or any(value < 1 for value in self.horizons): + raise ValueError("horizons must contain positive integers") + if len(self.horizons) > self.resources.max_horizons: + raise ValueError("horizons exceed the resource policy") + if tuple(sorted(set(self.horizons))) != self.horizons: + raise ValueError("horizons must be sorted and unique") + if self.fold_kind not in MODEL_FOLD_KIND_CODES: + raise ValueError("fold_kind must be expanding or rolling") + for name in ( + "minimum_training_observations", + "minimum_evaluation_observations", + "step_size", + ): + if int(getattr(self, name)) < 1: + raise ValueError(f"{name} must be positive") + if self.fold_kind == "rolling" and ( + self.rolling_window < self.minimum_training_observations + ): + raise ValueError( + "rolling_window must cover minimum training observations" + ) + if self.embargo_observations < 0: + raise ValueError("embargo_observations must be non-negative") + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between 0 and 16") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return a stable profile payload.""" + return { + "enabled": self.enabled, + "frequency_ms": self.frequency_ms, + "alignment_epoch_ms": self.alignment_epoch_ms, + "timezone": "UTC", + "closed_side": self.closed_side, + "label_side": self.label_side, + "midpoint_aggregation": self.midpoint_aggregation, + "spread_aggregation": self.spread_aggregation, + "minimum_observations_per_bin": (self.minimum_observations_per_bin), + "expected_closure_policy": self.expected_closure_policy, + "unexpected_missing_policy": self.unexpected_missing_policy, + "transform": self.transform, + "differencing_order": self.differencing_order, + "seasonal_differencing_order": (self.seasonal_differencing_order), + "seasonal_period": self.seasonal_period, + "horizons": list(self.horizons), + "fold_kind": self.fold_kind, + "minimum_training_observations": ( + self.minimum_training_observations + ), + "minimum_evaluation_observations": ( + self.minimum_evaluation_observations + ), + "step_size": self.step_size, + "rolling_window": self.rolling_window, + "embargo_observations": self.embargo_observations, + "rounding_digits": self.rounding_digits, + "resources": self.resources.to_metadata(), + } + + +@dataclass(frozen=True, slots=True) +class ClassicalModelInputResult: + """A bounded regularized frame plus serializable contract metadata.""" + + regularized_frame: Any + contract: Mapping[str, JSONValue] + folds: tuple[Mapping[str, JSONValue], ...] + + +def build_classical_model_input( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + profile: ClassicalModelInputProfile | None = None, + target: Any | None = None, +) -> ClassicalModelInputResult: + """Build a deterministic regular-grid view and leakage-safe folds.""" + import polars as pl + + selected = profile or ClassicalModelInputProfile(enabled=True) + base = _base_contract(fingerprint, selected) + if frame is None: + return _unavailable_result(base, "training_frame_unavailable") + input_was_enriched = "training_schema_version" in getattr( + frame, "columns", () + ) + try: + enriched = ensure_tick_training_features(frame, target=target) + except (AttributeError, TypeError, ValueError): + return _unavailable_result(base, "training_frame_unavailable") + missing = sorted( + set(MODEL_INPUT_REQUIRED_COLUMNS) + - set(getattr(enriched, "columns", ())) + ) + if missing: + base["missing_required_columns"] = cast(JSONValue, missing) + return _unavailable_result(base, "missing_required_columns") + + source_row_count = int(enriched.height) + source_truncated = source_row_count > selected.resources.max_source_rows + bounded = enriched.head(selected.resources.max_source_rows) + usable = bounded.filter( + pl.col("training_usable").fill_null(False) + & pl.col("timestamp_utc_ms").is_not_null() + & pl.col("mid").is_not_null() + & pl.col("mid").is_finite() + & pl.col("spread").is_not_null() + & pl.col("spread").is_finite() + ) + if usable.height < 1: + base["source"] = { + **cast(dict[str, JSONValue], base["source"]), + "row_count": source_row_count, + "bounded_row_count": int(bounded.height), + "usable_row_count": 0, + "structurally_unusable_row_count": int(bounded.height), + "source_rows_truncated": source_truncated, + "legacy_cache_enriched_on_read": not input_was_enriched, + } + return _unavailable_result(base, "no_usable_rows") + + grid, grid_metadata = _regularize(usable, selected) + transformed, transform_metadata = _apply_transforms(grid, selected) + folds = _build_folds(transformed, selected) + annotated = _annotate_primary_folds(transformed, folds) + limitations: list[str] = [] + if source_truncated: + limitations.append("source_row_limit") + if grid_metadata["truncated"] is True: + limitations.append("regularized_observation_limit") + valid_count = _valid_observation_count(annotated) + if valid_count < selected.minimum_training_observations: + limitations.append("insufficient_regularized_observations") + if not folds: + limitations.append("insufficient_folds") + if _int(transform_metadata["invalid_domain_count"]) > 0: + limitations.append("invalid_transform_domain") + status = "ready" if not limitations else "limited" + reason = limitations[0] if limitations else None + contract: dict[str, JSONValue] = { + **base, + "status": status, + "reason": reason, + "limitations": cast(JSONValue, limitations), + "derivation_id": _derivation_id(annotated, selected, fingerprint), + "source": { + **cast(dict[str, JSONValue], base["source"]), + "row_count": source_row_count, + "bounded_row_count": int(bounded.height), + "usable_row_count": int(usable.height), + "structurally_unusable_row_count": int( + bounded.height - usable.height + ), + "source_rows_truncated": source_truncated, + "legacy_cache_enriched_on_read": not input_was_enriched, + }, + "regularization": grid_metadata, + "transform_policy": transform_metadata, + "fold_policy": _fold_policy(selected, folds), + "dependency_policy": classical_model_dependency_status(), + "training_projection": _training_projection_metadata( + status, reason, selected, folds + ), + } + return ClassicalModelInputResult( + regularized_frame=annotated, + contract=contract, + folds=tuple(folds), + ) + + +def classical_model_input_contract_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + profile: ClassicalModelInputProfile | None = None, + target: Any | None = None, +) -> dict[str, JSONValue]: + """Return only the serializable contract for fingerprint/report use.""" + return dict( + build_classical_model_input( + frame, + fingerprint, + profile=profile, + target=target, + ).contract + ) + + +def project_classical_model_input_onto_training_frame( + frame: Any, + result: ClassicalModelInputResult, + *, + target: Any | None = None, +) -> Any: + """Project completed-bin scalars without leaking before bin close.""" + import polars as pl + + columns = set(getattr(frame, "columns", ())) + if not {"series_id", "period", "row_id"}.issubset(columns): + enriched = ensure_tick_training_features(frame, target=target) + else: + enriched = frame + missing = sorted({"series_id", "period", "row_id"} - set(enriched.columns)) + if missing: + raise ValueError( + "classical model projection requires identity columns: " + + ", ".join(missing) + ) + if "timestamp_utc_ms" not in enriched.columns: + return _with_empty_projection(enriched) + grid = result.regularized_frame + if getattr(grid, "height", 0) < 1: + return _with_empty_projection(enriched) + + right = _projection_frame(grid, result.contract) + left = enriched.drop( + [ + column + for column in CLASSICAL_MODEL_CONTRACT_COLUMNS + if column in enriched.columns + ] + ).with_row_index("__cm_original_order") + with_timestamp = left.filter(pl.col("timestamp_utc_ms").is_not_null()).sort( + "series_id", "period", "timestamp_utc_ms", "row_id" + ) + without_timestamp = left.filter(pl.col("timestamp_utc_ms").is_null()) + if with_timestamp.height: + projected = with_timestamp.join_asof( + right.sort( + "series_id", + "period", + "cm_input_available_at_utc_ms", + ), + left_on="timestamp_utc_ms", + right_on="cm_input_available_at_utc_ms", + by=["series_id", "period"], + strategy="backward", + check_sortedness=False, + ) + else: + projected = _with_empty_projection(with_timestamp) + if without_timestamp.height: + without_timestamp = _with_empty_projection(without_timestamp) + projected = pl.concat( + [projected, without_timestamp], how="diagonal_relaxed" + ) + projected = projected.sort("__cm_original_order").drop( + "__cm_original_order" + ) + return _ensure_projection_dtypes(projected) + + +def classical_model_fit_result( + *, + model_id: str, + family: str, + status: str, + reason: str = "", + warning_codes: Sequence[str] = (), + parameter_count: int = 0, + fitted_observation_count: int = 0, +) -> dict[str, JSONValue]: + """Create a bounded family-neutral fit-result contract.""" + if status not in MODEL_FIT_STATUSES: + raise ValueError("unsupported fit status") + if reason not in MODEL_FAILURE_REASON_CODES: + raise ValueError("unsupported fit failure reason") + warnings = sorted(set(str(value) for value in warning_codes))[:64] + return { + "schema_version": CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION, + "advisory": True, + "model_id": model_id, + "family": family, + "status": status, + "reason": reason or None, + "parameter_count": max(0, int(parameter_count)), + "fitted_observation_count": max(0, int(fitted_observation_count)), + "warning_codes": cast(JSONValue, warnings), + "warning_count": len(warnings), + "backend_exception_text_included": False, + } + + +def classical_model_evaluation_result( + *, + model_id: str, + status: str, + fold_count: int, + horizon_count: int, + metric_scale: str, + reason: str = "", +) -> dict[str, JSONValue]: + """Create a bounded evaluation-result contract without model fitting.""" + if status not in {"unavailable", "contract_ready", "evaluated"}: + raise ValueError("unsupported evaluation status") + return { + "schema_version": CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION, + "advisory": True, + "model_id": model_id, + "status": status, + "reason": reason or None, + "fold_count": max(0, int(fold_count)), + "horizon_count": max(0, int(horizon_count)), + "metric_scale": metric_scale, + "automatic_winner": False, + "full_forecasts_included": False, + "full_residuals_included": False, + } + + +def classical_model_dependency_status( + *, probe: bool = False +) -> dict[str, JSONValue]: + """Describe optional providers without making contracts environment-specific.""" + dependencies: list[dict[str, JSONValue]] = [] + for package, purpose in MODEL_OPTIONAL_DEPENDENCIES: + dependencies.append( + { + "package": package, + "available": ( + importlib.util.find_spec(package) is not None + if probe + else None + ), + "purpose": purpose, + } + ) + return { + "core_dependency_added": False, + "future_install_extra": "models", + "contract_available_without_optional_dependencies": True, + "rich_model_fitting_available": False, + "availability_basis": "runtime_probe" if probe else "not_probed", + "dependencies": cast(JSONValue, dependencies), + } + + +def classical_model_input_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = DEFAULT_MODEL_INPUT_SUMMARY_TARGET_LIMIT, +) -> dict[str, JSONValue] | None: + """Return a bounded report summary for opt-in model input contracts.""" + targets: list[dict[str, JSONValue]] = [] + statuses: Counter[str] = Counter() + for finding in findings: + fingerprint = _mapping(finding.metadata.get("time_series_fingerprint")) + contract = _mapping(fingerprint.get("classical_model_input")) + if not contract: + continue + status = _text(contract.get("status")) or "unavailable" + regularization = _mapping(contract.get("regularization")) + fold_policy = _mapping(contract.get("fold_policy")) + statuses[status] += 1 + targets.append( + { + "target_axis": dict(_mapping(contract.get("target_axis"))), + "status": status, + "reason": contract.get("reason"), + "regularized_observation_count": _int( + regularization.get("regularized_observation_count") + ), + "observed_bin_count": _int( + regularization.get("observed_bin_count") + ), + "expected_closure_count": _int( + regularization.get("expected_closure_count") + ), + "unexpected_missing_count": _int( + regularization.get("unexpected_missing_count") + ), + "fold_count": _int(fold_policy.get("fold_count")), + } + ) + if not targets: + return None + targets.sort(key=_target_sort_key) + limit = bounded_report_limit( + target_limit, + default_limit=DEFAULT_MODEL_INPUT_SUMMARY_TARGET_LIMIT, + allow_unbounded=True, + ) + included = limit.slice(targets) + omitted = len(targets) - len(included) + return { + "schema_version": CLASSICAL_MODEL_INPUT_SUMMARY_SCHEMA_VERSION, + "advisory": True, + "target_count": len(targets), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "status_counts": dict(sorted(statuses.items())), + "target_summaries": cast(JSONValue, included), + "limit_metadata": {"targets": limit.limit_payload()}, + } + + +def format_classical_model_input_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> tuple[str, ...]: + """Return concise human-readable model-input contract lines.""" + if not summary: + return () + statuses = _mapping(summary.get("status_counts")) + lines = [ + "", + "Classical model input contracts", + ( + f"targets: {_int(summary.get('target_count'))} " + f"ready: {_int(statuses.get('ready'))} " + f"limited: {_int(statuses.get('limited'))} " + f"unavailable: {_int(statuses.get('unavailable'))}" + ), + ] + for target in _mapping_rows(summary.get("target_summaries")): + axis = _mapping(target.get("target_axis")) + label = "/".join( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + lines.append( + f"- {label}: {target.get('status', 'unavailable')} " + f"bins={_int(target.get('regularized_observation_count'))} " + f"folds={_int(target.get('fold_count'))}" + ) + if summary.get("truncated") is True: + lines.append( + f"- {_int(summary.get('omitted_target_count'))} targets omitted" + ) + return tuple(lines) + + +def _base_contract( + fingerprint: Mapping[str, JSONValue], + profile: ClassicalModelInputProfile, +) -> dict[str, JSONValue]: + reference_fingerprint_id = _reference_fingerprint_id(fingerprint) + return { + "schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "advisory": True, + "base_grain": {"data_format": "ascii", "timeframe": "T"}, + "target_axis": dict(_mapping(fingerprint.get("target_axis"))), + "reference_fingerprint_id": reference_fingerprint_id, + "reference_fingerprint_basis": ( + "provided_fingerprint_id" + if _text(fingerprint.get("fingerprint_id")) + else "canonical_pre_contract_snapshot" + ), + "source": { + "kind": "enriched_training_frame", + "row_count": 0, + "usable_row_count": 0, + "legacy_cache_enriched_on_read": False, + }, + "training_schema_version": TRAINING_SCHEMA_VERSION, + "row_selection_policy": "training_usable_and_finite_mid_spread", + "configuration": profile.to_metadata(), + "calculation_basis": "regular_grid_from_enriched_ascii_ticks", + "canonical_row_identity": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "regularized_view_replaces_source_cache": False, + "derived_ohlc_is_canonical_source": False, + "hard_fail_quality_gate": False, + "model_fitting_in_scope": False, + } + + +def _unavailable_result( + base: Mapping[str, JSONValue], reason: str +) -> ClassicalModelInputResult: + import polars as pl + + contract: dict[str, JSONValue] = { + **dict(base), + "status": "unavailable", + "reason": reason, + "limitations": cast(JSONValue, [reason]), + "regularization": _empty_regularization(), + "transform_policy": _empty_transform_policy(base), + "fold_policy": _empty_fold_policy(base), + "dependency_policy": classical_model_dependency_status(), + "training_projection": _training_projection_metadata( + "unavailable", + reason, + _profile_from_base(base), + (), + ), + } + return ClassicalModelInputResult(pl.DataFrame(), contract, ()) + + +def _regularize( + frame: Any, profile: ClassicalModelInputProfile +) -> tuple[Any, dict[str, JSONValue]]: + import polars as pl + + frequency = profile.frequency_ms + alignment = profile.alignment_epoch_ms + working = frame.sort( + "series_id", "period", "timestamp_utc_ms", "row_id" + ).with_columns( + ( + ((pl.col("timestamp_utc_ms") - alignment) // frequency) * frequency + + alignment + ) + .cast(pl.Int64) + .alias("__cm_bin_start") + ) + grouped = ( + working.group_by("series_id", "period", "__cm_bin_start") + .agg( + pl.len().alias("observation_count"), + pl.col("row_id").first().alias("source_first_row_id"), + pl.col("row_id").last().alias("source_last_row_id"), + pl.col("mid").first().alias("mid_first"), + pl.col("mid").last().alias("mid_last"), + pl.col("mid").mean().alias("mid_mean"), + pl.col("mid").median().alias("mid_median"), + pl.col("mid").min().alias("mid_min"), + pl.col("mid").max().alias("mid_max"), + pl.col("spread").first().alias("spread_first"), + pl.col("spread").last().alias("spread_last"), + pl.col("spread").mean().alias("spread_mean"), + pl.col("spread").median().alias("spread_median"), + ) + .sort("series_id", "period", "__cm_bin_start") + ) + observed = { + ( + str(row["series_id"]), + str(row["period"]), + int(row["__cm_bin_start"]), + ): row + for row in grouped.to_dicts() + } + rows: list[_GridRow] = [] + truncated = False + group_bounds = ( + grouped.group_by("series_id", "period") + .agg( + pl.col("__cm_bin_start").min().alias("first_bin"), + pl.col("__cm_bin_start").max().alias("last_bin"), + ) + .sort("series_id", "period") + .to_dicts() + ) + for bounds in group_bounds: + series_id = str(bounds["series_id"]) + period = str(bounds["period"]) + first_bin = int(bounds["first_bin"]) + last_bin = int(bounds["last_bin"]) + for bin_start in range(first_bin, last_bin + frequency, frequency): + if len(rows) >= profile.resources.max_regularized_observations: + truncated = True + break + item = observed.get((series_id, period, bin_start)) + rows.append( + _grid_row( + series_id, + period, + bin_start, + item, + profile, + ) + ) + if truncated: + break + result = pl.DataFrame(rows) if rows else pl.DataFrame() + statuses = Counter( + _bin_status_name(row["cm_input_status_code"]) for row in rows + ) + metadata: dict[str, JSONValue] = { + "schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "basis": "regular_grid", + "source_basis": "enriched_ascii_tick_rows", + "timezone": "UTC", + "frequency_ms": profile.frequency_ms, + "alignment_epoch_ms": profile.alignment_epoch_ms, + "closed_side": profile.closed_side, + "label_side": profile.label_side, + "bin_interval": "[start,end)", + "midpoint_aggregation": profile.midpoint_aggregation, + "spread_aggregation": profile.spread_aggregation, + "derived_ohlc_fields": [ + "cm_input_mid_open", + "cm_input_mid_high", + "cm_input_mid_low", + "cm_input_mid_close", + ], + "minimum_observations_per_bin": (profile.minimum_observations_per_bin), + "duplicate_timestamp_policy": ( + "preserve_and_aggregate_in_row_id_order" + ), + "partial_or_unusable_row_policy": ( + "exclude_via_training_usable_before_regularization" + ), + "empty_bin_value_policy": "explicit_null", + "insufficient_bin_value_policy": "explicit_null", + "expected_closure_policy": profile.expected_closure_policy, + "expected_closure_grid_rows_retained": True, + "expected_closure_model_observations_omitted": ( + profile.expected_closure_policy == "omit" + ), + "unexpected_missing_policy": profile.unexpected_missing_policy, + "forward_fill_policy": "never", + "period_boundary_crossing": False, + "period_partitioned": True, + "rounding_digits": profile.rounding_digits, + "rounding_applies_to": ["cm_input_value"], + "regularized_observation_count": len(rows), + "observed_bin_count": statuses["observed"], + "expected_closure_count": statuses["expected_closure"], + "unexpected_missing_count": statuses["unexpected_missing"], + "insufficient_observation_bin_count": statuses[ + "insufficient_observations" + ], + "structurally_unavailable_count": statuses["insufficient_observations"], + "truncated": truncated, + "row_mapping_policy": "availability_safe_repetition_after_bin_close", + } + return result, metadata + + +def _grid_row( + series_id: str, + period: str, + bin_start: int, + item: Mapping[str, Any] | None, + profile: ClassicalModelInputProfile, +) -> _GridRow: + bin_end = bin_start + profile.frequency_ms + if item is None: + expected = ( + classify_histdata_timestamp(bin_start).session_state + == SESSION_STATE_WEEKEND_CLOSURE + ) + status = "expected_closure" if expected else "unexpected_missing" + count = 0 + else: + count = int(item.get("observation_count", 0) or 0) + expected = False + status = ( + "observed" + if count >= profile.minimum_observations_per_bin + else "insufficient_observations" + ) + observed_value = ( + _aggregate_value(item, "mid", profile.midpoint_aggregation) + if status == "observed" + else None + ) + spread_value = ( + _aggregate_value(item, "spread", profile.spread_aggregation) + if status == "observed" + else None + ) + return { + "series_id": series_id, + "period": period, + "cm_input_bin_start_utc_ms": bin_start, + "cm_input_bin_end_utc_ms": bin_end, + "cm_input_available_at_utc_ms": bin_end, + "cm_input_observation_count": count, + "cm_input_source_first_row_id": _optional_int( + item.get("source_first_row_id") if item else None + ), + "cm_input_source_last_row_id": _optional_int( + item.get("source_last_row_id") if item else None + ), + "cm_input_status_code": MODEL_BIN_STATUS_CODES[status], + "cm_input_expected_closure": expected, + "cm_input_unexpected_missing": status == "unexpected_missing", + "cm_input_mid_open": _optional_float( + item.get("mid_first") if item else None + ), + "cm_input_mid_high": _optional_float( + item.get("mid_max") if item else None + ), + "cm_input_mid_low": _optional_float( + item.get("mid_min") if item else None + ), + "cm_input_mid_close": _optional_float( + item.get("mid_last") if item else None + ), + "cm_input_mid_mean": _optional_float( + item.get("mid_mean") if item else None + ), + "cm_input_mid_median": _optional_float( + item.get("mid_median") if item else None + ), + "cm_input_spread": spread_value, + "cm_input_observed_value": observed_value, + "cm_input_value": observed_value, + "cm_input_transform_valid": observed_value is not None, + } + + +def _aggregate_value( + item: Mapping[str, Any] | None, prefix: str, aggregation: str +) -> float | None: + if item is None: + return None + return _optional_float(item.get(f"{prefix}_{aggregation}")) + + +def _apply_transforms( + frame: Any, profile: ClassicalModelInputProfile +) -> tuple[Any, dict[str, JSONValue]]: + import polars as pl + + if getattr(frame, "height", 0) < 1: + return frame, _empty_transform_policy_from_profile(profile) + rows = cast(list[dict[str, Any]], frame.to_dicts()) + invalid_domain_count = 0 + warmup_loss = 0 + groups: dict[tuple[str, str], list[int]] = {} + for index, row in enumerate(rows): + groups.setdefault( + (str(row["series_id"]), str(row["period"])), [] + ).append(index) + for indexes in groups.values(): + base: list[float | None] = [] + previous: float | None = None + for index in indexes: + value = _optional_float(rows[index].get("cm_input_observed_value")) + transformed: float | None + if value is None: + transformed = None + previous = None + elif profile.transform == "level": + transformed = value + elif profile.transform == "log_level": + transformed = math.log(value) if value > 0 else None + elif previous is None: + transformed = None + elif profile.transform == "return": + transformed = value / previous - 1 if previous != 0 else None + else: + transformed = ( + math.log(value / previous) + if value > 0 and previous > 0 + else None + ) + if value is not None and transformed is None: + if ( + profile.transform in {"log_level", "log_return"} + and value <= 0 + ): + invalid_domain_count += 1 + else: + warmup_loss += 1 + base.append(transformed) + if value is not None: + previous = value + values = base + for _ in range(profile.differencing_order): + values = _difference(values, 1) + for _ in range(profile.seasonal_differencing_order): + values = _difference(values, profile.seasonal_period) + for offset, index in enumerate(indexes): + original = base[offset] + value = values[offset] + if original is not None and value is None: + warmup_loss += 1 + rows[index]["cm_input_value"] = _rounded( + value, profile.rounding_digits + ) + rows[index]["cm_input_transform_valid"] = value is not None + result = pl.DataFrame(rows, infer_schema_length=None) + metadata = _empty_transform_policy_from_profile(profile) + metadata.update( + { + "invalid_domain_count": invalid_domain_count, + "warmup_loss": warmup_loss, + "transformed_observation_count": _valid_observation_count(result), + } + ) + return result, metadata + + +def _difference(values: Sequence[float | None], lag: int) -> list[float | None]: + output: list[float | None] = [] + for index, value in enumerate(values): + previous = values[index - lag] if index >= lag else None + output.append( + value - previous + if value is not None and previous is not None + else None + ) + return output + + +def _build_folds( + frame: Any, profile: ClassicalModelInputProfile +) -> list[dict[str, JSONValue]]: + if getattr(frame, "height", 0) < 1: + return [] + rows = cast(list[dict[str, Any]], frame.to_dicts()) + folds: list[dict[str, JSONValue]] = [] + groups: dict[tuple[str, str], list[int]] = {} + for index, row in enumerate(rows): + groups.setdefault( + (str(row["series_id"]), str(row["period"])), [] + ).append(index) + fold_id = 1 + for (series_id, period), indexes in sorted(groups.items()): + valid = [ + index + for index in indexes + if rows[index].get("cm_input_transform_valid") is True + ] + if len(valid) < ( + profile.minimum_training_observations + + profile.minimum_evaluation_observations + ): + continue + origins = valid[ + profile.minimum_training_observations - 1 :: profile.step_size + ] + for origin in origins: + future_valid = [index for index in valid if index > origin] + if len(future_valid) < profile.minimum_evaluation_observations: + continue + if profile.fold_kind == "rolling": + train_valid = [index for index in valid if index <= origin][ + -profile.rolling_window : + ] + else: + train_valid = [index for index in valid if index <= origin] + evaluation_start = origin + 1 + profile.embargo_observations + evaluation_end = min( + indexes[-1], + evaluation_start + + max( + profile.minimum_evaluation_observations, + max(profile.horizons), + ) + - 1, + ) + evaluation_valid = [ + index + for index in valid + if evaluation_start <= index <= evaluation_end + ] + if len(evaluation_valid) < profile.minimum_evaluation_observations: + continue + for horizon in profile.horizons: + target_index = origin + profile.embargo_observations + horizon + if target_index > indexes[-1]: + continue + target = rows[target_index] + if ( + profile.expected_closure_policy == "omit" + and target.get("cm_input_expected_closure") is True + ): + continue + status = ( + "valid" + if target.get("cm_input_transform_valid") is True + else "skipped" + ) + reason = "" if status == "valid" else "target_unavailable" + folds.append( + { + "schema_version": CLASSICAL_MODEL_FOLD_SCHEMA_VERSION, + "fold_id": fold_id, + "series_id": series_id, + "period": period, + "kind": profile.fold_kind, + "kind_code": MODEL_FOLD_KIND_CODES[profile.fold_kind], + "status": status, + "reason": reason or None, + "shuffle": False, + "future_values_visible": False, + "timestamp_required_as_identity": False, + "embargo_observations": profile.embargo_observations, + "horizon": horizon, + "training_observation_count": len(train_valid), + "training_start_index": train_valid[0], + "training_end_index": train_valid[-1], + "evaluation_start_index": evaluation_start, + "evaluation_end_index": evaluation_end, + "target_index": target_index, + "origin_bin_end_utc_ms": rows[origin][ + "cm_input_bin_end_utc_ms" + ], + "target_bin_end_utc_ms": target[ + "cm_input_bin_end_utc_ms" + ], + "origin_row_id": rows[origin].get( + "cm_input_source_last_row_id" + ), + "target_row_id": target.get( + "cm_input_source_last_row_id" + ), + "training_start_row_id": rows[train_valid[0]].get( + "cm_input_source_first_row_id" + ), + "training_end_row_id": rows[train_valid[-1]].get( + "cm_input_source_last_row_id" + ), + "evaluation_start_row_id": rows[evaluation_start].get( + "cm_input_source_first_row_id" + ), + "evaluation_end_row_id": rows[evaluation_end].get( + "cm_input_source_last_row_id" + ), + } + ) + fold_id += 1 + if len(folds) >= profile.resources.max_folds: + return folds + return folds + + +def _annotate_primary_folds( + frame: Any, folds: Sequence[Mapping[str, JSONValue]] +) -> Any: + import polars as pl + + if getattr(frame, "height", 0) < 1: + return frame + rows = cast(list[dict[str, Any]], frame.to_dicts()) + primary: dict[tuple[str, str, int], Mapping[str, JSONValue]] = {} + for fold in folds: + key = ( + _text(fold.get("series_id")), + _text(fold.get("period")), + _int(fold.get("target_bin_end_utc_ms")), + ) + primary.setdefault(key, fold) + for row in rows: + group = (str(row["series_id"]), str(row["period"])) + fold = primary.get( + ( + group[0], + group[1], + int(row["cm_input_bin_end_utc_ms"]), + ), + {}, + ) + row.update(_fold_projection_values(fold, row)) + return pl.DataFrame(rows, infer_schema_length=None) + + +def _fold_projection_values( + fold: Mapping[str, JSONValue], row: Mapping[str, Any] +) -> dict[str, Any]: + available = bool(fold) and fold.get("status") == "valid" + return { + "cm_fold_id": fold.get("fold_id"), + "cm_fold_kind_code": fold.get("kind_code"), + "cm_fold_origin_row_id": fold.get("origin_row_id"), + "cm_fold_target_row_id": fold.get("target_row_id"), + "cm_fold_horizon": fold.get("horizon"), + "cm_fold_training_start_row_id": fold.get("training_start_row_id"), + "cm_fold_training_end_row_id": fold.get("training_end_row_id"), + "cm_fold_evaluation_start_row_id": fold.get("evaluation_start_row_id"), + "cm_fold_evaluation_end_row_id": fold.get("evaluation_end_row_id"), + "cm_evaluation_status_code": ( + MODEL_EVALUATION_STATUS_CODES["contract_ready"] + if available + else MODEL_EVALUATION_STATUS_CODES["unavailable"] + ), + "cm_evaluation_target_available": available, + "cm_evaluation_forecast": None, + "cm_evaluation_actual": ( + row.get("cm_input_value") if available else None + ), + "cm_evaluation_error": None, + "cm_evaluation_diagnostic_only": available, + } + + +def _projection_frame(frame: Any, contract: Mapping[str, JSONValue]) -> Any: + import polars as pl + + status = _text(contract.get("status")) + reason = _text(contract.get("reason")) + derivation_id = _text(contract.get("derivation_id")) + configuration = _mapping(contract.get("configuration")) + selected = frame.with_columns( + [ + pl.lit(CLASSICAL_MODEL_INPUT_SCHEMA_VERSION).alias( + "cm_input_schema_version" + ), + pl.lit(derivation_id).alias("cm_input_derivation_id"), + pl.lit(MODEL_INPUT_STATUS_CODES.get(status, 0)) + .cast(pl.Int32) + .alias("cm_input_status_code"), + pl.lit(status == "ready").alias("cm_input_ready"), + pl.lit(MODEL_INPUT_REASON_CODES.get(reason, 99 if reason else 0)) + .cast(pl.Int32) + .alias("cm_input_exclusion_reason_code"), + pl.lit(_int(configuration.get("frequency_ms"))) + .cast(pl.Int64) + .alias("cm_input_frequency_ms"), + pl.lit( + MODEL_TRANSFORM_CODES.get( + _text(configuration.get("transform")), 0 + ) + ) + .cast(pl.Int32) + .alias("cm_input_transform_code"), + pl.lit(1).cast(pl.Int32).alias("cm_input_calculation_basis_code"), + pl.lit(CLASSICAL_MODEL_FOLD_SCHEMA_VERSION).alias( + "cm_fold_schema_version" + ), + pl.lit(CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION).alias( + "cm_evaluation_schema_version" + ), + pl.col("cm_input_transform_valid").alias("cm_input_available"), + ] + ) + keep = [ + "series_id", + "period", + "cm_input_available_at_utc_ms", + *[ + column + for column in CLASSICAL_MODEL_CONTRACT_COLUMNS + if column in selected.columns + ], + ] + return selected.select(list(dict.fromkeys(keep))) + + +def _with_empty_projection(frame: Any) -> Any: + import polars as pl + + expressions = [] + for name, dtype in _projection_dtypes().items(): + if name not in getattr(frame, "columns", ()): + expressions.append(pl.lit(None).cast(dtype).alias(name)) + return frame.with_columns(expressions) if expressions else frame + + +def _ensure_projection_dtypes(frame: Any) -> Any: + import polars as pl + + expressions = [] + for name, dtype in _projection_dtypes().items(): + if name in frame.columns: + expressions.append(pl.col(name).cast(dtype).alias(name)) + else: + expressions.append(pl.lit(None).cast(dtype).alias(name)) + return frame.with_columns(expressions) + + +def _projection_dtypes() -> dict[str, Any]: + import polars as pl + + strings = { + "cm_input_schema_version", + "cm_input_derivation_id", + "cm_fold_schema_version", + "cm_evaluation_schema_version", + } + booleans = { + "cm_input_ready", + "cm_input_available", + "cm_input_expected_closure", + "cm_input_unexpected_missing", + "cm_evaluation_target_available", + "cm_evaluation_diagnostic_only", + } + floats = { + "cm_input_observed_value", + "cm_input_value", + "cm_input_spread", + "cm_evaluation_forecast", + "cm_evaluation_actual", + "cm_evaluation_error", + } + return { + name: ( + pl.Utf8 + if name in strings + else ( + pl.Boolean + if name in booleans + else pl.Float64 if name in floats else pl.Int64 + ) + ) + for name in CLASSICAL_MODEL_CONTRACT_COLUMNS + } + + +def _fold_policy( + profile: ClassicalModelInputProfile, + folds: Sequence[Mapping[str, JSONValue]], +) -> dict[str, JSONValue]: + statuses = Counter(_text(fold.get("status")) for fold in folds) + retained = folds[: profile.resources.max_retained_diagnostics] + return { + "schema_version": CLASSICAL_MODEL_FOLD_SCHEMA_VERSION, + "kind": profile.fold_kind, + "shuffle": False, + "future_values_visible": False, + "timestamp_required_as_identity": False, + "minimum_training_observations": ( + profile.minimum_training_observations + ), + "minimum_evaluation_observations": ( + profile.minimum_evaluation_observations + ), + "step_size": profile.step_size, + "rolling_window": profile.rolling_window, + "embargo_observations": profile.embargo_observations, + "horizons": list(profile.horizons), + "horizon_unit": "regularized_grid_steps", + "incomplete_horizon_policy": "omit_unavailable_targets", + "fold_count": len(folds), + "valid_fold_count": statuses["valid"], + "skipped_fold_count": statuses["skipped"], + "folds_truncated": len(retained) < len(folds), + "fold_samples": [dict(fold) for fold in retained], + } + + +def _training_projection_metadata( + status: str, + reason: str | None, + profile: ClassicalModelInputProfile, + folds: Sequence[Mapping[str, JSONValue]], +) -> dict[str, JSONValue]: + return { + "schema_version": CLASSICAL_MODEL_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "row", + "identity_fields": ["series_id", "period", "row_id"], + "timestamp_required_as_identity": False, + "mapping_policy": "availability_safe_repetition_after_bin_close", + "column_names": list(CLASSICAL_MODEL_CONTRACT_COLUMNS), + "status_code": MODEL_INPUT_STATUS_CODES.get(status, 0), + "ready": status == "ready", + "reason_code": MODEL_INPUT_REASON_CODES.get( + reason or "", 99 if reason else 0 + ), + "frequency_ms": profile.frequency_ms, + "transform_code": MODEL_TRANSFORM_CODES[profile.transform], + "fold_count": len(folds), + "forecast_columns_populated": False, + "observed_columns_overwritten": False, + } + + +def _empty_regularization() -> dict[str, JSONValue]: + return { + "schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "basis": "regular_grid", + "regularized_observation_count": 0, + "observed_bin_count": 0, + "expected_closure_count": 0, + "unexpected_missing_count": 0, + "insufficient_observation_bin_count": 0, + "structurally_unavailable_count": 0, + "truncated": False, + "forward_fill_policy": "never", + } + + +def _empty_transform_policy( + base: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + return _empty_transform_policy_from_profile(_profile_from_base(base)) + + +def _empty_transform_policy_from_profile( + profile: ClassicalModelInputProfile, +) -> dict[str, JSONValue]: + inverse = { + "level": "identity", + "log_level": "exp", + "return": "compound_from_last_level", + "log_return": "exp_and_compound_from_last_level", + }[profile.transform] + return { + "transform": profile.transform, + "transform_code": MODEL_TRANSFORM_CODES[profile.transform], + "differencing_order": profile.differencing_order, + "seasonal_differencing_order": (profile.seasonal_differencing_order), + "seasonal_period": profile.seasonal_period, + "inverse_transform": inverse, + "original_scale_metrics_required": True, + "applied_explicitly": True, + "invalid_domain_count": 0, + "warmup_loss": 0, + "transformed_observation_count": 0, + "cross_period_state": False, + } + + +def _empty_fold_policy(base: Mapping[str, JSONValue]) -> dict[str, JSONValue]: + return _fold_policy(_profile_from_base(base), ()) + + +def _profile_from_base( + base: Mapping[str, JSONValue], +) -> ClassicalModelInputProfile: + configuration = _mapping(base.get("configuration")) + return ClassicalModelInputProfile( + enabled=bool(configuration.get("enabled", False)), + frequency_ms=max( + 1, + _int(configuration.get("frequency_ms")) + or DEFAULT_MODEL_FREQUENCY_MS, + ), + transform=_text(configuration.get("transform")) or "level", + horizons=tuple(_ints(configuration.get("horizons"))) + or DEFAULT_MODEL_HORIZONS, + ) + + +def _derivation_id( + frame: Any, + profile: ClassicalModelInputProfile, + fingerprint: Mapping[str, JSONValue], +) -> str: + samples: list[dict[str, JSONValue]] = [] + if getattr(frame, "height", 0): + for row in ( + frame.select( + "series_id", + "period", + "cm_input_bin_start_utc_ms", + "cm_input_observation_count", + "cm_input_value", + ) + .head(32) + .to_dicts() + ): + samples.append(cast(dict[str, JSONValue], row)) + payload = { + "schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "fingerprint_id": _reference_fingerprint_id(fingerprint), + "configuration": profile.to_metadata(), + "row_count": int(getattr(frame, "height", 0) or 0), + "samples": samples, + } + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("ascii") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _reference_fingerprint_id( + fingerprint: Mapping[str, JSONValue], +) -> str: + provided = _text(fingerprint.get("fingerprint_id")) + if provided: + return provided + snapshot = { + key: value + for key, value in fingerprint.items() + if key not in {"classical_model_input", "fingerprint_id"} + } + encoded = json.dumps( + snapshot, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("ascii") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _valid_observation_count(frame: Any) -> int: + if "cm_input_transform_valid" not in getattr(frame, "columns", ()): + return 0 + return int(frame.get_column("cm_input_transform_valid").sum() or 0) + + +def _bin_status_name(code: int) -> str: + for name, value in MODEL_BIN_STATUS_CODES.items(): + if value == code: + return name + return "unknown" + + +def _target_sort_key(value: Mapping[str, JSONValue]) -> tuple[str, ...]: + axis = _mapping(value.get("target_axis")) + return tuple( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + + +def _mapping(value: object) -> Mapping[str, JSONValue]: + return ( + cast(Mapping[str, JSONValue], value) + if isinstance(value, Mapping) + else {} + ) + + +def _mapping_rows(value: object) -> list[Mapping[str, JSONValue]]: + if not isinstance(value, list): + return [] + return [ + cast(Mapping[str, JSONValue], row) + for row in value + if isinstance(row, Mapping) + ] + + +def _text(value: object) -> str: + return str(value or "") + + +def _int(value: object) -> int: + try: + return int(cast(Any, value) or 0) + except (TypeError, ValueError): + return 0 + + +def _ints(value: object) -> list[int]: + if not isinstance(value, list): + return [] + return [_int(item) for item in value if _int(item) > 0] + + +def _optional_int(value: object) -> int | None: + if value is None: + return None + try: + return int(cast(Any, value)) + except (TypeError, ValueError): + return None + + +def _optional_float(value: object) -> float | None: + if value is None: + return None + try: + parsed = float(cast(Any, value)) + except (TypeError, ValueError): + return None + return parsed if math.isfinite(parsed) else None + + +def _rounded(value: float | None, digits: int) -> float | None: + return round(value, digits) if value is not None else None diff --git a/src/histdatacom/data_quality/contracts.py b/src/histdatacom/data_quality/contracts.py index b5b2b3fd..4dea2e21 100644 --- a/src/histdatacom/data_quality/contracts.py +++ b/src/histdatacom/data_quality/contracts.py @@ -198,6 +198,117 @@ def from_dict(cls, data: Mapping[str, Any]) -> "QualityTarget": ) +@dataclass(frozen=True, slots=True) +class QualitySkipEvent: + """Publish-safe evidence for one intentionally skipped rule evaluation.""" + + reason_code: str + rule_id: str + target_kind: QualityTargetKind = QualityTargetKind.UNKNOWN + data_format: str = "" + timeframe: str = "" + symbol: str = "" + period: str = "" + + def __post_init__(self) -> None: + """Normalize stable event vocabulary and target-axis fields.""" + object.__setattr__( + self, + "reason_code", + str(self.reason_code or "") + .strip() + .lower() + .replace("-", "_") + .replace(" ", "_"), + ) + object.__setattr__(self, "rule_id", str(self.rule_id or "").strip()) + object.__setattr__( + self, + "target_kind", + QualityTargetKind.from_value(self.target_kind), + ) + object.__setattr__( + self, + "data_format", + str(self.data_format or "").strip().lower(), + ) + object.__setattr__( + self, + "timeframe", + str(self.timeframe or "").strip().upper(), + ) + object.__setattr__( + self, + "symbol", + str(self.symbol or "").strip().upper(), + ) + object.__setattr__(self, "period", str(self.period or "").strip()) + + @classmethod + def from_target( + cls, + *, + reason_code: str, + rule_id: str, + target: QualityTarget, + ) -> "QualitySkipEvent": + """Create a skip event from stable, non-path target identity.""" + return cls( + reason_code=reason_code, + rule_id=rule_id, + target_kind=target.kind, + data_format=target.data_format, + timeframe=target.timeframe, + symbol=target.symbol, + period=target.period, + ) + + @property + def sort_key(self) -> tuple[str, str, str, str, str, str, str]: + """Return deterministic ordering independent of discovery order.""" + return ( + self.reason_code, + self.rule_id, + self.data_format, + self.timeframe, + self.symbol, + self.period, + self.target_kind.value, + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return the bounded path-free event representation.""" + return { + "reason_code": self.reason_code, + "rule_id": self.rule_id, + "target_kind": self.target_kind.value, + "target_axis": { + "data_format": self.data_format, + "timeframe": self.timeframe, + "symbol": self.symbol, + "period": self.period, + "kind": self.target_kind.value, + }, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "QualitySkipEvent": + """Restore a skip event from report metadata.""" + axis = data.get("target_axis") + target_axis = axis if isinstance(axis, Mapping) else {} + return cls( + reason_code=str(data.get("reason_code", "") or ""), + rule_id=str(data.get("rule_id", "") or ""), + target_kind=QualityTargetKind.from_value( + data.get("target_kind") or target_axis.get("kind") + ), + data_format=str(target_axis.get("data_format", "") or ""), + timeframe=str(target_axis.get("timeframe", "") or ""), + symbol=str(target_axis.get("symbol", "") or ""), + period=str(target_axis.get("period", "") or ""), + ) + + @dataclass(frozen=True, slots=True) class QualityLocation: """Optional record-level context for a quality finding.""" diff --git a/src/histdatacom/data_quality/engine.py b/src/histdatacom/data_quality/engine.py index c5628b63..25082fc6 100644 --- a/src/histdatacom/data_quality/engine.py +++ b/src/histdatacom/data_quality/engine.py @@ -10,6 +10,7 @@ QualityRule, QualityRuleResult, QualityRunRule, + QualitySkipEvent, QualityTarget, ) from histdatacom.data_quality.fingerprints import ( @@ -27,6 +28,10 @@ series_fingerprint_topology_attention_summary, series_fingerprint_topology_summary, ) +from histdatacom.data_quality.limits import ( + BoundedReportLimit, + bounded_report_limit, +) from histdatacom.data_quality.ticks import ( can_evaluate_tick_quality_bundle, evaluate_tick_quality_bundle, @@ -35,6 +40,15 @@ QualityProgressCallback = Callable[[Mapping[str, JSONValue]], None] +QUALITY_ENGINE_METADATA_KEY = "quality_engine" +QUALITY_ENGINE_SCHEMA_VERSION = "histdatacom.quality-engine.v1" +QUALITY_SKIP_EVENTS_SCHEMA_VERSION = "histdatacom.quality-skip-events.v1" +QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV = ( + "duplicate_archive_preferred_csv" +) +DEFAULT_QUALITY_SKIP_EVENT_LIMIT = 128 +DEFAULT_QUALITY_SKIP_COUNT_LIMIT = 64 + def evaluate_quality_rule( rule: QualityRule, @@ -62,8 +76,7 @@ def run_quality_assessment( run_rule_tuple = tuple(run_rules) base_metadata = dict(metadata or {}) csv_dimensions = _csv_target_dimensions(target_tuple) - skipped_duplicate_archive_rule_count = 0 - skipped_duplicate_archive_rule_counts: Counter[str] = Counter() + skip_events: list[QualitySkipEvent] = [] evaluation_plan: list[ tuple[int, QualityTarget, tuple[tuple[int, QualityRule], ...]] ] = [] @@ -77,8 +90,15 @@ def run_quality_assessment( target, csv_dimensions, ): - skipped_duplicate_archive_rule_count += 1 - skipped_duplicate_archive_rule_counts[str(rule.rule_id)] += 1 + skip_events.append( + QualitySkipEvent.from_target( + reason_code=( + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV + ), + rule_id=str(rule.rule_id), + target=target, + ) + ) rule_offset += 1 continue bundle_candidates = rule_tuple[rule_offset : rule_offset + 3] @@ -204,35 +224,32 @@ def run_quality_assessment( target=target, ) - if skipped_duplicate_archive_rule_count: - base_metadata["quality_engine"] = { - "target_count": len(target_tuple), - "rule_count": len(rule_tuple), - "target_rule_evaluation_count": len(rule_results_list), - "skipped_duplicate_archive_rule_evaluation_count": ( - skipped_duplicate_archive_rule_count - ), - "duplicate_archive_scan_policy": ( - "prefer_extracted_csv_for_non_inventory_rules" - ), - } rule_results = tuple(rule_results_list) - skipped_fingerprint_target_count = skipped_duplicate_archive_rule_counts[ - SERIES_FINGERPRINT_RULE_ID - ] + ordered_skip_events = tuple( + sorted(skip_events, key=lambda event: event.sort_key) + ) + if ordered_skip_events: + base_metadata[QUALITY_ENGINE_METADATA_KEY] = _quality_engine_metadata( + target_count=len(target_tuple), + rule_count=len(rule_tuple), + run_rule_count=len(run_rule_tuple), + executed_target_rule_count=len(rule_results_list), + skip_events=ordered_skip_events, + ) + fingerprint_skip_events = tuple( + event + for event in ordered_skip_events + if event.rule_id == SERIES_FINGERPRINT_RULE_ID + ) + skipped_fingerprint_target_count = len(fingerprint_skip_events) + skipped_fingerprint_reason_counts = Counter( + event.reason_code for event in fingerprint_skip_events + ) fingerprint_coverage_summary = series_fingerprint_coverage_summary( (finding for result in rule_results for finding in result.findings), discovered_target_count=len(target_tuple), skipped_fingerprint_target_count=skipped_fingerprint_target_count, - skipped_reason_counts=( - { - "duplicate_archive_preferred_csv": ( - skipped_fingerprint_target_count - ) - } - if skipped_fingerprint_target_count - else None - ), + skipped_reason_counts=skipped_fingerprint_reason_counts or None, ) if fingerprint_coverage_summary is not None: base_metadata[TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY] = ( @@ -342,6 +359,95 @@ def _merge_targets( return tuple(merged) +def _quality_engine_metadata( + *, + target_count: int, + rule_count: int, + run_rule_count: int, + executed_target_rule_count: int, + skip_events: tuple[QualitySkipEvent, ...], +) -> dict[str, JSONValue]: + skip_reason_counts = Counter(event.reason_code for event in skip_events) + duplicate_archive_skip_count = skip_reason_counts[ + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV + ] + planned_target_rule_count = target_count * rule_count + return { + "schema_version": QUALITY_ENGINE_SCHEMA_VERSION, + "target_count": target_count, + "rule_count": rule_count, + "run_rule_count": run_rule_count, + "planned_target_rule_evaluation_count": planned_target_rule_count, + "target_rule_evaluation_count": executed_target_rule_count, + "skipped_rule_evaluation_count": len(skip_events), + "skipped_duplicate_archive_rule_evaluation_count": ( + duplicate_archive_skip_count + ), + "duplicate_archive_scan_policy": ( + "prefer_extracted_csv_for_non_inventory_rules" + ), + "skip_events": _quality_skip_events_metadata(skip_events), + } + + +def _quality_skip_events_metadata( + skip_events: tuple[QualitySkipEvent, ...], +) -> dict[str, JSONValue]: + event_limit = bounded_report_limit( + None, + default_limit=DEFAULT_QUALITY_SKIP_EVENT_LIMIT, + maximum_limit=DEFAULT_QUALITY_SKIP_EVENT_LIMIT, + allow_unbounded=False, + ) + count_limit = bounded_report_limit( + None, + default_limit=DEFAULT_QUALITY_SKIP_COUNT_LIMIT, + maximum_limit=DEFAULT_QUALITY_SKIP_COUNT_LIMIT, + allow_unbounded=False, + ) + reason_counts = Counter(event.reason_code for event in skip_events) + rule_id_counts = Counter(event.rule_id for event in skip_events) + target_kind_counts = Counter( + event.target_kind.value for event in skip_events + ) + included_events = event_limit.slice(skip_events) + return { + "schema_version": QUALITY_SKIP_EVENTS_SCHEMA_VERSION, + "event_count": len(skip_events), + "included_event_count": len(included_events), + "omitted_event_count": max(0, len(skip_events) - len(included_events)), + "truncated": len(included_events) < len(skip_events), + "reason_counts": _bounded_quality_skip_counts( + reason_counts, + limit=count_limit, + ), + "rule_id_counts": _bounded_quality_skip_counts( + rule_id_counts, + limit=count_limit, + ), + "target_kind_counts": _bounded_quality_skip_counts( + target_kind_counts, + limit=count_limit, + ), + "events": [event.to_dict() for event in included_events], + "limit_metadata": { + "events": event_limit.count_payload(len(skip_events)), + "reasons": count_limit.count_payload(len(reason_counts)), + "rules": count_limit.count_payload(len(rule_id_counts)), + "target_kinds": count_limit.count_payload(len(target_kind_counts)), + }, + } + + +def _bounded_quality_skip_counts( + counts: Counter[str], + *, + limit: BoundedReportLimit, +) -> dict[str, JSONValue]: + ordered = sorted(counts.items(), key=lambda item: (-item[1], item[0])) + return {key: count for key, count in limit.slice(ordered)} + + def _csv_target_dimensions( targets: tuple[QualityTarget, ...], ) -> set[tuple[str, str, str, str]]: diff --git a/src/histdatacom/data_quality/exponential_smoothing.py b/src/histdatacom/data_quality/exponential_smoothing.py new file mode 100644 index 00000000..83959446 --- /dev/null +++ b/src/histdatacom/data_quality/exponential_smoothing.py @@ -0,0 +1,2086 @@ +"""Optional exponential-smoothing models over classical model contracts. + +The implementation is deliberately family-scoped. It consumes the regular +grid and rolling-origin folds from :mod:`classical_model_contracts`, fits only +explicitly configured specifications, and emits bounded advisory diagnostics. +The rich numerical backend is imported lazily so core fingerprinting remains a +low-dependency path. +""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.metadata +import json +import math +import re +import time +import warnings +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from statistics import median +from typing import Any, cast + +from histdatacom.data_quality.calendar import classify_histdata_timestamp +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + ClassicalModelInputProfile, + ClassicalModelInputResult, + build_classical_model_input, +) +from histdatacom.data_quality.contracts import QualityFinding +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.training_features import ( + EXPONENTIAL_SMOOTHING_COLUMNS, + ensure_tick_training_features, +) +from histdatacom.runtime_contracts import JSONValue + +EXPONENTIAL_SMOOTHING_SCHEMA_VERSION = "histdatacom.exponential-smoothing.v1" +EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION = ( + "histdatacom.exponential-smoothing-configuration.v1" +) +EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION = ( + "histdatacom.exponential-smoothing-fit-result.v1" +) +EXPONENTIAL_SMOOTHING_FORECAST_SCHEMA_VERSION = ( + "histdatacom.exponential-smoothing-forecast.v1" +) +EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION = ( + "histdatacom.exponential-smoothing-evaluation.v1" +) +EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.exponential-smoothing-training-projection.v1" +) +EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.exponential-smoothing-summary.v1" +) +EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_exponential_smoothing_summary" +) +EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY = "fingerprint_exponential_smoothing" + +DEFAULT_EXPONENTIAL_SMOOTHING_SUMMARY_TARGET_LIMIT = 16 +DEFAULT_EXPONENTIAL_SMOOTHING_ROLLING_WINDOWS = (5, 20) +DEFAULT_EXPONENTIAL_SMOOTHING_ROUNDING_DIGITS = 12 +MAX_EXPONENTIAL_SMOOTHING_SPECIFICATIONS = 16 +MAX_EXPONENTIAL_SMOOTHING_PARAMETER_BOUNDS = 32 + +EXPONENTIAL_SMOOTHING_FAMILIES = ( + "ses", + "holt", + "holt_winters", + "ets", +) +EXPONENTIAL_SMOOTHING_FAMILY_CODES = { + "ses": 1, + "holt": 2, + "holt_winters": 3, + "ets": 4, +} +EXPONENTIAL_SMOOTHING_COMPONENT_CODES = {"none": 0, "add": 1, "mul": 2} +EXPONENTIAL_SMOOTHING_INITIALIZATION_CODES = { + "estimated": 1, + "heuristic": 2, + "legacy-heuristic": 3, + "known": 4, +} +EXPONENTIAL_SMOOTHING_FIT_STATUS_CODES = { + "unavailable": 1, + "skipped": 2, + "failed": 3, + "limited": 4, + "fitted": 5, + "converged": 6, +} +EXPONENTIAL_SMOOTHING_REASON_CODES = { + "": 0, + "dependency_unavailable": 1, + "input_contract_unavailable": 2, + "insufficient_folds": 3, + "insufficient_data": 4, + "insufficient_seasonal_cycles": 5, + "invalid_multiplicative_domain": 6, + "invalid_configuration": 7, + "optimizer_failure": 8, + "numerical_failure": 9, + "resource_limit": 10, + "timeout": 11, + "backend_failure": 12, + "target_unavailable": 13, + "inverse_transform_unavailable": 14, +} +EXPONENTIAL_SMOOTHING_WARNING_CODES = { + "ConvergenceWarning": "convergence_warning", + "RuntimeWarning": "runtime_warning", + "ValueWarning": "value_warning", + "UserWarning": "user_warning", +} +EXPONENTIAL_SMOOTHING_BASELINE_CODES = { + "naive_random_walk": 1, + "rolling_mean": 2, + "rolling_median": 3, + "session_seasonal_naive": 4, +} + +_SPECIFICATION_ID = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") +_COMPONENTS = {"none", "add", "mul"} +_INITIALIZATION_METHODS = { + "estimated", + "heuristic", + "legacy-heuristic", + "known", +} +_BOUND_PARAMETER_NAMES = { + "smoothing_level", + "smoothing_trend", + "smoothing_seasonal", + "damping_trend", + "initial_level", + "initial_trend", +} + + +@dataclass(frozen=True, slots=True) +class ExponentialSmoothingSpecification: + """One explicit exponential-smoothing specification.""" + + specification_id: str = "ses" + family: str = "ses" + level: bool = True + error: str = "add" + trend: str = "none" + damped_trend: bool = False + seasonal: str = "none" + seasonal_periods: int = 0 + initialization_method: str = "estimated" + initial_level: float | None = None + initial_trend: float | None = None + initial_seasonal: tuple[float, ...] = () + optimized: bool = True + method: str = "" + use_brute: bool = False + remove_bias: bool = False + smoothing_level: float | None = None + smoothing_trend: float | None = None + smoothing_seasonal: float | None = None + damping_trend: float | None = None + parameter_bounds: tuple[tuple[str, float, float], ...] = () + max_iterations: int = 200 + + def __post_init__(self) -> None: + if not _SPECIFICATION_ID.fullmatch(self.specification_id): + raise ValueError("invalid exponential-smoothing specification_id") + if self.family not in EXPONENTIAL_SMOOTHING_FAMILIES: + raise ValueError("unsupported exponential-smoothing family") + if self.level is not True: + raise ValueError( + "exponential-smoothing level component is required" + ) + if self.error not in {"add", "mul"}: + raise ValueError("error must be add or mul") + if self.trend not in _COMPONENTS or self.seasonal not in _COMPONENTS: + raise ValueError("trend and seasonal must be none, add, or mul") + if self.initialization_method not in _INITIALIZATION_METHODS: + raise ValueError("unsupported initialization_method") + if self.family == "ses" and ( + self.trend != "none" or self.seasonal != "none" or self.damped_trend + ): + raise ValueError( + "SES cannot configure trend, damping, or seasonality" + ) + if self.family == "holt" and ( + self.trend == "none" or self.seasonal != "none" + ): + raise ValueError( + "Holt requires trend and cannot configure seasonality" + ) + if self.family == "holt_winters" and self.seasonal == "none": + raise ValueError( + "Holt-Winters requires an explicit seasonal component" + ) + if self.damped_trend and self.trend == "none": + raise ValueError("damped_trend requires a trend component") + if self.seasonal == "none" and self.seasonal_periods != 0: + raise ValueError("seasonal_periods requires a seasonal component") + if self.seasonal != "none" and self.seasonal_periods < 2: + raise ValueError("seasonal_periods must be at least 2") + if self.family == "ets" and not self.optimized: + raise ValueError("ETSModel requires optimized fitting") + if self.initialization_method == "known": + if self.initial_level is None: + raise ValueError("known initialization requires initial_level") + if self.trend != "none" and self.initial_trend is None: + raise ValueError( + "known trend initialization requires initial_trend" + ) + if ( + self.seasonal != "none" + and len(self.initial_seasonal) != self.seasonal_periods + ): + raise ValueError( + "known seasonal initialization must match seasonal_periods" + ) + for name in ( + "smoothing_level", + "smoothing_trend", + "smoothing_seasonal", + "damping_trend", + ): + value = getattr(self, name) + if value is not None and not 0.0 <= value <= 1.0: + raise ValueError(f"{name} must be between 0 and 1") + if self.max_iterations < 1: + raise ValueError("max_iterations must be positive") + if ( + len(self.parameter_bounds) + > MAX_EXPONENTIAL_SMOOTHING_PARAMETER_BOUNDS + ): + raise ValueError("too many exponential-smoothing parameter bounds") + names: set[str] = set() + for name, lower, upper in self.parameter_bounds: + if name not in _BOUND_PARAMETER_NAMES and not name.startswith( + "initial_seasonal." + ): + raise ValueError(f"unsupported parameter bound: {name}") + if ( + name in names + or not math.isfinite(lower) + or not math.isfinite(upper) + ): + raise ValueError("parameter bounds must be unique and finite") + if lower >= upper: + raise ValueError( + "parameter bound lower value must be below upper" + ) + names.add(name) + + def to_metadata(self) -> dict[str, JSONValue]: + """Return a stable JSON-compatible specification.""" + return { + "schema_version": EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION, + "specification_id": self.specification_id, + "family": self.family, + "family_code": EXPONENTIAL_SMOOTHING_FAMILY_CODES[self.family], + "level": self.level, + "error": self.error, + "trend": self.trend, + "damped_trend": self.damped_trend, + "seasonal": self.seasonal, + "seasonal_periods": self.seasonal_periods, + "initialization_method": self.initialization_method, + "initial_level": self.initial_level, + "initial_trend": self.initial_trend, + "initial_seasonal": cast(JSONValue, list(self.initial_seasonal)), + "optimized": self.optimized, + "method": self.method or None, + "use_brute": self.use_brute, + "remove_bias": self.remove_bias, + "smoothing_level": self.smoothing_level, + "smoothing_trend": self.smoothing_trend, + "smoothing_seasonal": self.smoothing_seasonal, + "damping_trend": self.damping_trend, + "parameter_bounds": cast( + JSONValue, + [ + {"parameter": name, "lower": lower, "upper": upper} + for name, lower, upper in self.parameter_bounds + ], + ), + "max_iterations": self.max_iterations, + "automatic_search": False, + } + + +def _default_specifications() -> tuple[ExponentialSmoothingSpecification, ...]: + return (ExponentialSmoothingSpecification(),) + + +@dataclass(frozen=True, slots=True) +class ExponentialSmoothingProfile: + """Explicit fitted-family controls; disabled by default.""" + + enabled: bool = False + specifications: tuple[ExponentialSmoothingSpecification, ...] = field( + default_factory=_default_specifications + ) + projection_specification_id: str = "ses" + projection_horizon: int = 1 + baseline_rolling_windows: tuple[int, ...] = ( + DEFAULT_EXPONENTIAL_SMOOTHING_ROLLING_WINDOWS + ) + rounding_digits: int = DEFAULT_EXPONENTIAL_SMOOTHING_ROUNDING_DIGITS + + def __post_init__(self) -> None: + if not self.specifications: + raise ValueError( + "at least one exponential-smoothing specification is required" + ) + if len(self.specifications) > MAX_EXPONENTIAL_SMOOTHING_SPECIFICATIONS: + raise ValueError("too many exponential-smoothing specifications") + identifiers = tuple( + spec.specification_id for spec in self.specifications + ) + if len(set(identifiers)) != len(identifiers): + raise ValueError( + "exponential-smoothing specification IDs must be unique" + ) + if self.projection_specification_id not in identifiers: + raise ValueError( + "projection_specification_id must select a specification" + ) + if self.projection_horizon < 1: + raise ValueError("projection_horizon must be positive") + if ( + not self.baseline_rolling_windows + or any(value < 1 for value in self.baseline_rolling_windows) + or tuple(sorted(set(self.baseline_rolling_windows))) + != self.baseline_rolling_windows + ): + raise ValueError( + "baseline_rolling_windows must be sorted unique positives" + ) + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between 0 and 16") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return stable profile metadata.""" + return { + "enabled": self.enabled, + "specifications": cast( + JSONValue, [spec.to_metadata() for spec in self.specifications] + ), + "projection_specification_id": self.projection_specification_id, + "projection_horizon": self.projection_horizon, + "baseline_rolling_windows": list(self.baseline_rolling_windows), + "rounding_digits": self.rounding_digits, + "automatic_search": False, + "automatic_winner": False, + } + + +@dataclass(frozen=True, slots=True) +class ExponentialSmoothingResult: + """Bounded diagnostics plus durable row-key annotations.""" + + diagnostics: Mapping[str, JSONValue] + annotations: tuple[Mapping[str, Any], ...] + input_result: ClassicalModelInputResult + + +@dataclass(frozen=True, slots=True) +class _Backend: + version: str + holt_winters: Any + ets_model: Any + + +@dataclass(frozen=True, slots=True) +class _FitOutcome: + status: str + reason: str + forecasts: tuple[float, ...] + parameters: Mapping[str, float] + warning_codes: tuple[str, ...] + converged: bool + fit_seconds: float + + +def exponential_smoothing_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: ExponentialSmoothingProfile | None = None, + target: Any | None = None, +) -> ExponentialSmoothingResult: + """Regularize the enriched tick frame and evaluate configured models.""" + selected_input = input_profile or ClassicalModelInputProfile(enabled=True) + input_result = build_classical_model_input( + frame, + fingerprint, + profile=selected_input, + target=target, + ) + return exponential_smoothing_from_model_input( + frame, + input_result, + fingerprint, + input_profile=selected_input, + profile=profile, + target=target, + ) + + +def exponential_smoothing_from_model_input( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile, + profile: ExponentialSmoothingProfile | None = None, + target: Any | None = None, +) -> ExponentialSmoothingResult: + """Fit explicit exponential-smoothing specifications over #421 folds.""" + selected = profile or ExponentialSmoothingProfile(enabled=True) + base = _base_payload(input_result, fingerprint, selected) + if selected.projection_horizon not in input_profile.horizons: + return _unavailable_result( + input_result, + base, + "invalid_configuration", + ) + if input_result.contract.get("status") == "unavailable": + return _unavailable_result( + input_result, + base, + "input_contract_unavailable", + ) + if not input_result.folds: + return _unavailable_result( + input_result, + base, + "insufficient_folds", + status="limited", + ) + backend = _load_backend() + if backend is None: + return _unavailable_result( + input_result, + base, + "dependency_unavailable", + ) + return _evaluate_models( + frame, + input_result, + fingerprint, + input_profile, + selected, + backend, + target=target, + ) + + +def exponential_smoothing_diagnostics_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: ExponentialSmoothingProfile | None = None, + target: Any | None = None, +) -> dict[str, JSONValue]: + """Return only the serializable fitted-family diagnostics.""" + return dict( + exponential_smoothing_from_training_frame( + frame, + fingerprint, + input_profile=input_profile, + profile=profile, + target=target, + ).diagnostics + ) + + +def project_exponential_smoothing_onto_training_frame( + frame: Any, + result: ExponentialSmoothingResult, + *, + target: Any | None = None, +) -> Any: + """Join bounded ETS annotations by durable row identity only.""" + import polars as pl + + columns = set(getattr(frame, "columns", ())) + if not {"series_id", "period", "row_id"}.issubset(columns): + enriched = ensure_tick_training_features(frame, target=target) + else: + enriched = frame + left = enriched.drop( + [ + name + for name in EXPONENTIAL_SMOOTHING_COLUMNS + if name in enriched.columns + ] + ).with_row_index("__cm_ets_original_order") + if result.annotations: + right = pl.DataFrame( + [dict(row) for row in result.annotations], + infer_schema_length=None, + ) + projected = left.join( + right, + on=["series_id", "period", "row_id"], + how="left", + validate="m:1", + ) + else: + projected = left + projected = _ensure_projection_columns(projected) + return projected.sort("__cm_ets_original_order").drop( + "__cm_ets_original_order" + ) + + +def exponential_smoothing_summary( + findings: Iterable[QualityFinding], + *, + target_limit: ( + int | None + ) = DEFAULT_EXPONENTIAL_SMOOTHING_SUMMARY_TARGET_LIMIT, +) -> dict[str, JSONValue] | None: + """Return bounded report metadata for exponential-smoothing results.""" + targets: list[dict[str, JSONValue]] = [] + statuses: Counter[str] = Counter() + for finding in findings: + fingerprint = _mapping(finding.metadata.get("time_series_fingerprint")) + payload = _mapping(fingerprint.get("exponential_smoothing")) + if not payload: + continue + status = _text(payload.get("status")) or "unavailable" + evaluation = _mapping(payload.get("evaluation")) + fit_summary = _mapping(payload.get("fit_summary")) + statuses[status] += 1 + targets.append( + { + "target_axis": dict(_mapping(payload.get("target_axis"))), + "status": status, + "reason": payload.get("reason"), + "model_count": _int(evaluation.get("model_count")), + "fit_attempt_count": _int(fit_summary.get("fit_attempt_count")), + "evaluated_fold_count": _int( + evaluation.get("evaluated_fold_count") + ), + "failed_fit_count": _int(fit_summary.get("failed_fit_count")), + } + ) + if not targets: + return None + targets.sort(key=_target_sort_key) + limit = bounded_report_limit( + target_limit, + default_limit=DEFAULT_EXPONENTIAL_SMOOTHING_SUMMARY_TARGET_LIMIT, + allow_unbounded=True, + ) + included = limit.slice(targets) + omitted = len(targets) - len(included) + return { + "schema_version": EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION, + "advisory": True, + "target_count": len(targets), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "status_counts": dict(sorted(statuses.items())), + "target_summaries": cast(JSONValue, included), + "limit_metadata": {"targets": limit.limit_payload()}, + } + + +def format_exponential_smoothing_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> tuple[str, ...]: + """Return concise human-readable fitted-family lines.""" + if not summary: + return () + statuses = _mapping(summary.get("status_counts")) + lines = [ + "", + "Exponential-smoothing models", + ( + f"targets: {_int(summary.get('target_count'))} " + f"ready: {_int(statuses.get('ready'))} " + f"limited: {_int(statuses.get('limited'))} " + f"unavailable: {_int(statuses.get('unavailable'))}" + ), + ] + for target in _mapping_rows(summary.get("target_summaries")): + axis = _mapping(target.get("target_axis")) + label = "/".join( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + lines.append( + f"- {label}: {_text(target.get('status'))} " + f"models={_int(target.get('model_count'))} " + f"fits={_int(target.get('fit_attempt_count'))} " + f"folds={_int(target.get('evaluated_fold_count'))}" + ) + if summary.get("truncated") is True: + lines.append( + f"- {_int(summary.get('omitted_target_count'))} targets omitted" + ) + return tuple(lines) + + +def _evaluate_models( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + input_profile: ClassicalModelInputProfile, + profile: ExponentialSmoothingProfile, + backend: _Backend, + *, + target: Any | None, +) -> ExponentialSmoothingResult: + rows = cast(list[dict[str, Any]], input_result.regularized_frame.to_dicts()) + folds = [dict(fold) for fold in input_result.folds] + origins = _folds_by_origin(folds) + resource_policy = input_profile.resources + specifications = profile.specifications[ + : resource_policy.max_candidate_orders + ] + limitations: list[str] = [] + estimated_working_memory_bytes = _estimated_working_memory_bytes( + len(rows), len(folds), len(specifications) + ) + if len(specifications) < len(profile.specifications): + limitations.append("resource_limit") + if estimated_working_memory_bytes > resource_policy.max_memory_bytes: + limitations.append("resource_limit") + specifications = () + started = time.monotonic() + fit_attempt_count = 0 + all_evaluations: list[dict[str, Any]] = [] + model_payloads: list[dict[str, JSONValue]] = [] + all_fit_samples: list[dict[str, JSONValue]] = [] + fit_statuses: Counter[str] = Counter() + fit_reasons: Counter[str] = Counter() + warning_counts: Counter[str] = Counter() + + for specification_code, specification in enumerate(specifications, start=1): + model_id = _model_id( + specification, + _text(input_result.contract.get("derivation_id")), + backend.version, + ) + model_evaluations: list[dict[str, Any]] = [] + model_fit_samples: list[dict[str, JSONValue]] = [] + for origin_key, origin_folds in origins: + if fit_attempt_count >= resource_policy.max_fit_attempts: + limitations.append("resource_limit") + break + if ( + time.monotonic() - started + >= resource_policy.max_wall_time_seconds + ): + limitations.append("timeout") + break + fit_attempt_count += 1 + origin = origin_folds[0] + training_start = _int(origin.get("training_start_index")) + training_end = _int(origin.get("training_end_index")) + segment_indexes = _trailing_contiguous_indexes( + rows, + training_start, + training_end, + _text(origin.get("series_id")), + _text(origin.get("period")), + ) + training_values = tuple( + _float(rows[index].get("cm_input_value")) + for index in segment_indexes + ) + max_horizon = max( + _int(fold.get("horizon")) for fold in origin_folds + ) + fit_outcome = _fit_specification( + specification, + training_values, + max_horizon, + backend, + minimum_required=_minimum_required_observations(specification), + ) + fit_statuses[fit_outcome.status] += 1 + if fit_outcome.reason: + fit_reasons[fit_outcome.reason] += 1 + warning_counts.update(fit_outcome.warning_codes) + fit_sample = _fit_sample( + fit_outcome, + specification, + model_id, + origin, + segment_indexes, + len(training_values), + ) + model_fit_samples.append(fit_sample) + all_fit_samples.append(fit_sample) + original_forecasts = _inverse_forecasts( + rows, + training_end, + fit_outcome.forecasts, + input_profile, + profile.rounding_digits, + ) + for fold in origin_folds: + evaluation = _fold_evaluation( + rows, + fold, + specification, + specification_code, + model_id, + fit_outcome, + original_forecasts, + profile.rounding_digits, + ) + model_evaluations.append(evaluation) + all_evaluations.append(evaluation) + model_payloads.append( + _model_evaluation_payload( + specification, + specification_code, + model_id, + model_evaluations, + model_fit_samples, + resource_policy.max_retained_diagnostics, + profile.rounding_digits, + ) + ) + if limitations and limitations[-1] in {"resource_limit", "timeout"}: + break + + references = _reference_baseline_payloads( + rows, + folds, + profile.baseline_rolling_windows, + profile.rounding_digits, + ) + annotations, projection_collisions = _build_annotations( + frame, + all_evaluations, + profile, + input_result, + target=target, + ) + evaluated_count = sum( + evaluation.get("status") == "evaluated" + for evaluation in all_evaluations + ) + skipped_evaluation_count = len(all_evaluations) - evaluated_count + failed_count = sum( + status in {"failed", "unavailable"} + for status in fit_statuses.elements() + ) + limited_fit_count = fit_statuses["limited"] + fit_statuses["skipped"] + if not evaluated_count: + limitations.append("insufficient_data") + limitations = list(dict.fromkeys(limitations)) + status = ( + "ready" + if not limitations and failed_count == 0 and limited_fit_count == 0 + else "limited" + ) + reason = ( + limitations[0] + if limitations + else ( + sorted(fit_reasons)[0] + if fit_reasons + else ("backend_failure" if failed_count else None) + ) + ) + if reason and status == "ready": + status = "limited" + diagnostics: dict[str, JSONValue] = { + **_base_payload(input_result, fingerprint, profile), + "status": status, + "reason": reason, + "limitations": cast(JSONValue, limitations), + "backend": { + "provider": "statsmodels", + "version": backend.version, + "available": True, + "import_basis": "optional_models_extra", + }, + "fit_summary": { + "schema_version": EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION, + "fit_attempt_count": fit_attempt_count, + "status_counts": dict(sorted(fit_statuses.items())), + "reason_counts": dict(sorted(fit_reasons.items())), + "warning_counts": dict(sorted(warning_counts.items())), + "failed_fit_count": failed_count, + "limited_fit_count": limited_fit_count, + "fit_samples": cast( + JSONValue, + all_fit_samples[: resource_policy.max_retained_diagnostics], + ), + "fit_samples_truncated": ( + len(all_fit_samples) > resource_policy.max_retained_diagnostics + ), + }, + "resource_usage": { + "limits": resource_policy.to_metadata(), + "estimated_working_memory_bytes": estimated_working_memory_bytes, + "memory_limit_exceeded": ( + estimated_working_memory_bytes + > resource_policy.max_memory_bytes + ), + "fit_attempt_count": fit_attempt_count, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "evaluation": { + "schema_version": EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin", + "original_scale": True, + "model_count": len(model_payloads), + "fold_count": len(folds), + "evaluated_fold_count": evaluated_count, + "skipped_evaluation_count": skipped_evaluation_count, + "models": cast(JSONValue, model_payloads), + "reference_baselines": cast(JSONValue, references), + "comparison_semantics": "descriptive_shared_folds_only", + "automatic_winner": False, + }, + "training_projection": _training_projection_metadata( + profile, + input_result, + len(annotations), + projection_collisions, + ), + "fit_duration_included": False, + } + return ExponentialSmoothingResult( + diagnostics=diagnostics, + annotations=annotations, + input_result=input_result, + ) + + +def _folds_by_origin( + folds: Sequence[Mapping[str, Any]], +) -> list[tuple[tuple[str, str, int], list[dict[str, Any]]]]: + grouped: dict[tuple[str, str, int], list[dict[str, Any]]] = {} + for fold in folds: + key = ( + _text(fold.get("series_id")), + _text(fold.get("period")), + _int(fold.get("origin_bin_end_utc_ms")), + ) + grouped.setdefault(key, []).append(dict(fold)) + return [ + (key, sorted(values, key=lambda item: _int(item.get("horizon")))) + for key, values in sorted(grouped.items()) + ] + + +def _estimated_working_memory_bytes( + row_count: int, + fold_count: int, + specification_count: int, +) -> int: + """Return a conservative deterministic bound for fitted-family work arrays.""" + row_storage = row_count * 16 * 8 + fold_storage = fold_count * 64 * 8 + model_storage = specification_count * 512 * 8 + return row_storage + fold_storage + model_storage + + +def _trailing_contiguous_indexes( + rows: Sequence[Mapping[str, Any]], + start: int, + end: int, + series_id: str, + period: str, +) -> tuple[int, ...]: + selected: list[int] = [] + for index in range(end, start - 1, -1): + row = rows[index] + if ( + _text(row.get("series_id")) != series_id + or _text(row.get("period")) != period + ): + break + value = _optional_float(row.get("cm_input_value")) + if value is None or row.get("cm_input_transform_valid") is not True: + if selected: + break + continue + selected.append(index) + return tuple(reversed(selected)) + + +def _fit_specification( + specification: ExponentialSmoothingSpecification, + values: Sequence[float], + horizon: int, + backend: _Backend, + *, + minimum_required: int, +) -> _FitOutcome: + if len(values) < minimum_required: + reason = ( + "insufficient_seasonal_cycles" + if specification.seasonal != "none" + else "insufficient_data" + ) + return _FitOutcome("skipped", reason, (), {}, (), False, 0.0) + if _uses_multiplicative_component(specification) and any( + value <= 0 for value in values + ): + return _FitOutcome( + "skipped", + "invalid_multiplicative_domain", + (), + {}, + (), + False, + 0.0, + ) + started = time.monotonic() + captured: list[warnings.WarningMessage] + try: + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + fitted = _statsmodels_fit(specification, values, backend) + raw_forecasts = fitted.forecast(horizon) + forecasts = tuple(float(value) for value in raw_forecasts) + if not forecasts or any( + not math.isfinite(value) for value in forecasts + ): + return _FitOutcome( + "failed", + "numerical_failure", + (), + {}, + _warning_codes(captured), + False, + time.monotonic() - started, + ) + warning_codes = _warning_codes(captured) + converged = _fit_converged( + fitted, warning_codes, specification.optimized + ) + status = ( + "limited" + if "convergence_warning" in warning_codes + else "converged" if converged else "fitted" + ) + reason = "optimizer_failure" if status == "limited" else "" + return _FitOutcome( + status, + reason, + forecasts, + _fitted_parameters(fitted), + warning_codes, + converged, + time.monotonic() - started, + ) + except (ArithmeticError, FloatingPointError, OverflowError): + return _FitOutcome( + "failed", + "numerical_failure", + (), + {}, + (), + False, + time.monotonic() - started, + ) + except (TypeError, ValueError) as exc: + return _FitOutcome( + "failed", + _configuration_failure_reason(exc), + (), + {}, + (), + False, + time.monotonic() - started, + ) + except Exception: # pragma: no cover - backend-specific safety boundary + return _FitOutcome( + "failed", + "backend_failure", + (), + {}, + (), + False, + time.monotonic() - started, + ) + + +def _statsmodels_fit( + specification: ExponentialSmoothingSpecification, + values: Sequence[float], + backend: _Backend, +) -> Any: + trend = None if specification.trend == "none" else specification.trend + seasonal = ( + None if specification.seasonal == "none" else specification.seasonal + ) + bounds = { + name: (lower, upper) + for name, lower, upper in specification.parameter_bounds + } + initialization = { + "initialization_method": specification.initialization_method, + "initial_level": specification.initial_level, + "initial_trend": specification.initial_trend, + "initial_seasonal": ( + list(specification.initial_seasonal) + if specification.initial_seasonal + else None + ), + } + if specification.family == "ets": + model = backend.ets_model( + values, + error=specification.error, + trend=trend, + damped_trend=specification.damped_trend, + seasonal=seasonal, + seasonal_periods=(specification.seasonal_periods or None), + bounds=(bounds or None), + missing="raise", + **initialization, + ) + return model.fit( + maxiter=specification.max_iterations, + disp=False, + ) + model = backend.holt_winters( + values, + trend=trend, + damped_trend=specification.damped_trend, + seasonal=seasonal, + seasonal_periods=(specification.seasonal_periods or None), + bounds=(bounds or None), + missing="raise", + use_boxcox=False, + **initialization, + ) + minimize_kwargs: dict[str, Any] | None = None + if specification.optimized: + minimize_kwargs = {"options": {"maxiter": specification.max_iterations}} + return model.fit( + smoothing_level=specification.smoothing_level, + smoothing_trend=specification.smoothing_trend, + smoothing_seasonal=specification.smoothing_seasonal, + damping_trend=specification.damping_trend, + optimized=specification.optimized, + remove_bias=specification.remove_bias, + method=(specification.method or None), + minimize_kwargs=minimize_kwargs, + use_brute=specification.use_brute, + ) + + +def _minimum_required_observations( + specification: ExponentialSmoothingSpecification, +) -> int: + if specification.seasonal != "none": + return max(4, 2 * specification.seasonal_periods) + if specification.trend != "none": + return 3 + return 2 + + +def _uses_multiplicative_component( + specification: ExponentialSmoothingSpecification, +) -> bool: + return "mul" in { + specification.error, + specification.trend, + specification.seasonal, + } + + +def _warning_codes( + captured: Sequence[warnings.WarningMessage], +) -> tuple[str, ...]: + values = { + EXPONENTIAL_SMOOTHING_WARNING_CODES.get( + item.category.__name__, "backend_warning" + ) + for item in captured + } + return tuple(sorted(values)) + + +def _fit_converged( + fitted: Any, + warning_codes: Sequence[str], + optimized: bool, +) -> bool: + if not optimized or "convergence_warning" in warning_codes: + return False + result = getattr(fitted, "mle_retvals", None) + if result is None: + return True + if isinstance(result, Mapping): + value = result.get("converged", result.get("success")) + else: + value = getattr(result, "success", getattr(result, "converged", None)) + return True if value is None else bool(value) + + +def _fitted_parameters(fitted: Any) -> dict[str, float]: + raw = getattr(fitted, "params", {}) + values: dict[str, float] = {} + items: Iterable[tuple[Any, Any]] + if isinstance(raw, Mapping): + items = raw.items() + else: + names = list(getattr(getattr(fitted, "model", None), "param_names", ())) + items = zip(names, raw, strict=False) + for name, value in items: + scalar = _optional_float(value) + if scalar is not None: + values[str(name)] = scalar + return dict(sorted(values.items())) + + +def _configuration_failure_reason(exc: Exception) -> str: + message = str(exc).lower() + if "seasonal" in message and ("cycle" in message or "period" in message): + return "insufficient_seasonal_cycles" + if "positive" in message or "strictly > 0" in message: + return "invalid_multiplicative_domain" + if "bound" in message or "initial" in message: + return "invalid_configuration" + return "backend_failure" + + +def _fit_sample( + outcome: _FitOutcome, + specification: ExponentialSmoothingSpecification, + model_id: str, + fold: Mapping[str, Any], + segment_indexes: Sequence[int], + observation_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION, + "model_id": model_id, + "specification_id": specification.specification_id, + "status": outcome.status, + "reason": outcome.reason or None, + "converged": outcome.converged, + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "observation_count": observation_count, + "segment_start_index": segment_indexes[0] if segment_indexes else None, + "segment_end_index": segment_indexes[-1] if segment_indexes else None, + "training_segment_policy": "trailing_contiguous_after_missing", + "parameters": dict(outcome.parameters), + "parameter_count": len(outcome.parameters), + "warning_codes": cast(JSONValue, list(outcome.warning_codes)), + "fit_duration_included": False, + "backend_exception_text_included": False, + } + + +def _fold_evaluation( + rows: Sequence[Mapping[str, Any]], + fold: Mapping[str, Any], + specification: ExponentialSmoothingSpecification, + specification_code: int, + model_id: str, + outcome: _FitOutcome, + original_forecasts: Sequence[float | None], + rounding_digits: int, +) -> dict[str, Any]: + horizon = _int(fold.get("horizon")) + forecast = ( + original_forecasts[horizon - 1] + if 0 < horizon <= len(original_forecasts) + else None + ) + transformed_forecast = ( + outcome.forecasts[horizon - 1] + if 0 < horizon <= len(outcome.forecasts) + else None + ) + target_index = _int(fold.get("target_index")) + actual = ( + _optional_float(rows[target_index].get("cm_input_observed_value")) + if 0 <= target_index < len(rows) + else None + ) + fold_valid = fold.get("status") == "valid" + if outcome.status in {"failed", "skipped", "unavailable"}: + status = "not_evaluated" + reason = outcome.reason + elif not fold_valid or actual is None: + status = "skipped" + reason = "target_unavailable" + elif forecast is None: + status = "skipped" + reason = "inverse_transform_unavailable" + else: + status = "evaluated" + reason = "" + error = ( + forecast - actual + if status == "evaluated" and forecast is not None and actual is not None + else None + ) + return { + "schema_version": EXPONENTIAL_SMOOTHING_FORECAST_SCHEMA_VERSION, + "status": status, + "reason": reason or None, + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "family_code": EXPONENTIAL_SMOOTHING_FAMILY_CODES[specification.family], + "error_component": specification.error, + "trend_component": specification.trend, + "seasonal_component": specification.seasonal, + "damped_trend": specification.damped_trend, + "initialization_method": specification.initialization_method, + "fit_status": outcome.status, + "fit_reason": outcome.reason or None, + "converged": outcome.converged, + "fold_id": _int(fold.get("fold_id")), + "origin_row_id": fold.get("origin_row_id"), + "target_row_id": fold.get("target_row_id"), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "target_bin_end_utc_ms": _int(fold.get("target_bin_end_utc_ms")), + "horizon": horizon, + "transformed_forecast": _rounded(transformed_forecast, rounding_digits), + "forecast": _rounded(forecast, rounding_digits), + "actual": _rounded(actual, rounding_digits), + "error": _rounded(error, rounding_digits), + "absolute_error": _rounded( + abs(error) if error is not None else None, rounding_digits + ), + "squared_error": _rounded( + error * error if error is not None else None, rounding_digits + ), + "original_scale": True, + "future_values_visible": False, + "automatic_winner": False, + } + + +def _model_evaluation_payload( + specification: ExponentialSmoothingSpecification, + specification_code: int, + model_id: str, + evaluations: Sequence[Mapping[str, Any]], + fit_samples: Sequence[Mapping[str, JSONValue]], + retained_limit: int, + rounding_digits: int, +) -> dict[str, JSONValue]: + statuses = Counter(_text(row.get("status")) for row in evaluations) + fit_statuses = Counter(_text(row.get("status")) for row in fit_samples) + reasons = Counter( + _text(row.get("reason")) + for row in evaluations + if _text(row.get("reason")) + ) + horizons = sorted({_int(row.get("horizon")) for row in evaluations}) + metrics = [ + _metrics_for_horizon(evaluations, horizon, rounding_digits) + for horizon in horizons + ] + return { + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "configuration": specification.to_metadata(), + "status": "ready" if statuses["evaluated"] else "limited", + "fit_status_counts": dict(sorted(fit_statuses.items())), + "evaluation_status_counts": dict(sorted(statuses.items())), + "reason_counts": dict(sorted(reasons.items())), + "horizon_metrics": cast(JSONValue, metrics), + "fold_results": cast( + JSONValue, [dict(row) for row in evaluations[:retained_limit]] + ), + "fold_results_truncated": len(evaluations) > retained_limit, + "fit_samples": cast( + JSONValue, [dict(row) for row in fit_samples[:retained_limit]] + ), + "fit_samples_truncated": len(fit_samples) > retained_limit, + "automatic_winner": False, + } + + +def _metrics_for_horizon( + evaluations: Sequence[Mapping[str, Any]], + horizon: int, + rounding_digits: int, +) -> dict[str, JSONValue]: + selected = [ + row + for row in evaluations + if _int(row.get("horizon")) == horizon + and row.get("status") == "evaluated" + ] + errors = [_float(row.get("error")) for row in selected] + absolute = [abs(value) for value in errors] + squared = [value * value for value in errors] + return { + "horizon": horizon, + "evaluation_count": len(selected), + "mae": ( + _rounded(sum(absolute) / len(absolute), rounding_digits) + if absolute + else None + ), + "rmse": ( + _rounded(math.sqrt(sum(squared) / len(squared)), rounding_digits) + if squared + else None + ), + "bias": ( + _rounded(sum(errors) / len(errors), rounding_digits) + if errors + else None + ), + "original_scale": True, + } + + +def _inverse_forecasts( + rows: Sequence[Mapping[str, Any]], + origin_index: int, + forecasts: Sequence[float], + profile: ClassicalModelInputProfile, + rounding_digits: int, +) -> tuple[float | None, ...]: + if not forecasts or not 0 <= origin_index < len(rows): + return () + origin = rows[origin_index] + group_indexes = [ + index + for index, row in enumerate(rows) + if _text(row.get("series_id")) == _text(origin.get("series_id")) + and _text(row.get("period")) == _text(origin.get("period")) + ] + try: + local_origin = group_indexes.index(origin_index) + except ValueError: + return tuple(None for _ in forecasts) + observed = [ + _optional_float(rows[index].get("cm_input_observed_value")) + for index in group_indexes + ] + base = _base_transform(observed, profile.transform) + stages: list[list[float | None]] = [base] + lags: list[int] = [] + for _ in range(profile.differencing_order): + stages.append(_difference(stages[-1], 1)) + lags.append(1) + for _ in range(profile.seasonal_differencing_order): + stages.append(_difference(stages[-1], profile.seasonal_period)) + lags.append(profile.seasonal_period) + current: list[float | None] = [float(value) for value in forecasts] + for stage_index in range(len(stages) - 1, 0, -1): + parent = stages[stage_index - 1] + lag = lags[stage_index - 1] + restored: list[float | None] = [] + for offset, value in enumerate(current, start=1): + reference_position = local_origin + offset - lag + if reference_position <= local_origin: + reference = ( + parent[reference_position] + if 0 <= reference_position < len(parent) + else None + ) + else: + prediction_position = reference_position - local_origin - 1 + reference = ( + restored[prediction_position] + if 0 <= prediction_position < len(restored) + else None + ) + restored.append( + value + reference + if value is not None and reference is not None + else None + ) + current = restored + if profile.transform == "level": + levels = current + elif profile.transform == "log_level": + levels = [ + ( + math.exp(value) + if value is not None and math.isfinite(value) + else None + ) + for value in current + ] + else: + previous = next( + ( + observed[index] + for index in range(local_origin, -1, -1) + if observed[index] is not None + ), + None, + ) + levels = [] + for value in current: + if value is None or previous is None: + levels.append(None) + previous = None + continue + candidate_level = ( + previous * (1.0 + value) + if profile.transform == "return" + else previous * math.exp(value) + ) + next_level: float | None = ( + candidate_level if math.isfinite(candidate_level) else None + ) + levels.append(next_level) + previous = next_level + return tuple(_rounded(value, rounding_digits) for value in levels) + + +def _base_transform( + observed: Sequence[float | None], transform: str +) -> list[float | None]: + values: list[float | None] = [] + previous: float | None = None + for value in observed: + if value is None: + transformed = None + previous = None + elif transform == "level": + transformed = value + elif transform == "log_level": + transformed = math.log(value) if value > 0 else None + elif previous is None: + transformed = None + elif transform == "return": + transformed = value / previous - 1.0 if previous != 0 else None + else: + transformed = ( + math.log(value / previous) + if value > 0 and previous > 0 + else None + ) + values.append(transformed) + if value is not None: + previous = value + return values + + +def _difference(values: Sequence[float | None], lag: int) -> list[float | None]: + output: list[float | None] = [] + for index, value in enumerate(values): + previous = values[index - lag] if index >= lag else None + output.append( + value - previous + if value is not None and previous is not None + else None + ) + return output + + +def _reference_baseline_payloads( + rows: Sequence[Mapping[str, Any]], + folds: Sequence[Mapping[str, Any]], + rolling_windows: Sequence[int], + rounding_digits: int, +) -> list[dict[str, JSONValue]]: + predictions: dict[str, list[dict[str, Any]]] = { + "naive_random_walk": [], + **{f"rolling_mean_{window}": [] for window in rolling_windows}, + **{f"rolling_median_{window}": [] for window in rolling_windows}, + "session_seasonal_naive": [], + } + for fold in folds: + if fold.get("status") != "valid": + continue + start = _int(fold.get("training_start_index")) + end = _int(fold.get("training_end_index")) + target_index = _int(fold.get("target_index")) + if not 0 <= target_index < len(rows): + continue + actual = _optional_float( + rows[target_index].get("cm_input_observed_value") + ) + if actual is None: + continue + training = [ + _optional_float(rows[index].get("cm_input_observed_value")) + for index in range(start, end + 1) + ] + valid = [value for value in training if value is not None] + if valid: + predictions["naive_random_walk"].append( + _baseline_evaluation(fold, valid[-1], actual) + ) + for window in rolling_windows: + if len(valid) >= window: + sample = valid[-window:] + predictions[f"rolling_mean_{window}"].append( + _baseline_evaluation( + fold, sum(sample) / len(sample), actual + ) + ) + predictions[f"rolling_median_{window}"].append( + _baseline_evaluation(fold, float(median(sample)), actual) + ) + target_timestamp = _int( + rows[target_index].get("cm_input_bin_start_utc_ms") + ) + target_session = classify_histdata_timestamp( + target_timestamp + ).session_state + session_value: float | None = None + for index in range(end, start - 1, -1): + value = _optional_float(rows[index].get("cm_input_observed_value")) + timestamp = _int(rows[index].get("cm_input_bin_start_utc_ms")) + if ( + value is not None + and classify_histdata_timestamp(timestamp).session_state + == target_session + ): + session_value = value + break + if session_value is not None: + predictions["session_seasonal_naive"].append( + _baseline_evaluation(fold, session_value, actual) + ) + payloads: list[dict[str, JSONValue]] = [] + for name, evaluations in predictions.items(): + base_name = ( + "rolling_mean" + if name.startswith("rolling_mean_") + else ( + "rolling_median" if name.startswith("rolling_median_") else name + ) + ) + baseline_window: int | None = ( + int(name.rsplit("_", 1)[1]) if name[-1:].isdigit() else None + ) + horizons = sorted({_int(row.get("horizon")) for row in evaluations}) + payloads.append( + { + "model": base_name, + "model_code": EXPONENTIAL_SMOOTHING_BASELINE_CODES[base_name], + "window": baseline_window, + "calculation_basis": "shared_regular_grid_folds", + "horizon_metrics": cast( + JSONValue, + [ + _metrics_for_horizon( + evaluations, horizon, rounding_digits + ) + for horizon in horizons + ], + ), + "evaluation_count": len(evaluations), + "automatic_winner": False, + } + ) + return payloads + + +def _baseline_evaluation( + fold: Mapping[str, Any], forecast: float, actual: float +) -> dict[str, Any]: + error = forecast - actual + return { + "status": "evaluated", + "horizon": _int(fold.get("horizon")), + "forecast": forecast, + "actual": actual, + "error": error, + } + + +def _build_annotations( + frame: Any | None, + evaluations: Sequence[Mapping[str, Any]], + profile: ExponentialSmoothingProfile, + input_result: ClassicalModelInputResult, + *, + target: Any | None, +) -> tuple[tuple[Mapping[str, Any], ...], int]: + if frame is None: + return (), 0 + try: + enriched = ensure_tick_training_features(frame, target=target) + except (AttributeError, TypeError, ValueError): + return (), 0 + source_rows = cast(list[dict[str, Any]], enriched.to_dicts()) + availability: dict[tuple[str, str], list[tuple[int, int]]] = {} + for row in source_rows: + timestamp = _optional_int(row.get("timestamp_utc_ms")) + row_id = _optional_int(row.get("row_id")) + if timestamp is None or row_id is None: + continue + availability.setdefault( + (_text(row.get("series_id")), _text(row.get("period"))), [] + ).append((timestamp, row_id)) + for values in availability.values(): + values.sort() + + forecast_rows: dict[tuple[str, str, int], dict[str, Any]] = {} + diagnostic_rows: dict[tuple[str, str, int], dict[str, Any]] = {} + collisions = 0 + selected = [ + row + for row in evaluations + if _text(row.get("specification_id")) + == profile.projection_specification_id + and _int(row.get("horizon")) == profile.projection_horizon + ] + selected.sort( + key=lambda row: ( + _text(row.get("series_id")), + _text(row.get("period")), + _int(row.get("origin_bin_end_utc_ms")), + _int(row.get("fold_id")), + ) + ) + for evaluation in selected: + group = ( + _text(evaluation.get("series_id")), + _text(evaluation.get("period")), + ) + forecast = _optional_float(evaluation.get("forecast")) + if forecast is not None: + row_id = _first_available_row_id( + availability.get(group, ()), + _int(evaluation.get("origin_bin_end_utc_ms")), + ) + if row_id is not None: + key = (*group, row_id) + if key in forecast_rows: + collisions += 1 + forecast_rows[key] = _annotation_row( + evaluation, + input_result, + row_id, + diagnostic=False, + ) + error = _optional_float(evaluation.get("error")) + if error is not None: + row_id = _first_available_row_id( + availability.get(group, ()), + _int(evaluation.get("target_bin_end_utc_ms")), + ) + if row_id is not None: + key = (*group, row_id) + if key in diagnostic_rows: + collisions += 1 + diagnostic_rows[key] = _annotation_row( + evaluation, + input_result, + row_id, + diagnostic=True, + ) + merged: dict[tuple[str, str, int], dict[str, Any]] = dict(forecast_rows) + for key, diagnostic in diagnostic_rows.items(): + if key not in merged: + merged[key] = diagnostic + continue + forecast_annotation = merged[key] + if forecast_annotation.get("cm_ets_fold_id") == diagnostic.get( + "cm_ets_fold_id" + ): + for name in ( + "cm_ets_actual", + "cm_ets_error", + "cm_ets_diagnostic_available", + "cm_ets_diagnostic_available_at_utc_ms", + ): + forecast_annotation[name] = diagnostic.get(name) + else: + collisions += 1 + return ( + tuple(merged[key] for key in sorted(merged)), + collisions, + ) + + +def _first_available_row_id( + values: Sequence[tuple[int, int]], threshold: int +) -> int | None: + return next( + (row_id for timestamp, row_id in values if timestamp >= threshold), None + ) + + +def _annotation_row( + evaluation: Mapping[str, Any], + input_result: ClassicalModelInputResult, + projection_row_id: int, + *, + diagnostic: bool, +) -> dict[str, Any]: + fit_status = _text(evaluation.get("fit_status")) or "unavailable" + trend = _text(evaluation.get("trend_component")) or "none" + seasonal = _text(evaluation.get("seasonal_component")) or "none" + error = _text(evaluation.get("error_component")) or "add" + initialization = ( + _text(evaluation.get("initialization_method")) or "estimated" + ) + row = { + "series_id": _text(evaluation.get("series_id")), + "period": _text(evaluation.get("period")), + "row_id": projection_row_id, + "cm_ets_schema_version": EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION, + "cm_ets_input_derivation_id": _text( + input_result.contract.get("derivation_id") + ), + "cm_ets_model_id": _text(evaluation.get("model_id")), + "cm_ets_family_code": _int(evaluation.get("family_code")), + "cm_ets_specification_code": _int(evaluation.get("specification_code")), + "cm_ets_error_code": EXPONENTIAL_SMOOTHING_COMPONENT_CODES[error], + "cm_ets_trend_code": EXPONENTIAL_SMOOTHING_COMPONENT_CODES[trend], + "cm_ets_seasonal_code": EXPONENTIAL_SMOOTHING_COMPONENT_CODES[seasonal], + "cm_ets_damped_trend": bool(evaluation.get("damped_trend", False)), + "cm_ets_initialization_code": EXPONENTIAL_SMOOTHING_INITIALIZATION_CODES[ + initialization + ], + "cm_ets_fit_status_code": EXPONENTIAL_SMOOTHING_FIT_STATUS_CODES.get( + fit_status, 1 + ), + "cm_ets_converged": bool(evaluation.get("converged", False)), + "cm_ets_fold_id": _int(evaluation.get("fold_id")), + "cm_ets_origin_row_id": evaluation.get("origin_row_id"), + "cm_ets_target_row_id": evaluation.get("target_row_id"), + "cm_ets_horizon": _int(evaluation.get("horizon")), + "cm_ets_forecast": _optional_float(evaluation.get("forecast")), + "cm_ets_forecast_available": not diagnostic, + "cm_ets_forecast_available_at_utc_ms": _int( + evaluation.get("origin_bin_end_utc_ms") + ), + "cm_ets_actual": ( + _optional_float(evaluation.get("actual")) if diagnostic else None + ), + "cm_ets_error": ( + _optional_float(evaluation.get("error")) if diagnostic else None + ), + "cm_ets_diagnostic_available": diagnostic, + "cm_ets_diagnostic_available_at_utc_ms": ( + _int(evaluation.get("target_bin_end_utc_ms")) + if diagnostic + else None + ), + "cm_ets_diagnostic_only": diagnostic, + "cm_ets_original_scale": True, + "cm_ets_training_eligible": not diagnostic, + } + return row + + +def _training_projection_metadata( + profile: ExponentialSmoothingProfile, + input_result: ClassicalModelInputResult, + annotation_count: int, + collision_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "row", + "identity_fields": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "mapping_policy": "first_source_row_at_or_after_availability", + "collision_policy": "latest_origin_wins", + "collision_count": collision_count, + "annotation_count": annotation_count, + "projection_specification_id": profile.projection_specification_id, + "projection_horizon": profile.projection_horizon, + "input_derivation_id": input_result.contract.get("derivation_id"), + "column_names": list(EXPONENTIAL_SMOOTHING_COLUMNS), + "forecast_time_values_use_future": False, + "diagnostics_marked_post_observation": True, + "observed_columns_overwritten": False, + } + + +def _base_payload( + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + profile: ExponentialSmoothingProfile, +) -> dict[str, JSONValue]: + input_contract = input_result.contract + input_regularization = _mapping(input_contract.get("regularization")) + return { + "schema_version": EXPONENTIAL_SMOOTHING_SCHEMA_VERSION, + "advisory": True, + "target_axis": dict(_mapping(input_contract.get("target_axis"))), + "reference_fingerprint_id": input_contract.get( + "reference_fingerprint_id" + ) + or fingerprint.get("fingerprint_id"), + "input_schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "input_derivation_id": input_contract.get("derivation_id"), + "input_status": input_contract.get("status"), + "calculation_basis": "regular_grid_rolling_origin", + "configuration": profile.to_metadata(), + "input_transform_policy": dict( + _mapping(input_contract.get("transform_policy")) + ), + "input_missingness_policy": { + "expected_closure_policy": input_regularization.get( + "expected_closure_policy" + ), + "unexpected_missing_policy": input_regularization.get( + "unexpected_missing_policy" + ), + "expected_closure_count": input_regularization.get( + "expected_closure_count" + ), + "unexpected_missing_count": input_regularization.get( + "unexpected_missing_count" + ), + "expected_closure_grid_rows_retained": input_regularization.get( + "expected_closure_grid_rows_retained" + ), + "expected_closure_model_observations_omitted": ( + input_regularization.get( + "expected_closure_model_observations_omitted" + ) + ), + }, + "missing_observation_policy": "reset_to_trailing_contiguous_segment", + "forward_fill_policy": "never", + "original_scale_forecasts": True, + "automatic_search": False, + "automatic_winner": False, + "hard_fail_quality_gate": False, + "fitted_objects_included": False, + "backend_exception_text_included": False, + } + + +def _unavailable_result( + input_result: ClassicalModelInputResult, + base: Mapping[str, JSONValue], + reason: str, + *, + status: str = "unavailable", +) -> ExponentialSmoothingResult: + configuration = _mapping(base.get("configuration")) + diagnostics: dict[str, JSONValue] = { + **dict(base), + "status": status, + "reason": reason, + "limitations": [reason], + "backend": { + "provider": "statsmodels", + "version": None, + "available": False, + "import_basis": "optional_models_extra", + }, + "fit_summary": { + "schema_version": EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION, + "fit_attempt_count": 0, + "status_counts": {}, + "reason_counts": {reason: 1}, + "warning_counts": {}, + "failed_fit_count": 0, + "limited_fit_count": 0, + "fit_samples": [], + "fit_samples_truncated": False, + }, + "evaluation": { + "schema_version": EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin", + "original_scale": True, + "model_count": len( + _mapping_rows(configuration.get("specifications")) + ), + "fold_count": len(input_result.folds), + "evaluated_fold_count": 0, + "skipped_evaluation_count": 0, + "models": [], + "reference_baselines": [], + "comparison_semantics": "descriptive_shared_folds_only", + "automatic_winner": False, + }, + "resource_usage": { + "limits": dict( + _mapping( + _mapping(input_result.contract.get("configuration")).get( + "resources" + ) + ) + ), + "estimated_working_memory_bytes": 0, + "memory_limit_exceeded": False, + "fit_attempt_count": 0, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "training_projection": { + "schema_version": EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "row", + "identity_fields": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "mapping_policy": "first_source_row_at_or_after_availability", + "collision_policy": "latest_origin_wins", + "collision_count": 0, + "annotation_count": 0, + "projection_specification_id": configuration.get( + "projection_specification_id" + ), + "projection_horizon": configuration.get("projection_horizon"), + "input_derivation_id": input_result.contract.get("derivation_id"), + "column_names": list(EXPONENTIAL_SMOOTHING_COLUMNS), + "forecast_time_values_use_future": False, + "diagnostics_marked_post_observation": True, + "observed_columns_overwritten": False, + }, + "fit_duration_included": False, + } + return ExponentialSmoothingResult(diagnostics, (), input_result) + + +def _ensure_projection_columns(frame: Any) -> Any: + import polars as pl + + expressions = [] + for name, dtype in _projection_dtypes().items(): + if name in frame.columns: + expressions.append(pl.col(name).cast(dtype).alias(name)) + else: + expressions.append(pl.lit(None).cast(dtype).alias(name)) + return frame.with_columns(expressions) + + +def _projection_dtypes() -> dict[str, Any]: + import polars as pl + + strings = { + "cm_ets_schema_version", + "cm_ets_input_derivation_id", + "cm_ets_model_id", + } + booleans = { + "cm_ets_damped_trend", + "cm_ets_converged", + "cm_ets_forecast_available", + "cm_ets_diagnostic_available", + "cm_ets_diagnostic_only", + "cm_ets_original_scale", + "cm_ets_training_eligible", + } + floats = {"cm_ets_forecast", "cm_ets_actual", "cm_ets_error"} + return { + name: ( + pl.Utf8 + if name in strings + else ( + pl.Boolean + if name in booleans + else pl.Float64 if name in floats else pl.Int64 + ) + ) + for name in EXPONENTIAL_SMOOTHING_COLUMNS + } + + +def _load_backend() -> _Backend | None: + try: + statsmodels = importlib.import_module("statsmodels") + holtwinters = importlib.import_module("statsmodels.tsa.holtwinters") + ets = importlib.import_module( + "statsmodels.tsa.exponential_smoothing.ets" + ) + version = getattr(statsmodels, "__version__", "") or ( + importlib.metadata.version("statsmodels") + ) + return _Backend( + version=str(version), + holt_winters=holtwinters.ExponentialSmoothing, + ets_model=ets.ETSModel, + ) + except ( + ImportError, + ModuleNotFoundError, + importlib.metadata.PackageNotFoundError, + ): + return None + + +def _model_id( + specification: ExponentialSmoothingSpecification, + input_derivation_id: str, + backend_version: str, +) -> str: + payload = { + "schema_version": EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION, + "input_derivation_id": input_derivation_id, + "backend": "statsmodels", + "backend_version": backend_version, + "specification": specification.to_metadata(), + } + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("ascii") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _target_sort_key(value: Mapping[str, JSONValue]) -> tuple[str, ...]: + axis = _mapping(value.get("target_axis")) + return tuple( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + + +def _mapping(value: object) -> Mapping[str, JSONValue]: + return ( + cast(Mapping[str, JSONValue], value) + if isinstance(value, Mapping) + else {} + ) + + +def _mapping_rows(value: object) -> list[Mapping[str, JSONValue]]: + if not isinstance(value, list): + return [] + return [ + cast(Mapping[str, JSONValue], row) + for row in value + if isinstance(row, Mapping) + ] + + +def _text(value: object) -> str: + return str(value or "") + + +def _int(value: object) -> int: + try: + return int(cast(Any, value) or 0) + except (TypeError, ValueError): + return 0 + + +def _optional_int(value: object) -> int | None: + if value is None or isinstance(value, bool): + return None + try: + return int(cast(Any, value)) + except (TypeError, ValueError): + return None + + +def _float(value: object) -> float: + result = _optional_float(value) + return result if result is not None else 0.0 + + +def _optional_float(value: object) -> float | None: + if value is None or isinstance(value, bool): + return None + try: + result = float(cast(Any, value)) + except (TypeError, ValueError): + return None + return result if math.isfinite(result) else None + + +def _rounded(value: float | None, digits: int) -> float | None: + return ( + round(value, digits) + if value is not None and math.isfinite(value) + else None + ) diff --git a/src/histdatacom/data_quality/fingerprint_contracts.py b/src/histdatacom/data_quality/fingerprint_contracts.py index 858b626a..a0fe9097 100644 --- a/src/histdatacom/data_quality/fingerprint_contracts.py +++ b/src/histdatacom/data_quality/fingerprint_contracts.py @@ -4,12 +4,65 @@ from collections.abc import Mapping from dataclasses import dataclass, field +from typing import cast +from histdatacom.data_quality.autoregressive import ( + AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY, + AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION, + AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION, + AUTOREGRESSIVE_FAMILIES, + AUTOREGRESSIVE_FIT_SCHEMA_VERSION, + AUTOREGRESSIVE_FIT_STATUS_CODES, + AUTOREGRESSIVE_FORECAST_SCHEMA_VERSION, + AUTOREGRESSIVE_REASON_CODES, + AUTOREGRESSIVE_SCHEMA_VERSION, + AUTOREGRESSIVE_SUMMARY_METADATA_KEY, + AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION, + AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION, + DEFAULT_AUTOREGRESSIVE_SUMMARY_TARGET_LIMIT, +) from histdatacom.data_quality.calendar import ( TIME_SERIES_FINGERPRINT_CALENDAR_REGIMES_SCHEMA_VERSION, ) +from histdatacom.data_quality.classical_baselines import ( + CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY, + CLASSICAL_BASELINE_SCHEMA_VERSION, + CLASSICAL_BASELINE_SUMMARY_METADATA_KEY, + CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION, + CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION, + DEFAULT_BASELINE_SUMMARY_TARGET_LIMIT, +) +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION, + CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION, + CLASSICAL_MODEL_FOLD_SCHEMA_VERSION, + CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY, + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY, + CLASSICAL_MODEL_INPUT_SUMMARY_SCHEMA_VERSION, + CLASSICAL_MODEL_TRAINING_PROJECTION_SCHEMA_VERSION, + DEFAULT_MODEL_INPUT_SUMMARY_TARGET_LIMIT, + MODEL_FAILURE_REASON_CODES, + MODEL_FIT_STATUSES, + MODEL_TRANSFORM_CODES, +) +from histdatacom.data_quality.classical_model_comparison import ( + CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY, + CLASSICAL_MODEL_COMPARISON_ELIGIBILITY_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY, + CLASSICAL_MODEL_COMPARISON_SUMMARY_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION, + CLASSICAL_MODEL_FIT_ACCOUNTING_SCHEMA_VERSION, + CLASSICAL_MODEL_SKILL_SCHEMA_VERSION, + CLASSICAL_MODEL_STABILITY_SCHEMA_VERSION, + COMPARISON_REASON_CODES, + DEFAULT_CLASSICAL_MODEL_COMPARISON_SUMMARY_TARGET_LIMIT, +) from histdatacom.data_quality.fingerprints import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, CROSS_SERIES_FINGERPRINT_RULE_ID, + CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, DEFAULT_FINGERPRINT_DISTRIBUTION_ATTENTION_LIMIT, DEFAULT_FINGERPRINT_DISTRIBUTION_FLAG_CACHE_FLOAT_PRECISION, DEFAULT_FINGERPRINT_DISTRIBUTION_FLAG_TRUNCATED, @@ -24,16 +77,20 @@ DEFAULT_FINGERPRINT_READINESS_RISK_REASON_LIMIT, DEFAULT_FINGERPRINT_READINESS_RISK_SECTION_LIMIT, DEFAULT_FINGERPRINT_READINESS_RISK_TARGET_LIMIT, + DEFAULT_FINGERPRINT_PARITY_SUMMARY_LIMIT, DEFAULT_FINGERPRINT_READINESS_SUMMARY_LIMIT, DEFAULT_FINGERPRINT_REGIME_COUNT_LIMIT, DEFAULT_FINGERPRINT_REGIME_SUMMARY_LIMIT, DEFAULT_FINGERPRINT_TOPOLOGY_ATTENTION_LIMIT, + DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT, DEFAULT_FINGERPRINT_TOPOLOGY_SUMMARY_LIMIT, SERIES_FINGERPRINT_RULE_ID, TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_CONDITIONAL_DISTRIBUTIONS_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY, TIME_SERIES_FINGERPRINT_COVERAGE_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DEPENDENCE_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DISTRIBUTION_ATTENTION_METADATA_KEY, TIME_SERIES_FINGERPRINT_DISTRIBUTION_ATTENTION_SCHEMA_VERSION, @@ -41,6 +98,9 @@ TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DYNAMICS_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY, @@ -54,6 +114,97 @@ TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_SCHEMA_VERSION, ) +from histdatacom.data_quality.exponential_smoothing import ( + DEFAULT_EXPONENTIAL_SMOOTHING_SUMMARY_TARGET_LIMIT, + EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY, + EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_FAMILIES, + EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_FORECAST_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_REASON_CODES, + EXPONENTIAL_SMOOTHING_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY, + EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.time import ( + TIMESTAMP_TOPOLOGY_INSPECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.synthetic_constraints import ( + DEFAULT_SYNTHETIC_CONSTRAINT_SUMMARY_LIMIT, + SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY, + SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY, + SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION, + SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION, + SYNTHETIC_VALIDATION_SCHEMA_VERSION, +) +from histdatacom.data_quality.synthetic_generation import ( + SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION, + SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION, + SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION, +) +from histdatacom.data_quality.seasonal_exogenous import ( + DEFAULT_SEASONAL_EXOGENOUS_SUMMARY_TARGET_LIMIT, + SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY, + SEASONAL_EXOGENOUS_CONFIGURATION_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_FAMILIES, + SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_FIT_STATUS_CODES, + SEASONAL_EXOGENOUS_FORECAST_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_REASON_CODES, + SEASONAL_EXOGENOUS_REGRESSOR_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY, + SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.state_space import ( + DEFAULT_STATE_SPACE_SUMMARY_TARGET_LIMIT, + STATE_SPACE_BOUNDED_PAYLOAD_KEY, + STATE_SPACE_CONFIGURATION_SCHEMA_VERSION, + STATE_SPACE_EVALUATION_SCHEMA_VERSION, + STATE_SPACE_FAMILIES, + STATE_SPACE_FIT_SCHEMA_VERSION, + STATE_SPACE_FIT_STATUS_CODES, + STATE_SPACE_FORECAST_SCHEMA_VERSION, + STATE_SPACE_REASON_CODES, + STATE_SPACE_SCHEMA_VERSION, + STATE_SPACE_STATE_RESULT_SCHEMA_VERSION, + STATE_SPACE_SUMMARY_METADATA_KEY, + STATE_SPACE_SUMMARY_SCHEMA_VERSION, + STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.volatility import ( + ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY, + DEFAULT_VOLATILITY_SUMMARY_TARGET_LIMIT, + VOLATILITY_BOUNDED_PAYLOAD_KEY, + VOLATILITY_CONFIGURATION_SCHEMA_VERSION, + VOLATILITY_DISTRIBUTIONS, + VOLATILITY_EVALUATION_SCHEMA_VERSION, + VOLATILITY_FAMILIES, + VOLATILITY_FIT_SCHEMA_VERSION, + VOLATILITY_FIT_STATUS_CODES, + VOLATILITY_FORECAST_SCHEMA_VERSION, + VOLATILITY_INPUT_DEFINITIONS, + VOLATILITY_MEAN_MODELS, + VOLATILITY_REASON_CODES, + VOLATILITY_SCHEMA_VERSION, + VOLATILITY_SUMMARY_METADATA_KEY, + VOLATILITY_SUMMARY_SCHEMA_VERSION, + VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.training_features import ( + AUTOREGRESSIVE_COLUMNS, + EXPONENTIAL_SMOOTHING_COLUMNS, + SEASONAL_EXOGENOUS_COLUMNS, + KALMAN_COLUMNS, + STATE_SPACE_COLUMNS, + VOLATILITY_COLUMNS, + CLASSICAL_MODEL_COMPARISON_COLUMNS, + training_feature_definitions, +) from histdatacom.histdata_ascii import TICK from histdatacom.runtime_contracts import JSONValue @@ -64,7 +215,17 @@ "histogram_bins", "max_rows", "rounding_digits", + "topology_inspection_sample_limit", "distribution_attention", + "cache_source_parity", + "classical_baselines", + "classical_model_input", + "exponential_smoothing", + "autoregressive", + "seasonal_exogenous", + "state_space", + "volatility", + "classical_model_comparison", ) FINGERPRINT_DISTRIBUTION_ATTENTION_CONFIG_KEYS = ( "invalid_row_min_count", @@ -89,6 +250,11 @@ FINGERPRINT_REGIME_BOUNDED_PAYLOAD_KEY = "fingerprint_regime" FINGERPRINT_READINESS_BOUNDED_PAYLOAD_KEY = "fingerprint_readiness" FINGERPRINT_READINESS_RISK_BOUNDED_PAYLOAD_KEY = "fingerprint_readiness_risk" +FINGERPRINT_CROSS_SERIES_BOUNDED_PAYLOAD_KEY = "fingerprint_cross_series" +FINGERPRINT_PARITY_BOUNDED_PAYLOAD_KEY = "fingerprint_parity" +FINGERPRINT_SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY = ( + SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY +) FINGERPRINT_SECTION_STATUSES = ("valid", "limited", "skipped", "unavailable") FINGERPRINT_DYNAMICS_STATUSES = ("ok", "limited", "unavailable") @@ -115,6 +281,8 @@ "no_computable_lags", "skipped_lags", "skipped_rolling_windows", + "stationarity_limited", + "stationarity_unavailable", "not_emitted", ) FINGERPRINT_TOPOLOGY_LIMITATIONS = ( @@ -137,7 +305,10 @@ "observed_sequence", "statistics computed over parsed row order without regular-grid imputation", ), - ("regular_grid", "reserved for future grid-regularized calculations"), + ( + "regular_grid", + "deterministic UTC grid derived from enriched ASCII tick rows", + ), ("limited", "section emitted with advisory limitations"), ("unavailable", "section could not compute enough contract data"), ) @@ -326,6 +497,13 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: metadata_key=TIME_SERIES_FINGERPRINT_TOPOLOGY_ATTENTION_METADATA_KEY, status="implemented", ), + FingerprintSchemaContract( + "fingerprint_topology_inspection", + TIMESTAMP_TOPOLOGY_INSPECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="temporal_topology.inspection_context", + status="implemented", + ), FingerprintSchemaContract( "fingerprint_distribution_summary", TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_SCHEMA_VERSION, @@ -383,6 +561,409 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: payload_path="time_series_fingerprint.stationarity_diagnostics", status="implemented", ), + FingerprintSchemaContract( + "fingerprint_decomposition", + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.decomposition", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_decomposition_training_projection", + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.decomposition.training_projection" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_baselines", + CLASSICAL_BASELINE_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.classical_baselines", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_baseline_training_projection", + CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.classical_baselines.training_projection" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_baseline_summary", + CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=CLASSICAL_BASELINE_SUMMARY_METADATA_KEY, + bounded_payload_key=CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_input", + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.classical_model_input", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_fold", + CLASSICAL_MODEL_FOLD_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.classical_model_input.fold_policy", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_fit_result", + CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_evaluation_result", + CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_training_projection", + CLASSICAL_MODEL_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.classical_model_input.training_projection" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_input_summary", + CLASSICAL_MODEL_INPUT_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY, + bounded_payload_key=CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_exponential_smoothing", + EXPONENTIAL_SMOOTHING_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.exponential_smoothing", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_exponential_smoothing_configuration", + EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.exponential_smoothing.configuration", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_exponential_smoothing_fit", + EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.exponential_smoothing.fit_summary", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_exponential_smoothing_forecast", + EXPONENTIAL_SMOOTHING_FORECAST_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_exponential_smoothing_evaluation", + EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.exponential_smoothing.evaluation", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_exponential_smoothing_training_projection", + EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.exponential_smoothing.training_projection" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_exponential_smoothing_summary", + EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY, + bounded_payload_key=EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_autoregressive", + AUTOREGRESSIVE_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.autoregressive", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_autoregressive_configuration", + AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.autoregressive.configuration", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_autoregressive_fit", + AUTOREGRESSIVE_FIT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.autoregressive.fit_summary", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_autoregressive_forecast", + AUTOREGRESSIVE_FORECAST_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_autoregressive_evaluation", + AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.autoregressive.evaluation", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_autoregressive_training_projection", + AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.autoregressive.training_projection" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_autoregressive_summary", + AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=AUTOREGRESSIVE_SUMMARY_METADATA_KEY, + bounded_payload_key=AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_seasonal_exogenous", + SEASONAL_EXOGENOUS_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.seasonal_exogenous", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_seasonal_exogenous_configuration", + SEASONAL_EXOGENOUS_CONFIGURATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.seasonal_exogenous.configuration", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_seasonal_exogenous_regressors", + SEASONAL_EXOGENOUS_REGRESSOR_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.seasonal_exogenous.regressors", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_seasonal_exogenous_fit", + SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.seasonal_exogenous.fit_summary", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_seasonal_exogenous_forecast", + SEASONAL_EXOGENOUS_FORECAST_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_seasonal_exogenous_evaluation", + SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.seasonal_exogenous.evaluation", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_seasonal_exogenous_training_projection", + SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.seasonal_exogenous.training_projection" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_seasonal_exogenous_summary", + SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY, + bounded_payload_key=SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_state_space", + STATE_SPACE_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.state_space", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_state_space_configuration", + STATE_SPACE_CONFIGURATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.state_space.configuration", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_state_space_fit", + STATE_SPACE_FIT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.state_space.fit_summary", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_kalman_state_result", + STATE_SPACE_STATE_RESULT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_state_space_forecast", + STATE_SPACE_FORECAST_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_state_space_evaluation", + STATE_SPACE_EVALUATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.state_space.evaluation", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_state_space_training_projection", + STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.state_space.training_projection", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_state_space_summary", + STATE_SPACE_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=STATE_SPACE_SUMMARY_METADATA_KEY, + bounded_payload_key=STATE_SPACE_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_volatility", + VOLATILITY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.volatility", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_volatility_configuration", + VOLATILITY_CONFIGURATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.volatility.configuration", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_volatility_fit", + VOLATILITY_FIT_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.volatility.fit_summary", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_volatility_forecast", + VOLATILITY_FORECAST_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_volatility_evaluation", + VOLATILITY_EVALUATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.volatility.evaluation", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_volatility_training_projection", + VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=("time_series_fingerprint.volatility.training_projection"), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_volatility_summary", + VOLATILITY_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=VOLATILITY_SUMMARY_METADATA_KEY, + bounded_payload_key=VOLATILITY_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_comparison", + CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.classical_model_comparison", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_comparison_eligibility", + CLASSICAL_MODEL_COMPARISON_ELIGIBILITY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.classical_model_comparison." + "comparison_records" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_skill", + CLASSICAL_MODEL_SKILL_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_stability", + CLASSICAL_MODEL_STABILITY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_fit_accounting", + CLASSICAL_MODEL_FIT_ACCOUNTING_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.classical_model_comparison.fit_accounting" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_comparison_training_projection", + CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path=( + "time_series_fingerprint.classical_model_comparison." + "training_projection" + ), + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_classical_model_comparison_summary", + CLASSICAL_MODEL_COMPARISON_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY, + bounded_payload_key=CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), FingerprintSchemaContract( "fingerprint_audit", TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION, @@ -390,6 +971,60 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: payload_path="time_series_fingerprint.fingerprint_audit", status="implemented", ), + FingerprintSchemaContract( + "fingerprint_cache_source_parity", + TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.cache_source_parity", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_cache_source_parity_summary", + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY, + bounded_payload_key=FINGERPRINT_PARITY_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_synthetic_constraints", + SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + payload_path="time_series_fingerprint.synthetic_constraints", + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_synthetic_constraint_summary", + SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + metadata_key=SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY, + bounded_payload_key=FINGERPRINT_SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY, + status="implemented", + ), + FingerprintSchemaContract( + "fingerprint_synthetic_validation", + SYNTHETIC_VALIDATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "synthetic_tick_generation", + SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "synthetic_tick_generation_configuration", + SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), + FingerprintSchemaContract( + "synthetic_tick_generation_validation", + SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION, + rule_id=SERIES_FINGERPRINT_RULE_ID, + status="implemented", + ), FingerprintSchemaContract( "fingerprint_readiness_summary", TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_SCHEMA_VERSION, @@ -408,10 +1043,11 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: ), FingerprintSchemaContract( "cross_series_fingerprint", - None, + CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, rule_id=CROSS_SERIES_FINGERPRINT_RULE_ID, - status="planned", - issue="#331", + metadata_key=CROSS_SERIES_FINGERPRINT_METADATA_KEY, + bounded_payload_key=FINGERPRINT_CROSS_SERIES_BOUNDED_PAYLOAD_KEY, + status="implemented", ), ) @@ -464,6 +1100,86 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: "regime_summary", "Fingerprint regimes", ), + FingerprintReportSurfaceContract( + "cache_source_parity", + "fingerprint_cache_source_parity_summary", + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY, + FINGERPRINT_PARITY_BOUNDED_PAYLOAD_KEY, + "cache_source_parity", + "Fingerprint cache/source parity", + ), + FingerprintReportSurfaceContract( + "synthetic_constraints", + "fingerprint_synthetic_constraint_summary", + SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY, + FINGERPRINT_SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY, + "synthetic_constraints", + "Synthetic fingerprint constraints", + ), + FingerprintReportSurfaceContract( + "classical_baselines", + "fingerprint_classical_baseline_summary", + CLASSICAL_BASELINE_SUMMARY_METADATA_KEY, + CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY, + "classical_baselines", + "Classical fingerprint baselines", + ), + FingerprintReportSurfaceContract( + "classical_model_input", + "fingerprint_classical_model_input_summary", + CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY, + CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY, + "classical_model_input", + "Classical model input contracts", + ), + FingerprintReportSurfaceContract( + "exponential_smoothing", + "fingerprint_exponential_smoothing_summary", + EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY, + EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY, + "exponential_smoothing", + "Exponential-smoothing models", + ), + FingerprintReportSurfaceContract( + "autoregressive", + "fingerprint_autoregressive_summary", + AUTOREGRESSIVE_SUMMARY_METADATA_KEY, + AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY, + "autoregressive", + "Autoregressive models", + ), + FingerprintReportSurfaceContract( + "seasonal_exogenous", + "fingerprint_seasonal_exogenous_summary", + SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY, + SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY, + "seasonal_exogenous", + "Seasonal and exogenous models", + ), + FingerprintReportSurfaceContract( + "state_space", + "fingerprint_state_space_summary", + STATE_SPACE_SUMMARY_METADATA_KEY, + STATE_SPACE_BOUNDED_PAYLOAD_KEY, + "state_space", + "State-space and Kalman models", + ), + FingerprintReportSurfaceContract( + "volatility", + "fingerprint_volatility_summary", + VOLATILITY_SUMMARY_METADATA_KEY, + VOLATILITY_BOUNDED_PAYLOAD_KEY, + "volatility", + "ARCH and GARCH volatility models", + ), + FingerprintReportSurfaceContract( + "classical_model_comparison", + "fingerprint_classical_model_comparison_summary", + CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY, + CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY, + "classical_model_comparison", + "Classical model comparison", + ), FingerprintReportSurfaceContract( "readiness_summary", "fingerprint_readiness_summary", @@ -480,6 +1196,14 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: "readiness_risk", "Fingerprint readiness risk", ), + FingerprintReportSurfaceContract( + "cross_series", + "cross_series_fingerprint", + CROSS_SERIES_FINGERPRINT_METADATA_KEY, + FINGERPRINT_CROSS_SERIES_BOUNDED_PAYLOAD_KEY, + "cross_series", + "Cross-series fingerprint", + ), ) IMPLEMENTED_FINGERPRINT_TARGET_SECTION_CONTRACTS = ( @@ -501,6 +1225,10 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: target_timeframes=(TICK,), schema_key="series_fingerprint", basis_values=("observed_sequence",), + extra={ + "optional_nested_schema_key": "fingerprint_topology_inspection", + "profile_controlled_by": ["topology_inspection_sample_limit"], + }, ), FingerprintTargetSectionContract( "calendar_regimes", @@ -560,6 +1288,376 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: ], }, ), + FingerprintTargetSectionContract( + "decomposition", + ( + "advisory trend, seasonality, residual, smoothing-window, and " + "structural-break proxies" + ), + target_timeframes=(TICK,), + schema_key="fingerprint_decomposition", + basis_values=("observed_sequence",), + row_order_values=("source_text_order", "cache_order"), + extra={ + "profile_controlled_by": [ + "rolling_windows", + "histogram_bins", + "rounding_digits", + ], + "stationarity_basis": "stationarity_diagnostics", + "training_projection_grain": "period", + "training_projection_identity": [ + "series_id", + "period", + "row_id", + ], + }, + ), + FingerprintTargetSectionContract( + "cache_source_parity", + "opt-in source/cache and enriched training projection comparison", + target_timeframes=(TICK,), + schema_key="fingerprint_cache_source_parity", + basis_values=("text_scan", "direct_cache", "fresh_sibling_cache"), + extra={ + "profile_controlled_by": ["cache_source_parity"], + "default_enabled": False, + "advisory": True, + }, + ), + FingerprintTargetSectionContract( + "classical_baselines", + "opt-in deterministic advisory baselines over enriched tick rows", + target_timeframes=(TICK,), + schema_key="fingerprint_classical_baselines", + basis_values=("observed_sequence_walk_forward",), + row_order_values=("series_id_period_row_id",), + extra={ + "issue": "#332", + "profile_controlled_by": ["classical_baselines"], + "default_enabled": False, + "advisory": True, + "base_grain": "ascii/T", + "metric": "mid", + "timestamp_required": False, + "model_families": [ + "naive_random_walk", + "rolling_mean", + "rolling_median", + "session_seasonal_naive", + ], + }, + ), + FingerprintTargetSectionContract( + "classical_model_input", + "opt-in regularized input and evaluation contract over enriched tick rows", + target_timeframes=(TICK,), + schema_key="fingerprint_classical_model_input", + basis_values=("regular_grid_from_enriched_ascii_ticks",), + row_order_values=("series_id_period_row_id",), + extra={ + "issue": "#421", + "profile_controlled_by": ["classical_model_input"], + "default_enabled": False, + "advisory": True, + "base_grain": "ascii/T", + "derived_grain": "regular_grid", + "timestamp_required_as_identity": False, + "augmented_column_prefixes": [ + "cm_input_", + "cm_fold_", + "cm_evaluation_", + ], + "model_fitting_in_scope": False, + "aggregations": ["first", "last", "mean", "median"], + "transforms": list(MODEL_TRANSFORM_CODES), + "fold_kinds": ["expanding", "rolling"], + "fit_statuses": list(MODEL_FIT_STATUSES), + "failure_reason_codes": list(MODEL_FAILURE_REASON_CODES), + "row_mapping_policy": ( + "availability_safe_repetition_after_bin_close" + ), + }, + ), + FingerprintTargetSectionContract( + "exponential_smoothing", + "opt-in fitted SES, Holt, Holt-Winters, and ETS diagnostics", + target_timeframes=(TICK,), + schema_key="fingerprint_exponential_smoothing", + basis_values=("regular_grid_rolling_origin",), + row_order_values=("series_id_period_row_id",), + extra={ + "issue": "#422", + "profile_controlled_by": [ + "classical_model_input", + "exponential_smoothing", + ], + "default_enabled": False, + "advisory": True, + "base_grain": "ascii/T", + "derived_grain": "regular_grid", + "optional_dependency_extra": "models", + "backend": "statsmodels", + "model_families": list(EXPONENTIAL_SMOOTHING_FAMILIES), + "failure_reason_codes": list(EXPONENTIAL_SMOOTHING_REASON_CODES), + "augmented_column_prefixes": ["cm_ets_"], + "augmented_columns": [ + { + "name": definition.name, + "dtype": definition.dtype, + "nullable": definition.nullable, + "grain": definition.grain, + "source": definition.source, + } + for definition in training_feature_definitions() + if definition.name in EXPONENTIAL_SMOOTHING_COLUMNS + ], + "automatic_search": False, + "automatic_winner": False, + "row_mapping_policy": ("first_source_row_at_or_after_availability"), + }, + ), + FingerprintTargetSectionContract( + "autoregressive", + "opt-in fitted explicit-order AR, ARMA, and ARIMA diagnostics", + target_timeframes=(TICK,), + schema_key="fingerprint_autoregressive", + basis_values=("regular_grid_rolling_origin",), + row_order_values=("series_id_period_row_id",), + extra={ + "issue": "#423", + "profile_controlled_by": [ + "classical_model_input", + "autoregressive", + ], + "default_enabled": False, + "advisory": True, + "base_grain": "ascii/T", + "derived_grain": "regular_grid", + "optional_dependency_extra": "models", + "backend": "statsmodels", + "model_families": list(AUTOREGRESSIVE_FAMILIES), + "fit_statuses": list(AUTOREGRESSIVE_FIT_STATUS_CODES), + "failure_reason_codes": list(AUTOREGRESSIVE_REASON_CODES), + "augmented_column_prefixes": [ + "cm_ar_", + "cm_arma_", + "cm_arima_", + ], + "augmented_columns": [ + { + "name": definition.name, + "dtype": definition.dtype, + "nullable": definition.nullable, + "grain": definition.grain, + "source": definition.source, + } + for definition in training_feature_definitions() + if definition.name in AUTOREGRESSIVE_COLUMNS + ], + "explicit_order_configuration": True, + "automatic_order_selection": False, + "automatic_winner": False, + "row_mapping_policy": ("first_source_row_at_or_after_availability"), + }, + ), + FingerprintTargetSectionContract( + "seasonal_exogenous", + "opt-in fitted explicit-order SARIMA, ARIMAX, and SARIMAX diagnostics", + target_timeframes=(TICK,), + schema_key="fingerprint_seasonal_exogenous", + basis_values=("regular_grid_rolling_origin",), + row_order_values=("series_id_period_row_id",), + extra={ + "issue": "#424", + "profile_controlled_by": [ + "classical_model_input", + "seasonal_exogenous", + ], + "default_enabled": False, + "advisory": True, + "base_grain": "ascii/T", + "derived_grain": "regular_grid", + "optional_dependency_extra": "models", + "backend": "statsmodels", + "model_families": list(SEASONAL_EXOGENOUS_FAMILIES), + "fit_statuses": list(SEASONAL_EXOGENOUS_FIT_STATUS_CODES), + "failure_reason_codes": list(SEASONAL_EXOGENOUS_REASON_CODES), + "augmented_column_prefixes": [ + "cm_sarima_", + "cm_arimax_", + "cm_sarimax_", + ], + "augmented_columns": [ + { + "name": definition.name, + "dtype": definition.dtype, + "nullable": definition.nullable, + "grain": definition.grain, + "source": definition.source, + } + for definition in training_feature_definitions() + if definition.name in SEASONAL_EXOGENOUS_COLUMNS + ], + "explicit_order_configuration": True, + "deterministic_regressor_contract": True, + "future_regressor_policy": "calendar_known_in_advance_only", + "automatic_order_selection": False, + "automatic_winner": False, + "row_mapping_policy": ("first_source_row_at_or_after_availability"), + }, + ), + FingerprintTargetSectionContract( + "state_space", + "opt-in structural state-space and leakage-safe Kalman diagnostics", + target_timeframes=(TICK,), + schema_key="fingerprint_state_space", + basis_values=("regular_grid_rolling_origin",), + row_order_values=("series_id_period_row_id",), + extra={ + "issue": "#425", + "profile_controlled_by": [ + "classical_model_input", + "state_space", + ], + "default_enabled": False, + "advisory": True, + "base_grain": "ascii/T", + "derived_grain": "regular_grid", + "optional_dependency_extra": "models", + "backend": "statsmodels", + "model_families": list(STATE_SPACE_FAMILIES), + "fit_statuses": list(STATE_SPACE_FIT_STATUS_CODES), + "failure_reason_codes": list(STATE_SPACE_REASON_CODES), + "augmented_column_prefixes": [ + "cm_state_space_", + "cm_kalman_", + ], + "augmented_columns": [ + { + "name": definition.name, + "dtype": definition.dtype, + "nullable": definition.nullable, + "grain": definition.grain, + "source": definition.source, + } + for definition in training_feature_definitions() + if definition.name in (*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS) + ], + "explicit_component_configuration": True, + "filtered_state_policy": "forecast_origin_information_only", + "smoothed_state_policy": "retrospective_diagnostic_only", + "missing_observation_policy": "prediction_only_transition", + "automatic_component_selection": False, + "automatic_winner": False, + "row_mapping_policy": "first_source_row_at_or_after_availability", + }, + ), + FingerprintTargetSectionContract( + "volatility", + "opt-in symmetric ARCH/GARCH conditional-variance diagnostics", + target_timeframes=(TICK,), + schema_key="fingerprint_volatility", + basis_values=("regular_grid_rolling_origin",), + row_order_values=("series_id_period_row_id",), + extra={ + "issue": "#426", + "profile_controlled_by": [ + "classical_model_input", + "volatility", + ], + "default_enabled": False, + "advisory": True, + "base_grain": "ascii/T", + "derived_grain": "regular_grid", + "optional_dependency_extra": "models", + "backend": "arch", + "model_families": list(VOLATILITY_FAMILIES), + "input_definitions": list(VOLATILITY_INPUT_DEFINITIONS), + "mean_models": list(VOLATILITY_MEAN_MODELS), + "distributions": list(VOLATILITY_DISTRIBUTIONS), + "fit_statuses": list(VOLATILITY_FIT_STATUS_CODES), + "failure_reason_codes": list(VOLATILITY_REASON_CODES), + "augmented_column_prefixes": ["cm_arch_", "cm_garch_"], + "augmented_columns": [ + { + "name": definition.name, + "dtype": definition.dtype, + "nullable": definition.nullable, + "grain": definition.grain, + "source": definition.source, + } + for definition in training_feature_definitions() + if definition.name in VOLATILITY_COLUMNS + ], + "explicit_order_configuration": True, + "mean_and_variance_metrics_separate": True, + "realized_variance_proxy_explicit": True, + "asymmetric_extension_registry": cast( + JSONValue, list(ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY) + ), + "asymmetric_fitting_enabled": False, + "automatic_order_selection": False, + "automatic_winner": False, + "row_mapping_policy": "first_source_row_at_or_after_availability", + }, + ), + FingerprintTargetSectionContract( + "classical_model_comparison", + "family-neutral skill, stability, failure, and eligibility reporting", + target_timeframes=(TICK,), + schema_key="fingerprint_classical_model_comparison", + basis_values=("saved_bounded_model_evaluation_artifacts",), + row_order_values=("series_id_period_row_id",), + extra={ + "issue": "#427", + "profile_controlled_by": ["classical_model_comparison"], + "default_enabled": False, + "advisory": True, + "base_grain": "ascii/T", + "derived_grain": "regular_grid_rolling_origin", + "model_fitting_in_scope": False, + "mean_and_variance_metrics_separate": True, + "explicit_reference_baselines": True, + "multiple_horizons": True, + "selection_policy": "none", + "failure_reason_codes": list(COMPARISON_REASON_CODES), + "augmented_column_prefixes": [ + "cm_comparison_", + "cm_skill_", + "cm_stability_", + ], + "augmented_columns": [ + { + "name": definition.name, + "dtype": definition.dtype, + "nullable": definition.nullable, + "grain": definition.grain, + "source": definition.source, + } + for definition in training_feature_definitions() + if definition.name in CLASSICAL_MODEL_COMPARISON_COLUMNS + ], + "row_mapping_policy": "first_source_row_at_or_after_availability", + }, + ), + FingerprintTargetSectionContract( + "synthetic_constraints", + "generator-facing defects, stylized facts, artifacts, and validation contract", + target_timeframes=(TICK,), + schema_key="fingerprint_synthetic_constraints", + basis_values=("enriched_training_frame", "fingerprint_fallback"), + extra={ + "issue": "#81", + "constraint_issue": "#333", + "advisory": True, + "base_grain": "ascii/T", + "generation_in_scope": True, + "generation_method": "empirical_block_bootstrap", + "generator_schema_key": "synthetic_tick_generation", + "non_tick_input_constraints_supported": False, + }, + ), FingerprintTargetSectionContract( "fingerprint_audit", "machine-readable expected/emitted/skipped section accounting and readiness", @@ -568,22 +1666,27 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: ), ) -PLANNED_FINGERPRINT_TARGET_SECTION_CONTRACTS = ( - FingerprintPlannedSectionContract("decomposition", "#330"), - FingerprintPlannedSectionContract("synthetic_constraints", "#333"), -) -PLANNED_FINGERPRINT_RUN_SECTION_CONTRACTS = ( +PLANNED_FINGERPRINT_TARGET_SECTION_CONTRACTS: tuple[ + FingerprintPlannedSectionContract, ... +] = () +IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS = ( FingerprintRunSectionContract( "cross_series_fingerprint", - "planned", + "implemented", CROSS_SERIES_FINGERPRINT_RULE_ID, "#331", ), ) +PLANNED_FINGERPRINT_RUN_SECTION_CONTRACTS: tuple[ + FingerprintRunSectionContract, ... +] = () FINGERPRINT_SECTION_LIMIT_DEFAULTS = { "topology_summary_target_limit": DEFAULT_FINGERPRINT_TOPOLOGY_SUMMARY_LIMIT, "topology_attention_target_limit": DEFAULT_FINGERPRINT_TOPOLOGY_ATTENTION_LIMIT, + "topology_inspection_sample_limit": ( + DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT + ), "distribution_summary_target_limit": DEFAULT_FINGERPRINT_DISTRIBUTION_SUMMARY_LIMIT, "distribution_attention_target_limit": ( DEFAULT_FINGERPRINT_DISTRIBUTION_ATTENTION_LIMIT @@ -601,6 +1704,30 @@ def to_discovery_payload(self) -> dict[str, JSONValue]: "readiness_risk_reason_limit": ( DEFAULT_FINGERPRINT_READINESS_RISK_REASON_LIMIT ), + "parity_summary_target_limit": DEFAULT_FINGERPRINT_PARITY_SUMMARY_LIMIT, + "synthetic_constraint_summary_target_limit": ( + DEFAULT_SYNTHETIC_CONSTRAINT_SUMMARY_LIMIT + ), + "classical_baseline_summary_target_limit": ( + DEFAULT_BASELINE_SUMMARY_TARGET_LIMIT + ), + "classical_model_input_summary_target_limit": ( + DEFAULT_MODEL_INPUT_SUMMARY_TARGET_LIMIT + ), + "exponential_smoothing_summary_target_limit": ( + DEFAULT_EXPONENTIAL_SMOOTHING_SUMMARY_TARGET_LIMIT + ), + "autoregressive_summary_target_limit": ( + DEFAULT_AUTOREGRESSIVE_SUMMARY_TARGET_LIMIT + ), + "seasonal_exogenous_summary_target_limit": ( + DEFAULT_SEASONAL_EXOGENOUS_SUMMARY_TARGET_LIMIT + ), + "state_space_summary_target_limit": DEFAULT_STATE_SPACE_SUMMARY_TARGET_LIMIT, + "volatility_summary_target_limit": DEFAULT_VOLATILITY_SUMMARY_TARGET_LIMIT, + "classical_model_comparison_summary_target_limit": ( + DEFAULT_CLASSICAL_MODEL_COMPARISON_SUMMARY_TARGET_LIMIT + ), } FINGERPRINT_DISTRIBUTION_ATTENTION_DEFAULTS = { "invalid_row_min_count": DEFAULT_FINGERPRINT_DISTRIBUTION_INVALID_ROW_MIN_COUNT, diff --git a/src/histdatacom/data_quality/fingerprint_discovery.py b/src/histdatacom/data_quality/fingerprint_discovery.py index 04ccac92..dc92a280 100644 --- a/src/histdatacom/data_quality/fingerprint_discovery.py +++ b/src/histdatacom/data_quality/fingerprint_discovery.py @@ -25,11 +25,13 @@ FINGERPRINT_SKIP_REASON_CODES, FINGERPRINT_TOPOLOGY_LIMITATIONS, FingerprintReportSurfaceContract, + IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS, IMPLEMENTED_FINGERPRINT_TARGET_SECTION_CONTRACTS, PLANNED_FINGERPRINT_RUN_SECTION_CONTRACTS, PLANNED_FINGERPRINT_TARGET_SECTION_CONTRACTS, ) from histdatacom.data_quality.fingerprints import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, FINGERPRINT_AUDIT_SECTIONS, FINGERPRINT_DYNAMICS_SECTIONS, SERIES_FINGERPRINT_RULE_ID, @@ -244,6 +246,9 @@ def fingerprint_contract_audit( "implemented_target_section_count": len( IMPLEMENTED_FINGERPRINT_TARGET_SECTION_CONTRACTS ), + "implemented_run_section_count": len( + IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS + ), "planned_target_section_count": len( PLANNED_FINGERPRINT_TARGET_SECTION_CONTRACTS ), @@ -595,6 +600,10 @@ def _audit_section_contracts( contract.to_discovery_payload() for contract in IMPLEMENTED_FINGERPRINT_TARGET_SECTION_CONTRACTS ] + expected_implemented_runs = [ + contract.to_discovery_payload() + for contract in IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS + ] expected_planned_targets = [ contract.to_discovery_payload() for contract in PLANNED_FINGERPRINT_TARGET_SECTION_CONTRACTS @@ -611,6 +620,14 @@ def _audit_section_contracts( code="implemented_target_sections_mismatch", message="implemented target section discovery must be generated from the registry", ) + _compare_value( + findings, + path="sections.implemented.run_sections", + expected=expected_implemented_runs, + actual=implemented.get("run_sections"), + code="implemented_run_sections_mismatch", + message="implemented run section discovery must be generated from the registry", + ) _compare_value( findings, path="sections.planned.target_sections", @@ -646,6 +663,9 @@ def _audit_section_contracts( implemented_names = { contract.name for contract in IMPLEMENTED_FINGERPRINT_TARGET_SECTION_CONTRACTS + } | { + contract.name + for contract in IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS } planned_names = { contract.name @@ -666,6 +686,7 @@ def _audit_section_contracts( ) return ( len(expected_implemented) + + len(expected_implemented_runs) + len(expected_planned_targets) + len(expected_planned_runs) + 2, @@ -1240,7 +1261,8 @@ def _schema_payload() -> dict[str, JSONValue]: def _metadata_key_payload() -> dict[str, JSONValue]: return { "finding_metadata": { - "series_fingerprint": TIME_SERIES_FINGERPRINT_METADATA_KEY + "series_fingerprint": TIME_SERIES_FINGERPRINT_METADATA_KEY, + "cross_series_fingerprint": CROSS_SERIES_FINGERPRINT_METADATA_KEY, }, "report_metadata": { surface.key: surface.report_metadata_key @@ -1254,7 +1276,7 @@ def _metadata_key_payload() -> dict[str, JSONValue]: def _target_capability_payload() -> dict[str, JSONValue]: - run_contract = PLANNED_FINGERPRINT_RUN_SECTION_CONTRACTS[0] + run_contract = IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS[0] return { "supported_target_kinds": _json_strings(("csv", "zip", "cache")), "supported_data_format": "ascii", @@ -1277,6 +1299,10 @@ def _section_payload() -> dict[str, JSONValue]: contract.to_discovery_payload() for contract in IMPLEMENTED_FINGERPRINT_TARGET_SECTION_CONTRACTS ], + "run_sections": [ + contract.to_discovery_payload() + for contract in IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS + ], }, "planned": { "target_sections": [ diff --git a/src/histdatacom/data_quality/fingerprint_next_work.py b/src/histdatacom/data_quality/fingerprint_next_work.py new file mode 100644 index 00000000..06c3894b --- /dev/null +++ b/src/histdatacom/data_quality/fingerprint_next_work.py @@ -0,0 +1,958 @@ +"""Bounded next-work recommendations from saved fingerprint reports.""" + +from __future__ import annotations + +import hashlib +from collections import Counter +from collections.abc import Mapping, Sequence +from typing import Any, cast + +from histdatacom.data_quality.contracts import QualityReport, QualityTarget +from histdatacom.data_quality.fingerprint_discovery import ( + fingerprint_schema_discovery, +) +from histdatacom.data_quality.fingerprints import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY, + TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_METADATA_KEY, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY, + TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY, + TIME_SERIES_FINGERPRINT_REGIME_SUMMARY_METADATA_KEY, + TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY, +) +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.reporting import ( + fingerprint_readiness_risk_summary, + quality_report_to_json, +) +from histdatacom.data_quality.training_features import ( + IDENTITY_COLUMNS, + TRAINING_REQUIRED_COLUMNS, + TRAINING_SCHEMA_VERSION, +) +from histdatacom.histdata_ascii import TICK +from histdatacom.publication_safety import ( + publish_safe_json_mapping, + publish_safe_path, +) +from histdatacom.runtime_contracts import JSONValue + +FINGERPRINT_NEXT_WORK_SCHEMA_VERSION = "histdatacom.fingerprint-next-work.v1" +DEFAULT_FINGERPRINT_NEXT_WORK_ALTERNATE_LIMIT = 3 +DEFAULT_FINGERPRINT_NEXT_WORK_TARGET_AXIS_LIMIT = 5 + +_PREREQUISITE_SECTIONS = { + "decomposition": ("stationarity_diagnostics",), + "classical_baseline_diagnostics": ( + "stationarity_diagnostics", + "decomposition", + ), + "synthetic_constraints": ( + "stationarity_diagnostics", + "decomposition", + "cross_series_fingerprint", + "ascii_tick_training_substrate", + ), +} +_PREREQUISITE_ISSUES = { + "synthetic_constraints": ("#331",), +} +_DOWNSTREAM_CONSUMERS = { + "stationarity_diagnostics": ( + "decomposition", + "classical_baseline_diagnostics (#332)", + "synthetic_constraints (#333)", + ), + "decomposition": ( + "classical_baseline_diagnostics (#332)", + "synthetic_constraints (#333)", + ), + "cross_series_fingerprint": ( + "classical_baseline_diagnostics (#332)", + "synthetic_constraints (#333)", + ), +} + + +def fingerprint_next_work_recommendation( + reports: Sequence[tuple[str, QualityReport]], + *, + alternate_limit: int | None = None, + target_axis_limit: int | None = None, + discovery: Mapping[str, JSONValue] | None = None, +) -> dict[str, JSONValue]: + """Recommend bounded fingerprint work from already-saved reports.""" + alternate_state = bounded_report_limit( + alternate_limit, + default_limit=DEFAULT_FINGERPRINT_NEXT_WORK_ALTERNATE_LIMIT, + ) + target_state = bounded_report_limit( + target_axis_limit, + default_limit=DEFAULT_FINGERPRINT_NEXT_WORK_TARGET_AXIS_LIMIT, + ) + discovery_payload = dict(discovery or fingerprint_schema_discovery()) + report_inputs = _report_inputs(reports) + evidence = _collect_evidence(reports) + candidates = _risk_candidates(evidence) + candidates.extend(_report_gap_candidates(evidence)) + candidates.extend(_planned_candidates(evidence, discovery_payload)) + candidates.sort(key=_candidate_sort_key) + + ranked = [ + _ranked_candidate(candidate, rank, target_state) + for rank, candidate in enumerate(candidates, start=1) + ] + recommendation = ranked[0] if ranked else None + all_alternates = ranked[1:] + alternates = alternate_state.slice(all_alternates) + included_count = (1 if recommendation is not None else 0) + len(alternates) + omitted_count = max(0, len(ranked) - included_count) + payload: dict[str, JSONValue] = { + "schema_version": FINGERPRINT_NEXT_WORK_SCHEMA_VERSION, + "status": "recommended" if recommendation else "no_work", + "input_report_count": len(report_inputs), + "input_reports": cast(JSONValue, report_inputs), + "recommendation_count": len(ranked), + "included_recommendation_count": included_count, + "omitted_recommendation_count": omitted_count, + "truncated": omitted_count > 0, + "limit_metadata": { + "alternates": alternate_state.count_payload(len(all_alternates)), + "representative_target_axes": target_state.limit_payload(), + }, + "basis": { + "source": "saved_quality_reports", + "market_data_rescanned": False, + "repository_workflow_inspected": False, + "discovery_schema_version": discovery_payload.get("schema_version"), + "fingerprint_evidence_report_count": evidence[ + "fingerprint_evidence_report_count" + ], + "eligible_ascii_tick_target_count": evidence[ + "eligible_target_count" + ], + "ignored_non_base_target_count": evidence["ignored_target_count"], + "base_grain": {"data_format": "ascii", "timeframe": TICK}, + "training_substrate": evidence["training_substrate"], + "cross_series": evidence["cross_series"], + "saved_surface_presence_counts": evidence[ + "saved_surface_presence_counts" + ], + }, + "recommendation": recommendation, + "alternates": cast(JSONValue, alternates), + "no_work_reason": ( + "no fingerprint evidence or registered product gap was present" + if recommendation is None + else None + ), + "non_goals": [ + "does not rescan market data", + "does not create, update, close, or rank GitHub issues", + "does not inspect CI, branches, pull requests, or releases", + "does not change quality pass/fail status", + ], + } + safe_payload: object = publish_safe_json_mapping(payload) + if not isinstance(safe_payload, Mapping): + return {} + return { + str(key): cast(JSONValue, value) for key, value in safe_payload.items() + } + + +def format_fingerprint_next_work( + payload: Mapping[str, JSONValue], +) -> str: + """Return concise human-readable next-work recommendation text.""" + lines = [ + "Next fingerprint work", + f"reports: {payload.get('input_report_count', 0)}", + ] + recommendation = _mapping(payload.get("recommendation")) + if not recommendation: + lines.append( + f"status: no work ({payload.get('no_work_reason', 'no evidence')})" + ) + return "\n".join(lines) + lines.extend(_format_candidate_lines(recommendation, primary=True)) + alternates = _mapping_rows(payload.get("alternates")) + if alternates: + lines.append("alternates:") + for alternate in alternates: + issue = str(alternate.get("issue_reference") or "") + issue_text = f" {issue}" if issue else "" + lines.append( + f"- #{alternate.get('rank', '?')} " + f"{alternate.get('capability', 'unknown')}{issue_text}: " + f"{alternate.get('rationale', '')}" + ) + omitted = _int(payload.get("omitted_recommendation_count")) + if omitted: + lines.append(f"additional recommendations omitted: {omitted}") + return "\n".join(lines) + + +def _format_candidate_lines( + candidate: Mapping[str, JSONValue], + *, + primary: bool, +) -> list[str]: + issue = str(candidate.get("issue_reference") or "") + issue_text = f" ({issue})" if issue else "" + prefix = "recommendation" if primary else "candidate" + lines = [ + f"{prefix}: #{candidate.get('rank', 1)} " + f"{candidate.get('capability', 'unknown')}{issue_text}", + f"confidence: {candidate.get('confidence', 'unknown')}", + f"why: {candidate.get('rationale', '')}", + f"affected ascii/T targets: {candidate.get('affected_target_count', 0)}", + ] + reasons = _string_rows(candidate.get("reason_codes")) + if reasons: + lines.append("reasons: " + ", ".join(reasons)) + prerequisites = _string_rows(candidate.get("prerequisite_sections")) + if prerequisites: + lines.append("prerequisites: " + ", ".join(prerequisites)) + criteria = _string_rows(candidate.get("suggested_acceptance_criteria")) + if criteria: + lines.append("suggested acceptance criteria:") + lines.extend(f"- {criterion}" for criterion in criteria) + return lines + + +def _report_inputs( + reports: Sequence[tuple[str, QualityReport]], +) -> list[dict[str, JSONValue]]: + inputs: list[dict[str, JSONValue]] = [] + for index, (name, report) in enumerate(reports, start=1): + risk = _saved_risk_summary(report) + encoded = quality_report_to_json(report).encode("utf-8") + inputs.append( + { + "report_name": publish_safe_path(name) or f"report-{index}", + "content_sha256": hashlib.sha256(encoded).hexdigest(), + "target_count": len(report.targets), + "fingerprint_evidence": _has_fingerprint_evidence(report), + "risk_target_count": _int(risk.get("risk_target_count")), + "risk_payload_truncated": risk.get("truncated") is True, + } + ) + return inputs + + +def _collect_evidence( + reports: Sequence[tuple[str, QualityReport]], +) -> dict[str, Any]: + section_rows: dict[str, dict[str, Any]] = {} + section_status_counts: dict[str, Counter[str]] = {} + all_axes: dict[str, dict[str, JSONValue]] = {} + report_surface_gap_count = 0 + truncated_risk_report_count = 0 + fingerprint_evidence_report_count = 0 + eligible_target_count = 0 + ignored_target_count = 0 + cross_payloads: list[Mapping[str, JSONValue]] = [] + training_versions: Counter[str] = Counter() + surface_presence_counts: Counter[str] = Counter() + + for _, report in reports: + if _has_fingerprint_evidence(report): + fingerprint_evidence_report_count += 1 + for target in report.targets: + axis = _target_axis(target) + if _is_ascii_tick_axis(axis): + all_axes[_axis_key(axis)] = axis + eligible_target_count += 1 + else: + ignored_target_count += 1 + risk = _saved_risk_summary(report) + _record_surface_presence(surface_presence_counts, report, risk) + if risk.get("truncated") is True: + truncated_risk_report_count += 1 + report_surface_gap_count += _report_surface_gap_count(risk) + _merge_section_status_counts(section_status_counts, risk) + if not _mapping(risk.get("section_status_counts")): + _merge_readiness_section_status_counts( + section_status_counts, + report, + ) + for target_risk in _mapping_rows(risk.get("target_risks")): + axis = _mapping(target_risk.get("target_axis")) + if not _is_ascii_tick_axis(axis): + continue + safe_axis = cast(dict[str, JSONValue], dict(axis)) + all_axes[_axis_key(safe_axis)] = safe_axis + for section_risk in _mapping_rows(target_risk.get("section_risks")): + _merge_section_risk( + section_rows, + safe_axis, + section_risk, + ) + cross = _mapping( + report.metadata.get(CROSS_SERIES_FINGERPRINT_METADATA_KEY) + ) + if cross: + cross_payloads.append(cross) + version = str( + _mapping(cross.get("row_identity")).get( + "training_schema_version" + ) + or "" + ) + if version: + training_versions[version] += 1 + metadata_version = str( + report.metadata.get("training_schema_version") or "" + ) + if metadata_version: + training_versions[metadata_version] += 1 + for finding in report.findings: + version = str(finding.metadata.get("training_schema_version") or "") + if version: + training_versions[version] += 1 + + cross_series = _cross_series_evidence(cross_payloads) + training_substrate = _training_substrate_evidence( + training_versions, + cross_series, + ) + return { + "section_rows": section_rows, + "section_status_counts": section_status_counts, + "all_axes": list(all_axes.values()), + "report_surface_gap_count": report_surface_gap_count, + "truncated_risk_report_count": truncated_risk_report_count, + "fingerprint_evidence_report_count": ( + fingerprint_evidence_report_count + ), + "eligible_target_count": eligible_target_count, + "ignored_target_count": ignored_target_count, + "cross_series": cross_series, + "training_substrate": training_substrate, + "saved_surface_presence_counts": dict( + sorted(surface_presence_counts.items()) + ), + } + + +def _merge_section_risk( + rows: dict[str, dict[str, Any]], + axis: dict[str, JSONValue], + section_risk: Mapping[str, JSONValue], +) -> None: + section = str(section_risk.get("section") or "unknown") + row = rows.setdefault( + section, + { + "axes": {}, + "reason_counts": Counter(), + "status_counts": Counter(), + "risk_score": 0, + }, + ) + row["axes"][_axis_key(axis)] = axis + row["risk_score"] += _int(section_risk.get("score")) + row["status_counts"][str(section_risk.get("status") or "unknown")] += 1 + reasons = _string_rows(section_risk.get("reasons")) + if not reasons: + reasons = ["section_not_ready"] + row["reason_counts"].update(reasons) + + +def _risk_candidates(evidence: Mapping[str, Any]) -> list[dict[str, Any]]: + candidates = [] + for section, row in evidence["section_rows"].items(): + axes = list(row["axes"].values()) + reason_counts: Counter[str] = row["reason_counts"] + status_counts: Counter[str] = row["status_counts"] + reasons = _ordered_counter_keys(reason_counts) + statuses = _ordered_counter_keys(status_counts) + severity_priority = ( + 500 + if set(statuses) & {"missing", "unavailable", "skipped"} + else 470 + ) + candidates.append( + { + "_priority": severity_priority, + "_score": int(row["risk_score"]), + "kind": "section_readiness", + "capability": section, + "section": section, + "issue_reference": None, + "rationale": ( + f"{len(axes)} ascii/T target(s) report {section} as " + f"{', '.join(statuses)}; address the emitted reasons " + "before advancing dependent fingerprint work." + ), + "affected_target_count": len(axes), + "representative_target_axes": axes, + "reason_codes": reasons, + "reason_counts": dict(sorted(reason_counts.items())), + "prerequisite_sections": [], + "prerequisite_issues": [], + "downstream_consumers": list( + _DOWNSTREAM_CONSUMERS.get(section, ()) + ), + "confidence": "high" if len(axes) > 1 else "medium", + "basis": { + "source": "fingerprint_readiness_risk", + "risk_score": int(row["risk_score"]), + "status_counts": dict(sorted(status_counts.items())), + }, + "suggested_acceptance_criteria": [ + f"Make {section} valid or explicitly not applicable for the affected ascii/T targets.", + "Preserve the enriched single-row training surface and deterministic row identity.", + "Expose bounded reason codes and representative target axes in saved reports.", + "Add deterministic tests for missing, limited, and valid evidence.", + ], + } + ) + return candidates + + +def _report_gap_candidates( + evidence: Mapping[str, Any], +) -> list[dict[str, Any]]: + candidates: list[dict[str, Any]] = [] + axes = list(evidence["all_axes"]) + gap_count = int(evidence["report_surface_gap_count"]) + if gap_count: + candidates.append( + _generic_gap_candidate( + priority=460, + capability="fingerprint_report_surfaces", + rationale=( + f"Saved readiness metadata reports {gap_count} missing or " + "incomplete fingerprint report surface state(s)." + ), + axes=axes, + reasons=["missing_report_surface"], + ) + ) + truncated_count = int(evidence["truncated_risk_report_count"]) + if truncated_count: + candidates.append( + _generic_gap_candidate( + priority=430, + capability="fingerprint_readiness_drill_down", + rationale=( + f"{truncated_count} saved risk report(s) omit ranked target " + "evidence needed to compare all affected sections." + ), + axes=axes, + reasons=["readiness_risk_truncated"], + ) + ) + return candidates + + +def _generic_gap_candidate( + *, + priority: int, + capability: str, + rationale: str, + axes: list[dict[str, JSONValue]], + reasons: list[str], +) -> dict[str, Any]: + return { + "_priority": priority, + "_score": len(axes), + "kind": "report_gap", + "capability": capability, + "section": None, + "issue_reference": None, + "rationale": rationale, + "affected_target_count": len(axes), + "representative_target_axes": axes, + "reason_codes": reasons, + "reason_counts": {reason: 1 for reason in reasons}, + "prerequisite_sections": [], + "prerequisite_issues": [], + "downstream_consumers": [], + "confidence": "high", + "basis": {"source": "saved_report_surface_metadata"}, + "suggested_acceptance_criteria": [ + "Emit the missing report surface from saved fingerprint evidence.", + "Keep the output bounded with explicit truncation metadata.", + "Add JSON, human-rendering, and compatibility tests.", + ], + } + + +def _planned_candidates( + evidence: Mapping[str, Any], + discovery: Mapping[str, JSONValue], +) -> list[dict[str, Any]]: + if not int(evidence["fingerprint_evidence_report_count"]): + return [] + sections = _mapping(discovery.get("sections")) + implemented = _mapping(sections.get("implemented")) + planned = _mapping(sections.get("planned")) + implemented_names = { + str(row.get("name") or "") + for row in ( + _mapping_rows(implemented.get("target_sections")) + + _mapping_rows(implemented.get("run_sections")) + ) + } + candidates = [] + planned_rows = _mapping_rows( + planned.get("target_sections") + ) + _mapping_rows(planned.get("run_sections")) + for row in planned_rows: + capability = str(row.get("name") or "") + if not capability: + continue + prerequisites = list(_PREREQUISITE_SECTIONS.get(capability, ())) + prerequisite_evidence = _prerequisite_evidence( + prerequisites, + implemented_names, + evidence, + ) + blockers = [ + str(item["section"]) + for item in prerequisite_evidence + if item["ready"] is not True + ] + reasons = ( + [f"prerequisite_not_ready:{section}" for section in blockers] + if blockers + else ["registered_planned_capability"] + ) + issue = str(row.get("issue") or "") or None + candidates.append( + { + "_priority": 210 if not blockers else 170, + "_score": int(evidence["eligible_target_count"]), + "kind": "planned_capability", + "capability": capability, + "section": capability, + "issue_reference": issue, + "rationale": _planned_rationale( + capability, + blockers, + int(evidence["eligible_target_count"]), + ), + "affected_target_count": int(evidence["eligible_target_count"]), + "representative_target_axes": list(evidence["all_axes"]), + "reason_codes": reasons, + "reason_counts": {reason: 1 for reason in reasons}, + "prerequisite_sections": prerequisites, + "prerequisite_issues": list( + _PREREQUISITE_ISSUES.get(capability, ()) + ), + "prerequisite_evidence": prerequisite_evidence, + "downstream_consumers": list( + _DOWNSTREAM_CONSUMERS.get(capability, ()) + ), + "confidence": "high" if not blockers else "low", + "basis": { + "source": "fingerprint_discovery_and_saved_reports", + "registry_status": row.get("status"), + "training_substrate": evidence["training_substrate"], + "cross_series": evidence["cross_series"], + }, + "suggested_acceptance_criteria": _planned_acceptance_criteria( + capability + ), + } + ) + return candidates + + +def _prerequisite_evidence( + prerequisites: Sequence[str], + implemented_names: set[str], + evidence: Mapping[str, Any], +) -> list[dict[str, JSONValue]]: + rows: list[dict[str, JSONValue]] = [] + status_counts = evidence["section_status_counts"] + cross = evidence["cross_series"] + for section in prerequisites: + if section == "ascii_tick_training_substrate": + training = evidence["training_substrate"] + observed_status = str( + training.get("training_facing_columns_status") or "unknown" + ) + rows.append( + { + "section": section, + "implemented": training.get( + "legacy_raw_cache_enrichment_on_read_supported" + ) + is True, + "observed_valid_target_count": _int( + evidence.get("eligible_target_count") + ), + "ready": observed_status in {"confirmed", "partial"}, + "basis": f"training_columns_{observed_status}", + } + ) + continue + if section == "cross_series_fingerprint": + observed_count = _int(cross.get("observed_report_count")) + ready = section in implemented_names and observed_count > 0 + rows.append( + { + "section": section, + "implemented": section in implemented_names, + "observed_valid_target_count": observed_count, + "ready": ready, + "basis": "cross_series_report_metadata", + } + ) + continue + counts: Counter[str] = status_counts.get(section, Counter()) + valid_count = counts.get("valid", 0) + counts.get("computed", 0) + ready = section in implemented_names and valid_count > 0 + rows.append( + { + "section": section, + "implemented": section in implemented_names, + "observed_valid_target_count": valid_count, + "ready": ready, + "basis": "readiness_section_status_counts", + } + ) + return rows + + +def _planned_rationale( + capability: str, + blockers: Sequence[str], + target_count: int, +) -> str: + if blockers: + return ( + f"{capability} is registered as planned, but saved evidence does " + f"not yet confirm {', '.join(blockers)}; complete those product " + "prerequisites first." + ) + return ( + f"{capability} is the next registered capability and its known " + f"prerequisites are present for {target_count} ascii/T target(s)." + ) + + +def _planned_acceptance_criteria(capability: str) -> list[str]: + criteria = [ + f"Implement {capability} from existing fingerprint and readiness evidence without rescanning in this recommendation command.", + "Preserve ascii/T as the only base grain and the enriched single-row training surface.", + "Keep row identity durable across duplicate timestamps and legacy cache enrichment.", + "Emit deterministic bounded JSON and concise human-readable evidence.", + ] + if capability in {"synthetic_constraints", "cross_series_fingerprint"}: + criteria.append( + "Include bounded duplicate-timestamp, unequal-range, and triangle evidence." + ) + return criteria + + +def _ranked_candidate( + candidate: Mapping[str, Any], + rank: int, + target_state: Any, +) -> dict[str, JSONValue]: + result = { + key: cast(JSONValue, value) + for key, value in candidate.items() + if not key.startswith("_") and key != "representative_target_axes" + } + axes = list(candidate.get("representative_target_axes", ())) + included_axes = target_state.slice(axes) + result.update( + { + "rank": rank, + "representative_target_axes": cast(JSONValue, included_axes), + "target_axis_limit_metadata": target_state.count_payload(len(axes)), + } + ) + return result + + +def _candidate_sort_key(candidate: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + -int(candidate.get("_priority", 0)), + -int(candidate.get("_score", 0)), + -int(candidate.get("affected_target_count", 0)), + str(candidate.get("capability") or ""), + str(candidate.get("issue_reference") or ""), + ) + + +def _saved_risk_summary(report: QualityReport) -> dict[str, JSONValue]: + saved = _mapping( + report.metadata.get(TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY) + ) + if saved: + return cast(dict[str, JSONValue], dict(saved)) + computed = fingerprint_readiness_risk_summary( + report, + target_limit=-1, + section_limit=-1, + reason_limit=-1, + ) + return computed or {} + + +def _has_fingerprint_evidence(report: QualityReport) -> bool: + if _saved_risk_summary(report): + return True + if _mapping( + report.metadata.get( + TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY + ) + ): + return True + return any( + _mapping(finding.metadata.get(TIME_SERIES_FINGERPRINT_METADATA_KEY)) + for finding in report.findings + ) + + +def _merge_section_status_counts( + target: dict[str, Counter[str]], + risk: Mapping[str, JSONValue], +) -> None: + for section, raw_counts in _mapping( + risk.get("section_status_counts") + ).items(): + counts = target.setdefault(section, Counter()) + for status, count in _mapping(raw_counts).items(): + counts[status] += _int(count) + + +def _merge_readiness_section_status_counts( + target: dict[str, Counter[str]], + report: QualityReport, +) -> None: + readiness = _mapping( + report.metadata.get( + TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY + ) + ) + for summary in _mapping_rows(readiness.get("target_summaries")): + axis = _mapping(summary.get("target_axis")) + if not _is_ascii_tick_axis(axis): + continue + for section, status in _mapping( + summary.get("section_statuses") + ).items(): + target.setdefault(section, Counter())[str(status)] += 1 + + +def _record_surface_presence( + counts: Counter[str], + report: QualityReport, + risk: Mapping[str, JSONValue], +) -> None: + surfaces = { + "coverage": TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY, + "topology": TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY, + "distribution": ( + TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_METADATA_KEY + ), + "regime": TIME_SERIES_FINGERPRINT_REGIME_SUMMARY_METADATA_KEY, + "readiness": TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY, + "readiness_risk": (TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY), + "cross_series": CROSS_SERIES_FINGERPRINT_METADATA_KEY, + } + for name, key in surfaces.items(): + if _mapping(report.metadata.get(key)): + counts[name] += 1 + if risk and not _mapping( + report.metadata.get(TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY) + ): + counts["readiness_risk"] += 1 + for finding in report.findings: + fingerprint = _mapping( + finding.metadata.get(TIME_SERIES_FINGERPRINT_METADATA_KEY) + ) + for section in ( + "conditional_distributions", + "microstructure_dynamics", + "dependence", + "stationarity_diagnostics", + "decomposition", + ): + if _mapping(fingerprint.get(section)): + counts[section] += 1 + + +def _report_surface_gap_count(risk: Mapping[str, JSONValue]) -> int: + evidence = _mapping(risk.get("report_surface_evidence")) + gap_count = 0 + for key in ( + "report_metadata_state_counts", + "bounded_payload_state_counts", + "cli_summary_state_counts", + ): + for state, count in _mapping(evidence.get(key)).items(): + if state != "present": + gap_count += _int(count) + return gap_count + + +def _cross_series_evidence( + payloads: Sequence[Mapping[str, JSONValue]], +) -> dict[str, JSONValue]: + identity_columns: set[str] = set() + training_versions: Counter[str] = Counter() + status_counts: Counter[str] = Counter() + group_count = 0 + incomplete_group_count = 0 + duplicate_timestamp_row_count = 0 + unequal_range_group_count = 0 + triangle_candidate_count = 0 + triangle_compared_timestamp_count = 0 + cache_source_count = 0 + for payload in payloads: + status_counts[str(payload.get("status") or "unknown")] += 1 + group_count += _int(payload.get("group_count")) + incomplete_group_count += _int(payload.get("incomplete_group_count")) + row_identity = _mapping(payload.get("row_identity")) + identity_columns.update(_string_rows(row_identity.get("columns"))) + duplicate_timestamp_row_count += _int( + row_identity.get("duplicate_timestamp_row_count") + ) + version = str(row_identity.get("training_schema_version") or "") + if version: + training_versions[version] += 1 + cache_source_count += sum( + _int(value) + for value in _mapping(payload.get("cache_source_counts")).values() + ) + triangular = _mapping(payload.get("triangular_consistency")) + triangle_candidate_count += _int(triangular.get("candidate_count")) + triangle_compared_timestamp_count += _int( + triangular.get("compared_timestamp_count") + ) + for group in _mapping_rows(payload.get("groups")): + coverage = _mapping(group.get("coverage_ranges")) + if coverage.get("unequal_ranges") is True: + unequal_range_group_count += 1 + for panel in _mapping_rows(payload.get("panel_coverage")): + if panel.get("unequal_period_ranges") is True: + unequal_range_group_count += 1 + return { + "observed_report_count": len(payloads), + "status_counts": dict(sorted(status_counts.items())), + "group_count": group_count, + "incomplete_group_count": incomplete_group_count, + "row_identity_columns": cast( + JSONValue, + sorted(identity_columns), + ), + "training_schema_version_counts": dict( + sorted(training_versions.items()) + ), + "duplicate_timestamp_row_count": duplicate_timestamp_row_count, + "unequal_range_group_count": unequal_range_group_count, + "triangle_candidate_count": triangle_candidate_count, + "triangle_compared_timestamp_count": triangle_compared_timestamp_count, + "cache_source_count": cache_source_count, + } + + +def _training_substrate_evidence( + versions: Counter[str], + cross_series: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + for version, count in _mapping( + cross_series.get("training_schema_version_counts") + ).items(): + versions[version] += _int(count) + identity_columns = set( + _string_rows(cross_series.get("row_identity_columns")) + ) + current_count = versions.get(TRAINING_SCHEMA_VERSION, 0) + if current_count: + column_status = "confirmed" + elif identity_columns: + column_status = "partial" + else: + column_status = "unknown" + required_identity = set(IDENTITY_COLUMNS) + observed_identity = required_identity & identity_columns + cache_source_count = _int(cross_series.get("cache_source_count")) + return { + "schema_version": TRAINING_SCHEMA_VERSION, + "observed_schema_version_counts": dict(sorted(versions.items())), + "training_facing_columns_status": column_status, + "required_column_count": len(TRAINING_REQUIRED_COLUMNS), + "required_identity_column_count": len(required_identity), + "observed_identity_column_count": len(observed_identity), + "observed_identity_columns": cast( + JSONValue, + sorted(observed_identity), + ), + "single_row_training_surface": True, + "legacy_raw_cache_enrichment_on_read_supported": True, + "observed_cache_projection_count": cache_source_count, + "observed_enriched_cache_projection": ( + cache_source_count > 0 and current_count > 0 + ), + "timestamp_is_durable_identity": False, + } + + +def _target_axis(target: QualityTarget) -> dict[str, JSONValue]: + return { + "data_format": target.data_format, + "symbol": target.symbol, + "timeframe": target.timeframe, + "period": target.period, + "kind": target.kind.value, + } + + +def _is_ascii_tick_axis(axis: Mapping[str, JSONValue]) -> bool: + return ( + str(axis.get("data_format") or "").lower() == "ascii" + and str(axis.get("timeframe") or "").upper() == TICK + ) + + +def _axis_key(axis: Mapping[str, JSONValue]) -> str: + return "|".join( + str(axis.get(key) or "") + for key in ("data_format", "symbol", "timeframe", "period", "kind") + ) + + +def _ordered_counter_keys(counter: Counter[str]) -> list[str]: + return [ + key + for key, _ in sorted( + counter.items(), key=lambda item: (-item[1], item[0]) + ) + ] + + +def _mapping(value: object) -> dict[str, Any]: + if isinstance(value, Mapping): + return {str(key): item for key, item in value.items()} + return {} + + +def _mapping_rows(value: object) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + return [_mapping(item) for item in value if isinstance(item, Mapping)] + + +def _string_rows(value: object) -> list[str]: + if not isinstance(value, (list, tuple)): + return [] + return [str(item) for item in value if str(item)] + + +def _int(value: object) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int): + return value + return 0 diff --git a/src/histdatacom/data_quality/fingerprints.py b/src/histdatacom/data_quality/fingerprints.py index 5b102934..f2d733a4 100644 --- a/src/histdatacom/data_quality/fingerprints.py +++ b/src/histdatacom/data_quality/fingerprints.py @@ -11,6 +11,8 @@ from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from decimal import Decimal, InvalidOperation, localcontext +from io import StringIO +from itertools import islice from pathlib import Path from typing import Any, cast @@ -26,6 +28,7 @@ SESSION_MARKET_CLOSED, SESSION_NO_ACTIVE_WINDOW, SESSION_STATE_WEEKEND_CLOSURE, + SOURCE_WEEKDAY_NAMES, calendar_regime_payload_for_target, classify_histdata_source_timestamp, classify_histdata_timestamp, @@ -34,16 +37,69 @@ HistDataCalendarProfile, default_calendar_profile, ) +from histdatacom.data_quality.classical_baselines import ( + ClassicalBaselineProfile, + classical_baseline_diagnostics_from_training_frame, +) +from histdatacom.data_quality.classical_model_contracts import ( + ClassicalModelInputProfile, + build_classical_model_input, +) +from histdatacom.data_quality.classical_model_comparison import ( + ClassicalModelComparisonProfile, + classical_model_comparison_from_saved_results, +) +from histdatacom.data_quality.autoregressive import ( + AutoregressiveProfile, + autoregressive_from_model_input, +) +from histdatacom.data_quality.exponential_smoothing import ( + ExponentialSmoothingProfile, + exponential_smoothing_from_model_input, +) from histdatacom.data_quality.limits import ( BoundedReportLimit, bounded_report_limit, ) -from histdatacom.data_quality.polars_cache import read_quality_polars_cache +from histdatacom.data_quality.polars_cache import ( + read_fingerprint_parity_polars_cache, + read_quality_polars_cache, +) +from histdatacom.data_quality.seasonal_exogenous import ( + SeasonalExogenousProfile, + seasonal_exogenous_from_model_input, +) +from histdatacom.data_quality.state_space import ( + StateSpaceProfile, + state_space_from_model_input, +) +from histdatacom.data_quality.volatility import ( + VolatilityProfile, + volatility_from_model_input, +) from histdatacom.data_quality.remediation import ( remediation_hint_payloads_for_flags, ) -from histdatacom.data_quality.symbols import symbol_metadata_for -from histdatacom.data_quality.time import timestamp_topology_payload_for_target +from histdatacom.data_quality.symbols import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY as CROSS_SERIES_FINGERPRINT_METADATA_KEY, + CROSS_SERIES_FINGERPRINT_RULE_ID as CROSS_SERIES_FINGERPRINT_RULE_ID, + CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION as CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, + HistDataCrossSeriesFingerprintRule as HistDataCrossSeriesFingerprintRule, + symbol_metadata_for, +) +from histdatacom.data_quality.synthetic_constraints import ( + synthetic_constraints_from_fingerprint, +) +from histdatacom.data_quality.time import ( + DEFAULT_TIMESTAMP_INSPECTION_SAMPLE_LIMIT, + timestamp_topology_payload_for_target, +) +from histdatacom.data_quality.training_features import ( + TRAINING_REQUIRED_COLUMNS, + TRAINING_SCHEMA_VERSION, + ensure_tick_training_features, + quality_report_from_training_features, +) from histdatacom.data_quality.ticks import ( DEFAULT_TICK_MICROSTRUCTURE_THRESHOLDS, DEFAULT_TICK_SPREAD_REGIME_THRESHOLDS, @@ -52,7 +108,10 @@ TICK, columns_for_timeframe, delimiter_for_timeframe, + format_influx_line, normalize_ascii_row, + parse_ascii_lines, + to_polars_frame, ) from histdatacom.publication_safety import publish_safe_path from histdatacom.runtime_contracts import JSONValue @@ -90,6 +149,12 @@ TIME_SERIES_FINGERPRINT_STATIONARITY_SCHEMA_VERSION = ( "histdatacom.time-series-fingerprint-stationarity.v1" ) +TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION = ( + "histdatacom.time-series-fingerprint-decomposition.v1" +) +TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.time-series-fingerprint-decomposition-training-projection.v1" +) TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION = ( "histdatacom.time-series-fingerprint-audit.v1" ) @@ -99,6 +164,12 @@ TIME_SERIES_FINGERPRINT_READINESS_RISK_SCHEMA_VERSION = ( "histdatacom.time-series-fingerprint-readiness-risk.v1" ) +TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION = ( + "histdatacom.time-series-fingerprint-cache-source-parity.v1" +) +TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.time-series-fingerprint-cache-source-parity-summary.v1" +) TIME_SERIES_FINGERPRINT_METADATA_KEY = "time_series_fingerprint" TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY = ( "time_series_fingerprint_coverage" @@ -124,8 +195,10 @@ TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY = ( "time_series_fingerprint_readiness_risk" ) +TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_cache_source_parity_summary" +) SERIES_FINGERPRINT_RULE_ID = "fingerprint.series" -CROSS_SERIES_FINGERPRINT_RULE_ID = "fingerprint.cross_series" SERIES_FINGERPRINT_SUMMARY_CODE = "FINGERPRINT_SERIES_SUMMARY" SERIES_FINGERPRINT_SOURCE_UNAVAILABLE_CODE = "FINGERPRINT_SOURCE_UNAVAILABLE" @@ -143,6 +216,9 @@ DEFAULT_FINGERPRINT_HISTOGRAM_BINS = 32 DEFAULT_FINGERPRINT_MAX_ROWS = 1_000_000 DEFAULT_FINGERPRINT_ROUNDING_DIGITS = 12 +DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT = ( + DEFAULT_TIMESTAMP_INSPECTION_SAMPLE_LIMIT +) DEFAULT_FINGERPRINT_TOPOLOGY_SUMMARY_LIMIT = 128 DEFAULT_FINGERPRINT_TOPOLOGY_ATTENTION_LIMIT = 32 DEFAULT_FINGERPRINT_DISTRIBUTION_SUMMARY_LIMIT = 128 @@ -162,6 +238,9 @@ DEFAULT_FINGERPRINT_DISTRIBUTION_NEGATIVE_SPREAD_MIN_RATE = 0.0 DEFAULT_FINGERPRINT_DISTRIBUTION_FLAG_TRUNCATED = True DEFAULT_FINGERPRINT_DISTRIBUTION_FLAG_CACHE_FLOAT_PRECISION = True +DEFAULT_FINGERPRINT_PARITY_MISMATCH_LIMIT = 16 +DEFAULT_FINGERPRINT_PARITY_SUMMARY_LIMIT = 32 +_CALENDAR_POLICY_CONTEXT_TEXT_LIMIT = 128 SUPPORTED_SERIES_FINGERPRINT_TIMEFRAMES = (TICK,) SUPPORTED_SERIES_FINGERPRINT_KINDS = ( QualityTargetKind.CSV, @@ -196,6 +275,17 @@ "microstructure_dynamics", "dependence", "stationarity_diagnostics", + "decomposition", + "cache_source_parity", + "classical_baselines", + "classical_model_input", + "exponential_smoothing", + "autoregressive", + "seasonal_exogenous", + "state_space", + "volatility", + "classical_model_comparison", + "synthetic_constraints", ) FINGERPRINT_DYNAMICS_SECTIONS = ("microstructure_dynamics",) @@ -243,6 +333,21 @@ def to_metadata(self) -> dict[str, JSONValue]: } +@dataclass(frozen=True, slots=True) +class HistDataFingerprintParityProfile: + """Opt-in bounded cache/source parity controls.""" + + enabled: bool = False + mismatch_limit: int = DEFAULT_FINGERPRINT_PARITY_MISMATCH_LIMIT + + def to_metadata(self) -> dict[str, JSONValue]: + """Return a JSON-compatible representation.""" + return { + "enabled": self.enabled, + "mismatch_limit": self.mismatch_limit, + } + + @dataclass(frozen=True, slots=True) class HistDataFingerprintProfile: """Operator-tunable limits for deterministic fingerprint summaries.""" @@ -253,6 +358,9 @@ class HistDataFingerprintProfile: histogram_bins: int = DEFAULT_FINGERPRINT_HISTOGRAM_BINS max_rows: int = DEFAULT_FINGERPRINT_MAX_ROWS rounding_digits: int = DEFAULT_FINGERPRINT_ROUNDING_DIGITS + topology_inspection_sample_limit: int = ( + DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT + ) calendar_profile: HistDataCalendarProfile = field( default_factory=default_calendar_profile, repr=False, @@ -261,6 +369,29 @@ class HistDataFingerprintProfile: distribution_attention: HistDataFingerprintDistributionAttentionProfile = ( field(default_factory=HistDataFingerprintDistributionAttentionProfile) ) + cache_source_parity: HistDataFingerprintParityProfile = field( + default_factory=HistDataFingerprintParityProfile + ) + classical_baselines: ClassicalBaselineProfile = field( + default_factory=ClassicalBaselineProfile + ) + classical_model_input: ClassicalModelInputProfile = field( + default_factory=ClassicalModelInputProfile + ) + exponential_smoothing: ExponentialSmoothingProfile = field( + default_factory=ExponentialSmoothingProfile + ) + autoregressive: AutoregressiveProfile = field( + default_factory=AutoregressiveProfile + ) + seasonal_exogenous: SeasonalExogenousProfile = field( + default_factory=SeasonalExogenousProfile + ) + state_space: StateSpaceProfile = field(default_factory=StateSpaceProfile) + volatility: VolatilityProfile = field(default_factory=VolatilityProfile) + classical_model_comparison: ClassicalModelComparisonProfile = field( + default_factory=ClassicalModelComparisonProfile + ) def to_metadata(self) -> dict[str, JSONValue]: """Return a JSON-compatible representation.""" @@ -271,9 +402,23 @@ def to_metadata(self) -> dict[str, JSONValue]: "histogram_bins": self.histogram_bins, "max_rows": self.max_rows, "rounding_digits": self.rounding_digits, + "topology_inspection_sample_limit": ( + self.topology_inspection_sample_limit + ), "distribution_attention": ( self.distribution_attention.to_metadata() ), + "cache_source_parity": self.cache_source_parity.to_metadata(), + "classical_baselines": self.classical_baselines.to_metadata(), + "classical_model_input": (self.classical_model_input.to_metadata()), + "exponential_smoothing": self.exponential_smoothing.to_metadata(), + "autoregressive": self.autoregressive.to_metadata(), + "seasonal_exogenous": self.seasonal_exogenous.to_metadata(), + "state_space": self.state_space.to_metadata(), + "volatility": self.volatility.to_metadata(), + "classical_model_comparison": ( + self.classical_model_comparison.to_metadata() + ), } @@ -839,6 +984,127 @@ def series_fingerprint_regime_summary( } +def series_fingerprint_parity_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = DEFAULT_FINGERPRINT_PARITY_SUMMARY_LIMIT, +) -> dict[str, JSONValue] | None: + """Return a bounded cache/source parity rollup from fingerprint findings.""" + target_limit_state = bounded_report_limit( + target_limit, + default_limit=DEFAULT_FINGERPRINT_PARITY_SUMMARY_LIMIT, + ) + targets: list[dict[str, JSONValue]] = [] + status_counts: Counter[str] = Counter() + mismatch_code_counts: Counter[str] = Counter() + cache_source_counts: Counter[str] = Counter() + freshness_counts: Counter[str] = Counter() + computed_from_counts: Counter[str] = Counter() + for finding in findings: + fingerprint = _payload_mapping( + finding.metadata.get(TIME_SERIES_FINGERPRINT_METADATA_KEY) + ) + parity = _payload_mapping(fingerprint.get("cache_source_parity")) + if not parity: + continue + bases = _payload_mapping(parity.get("bases")) + raw_cache = _payload_mapping(bases.get("raw_cache")) + enriched_cache = _payload_mapping(bases.get("enriched_cache")) + calendar = _payload_mapping(fingerprint.get("calendar_regimes")) + status = _summary_key(parity.get("status")) + status_counts[status] += 1 + mismatch_codes = [ + str(item) for item in _string_list(parity.get("mismatch_codes")) + ] + mismatch_code_counts.update(mismatch_codes) + cache_source = _optional_summary_key(raw_cache.get("cache_source")) + if cache_source: + cache_source_counts[cache_source] += 1 + freshness = _optional_summary_key(raw_cache.get("freshness")) + if freshness: + freshness_counts[freshness] += 1 + computed_from = _optional_summary_key(calendar.get("computed_from")) + if computed_from: + computed_from_counts[computed_from] += 1 + target_summary: dict[str, JSONValue] = { + "target_axis": dict(_payload_mapping(parity.get("target_axis"))), + "status": status, + "compared_section_count": _int_payload( + parity.get("compared_section_count") + ), + "mismatched_section_count": _int_payload( + parity.get("mismatched_section_count") + ), + "mismatch_codes": cast(JSONValue, mismatch_codes), + "skipped_reasons": cast( + JSONValue, + [ + str(item) + for item in _string_list(parity.get("skipped_reasons")) + ], + ), + "cache_source": cache_source, + "freshness": freshness, + "computed_from": computed_from, + "training_substrate": { + "status": enriched_cache.get("status"), + "training_schema_version": enriched_cache.get( + "training_schema_version" + ), + "cache_was_enriched": enriched_cache.get("cache_was_enriched"), + "legacy_cache_enriched_on_read": enriched_cache.get( + "legacy_cache_enriched_on_read" + ), + }, + } + targets.append(target_summary) + if not targets: + return None + targets.sort(key=_fingerprint_parity_target_sort_key) + included: list[JSONValue] = [ + dict(item) for item in target_limit_state.slice(targets) + ] + omitted_count = max(0, len(targets) - len(included)) + return { + "schema_version": TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION, + "rule_id": SERIES_FINGERPRINT_RULE_ID, + "target_count": len(targets), + "compared_target_count": sum( + count + for status, count in status_counts.items() + if status in {"match", "mismatch"} + ), + "matching_target_count": status_counts.get("match", 0), + "mismatched_target_count": status_counts.get("mismatch", 0), + "not_compared_target_count": status_counts.get("not_compared", 0), + "included_target_count": len(included), + "omitted_target_count": omitted_count, + "truncated": omitted_count > 0, + "limit_metadata": {"targets": target_limit_state.limit_payload()}, + "status_counts": _counter_payload(status_counts), + "mismatch_code_counts": _counter_payload(mismatch_code_counts), + "computed_from_counts": _counter_payload(computed_from_counts), + "cache_source_counts": _counter_payload(cache_source_counts), + "freshness_counts": _counter_payload(freshness_counts), + "target_summaries": included, + } + + +def _fingerprint_parity_target_sort_key( + target: Mapping[str, JSONValue], +) -> tuple[int, str, str, str, str, str]: + status_rank = {"mismatch": 0, "not_compared": 1, "match": 2} + axis = _payload_mapping(target.get("target_axis")) + return ( + status_rank.get(_summary_key(target.get("status")), 99), + _summary_key(axis.get("data_format")), + _summary_key(axis.get("timeframe")), + _summary_key(axis.get("symbol")), + _summary_key(axis.get("period")), + _summary_key(axis.get("kind")), + ) + + def series_fingerprint_readiness_summary( findings: Iterable[QualityFinding], *, @@ -898,6 +1164,16 @@ def series_fingerprint_readiness_summary( stationarity_recommended_transform_counts: Counter[str] = Counter() stationarity_computed_window_count = 0 stationarity_skipped_window_count = 0 + decomposition_status_counts: Counter[str] = Counter() + decomposition_reason_counts: Counter[str] = Counter() + decomposition_basis_counts: Counter[str] = Counter() + decomposition_limitation_counts: Counter[str] = Counter() + decomposition_skipped_window_reason_counts: Counter[str] = Counter() + decomposition_stationarity_status_counts: Counter[str] = Counter() + decomposition_structural_break_status_counts: Counter[str] = Counter() + decomposition_computed_window_count = 0 + decomposition_skipped_window_count = 0 + decomposition_structural_break_candidate_count = 0 topology_limitation_counts: Counter[str] = Counter() dynamics_limitation_counts: Counter[str] = Counter() row_order_counts: Counter[str] = Counter() @@ -991,6 +1267,53 @@ def series_fingerprint_readiness_summary( stationarity_skipped_window_count += _int_payload( stationarity.get("skipped_window_count") ) + decomposition = _payload_mapping(item.get("decomposition")) + decomposition_status_counts[ + _summary_key(decomposition.get("status")) + ] += 1 + decomposition_reason = _optional_summary_key( + decomposition.get("reason") + ) + if decomposition_reason: + decomposition_reason_counts[decomposition_reason] += 1 + decomposition_basis = _summary_key( + decomposition.get("calculation_basis") + ) + if decomposition_basis != "unknown": + decomposition_basis_counts[decomposition_basis] += 1 + decomposition_limitation_counts.update( + _summary_key(value) + for value in _string_list(decomposition.get("limitations")) + ) + decomposition_skipped_window_reason_counts.update( + _counter_from_mapping( + _payload_mapping( + decomposition.get("skipped_window_reason_counts") + ) + ) + ) + decomposition_computed_window_count += _int_payload( + decomposition.get("computed_window_count") + ) + decomposition_skipped_window_count += _int_payload( + decomposition.get("skipped_window_count") + ) + decomposition_stationarity_status_counts[ + _summary_key( + _payload_mapping(decomposition.get("stationarity")).get( + "status" + ) + ) + ] += 1 + structural_break = _payload_mapping( + decomposition.get("structural_break") + ) + decomposition_structural_break_status_counts[ + _summary_key(structural_break.get("status")) + ] += 1 + decomposition_structural_break_candidate_count += _int_payload( + structural_break.get("candidate_count") + ) profile_complete_count = sum( 1 @@ -1067,6 +1390,36 @@ def series_fingerprint_readiness_summary( "stationarity_skipped_window_count": ( stationarity_skipped_window_count ), + "decomposition_status_counts": _counter_payload( + decomposition_status_counts + ), + "decomposition_reason_counts": _counter_payload( + decomposition_reason_counts + ), + "decomposition_basis_counts": _counter_payload( + decomposition_basis_counts + ), + "decomposition_limitation_counts": _counter_payload( + decomposition_limitation_counts + ), + "decomposition_skipped_window_reason_counts": _counter_payload( + decomposition_skipped_window_reason_counts + ), + "decomposition_stationarity_status_counts": _counter_payload( + decomposition_stationarity_status_counts + ), + "decomposition_structural_break_status_counts": _counter_payload( + decomposition_structural_break_status_counts + ), + "decomposition_computed_window_count": ( + decomposition_computed_window_count + ), + "decomposition_skipped_window_count": ( + decomposition_skipped_window_count + ), + "decomposition_structural_break_candidate_count": ( + decomposition_structural_break_candidate_count + ), "topology_limitation_counts": _counter_payload( topology_limitation_counts ), @@ -1249,6 +1602,7 @@ def _fingerprint_readiness_risk_target( "conditional_distributions", "dependence", "stationarity_diagnostics", + "decomposition", "temporal_topology", *FINGERPRINT_DYNAMICS_SECTIONS, }: @@ -1269,6 +1623,7 @@ def _fingerprint_readiness_risk_target( _add_dynamics_risks(section_risks, reason_counts, target, axis) _add_dependence_risks(section_risks, reason_counts, target) _add_stationarity_risks(section_risks, reason_counts, target) + _add_decomposition_risks(section_risks, reason_counts, target) _add_regime_risks(section_risks, reason_counts, regime or {}) compact_sections = _bounded_section_risks( section_risks, @@ -1365,6 +1720,8 @@ def _fingerprint_section_is_non_applicable( and timeframe != TICK or section == "stationarity_diagnostics" and timeframe not in SUPPORTED_SERIES_FINGERPRINT_TIMEFRAMES + or section == "decomposition" + and timeframe not in SUPPORTED_SERIES_FINGERPRINT_TIMEFRAMES ) @@ -1455,6 +1812,53 @@ def _add_stationarity_risks( ) +def _add_decomposition_risks( + section_risks: list[dict[str, JSONValue]], + reason_counts: Counter[str], + target: Mapping[str, JSONValue], +) -> None: + section_statuses = _payload_mapping(target.get("section_statuses")) + if "decomposition" not in section_statuses: + return + decomposition = _payload_mapping(target.get("decomposition")) + status = _summary_key(decomposition.get("status")) + reasons: list[str] = [] + primary_reason = _optional_summary_key(decomposition.get("reason")) + if primary_reason: + reasons.append(primary_reason) + reasons.extend( + _summary_key(reason) + for reason in _string_list(decomposition.get("limitations")) + ) + skipped_window_count = _int_payload( + decomposition.get("skipped_window_count") + ) + if skipped_window_count: + reasons.append("skipped_rolling_windows") + reasons.extend( + _counter_from_mapping( + _payload_mapping( + decomposition.get("skipped_window_reason_counts") + ) + ).elements() + ) + if status in {"valid", "ok"} and not reasons: + return + if status == "skipped" and not reasons: + reasons.append("not_emitted") + _add_section_risk( + section_risks, + reason_counts, + section="decomposition", + status=status, + reasons=tuple(_ordered_unique(reasons)), + base_score=( + _fingerprint_status_risk_score(status) + + min(20, skipped_window_count * 2) + ), + ) + + def _add_regime_risks( section_risks: list[dict[str, JSONValue]], reason_counts: Counter[str], @@ -1779,6 +2183,12 @@ def _fingerprint_readiness_target_summary( section_statuses=section_statuses, skipped_sections=skipped_sections, ) + decomposition_readiness = _fingerprint_readiness_decomposition_summary( + finding, + payload, + section_statuses=section_statuses, + skipped_sections=skipped_sections, + ) return { "target_axis": target_axis, @@ -1816,6 +2226,7 @@ def _fingerprint_readiness_target_summary( ), "dependence": dependence_readiness, "stationarity_diagnostics": stationarity_readiness, + "decomposition": decomposition_readiness, } @@ -2287,6 +2698,156 @@ def _empty_stationarity_readiness( } +def _fingerprint_readiness_decomposition_summary( + finding: QualityFinding, + payload: Mapping[str, JSONValue], + *, + section_statuses: Mapping[str, JSONValue], + skipped_sections: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + decomposition = _payload_mapping(payload.get("decomposition")) + if not decomposition: + status = _summary_key( + section_statuses.get("decomposition") or "skipped" + ) + if status not in {"skipped", "unavailable"}: + status = "skipped" + skipped = _payload_mapping(skipped_sections.get("decomposition")) + reason = _optional_summary_key(skipped.get("reason")) + if reason is None: + reason = _fingerprint_section_skip_reason( + "decomposition", + payload, + target=finding.target, + ) + return _empty_decomposition_readiness(status=status, reason=reason) + + sample_counts = _payload_mapping(decomposition.get("sample_counts")) + stationarity_basis = _payload_mapping( + decomposition.get("stationarity_basis") + ) + structural_break = _payload_mapping( + decomposition.get("structural_break_proxy") + ) + trend = _payload_mapping(decomposition.get("trend_proxy")) + strongest = _payload_mapping(structural_break.get("strongest_candidate")) + return { + "status": _decomposition_section_status(decomposition), + "reason": _optional_summary_key(decomposition.get("reason")), + "basis": _summary_key(decomposition.get("basis")), + "calculation_basis": _summary_key( + decomposition.get("calculation_basis") + ), + "row_order": _summary_key(decomposition.get("row_order")), + "computed_from": _summary_key(decomposition.get("computed_from")), + "cache_source": _optional_summary_key( + decomposition.get("cache_source") + ), + "regular_grid": decomposition.get("regular_grid") is True, + "metric": _summary_key(decomposition.get("metric")), + "limitations": _string_list(decomposition.get("limitations")), + "row_count": _int_payload(decomposition.get("row_count")), + "sampled_row_count": _int_payload( + decomposition.get("sampled_row_count") + ), + "usable_row_count": _int_payload(decomposition.get("usable_row_count")), + "invalid_row_count": _int_payload( + decomposition.get("invalid_row_count") + ), + "partial_row_count": _int_payload( + decomposition.get("partial_row_count") + ), + "truncated": decomposition.get("truncated") is True, + "level_sample_count": _int_payload(sample_counts.get("level")), + "return_sample_count": _int_payload(sample_counts.get("return")), + "windows": list(_int_sequence_payload(decomposition.get("windows"))), + "rounding_digits": _int_payload(decomposition.get("rounding_digits")), + "computed_window_count": _int_payload( + decomposition.get("computed_window_count") + ), + "skipped_window_count": _int_payload( + decomposition.get("skipped_window_count") + ), + "skipped_window_reason_counts": _counter_payload( + _counter_from_mapping( + _payload_mapping( + decomposition.get("skipped_window_reason_counts") + ) + ) + ), + "stationarity": { + "status": _summary_key(stationarity_basis.get("status")), + "reason": _optional_summary_key(stationarity_basis.get("reason")), + "stationarity_status": _summary_key( + stationarity_basis.get("stationarity_status") + ), + "zero_variance_metrics": _string_list( + stationarity_basis.get("zero_variance_metrics") + ), + "recommended_transforms": _string_list( + stationarity_basis.get("recommended_transforms") + ), + }, + "trend": { + "status": _summary_key(trend.get("status")), + "direction": _summary_key(trend.get("direction")), + "trend_strength": _optional_float_payload( + trend.get("trend_strength") + ), + }, + "structural_break": { + "status": _summary_key(structural_break.get("status")), + "candidate_count": _int_payload( + structural_break.get("candidate_count") + ), + "strongest_score": ( + _optional_float_payload(strongest.get("score")) + if strongest + else None + ), + }, + "training_projection": dict( + _payload_mapping(decomposition.get("training_projection")) + ), + } + + +def _empty_decomposition_readiness( + *, + status: str, + reason: str | None, +) -> dict[str, JSONValue]: + return { + "status": status, + "reason": reason, + "basis": "unknown", + "calculation_basis": "unknown", + "row_order": "unknown", + "computed_from": "unknown", + "cache_source": None, + "regular_grid": False, + "metric": "unknown", + "limitations": [], + "row_count": 0, + "sampled_row_count": 0, + "usable_row_count": 0, + "invalid_row_count": 0, + "partial_row_count": 0, + "truncated": False, + "level_sample_count": 0, + "return_sample_count": 0, + "windows": [], + "rounding_digits": 0, + "computed_window_count": 0, + "skipped_window_count": 0, + "skipped_window_reason_counts": {}, + "stationarity": {}, + "trend": {}, + "structural_break": {}, + "training_projection": {}, + } + + def _compact_stationarity_window_summary( value: JSONValue, ) -> dict[str, JSONValue]: @@ -2415,33 +2976,6 @@ def _compact_numeric_summary(value: JSONValue) -> dict[str, JSONValue]: } -def _compact_flatline_summary(value: JSONValue) -> dict[str, JSONValue]: - flatline = _payload_mapping(value) - if not flatline: - return {} - return { - "zero_return_count": _int_payload(flatline.get("zero_return_count")), - "zero_return_rate": _optional_float_payload( - flatline.get("zero_return_rate") - ), - "zero_return_run_count": _int_payload( - flatline.get("zero_return_run_count") - ), - "ohlc_flatline_row_count": _int_payload( - flatline.get("ohlc_flatline_row_count") - ), - "ohlc_flatline_rate": _optional_float_payload( - flatline.get("ohlc_flatline_rate") - ), - "ohlc_flatline_run_count": _int_payload( - flatline.get("ohlc_flatline_run_count") - ), - "ohlc_flatline_affected_row_count": _int_payload( - flatline.get("ohlc_flatline_affected_row_count") - ), - } - - def _compact_event_summary( value: JSONValue, *, @@ -2470,6 +3004,7 @@ def _fingerprint_readiness_target_sort_key( axis = _payload_mapping(target.get("target_axis")) dependence = _payload_mapping(target.get("dependence")) stationarity = _payload_mapping(target.get("stationarity_diagnostics")) + decomposition = _payload_mapping(target.get("decomposition")) readiness_rank = min( _fingerprint_readiness_status_rank( _summary_key(target.get("applicable_dynamics_status")) @@ -2480,6 +3015,9 @@ def _fingerprint_readiness_target_sort_key( _fingerprint_readiness_status_rank( _summary_key(stationarity.get("status")) ), + _fingerprint_readiness_status_rank( + _summary_key(decomposition.get("status")) + ), ) return ( readiness_rank, @@ -3096,7 +3634,10 @@ def _series_fingerprint_payload( "schema_version": TIME_SERIES_FINGERPRINT_SCHEMA_VERSION, "target_axis": _target_axis(target), "coverage": _empty_coverage(parsed_row_count=None), - "temporal_topology": timestamp_topology_payload_for_target(target), + "temporal_topology": timestamp_topology_payload_for_target( + target, + inspection_sample_limit=profile.topology_inspection_sample_limit, + ), "source": _unavailable_source( target, reason=_unsupported_reason(target), @@ -3149,6 +3690,7 @@ def _series_fingerprint_payload( payload, target=target, profile=profile, + training_frame=cache.frame, ) if target.kind is QualityTargetKind.CACHE: @@ -3229,10 +3771,16 @@ def _series_fingerprint_payload( "kind": "csv_text", "path": publish_safe_path(target.path), } + training_frame = _training_frame_from_text( + text_payload.text, + target=target, + profile=profile, + ) return _finalize_fingerprint_payload( payload, target=target, profile=profile, + training_frame=training_frame, ) @@ -3241,153 +3789,997 @@ def _finalize_fingerprint_payload( *, target: QualityTarget, profile: HistDataFingerprintProfile, + training_frame: Any | None = None, ) -> dict[str, JSONValue]: + if profile.cache_source_parity.enabled: + payload["cache_source_parity"] = _cache_source_parity_payload( + target, + profile=profile, + ) payload["fingerprint_audit"] = _fingerprint_audit_payload( payload, target=target, profile=profile, ) + if not _unsupported_reason(target): + payload["synthetic_constraints"] = ( + synthetic_constraints_from_fingerprint( + payload, + training_frame=training_frame, + target=target, + ) + ) + if profile.classical_baselines.enabled: + payload["classical_baselines"] = ( + classical_baseline_diagnostics_from_training_frame( + training_frame, + payload, + profile=profile.classical_baselines, + target=target, + ) + ) + model_input_result = None + if ( + profile.classical_model_input.enabled + or profile.exponential_smoothing.enabled + or profile.autoregressive.enabled + or profile.seasonal_exogenous.enabled + or profile.state_space.enabled + or profile.volatility.enabled + or profile.classical_model_comparison.enabled + ): + model_input_result = build_classical_model_input( + training_frame, + payload, + profile=profile.classical_model_input, + target=target, + ) + if ( + profile.classical_model_input.enabled + and model_input_result is not None + ): + payload["classical_model_input"] = dict(model_input_result.contract) + if ( + profile.exponential_smoothing.enabled + and model_input_result is not None + ): + payload["exponential_smoothing"] = dict( + exponential_smoothing_from_model_input( + training_frame, + model_input_result, + payload, + input_profile=profile.classical_model_input, + profile=profile.exponential_smoothing, + target=target, + ).diagnostics + ) + if profile.autoregressive.enabled and model_input_result is not None: + payload["autoregressive"] = dict( + autoregressive_from_model_input( + training_frame, + model_input_result, + payload, + input_profile=profile.classical_model_input, + profile=profile.autoregressive, + exponential_smoothing=_payload_mapping( + payload.get("exponential_smoothing") + ), + target=target, + ).diagnostics + ) + if ( + profile.seasonal_exogenous.enabled + and model_input_result is not None + ): + payload["seasonal_exogenous"] = dict( + seasonal_exogenous_from_model_input( + training_frame, + model_input_result, + payload, + input_profile=profile.classical_model_input, + profile=profile.seasonal_exogenous, + calendar_profile=profile.calendar_profile, + exponential_smoothing=_payload_mapping( + payload.get("exponential_smoothing") + ), + autoregressive=_payload_mapping( + payload.get("autoregressive") + ), + target=target, + ).diagnostics + ) + if profile.state_space.enabled and model_input_result is not None: + payload["state_space"] = dict( + state_space_from_model_input( + training_frame, + model_input_result, + payload, + input_profile=profile.classical_model_input, + profile=profile.state_space, + exponential_smoothing=_payload_mapping( + payload.get("exponential_smoothing") + ), + autoregressive=_payload_mapping( + payload.get("autoregressive") + ), + seasonal_exogenous=_payload_mapping( + payload.get("seasonal_exogenous") + ), + target=target, + ).diagnostics + ) + if profile.volatility.enabled and model_input_result is not None: + payload["volatility"] = dict( + volatility_from_model_input( + training_frame, + model_input_result, + payload, + input_profile=profile.classical_model_input, + profile=profile.volatility, + exponential_smoothing=_payload_mapping( + payload.get("exponential_smoothing") + ), + autoregressive=_payload_mapping( + payload.get("autoregressive") + ), + seasonal_exogenous=_payload_mapping( + payload.get("seasonal_exogenous") + ), + state_space=_payload_mapping(payload.get("state_space")), + target=target, + ).diagnostics + ) + if profile.classical_model_comparison.enabled: + payload["classical_model_comparison"] = dict( + classical_model_comparison_from_saved_results( + training_frame, + payload, + model_input=( + model_input_result.contract + if model_input_result is not None + else _payload_mapping( + payload.get("classical_model_input") + ) + ), + classical_baselines=_payload_mapping( + payload.get("classical_baselines") + ), + exponential_smoothing=_payload_mapping( + payload.get("exponential_smoothing") + ), + autoregressive=_payload_mapping( + payload.get("autoregressive") + ), + seasonal_exogenous=_payload_mapping( + payload.get("seasonal_exogenous") + ), + state_space=_payload_mapping(payload.get("state_space")), + volatility=_payload_mapping(payload.get("volatility")), + profile=profile.classical_model_comparison, + target=target, + ).diagnostics + ) + payload["fingerprint_audit"] = _fingerprint_audit_payload( + payload, + target=target, + profile=profile, + ) payload["fingerprint_id"] = _fingerprint_id(payload) return payload -def _fingerprint_audit_payload( - payload: Mapping[str, JSONValue], +def _training_frame_from_text( + text: str, *, target: QualityTarget, profile: HistDataFingerprintProfile, -) -> dict[str, JSONValue]: - expected = _fingerprint_expected_sections(target) - emitted = [ - section for section in FINGERPRINT_AUDIT_SECTIONS if section in payload - ] - status_sections = _ordered_unique((*expected, *emitted)) - skipped: dict[str, JSONValue] = {} - for section in expected: - if section not in payload: - skipped[section] = _fingerprint_section_skip_payload( - section, - payload, - target=target, - ) - section_statuses: dict[str, JSONValue] = {} - for section in status_sections: - section_statuses[section] = _fingerprint_section_status( - section, - payload, +) -> Any | None: + try: + batch = parse_ascii_lines( + target.timeframe, + islice(StringIO(text), max(1, _profile_max_rows(profile))), ) - unsupported_reason = _unsupported_reason(target) - source = _payload_mapping(payload.get("source")) - source_reason = _optional_summary_key(source.get("reason")) - target_capability: dict[str, JSONValue] = { - "supported": unsupported_reason == "", - "unsupported_reason": unsupported_reason or None, - } - source_status: dict[str, JSONValue] = { - "kind": _summary_key(source.get("kind")), - "readable": source.get("kind") != "unavailable", - "reason": source_reason, + return to_polars_frame(batch) + except (OSError, TypeError, ValueError): + return None + + +def _cache_source_parity_payload( + target: QualityTarget, + *, + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + mismatch_limit = bounded_report_limit( + profile.cache_source_parity.mismatch_limit, + default_limit=DEFAULT_FINGERPRINT_PARITY_MISMATCH_LIMIT, + ) + bases: dict[str, JSONValue] = { + "raw_source": _parity_unavailable_basis("source_unavailable"), + "raw_cache": _parity_unavailable_basis("cache_unavailable"), + "enriched_cache": _parity_unavailable_basis( + "cache_enrichment_unavailable" + ), + "quality_report": _parity_unavailable_basis( + "quality_report_projection_unavailable" + ), + "influx_projection": _parity_unavailable_basis( + "influx_projection_unavailable" + ), } - conditional_distribution_eligibility: dict[str, JSONValue] = { - "tick_spread": _conditional_tick_spread_eligibility( - payload, - target=target, + unsupported_reason = _unsupported_reason(target) + if unsupported_reason: + return _finalize_parity_payload( + target, + bases=bases, + comparisons=[], + skipped_reasons=[unsupported_reason], + mismatch_limit=mismatch_limit, ) - } - dynamics_readiness: dict[str, JSONValue] = { - "microstructure_dynamics": _fingerprint_dynamics_readiness( - "microstructure_dynamics", - payload, - target=target, + required_columns = columns_for_timeframe(target.timeframe) + cache = read_fingerprint_parity_polars_cache( + target, + required_columns=required_columns, + ) + if cache is not None: + raw_cache_columns = set(getattr(cache.frame, "columns", ())) + bases["raw_cache"] = { + "status": "available", + "path": publish_safe_path(str(cache.path)), + "cache_source": cache.source, + "fresh": cache.fresh, + "freshness": _parity_freshness_status(cache.fresh), + "row_count": int(getattr(cache.frame, "height", 0) or 0), + "column_count": len(raw_cache_columns), + "training_schema_present": ( + "training_schema_version" in raw_cache_columns + ), + "training_required_columns_present": set( + TRAINING_REQUIRED_COLUMNS + ).issubset(raw_cache_columns), + } + if target.kind not in {QualityTargetKind.CSV, QualityTargetKind.ZIP}: + return _finalize_parity_payload( + target, + bases=bases, + comparisons=[], + skipped_reasons=["source_target_unavailable"], + mismatch_limit=mismatch_limit, + ) + try: + text_payload = _read_text_payload(target) + except (OSError, UnicodeDecodeError, ValueError, zipfile.BadZipFile) as exc: + bases["raw_source"] = _parity_unavailable_basis( + "source_unreadable", + detail=type(exc).__name__, + ) + return _finalize_parity_payload( + target, + bases=bases, + comparisons=[], + skipped_reasons=["source_unreadable"], + mismatch_limit=mismatch_limit, + ) + + source_coverage = _coverage_from_text( + text_payload.text, + timeframe=target.timeframe, + ) + bases["raw_source"] = { + "status": "available", + "kind": ( + "zip_member" if target.kind is QualityTargetKind.ZIP else "csv_text" ), + "path": publish_safe_path(target.path), + "member": text_payload.source_member or None, + "row_count": _int_payload(source_coverage.get("row_count")), } - stationarity_readiness = _fingerprint_stationarity_readiness( - payload, - target=target, + if cache is None: + return _finalize_parity_payload( + target, + bases=bases, + comparisons=[], + skipped_reasons=["cache_unavailable"], + mismatch_limit=mismatch_limit, + ) + + cache_target = QualityTarget( + path=str(cache.path), + kind=QualityTargetKind.CACHE, + data_format=target.data_format, + timeframe=target.timeframe, + symbol=target.symbol, + period=target.period, + metadata=dict(target.metadata), ) - audit_payload: dict[str, JSONValue] = { - "schema_version": TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION, - "sections_expected": [section for section in expected], - "sections_emitted": [section for section in emitted], - "sections_skipped": skipped, - "section_statuses": section_statuses, - "target_capability": target_capability, - "source_status": source_status, - "conditional_distribution_eligibility": ( - conditional_distribution_eligibility + source_sections: dict[str, Mapping[str, JSONValue]] = { + "coverage": source_coverage, + "temporal_topology": timestamp_topology_payload_for_target( + target, + inspection_sample_limit=0, + prefer_cache=False, ), - "profile_completeness": _fingerprint_profile_completeness( - payload, + "calendar_regimes": calendar_regime_payload_for_target( + target, + calendar_profile=profile.calendar_profile, + prefer_cache=False, + ), + "conditional_distributions": _conditional_distributions( + target, + frame=None, + text=text_payload.text, profile=profile, ), - "dynamics_readiness": dynamics_readiness, - "stationarity_readiness": stationarity_readiness, } - return audit_payload + cache_sections: dict[str, Mapping[str, JSONValue]] = { + "coverage": _coverage_from_frame(cache.frame), + "temporal_topology": timestamp_topology_payload_for_target( + cache_target, + inspection_sample_limit=0, + ), + "calendar_regimes": calendar_regime_payload_for_target( + cache_target, + calendar_profile=profile.calendar_profile, + ), + "conditional_distributions": _conditional_distributions( + cache_target, + frame=cache.frame, + text=None, + profile=profile, + ), + } + comparisons = _fingerprint_section_parity_comparisons( + source_sections, + cache_sections, + mismatch_limit=mismatch_limit, + ) + training_comparisons, training_bases = _training_projection_parity( + target, + cache_target=cache_target, + source_text=text_payload.text, + cache_frame=cache.frame, + profile=profile, + mismatch_limit=mismatch_limit, + ) + comparisons.extend(training_comparisons) + bases.update(training_bases) + if cache.fresh is False: + comparisons.insert( + 0, + { + "section": "cache_freshness", + "status": "mismatch", + "compared_field_count": 1, + "mismatch_field_count": 1, + "mismatch_fields": ["mtime_ns"], + "mismatch_codes": ["fingerprint_cache_source_stale_cache"], + }, + ) + return _finalize_parity_payload( + target, + bases=bases, + comparisons=comparisons, + skipped_reasons=[], + mismatch_limit=mismatch_limit, + ) -def _fingerprint_expected_sections( - target: QualityTarget, -) -> tuple[str, ...]: - sections = ["coverage", "temporal_topology"] - if target.timeframe in SUPPORTED_SERIES_FINGERPRINT_TIMEFRAMES: - sections.append("calendar_regimes") - if target.timeframe == TICK: - sections.extend( +def _fingerprint_section_parity_comparisons( + source: Mapping[str, Mapping[str, JSONValue]], + cache: Mapping[str, Mapping[str, JSONValue]], + *, + mismatch_limit: BoundedReportLimit, +) -> list[dict[str, JSONValue]]: + specifications = ( + ( + "coverage", ( - "tick_distribution", - "conditional_distributions", - "microstructure_dynamics", - "dependence", - "stationarity_diagnostics", - ) + "row_count", + "parsed_row_count", + "start_timestamp_utc_ms", + "end_timestamp_utc_ms", + ), + ), + ( + "temporal_topology", + ( + "row_count", + "parsed_row_count", + "invalid_timestamp_count", + "non_monotonic_count", + "duplicate_timestamp_count", + "suspicious_gap_count", + "expected_session_closure_count", + "weekend_activity_count", + "gap_bucket_counts", + ), + ), + ( + "calendar_regimes", + ( + "row_count", + "parsed_row_count", + "invalid_timestamp_count", + "session_state_counts", + "active_session_counts", + "clock_session_counts", + "overlap_counts", + "special_tag_counts", + "holiday_tag_counts", + "event_tag_counts", + "hour_of_day_counts", + "day_of_week_counts", + ), + ), + ( + "conditional_distributions", + ( + "row_count", + "sampled_row_count", + "usable_row_count", + "invalid_row_count", + "by_active_session", + "by_special_tag", + ), + ), + ) + return [ + _parity_section_comparison( + section, + source.get(section, {}), + cache.get(section, {}), + fields=fields, + mismatch_limit=mismatch_limit, ) - return tuple(sections) - - -def _ordered_unique(values: Iterable[str]) -> tuple[str, ...]: - seen: set[str] = set() - ordered: list[str] = [] - for value in values: - if value in seen: - continue - seen.add(value) - ordered.append(value) - return tuple(ordered) + for section, fields in specifications + ] -def _fingerprint_section_skip_payload( +def _parity_section_comparison( section: str, - payload: Mapping[str, JSONValue], + source: Mapping[str, JSONValue], + cache: Mapping[str, JSONValue], *, - target: QualityTarget, + fields: tuple[str, ...], + mismatch_limit: BoundedReportLimit, ) -> dict[str, JSONValue]: - reason = _fingerprint_section_skip_reason( - section, - payload, - target=target, - ) - skipped: dict[str, JSONValue] = {"reason": reason} - details = _fingerprint_section_skip_details(section, target) - if details: - skipped["details"] = details - return skipped + if not source or not cache: + return { + "section": section, + "status": "skipped", + "reason": "section_unavailable", + "compared_field_count": 0, + "mismatch_field_count": 0, + "mismatch_fields": [], + "mismatch_codes": [], + } + mismatches = [ + field for field in fields if source.get(field) != cache.get(field) + ] + codes = _parity_section_mismatch_codes(section, mismatches) + included = mismatch_limit.slice(mismatches) + return { + "section": section, + "status": "mismatch" if mismatches else "match", + "compared_field_count": len(fields), + "mismatch_field_count": len(mismatches), + "included_mismatch_field_count": len(included), + "omitted_mismatch_field_count": max(0, len(mismatches) - len(included)), + "truncated": len(included) < len(mismatches), + "mismatch_fields": cast(JSONValue, included), + "mismatch_codes": cast(JSONValue, codes), + } -def _fingerprint_section_skip_details( +def _parity_section_mismatch_codes( section: str, - target: QualityTarget, -) -> dict[str, JSONValue]: - if section == "conditional_distributions": - return { - "metric": "tick_spread", - "grouped_by": ["active_session", "special_tag"], + fields: list[str], +) -> list[str]: + if not fields: + return [] + if section == "coverage": + codes = [] + if set(fields) & {"row_count", "parsed_row_count"}: + codes.append("fingerprint_cache_source_row_count_mismatch") + if set(fields) & {"start_timestamp_utc_ms", "end_timestamp_utc_ms"}: + codes.append("fingerprint_cache_source_coverage_bounds_mismatch") + return codes + return [ + { + "temporal_topology": "fingerprint_cache_source_topology_mismatch", + "calendar_regimes": "fingerprint_cache_source_calendar_mismatch", + "conditional_distributions": ( + "fingerprint_cache_source_conditioned_spread_mismatch" + ), + }[section] + ] + + +def _training_projection_parity( + target: QualityTarget, + *, + cache_target: QualityTarget, + source_text: str, + cache_frame: Any, + profile: HistDataFingerprintProfile, + mismatch_limit: BoundedReportLimit, +) -> tuple[list[dict[str, JSONValue]], dict[str, JSONValue]]: + sample_limit = max(1, _profile_max_rows(profile)) + try: + source_batch = parse_ascii_lines( + target.timeframe, + islice(StringIO(source_text), sample_limit), + ) + source_frame = to_polars_frame(source_batch) + cache_sample = cache_frame.head(sample_limit) + enriched_source = ensure_tick_training_features( + source_frame, + target=target, + ) + enriched_cache = ensure_tick_training_features( + cache_sample, + target=target, + ) + except (OSError, TypeError, ValueError) as exc: + comparison: dict[str, JSONValue] = { + "section": "training_substrate", + "status": "skipped", + "reason": "training_enrichment_unavailable", + "error_type": type(exc).__name__, + "compared_field_count": 0, + "mismatch_field_count": 0, + "mismatch_fields": [], + "mismatch_codes": [], + } + return [comparison], {} + + required = set(TRAINING_REQUIRED_COLUMNS) + source_columns = set(enriched_source.columns) + cache_columns = set(enriched_cache.columns) + missing_columns = sorted( + (required - source_columns) | (required - cache_columns) + ) + column_comparison = _bounded_training_comparison( + "training_columns", + missing_columns, + code="fingerprint_cache_source_training_columns_mismatch", + compared_count=len(required), + mismatch_limit=mismatch_limit, + ) + identity_mismatches = _identity_mismatch_columns( + enriched_source, + enriched_cache, + ) + identity_comparison = _bounded_training_comparison( + "row_identity", + identity_mismatches, + code="fingerprint_cache_source_row_identity_mismatch", + compared_count=5, + mismatch_limit=mismatch_limit, + ) + source_duplicates = _duplicate_timestamp_row_count(enriched_source) + cache_duplicates = _duplicate_timestamp_row_count(enriched_cache) + duplicate_fields = ( + [] + if source_duplicates == cache_duplicates + else ["duplicate_timestamp_row_count"] + ) + duplicate_comparison = _bounded_training_comparison( + "duplicate_timestamps", + duplicate_fields, + code="fingerprint_cache_source_duplicate_timestamp_mismatch", + compared_count=1, + mismatch_limit=mismatch_limit, + ) + + report = quality_report_from_training_features( + enriched_cache, + target=cache_target, + ) + report_fields = [] + if ( + report.metadata.get("training_schema_version") + != TRAINING_SCHEMA_VERSION + ): + report_fields.append("training_schema_version") + if _int_payload(report.metadata.get("row_count")) != enriched_cache.height: + report_fields.append("row_count") + report_comparison = _bounded_training_comparison( + "quality_report_projection", + report_fields, + code="fingerprint_cache_source_quality_report_projection_mismatch", + compared_count=2, + mismatch_limit=mismatch_limit, + ) + influx_fields = _influx_projection_missing_fields( + enriched_cache, + target=target, + ) + influx_comparison = _bounded_training_comparison( + "influx_projection", + influx_fields, + code="fingerprint_cache_source_influx_projection_mismatch", + compared_count=6, + mismatch_limit=mismatch_limit, + ) + raw_cache_columns = set(getattr(cache_frame, "columns", ())) + cache_was_enriched = required.issubset(raw_cache_columns) + sampled_count = min(int(cache_frame.height), sample_limit) + bases: dict[str, JSONValue] = { + "enriched_cache": { + "status": "available", + "training_schema_version": TRAINING_SCHEMA_VERSION, + "cache_was_enriched": cache_was_enriched, + "legacy_cache_enriched_on_read": not cache_was_enriched, + "required_column_count": len(required), + "column_count": len(cache_columns), + "sampled_row_count": sampled_count, + "source_row_count": int(cache_frame.height), + "truncated": sampled_count < int(cache_frame.height), + }, + "quality_report": { + "status": "available", + "projection_kind": "audit_from_enriched_rows", + "training_schema_version": report.metadata.get( + "training_schema_version" + ), + "row_count": report.metadata.get("row_count"), + }, + "influx_projection": { + "status": "available" if not influx_fields else "limited", + "projection_kind": "same_point_enriched_fields", + "missing_required_field_count": len(influx_fields), + }, + } + return ( + [ + column_comparison, + identity_comparison, + duplicate_comparison, + report_comparison, + influx_comparison, + ], + bases, + ) + + +def _bounded_training_comparison( + section: str, + mismatches: list[str], + *, + code: str, + compared_count: int, + mismatch_limit: BoundedReportLimit, +) -> dict[str, JSONValue]: + included = mismatch_limit.slice(mismatches) + return { + "section": section, + "status": "mismatch" if mismatches else "match", + "compared_field_count": compared_count, + "mismatch_field_count": len(mismatches), + "included_mismatch_field_count": len(included), + "omitted_mismatch_field_count": max(0, len(mismatches) - len(included)), + "truncated": len(included) < len(mismatches), + "mismatch_fields": cast(JSONValue, included), + "mismatch_codes": cast(JSONValue, [code] if mismatches else []), + } + + +def _identity_mismatch_columns(source: Any, cache: Any) -> list[str]: + identity_columns = ( + "series_id", + "period", + "row_id", + "source_row_number", + "event_seq", + ) + mismatches = [] + for column in identity_columns: + if column not in source.columns or column not in cache.columns: + mismatches.append(column) + continue + source_values = source.get_column(column).to_list() + cache_values = cache.get_column(column).to_list() + if source_values != cache_values: + mismatches.append(column) + return mismatches + + +def _duplicate_timestamp_row_count(frame: Any) -> int: + if "datetime" not in frame.columns or frame.is_empty(): + return 0 + return max( + 0, + int(frame.height) - int(frame.get_column("datetime").n_unique()), + ) + + +def _influx_projection_missing_fields( + frame: Any, + *, + target: QualityTarget, +) -> list[str]: + if frame.is_empty(): + return ["sample_row"] + columns = tuple(frame.columns) + row = frame.row(0) + line = format_influx_line( + target.symbol, + target.data_format, + target.timeframe, + row, + columns=columns, + ) + try: + field_text = line.split(" ", 2)[1] + except IndexError: + return ["line_protocol_fields"] + emitted = { + item.split("=", 1)[0] for item in field_text.split(",") if "=" in item + } + required = ( + "source_row_number", + "event_seq", + "quality_status_code", + "quality_finding_count", + "training_usable", + "training_weight", + ) + return [field for field in required if field not in emitted] + + +def _finalize_parity_payload( + target: QualityTarget, + *, + bases: Mapping[str, JSONValue], + comparisons: list[dict[str, JSONValue]], + skipped_reasons: list[str], + mismatch_limit: BoundedReportLimit, +) -> dict[str, JSONValue]: + mismatch_codes = sorted( + { + code + for comparison in comparisons + for code in ( + str(item) + for item in _string_list(comparison.get("mismatch_codes")) + ) + } + ) + included_codes = mismatch_limit.slice(mismatch_codes) + mismatch_count = sum( + 1 for item in comparisons if item.get("status") == "mismatch" + ) + compared_count = sum( + 1 for item in comparisons if item.get("status") in {"match", "mismatch"} + ) + status = "not_compared" + if compared_count: + status = "mismatch" if mismatch_count else "match" + return { + "schema_version": TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION, + "status": status, + "advisory": True, + "target_axis": _target_axis(target), + "base_grain": { + "data_format": target.data_format, + "timeframe": target.timeframe, + }, + "compared_section_count": compared_count, + "matching_section_count": sum( + 1 for item in comparisons if item.get("status") == "match" + ), + "mismatched_section_count": mismatch_count, + "skipped_section_count": sum( + 1 for item in comparisons if item.get("status") == "skipped" + ), + "mismatch_code_count": len(mismatch_codes), + "included_mismatch_code_count": len(included_codes), + "omitted_mismatch_code_count": max( + 0, len(mismatch_codes) - len(included_codes) + ), + "truncated": len(included_codes) < len(mismatch_codes), + "limit_metadata": {"mismatches": mismatch_limit.limit_payload()}, + "mismatch_codes": cast(JSONValue, included_codes), + "skipped_reasons": cast(JSONValue, sorted(set(skipped_reasons))), + "bases": dict(bases), + "comparisons": cast(JSONValue, comparisons), + } + + +def _parity_unavailable_basis( + reason: str, + *, + detail: str = "", +) -> dict[str, JSONValue]: + payload: dict[str, JSONValue] = { + "status": "unavailable", + "reason": reason, + } + if detail: + payload["detail"] = detail + return payload + + +def _parity_freshness_status(fresh: bool | None) -> str: + if fresh is True: + return "fresh" + if fresh is False: + return "stale" + return "not_applicable" + + +def _fingerprint_audit_payload( + payload: Mapping[str, JSONValue], + *, + target: QualityTarget, + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + expected = _fingerprint_expected_sections(target) + if profile.cache_source_parity.enabled: + expected = (*expected, "cache_source_parity") + if profile.classical_baselines.enabled: + expected = (*expected, "classical_baselines") + if profile.classical_model_input.enabled: + expected = (*expected, "classical_model_input") + if profile.exponential_smoothing.enabled: + expected = (*expected, "exponential_smoothing") + if profile.autoregressive.enabled: + expected = (*expected, "autoregressive") + if profile.seasonal_exogenous.enabled: + expected = (*expected, "seasonal_exogenous") + if profile.state_space.enabled: + expected = (*expected, "state_space") + if profile.volatility.enabled: + expected = (*expected, "volatility") + if profile.classical_model_comparison.enabled: + expected = (*expected, "classical_model_comparison") + emitted = [ + section for section in FINGERPRINT_AUDIT_SECTIONS if section in payload + ] + status_sections = _ordered_unique((*expected, *emitted)) + skipped: dict[str, JSONValue] = {} + for section in expected: + if section not in payload: + skipped[section] = _fingerprint_section_skip_payload( + section, + payload, + target=target, + ) + section_statuses: dict[str, JSONValue] = {} + for section in status_sections: + section_statuses[section] = _fingerprint_section_status( + section, + payload, + ) + unsupported_reason = _unsupported_reason(target) + source = _payload_mapping(payload.get("source")) + source_reason = _optional_summary_key(source.get("reason")) + target_capability: dict[str, JSONValue] = { + "supported": unsupported_reason == "", + "unsupported_reason": unsupported_reason or None, + } + source_status: dict[str, JSONValue] = { + "kind": _summary_key(source.get("kind")), + "readable": source.get("kind") != "unavailable", + "reason": source_reason, + } + conditional_distribution_eligibility: dict[str, JSONValue] = { + "tick_spread": _conditional_tick_spread_eligibility( + payload, + target=target, + ) + } + dynamics_readiness: dict[str, JSONValue] = { + "microstructure_dynamics": _fingerprint_dynamics_readiness( + "microstructure_dynamics", + payload, + target=target, + ), + } + stationarity_readiness = _fingerprint_stationarity_readiness( + payload, + target=target, + ) + decomposition_readiness = _fingerprint_decomposition_readiness( + payload, + target=target, + ) + audit_payload: dict[str, JSONValue] = { + "schema_version": TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION, + "sections_expected": [section for section in expected], + "sections_emitted": [section for section in emitted], + "sections_skipped": skipped, + "section_statuses": section_statuses, + "target_capability": target_capability, + "source_status": source_status, + "conditional_distribution_eligibility": ( + conditional_distribution_eligibility + ), + "profile_completeness": _fingerprint_profile_completeness( + payload, + profile=profile, + ), + "dynamics_readiness": dynamics_readiness, + "stationarity_readiness": stationarity_readiness, + "decomposition_readiness": decomposition_readiness, + } + return audit_payload + + +def _fingerprint_expected_sections( + target: QualityTarget, +) -> tuple[str, ...]: + sections = ["coverage", "temporal_topology"] + if target.timeframe in SUPPORTED_SERIES_FINGERPRINT_TIMEFRAMES: + sections.append("calendar_regimes") + if target.timeframe == TICK: + sections.extend( + ( + "tick_distribution", + "conditional_distributions", + "microstructure_dynamics", + "dependence", + "stationarity_diagnostics", + "decomposition", + "synthetic_constraints", + ) + ) + return tuple(sections) + + +def _ordered_unique(values: Iterable[str]) -> tuple[str, ...]: + seen: set[str] = set() + ordered: list[str] = [] + for value in values: + if value in seen: + continue + seen.add(value) + ordered.append(value) + return tuple(ordered) + + +def _fingerprint_section_skip_payload( + section: str, + payload: Mapping[str, JSONValue], + *, + target: QualityTarget, +) -> dict[str, JSONValue]: + reason = _fingerprint_section_skip_reason( + section, + payload, + target=target, + ) + skipped: dict[str, JSONValue] = {"reason": reason} + details = _fingerprint_section_skip_details(section, target) + if details: + skipped["details"] = details + return skipped + + +def _fingerprint_section_skip_details( + section: str, + target: QualityTarget, +) -> dict[str, JSONValue]: + if section == "conditional_distributions": + return { + "metric": "tick_spread", + "grouped_by": ["active_session", "special_tag"], } if section in { "tick_distribution", "microstructure_dynamics", "dependence", "stationarity_diagnostics", + "decomposition", + "classical_baselines", + "classical_model_input", + "exponential_smoothing", + "autoregressive", + "seasonal_exogenous", + "state_space", + "volatility", + "classical_model_comparison", + "synthetic_constraints", }: return {"timeframe": target.timeframe} return {} @@ -3432,6 +4824,15 @@ def _section_timeframe_mismatch( "tick_distribution", "conditional_distributions", "microstructure_dynamics", + "synthetic_constraints", + "classical_baselines", + "classical_model_input", + "exponential_smoothing", + "autoregressive", + "seasonal_exogenous", + "state_space", + "volatility", + "classical_model_comparison", } and target.timeframe != TICK ) @@ -3443,6 +4844,10 @@ def _section_timeframe_mismatch( section == "stationarity_diagnostics" and target.timeframe not in SUPPORTED_SERIES_FINGERPRINT_TIMEFRAMES ) + or ( + section == "decomposition" + and target.timeframe not in SUPPORTED_SERIES_FINGERPRINT_TIMEFRAMES + ) ) @@ -3482,6 +4887,98 @@ def _fingerprint_section_status( return _dependence_section_status(_payload_mapping(payload[section])) if section == "stationarity_diagnostics": return _stationarity_section_status(_payload_mapping(payload[section])) + if section == "decomposition": + return _decomposition_section_status(_payload_mapping(payload[section])) + if section == "cache_source_parity": + parity_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if parity_status == "match": + return "valid" + if parity_status == "mismatch": + return "limited" + return "unavailable" + if section == "synthetic_constraints": + constraint_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if constraint_status == "ready": + return "valid" + if constraint_status == "limited": + return "limited" + return "unavailable" + if section == "classical_baselines": + baseline_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if baseline_status == "ready": + return "valid" + if baseline_status == "limited": + return "limited" + return "unavailable" + if section == "classical_model_input": + input_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if input_status == "ready": + return "valid" + if input_status == "limited": + return "limited" + return "unavailable" + if section == "exponential_smoothing": + model_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if model_status == "ready": + return "valid" + if model_status == "limited": + return "limited" + return "unavailable" + if section == "autoregressive": + model_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if model_status == "ready": + return "valid" + if model_status == "limited": + return "limited" + return "unavailable" + if section == "seasonal_exogenous": + model_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if model_status == "ready": + return "valid" + if model_status == "limited": + return "limited" + return "unavailable" + if section == "state_space": + model_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if model_status == "ready": + return "valid" + if model_status == "limited": + return "limited" + return "unavailable" + if section == "volatility": + model_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if model_status == "ready": + return "valid" + if model_status == "limited": + return "limited" + return "unavailable" + if section == "classical_model_comparison": + model_status = _summary_key( + _payload_mapping(payload[section]).get("status") + ) + if model_status == "ready": + return "valid" + if model_status == "limited": + return "limited" + return "unavailable" return "valid" @@ -3573,6 +5070,17 @@ def _stationarity_section_status( return "limited" +def _decomposition_section_status( + decomposition: Mapping[str, JSONValue], +) -> str: + status = _summary_key(decomposition.get("decomposition_status")) + if status == "ok": + return "valid" + if status == "unavailable": + return "unavailable" + return "limited" + + def _conditional_tick_spread_eligibility( payload: Mapping[str, JSONValue], *, @@ -3780,13 +5288,127 @@ def _fingerprint_stationarity_readiness( return readiness -def _string_list(value: object) -> list[JSONValue]: - if not isinstance(value, list): - return [] - return [str(item) for item in value if str(item or "").strip()] - - -def _payload_mapping(value: object) -> Mapping[str, JSONValue]: +def _fingerprint_decomposition_readiness( + payload: Mapping[str, JSONValue], + *, + target: QualityTarget, +) -> dict[str, JSONValue]: + section = "decomposition" + if section not in payload: + return { + "status": "skipped", + "reason": _fingerprint_section_skip_reason( + section, + payload, + target=target, + ), + } + decomposition = _payload_mapping(payload.get(section)) + sample_counts = _payload_mapping(decomposition.get("sample_counts")) + stationarity = _payload_mapping(decomposition.get("stationarity_basis")) + structural_break = _payload_mapping( + decomposition.get("structural_break_proxy") + ) + trend = _payload_mapping(decomposition.get("trend_proxy")) + readiness: dict[str, JSONValue] = { + "status": _decomposition_section_status(decomposition), + "reason": _optional_summary_key(decomposition.get("reason")), + "basis": _summary_key(decomposition.get("basis")), + "calculation_basis": _summary_key( + decomposition.get("calculation_basis") + ), + "row_order": _summary_key(decomposition.get("row_order")), + "computed_from": _summary_key(decomposition.get("computed_from")), + "cache_source": _optional_summary_key( + decomposition.get("cache_source") + ), + "regular_grid": decomposition.get("regular_grid") is True, + "metric": _summary_key(decomposition.get("metric")), + "limitations": _string_list(decomposition.get("limitations")), + "row_count": _int_payload(decomposition.get("row_count")), + "sampled_row_count": _int_payload( + decomposition.get("sampled_row_count") + ), + "usable_row_count": _int_payload(decomposition.get("usable_row_count")), + "invalid_row_count": _int_payload( + decomposition.get("invalid_row_count") + ), + "partial_row_count": _int_payload( + decomposition.get("partial_row_count") + ), + "truncated": decomposition.get("truncated") is True, + "level_sample_count": _int_payload(sample_counts.get("level")), + "return_sample_count": _int_payload(sample_counts.get("return")), + "windows": list(_int_sequence_payload(decomposition.get("windows"))), + "rounding_digits": _int_payload(decomposition.get("rounding_digits")), + "computed_window_count": _int_payload( + decomposition.get("computed_window_count") + ), + "skipped_window_count": _int_payload( + decomposition.get("skipped_window_count") + ), + "skipped_window_reason_counts": _counter_payload( + _counter_from_mapping( + _payload_mapping( + decomposition.get("skipped_window_reason_counts") + ) + ) + ), + "stationarity": { + "status": _summary_key(stationarity.get("status")), + "stationarity_status": _summary_key( + stationarity.get("stationarity_status") + ), + "computed_window_count": _int_payload( + stationarity.get("computed_window_count") + ), + "skipped_window_count": _int_payload( + stationarity.get("skipped_window_count") + ), + "zero_variance_metrics": _string_list( + stationarity.get("zero_variance_metrics") + ), + "recommended_transforms": _string_list( + stationarity.get("recommended_transforms") + ), + }, + "trend": { + "status": _summary_key(trend.get("status")), + "direction": _summary_key(trend.get("direction")), + "slope_per_observation": trend.get("slope_per_observation"), + "trend_strength": trend.get("trend_strength"), + }, + "structural_break": { + "status": _summary_key(structural_break.get("status")), + "candidate_count": _int_payload( + structural_break.get("candidate_count") + ), + "strongest_candidate": dict( + _payload_mapping(structural_break.get("strongest_candidate")) + ), + }, + "training_projection": dict( + _payload_mapping(decomposition.get("training_projection")) + ), + } + if readiness["status"] in {"limited", "unavailable"} and not readiness.get( + "reason" + ): + readiness["reason"] = _summary_key( + _string_list(decomposition.get("limitations"))[0] + if _string_list(decomposition.get("limitations")) + else decomposition.get("decomposition_status") + ) + return readiness + + +def _string_list(value: object) -> list[JSONValue]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if str(item or "").strip()] + + +def _payload_mapping(value: object) -> Mapping[str, JSONValue]: if isinstance(value, Mapping): return cast(Mapping[str, JSONValue], value) return {} @@ -3868,8 +5490,17 @@ def _series_fingerprint_topology_target_summaries( if not topology: continue target_axis = _payload_mapping(payload.get("target_axis")) + calendar = _payload_mapping(payload.get("calendar_regimes")) + calendar_policy = _compact_calendar_policy( + _payload_mapping(calendar.get("calendar_policy")) + ) target_summaries.append( - _topology_target_summary(finding, target_axis, topology) + _topology_target_summary( + finding, + target_axis, + topology, + calendar_policy=calendar_policy, + ) ) return target_summaries @@ -3878,9 +5509,11 @@ def _topology_target_summary( finding: QualityFinding, target_axis: Mapping[str, JSONValue], topology: Mapping[str, JSONValue], + *, + calendar_policy: Mapping[str, JSONValue], ) -> dict[str, JSONValue]: flags = _topology_flags(topology) - return { + summary: dict[str, JSONValue] = { "target_axis": _topology_target_axis(finding, target_axis), "row_count": _int_payload(topology.get("row_count")), "parsed_row_count": _optional_int_payload( @@ -3914,6 +5547,56 @@ def _topology_target_summary( "status": _topology_status(topology), "flags": flags, } + inspection_context = _payload_mapping(topology.get("inspection_context")) + if inspection_context: + summary["inspection_context"] = dict(inspection_context) + if calendar_policy: + summary["calendar_policy"] = dict(calendar_policy) + return summary + + +def _compact_calendar_policy( + policy: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + """Return bounded calendar policy fields needed by remediation guidance.""" + if not policy: + return {} + compact: dict[str, JSONValue] = {} + for key in ( + "source_timezone", + "canonical_timezone", + "holiday_calendar_source", + "holiday_calendar_complete", + "holiday_calendar_static_advisory", + "weekend_activity_policy", + "expected_session_closure_policy", + ): + value = policy.get(key) + if value is not None: + compact[key] = _bounded_calendar_policy_value(value) + profile = _payload_mapping(policy.get("calendar_profile")) + compact_profile: dict[str, JSONValue] = {} + for key in ( + "name", + "source", + "version", + "complete", + "static_advisory", + "weekend_activity_policy", + "expected_session_closure_policy", + ): + value = profile.get(key) + if value is not None: + compact_profile[key] = _bounded_calendar_policy_value(value) + if compact_profile: + compact["calendar_profile"] = compact_profile + return compact + + +def _bounded_calendar_policy_value(value: JSONValue) -> JSONValue: + if isinstance(value, str): + return value[:_CALENDAR_POLICY_CONTEXT_TEXT_LIMIT] + return value def _topology_target_axis( @@ -4026,18 +5709,29 @@ def _topology_attention_target_summary( ) -> dict[str, JSONValue] | None: flags = _topology_summary_flags(target.get("flags")) flag_set = set(flags) + calendar_policy = _payload_mapping(target.get("calendar_policy")) attention_flags = [ flag for flag in ACTIONABLE_TOPOLOGY_FLAGS if flag in flag_set ] + if ( + "expected_session_closures" in flag_set + and _summary_key(calendar_policy.get("expected_session_closure_policy")) + == "unexpected" + ): + attention_flags.append("expected_session_closures") if not attention_flags: return None - attention_level = _topology_attention_level(attention_flags) - return { + attention_level = _topology_attention_level( + attention_flags, + calendar_policy=calendar_policy, + ) + summary: dict[str, JSONValue] = { "target_axis": _topology_attention_axis(target), "attention_level": attention_level, "attention_flags": list(attention_flags), "remediation_hints": remediation_hint_payloads_for_flags( - attention_flags + attention_flags, + calendar_policy=calendar_policy, ), "flags": list(flags), "status": _summary_key(target.get("status")), @@ -4061,6 +5755,78 @@ def _topology_attention_target_summary( "computed_from": _summary_key(target.get("computed_from")), "cache_source": _optional_summary_key(target.get("cache_source")), } + if calendar_policy: + summary["calendar_policy"] = dict(calendar_policy) + inspection_context = _topology_attention_inspection_context( + target, + attention_flags, + calendar_policy=calendar_policy, + ) + if inspection_context: + summary["inspection_context"] = inspection_context + return summary + + +def _topology_attention_inspection_context( + target: Mapping[str, JSONValue], + attention_flags: list[str], + *, + calendar_policy: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + raw = _payload_mapping(target.get("inspection_context")) + if not raw: + return {} + context: dict[str, JSONValue] = {} + schema_version = raw.get("schema_version") + if schema_version is not None: + context["schema_version"] = schema_version + section_flags = ( + ("invalid_timestamps", "invalid_timestamps"), + ("non_monotonic_timestamps", "non_monotonic_timestamps"), + ("duplicate_timestamps", "duplicate_timestamps"), + ("suspicious_gaps", "suspicious_gaps"), + ("weekend_activity", "weekend_activity"), + ) + attention_flag_set = set(attention_flags) + target_axis = _topology_attention_axis(target) + for section_name, flag in section_flags: + section = _payload_mapping(raw.get(section_name)) + if flag not in attention_flag_set or not section: + continue + hints = remediation_hint_payloads_for_flags( + (flag,), + calendar_policy=calendar_policy, + ) + linked = dict(section) + hint = hints[0] if hints and isinstance(hints[0], Mapping) else {} + policy_context = _payload_mapping(hint.get("policy_context")) + actionable = policy_context.get("actionable") is not False + linked["actionable"] = actionable + linked["target_axis"] = target_axis + if hints: + linked["next_action" if actionable else "policy_note"] = hints[0] + context[section_name] = linked + expected_closures = _payload_mapping(raw.get("expected_session_closures")) + if "expected_session_closures" in attention_flag_set and expected_closures: + linked = dict(expected_closures) + hints = remediation_hint_payloads_for_flags( + ("expected_session_closures",), + calendar_policy=calendar_policy, + ) + linked["actionable"] = True + linked["target_axis"] = target_axis + if hints: + linked["next_action"] = hints[0] + context["expected_session_closures"] = linked + elif "suspicious_gaps" in attention_flag_set and expected_closures: + contextual = dict(expected_closures) + contextual["actionable"] = False + contextual["contextual_for"] = "suspicious_gaps" + contextual["target_axis"] = target_axis + context["expected_session_closures"] = contextual + if len(context) == int("schema_version" in context): + return {} + return context def _topology_attention_axis( @@ -4087,7 +5853,11 @@ def _topology_summary_flags(value: object) -> tuple[str, ...]: return tuple(flags) -def _topology_attention_level(flags: list[str]) -> str: +def _topology_attention_level( + flags: list[str], + *, + calendar_policy: Mapping[str, JSONValue], +) -> str: flag_set = set(flags) if "unavailable_topology" in flag_set: return "unavailable" @@ -4095,6 +5865,12 @@ def _topology_attention_level(flags: list[str]) -> str: return "structural" if flag_set & {"duplicate_timestamps", "suspicious_gaps"}: return "sequence" + if ( + flag_set == {"weekend_activity"} + and _summary_key(calendar_policy.get("weekend_activity_policy")) + == "allowed" + ): + return "contextual" return "session" @@ -4126,6 +5902,7 @@ def _topology_attention_level_rank(level: str) -> int: "structural": 1, "sequence": 2, "session": 3, + "contextual": 4, } return ranks.get(level, 99) @@ -4280,9 +6057,10 @@ def _add_dynamics_payload( profile: HistDataFingerprintProfile, ) -> None: if target.timeframe == TICK: - microstructure_dynamics, dependence, stationarity = ( + microstructure_dynamics, dependence, stationarity, decomposition = ( _tick_sequence_payloads( payload, + target=target, frame=frame, text=text, profile=profile, @@ -4291,15 +6069,22 @@ def _add_dynamics_payload( payload["microstructure_dynamics"] = microstructure_dynamics payload["dependence"] = dependence payload["stationarity_diagnostics"] = stationarity + payload["decomposition"] = decomposition def _tick_sequence_payloads( payload: Mapping[str, JSONValue], *, + target: QualityTarget, frame: Any | None, text: str | None, profile: HistDataFingerprintProfile, -) -> tuple[dict[str, JSONValue], dict[str, JSONValue], dict[str, JSONValue]]: +) -> tuple[ + dict[str, JSONValue], + dict[str, JSONValue], + dict[str, JSONValue], + dict[str, JSONValue], +]: if frame is not None: rows, row_count, usable_row_count, invalid_row_count = ( _tick_dynamics_rows_from_frame(frame, profile) @@ -4333,6 +6118,11 @@ def _tick_sequence_payloads( partial_row_count=partial_row_count, profile=profile, ) + stationarity = _tick_stationarity_payload( + rows, + base=base, + profile=profile, + ) return ( _tick_microstructure_dynamics_payload( rows, @@ -4340,7 +6130,14 @@ def _tick_sequence_payloads( profile=profile, ), _tick_dependence_payload(rows, base=base, profile=profile), - _tick_stationarity_payload(rows, base=base, profile=profile), + stationarity, + _tick_decomposition_payload( + rows, + base=base, + stationarity=stationarity, + target=target, + profile=profile, + ), ) @@ -4798,6 +6595,745 @@ def _tick_stationarity_payload( ) +def _tick_decomposition_payload( + rows: list[_TickDynamicsRow], + *, + base: dict[str, JSONValue], + stationarity: Mapping[str, JSONValue], + target: QualityTarget, + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + mids = [(row.bid + row.ask) / 2.0 for row in rows] + returns: list[float] = [] + absolute_returns_by_row: list[float | None] = [] + previous_mid: float | None = None + for mid in mids: + row_return: float | None = None + if previous_mid is not None and previous_mid > 0.0 and mid > 0.0: + row_return = math.log(mid / previous_mid) + returns.append(row_return) + absolute_returns_by_row.append( + abs(row_return) if row_return is not None else None + ) + previous_mid = mid + return _decomposition_payload( + base, + profile=profile, + target=target, + metric="mid_price", + timestamps=[row.timestamp_utc_ms for row in rows], + level_values=mids, + return_values=returns, + absolute_returns_by_row=absolute_returns_by_row, + stationarity=stationarity, + ) + + +def _decomposition_payload( + base: Mapping[str, JSONValue], + *, + profile: HistDataFingerprintProfile, + target: QualityTarget, + metric: str, + timestamps: list[int], + level_values: Iterable[float], + return_values: Iterable[float], + absolute_returns_by_row: list[float | None], + stationarity: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + levels = _finite_values(level_values) + returns = _finite_values(return_values) + trend = _decomposition_trend_payload(levels, profile) + residual = _decomposition_residual_payload(levels, profile) + seasonality = _decomposition_seasonality_payload( + timestamps, + levels, + absolute_returns_by_row, + target=target, + profile=profile, + ) + smoothing_windows, computed_window_count, skipped_reason_counts = ( + _decomposition_smoothing_windows_payload(levels, returns, profile) + ) + structural_break = _decomposition_structural_break_payload(levels, profile) + stationarity_basis = _decomposition_stationarity_basis(stationarity) + skipped_window_count = sum(skipped_reason_counts.values()) + limitations = _decomposition_limitations( + base, + stationarity_basis=stationarity_basis, + level_count=len(levels), + return_count=len(returns), + computed_window_count=computed_window_count, + skipped_window_count=skipped_window_count, + ) + decomposition_status = _decomposition_status( + base, + stationarity_basis=stationarity_basis, + level_count=len(levels), + return_count=len(returns), + limitations=limitations, + ) + + result = dict(base) + result.update( + { + "schema_version": ( + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION + ), + "decomposition_status": decomposition_status, + "calculation_basis": "observed_sequence", + "metric": metric, + "sample_counts": { + "level": len(levels), + "return": len(returns), + }, + "windows": [int(window) for window in profile.rolling_windows], + "rounding_digits": int(profile.rounding_digits), + "stationarity_basis": stationarity_basis, + "trend_proxy": trend, + "seasonality_proxy": seasonality, + "residual_proxy": residual, + "smoothing_windows": smoothing_windows, + "computed_window_count": computed_window_count, + "skipped_window_count": skipped_window_count, + "skipped_window_reason_counts": _counter_payload( + skipped_reason_counts + ), + "structural_break_proxy": structural_break, + "limitations": [value for value in limitations], + } + ) + if decomposition_status in {"limited", "unavailable"}: + result["reason"] = _decomposition_status_reason( + limitations, + stationarity_basis=stationarity_basis, + ) + result["training_projection"] = decomposition_training_projection(result) + return result + + +def _decomposition_stationarity_basis( + stationarity: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + if not stationarity: + return { + "status": "unavailable", + "reason": "stationarity_unavailable", + "stationarity_status": "unavailable", + "calculation_basis": "unknown", + "computed_window_count": 0, + "skipped_window_count": 0, + "skipped_window_reason_counts": {}, + "zero_variance_metrics": [], + "recommended_transforms": [], + "limitations": ["stationarity_unavailable"], + } + return { + "status": _stationarity_section_status(stationarity), + "reason": _optional_summary_key(stationarity.get("reason")), + "stationarity_status": _summary_key( + stationarity.get("stationarity_status") + ), + "calculation_basis": _summary_key( + stationarity.get("calculation_basis") + ), + "computed_window_count": _int_payload( + stationarity.get("computed_window_count") + ), + "skipped_window_count": _int_payload( + stationarity.get("skipped_window_count") + ), + "skipped_window_reason_counts": _counter_payload( + _counter_from_mapping( + _payload_mapping( + stationarity.get("skipped_window_reason_counts") + ) + ) + ), + "zero_variance_metrics": _string_list( + stationarity.get("zero_variance_metrics") + ), + "recommended_transforms": _string_list( + stationarity.get("recommended_transforms") + ), + "limitations": _string_list(stationarity.get("limitations")), + } + + +def _decomposition_trend_payload( + levels: list[float], + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + components = _linear_trend_components(levels) + if components is None: + return { + "status": "skipped", + "reason": "insufficient_sample_count", + "sample_count": len(levels), + "required_sample_count": 2, + } + slope, intercept, fitted_values, _residuals, trend_strength = components + first_fitted = fitted_values[0] + last_fitted = fitted_values[-1] + return { + "status": "computed", + "index_basis": "observation_index", + "sample_count": len(levels), + "slope_per_observation": _rounded(slope, profile), + "intercept": _rounded(intercept, profile), + "fitted_first": _rounded(first_fitted, profile), + "fitted_last": _rounded(last_fitted, profile), + "direction": _trend_direction(slope, profile), + "trend_strength": _rounded(trend_strength, profile), + "fitted_change_first_to_last": _stationarity_change_payload( + first_fitted, + last_fitted, + profile, + ), + } + + +def _decomposition_residual_payload( + levels: list[float], + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + components = _linear_trend_components(levels) + if components is None: + return { + "status": "skipped", + "reason": "insufficient_sample_count", + "sample_count": len(levels), + "required_sample_count": 2, + } + _slope, _intercept, _fitted_values, residuals, _trend_strength = components + level_variance = _population_variance(levels) + residual_variance = _population_variance(residuals) + return { + "status": "computed", + "basis": "linear_trend_residual", + "sample_count": len(levels), + "level_variance": _rounded(level_variance, profile), + "residual_variance": _rounded(residual_variance, profile), + "residual_to_level_variance_ratio": ( + _rounded(residual_variance / level_variance, profile) + if level_variance > 0.0 + else None + ), + "residual": _numeric_summary(residuals, profile), + "absolute_residual": _numeric_summary( + [abs(value) for value in residuals], + profile, + ), + } + + +def _linear_trend_components( + values: list[float], +) -> tuple[float, float, list[float], list[float], float] | None: + sample_count = len(values) + if sample_count < 2: + return None + x_mean = (sample_count - 1) / 2.0 + y_mean = _mean(values) + denominator = sum((index - x_mean) ** 2 for index in range(sample_count)) + if denominator <= 0.0: + return None + slope = ( + sum( + (index - x_mean) * (value - y_mean) + for index, value in enumerate(values) + ) + / denominator + ) + intercept = y_mean - slope * x_mean + fitted_values = [intercept + slope * index for index in range(sample_count)] + residuals = [ + value - fitted + for value, fitted in zip(values, fitted_values, strict=True) + ] + total_variance = _population_variance(values) + residual_variance = _population_variance(residuals) + if total_variance > 0.0: + trend_strength = max( + 0.0, min(1.0, 1.0 - residual_variance / total_variance) + ) + else: + trend_strength = 1.0 if residual_variance <= 0.0 else 0.0 + return slope, intercept, fitted_values, residuals, trend_strength + + +def _trend_direction( + slope: float, + profile: HistDataFingerprintProfile, +) -> str: + tolerance = 10 ** (-max(0, int(profile.rounding_digits))) + if slope > tolerance: + return "increasing" + if slope < -tolerance: + return "decreasing" + return "flat" + + +def _decomposition_seasonality_payload( + timestamps: list[int], + levels: list[float], + absolute_returns_by_row: list[float | None], + *, + target: QualityTarget, + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + by_hour: dict[str, list[float]] = {} + returns_by_hour: dict[str, list[float]] = {} + by_weekday: dict[str, list[float]] = {} + returns_by_weekday: dict[str, list[float]] = {} + by_session: dict[str, list[float]] = {} + returns_by_session: dict[str, list[float]] = {} + usable_count = min( + len(timestamps), len(levels), len(absolute_returns_by_row) + ) + + for index in range(usable_count): + classification = classify_histdata_timestamp( + timestamps[index], + calendar_profile=profile.calendar_profile, + asset_class=_target_asset_class(target), + ) + source_datetime = classification.source_datetime + hour_key = f"{source_datetime.hour:02d}" + weekday_key = SOURCE_WEEKDAY_NAMES[source_datetime.weekday()] + active_sessions = tuple(classification.active_sessions) or ( + (SESSION_MARKET_CLOSED,) + if classification.session_state == SESSION_STATE_WEEKEND_CLOSURE + else (SESSION_NO_ACTIVE_WINDOW,) + ) + _record_decomposition_bucket( + by_hour, + returns_by_hour, + hour_key, + level=levels[index], + absolute_return=absolute_returns_by_row[index], + ) + _record_decomposition_bucket( + by_weekday, + returns_by_weekday, + weekday_key, + level=levels[index], + absolute_return=absolute_returns_by_row[index], + ) + for session in active_sessions: + _record_decomposition_bucket( + by_session, + returns_by_session, + str(session), + level=levels[index], + absolute_return=absolute_returns_by_row[index], + ) + + status = "computed" if usable_count else "skipped" + result: dict[str, JSONValue] = { + "status": status, + "sample_count": usable_count, + "grouped_by": ["source_hour", "source_weekday", "active_session"], + "by_source_hour": _decomposition_bucket_group_payload( + by_hour, + returns_by_hour, + profile, + ), + "by_source_weekday": _decomposition_bucket_group_payload( + by_weekday, + returns_by_weekday, + profile, + ), + "by_active_session": _decomposition_bucket_group_payload( + by_session, + returns_by_session, + profile, + ), + } + if status == "skipped": + result["reason"] = "insufficient_sample_count" + return result + + +def _record_decomposition_bucket( + level_buckets: dict[str, list[float]], + return_buckets: dict[str, list[float]], + bucket: str, + *, + level: float, + absolute_return: float | None, +) -> None: + level_buckets.setdefault(bucket, []).append(level) + if absolute_return is not None: + return_buckets.setdefault(bucket, []).append(absolute_return) + + +def _decomposition_bucket_group_payload( + level_buckets: Mapping[str, list[float]], + return_buckets: Mapping[str, list[float]], + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + limit = max(1, int(profile.histogram_bins)) + bucket_names = sorted(level_buckets) + included_names = bucket_names[:limit] + buckets: dict[str, JSONValue] = {} + level_means: list[float] = [] + return_means: list[float] = [] + for bucket in included_names: + bucket_levels = _finite_values(level_buckets.get(bucket, ())) + bucket_returns = _finite_values(return_buckets.get(bucket, ())) + level_mean = _mean(bucket_levels) if bucket_levels else None + return_mean = _mean(bucket_returns) if bucket_returns else None + if level_mean is not None: + level_means.append(level_mean) + if return_mean is not None: + return_means.append(return_mean) + buckets[bucket] = { + "level_count": len(bucket_levels), + "level_mean": ( + _rounded(level_mean, profile) + if level_mean is not None + else None + ), + "absolute_return_count": len(bucket_returns), + "absolute_return_mean": ( + _rounded(return_mean, profile) + if return_mean is not None + else None + ), + } + dominant_bucket = None + if bucket_names: + dominant_bucket = max( + bucket_names, + key=lambda item: (len(level_buckets.get(item, ())), item), + ) + return { + "bucket_count": len(bucket_names), + "included_bucket_count": len(included_names), + "omitted_bucket_count": max(0, len(bucket_names) - len(included_names)), + "truncated": len(included_names) < len(bucket_names), + "dominant_bucket": dominant_bucket, + "buckets": buckets, + "level_mean_dispersion": _numeric_summary(level_means, profile), + "absolute_return_mean_dispersion": _numeric_summary( + return_means, + profile, + ), + } + + +def _decomposition_smoothing_windows_payload( + levels: list[float], + returns: list[float], + profile: HistDataFingerprintProfile, +) -> tuple[dict[str, JSONValue], int, Counter[str]]: + windows: dict[str, JSONValue] = {} + skipped_reason_counts: Counter[str] = Counter() + computed_window_count = 0 + for window in profile.rolling_windows: + window_payload = _decomposition_smoothing_window_payload( + levels, + returns, + window=int(window), + profile=profile, + ) + windows[str(window)] = window_payload + if _summary_key(window_payload.get("status")) == "computed": + computed_window_count += 1 + else: + reason = _optional_summary_key(window_payload.get("reason")) + if reason: + skipped_reason_counts[reason] += 1 + return windows, computed_window_count, skipped_reason_counts + + +def _decomposition_smoothing_window_payload( + levels: list[float], + returns: list[float], + *, + window: int, + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + sample_counts: dict[str, JSONValue] = { + "level": len(levels), + "return": len(returns), + } + required_sample_count = max(2, window * 2) + if ( + window <= 0 + or len(levels) < required_sample_count + or len(returns) < required_sample_count + ): + return { + "status": "skipped", + "reason": "insufficient_sample_count", + "window": window, + "sample_counts": sample_counts, + "required_sample_count": required_sample_count, + } + level_means = _rolling_stat_values(levels, window, statistic="mean") + level_variances = _rolling_stat_values( + levels, + window, + statistic="variance", + ) + absolute_return_means = _rolling_stat_values( + [abs(value) for value in returns], + window, + statistic="mean", + ) + return { + "status": "computed", + "window": window, + "sample_counts": sample_counts, + "required_sample_count": required_sample_count, + "level_smoothed_mean": _numeric_summary(level_means, profile), + "level_smoothed_variance": _numeric_summary(level_variances, profile), + "level_smoothed_mean_drift": _stationarity_change_payload( + level_means[0], + level_means[-1], + profile, + ), + "absolute_return_smoothed_mean": _numeric_summary( + absolute_return_means, + profile, + ), + "absolute_return_smoothed_mean_drift": _stationarity_change_payload( + absolute_return_means[0], + absolute_return_means[-1], + profile, + ), + } + + +def _rolling_stat_values( + values: list[float], + window: int, + *, + statistic: str, +) -> list[float]: + if window <= 0 or len(values) < window: + return [] + return [ + _stationarity_statistic(values[index : index + window], statistic) + for index in range(0, len(values) - window + 1) + ] + + +def _decomposition_structural_break_payload( + levels: list[float], + profile: HistDataFingerprintProfile, +) -> dict[str, JSONValue]: + sample_count = len(levels) + minimum_segment_size = 2 + if sample_count < minimum_segment_size * 2: + return { + "status": "skipped", + "reason": "insufficient_sample_count", + "sample_count": sample_count, + "required_sample_count": minimum_segment_size * 2, + "minimum_segment_size": minimum_segment_size, + "candidate_count": 0, + "candidates": [], + } + candidates: list[dict[str, JSONValue]] = [] + for split_index in range( + minimum_segment_size, + sample_count - minimum_segment_size + 1, + ): + before = levels[:split_index] + after = levels[split_index:] + before_mean = _mean(before) + after_mean = _mean(after) + mean_shift = after_mean - before_mean + pooled_variance = ( + _population_variance(before) + _population_variance(after) + ) / 2.0 + if pooled_variance > 0.0: + score = abs(mean_shift) / math.sqrt(pooled_variance) + score_basis = "absolute_mean_shift_over_pooled_std" + else: + score = abs(mean_shift) * sample_count + score_basis = "scaled_absolute_mean_shift_zero_variance" + candidates.append( + { + "split_index": split_index, + "before_count": len(before), + "after_count": len(after), + "before_mean": _rounded(before_mean, profile), + "after_mean": _rounded(after_mean, profile), + "mean_shift": _rounded(mean_shift, profile), + "absolute_mean_shift": _rounded(abs(mean_shift), profile), + "score": _rounded(score, profile), + "score_basis": score_basis, + } + ) + ranked = sorted( + candidates, + key=lambda item: ( + -(_optional_float_payload(item.get("score")) or 0.0), + _int_payload(item.get("split_index")), + ), + ) + limit = max(1, int(profile.histogram_bins)) + included = ranked[:limit] + included_candidates: list[JSONValue] = [dict(item) for item in included] + return { + "status": "computed", + "basis": "two_segment_mean_shift", + "sample_count": sample_count, + "minimum_segment_size": minimum_segment_size, + "candidate_count": len(candidates), + "included_candidate_count": len(included), + "omitted_candidate_count": max(0, len(candidates) - len(included)), + "truncated": len(included) < len(candidates), + "strongest_candidate": ( + included_candidates[0] if included_candidates else None + ), + "candidates": included_candidates, + } + + +def _decomposition_limitations( + base: Mapping[str, JSONValue], + *, + stationarity_basis: Mapping[str, JSONValue], + level_count: int, + return_count: int, + computed_window_count: int, + skipped_window_count: int, +) -> tuple[str, ...]: + limitations = [ + str(value) for value in _string_list(base.get("limitations")) + ] + if level_count < 3 or return_count < 1: + limitations.append("insufficient_sample_count") + stationarity_status = _summary_key(stationarity_basis.get("status")) + if stationarity_status == "unavailable": + limitations.append("stationarity_unavailable") + elif stationarity_status == "limited": + limitations.append("stationarity_limited") + if _string_list(stationarity_basis.get("zero_variance_metrics")): + limitations.append("zero_variance") + if skipped_window_count > 0 or _int_payload( + stationarity_basis.get("skipped_window_count") + ): + limitations.append("skipped_rolling_windows") + if computed_window_count <= 0: + limitations.append("insufficient_sample_count") + return _ordered_unique(limitations) + + +def _decomposition_status( + base: Mapping[str, JSONValue], + *, + stationarity_basis: Mapping[str, JSONValue], + level_count: int, + return_count: int, + limitations: tuple[str, ...], +) -> str: + if _summary_key(base.get("sequence_status")) == "unavailable": + return "unavailable" + if level_count < 3 or return_count < 1: + return "unavailable" + if _summary_key(stationarity_basis.get("status")) == "unavailable": + return "unavailable" + if limitations: + return "limited" + return "ok" + + +def _decomposition_status_reason( + limitations: tuple[str, ...], + *, + stationarity_basis: Mapping[str, JSONValue], +) -> str: + if limitations: + return _summary_key(limitations[0]) + reason = _optional_summary_key(stationarity_basis.get("reason")) + if reason: + return reason + return "limited" + + +def decomposition_training_projection( + decomposition: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + """Return flat period-grain decomposition scalars for training rows.""" + trend = _payload_mapping(decomposition.get("trend_proxy")) + residual = _payload_mapping(decomposition.get("residual_proxy")) + structural = _payload_mapping(decomposition.get("structural_break_proxy")) + strongest = _payload_mapping(structural.get("strongest_candidate")) + stationarity = _payload_mapping(decomposition.get("stationarity_basis")) + status = _summary_key(decomposition.get("decomposition_status")) + direction = _summary_key(trend.get("direction")) + stationarity_status = _summary_key(stationarity.get("status")) + return { + "schema_version": ( + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION + ), + "grain": "period", + "identity_fields": ["series_id", "period", "row_id"], + "values": { + "decomposition_status_code": { + "unavailable": 1, + "limited": 2, + "ok": 3, + }.get(status, 0), + "decomposition_training_ready": status in {"limited", "ok"}, + "decomposition_trend_direction_code": { + "decreasing": -1, + "flat": 0, + "increasing": 1, + }.get(direction, 0), + "decomposition_trend_slope": trend.get("slope_per_observation"), + "decomposition_trend_strength": trend.get("trend_strength"), + "decomposition_residual_variance_ratio": residual.get( + "residual_to_level_variance_ratio" + ), + "decomposition_computed_window_count": _int_payload( + decomposition.get("computed_window_count") + ), + "decomposition_structural_break_candidate_count": _int_payload( + structural.get("candidate_count") + ), + "decomposition_structural_break_split_index": ( + _int_payload(strongest.get("split_index")) + if strongest + else None + ), + "decomposition_structural_break_score": strongest.get("score"), + "decomposition_stationarity_status_code": { + "unavailable": 1, + "limited": 2, + "valid": 3, + }.get(stationarity_status, 0), + }, + } + + +def project_decomposition_onto_training_frame( + frame: Any, + decomposition: Mapping[str, JSONValue], +) -> Any: + """Project period-grain decomposition scalars onto enriched tick rows.""" + required = {"series_id", "period", "row_id"} + columns = set(getattr(frame, "columns", ())) + missing = sorted(required - columns) + if missing: + raise ValueError( + "decomposition training projection requires enriched ASCII tick " + f"identity columns: {', '.join(missing)}" + ) + import polars as pl + + projection = decomposition_training_projection(decomposition) + values = _payload_mapping(projection.get("values")) + expressions = [pl.lit(value).alias(name) for name, value in values.items()] + return frame.with_columns(expressions) + + def _stationarity_payload( base: Mapping[str, JSONValue], *, diff --git a/src/histdatacom/data_quality/ingestion.py b/src/histdatacom/data_quality/ingestion.py index 62cd980c..c95979a3 100644 --- a/src/histdatacom/data_quality/ingestion.py +++ b/src/histdatacom/data_quality/ingestion.py @@ -22,6 +22,7 @@ read_quality_polars_cache, ) from histdatacom.histdata_ascii import ( + TICK, columns_for_timeframe, delimiter_for_timeframe, parse_histdata_datetime_to_utc_ms, @@ -471,23 +472,37 @@ def ingestion_quality_rules() -> tuple[QualityRule, ...]: def _is_ingestion_count_target(target: QualityTarget) -> bool: return ( - _is_ascii_text_target(target) or target.kind is QualityTargetKind.CACHE + target.data_format == "ascii" + and target.timeframe == TICK + and ( + target.kind + in { + QualityTargetKind.CSV, + QualityTargetKind.ZIP, + QualityTargetKind.CACHE, + } + ) ) def _is_ascii_schema_target(target: QualityTarget) -> bool: return _is_ascii_text_target(target) or ( target.data_format == "ascii" + and target.timeframe == TICK and target.kind is QualityTargetKind.CACHE - and bool(target.timeframe) ) def _is_ascii_text_target(target: QualityTarget) -> bool: - return target.data_format == "ascii" and target.kind in { - QualityTargetKind.CSV, - QualityTargetKind.ZIP, - } + return ( + target.data_format == "ascii" + and target.timeframe == TICK + and target.kind + in { + QualityTargetKind.CSV, + QualityTargetKind.ZIP, + } + ) def _profile_ingestion_target(target: QualityTarget) -> _IngestionProfile: diff --git a/src/histdatacom/data_quality/polars_cache.py b/src/histdatacom/data_quality/polars_cache.py index d37f5855..88604e51 100644 --- a/src/histdatacom/data_quality/polars_cache.py +++ b/src/histdatacom/data_quality/polars_cache.py @@ -17,6 +17,9 @@ class FreshPolarsCache: path: Path frame: Any source: str = "sibling" + fresh: bool | None = True + source_mtime_ns: int | None = None + cache_mtime_ns: int | None = None def read_fresh_sibling_polars_cache( @@ -25,28 +28,65 @@ def read_fresh_sibling_polars_cache( required_columns: tuple[str, ...], ) -> FreshPolarsCache | None: """Return a fresh sibling Polars cache for a CSV target, if available.""" - if target.kind is not QualityTargetKind.CSV: + cache = read_fingerprint_parity_polars_cache( + target, + required_columns=required_columns, + ) + if ( + cache is None + or target.kind is not QualityTargetKind.CSV + or cache.fresh is not True + ): return None + return cache - csv_path = Path(target.path) - cache_path = csv_path.with_name(CACHE_FILENAME) - try: - csv_stat = csv_path.stat() - cache_stat = cache_path.stat() - except OSError: - return None - if cache_stat.st_mtime_ns < csv_stat.st_mtime_ns: + +def read_fingerprint_parity_polars_cache( + target: QualityTarget, + *, + required_columns: tuple[str, ...], +) -> FreshPolarsCache | None: + """Return a readable direct or sibling cache with freshness evidence.""" + if target.kind is QualityTargetKind.CACHE: + cache = read_quality_polars_cache( + target, + required_columns=required_columns, + ) + if cache is None: + return None + try: + cache_mtime_ns = cache.path.stat().st_mtime_ns + except OSError: + cache_mtime_ns = None + return FreshPolarsCache( + path=cache.path, + frame=cache.frame, + source="direct", + fresh=None, + cache_mtime_ns=cache_mtime_ns, + ) + if target.kind not in {QualityTargetKind.CSV, QualityTargetKind.ZIP}: return None + source_path = Path(target.path) + cache_path = source_path.with_name(CACHE_FILENAME) try: + source_stat = source_path.stat() + cache_stat = cache_path.stat() frame = read_polars_cache(cache_path) except (OSError, ValueError): return None - columns = set(getattr(frame, "columns", ())) if not set(required_columns).issubset(columns): return None - return FreshPolarsCache(path=cache_path, frame=frame, source="sibling") + return FreshPolarsCache( + path=cache_path, + frame=frame, + source="sibling", + fresh=cache_stat.st_mtime_ns >= source_stat.st_mtime_ns, + source_mtime_ns=source_stat.st_mtime_ns, + cache_mtime_ns=cache_stat.st_mtime_ns, + ) def read_quality_polars_cache( diff --git a/src/histdatacom/data_quality/preflight.py b/src/histdatacom/data_quality/preflight.py index d50db5d5..aef85295 100644 --- a/src/histdatacom/data_quality/preflight.py +++ b/src/histdatacom/data_quality/preflight.py @@ -65,6 +65,9 @@ "histdatacom.quality-preflight-evidence.v1" ) QUALITY_PREFLIGHT_EVIDENCE_ARTIFACTS_METADATA_KEY = "artifacts" +QUALITY_PREFLIGHT_VALIDATION_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.quality-preflight-validation-evidence.v1" +) QUALITY_PREFLIGHT_INSPECTION_SCHEMA_VERSION = ( "histdatacom.quality-preflight-inspection.v1" ) @@ -82,6 +85,7 @@ ) QUALITY_PREFLIGHT_FINGERPRINT_CONTRACT_FINDING_DISPLAY_LIMIT = 8 QUALITY_PREFLIGHT_BOUNDED_PAYLOAD_CONTRACT_FINDING_DISPLAY_LIMIT = 8 +QUALITY_PREFLIGHT_REMEDIATION_PLAN_DISPLAY_LIMIT = 5 DEFAULT_QUALITY_PREFLIGHT_VALIDATION_REPORT_DIR = ( Path(".histdatacom") / "closure-readiness" ) @@ -131,6 +135,7 @@ def run_cache_quality_preflight( run_validation: bool = False, validation_runner: ValidationRunner | None = None, clock: Callable[[], float] = perf_counter, + validation_clock: Callable[[], float] = perf_counter, utc_now: Callable[[], datetime] = _utc_now, ) -> dict[str, JSONValue]: """Benchmark a bounded cache sample and estimate full quality runtime.""" @@ -187,6 +192,7 @@ def run_cache_quality_preflight( validation_report_path=validation_report_path, run_validation=run_validation, validation_runner=validation_runner, + validation_clock=validation_clock, ) @@ -195,6 +201,87 @@ def quality_preflight_to_json(payload: Mapping[str, JSONValue]) -> str: return json.dumps(payload, indent=2, sort_keys=True) + "\n" +def quality_preflight_validation_evidence_payload( + payload: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + """Return bounded validation evidence extracted from one preflight.""" + evidence = _mapping(payload.get("evidence")) + source = _mapping(evidence.get("validation_source")) + commands = [ + _validation_artifact_command(row) + for row in _list_of_mappings(evidence.get("validation_commands")) + ] + status_counts: dict[str, JSONValue] = { + status: sum(1 for row in commands if row.get("status") == status) + for status in ("pass", "fail", "skipped", "not-run") + } + command_payloads: list[JSONValue] = list(commands) + validation_payload: dict[str, JSONValue] = { + "schema_version": ( + QUALITY_PREFLIGHT_VALIDATION_EVIDENCE_SCHEMA_VERSION + ), + "report_kind": "quality-preflight-validation-evidence", + "generated_at_utc": str(payload.get("generated_at_utc", "")), + "state": _validation_artifact_state(commands), + "source": source, + "command_count": len(commands), + "status_counts": status_counts, + "commands": command_payloads, + } + safe_payload: dict[str, JSONValue] = publish_safe_json_mapping( + validation_payload + ) + return safe_payload + + +def quality_preflight_validation_evidence_to_json( + payload: Mapping[str, JSONValue], +) -> str: + """Return deterministic JSON for validation evidence.""" + return json.dumps(payload, indent=2, sort_keys=True) + "\n" + + +def _validation_artifact_command( + row: Mapping[str, Any], +) -> dict[str, JSONValue]: + """Normalize one validation row for the dedicated evidence artifact.""" + safe_payload: dict[str, JSONValue] = publish_safe_json_mapping( + { + "name": str(row.get("name", "")), + "command": str(row.get("command", "")), + "status": _normalized_validation_status(row.get("status")), + "exit_code": ( + _int_value(row.get("returncode")) + if "returncode" in row + else None + ), + "duration_seconds": ( + round(_float_value(row.get("duration_seconds")), 6) + if "duration_seconds" in row + else None + ), + "summary": _validation_summary(row), + "output_artifact_path": str( + row.get("output_artifact_path", "") or "" + ), + } + ) + return safe_payload + + +def _validation_artifact_state( + commands: Sequence[Mapping[str, JSONValue]], +) -> str: + statuses = {str(row.get("status", "")) for row in commands} + if "fail" in statuses: + return "fail" + if statuses == {"not-run"} or not statuses: + return "planned" + if statuses <= {"pass", "skipped"} and "pass" in statuses: + return "pass" + return "partial" + + def write_quality_preflight_report( payload: Mapping[str, JSONValue], path: str | Path, @@ -302,6 +389,7 @@ def quality_preflight_to_markdown(payload: Mapping[str, JSONValue]) -> str: quality_summary = _mapping(quality.get("summary")) remediation_audit = _mapping(quality.get("remediation_catalog_audit")) remediation_audit_summary = _mapping(remediation_audit.get("summary")) + remediation_plan = _mapping(remediation_audit.get("remediation_plan")) decision = _mapping(safe_payload.get("decision")) cache_inventory = _mapping(safe_payload.get("cache_inventory")) lines = [ @@ -577,11 +665,55 @@ def quality_preflight_to_markdown(payload: Mapping[str, JSONValue]) -> str: ) ), ), + ( + "Actionable warning/error gaps", + str( + remediation_audit_summary.get( + "unmapped_actionable_warning_error_gap_count", + 0, + ) + ), + ), + ( + "Intentional warning/error boundaries", + str( + remediation_audit_summary.get( + "intentionally_unremediable_warning_error_code_count", + 0, + ) + ), + ), + ( + "Attribution-blocked warning/error gaps", + str( + remediation_audit_summary.get( + "blocked_by_attribution_warning_error_code_count", + 0, + ) + ), + ), + ( + "Remediation plan candidates", + str(remediation_plan.get("plan_item_count", 0)), + ), ), ), "", ] ) + plan_rows = _remediation_plan_markdown_rows(remediation_plan) + if plan_rows: + lines.extend( + [ + "#### Top Remediation Plan Items", + "", + *_markdown_table( + ("Rank", "Gap", "Fixability", "Selector", "Action"), + plan_rows, + ), + "", + ] + ) lines.extend(_operational_markdown_lines(operational, runtime_cleanup)) lines.extend( [ @@ -972,6 +1104,7 @@ def format_quality_preflight_console_summary( quality_summary = _mapping(quality.get("summary")) remediation_audit = _mapping(quality.get("remediation_catalog_audit")) remediation_audit_summary = _mapping(remediation_audit.get("summary")) + remediation_plan = _mapping(remediation_audit.get("remediation_plan")) decision = _mapping(payload.get("decision")) diagnostics = _mapping(payload.get("diagnostics")) evidence = _mapping(payload.get("evidence")) @@ -1029,8 +1162,26 @@ def format_quality_preflight_console_summary( "known_warning_error_gaps=" f"{remediation_audit_summary.get('unmapped_warning_error_gap_count', 0)} " "observed_unmapped_warning_error_groups=" - f"{remediation_audit_summary.get('report_unmapped_warning_error_group_count', 0)}" + f"{remediation_audit_summary.get('report_unmapped_warning_error_group_count', 0)} " + "actionable_warning_error_gaps=" + f"{remediation_audit_summary.get('unmapped_actionable_warning_error_gap_count', 0)} " + "intentional_boundaries=" + f"{remediation_audit_summary.get('intentionally_unremediable_warning_error_code_count', 0)}" ) + plan_items = _list_of_mappings(remediation_plan.get("items")) + if plan_items: + first_plan = plan_items[0] + first_fixability = _mapping(first_plan.get("fixability")) + first_action = _mapping(first_plan.get("suggested_action")) + lines.append( + "sample remediation plan: candidates=" + f"{remediation_plan.get('plan_item_count', 0)} " + f"top={first_plan.get('finding_code', 'unknown')} " + "fixability=" + f"{first_fixability.get('level', 'unknown')}/" + f"{first_fixability.get('score', 0)} " + f"action={first_action.get('action_kind', 'inspect')}" + ) if fingerprint_contract: lines.append( "fingerprint contract audit: " @@ -1264,6 +1415,7 @@ def _payload( validation_report_path: str | Path | None, run_validation: bool, validation_runner: ValidationRunner | None, + validation_clock: Callable[[], float], ) -> dict[str, JSONValue]: target_bytes = sum(item.size_bytes for item in cache_targets) sample_bytes = _int_value(benchmark.get("sample_cache_bytes")) @@ -1358,6 +1510,7 @@ def _payload( validation_report_path=validation_report_path, run_validation=run_validation, validation_runner=validation_runner, + validation_clock=validation_clock, ) if _fingerprint_contract_audit_failed( payload["evidence"] @@ -1388,6 +1541,7 @@ def _quality_preflight_evidence_payload( validation_report_path: str | Path | None, run_validation: bool, validation_runner: ValidationRunner | None, + validation_clock: Callable[[], float], ) -> dict[str, JSONValue]: validation = _validation_commands_payload() validation_source: dict[str, JSONValue] = { @@ -1408,7 +1562,8 @@ def _quality_preflight_evidence_payload( ) if run_validation: bundle = _run_quality_preflight_validation_bundle( - runner=validation_runner + runner=validation_runner, + clock=validation_clock, ) validation_source = _combine_validation_sources( validation_source, @@ -1593,6 +1748,10 @@ def _validation_commands_payload() -> list[JSONValue]: ), ("full-pytest", "python -m pytest"), ("full-pre-commit", "python -m pre_commit run --all-files"), + ( + "readme-help-sync", + "python scripts/sync_readme_cli_help.py --check", + ), ("git-diff-check", "git diff --check"), ) return [ @@ -1928,8 +2087,10 @@ def _validation_row_from_result( ) payload: dict[str, JSONValue] = { **row, + "command": str(result.get("command") or row.get("command") or ""), "status": status, "reason": _bounded_text(reason), + "summary": _validation_summary(result), "source": dict(source), } if "returncode" in result: @@ -1940,9 +2101,39 @@ def _validation_row_from_result( payload["stdout_tail"] = stdout_tail if stderr_tail: payload["stderr_tail"] = stderr_tail + if "duration_seconds" in result: + payload["duration_seconds"] = round( + max(0.0, _float_value(result.get("duration_seconds"))), 6 + ) + output_artifact_path = str( + result.get("output_artifact_path") or result.get("log_path") or "" + ) + if output_artifact_path: + payload["output_artifact_path"] = str( + publish_safe_path(output_artifact_path) + ) return payload +def _validation_summary(result: Mapping[str, Any]) -> str: + """Return one bounded, publish-safe validation summary.""" + explicit = str(result.get("summary", "") or "") + if explicit: + return _bounded_text(explicit) + status = _normalized_validation_status(result.get("status")) + stream = ( + str(result.get("stderr_tail", "") or "") + if status == "fail" + else str(result.get("stdout_tail", "") or "") + ) + if stream: + return _bounded_text(stream) + reason = str(result.get("reason", "") or "") + if reason: + return _bounded_text(reason) + return f"validation command {status}" + + def _normalized_validation_status(value: object) -> str: status = str(value or "").lower() if status in {"pass", "passed", "success", "succeeded"}: @@ -1962,10 +2153,12 @@ def _validation_target_name(result: Mapping[str, JSONValue]) -> str: normalized_name = name.removeprefix("gate-") aliases = { "pytest": "full-pytest", + "full-tests": "full-pytest", "full-pytest": "full-pytest", "pre-commit": "full-pre-commit", "pre_commit": "full-pre-commit", "full-pre-commit": "full-pre-commit", + "readme-help-sync": "readme-help-sync", "git-diff-check": "git-diff-check", "focused-quality-preflight-tests": "focused-quality-preflight-tests", } @@ -1982,6 +2175,8 @@ def _validation_target_name(result: Mapping[str, JSONValue]) -> str: return "full-pytest" if "pre-commit" in command_key or "pre_commit" in command: return "full-pre-commit" + if "sync-readme-cli-help.py" in command_key and "--check" in command_key: + return "readme-help-sync" if "git diff --check" in command_key: return "git-diff-check" return "" @@ -1997,12 +2192,13 @@ def _validation_command_names() -> set[str]: def _run_quality_preflight_validation_bundle( *, runner: ValidationRunner | None = None, + clock: Callable[[], float] = perf_counter, ) -> dict[str, JSONValue]: source: dict[str, JSONValue] = { "state": "generated", "reason": ( - "bounded quality preflight validation ran focused tests and " - "git diff check only" + "bounded quality preflight validation ran focused tests, README " + "help sync, and git diff checks only" ), } commands: tuple[tuple[str, tuple[str, ...]], ...] = ( @@ -2017,10 +2213,18 @@ def _run_quality_preflight_validation_bundle( "-q", ), ), + ( + "readme-help-sync", + ( + sys.executable, + "scripts/sync_readme_cli_help.py", + "--check", + ), + ), ("git-diff-check", ("git", "diff", "--check")), ) results: list[JSONValue] = [ - _run_validation_command(name, command, runner=runner) + _run_validation_command(name, command, runner=runner, clock=clock) for name, command in commands ] return {"source": source, "results": results} @@ -2031,7 +2235,9 @@ def _run_validation_command( command: Sequence[str], *, runner: ValidationRunner | None, + clock: Callable[[], float], ) -> dict[str, JSONValue]: + started_at = clock() if runner is None: result = subprocess.run( list(command), @@ -2041,11 +2247,19 @@ def _run_validation_command( ) else: result = runner(tuple(command)) + duration_seconds = max(0.0, clock() - started_at) + status = "pass" if result.returncode == 0 else "fail" + summary = _bounded_text( + (result.stderr if status == "fail" else result.stdout) + or f"validation command {status}" + ) return { "name": name, "command": _display_validation_command(name), - "status": "pass" if result.returncode == 0 else "fail", + "status": status, "returncode": result.returncode, + "duration_seconds": round(duration_seconds, 6), + "summary": summary, "stdout_tail": _bounded_text(result.stdout or ""), "stderr_tail": _bounded_text(result.stderr or ""), } @@ -3018,6 +3232,34 @@ def _fingerprint_contract_audit_markdown_rows( ) +def _remediation_plan_markdown_rows( + plan: Mapping[str, Any], +) -> tuple[tuple[str, str, str, str, str], ...]: + rows: list[tuple[str, str, str, str, str]] = [] + for item in _list_of_mappings(plan.get("items"))[ + :QUALITY_PREFLIGHT_REMEDIATION_PLAN_DISPLAY_LIMIT + ]: + fixability = _mapping(item.get("fixability")) + selector = _mapping(item.get("suggested_selector")) + action = _mapping(item.get("suggested_action")) + rows.append( + ( + str(item.get("rank", "")), + ( + f"{item.get('rule_id', 'unknown')}:" + f"{item.get('finding_code', 'unknown')}" + ), + ( + f"{fixability.get('level', 'unknown')}/" + f"{fixability.get('score', 0)}" + ), + str(selector.get("shape", "unknown")), + str(action.get("action_kind", "inspect")), + ) + ) + return tuple(rows) + + def _fingerprint_contract_finding_rows( audit: Mapping[str, Any], ) -> tuple[tuple[str, str, str, str], ...]: @@ -3294,6 +3536,7 @@ def _format_duration(value: object) -> str: "QUALITY_PREFLIGHT_LARGE_CACHE_TARGET_COUNT", "QUALITY_PREFLIGHT_INSPECTION_SCHEMA_VERSION", "QUALITY_PREFLIGHT_SCHEMA_VERSION", + "QUALITY_PREFLIGHT_VALIDATION_EVIDENCE_SCHEMA_VERSION", "QUALITY_PREFLIGHT_VALIDATION_REPORT_LATEST", "QualityDiscoveryError", "discover_latest_quality_preflight_validation_report", @@ -3304,8 +3547,12 @@ def _format_duration(value: object) -> str: "load_quality_preflight_evidence", "quality_preflight_to_markdown", "quality_preflight_to_json", + "quality_preflight_validation_evidence_payload", + "quality_preflight_validation_evidence_to_json", "quality_run_preflight_warning", "run_cache_quality_preflight", "write_quality_preflight_markdown_report", + "write_quality_preflight_evidence_artifact", "write_quality_preflight_report", + "register_quality_preflight_evidence_artifact", ] diff --git a/src/histdatacom/data_quality/profiles.py b/src/histdatacom/data_quality/profiles.py index 37939e3a..40479849 100644 --- a/src/histdatacom/data_quality/profiles.py +++ b/src/histdatacom/data_quality/profiles.py @@ -13,18 +13,53 @@ HistDataCalendarProfile, calendar_profile_from_mapping, ) +from histdatacom.data_quality.autoregressive import ( + AutoregressiveProfile, + AutoregressiveSpecification, +) +from histdatacom.data_quality.classical_baselines import ( + MAX_BASELINE_ROLLING_WINDOWS, + ClassicalBaselineProfile, +) +from histdatacom.data_quality.classical_model_contracts import ( + ClassicalModelInputProfile, + ClassicalModelResourcePolicy, +) +from histdatacom.data_quality.classical_model_comparison import ( + ClassicalModelComparisonProfile, +) from histdatacom.data_quality.contracts import QualitySeverity +from histdatacom.data_quality.exponential_smoothing import ( + ExponentialSmoothingProfile, + ExponentialSmoothingSpecification, +) from histdatacom.data_quality.fingerprints import ( DEFAULT_FINGERPRINT_HISTOGRAM_BINS, DEFAULT_FINGERPRINT_LAGS, DEFAULT_FINGERPRINT_MAX_ROWS, + DEFAULT_FINGERPRINT_PARITY_MISMATCH_LIMIT, DEFAULT_FINGERPRINT_QUANTILES, DEFAULT_FINGERPRINT_ROLLING_WINDOWS, DEFAULT_FINGERPRINT_ROUNDING_DIGITS, + DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT, SERIES_FINGERPRINT_RULE_ID, HistDataFingerprintDistributionAttentionProfile, + HistDataFingerprintParityProfile, HistDataFingerprintProfile, ) +from histdatacom.data_quality.seasonal_exogenous import ( + CalendarRegressorProfile, + SeasonalExogenousProfile, + SeasonalExogenousSpecification, +) +from histdatacom.data_quality.state_space import ( + StateSpaceProfile, + StateSpaceSpecification, +) +from histdatacom.data_quality.volatility import ( + VolatilityProfile, + VolatilitySpecification, +) from histdatacom.data_quality.ingestion import ( ASCII_ROW_COUNT_INGESTION_RULE_ID, DEFAULT_MIN_ROW_COUNT, @@ -58,6 +93,9 @@ from histdatacom.runtime_contracts import JSONValue QUALITY_PROFILE_SCHEMA_VERSION = "histdatacom.quality-profile.v1" +QUALITY_PROFILE_RESOLUTION_SCHEMA_VERSION = ( + "histdatacom.quality-profile-resolution.v1" +) DEFAULT_QUALITY_PROFILE_NAME = "default" DEFAULT_QUALITY_PROFILE_SOURCE = "default" OPERATOR_QUALITY_PROFILE_SOURCE = "operator-config" @@ -474,7 +512,17 @@ def fingerprint_profile(self) -> HistDataFingerprintProfile: "histogram_bins", "max_rows", "rounding_digits", + "topology_inspection_sample_limit", "distribution_attention", + "cache_source_parity", + "classical_baselines", + "classical_model_input", + "exponential_smoothing", + "autoregressive", + "seasonal_exogenous", + "state_space", + "volatility", + "classical_model_comparison", }, SERIES_FINGERPRINT_RULE_ID, ) @@ -518,6 +566,14 @@ def fingerprint_profile(self) -> HistDataFingerprintProfile: minimum=0, path=SERIES_FINGERPRINT_RULE_ID, ), + topology_inspection_sample_limit=_int_field( + config, + "topology_inspection_sample_limit", + DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT, + minimum=0, + maximum=DEFAULT_FINGERPRINT_TOPOLOGY_INSPECTION_SAMPLE_LIMIT, + path=SERIES_FINGERPRINT_RULE_ID, + ), calendar_profile=self.calendar_profile(), distribution_attention=( _fingerprint_distribution_attention_profile( @@ -529,6 +585,80 @@ def fingerprint_profile(self) -> HistDataFingerprintProfile: path=f"{SERIES_FINGERPRINT_RULE_ID}.distribution_attention", ) ), + cache_source_parity=_fingerprint_parity_profile( + _mapping_field( + config, + "cache_source_parity", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=f"{SERIES_FINGERPRINT_RULE_ID}.cache_source_parity", + ), + classical_baselines=_classical_baseline_profile( + _mapping_field( + config, + "classical_baselines", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=f"{SERIES_FINGERPRINT_RULE_ID}.classical_baselines", + ), + classical_model_input=_classical_model_input_profile( + _mapping_field( + config, + "classical_model_input", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=(f"{SERIES_FINGERPRINT_RULE_ID}.classical_model_input"), + ), + exponential_smoothing=_exponential_smoothing_profile( + _mapping_field( + config, + "exponential_smoothing", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=f"{SERIES_FINGERPRINT_RULE_ID}.exponential_smoothing", + ), + autoregressive=_autoregressive_profile( + _mapping_field( + config, + "autoregressive", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=f"{SERIES_FINGERPRINT_RULE_ID}.autoregressive", + ), + seasonal_exogenous=_seasonal_exogenous_profile( + _mapping_field( + config, + "seasonal_exogenous", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=f"{SERIES_FINGERPRINT_RULE_ID}.seasonal_exogenous", + ), + state_space=_state_space_profile( + _mapping_field( + config, + "state_space", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=f"{SERIES_FINGERPRINT_RULE_ID}.state_space", + ), + volatility=_volatility_profile( + _mapping_field( + config, + "volatility", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=f"{SERIES_FINGERPRINT_RULE_ID}.volatility", + ), + classical_model_comparison=_classical_model_comparison_profile( + _mapping_field( + config, + "classical_model_comparison", + path=SERIES_FINGERPRINT_RULE_ID, + ), + path=( + f"{SERIES_FINGERPRINT_RULE_ID}.classical_model_comparison" + ), + ), ) def reporting_profile(self) -> QualityReportingProfile: @@ -573,13 +703,224 @@ def to_metadata(self) -> dict[str, JSONValue]: return payload +@dataclass(frozen=True, slots=True) +class QualityProfileValueSource: + """Value-level provenance retained while a quality profile is resolved.""" + + path: str + value: JSONValue + source: str + profile_name: str = "" + source_path: str = "" + selected_by: str = "" + override: bool = False + previous_source: str = "" + previous_value: JSONValue | None = None + previous_value_present: bool = False + + def to_payload(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible provenance metadata.""" + payload: dict[str, JSONValue] = { + "path": self.path, + "value": self.value, + "source": self.source, + } + if self.profile_name: + payload["profile_name"] = self.profile_name + if self.source_path: + payload["source_path"] = self.source_path + if self.selected_by: + payload["selected_by"] = self.selected_by + if self.override: + payload["override"] = True + payload["previous_source"] = self.previous_source or "unknown" + payload["overridden_source"] = self.previous_source or "unknown" + if self.previous_value_present: + payload["previous_value"] = self.previous_value + return payload + + +@dataclass(frozen=True, slots=True) +class QualityProfileResolution: + """Resolved quality profile plus deterministic value provenance.""" + + profile: QualityProfile + value_sources: tuple[QualityProfileValueSource, ...] + input_channels: tuple[dict[str, JSONValue], ...] + schema_version: str = QUALITY_PROFILE_RESOLUTION_SCHEMA_VERSION + + def to_payload(self) -> dict[str, JSONValue]: + """Return the resolution contract as JSON-compatible metadata.""" + return { + "schema_version": self.schema_version, + "resolved_profile": _expanded_quality_profile_payload(self.profile), + "input_channels": cast(JSONValue, list(self.input_channels)), + "effective_value_sources": cast( + JSONValue, + [item.to_payload() for item in self.value_sources], + ), + } + + def default_quality_profile() -> QualityProfile: """Return the deterministic default profile.""" return QualityProfile() +def resolve_quality_profile( + payload: Mapping[str, Any] | None = None, + *, + source: str = OPERATOR_QUALITY_PROFILE_SOURCE, + source_path: str = "", + config_path: str = "", + selected_by: str = "", +) -> QualityProfileResolution: + """Resolve a profile while retaining source metadata for every value.""" + raw_payload = dict(payload or {}) + profile = _quality_profile_from_mapping( + raw_payload, + source=source, + source_path=source_path, + ) + source_kind = ( + "built_in_default" + if not raw_payload + else quality_profile_source_kind(source) + ) + explicit_values = dict(_flatten_profile_mapping(raw_payload)) + sources: list[QualityProfileValueSource] = [] + for path, value in _flatten_profile_mapping( + _expanded_quality_profile_payload(profile) + ): + value_source = _resolved_profile_value_source( + path, + explicit_values=explicit_values, + source_kind=source_kind, + ) + sources.append( + QualityProfileValueSource( + path=path, + value=value, + source=value_source, + profile_name=( + profile.name + if value_source not in {"built_in_default", "named_profile"} + else "" + ), + source_path=( + source_path if value_source != "built_in_default" else "" + ), + selected_by=( + selected_by + if value_source not in {"built_in_default", "named_profile"} + else "" + ), + ) + ) + channels: list[dict[str, JSONValue]] = [ + { + "kind": "built_in_default", + "description": "Built-in quality profile defaults.", + } + ] + if config_path: + _append_profile_input_channel( + channels, + {"kind": "yaml_config", "path": config_path}, + ) + if _is_named_quality_profile(profile, raw_payload): + _append_profile_input_channel( + channels, + {"kind": "named_profile", "profile_name": profile.name}, + ) + if raw_payload: + channel: dict[str, JSONValue] = { + "kind": source_kind, + "profile_name": profile.name, + } + if source_path: + channel["source_path"] = source_path + if selected_by: + channel["selected_by"] = selected_by + _append_profile_input_channel(channels, channel) + return QualityProfileResolution( + profile=profile, + value_sources=tuple(sorted(sources, key=lambda item: item.path)), + input_channels=tuple(channels), + ) + + +def apply_quality_profile_overrides( + resolution: QualityProfileResolution, + overrides: Mapping[str, JSONValue], + *, + source: str, + source_path: str = "", +) -> QualityProfileResolution: + """Apply value overrides while retaining previous source and value facts.""" + if not overrides: + return resolution + payload = resolution.profile.to_request_payload() + if resolution.profile.is_default: + payload["name"] = "operator" + payload["source"] = _quality_profile_contract_source(source) + override_pointers: set[str] = set() + for path, value in sorted(overrides.items()): + pointer = _profile_pointer(path) + override_pointers.add(pointer) + _set_profile_pointer(payload, pointer, value) + profile = _quality_profile_from_mapping(payload) + before = {item.path: item for item in resolution.value_sources} + sources: list[QualityProfileValueSource] = [] + for path, value in _flatten_profile_mapping( + _expanded_quality_profile_payload(profile) + ): + previous = before.get(path) + changed = previous is None or previous.value != value + explicitly_overridden = path in override_pointers + if not changed and not explicitly_overridden and previous is not None: + sources.append(previous) + continue + sources.append( + QualityProfileValueSource( + path=path, + value=value, + source=source, + profile_name=profile.name, + source_path=source_path, + override=True, + previous_source=(previous.source if previous else "unknown"), + previous_value=(previous.value if previous else None), + previous_value_present=previous is not None, + ) + ) + channels = [dict(channel) for channel in resolution.input_channels] + _append_profile_input_channel( + channels, + { + "kind": source, + "paths": cast(JSONValue, sorted(override_pointers)), + }, + ) + return QualityProfileResolution( + profile=profile, + value_sources=tuple(sorted(sources, key=lambda item: item.path)), + input_channels=tuple(channels), + ) + + def load_quality_profile_file(path: str | Path) -> QualityProfile: """Load and validate a JSON quality profile file.""" + return load_quality_profile_file_resolution(path).profile + + +def load_quality_profile_file_resolution( + path: str | Path, + *, + config_path: str = "", + selected_by: str = "", +) -> QualityProfileResolution: + """Load a JSON profile while preserving file and selection provenance.""" profile_path = Path(path).expanduser() try: payload = json.loads(profile_path.read_text(encoding="utf-8")) @@ -592,10 +933,12 @@ def load_quality_profile_file(path: str | Path) -> QualityProfile: if not isinstance(payload, Mapping): msg = "quality profile JSON root must be an object" raise QualityProfileError(msg) - return quality_profile_from_mapping( + return resolve_quality_profile( payload, source="file", source_path=str(profile_path), + config_path=config_path, + selected_by=selected_by, ) @@ -606,6 +949,20 @@ def quality_profile_from_mapping( source_path: str = "", ) -> QualityProfile: """Validate and return a quality profile from a mapping payload.""" + return _quality_profile_from_mapping( + payload, + source=source, + source_path=source_path, + ) + + +def _quality_profile_from_mapping( + payload: Mapping[str, Any] | None, + *, + source: str = OPERATOR_QUALITY_PROFILE_SOURCE, + source_path: str = "", +) -> QualityProfile: + """Construct a profile without discarding resolver input metadata.""" if not payload: return default_quality_profile() _reject_unknown_keys(payload, _TOP_LEVEL_KEYS, "quality_profile") @@ -633,6 +990,19 @@ def quality_profile_from_mapping( return profile +def quality_profile_resolution_from_value( + value: Mapping[str, Any] | QualityProfile | None, +) -> QualityProfileResolution: + """Normalize a public profile value into the structured resolution form.""" + if isinstance(value, QualityProfile): + return resolve_quality_profile( + value.to_request_payload(), + source=value.source, + source_path=value.source_path, + ) + return resolve_quality_profile(value) + + def quality_profile_from_value( value: Mapping[str, Any] | QualityProfile | None, ) -> QualityProfile: @@ -651,6 +1021,153 @@ def quality_profile_metadata( return quality_profile_from_value(value).to_metadata() +def quality_profile_source_kind(value: str | QualityProfile) -> str: + """Return a stable public provenance kind for a profile source.""" + source = value.source if isinstance(value, QualityProfile) else str(value) + aliases = { + "default": "built_in_default", + "file": "profile_file", + "api-options": "api_options", + "cli-options": "cli_options", + "operator-config": "operator_config", + "yaml-config": "yaml_config", + "cli-override": "cli_override", + } + return aliases.get(source, source.replace("-", "_") or "unknown") + + +def _expanded_quality_profile_payload( + profile: QualityProfile, +) -> dict[str, JSONValue]: + payload = profile.to_request_payload() + if "reporting" not in payload: + payload["reporting"] = profile.reporting_profile().to_metadata() + return payload + + +def _flatten_profile_mapping( + value: Mapping[str, Any], + *, + prefix: str = "", +) -> list[tuple[str, JSONValue]]: + flattened: list[tuple[str, JSONValue]] = [] + for key in sorted(value, key=str): + path = f"{prefix}/{_profile_pointer_token(str(key))}" + item = value[key] + if isinstance(item, Mapping): + if item: + flattened.extend(_flatten_profile_mapping(item, prefix=path)) + else: + flattened.append((path, {})) + else: + flattened.append((path, cast(JSONValue, item))) + return flattened + + +def _resolved_profile_value_source( + path: str, + *, + explicit_values: Mapping[str, JSONValue], + source_kind: str, +) -> str: + if path == "/name" and path in explicit_values: + return "named_profile" + if path in explicit_values: + return source_kind + if ( + path in {"/source", "/source_path"} + and source_kind != "built_in_default" + ): + return source_kind + return "built_in_default" + + +def _is_named_quality_profile( + profile: QualityProfile, + raw_payload: Mapping[str, Any], +) -> bool: + return "name" in raw_payload and profile.name not in { + DEFAULT_QUALITY_PROFILE_NAME, + "operator", + } + + +def _append_profile_input_channel( + channels: list[dict[str, JSONValue]], + channel: Mapping[str, JSONValue], +) -> None: + kind = str(channel.get("kind") or "") + for existing in channels: + if existing.get("kind") != kind: + continue + for key, value in channel.items(): + if key == "paths": + current = existing.get("paths") + current_paths = ( + list(current) if isinstance(current, list) else [] + ) + incoming = list(value) if isinstance(value, list) else [] + existing["paths"] = cast( + JSONValue, + sorted({str(item) for item in (*current_paths, *incoming)}), + ) + elif key != "kind": + existing[key] = value + return + channels.append(dict(channel)) + + +def _profile_pointer(path: str) -> str: + if path.startswith("/"): + return path + return "/" + "/".join( + _profile_pointer_token(part) for part in path.split(".") if part + ) + + +def _profile_pointer_token(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def _profile_pointer_token_unescape(value: str) -> str: + return value.replace("~1", "/").replace("~0", "~") + + +def _set_profile_pointer( + payload: dict[str, JSONValue], + pointer: str, + value: JSONValue, +) -> None: + tokens = [ + _profile_pointer_token_unescape(token) + for token in pointer.strip("/").split("/") + if token + ] + if not tokens: + raise QualityProfileError( + "quality profile override path cannot be empty" + ) + current: dict[str, JSONValue] = payload + for token in tokens[:-1]: + existing = current.get(token) + if isinstance(existing, dict): + child = existing + else: + child = {} + current[token] = child + current = child + current[tokens[-1]] = value + + +def _quality_profile_contract_source(source: str) -> str: + aliases = { + "api_options": "api-options", + "cli_override": "cli-options", + "yaml_config": "yaml-config", + } + return aliases.get(source, source.replace("_", "-")) + + def validate_quality_profile(profile: QualityProfile) -> None: """Eagerly validate every configured rule stanza.""" profile.row_count_profile() @@ -1202,53 +1719,1639 @@ def _fingerprint_distribution_attention_profile( ) -def _quality_reporting_profile( +def _fingerprint_parity_profile( value: Mapping[str, JSONValue], -) -> QualityReportingProfile: + *, + path: str, +) -> HistDataFingerprintParityProfile: + _reject_unknown_keys(value, {"enabled", "mismatch_limit"}, path) + return HistDataFingerprintParityProfile( + enabled=_bool_field(value, "enabled", False, path=path), + mismatch_limit=_int_field( + value, + "mismatch_limit", + DEFAULT_FINGERPRINT_PARITY_MISMATCH_LIMIT, + minimum=0, + path=path, + ), + ) + + +def _classical_baseline_profile( + value: Mapping[str, JSONValue], + *, + path: str, +) -> ClassicalBaselineProfile: + base = ClassicalBaselineProfile() _reject_unknown_keys( value, - {"remediation_catalog_audit"}, - "quality_profile.reporting", + { + "enabled", + "evaluation_fraction", + "minimum_training_rows", + "minimum_evaluation_rows", + "rolling_windows", + "session_seasonal_enabled", + "rounding_digits", + }, + path, ) - remediation_catalog_audit = _mapping_field( + rolling_windows = _fingerprint_int_sequence( value, - "remediation_catalog_audit", - path="quality_profile.reporting", - ) - _reject_unknown_keys( - remediation_catalog_audit, - {"enabled"}, - "quality_profile.reporting.remediation_catalog_audit", + "rolling_windows", + base.rolling_windows, + path=path, ) - return QualityReportingProfile( - remediation_catalog_audit=QualityRemediationCatalogAuditProfile( - enabled=_bool_field( - remediation_catalog_audit, - "enabled", - False, - path="quality_profile.reporting.remediation_catalog_audit", - ) + if len(rolling_windows) > MAX_BASELINE_ROLLING_WINDOWS: + raise QualityProfileError( + f"{path}.rolling_windows supports at most " + f"{MAX_BASELINE_ROLLING_WINDOWS} values" ) + return ClassicalBaselineProfile( + enabled=_bool_field(value, "enabled", base.enabled, path=path), + evaluation_fraction=_float_field( + value, + "evaluation_fraction", + base.evaluation_fraction, + minimum=0.01, + maximum=0.99, + path=path, + ), + minimum_training_rows=_int_field( + value, + "minimum_training_rows", + base.minimum_training_rows, + minimum=1, + path=path, + ), + minimum_evaluation_rows=_int_field( + value, + "minimum_evaluation_rows", + base.minimum_evaluation_rows, + minimum=1, + path=path, + ), + rolling_windows=rolling_windows, + session_seasonal_enabled=_bool_field( + value, + "session_seasonal_enabled", + base.session_seasonal_enabled, + path=path, + ), + rounding_digits=_int_field( + value, + "rounding_digits", + base.rounding_digits, + minimum=0, + maximum=16, + path=path, + ), ) -def _mapping_field( - mapping: Mapping[str, Any], - key: str, +def _classical_model_input_profile( + value: Mapping[str, JSONValue], *, path: str, -) -> dict[str, JSONValue]: - value = mapping.get(key, {}) - if value in (None, ""): - return {} - return _expect_mapping(value, path=f"{path}.{key}") +) -> ClassicalModelInputProfile: + base = ClassicalModelInputProfile() + _reject_unknown_keys( + value, + { + "enabled", + "frequency_ms", + "alignment_epoch_ms", + "closed_side", + "label_side", + "midpoint_aggregation", + "spread_aggregation", + "minimum_observations_per_bin", + "expected_closure_policy", + "unexpected_missing_policy", + "transform", + "differencing_order", + "seasonal_differencing_order", + "seasonal_period", + "horizons", + "fold_kind", + "minimum_training_observations", + "minimum_evaluation_observations", + "step_size", + "rolling_window", + "embargo_observations", + "rounding_digits", + "resources", + }, + path, + ) + resources = _classical_model_resource_policy( + _mapping_field(value, "resources", path=path), + path=f"{path}.resources", + ) + try: + return ClassicalModelInputProfile( + enabled=_bool_field(value, "enabled", base.enabled, path=path), + frequency_ms=_int_field( + value, + "frequency_ms", + base.frequency_ms, + minimum=1, + path=path, + ), + alignment_epoch_ms=_int_field( + value, + "alignment_epoch_ms", + base.alignment_epoch_ms, + path=path, + ), + closed_side=_string_field( + value, "closed_side", base.closed_side, path=path + ), + label_side=_string_field( + value, "label_side", base.label_side, path=path + ), + midpoint_aggregation=_string_field( + value, + "midpoint_aggregation", + base.midpoint_aggregation, + path=path, + ), + spread_aggregation=_string_field( + value, + "spread_aggregation", + base.spread_aggregation, + path=path, + ), + minimum_observations_per_bin=_int_field( + value, + "minimum_observations_per_bin", + base.minimum_observations_per_bin, + minimum=1, + path=path, + ), + expected_closure_policy=_string_field( + value, + "expected_closure_policy", + base.expected_closure_policy, + path=path, + ), + unexpected_missing_policy=_string_field( + value, + "unexpected_missing_policy", + base.unexpected_missing_policy, + path=path, + ), + transform=_string_field( + value, "transform", base.transform, path=path + ), + differencing_order=_int_field( + value, + "differencing_order", + base.differencing_order, + minimum=0, + maximum=2, + path=path, + ), + seasonal_differencing_order=_int_field( + value, + "seasonal_differencing_order", + base.seasonal_differencing_order, + minimum=0, + maximum=1, + path=path, + ), + seasonal_period=_int_field( + value, + "seasonal_period", + base.seasonal_period, + minimum=0, + path=path, + ), + horizons=_fingerprint_int_sequence( + value, + "horizons", + base.horizons, + path=path, + ), + fold_kind=_string_field( + value, "fold_kind", base.fold_kind, path=path + ), + minimum_training_observations=_int_field( + value, + "minimum_training_observations", + base.minimum_training_observations, + minimum=1, + path=path, + ), + minimum_evaluation_observations=_int_field( + value, + "minimum_evaluation_observations", + base.minimum_evaluation_observations, + minimum=1, + path=path, + ), + step_size=_int_field( + value, + "step_size", + base.step_size, + minimum=1, + path=path, + ), + rolling_window=_int_field( + value, + "rolling_window", + base.rolling_window, + minimum=0, + path=path, + ), + embargo_observations=_int_field( + value, + "embargo_observations", + base.embargo_observations, + minimum=0, + path=path, + ), + rounding_digits=_int_field( + value, + "rounding_digits", + base.rounding_digits, + minimum=0, + maximum=16, + path=path, + ), + resources=resources, + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc -def _expect_mapping(value: Any, *, path: str) -> dict[str, JSONValue]: - if not isinstance(value, Mapping): - msg = f"{path} must be an object" - raise QualityProfileError(msg) - return _json_mapping(value) +def _classical_model_resource_policy( + value: Mapping[str, JSONValue], + *, + path: str, +) -> ClassicalModelResourcePolicy: + base = ClassicalModelResourcePolicy() + keys = set(base.to_metadata()) + _reject_unknown_keys(value, keys, path) + try: + return ClassicalModelResourcePolicy( + **{ + key: _int_field( + value, + key, + int(getattr(base, key)), + minimum=1, + path=path, + ) + for key in keys + } + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _exponential_smoothing_profile( + value: Mapping[str, JSONValue], + *, + path: str, +) -> ExponentialSmoothingProfile: + base = ExponentialSmoothingProfile() + _reject_unknown_keys( + value, + { + "enabled", + "specifications", + "projection_specification_id", + "projection_horizon", + "baseline_rolling_windows", + "rounding_digits", + }, + path, + ) + specifications = base.specifications + if "specifications" in value: + raw = value["specifications"] + if not isinstance(raw, list) or not raw: + raise QualityProfileError( + f"{path}.specifications must be a non-empty list" + ) + parsed: list[ExponentialSmoothingSpecification] = [] + for index, item in enumerate(raw): + parsed.append( + _exponential_smoothing_specification( + _expect_mapping( + item, path=f"{path}.specifications[{index}]" + ), + path=f"{path}.specifications[{index}]", + ) + ) + specifications = tuple(parsed) + try: + return ExponentialSmoothingProfile( + enabled=_bool_field(value, "enabled", base.enabled, path=path), + specifications=specifications, + projection_specification_id=_string_field( + value, + "projection_specification_id", + ( + specifications[0].specification_id + if "projection_specification_id" not in value + else base.projection_specification_id + ), + path=path, + ), + projection_horizon=_int_field( + value, + "projection_horizon", + base.projection_horizon, + minimum=1, + path=path, + ), + baseline_rolling_windows=_fingerprint_int_sequence( + value, + "baseline_rolling_windows", + base.baseline_rolling_windows, + path=path, + ), + rounding_digits=_int_field( + value, + "rounding_digits", + base.rounding_digits, + minimum=0, + maximum=16, + path=path, + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _autoregressive_profile( + value: Mapping[str, JSONValue], + *, + path: str, +) -> AutoregressiveProfile: + base = AutoregressiveProfile() + _reject_unknown_keys( + value, + { + "enabled", + "specifications", + "projection_specification_ids", + "projection_horizon", + "baseline_rolling_windows", + "compare_exponential_smoothing", + "rounding_digits", + }, + path, + ) + specifications = base.specifications + if "specifications" in value: + raw = value["specifications"] + if not isinstance(raw, list) or not raw: + raise QualityProfileError( + f"{path}.specifications must be a non-empty list" + ) + specifications = tuple( + _autoregressive_specification( + _expect_mapping(item, path=f"{path}.specifications[{index}]"), + path=f"{path}.specifications[{index}]", + ) + for index, item in enumerate(raw) + ) + projection_ids = base.projection_specification_ids + if "projection_specification_ids" in value: + raw_ids = value["projection_specification_ids"] + if not isinstance(raw_ids, list) or not raw_ids: + raise QualityProfileError( + f"{path}.projection_specification_ids must be a non-empty list" + ) + if not all(isinstance(item, str) and item for item in raw_ids): + raise QualityProfileError( + f"{path}.projection_specification_ids must contain strings" + ) + projection_ids = tuple(cast(list[str], raw_ids)) + elif "specifications" in value: + by_family: dict[str, str] = {} + for specification in specifications: + by_family.setdefault( + specification.family, specification.specification_id + ) + projection_ids = tuple(by_family.values()) + try: + return AutoregressiveProfile( + enabled=_bool_field(value, "enabled", base.enabled, path=path), + specifications=specifications, + projection_specification_ids=projection_ids, + projection_horizon=_int_field( + value, + "projection_horizon", + base.projection_horizon, + minimum=1, + path=path, + ), + baseline_rolling_windows=_fingerprint_int_sequence( + value, + "baseline_rolling_windows", + base.baseline_rolling_windows, + path=path, + ), + compare_exponential_smoothing=_bool_field( + value, + "compare_exponential_smoothing", + base.compare_exponential_smoothing, + path=path, + ), + rounding_digits=_int_field( + value, + "rounding_digits", + base.rounding_digits, + minimum=0, + maximum=16, + path=path, + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _autoregressive_specification( + value: Mapping[str, JSONValue], + *, + path: str, +) -> AutoregressiveSpecification: + _reject_unknown_keys( + value, + { + "specification_id", + "family", + "p", + "d", + "q", + "trend", + "initialization_method", + "estimation_method", + "enforce_stationarity", + "enforce_invertibility", + "concentrate_scale", + "fixed_parameters", + "max_iterations", + }, + path, + ) + if ( + "specification_id" not in value + or "family" not in value + or "p" not in value + ): + raise QualityProfileError( + f"{path} requires specification_id, family, and p" + ) + family = _string_field(value, "family", "", path=path) + fixed_parameters: tuple[tuple[str, float], ...] = () + if "fixed_parameters" in value: + raw_fixed = _expect_mapping( + value["fixed_parameters"], path=f"{path}.fixed_parameters" + ) + parsed_fixed: list[tuple[str, float]] = [] + for name, raw_value in sorted(raw_fixed.items()): + if isinstance(raw_value, bool): + raise QualityProfileError( + f"{path}.fixed_parameters.{name} must be a number" + ) + try: + parsed_fixed.append((name, float(cast(Any, raw_value)))) + except (TypeError, ValueError) as exc: + raise QualityProfileError( + f"{path}.fixed_parameters.{name} must be a number" + ) from exc + fixed_parameters = tuple(parsed_fixed) + try: + return AutoregressiveSpecification( + specification_id=_string_field( + value, "specification_id", "", path=path + ), + family=family, + p=_int_field(value, "p", 0, minimum=0, path=path), + d=_int_field(value, "d", 0, minimum=0, path=path), + q=_int_field(value, "q", 0, minimum=0, path=path), + trend=_string_field(value, "trend", "n", path=path), + initialization_method=_string_field( + value, "initialization_method", "default", path=path + ), + estimation_method=_string_field( + value, "estimation_method", "statespace", path=path + ), + enforce_stationarity=_bool_field( + value, "enforce_stationarity", True, path=path + ), + enforce_invertibility=_bool_field( + value, "enforce_invertibility", True, path=path + ), + concentrate_scale=_bool_field( + value, "concentrate_scale", False, path=path + ), + fixed_parameters=fixed_parameters, + max_iterations=_int_field( + value, "max_iterations", 200, minimum=1, path=path + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _seasonal_exogenous_profile( + value: Mapping[str, JSONValue], + *, + path: str, +) -> SeasonalExogenousProfile: + base = SeasonalExogenousProfile() + _reject_unknown_keys( + value, + { + "enabled", + "specifications", + "projection_specification_ids", + "projection_horizon", + "regressor_profile", + "baseline_rolling_windows", + "compare_exponential_smoothing", + "compare_autoregressive", + "rounding_digits", + }, + path, + ) + specifications = base.specifications + if "specifications" in value: + raw = value["specifications"] + if not isinstance(raw, list) or not raw: + raise QualityProfileError( + f"{path}.specifications must be a non-empty list" + ) + specifications = tuple( + _seasonal_exogenous_specification( + _expect_mapping(item, path=f"{path}.specifications[{index}]"), + path=f"{path}.specifications[{index}]", + ) + for index, item in enumerate(raw) + ) + projection_ids = base.projection_specification_ids + if "projection_specification_ids" in value: + raw_ids = value["projection_specification_ids"] + if not isinstance(raw_ids, list) or not raw_ids: + raise QualityProfileError( + f"{path}.projection_specification_ids must be a non-empty list" + ) + if not all(isinstance(item, str) and item for item in raw_ids): + raise QualityProfileError( + f"{path}.projection_specification_ids must contain strings" + ) + projection_ids = tuple(cast(list[str], raw_ids)) + elif "specifications" in value: + by_family: dict[str, str] = {} + for specification in specifications: + by_family.setdefault( + specification.family, specification.specification_id + ) + projection_ids = tuple(by_family.values()) + try: + return SeasonalExogenousProfile( + enabled=_bool_field(value, "enabled", base.enabled, path=path), + specifications=specifications, + projection_specification_ids=projection_ids, + projection_horizon=_int_field( + value, + "projection_horizon", + base.projection_horizon, + minimum=1, + path=path, + ), + regressor_profile=_calendar_regressor_profile( + _mapping_field(value, "regressor_profile", path=path), + path=f"{path}.regressor_profile", + ), + baseline_rolling_windows=_fingerprint_int_sequence( + value, + "baseline_rolling_windows", + base.baseline_rolling_windows, + path=path, + ), + compare_exponential_smoothing=_bool_field( + value, + "compare_exponential_smoothing", + base.compare_exponential_smoothing, + path=path, + ), + compare_autoregressive=_bool_field( + value, + "compare_autoregressive", + base.compare_autoregressive, + path=path, + ), + rounding_digits=_int_field( + value, + "rounding_digits", + base.rounding_digits, + minimum=0, + maximum=16, + path=path, + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _state_space_profile( + value: Mapping[str, JSONValue], + *, + path: str, +) -> StateSpaceProfile: + base = StateSpaceProfile() + _reject_unknown_keys( + value, + { + "enabled", + "specifications", + "projection_specification_id", + "projection_horizon", + "max_state_dimension", + "max_component_count", + "max_prediction_only_gap", + "max_retained_states", + "baseline_rolling_windows", + "compare_exponential_smoothing", + "compare_autoregressive", + "compare_seasonal_exogenous", + "rounding_digits", + }, + path, + ) + specifications = base.specifications + if "specifications" in value: + raw = value["specifications"] + if not isinstance(raw, list) or not raw: + raise QualityProfileError( + f"{path}.specifications must be a non-empty list" + ) + specifications = tuple( + _state_space_specification( + _expect_mapping(item, path=f"{path}.specifications[{index}]"), + path=f"{path}.specifications[{index}]", + ) + for index, item in enumerate(raw) + ) + projection_id = _string_field( + value, + "projection_specification_id", + ( + specifications[0].specification_id + if "specifications" in value + else base.projection_specification_id + ), + path=path, + ) + try: + return StateSpaceProfile( + enabled=_bool_field(value, "enabled", base.enabled, path=path), + specifications=specifications, + projection_specification_id=projection_id, + projection_horizon=_int_field( + value, + "projection_horizon", + base.projection_horizon, + minimum=1, + path=path, + ), + max_state_dimension=_int_field( + value, + "max_state_dimension", + base.max_state_dimension, + minimum=1, + maximum=256, + path=path, + ), + max_component_count=_int_field( + value, + "max_component_count", + base.max_component_count, + minimum=1, + maximum=16, + path=path, + ), + max_prediction_only_gap=_int_field( + value, + "max_prediction_only_gap", + base.max_prediction_only_gap, + minimum=0, + path=path, + ), + max_retained_states=_int_field( + value, + "max_retained_states", + base.max_retained_states, + minimum=1, + maximum=64, + path=path, + ), + baseline_rolling_windows=_fingerprint_int_sequence( + value, + "baseline_rolling_windows", + base.baseline_rolling_windows, + path=path, + ), + compare_exponential_smoothing=_bool_field( + value, + "compare_exponential_smoothing", + base.compare_exponential_smoothing, + path=path, + ), + compare_autoregressive=_bool_field( + value, + "compare_autoregressive", + base.compare_autoregressive, + path=path, + ), + compare_seasonal_exogenous=_bool_field( + value, + "compare_seasonal_exogenous", + base.compare_seasonal_exogenous, + path=path, + ), + rounding_digits=_int_field( + value, + "rounding_digits", + base.rounding_digits, + minimum=0, + maximum=16, + path=path, + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _state_space_specification( + value: Mapping[str, JSONValue], + *, + path: str, +) -> StateSpaceSpecification: + _reject_unknown_keys( + value, + { + "specification_id", + "family", + "irregular", + "stochastic_level", + "stochastic_trend", + "seasonal_period", + "seasonal_cycle_ms", + "stochastic_seasonal", + "cycle", + "stochastic_cycle", + "damped_cycle", + "autoregressive_order", + "initialization_method", + "approximate_diffuse_variance", + "optimizer", + "fixed_parameters", + "max_iterations", + }, + path, + ) + if "specification_id" not in value or "family" not in value: + raise QualityProfileError( + f"{path} requires specification_id and family" + ) + fixed_parameters: tuple[tuple[str, float], ...] = () + if "fixed_parameters" in value: + raw_fixed = _expect_mapping( + value["fixed_parameters"], path=f"{path}.fixed_parameters" + ) + parsed: list[tuple[str, float]] = [] + for name, raw_value in sorted(raw_fixed.items()): + if isinstance(raw_value, bool): + raise QualityProfileError( + f"{path}.fixed_parameters.{name} must be a number" + ) + try: + parsed.append((name, float(cast(Any, raw_value)))) + except (TypeError, ValueError) as exc: + raise QualityProfileError( + f"{path}.fixed_parameters.{name} must be a number" + ) from exc + fixed_parameters = tuple(parsed) + family = _string_field(value, "family", "", path=path) + default_trend = family == "local_linear_trend" + try: + return StateSpaceSpecification( + specification_id=_string_field( + value, "specification_id", "", path=path + ), + family=family, + irregular=_bool_field(value, "irregular", True, path=path), + stochastic_level=_bool_field( + value, "stochastic_level", True, path=path + ), + stochastic_trend=_bool_field( + value, "stochastic_trend", default_trend, path=path + ), + seasonal_period=_int_field( + value, "seasonal_period", 0, minimum=0, path=path + ), + seasonal_cycle_ms=_int_field( + value, "seasonal_cycle_ms", 0, minimum=0, path=path + ), + stochastic_seasonal=_bool_field( + value, "stochastic_seasonal", True, path=path + ), + cycle=_bool_field(value, "cycle", False, path=path), + stochastic_cycle=_bool_field( + value, "stochastic_cycle", False, path=path + ), + damped_cycle=_bool_field(value, "damped_cycle", False, path=path), + autoregressive_order=_int_field( + value, "autoregressive_order", 0, minimum=0, path=path + ), + initialization_method=_string_field( + value, "initialization_method", "default", path=path + ), + approximate_diffuse_variance=_float_field( + value, + "approximate_diffuse_variance", + 1_000_000.0, + minimum=0.0, + path=path, + ), + optimizer=_string_field(value, "optimizer", "lbfgs", path=path), + fixed_parameters=fixed_parameters, + max_iterations=_int_field( + value, "max_iterations", 200, minimum=1, path=path + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _volatility_profile( + value: Mapping[str, JSONValue], *, path: str +) -> VolatilityProfile: + base = VolatilityProfile() + _reject_unknown_keys( + value, + { + "enabled", + "specifications", + "projection_specification_ids", + "projection_horizon", + "realized_variance_proxy", + "annualization_periods", + "baseline_rolling_windows", + "ewma_decay", + "maximum_persistence", + "maximum_covariance_condition_number", + "boundary_tolerance", + "compare_exponential_smoothing", + "compare_autoregressive", + "compare_seasonal_exogenous", + "compare_state_space", + "rounding_digits", + }, + path, + ) + specifications = base.specifications + if "specifications" in value: + raw = value["specifications"] + if not isinstance(raw, list) or not raw: + raise QualityProfileError( + f"{path}.specifications must be a non-empty list" + ) + specifications = tuple( + _volatility_specification( + _expect_mapping(item, path=f"{path}.specifications[{index}]"), + path=f"{path}.specifications[{index}]", + ) + for index, item in enumerate(raw) + ) + projection_ids = base.projection_specification_ids + if "projection_specification_ids" in value: + raw_ids = value["projection_specification_ids"] + if ( + not isinstance(raw_ids, list) + or not raw_ids + or not all(isinstance(item, str) and item for item in raw_ids) + ): + raise QualityProfileError( + f"{path}.projection_specification_ids must contain strings" + ) + projection_ids = tuple(cast(list[str], raw_ids)) + elif "specifications" in value: + by_family: dict[str, str] = {} + for specification in specifications: + by_family.setdefault( + specification.family, specification.specification_id + ) + projection_ids = tuple(by_family.values()) + try: + return VolatilityProfile( + enabled=_bool_field(value, "enabled", base.enabled, path=path), + specifications=specifications, + projection_specification_ids=projection_ids, + projection_horizon=_int_field( + value, + "projection_horizon", + base.projection_horizon, + minimum=1, + path=path, + ), + realized_variance_proxy=_string_field( + value, + "realized_variance_proxy", + base.realized_variance_proxy, + path=path, + ), + annualization_periods=_int_field( + value, + "annualization_periods", + base.annualization_periods, + minimum=0, + path=path, + ), + baseline_rolling_windows=_fingerprint_int_sequence( + value, + "baseline_rolling_windows", + base.baseline_rolling_windows, + path=path, + ), + ewma_decay=_float_field( + value, "ewma_decay", base.ewma_decay, path=path + ), + maximum_persistence=_float_field( + value, + "maximum_persistence", + base.maximum_persistence, + path=path, + ), + maximum_covariance_condition_number=_float_field( + value, + "maximum_covariance_condition_number", + base.maximum_covariance_condition_number, + minimum=1.0, + path=path, + ), + boundary_tolerance=_float_field( + value, + "boundary_tolerance", + base.boundary_tolerance, + path=path, + ), + compare_exponential_smoothing=_bool_field( + value, + "compare_exponential_smoothing", + base.compare_exponential_smoothing, + path=path, + ), + compare_autoregressive=_bool_field( + value, + "compare_autoregressive", + base.compare_autoregressive, + path=path, + ), + compare_seasonal_exogenous=_bool_field( + value, + "compare_seasonal_exogenous", + base.compare_seasonal_exogenous, + path=path, + ), + compare_state_space=_bool_field( + value, + "compare_state_space", + base.compare_state_space, + path=path, + ), + rounding_digits=_int_field( + value, + "rounding_digits", + base.rounding_digits, + minimum=0, + maximum=16, + path=path, + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _classical_model_comparison_profile( + value: Mapping[str, JSONValue], *, path: str +) -> ClassicalModelComparisonProfile: + base = ClassicalModelComparisonProfile() + _reject_unknown_keys( + value, + { + "enabled", + "mean_reference_baseline", + "variance_reference_baseline", + "near_zero_tolerance", + "minimum_stability_folds", + "drift_tolerance", + "max_models", + "max_horizons", + "max_comparisons", + "max_reason_codes", + "max_samples", + "rounding_digits", + }, + path, + ) + try: + return ClassicalModelComparisonProfile( + enabled=_bool_field(value, "enabled", base.enabled, path=path), + mean_reference_baseline=_string_field( + value, + "mean_reference_baseline", + base.mean_reference_baseline, + path=path, + ), + variance_reference_baseline=_string_field( + value, + "variance_reference_baseline", + base.variance_reference_baseline, + path=path, + ), + near_zero_tolerance=_float_field( + value, + "near_zero_tolerance", + base.near_zero_tolerance, + minimum=0.0, + path=path, + ), + minimum_stability_folds=_int_field( + value, + "minimum_stability_folds", + base.minimum_stability_folds, + minimum=2, + path=path, + ), + drift_tolerance=_float_field( + value, + "drift_tolerance", + base.drift_tolerance, + minimum=0.0, + path=path, + ), + max_models=_int_field( + value, "max_models", base.max_models, minimum=1, path=path + ), + max_horizons=_int_field( + value, + "max_horizons", + base.max_horizons, + minimum=1, + path=path, + ), + max_comparisons=_int_field( + value, + "max_comparisons", + base.max_comparisons, + minimum=1, + path=path, + ), + max_reason_codes=_int_field( + value, + "max_reason_codes", + base.max_reason_codes, + minimum=1, + path=path, + ), + max_samples=_int_field( + value, + "max_samples", + base.max_samples, + minimum=1, + path=path, + ), + rounding_digits=_int_field( + value, + "rounding_digits", + base.rounding_digits, + minimum=0, + maximum=16, + path=path, + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _volatility_specification( + value: Mapping[str, JSONValue], *, path: str +) -> VolatilitySpecification: + _reject_unknown_keys( + value, + { + "specification_id", + "family", + "input_definition", + "mean_model", + "mean_model_reference_id", + "distribution", + "innovation_order", + "variance_order", + "scale_factor", + "variance_initialization", + "initial_variance", + "covariance_type", + "optimizer_tolerance", + "parameter_bounds", + "max_iterations", + }, + path, + ) + if "specification_id" not in value or "family" not in value: + raise QualityProfileError( + f"{path} requires specification_id and family" + ) + family = _string_field(value, "family", "", path=path) + try: + return VolatilitySpecification( + specification_id=_string_field( + value, "specification_id", "", path=path + ), + family=family, + input_definition=_string_field( + value, "input_definition", "raw_return", path=path + ), + mean_model=_string_field(value, "mean_model", "zero", path=path), + mean_model_reference_id=_optional_string_profile_field( + value, "mean_model_reference_id", "", path=path + ), + distribution=_string_field( + value, "distribution", "normal", path=path + ), + innovation_order=_int_field( + value, "innovation_order", 1, minimum=1, path=path + ), + variance_order=_int_field( + value, + "variance_order", + 0 if family == "arch" else 1, + minimum=0, + path=path, + ), + scale_factor=_float_field( + value, "scale_factor", 100.0, minimum=0.0, path=path + ), + variance_initialization=_string_field( + value, + "variance_initialization", + "backend_default", + path=path, + ), + initial_variance=_optional_float_profile_field( + value, "initial_variance", None, minimum=0.0, path=path + ), + covariance_type=_string_field( + value, "covariance_type", "robust", path=path + ), + optimizer_tolerance=_optional_float_profile_field( + value, + "optimizer_tolerance", + None, + minimum=0.0, + path=path, + ), + parameter_bounds=cast( + tuple[tuple[str, float | None, float | None], ...], + _parameter_bounds_profile_field( + value, "parameter_bounds", path=path + ), + ), + max_iterations=_int_field( + value, "max_iterations", 200, minimum=1, path=path + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _calendar_regressor_profile( + value: Mapping[str, JSONValue], + *, + path: str, +) -> CalendarRegressorProfile: + base = CalendarRegressorProfile() + _reject_unknown_keys( + value, + { + "allow_partial_calendar", + "require_complete_calendar_for", + "max_regressors", + }, + path, + ) + required = base.require_complete_calendar_for + if "require_complete_calendar_for" in value: + raw = value["require_complete_calendar_for"] + if not isinstance(raw, list) or not all( + isinstance(item, str) and item for item in raw + ): + raise QualityProfileError( + f"{path}.require_complete_calendar_for must contain strings" + ) + required = tuple(cast(list[str], raw)) + try: + return CalendarRegressorProfile( + allow_partial_calendar=_bool_field( + value, + "allow_partial_calendar", + base.allow_partial_calendar, + path=path, + ), + require_complete_calendar_for=required, + max_regressors=_int_field( + value, + "max_regressors", + base.max_regressors, + minimum=1, + maximum=64, + path=path, + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _seasonal_exogenous_specification( + value: Mapping[str, JSONValue], + *, + path: str, +) -> SeasonalExogenousSpecification: + _reject_unknown_keys( + value, + { + "specification_id", + "family", + "p", + "d", + "q", + "seasonal_p", + "seasonal_d", + "seasonal_q", + "seasonal_period", + "seasonal_cycle_ms", + "trend", + "initialization_method", + "optimizer", + "enforce_stationarity", + "enforce_invertibility", + "concentrate_scale", + "use_exact_diffuse", + "regressor_names", + "fixed_parameters", + "max_iterations", + }, + path, + ) + required_keys = {"specification_id", "family", "p"} + if not required_keys.issubset(value): + raise QualityProfileError( + f"{path} requires specification_id, family, and p" + ) + regressors: tuple[str, ...] = () + if "regressor_names" in value: + raw = value["regressor_names"] + if not isinstance(raw, list) or not all( + isinstance(item, str) and item for item in raw + ): + raise QualityProfileError( + f"{path}.regressor_names must contain strings" + ) + regressors = tuple(cast(list[str], raw)) + fixed_parameters: tuple[tuple[str, float], ...] = () + if "fixed_parameters" in value: + raw_fixed = _expect_mapping( + value["fixed_parameters"], path=f"{path}.fixed_parameters" + ) + parsed: list[tuple[str, float]] = [] + for name, raw_value in sorted(raw_fixed.items()): + if isinstance(raw_value, bool): + raise QualityProfileError( + f"{path}.fixed_parameters.{name} must be a number" + ) + try: + parsed.append((name, float(cast(Any, raw_value)))) + except (TypeError, ValueError) as exc: + raise QualityProfileError( + f"{path}.fixed_parameters.{name} must be a number" + ) from exc + fixed_parameters = tuple(parsed) + try: + return SeasonalExogenousSpecification( + specification_id=_string_field( + value, "specification_id", "", path=path + ), + family=_string_field(value, "family", "", path=path), + p=_int_field(value, "p", 0, minimum=0, path=path), + d=_int_field(value, "d", 0, minimum=0, path=path), + q=_int_field(value, "q", 0, minimum=0, path=path), + seasonal_p=_int_field(value, "seasonal_p", 0, minimum=0, path=path), + seasonal_d=_int_field(value, "seasonal_d", 0, minimum=0, path=path), + seasonal_q=_int_field(value, "seasonal_q", 0, minimum=0, path=path), + seasonal_period=_int_field( + value, "seasonal_period", 0, minimum=0, path=path + ), + seasonal_cycle_ms=_int_field( + value, "seasonal_cycle_ms", 0, minimum=0, path=path + ), + trend=_string_field(value, "trend", "n", path=path), + initialization_method=_string_field( + value, "initialization_method", "default", path=path + ), + optimizer=_string_field(value, "optimizer", "lbfgs", path=path), + enforce_stationarity=_bool_field( + value, "enforce_stationarity", True, path=path + ), + enforce_invertibility=_bool_field( + value, "enforce_invertibility", True, path=path + ), + concentrate_scale=_bool_field( + value, "concentrate_scale", False, path=path + ), + use_exact_diffuse=_bool_field( + value, "use_exact_diffuse", False, path=path + ), + regressor_names=regressors, + fixed_parameters=fixed_parameters, + max_iterations=_int_field( + value, "max_iterations", 200, minimum=1, path=path + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _exponential_smoothing_specification( + value: Mapping[str, JSONValue], + *, + path: str, +) -> ExponentialSmoothingSpecification: + base = ExponentialSmoothingSpecification() + _reject_unknown_keys( + value, + { + "specification_id", + "family", + "level", + "error", + "trend", + "damped_trend", + "seasonal", + "seasonal_periods", + "initialization_method", + "initial_level", + "initial_trend", + "initial_seasonal", + "optimized", + "method", + "use_brute", + "remove_bias", + "smoothing_level", + "smoothing_trend", + "smoothing_seasonal", + "damping_trend", + "parameter_bounds", + "max_iterations", + }, + path, + ) + try: + return ExponentialSmoothingSpecification( + specification_id=_string_field( + value, + "specification_id", + base.specification_id, + path=path, + ), + family=_string_field(value, "family", base.family, path=path), + level=_bool_field(value, "level", base.level, path=path), + error=_string_field(value, "error", base.error, path=path), + trend=_string_field(value, "trend", base.trend, path=path), + damped_trend=_bool_field( + value, "damped_trend", base.damped_trend, path=path + ), + seasonal=_string_field(value, "seasonal", base.seasonal, path=path), + seasonal_periods=_int_field( + value, + "seasonal_periods", + base.seasonal_periods, + minimum=0, + path=path, + ), + initialization_method=_string_field( + value, + "initialization_method", + base.initialization_method, + path=path, + ), + initial_level=_optional_float_profile_field( + value, "initial_level", base.initial_level, path=path + ), + initial_trend=_optional_float_profile_field( + value, "initial_trend", base.initial_trend, path=path + ), + initial_seasonal=_float_tuple_profile_field( + value, + "initial_seasonal", + base.initial_seasonal, + path=path, + ), + optimized=_bool_field( + value, "optimized", base.optimized, path=path + ), + method=_optional_string_profile_field( + value, "method", base.method, path=path + ), + use_brute=_bool_field( + value, "use_brute", base.use_brute, path=path + ), + remove_bias=_bool_field( + value, "remove_bias", base.remove_bias, path=path + ), + smoothing_level=_optional_float_profile_field( + value, + "smoothing_level", + base.smoothing_level, + minimum=0.0, + maximum=1.0, + path=path, + ), + smoothing_trend=_optional_float_profile_field( + value, + "smoothing_trend", + base.smoothing_trend, + minimum=0.0, + maximum=1.0, + path=path, + ), + smoothing_seasonal=_optional_float_profile_field( + value, + "smoothing_seasonal", + base.smoothing_seasonal, + minimum=0.0, + maximum=1.0, + path=path, + ), + damping_trend=_optional_float_profile_field( + value, + "damping_trend", + base.damping_trend, + minimum=0.0, + maximum=1.0, + path=path, + ), + parameter_bounds=_parameter_bounds_profile_field( + value, "parameter_bounds", path=path + ), + max_iterations=_int_field( + value, + "max_iterations", + base.max_iterations, + minimum=1, + path=path, + ), + ) + except ValueError as exc: + raise QualityProfileError(f"{path}: {exc}") from exc + + +def _optional_float_profile_field( + mapping: Mapping[str, JSONValue], + key: str, + default: float | None, + *, + minimum: float | None = None, + maximum: float | None = None, + path: str, +) -> float | None: + if key not in mapping or mapping[key] is None: + return default + return _float_field( + mapping, + key, + 0.0, + minimum=minimum, + maximum=maximum, + path=path, + ) + + +def _float_tuple_profile_field( + mapping: Mapping[str, JSONValue], + key: str, + default: tuple[float, ...], + *, + path: str, +) -> tuple[float, ...]: + if key not in mapping: + return default + value = mapping[key] + if not isinstance(value, list): + raise QualityProfileError(f"{path}.{key} must be a list of numbers") + parsed: list[float] = [] + for index, item in enumerate(value): + if isinstance(item, bool): + raise QualityProfileError(f"{path}.{key}[{index}] must be a number") + try: + parsed.append(float(item)) # type: ignore[arg-type] + except (TypeError, ValueError) as exc: + raise QualityProfileError( + f"{path}.{key}[{index}] must be a number" + ) from exc + return tuple(parsed) + + +def _optional_string_profile_field( + mapping: Mapping[str, JSONValue], + key: str, + default: str, + *, + path: str, +) -> str: + if key not in mapping or mapping[key] in (None, ""): + return default + return _string_field(mapping, key, default, path=path) + + +def _parameter_bounds_profile_field( + mapping: Mapping[str, JSONValue], + key: str, + *, + path: str, +) -> tuple[tuple[str, float, float], ...]: + if key not in mapping: + return () + value = mapping[key] + if not isinstance(value, list): + raise QualityProfileError(f"{path}.{key} must be a list of objects") + parsed: list[tuple[str, float, float]] = [] + for index, item in enumerate(value): + bound_path = f"{path}.{key}[{index}]" + bound = _expect_mapping(item, path=bound_path) + _reject_unknown_keys(bound, {"parameter", "lower", "upper"}, bound_path) + parameter = _string_field(bound, "parameter", "", path=bound_path) + lower = _float_field(bound, "lower", 0.0, path=bound_path) + upper = _float_field(bound, "upper", 0.0, path=bound_path) + parsed.append((parameter, lower, upper)) + return tuple(parsed) + + +def _quality_reporting_profile( + value: Mapping[str, JSONValue], +) -> QualityReportingProfile: + _reject_unknown_keys( + value, + {"remediation_catalog_audit"}, + "quality_profile.reporting", + ) + remediation_catalog_audit = _mapping_field( + value, + "remediation_catalog_audit", + path="quality_profile.reporting", + ) + _reject_unknown_keys( + remediation_catalog_audit, + {"enabled"}, + "quality_profile.reporting.remediation_catalog_audit", + ) + return QualityReportingProfile( + remediation_catalog_audit=QualityRemediationCatalogAuditProfile( + enabled=_bool_field( + remediation_catalog_audit, + "enabled", + False, + path="quality_profile.reporting.remediation_catalog_audit", + ) + ) + ) + + +def _mapping_field( + mapping: Mapping[str, Any], + key: str, + *, + path: str, +) -> dict[str, JSONValue]: + value = mapping.get(key, {}) + if value in (None, ""): + return {} + return _expect_mapping(value, path=f"{path}.{key}") + + +def _expect_mapping(value: Any, *, path: str) -> dict[str, JSONValue]: + if not isinstance(value, Mapping): + msg = f"{path} must be an object" + raise QualityProfileError(msg) + return _json_mapping(value) def _json_mapping(value: Mapping[str, Any]) -> dict[str, JSONValue]: @@ -1274,6 +3377,7 @@ def _int_field( default: int, *, minimum: int | None = None, + maximum: int | None = None, path: str, ) -> int: if key not in mapping: @@ -1290,6 +3394,9 @@ def _int_field( if minimum is not None and parsed < minimum: msg = f"{path}.{key} must be >= {minimum}" raise QualityProfileError(msg) + if maximum is not None and parsed > maximum: + msg = f"{path}.{key} must be <= {maximum}" + raise QualityProfileError(msg) return parsed @@ -1431,6 +3538,22 @@ def _bool_field( return value +def _string_field( + mapping: Mapping[str, JSONValue], + key: str, + default: str, + *, + path: str, +) -> str: + if key not in mapping: + return default + value = mapping[key] + if not isinstance(value, str) or not value.strip(): + msg = f"{path}.{key} must be a non-empty string" + raise QualityProfileError(msg) + return value.strip() + + def _reject_unknown_keys( mapping: Mapping[str, Any], allowed: set[str] | frozenset[str], diff --git a/src/histdatacom/data_quality/remediation.py b/src/histdatacom/data_quality/remediation.py index 59693f4e..f921e8b7 100644 --- a/src/histdatacom/data_quality/remediation.py +++ b/src/histdatacom/data_quality/remediation.py @@ -3,11 +3,153 @@ from __future__ import annotations from collections.abc import Iterable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field, replace +from enum import Enum from histdatacom.data_quality.contracts import QualityFinding from histdatacom.runtime_contracts import JSONValue +CALENDAR_POLICY_REMEDIATION_CONTEXT_SCHEMA_VERSION = ( + "histdatacom.calendar-policy-remediation-context.v1" +) +_POLICY_CONTEXT_TEXT_LIMIT = 128 + + +class RemediationActionability(str, Enum): + """Stable remediation actionability and boundary classifications.""" + + REMEDIABLE_DEFECT = "remediable_defect" + POLICY_OR_PROFILE_DECISION = "policy_or_profile_decision" + UNSUPPORTED_FORMAT_OR_CAPABILITY = "unsupported_format_or_capability" + EXPECTED_ARTIFACT_OR_CONTEXT = "expected_artifact_or_context" + NEEDS_RULE_ATTRIBUTION = "needs_rule_attribution" + NEEDS_DIAGNOSTIC_CONTEXT = "needs_diagnostic_context" + UNSAFE_TO_AUTOMATE = "unsafe_to_automate" + INFORMATIONAL_ONLY = "informational_only" + + +@dataclass(frozen=True, slots=True) +class RemediationActionabilityDecision: + """One deterministic actionability decision and its stable reason.""" + + actionability: RemediationActionability + reason: str + + def to_payload(self) -> dict[str, JSONValue]: + """Return the public JSON-compatible decision payload.""" + return { + "actionability": self.actionability.value, + "actionability_reason": self.reason, + } + + +_POLICY_OR_PROFILE_FINDING_MARKERS = ( + "_POLICY_", + "_POLICY_MISSING", + "_PROFILE_", + "_PROFILE_MISSING", + "EXPECTED_SESSION_CLOSURE", + "WEEKEND_ACTIVITY", +) +_EXPECTED_ARTIFACT_OR_CONTEXT_FINDING_MARKERS = ( + "EXPECTED_ARTIFACT_OR_CONTEXT", + "EXPECTED_CONTEXT", + "PROVENANCE_MANIFEST_UNAVAILABLE", +) +_NEEDS_DIAGNOSTIC_CONTEXT_FINDING_MARKERS = ( + "DIAGNOSTIC_CONTEXT_MISSING", + "DIAGNOSTICS_UNAVAILABLE", + "NEEDS_DIAGNOSTIC_CONTEXT", +) +_UNSAFE_TO_AUTOMATE_FINDING_MARKERS = ( + "UNSAFE_TO_AUTOMATE", + "DESTRUCTIVE_REPAIR_REQUIRED", +) + + +def classify_remediation_actionability( + *, + rule_id: str, + finding_code: str, + severity: str, + mapped: bool, + attribution_status: str = "exact", +) -> RemediationActionabilityDecision: + """Classify remediation actionability without weakening defect coverage.""" + normalized_rule = rule_id.strip().lower() + normalized_code = finding_code.strip().upper() + normalized_severity = severity.strip().lower() + + if mapped: + return RemediationActionabilityDecision( + RemediationActionability.REMEDIABLE_DEFECT, + "mapped_remediation_hint", + ) + if normalized_severity == "info": + return RemediationActionabilityDecision( + RemediationActionability.INFORMATIONAL_ONLY, + "informational_severity", + ) + if ( + attribution_status == "unresolved" + or normalized_rule + in { + "", + "unknown", + } + or normalized_rule.endswith(".unresolved") + ): + return RemediationActionabilityDecision( + RemediationActionability.NEEDS_RULE_ATTRIBUTION, + "unresolved_rule_attribution", + ) + if normalized_rule == "inventory.format_support": + return RemediationActionabilityDecision( + RemediationActionability.UNSUPPORTED_FORMAT_OR_CAPABILITY, + "unsupported_format_rule", + ) + if "UNSUPPORTED" in normalized_code or "NOT_SUPPORTED" in normalized_code: + return RemediationActionabilityDecision( + RemediationActionability.UNSUPPORTED_FORMAT_OR_CAPABILITY, + "unsupported_capability_finding", + ) + if any( + marker in normalized_code + for marker in _POLICY_OR_PROFILE_FINDING_MARKERS + ): + return RemediationActionabilityDecision( + RemediationActionability.POLICY_OR_PROFILE_DECISION, + "policy_or_profile_context_required", + ) + if any( + marker in normalized_code + for marker in _EXPECTED_ARTIFACT_OR_CONTEXT_FINDING_MARKERS + ): + return RemediationActionabilityDecision( + RemediationActionability.EXPECTED_ARTIFACT_OR_CONTEXT, + "expected_artifact_or_context", + ) + if any( + marker in normalized_code + for marker in _NEEDS_DIAGNOSTIC_CONTEXT_FINDING_MARKERS + ): + return RemediationActionabilityDecision( + RemediationActionability.NEEDS_DIAGNOSTIC_CONTEXT, + "missing_diagnostic_context", + ) + if any( + marker in normalized_code + for marker in _UNSAFE_TO_AUTOMATE_FINDING_MARKERS + ): + return RemediationActionabilityDecision( + RemediationActionability.UNSAFE_TO_AUTOMATE, + "unsafe_automatic_repair", + ) + return RemediationActionabilityDecision( + RemediationActionability.REMEDIABLE_DEFECT, + "unmapped_warning_or_error", + ) + @dataclass(frozen=True, slots=True) class QualityRemediationHint: @@ -19,6 +161,10 @@ class QualityRemediationHint: rule_id: str = "" flag: str = "" finding_code: str = "" + policy_context: Mapping[str, JSONValue] = field( + default_factory=dict, + hash=False, + ) def to_payload(self) -> dict[str, JSONValue]: payload: dict[str, JSONValue] = { @@ -32,6 +178,8 @@ def to_payload(self) -> dict[str, JSONValue]: payload["flag"] = self.flag if self.finding_code: payload["finding_code"] = self.finding_code + if self.policy_context: + payload["policy_context"] = dict(self.policy_context) return payload @@ -82,6 +230,13 @@ def to_payload(self) -> dict[str, JSONValue]: rule_id=FINGERPRINT_SERIES_RULE_ID, flag="weekend_activity", ), + "expected_session_closures": QualityRemediationHint( + code="inspect_unexpected_session_closure", + message="inspect session closure marked unexpected by the active profile", + action_kind="inspect", + rule_id=FINGERPRINT_SERIES_RULE_ID, + flag="expected_session_closures", + ), } REMEDIATION_HINTS_BY_FINDING: Mapping[ @@ -197,6 +352,8 @@ def to_payload(self) -> dict[str, JSONValue]: def remediation_hints_for_flags( flags: Iterable[str], + *, + calendar_policy: Mapping[str, JSONValue] | None = None, ) -> tuple[QualityRemediationHint, ...]: hints: list[QualityRemediationHint] = [] seen: set[str] = set() @@ -206,14 +363,125 @@ def remediation_hints_for_flags( seen.add(flag) hint = TOPOLOGY_REMEDIATION_HINTS_BY_FLAG.get(flag) if hint is not None: - hints.append(hint) + policy_hint = _policy_aware_topology_hint( + hint, + calendar_policy=calendar_policy, + ) + if policy_hint is not None: + hints.append(policy_hint) return tuple(hints) def remediation_hint_payloads_for_flags( flags: Iterable[str], + *, + calendar_policy: Mapping[str, JSONValue] | None = None, ) -> list[JSONValue]: - return [hint.to_payload() for hint in remediation_hints_for_flags(flags)] + return [ + hint.to_payload() + for hint in remediation_hints_for_flags( + flags, + calendar_policy=calendar_policy, + ) + ] + + +def _policy_aware_topology_hint( + hint: QualityRemediationHint, + *, + calendar_policy: Mapping[str, JSONValue] | None, +) -> QualityRemediationHint | None: + if hint.flag not in {"weekend_activity", "expected_session_closures"}: + return hint + context = _calendar_policy_remediation_context( + calendar_policy, + flag=hint.flag, + ) + if not context: + return hint if hint.flag == "weekend_activity" else None + if hint.flag == "expected_session_closures": + if context.get("expected_session_closure_policy") != "unexpected": + return None + return replace(hint, policy_context=context) + + weekend_policy = str(context.get("weekend_activity_policy") or "") + if weekend_policy == "strict": + return replace( + hint, + message="inspect weekend activity against strict no-weekend policy", + action_kind="inspect", + policy_context=context, + ) + if weekend_policy == "allowed": + return replace( + hint, + message="weekend activity is allowed by the active profile", + action_kind="context", + policy_context=context, + ) + return replace( + hint, + message="verify weekend-session profile assumptions", + action_kind="verify", + policy_context=context, + ) + + +def _calendar_policy_remediation_context( + calendar_policy: Mapping[str, JSONValue] | None, + *, + flag: str, +) -> dict[str, JSONValue]: + if not calendar_policy: + return {} + profile_value = calendar_policy.get("calendar_profile") + profile = profile_value if isinstance(profile_value, Mapping) else {} + weekend_policy = _bounded_policy_text( + calendar_policy.get("weekend_activity_policy") + or profile.get("weekend_activity_policy") + ) + closure_policy = _bounded_policy_text( + calendar_policy.get("expected_session_closure_policy") + or profile.get("expected_session_closure_policy") + ) + if flag == "weekend_activity" and not weekend_policy: + return {} + if flag == "expected_session_closures" and not closure_policy: + return {} + context: dict[str, JSONValue] = { + "schema_version": CALENDAR_POLICY_REMEDIATION_CONTEXT_SCHEMA_VERSION, + "flag": flag, + "actionable": not ( + flag == "weekend_activity" and weekend_policy == "allowed" + ), + } + optional_text = { + "profile_name": profile.get("name"), + "profile_source": profile.get("source") + or calendar_policy.get("holiday_calendar_source"), + "profile_version": profile.get("version"), + "source_timezone": calendar_policy.get("source_timezone"), + "canonical_timezone": calendar_policy.get("canonical_timezone"), + } + for key, value in optional_text.items(): + text = _bounded_policy_text(value) + if text: + context[key] = text + complete = calendar_policy.get("holiday_calendar_complete") + if isinstance(complete, bool): + context["calendar_complete"] = complete + static_advisory = calendar_policy.get("holiday_calendar_static_advisory") + if isinstance(static_advisory, bool): + context["calendar_static_advisory"] = static_advisory + if weekend_policy: + context["weekend_activity_policy"] = weekend_policy + if closure_policy: + context["expected_session_closure_policy"] = closure_policy + return context + + +def _bounded_policy_text(value: object) -> str: + return str(value or "").strip()[:_POLICY_CONTEXT_TEXT_LIMIT] def remediation_hints_for_finding_code( diff --git a/src/histdatacom/data_quality/remediation_audit.py b/src/histdatacom/data_quality/remediation_audit.py index 7ead97b9..fe5c5da3 100644 --- a/src/histdatacom/data_quality/remediation_audit.py +++ b/src/histdatacom/data_quality/remediation_audit.py @@ -19,6 +19,9 @@ bounded_report_limit, ) from histdatacom.data_quality.remediation import ( + RemediationActionability, + RemediationActionabilityDecision, + classify_remediation_actionability, remediation_hints_for_finding_code, ) from histdatacom.data_quality.reporting import ( @@ -32,6 +35,9 @@ QUALITY_REMEDIATION_CATALOG_AUDIT_SCHEMA_VERSION = ( "histdatacom.quality-remediation-catalog-audit.v1" ) +QUALITY_REMEDIATION_PLAN_SCHEMA_VERSION = ( + "histdatacom.quality-remediation-plan.v1" +) DEFAULT_REMEDIATION_CATALOG_AUDIT_CODE_LIMIT = ( QUALITY_PAYLOAD_REMEDIATION_COVERAGE_GROUP_LIMIT ) @@ -43,6 +49,89 @@ _SEVERITY_RANK = {"info": 1, "warning": 2, "error": 3} _SEVERITY_SORT = {"error": 0, "warning": 1, "info": 2} +_ATTRIBUTION_STATUS_SORT = {"unresolved": 0, "inferred": 1, "exact": 2} +_ACTIONABILITY_SORT = { + RemediationActionability.REMEDIABLE_DEFECT.value: 0, + RemediationActionability.NEEDS_DIAGNOSTIC_CONTEXT.value: 1, + RemediationActionability.NEEDS_RULE_ATTRIBUTION.value: 2, + RemediationActionability.UNSAFE_TO_AUTOMATE.value: 3, + RemediationActionability.POLICY_OR_PROFILE_DECISION.value: 4, + RemediationActionability.UNSUPPORTED_FORMAT_OR_CAPABILITY.value: 5, + RemediationActionability.EXPECTED_ARTIFACT_OR_CONTEXT.value: 6, + RemediationActionability.INFORMATIONAL_ONLY.value: 7, +} +_ACTIONABILITY_FIXABILITY_SCORE = { + RemediationActionability.REMEDIABLE_DEFECT.value: 45, + RemediationActionability.NEEDS_DIAGNOSTIC_CONTEXT.value: 15, + RemediationActionability.NEEDS_RULE_ATTRIBUTION.value: 10, + RemediationActionability.UNSAFE_TO_AUTOMATE.value: 8, + RemediationActionability.POLICY_OR_PROFILE_DECISION.value: 8, + RemediationActionability.UNSUPPORTED_FORMAT_OR_CAPABILITY.value: 5, + RemediationActionability.EXPECTED_ARTIFACT_OR_CONTEXT.value: 5, + RemediationActionability.INFORMATIONAL_ONLY.value: 3, +} +_BLOCKED_PLAN_ACTIONABILITIES = frozenset( + { + RemediationActionability.NEEDS_DIAGNOSTIC_CONTEXT.value, + RemediationActionability.NEEDS_RULE_ATTRIBUTION.value, + } +) +_BOUNDARY_PLAN_ACTIONABILITIES = frozenset( + { + RemediationActionability.POLICY_OR_PROFILE_DECISION.value, + RemediationActionability.UNSUPPORTED_FORMAT_OR_CAPABILITY.value, + RemediationActionability.EXPECTED_ARTIFACT_OR_CONTEXT.value, + RemediationActionability.UNSAFE_TO_AUTOMATE.value, + RemediationActionability.INFORMATIONAL_ONLY.value, + } +) +_REBUILD_ACTION_MARKERS = ( + "CACHE_SCHEMA", + "CORRUPT", + "CRC", + "MISSING", + "UNAVAILABLE", +) +_REPAIR_ACTION_MARKERS = ( + "DUPLICATE", + "INVALID", + "MISMATCH", + "NEGATIVE", + "NON_MONOTONIC", + "PRECISION", + "UNREADABLE", +) +_VERIFY_ACTION_MARKERS = ( + "EXPECTED", + "POLICY", + "UNSUPPORTED", +) +_REMEDIATION_PLAN_DISPLAY_LIMIT = 5 +_FINDING_CODE_RULE_PREFIXES = ( + ("ASCII_TICK_SPREAD_REGIME_", "ticks.ascii.spread_regimes"), + ("ASCII_TICK_MICROSTRUCTURE_", "ticks.ascii.microstructure"), + ("ASCII_TICK_ONE_SIDED_", "ticks.ascii.microstructure"), + ("ASCII_TICK_STALE_", "ticks.ascii.microstructure"), + ("ASCII_TICK_BURST_", "ticks.ascii.microstructure"), + ("ASCII_TICK_BID_ASK_", "ticks.ascii.spread"), + ("ASCII_TICK_NEGATIVE_SPREAD", "ticks.ascii.spread"), + ("ASCII_TICK_ZERO_SPREAD", "ticks.ascii.spread"), + ("ASCII_TICK_SPREAD_", "ticks.ascii.spread"), + ("DOMAIN_CALENDAR_", "domain.calendar_sessions"), + ("DOMAIN_CROSS_INSTRUMENT_", "domain.cross_instrument_consistency"), + ("PROVENANCE_", "provenance.manifest.lineage"), + ("ASCII_TIMESTAMP_CONTINUITY_", "time.ascii.continuity"), + ("ASCII_TIMESTAMP_EXPECTED_SESSION_CLOSURE_GAP", "time.ascii.gaps"), + ("ASCII_TIMESTAMP_SUSPICIOUS_GAP", "time.ascii.gaps"), + ("ASCII_TIMESTAMP_WEEKEND_ACTIVITY", "time.ascii.gaps"), + ("ASCII_TIMESTAMP_GAP_", "time.ascii.gaps"), + ("ASCII_TIMESTAMP_SOURCE_", "time.ascii.est_no_dst"), + ("ASCII_TIMESTAMP_EST_NO_DST_", "time.ascii.est_no_dst"), + ("ASCII_TIMESTAMP_UTC_", "time.ascii.est_no_dst"), + ("HISTDATA_FORMAT_", "inventory.format_support"), + ("COVERAGE_", "inventory.coverage.manifest"), + ("FINGERPRINT_", "fingerprint.series"), +) _T = TypeVar("_T") @@ -56,6 +145,17 @@ class KnownQualityFindingCode: source: str = "" severity_source: str = "" source_family: str = "" + source_helper: str = "" + finding_code_prefix: str = "" + attribution_status: str = "exact" + attribution_reason: str = "provided_rule_id" + + +@dataclass(frozen=True, slots=True) +class _RuleAttribution: + rule_id: str + status: str + reason: str @dataclass(slots=True) @@ -67,6 +167,10 @@ class _CodeAggregate: severity_counts: Counter[str] = field(default_factory=Counter) source_counts: Counter[str] = field(default_factory=Counter) source_family_counts: Counter[str] = field(default_factory=Counter) + source_helper_counts: Counter[str] = field(default_factory=Counter) + finding_code_prefix_counts: Counter[str] = field(default_factory=Counter) + attribution_status_counts: Counter[str] = field(default_factory=Counter) + attribution_reason_counts: Counter[str] = field(default_factory=Counter) @property def max_severity(self) -> str: @@ -192,6 +296,10 @@ def audit_remediation_catalog( report_gap_evidence=report_gap_evidence, source_limit=source_limit_state.effective_limit, ) + remediation_plan = _remediation_plan_payload( + ranked_gaps, + limit=code_limit_state, + ) included_ranked_gaps = list(code_limit_state.slice(ranked_gaps)) included_unmapped_known = code_limit_state.slice(unmapped_known) report_summary = _report_coverage_summary(report_payloads) @@ -220,12 +328,17 @@ def audit_remediation_catalog( for aggregate in included_unmapped_known ], "ranked_gaps": cast(JSONValue, included_ranked_gaps), + "remediation_plan": remediation_plan, "report_coverage": cast(JSONValue, report_payloads), "payload_limits": { "ranked_gaps": _payload_limit_metadata( len(ranked_gaps), code_limit_state, ), + "remediation_plan": _payload_limit_metadata( + len(ranked_gaps), + code_limit_state, + ), "known_unmapped_codes": _payload_limit_metadata( len(unmapped_known), code_limit_state, @@ -255,6 +368,34 @@ def audit_remediation_catalog( ), code_limit_state, ), + "attribution_reason_counts": _payload_limit_metadata( + _counter_distinct_count( + reason + for aggregate in known_aggregates.values() + for reason in aggregate.attribution_reason_counts + ), + rule_limit_state, + ), + "unresolved_source_helper_counts": _payload_limit_metadata( + _counter_distinct_count( + helper + for aggregate in known_aggregates.values() + if aggregate.attribution_status_counts["unresolved"] + for helper in aggregate.source_helper_counts + ), + rule_limit_state, + ), + "unresolved_finding_code_prefix_counts": ( + _payload_limit_metadata( + _counter_distinct_count( + prefix + for aggregate in known_aggregates.values() + if aggregate.attribution_status_counts["unresolved"] + for prefix in aggregate.finding_code_prefix_counts + ), + rule_limit_state, + ) + ), "report_unmapped_groups": { **code_limit_state.limit_payload(), "target_axis_limit": target_axis_limit_state.effective_limit, @@ -326,7 +467,39 @@ def format_remediation_catalog_audit( "warning/error gaps: " f"{_int_value(summary, 'unmapped_warning_error_gap_count')}" ), + ( + "attribution occurrences: " + f"exact={_int_value(summary, 'exact_attribution_occurrence_count')} " + "inferred=" + f"{_int_value(summary, 'inferred_attribution_occurrence_count')} " + "unresolved=" + f"{_int_value(summary, 'unresolved_attribution_occurrence_count')}" + ), + ( + "actionability gaps: " + "actionable=" + f"{_int_value(summary, 'unmapped_actionable_warning_error_gap_count')} " + "blocked_attribution=" + f"{_int_value(summary, 'blocked_by_attribution_warning_error_code_count')} " + "blocked_diagnostics=" + f"{_int_value(summary, 'blocked_by_missing_diagnostics_warning_error_code_count')} " + "intentional_boundary=" + f"{_int_value(summary, 'intentionally_unremediable_warning_error_code_count')}" + ), ] + code_counts = _mapping_payload(payload.get("known_code_counts")) + unresolved_families = _format_named_counts( + code_counts.get("unresolved_source_family_counts"), + name_key="source_family", + ) + if unresolved_families: + lines.append(f"unresolved families: {unresolved_families}") + unresolved_helpers = _format_named_counts( + code_counts.get("unresolved_source_helper_counts"), + name_key="source_helper", + ) + if unresolved_helpers: + lines.append(f"unresolved helpers: {unresolved_helpers}") report_count = _int_value(summary, "report_count") if report_count: lines.append( @@ -336,6 +509,23 @@ def format_remediation_catalog_audit( "unmapped warning/error groups: " f"{_int_value(summary, 'report_unmapped_warning_error_group_count')}" ) + remediation_plan = _mapping_payload(payload.get("remediation_plan")) + plan_items = _list_payload(remediation_plan.get("items")) + lines.extend(("", "Remediation plan")) + if not plan_items: + lines.append("- none") + else: + lines.extend( + f"- {_format_remediation_plan_item(item)}" + for item in plan_items[:_REMEDIATION_PLAN_DISPLAY_LIMIT] + ) + hidden_plan_items = max( + 0, + _int_value(remediation_plan, "plan_item_count") + - min(len(plan_items), _REMEDIATION_PLAN_DISPLAY_LIMIT), + ) + if hidden_plan_items: + lines.append(f"- additional plan items: {hidden_plan_items}") ranked_groups = [ item for item in _list_payload(payload.get("ranked_gaps")) @@ -403,22 +593,29 @@ def _known_findings_from_source( if not code or not _looks_like_finding_code(code): continue severity, severity_source = _severity_from_call(node) - rule_id = _rule_id_from_call( + attribution = _rule_attribution_from_call( node, + code, + tree, constants, class_rule_ids=class_rule_ids, parents=parents, source_family=source_family, ) + source_helper = _nearest_function_name(node, parents) source = _relative_source(path, root=root, line_number=node.lineno) findings.append( KnownQualityFindingCode( - rule_id=rule_id, + rule_id=attribution.rule_id, finding_code=code, severity=severity, source=source, severity_source=severity_source, source_family=source_family, + source_helper=source_helper, + finding_code_prefix=_finding_code_prefix(code), + attribution_status=attribution.status, + attribution_reason=attribution.reason, ) ) return tuple(findings) @@ -445,6 +642,19 @@ def _known_code_aggregates( family = known.source_family or _source_family_from_source(known.source) if family: aggregate.source_family_counts[family] += 1 + if known.source_helper: + aggregate.source_helper_counts[known.source_helper] += 1 + prefix = known.finding_code_prefix or _finding_code_prefix( + known.finding_code + ) + if prefix: + aggregate.finding_code_prefix_counts[prefix] += 1 + aggregate.attribution_status_counts[ + known.attribution_status or "unresolved" + ] += 1 + aggregate.attribution_reason_counts[ + known.attribution_reason or "no_rule_context" + ] += 1 aggregate.mapped = aggregate.mapped or _known_code_is_mapped(known) return aggregates @@ -486,6 +696,10 @@ def _report_coverage_summary( mapped_finding_count = 0 unmapped_finding_count = 0 unmapped_warning_error_group_count = 0 + unmapped_actionable_warning_error_group_count = 0 + intentionally_unremediable_warning_error_finding_count = 0 + blocked_by_attribution_warning_error_finding_count = 0 + blocked_by_missing_diagnostics_warning_error_finding_count = 0 for payload in report_payloads: coverage = _mapping_payload(payload.get("remediation_coverage")) finding_count += _int_value(coverage, "finding_count") @@ -498,6 +712,24 @@ def _report_coverage_summary( coverage, "unmapped_warning_error_group_count", ) + unmapped_actionable_warning_error_group_count += _int_value( + coverage, + "unmapped_actionable_warning_error_group_count", + ) + intentionally_unremediable_warning_error_finding_count += _int_value( + coverage, + "intentionally_unremediable_warning_error_finding_count", + ) + blocked_by_attribution_warning_error_finding_count += _int_value( + coverage, + "blocked_by_attribution_warning_error_finding_count", + ) + blocked_by_missing_diagnostics_warning_error_finding_count += ( + _int_value( + coverage, + "blocked_by_missing_diagnostics_warning_error_finding_count", + ) + ) return { "report_count": report_count, "report_finding_count": finding_count, @@ -506,6 +738,18 @@ def _report_coverage_summary( "report_unmapped_warning_error_group_count": ( unmapped_warning_error_group_count ), + "report_unmapped_actionable_warning_error_group_count": ( + unmapped_actionable_warning_error_group_count + ), + "report_intentionally_unremediable_warning_error_finding_count": ( + intentionally_unremediable_warning_error_finding_count + ), + "report_blocked_by_attribution_warning_error_finding_count": ( + blocked_by_attribution_warning_error_finding_count + ), + "report_blocked_by_missing_diagnostics_warning_error_finding_count": ( + blocked_by_missing_diagnostics_warning_error_finding_count + ), } @@ -584,6 +828,39 @@ def _audit_summary( report_summary, "report_unmapped_warning_error_group_count", ) + attribution_status_counts: Counter[str] = Counter() + actionability_counts: Counter[str] = Counter() + unmapped_actionable_warning_error_code_count = 0 + intentionally_unremediable_warning_error_code_count = 0 + blocked_by_attribution_warning_error_code_count = 0 + blocked_by_missing_diagnostics_warning_error_code_count = 0 + for aggregate in aggregates.values(): + attribution_status_counts.update(aggregate.attribution_status_counts) + decision = _code_aggregate_actionability(aggregate) + actionability_counts[decision.actionability.value] += 1 + if aggregate.mapped or aggregate.max_severity not in { + "error", + "warning", + }: + continue + if decision.actionability is RemediationActionability.REMEDIABLE_DEFECT: + unmapped_actionable_warning_error_code_count += 1 + elif ( + decision.actionability + is RemediationActionability.NEEDS_RULE_ATTRIBUTION + ): + blocked_by_attribution_warning_error_code_count += 1 + elif ( + decision.actionability + is RemediationActionability.NEEDS_DIAGNOSTIC_CONTEXT + ): + blocked_by_missing_diagnostics_warning_error_code_count += 1 + else: + intentionally_unremediable_warning_error_code_count += 1 + report_actionable_gap_count = _int_value( + report_summary, + "report_unmapped_actionable_warning_error_group_count", + ) return { "known_code_count": known_code_count, "known_finding_occurrence_count": sum( @@ -599,6 +876,32 @@ def _audit_summary( "unmapped_warning_error_gap_count": ( unmapped_warning_error_code_count + report_gap_count ), + "exact_attribution_occurrence_count": attribution_status_counts[ + "exact" + ], + "inferred_attribution_occurrence_count": attribution_status_counts[ + "inferred" + ], + "unresolved_attribution_occurrence_count": attribution_status_counts[ + "unresolved" + ], + "actionability_counts": _counter_payload(actionability_counts), + "unmapped_actionable_warning_error_code_count": ( + unmapped_actionable_warning_error_code_count + ), + "unmapped_actionable_warning_error_gap_count": ( + unmapped_actionable_warning_error_code_count + + report_actionable_gap_count + ), + "intentionally_unremediable_warning_error_code_count": ( + intentionally_unremediable_warning_error_code_count + ), + "blocked_by_attribution_warning_error_code_count": ( + blocked_by_attribution_warning_error_code_count + ), + "blocked_by_missing_diagnostics_warning_error_code_count": ( + blocked_by_missing_diagnostics_warning_error_code_count + ), **dict(report_summary), } @@ -634,6 +937,291 @@ def _ranked_gap_payloads( return [{**gap, "rank": index} for index, gap in enumerate(ranked, start=1)] +def _remediation_plan_payload( + ranked_gaps: Sequence[JSONValue], + *, + limit: BoundedReportLimit, +) -> dict[str, JSONValue]: + """Turn ranked catalog gaps into bounded catalog-edit plan inputs.""" + plan_items = sorted( + ( + _remediation_plan_item(gap) + for gap in ranked_gaps + if isinstance(gap, Mapping) + ), + key=_remediation_plan_item_sort_key, + ) + ranked_items = [ + {**item, "rank": index} + for index, item in enumerate(plan_items, start=1) + ] + included_items = list(limit.slice(ranked_items)) + actionability_counts = Counter( + _optional_string(item, "actionability") or "unknown" + for item in plan_items + ) + fixability_counts = Counter( + _optional_string(_mapping_payload(item.get("fixability")), "level") + or "unknown" + for item in plan_items + ) + return { + "schema_version": QUALITY_REMEDIATION_PLAN_SCHEMA_VERSION, + "plan_item_count": len(plan_items), + "included_plan_item_count": len(included_items), + "omitted_plan_item_count": max( + 0, + len(plan_items) - len(included_items), + ), + "truncated": len(included_items) < len(plan_items), + "actionability_counts": _counter_payload(actionability_counts), + "fixability_counts": _counter_payload(fixability_counts), + "items": cast(JSONValue, included_items), + } + + +def _remediation_plan_item( + gap: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + selector = _suggested_remediation_selector(gap) + action = _suggested_remediation_action(gap) + fixability = _remediation_fixability( + gap, + selector=selector, + action=action, + ) + missing_fields = _remediation_plan_missing_fields( + gap, + selector=selector, + action=action, + ) + finding_code = _optional_string(gap, "finding_code") + action_kind = _optional_string(action, "action_kind") or "inspect" + prefix = _optional_string(selector, "finding_code_prefix") + return { + "catalog_gap_rank": _int_value(gap, "rank"), + "finding_code": finding_code, + "rule_id": _optional_string(gap, "rule_id") or "unknown", + "source_family": _optional_string(gap, "source_family"), + "mapped": bool(gap.get("mapped")), + "max_severity": _optional_string(gap, "max_severity") or "info", + "severity_counts": _mapping_payload(gap.get("severity_counts")), + "actionability": _optional_string(gap, "actionability"), + "actionability_reason": _optional_string( + gap, + "actionability_reason", + ), + "attribution_status": _optional_string(gap, "attribution_status"), + "attribution_reason": _optional_string(gap, "attribution_reason"), + "suggested_selector": selector, + "suggested_hint_code_prefix": ( + f"{action_kind}_{prefix.lower()}" if prefix else action_kind + ), + "draft_hint_code": ( + f"{action_kind}_{finding_code.lower()}" + if finding_code + else f"{action_kind}_finding" + ), + "suggested_action": action, + "fixability": fixability, + "missing_fields": cast(JSONValue, missing_fields), + "evidence": { + "known_source_occurrence_count": _int_value( + gap, + "known_source_occurrence_count", + ), + "report_occurrence_count": _int_value( + gap, + "report_occurrence_count", + ), + "report_group_count": _int_value(gap, "report_group_count"), + "sources": gap.get("sources", []), + "reports": gap.get("reports", []), + "omitted_source_count": _int_value( + gap, + "omitted_source_count", + ), + "omitted_report_source_count": _int_value( + gap, + "omitted_report_source_count", + ), + }, + "rank_reasons": gap.get("rank_reasons", []), + } + + +def _suggested_remediation_selector( + gap: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + finding_code = _optional_string(gap, "finding_code") + rule_id = _optional_string(gap, "rule_id") + attribution = _optional_string(gap, "attribution_status") + rule_is_specific = bool( + rule_id and rule_id != "unknown" and not rule_id.endswith(".unresolved") + ) + prefix_counts = _list_payload(gap.get("finding_code_prefix_counts")) + prefix = ( + _optional_string(prefix_counts[0], "finding_code_prefix") + if prefix_counts + else _finding_code_prefix(finding_code) + ) + if rule_is_specific and finding_code: + confidence = ( + "high" if attribution in {"exact", "runtime_report"} else "medium" + ) + basis = ( + "reported_rule_and_finding" + if attribution == "runtime_report" + else f"{attribution or 'unknown'}_rule_attribution" + ) + return { + "shape": "exact_rule_and_finding", + "rule_id": rule_id, + "finding_code": finding_code, + "finding_code_prefix": prefix, + "confidence": confidence, + "basis": basis, + } + return { + "shape": "finding_family", + "rule_id": rule_id or "unknown", + "finding_code": finding_code, + "finding_code_prefix": prefix, + "confidence": "low", + "basis": "rule_attribution_required", + } + + +def _suggested_remediation_action( + gap: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + finding_code = _optional_string(gap, "finding_code").upper() + actionability = _optional_string(gap, "actionability") + marker = _first_marker(finding_code, _REBUILD_ACTION_MARKERS) + action_kind = "rebuild" + if not marker: + marker = _first_marker(finding_code, _REPAIR_ACTION_MARKERS) + action_kind = "repair" + if not marker: + marker = _first_marker(finding_code, _VERIFY_ACTION_MARKERS) + action_kind = "verify" + if not marker: + action_kind = "inspect" + confidence = "high" if marker else "low" + basis = ( + f"finding_code_marker={marker.lower()}" + if marker + else "no_concrete_action_marker" + ) + if actionability in _BOUNDARY_PLAN_ACTIONABILITIES: + confidence = "low" + basis = f"actionability_boundary={actionability}" + if actionability in _BLOCKED_PLAN_ACTIONABILITIES: + confidence = "low" + basis = f"actionability_blocker={actionability}" + return { + "action_kind": action_kind, + "confidence": confidence, + "basis": basis, + "concrete": confidence == "high", + } + + +def _remediation_fixability( + gap: Mapping[str, JSONValue], + *, + selector: Mapping[str, JSONValue], + action: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + actionability = _optional_string(gap, "actionability") + attribution = _optional_string(gap, "attribution_status") + severity = _optional_string(gap, "max_severity") + selector_shape = _optional_string(selector, "shape") + concrete = bool(action.get("concrete")) + known_sources = _int_value(gap, "known_source_occurrence_count") + report_occurrences = _int_value(gap, "report_occurrence_count") + score = _ACTIONABILITY_FIXABILITY_SCORE.get(actionability, 0) + score += 20 if selector_shape == "exact_rule_and_finding" else 5 + if attribution in {"exact", "runtime_report"}: + score += 10 + elif attribution == "inferred": + score += 6 + score += {"error": 10, "warning": 7, "info": 2}.get(severity, 0) + score += min(5, known_sources) + if report_occurrences: + score += min(20, 10 + report_occurrences * 2) + if concrete: + score += 10 + score = min(100, score) + if actionability in _BLOCKED_PLAN_ACTIONABILITIES: + score = min(score, 24) + level = "blocked" + elif actionability in _BOUNDARY_PLAN_ACTIONABILITIES: + score = min(score, 49) + level = "low" + elif score >= 75: + level = "high" + elif score >= 50: + level = "medium" + elif score >= 25: + level = "low" + else: + level = "blocked" + confidence = "high" if score >= 75 else "medium" if score >= 50 else "low" + basis: list[JSONValue] = [ + f"actionability={actionability or 'unknown'}", + f"selector={selector_shape or 'unknown'}", + f"attribution={attribution or 'unknown'}", + f"severity={severity or 'unknown'}", + f"concrete_action={str(concrete).lower()}", + f"known_sources={known_sources}", + f"report_occurrences={report_occurrences}", + ] + return { + "score": score, + "level": level, + "confidence": confidence, + "basis": basis, + } + + +def _remediation_plan_missing_fields( + gap: Mapping[str, JSONValue], + *, + selector: Mapping[str, JSONValue], + action: Mapping[str, JSONValue], +) -> list[JSONValue]: + missing: list[JSONValue] = ["message"] + if _optional_string(selector, "shape") != "exact_rule_and_finding": + missing.append("exact_rule_id") + if _optional_string(action, "confidence") != "high": + missing.append("action_kind_confirmation") + actionability = _optional_string(gap, "actionability") + if actionability in _BLOCKED_PLAN_ACTIONABILITIES: + missing.append("blocking_evidence") + elif actionability in _BOUNDARY_PLAN_ACTIONABILITIES: + missing.append("boundary_decision") + return missing + + +def _remediation_plan_item_sort_key( + item: Mapping[str, JSONValue], +) -> tuple[int, int, int, str, str]: + fixability = _mapping_payload(item.get("fixability")) + level = _optional_string(fixability, "level") + return ( + 1 if level == "blocked" else 0, + -_int_value(fixability, "score"), + _int_value(item, "catalog_gap_rank"), + _optional_string(item, "finding_code"), + _optional_string(item, "rule_id"), + ) + + +def _first_marker(value: str, markers: Sequence[str]) -> str: + return next((marker for marker in markers if marker in value), "") + + def _ranked_gap_payload( aggregate: _CodeAggregate, *, @@ -658,6 +1246,7 @@ def _ranked_gap_payload( limit=source_limit, ) source_family = _primary_source_family(aggregate.source_family_counts) + actionability = _code_aggregate_actionability(aggregate) payload: dict[str, JSONValue] = { "finding_code": aggregate.finding_code, "rule_id": aggregate.rule_id, @@ -666,6 +1255,27 @@ def _ranked_gap_payload( "severity_counts": severity_counts, "source_family": source_family, "source_family_counts": source_family_counts, + "source_helper_counts": _named_counter_payloads( + aggregate.source_helper_counts, + key_name="source_helper", + limit=source_limit, + ), + "finding_code_prefix_counts": _named_counter_payloads( + aggregate.finding_code_prefix_counts, + key_name="finding_code_prefix", + limit=source_limit, + ), + "attribution_status": _primary_attribution_status(aggregate), + "attribution_reason": _primary_counter_key( + aggregate.attribution_reason_counts + ), + "attribution_status_counts": _counter_payload( + aggregate.attribution_status_counts + ), + "attribution_reason_counts": _counter_payload( + aggregate.attribution_reason_counts + ), + **actionability.to_payload(), "known_source_occurrence_count": aggregate.occurrence_count, "source_count": len(aggregate.source_counts), "included_source_count": len(sources), @@ -682,6 +1292,7 @@ def _ranked_gap_payload( aggregate, report_occurrence_count=report_occurrence_count, source_family=source_family, + actionability=actionability.actionability.value, ), } return payload @@ -698,6 +1309,13 @@ def _report_gap_payload( limit=source_limit, ) source_family = _source_family_from_rule_id(aggregate.rule_id) + actionability = classify_remediation_actionability( + rule_id=aggregate.rule_id, + finding_code=aggregate.finding_code, + severity=aggregate.max_severity, + mapped=False, + attribution_status="runtime_report", + ) return { "finding_code": aggregate.finding_code, "rule_id": aggregate.rule_id, @@ -706,6 +1324,13 @@ def _report_gap_payload( "severity_counts": _counter_payload(aggregate.severity_counts), "source_family": source_family, "source_family_counts": [], + "source_helper_counts": [], + "finding_code_prefix_counts": [], + "attribution_status": "runtime_report", + "attribution_reason": "report_rule_id", + "attribution_status_counts": {}, + "attribution_reason_counts": {}, + **actionability.to_payload(), "known_source_occurrence_count": 0, "source_count": 0, "included_source_count": 0, @@ -723,16 +1348,18 @@ def _report_gap_payload( "rank_reasons": _report_rank_reasons( aggregate, source_family=source_family, + actionability=actionability.actionability.value, ), } def _ranked_gap_sort_key( gap: Mapping[str, JSONValue], -) -> tuple[int, int, int, int, int, int, str, str, str]: +) -> tuple[int, int, int, int, int, int, int, str, str, str]: severity_counts = _mapping_payload(gap.get("severity_counts")) max_severity = _optional_string(gap, "max_severity") return ( + _ACTIONABILITY_SORT.get(_optional_string(gap, "actionability"), 9), 0 if max_severity in {"error", "warning"} else 1, _SEVERITY_SORT.get(max_severity, 9), -_int_value(severity_counts, "error"), @@ -750,8 +1377,10 @@ def _rank_reasons( *, report_occurrence_count: int, source_family: str, + actionability: str, ) -> list[JSONValue]: reasons: list[JSONValue] = [ + f"actionability={actionability}", f"severity={aggregate.max_severity}", f"source_family={source_family or 'unknown'}", f"known_sources={aggregate.occurrence_count}", @@ -765,8 +1394,10 @@ def _report_rank_reasons( aggregate: _ReportGapAggregate, *, source_family: str, + actionability: str, ) -> list[JSONValue]: return [ + f"actionability={actionability}", f"severity={aggregate.max_severity}", f"source_family={source_family or 'unknown'}", f"report_occurrences={aggregate.occurrence_count}", @@ -787,6 +1418,11 @@ def _known_code_counts( unmapped_rule_id_counts: Counter[str] = Counter() unmapped_source_family_counts: Counter[str] = Counter() unmapped_finding_code_counts: Counter[str] = Counter() + attribution_status_counts: Counter[str] = Counter() + attribution_reason_counts: Counter[str] = Counter() + unresolved_source_family_counts: Counter[str] = Counter() + unresolved_source_helper_counts: Counter[str] = Counter() + unresolved_finding_code_prefix_counts: Counter[str] = Counter() for aggregate in aggregates.values(): severity_counts.update(aggregate.severity_counts) rule_id_counts[aggregate.rule_id] += aggregate.occurrence_count @@ -794,6 +1430,18 @@ def _known_code_counts( finding_code_counts[ aggregate.finding_code ] += aggregate.occurrence_count + attribution_status_counts.update(aggregate.attribution_status_counts) + attribution_reason_counts.update(aggregate.attribution_reason_counts) + if aggregate.attribution_status_counts["unresolved"]: + unresolved_source_family_counts.update( + aggregate.source_family_counts + ) + unresolved_source_helper_counts.update( + aggregate.source_helper_counts + ) + unresolved_finding_code_prefix_counts.update( + aggregate.finding_code_prefix_counts + ) if not aggregate.mapped: unmapped_rule_id_counts[ aggregate.rule_id @@ -819,6 +1467,29 @@ def _known_code_counts( key_name="finding_code", limit=code_limit, ), + "attribution_status_counts": _counter_payload( + attribution_status_counts + ), + "attribution_reason_counts": _named_counter_payloads( + attribution_reason_counts, + key_name="attribution_reason", + limit=rule_limit, + ), + "unresolved_source_family_counts": _named_counter_payloads( + unresolved_source_family_counts, + key_name="source_family", + limit=rule_limit, + ), + "unresolved_source_helper_counts": _named_counter_payloads( + unresolved_source_helper_counts, + key_name="source_helper", + limit=rule_limit, + ), + "unresolved_finding_code_prefix_counts": _named_counter_payloads( + unresolved_finding_code_prefix_counts, + key_name="finding_code_prefix", + limit=rule_limit, + ), "unmapped_rule_id_counts": _named_counter_payloads( unmapped_rule_id_counts, key_name="rule_id", @@ -852,6 +1523,7 @@ def _code_aggregate_payload( key_name="source_family", limit=source_limit, ) + actionability = _code_aggregate_actionability(aggregate) return { "rule_id": aggregate.rule_id, "finding_code": aggregate.finding_code, @@ -861,6 +1533,27 @@ def _code_aggregate_payload( "severity_counts": _counter_payload(aggregate.severity_counts), "source_family": _primary_source_family(aggregate.source_family_counts), "source_family_counts": source_family_counts, + "source_helper_counts": _named_counter_payloads( + aggregate.source_helper_counts, + key_name="source_helper", + limit=source_limit, + ), + "finding_code_prefix_counts": _named_counter_payloads( + aggregate.finding_code_prefix_counts, + key_name="finding_code_prefix", + limit=source_limit, + ), + "attribution_status": _primary_attribution_status(aggregate), + "attribution_reason": _primary_counter_key( + aggregate.attribution_reason_counts + ), + "attribution_status_counts": _counter_payload( + aggregate.attribution_status_counts + ), + "attribution_reason_counts": _counter_payload( + aggregate.attribution_reason_counts + ), + **actionability.to_payload(), "source_count": len(aggregate.source_counts), "included_source_count": len(sources), "omitted_source_count": max( @@ -871,6 +1564,18 @@ def _code_aggregate_payload( } +def _code_aggregate_actionability( + aggregate: _CodeAggregate, +) -> RemediationActionabilityDecision: + return classify_remediation_actionability( + rule_id=aggregate.rule_id, + finding_code=aggregate.finding_code, + severity=aggregate.max_severity, + mapped=aggregate.mapped, + attribution_status=_primary_attribution_status(aggregate), + ) + + def _normalized_reports( reports: Iterable[QualityReport | tuple[str, QualityReport]], ) -> tuple[tuple[str, QualityReport], ...]: @@ -946,33 +1651,285 @@ def _string_keyword(node: ast.Call, name: str) -> str: return "" -def _rule_id_from_call( +def _rule_attribution_from_call( node: ast.Call, + finding_code: str, + tree: ast.AST, constants: Mapping[str, str], *, class_rule_ids: Mapping[str, str], parents: Mapping[ast.AST, ast.AST], source_family: str, -) -> str: +) -> _RuleAttribution: + explicit = _explicit_rule_attribution( + node, + constants, + class_rule_ids=class_rule_ids, + parents=parents, + source_family=source_family, + ) + if explicit is not None: + return explicit + + helper_candidates = _enclosing_function_rule_candidates( + node, + tree, + constants, + class_rule_ids=class_rule_ids, + parents=parents, + source_family=source_family, + ) + if len(helper_candidates) == 1: + return _RuleAttribution( + rule_id=next(iter(helper_candidates)), + status="inferred", + reason="unique_helper_rule", + ) + + prefix_rule_id = _finding_code_rule_id(finding_code) + if prefix_rule_id and ( + not helper_candidates or prefix_rule_id in helper_candidates + ): + return _RuleAttribution( + rule_id=prefix_rule_id, + status="inferred", + reason="finding_code_prefix", + ) + + module_rule_ids = set(class_rule_ids.values()) + if len(module_rule_ids) == 1: + return _RuleAttribution( + rule_id=next(iter(module_rule_ids)), + status="inferred", + reason="unique_module_rule", + ) + + if len(helper_candidates) > 1: + reason = "ambiguous_helper_rules" + elif len(module_rule_ids) > 1: + reason = "multiple_module_rules" + else: + reason = "no_rule_context" + return _RuleAttribution( + rule_id=_unresolved_rule_id(source_family), + status="unresolved", + reason=reason, + ) + + +def _explicit_rule_attribution( + node: ast.Call, + constants: Mapping[str, str], + *, + class_rule_ids: Mapping[str, str], + parents: Mapping[ast.AST, ast.AST], + source_family: str, +) -> _RuleAttribution | None: for keyword in node.keywords: if keyword.arg != "rule_id": continue value = keyword.value if isinstance(value, ast.Constant) and isinstance(value.value, str): - return value.value - if isinstance(value, ast.Name): - return constants.get( - value.id, - _unresolved_rule_id(source_family), + return _RuleAttribution( + rule_id=value.value, + status="exact", + reason="literal_rule_id", + ) + if isinstance(value, ast.Name) and value.id in constants: + return _RuleAttribution( + rule_id=constants[value.id], + status="exact", + reason="module_constant", ) - return _rule_id_from_expression( + rule_id = _rule_id_from_expression( value, constants, class_rule_ids=class_rule_ids, parents=parents, source_family=source_family, ) - return _unresolved_rule_id(source_family) + if not rule_id.endswith(".unresolved"): + is_self = ( + isinstance(value, ast.Attribute) + and isinstance(value.value, ast.Name) + and value.value.id == "self" + ) + return _RuleAttribution( + rule_id=rule_id, + status="exact" if is_self else "inferred", + reason=("class_rule_id" if is_self else "local_rule_object"), + ) + return None + + +def _enclosing_function_rule_candidates( + node: ast.AST, + tree: ast.AST, + constants: Mapping[str, str], + *, + class_rule_ids: Mapping[str, str], + parents: Mapping[ast.AST, ast.AST], + source_family: str, +) -> set[str]: + function = _nearest_function(node, parents) + if function is None: + return set() + return _function_rule_candidates( + function, + tree, + constants, + class_rule_ids=class_rule_ids, + parents=parents, + source_family=source_family, + visited=set(), + ) + + +def _function_rule_candidates( + function: ast.FunctionDef | ast.AsyncFunctionDef, + tree: ast.AST, + constants: Mapping[str, str], + *, + class_rule_ids: Mapping[str, str], + parents: Mapping[ast.AST, ast.AST], + source_family: str, + visited: set[int], +) -> set[str]: + identity = id(function) + if identity in visited: + return set() + visited = {*visited, identity} + candidates: set[str] = set() + + class_rule_id = _nearest_class_rule_id( + function, + parents=parents, + class_rule_ids=class_rule_ids, + ) + if class_rule_id: + candidates.add(class_rule_id) + candidates.update( + _function_rule_id_annotations(function, class_rule_ids).values() + ) + candidates.update( + _function_rule_id_assertions(function, class_rule_ids).values() + ) + candidates.update( + _function_rule_id_assignments(function, class_rule_ids).values() + ) + default_rule_id = _function_default_rule_id(function, constants) + if default_rule_id: + candidates.add(default_rule_id) + + if class_rule_id: + return candidates + + for call in _calls_to_function(tree, function.name): + argument_rule_id = _call_rule_id_argument( + call, + function, + constants, + class_rule_ids=class_rule_ids, + parents=parents, + source_family=source_family, + ) + if argument_rule_id: + candidates.add(argument_rule_id) + caller_class_rule_id = _nearest_class_rule_id( + call, + parents=parents, + class_rule_ids=class_rule_ids, + ) + if caller_class_rule_id: + candidates.add(caller_class_rule_id) + continue + caller = _nearest_function(call, parents) + if caller is None or caller is function: + continue + candidates.update( + _function_rule_candidates( + caller, + tree, + constants, + class_rule_ids=class_rule_ids, + parents=parents, + source_family=source_family, + visited=visited, + ) + ) + return candidates + + +def _calls_to_function( + tree: ast.AST, function_name: str +) -> tuple[ast.Call, ...]: + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and _call_function_name(node.func) == function_name + ] + return tuple(sorted(calls, key=lambda item: (item.lineno, item.col_offset))) + + +def _call_function_name(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return "" + + +def _call_rule_id_argument( + call: ast.Call, + function: ast.FunctionDef | ast.AsyncFunctionDef, + constants: Mapping[str, str], + *, + class_rule_ids: Mapping[str, str], + parents: Mapping[ast.AST, ast.AST], + source_family: str, +) -> str: + value: ast.AST | None = None + for keyword in call.keywords: + if keyword.arg == "rule_id": + value = keyword.value + break + if value is None: + args = list(function.args.posonlyargs) + list(function.args.args) + rule_indexes = [ + index for index, arg in enumerate(args) if arg.arg == "rule_id" + ] + if rule_indexes and rule_indexes[0] < len(call.args): + value = call.args[rule_indexes[0]] + if value is None: + return "" + rule_id = _rule_id_from_expression( + value, + constants, + class_rule_ids=class_rule_ids, + parents=parents, + source_family=source_family, + ) + return "" if rule_id.endswith(".unresolved") else rule_id + + +def _function_default_rule_id( + function: ast.FunctionDef | ast.AsyncFunctionDef, + constants: Mapping[str, str], +) -> str: + positional = list(function.args.posonlyargs) + list(function.args.args) + defaults = list(function.args.defaults) + if defaults: + for arg, default in zip(positional[-len(defaults) :], defaults): + if arg.arg == "rule_id": + return _rule_id_literal(default, constants) + for arg, kw_default in zip( + function.args.kwonlyargs, + function.args.kw_defaults, + ): + if arg.arg == "rule_id": + return _rule_id_literal(kw_default, constants) + return "" def _rule_id_from_expression( @@ -1056,7 +2013,10 @@ def _function_rule_id_for_name( if name in annotations: return annotations[name] asserted = _function_rule_id_assertions(function, class_rule_ids) - return asserted.get(name, "") + if name in asserted: + return asserted[name] + assigned = _function_rule_id_assignments(function, class_rule_ids) + return assigned.get(name, "") def _nearest_function( @@ -1071,6 +2031,14 @@ def _nearest_function( return None +def _nearest_function_name( + node: ast.AST, + parents: Mapping[ast.AST, ast.AST], +) -> str: + function = _nearest_function(node, parents) + return function.name if function is not None else "" + + def _function_rule_id_annotations( function: ast.FunctionDef | ast.AsyncFunctionDef, class_rule_ids: Mapping[str, str], @@ -1112,6 +2080,35 @@ def _function_rule_id_assertions( return asserted +def _function_rule_id_assignments( + function: ast.FunctionDef | ast.AsyncFunctionDef, + class_rule_ids: Mapping[str, str], +) -> dict[str, str]: + assigned: dict[str, str] = {} + for node in ast.walk(function): + target: ast.Name | None = None + value: ast.AST | None = None + annotation: ast.AST | None = None + if isinstance(node, ast.AnnAssign) and isinstance( + node.target, ast.Name + ): + target = node.target + value = node.value + annotation = node.annotation + elif isinstance(node, ast.Assign) and len(node.targets) == 1: + if isinstance(node.targets[0], ast.Name): + target = node.targets[0] + value = node.value + if target is None: + continue + class_name = _annotation_name(annotation) + if not class_name and isinstance(value, ast.Call): + class_name = _annotation_name(value.func) + if class_name in class_rule_ids: + assigned[target.id] = class_rule_ids[class_name] + return assigned + + def _annotation_name(node: ast.AST | None) -> str: if isinstance(node, ast.Name): return node.id @@ -1120,6 +2117,21 @@ def _annotation_name(node: ast.AST | None) -> str: return "" +def _finding_code_rule_id(finding_code: str) -> str: + for prefix, rule_id in _FINDING_CODE_RULE_PREFIXES: + if finding_code.startswith(prefix): + return rule_id + return "" + + +def _finding_code_prefix(finding_code: str) -> str: + for prefix, _rule_id in _FINDING_CODE_RULE_PREFIXES: + if finding_code.startswith(prefix): + return prefix.removesuffix("_") + parts = [part for part in finding_code.split("_") if part] + return "_".join(parts[:3]) + + def _call_name_is(node: ast.AST, name: str) -> bool: return isinstance(node, ast.Name) and node.id == name @@ -1164,6 +2176,21 @@ def _primary_source_family(counter: Counter[str]) -> str: return sorted(counter, key=lambda item: (-counter[item], item))[0] +def _primary_counter_key(counter: Counter[str]) -> str: + if not counter: + return "" + return sorted(counter, key=lambda item: (-counter[item], item))[0] + + +def _primary_attribution_status(aggregate: _CodeAggregate) -> str: + if not aggregate.attribution_status_counts: + return "unresolved" + return min( + aggregate.attribution_status_counts, + key=lambda item: (_ATTRIBUTION_STATUS_SORT.get(item, 9), item), + ) + + def _max_severity(counter: Counter[str]) -> str: return max( counter, @@ -1248,6 +2275,18 @@ def _format_code_group(group: Mapping[str, JSONValue]) -> str: omitted = _int_value(group, "omitted_source_count") if omitted: suffix += f" (+{omitted} more sources)" + attribution = _optional_string(group, "attribution_status") + attribution_reason = _optional_string(group, "attribution_reason") + if attribution: + suffix += f" attribution={attribution}" + if attribution_reason: + suffix += f"({attribution_reason})" + actionability = _optional_string(group, "actionability") + actionability_reason = _optional_string(group, "actionability_reason") + if actionability: + suffix += f" actionability={actionability}" + if actionability_reason: + suffix += f"({actionability_reason})" return ( f"{_optional_string(group, 'max_severity') or 'info'} " f"{_optional_string(group, 'finding_code')} " @@ -1260,6 +2299,16 @@ def _format_code_group(group: Mapping[str, JSONValue]) -> str: def _format_ranked_gap(group: Mapping[str, JSONValue]) -> str: reasons = _string_list(group.get("rank_reasons")) reason_text = f" reasons={'; '.join(reasons)}" if reasons else "" + attribution = _optional_string(group, "attribution_status") or "unknown" + attribution_reason = _optional_string(group, "attribution_reason") + attribution_text = f" attribution={attribution}" + if attribution_reason: + attribution_text += f"({attribution_reason})" + actionability = _optional_string(group, "actionability") or "unknown" + actionability_reason = _optional_string(group, "actionability_reason") + actionability_text = f" actionability={actionability}" + if actionability_reason: + actionability_text += f"({actionability_reason})" return ( f"#{_int_value(group, 'rank')} " f"{_optional_string(group, 'max_severity') or 'info'} " @@ -1268,10 +2317,41 @@ def _format_ranked_gap(group: Mapping[str, JSONValue]) -> str: f"rule={_optional_string(group, 'rule_id') or 'unknown'} " f"reports={_int_value(group, 'report_occurrence_count')} " f"known_sources={_int_value(group, 'known_source_occurrence_count')}" + f"{attribution_text}" + f"{actionability_text}" f"{reason_text}" ) +def _format_remediation_plan_item( + item: Mapping[str, JSONValue], +) -> str: + selector = _mapping_payload(item.get("suggested_selector")) + action = _mapping_payload(item.get("suggested_action")) + fixability = _mapping_payload(item.get("fixability")) + missing = _string_list(item.get("missing_fields")) + missing_text = ",".join(missing) if missing else "none" + return ( + f"#{_int_value(item, 'rank')} " + f"{_optional_string(fixability, 'level') or 'unknown'}" + f"/{_int_value(fixability, 'score')} " + f"{_optional_string(item, 'finding_code')} " + f"selector={_optional_string(selector, 'shape') or 'unknown'} " + f"action={_optional_string(action, 'action_kind') or 'inspect'} " + f"confidence={_optional_string(fixability, 'confidence') or 'low'} " + f"missing={missing_text}" + ) + + +def _format_named_counts(value: object, *, name_key: str) -> str: + parts = [] + for item in _list_payload(value): + name = _optional_string(item, name_key) + if name: + parts.append(f"{name}={_int_value(item, 'count')}") + return ", ".join(parts) + + def _payload_limit_metadata( total_count: int, limit: int | BoundedReportLimit | None, diff --git a/src/histdatacom/data_quality/repair_plan.py b/src/histdatacom/data_quality/repair_plan.py new file mode 100644 index 00000000..e5789485 --- /dev/null +++ b/src/histdatacom/data_quality/repair_plan.py @@ -0,0 +1,647 @@ +"""Deterministic non-mutating repair plans for saved quality reports.""" + +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import cast + +from histdatacom.data_quality.contracts import QualityFinding, QualityReport +from histdatacom.data_quality.limits import ( + BoundedReportLimit, + bounded_report_limit, +) +from histdatacom.data_quality.remediation import ( + QualityRemediationHint, + remediation_hints_for_finding_code, +) +from histdatacom.publication_safety import ( + publish_safe_json_mapping, + publish_safe_path, +) +from histdatacom.runtime_contracts import JSONValue + +QUALITY_REPAIR_PLAN_SCHEMA_VERSION = "histdatacom.quality-repair-plan.v1" +DEFAULT_QUALITY_REPAIR_PLAN_ITEM_LIMIT = 16 +MAXIMUM_QUALITY_REPAIR_PLAN_ITEM_LIMIT = 64 +DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT = 8 +MAXIMUM_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT = 32 +QUALITY_REPAIR_PLAN_DISPLAY_LIMIT = 8 + +_ZIP_INVENTORY_RULE_ID = "inventory.zip.integrity" + + +@dataclass(frozen=True, slots=True) +class _OperationSpec: + category: str + next_step: str + preconditions: tuple[str, ...] + evidence_needed: tuple[str, ...] + required_evidence_groups: tuple[tuple[str, ...], ...] + priority: int + + +_OPERATION_SPECS: Mapping[str, _OperationSpec] = { + "ZIP_UNREADABLE": _OperationSpec( + category="restore_read_access", + next_step=( + "Restore read access to the archive or replace it with a readable copy." + ), + preconditions=( + "Confirm the archive is expected at the reported target axis.", + "Inspect ownership, permissions, and filesystem availability manually.", + ), + evidence_needed=("read failure type", "expected archive identity"), + required_evidence_groups=(), + priority=10, + ), + "ZIP_CORRUPT": _OperationSpec( + category="redownload_archive", + next_step=( + "Replace the corrupt archive with a trusted copy from the original source." + ), + preconditions=( + "Confirm the archive target identity and period.", + "Retain the original until the replacement passes ZIP integrity checks.", + ), + evidence_needed=("archive target identity", "replacement provenance"), + required_evidence_groups=(), + priority=20, + ), + "ZIP_CRC_ERROR": _OperationSpec( + category="redownload_archive", + next_step=( + "Replace the CRC-failed archive with a trusted copy and rerun inventory checks." + ), + preconditions=( + "Record the failing member before replacing the archive.", + "Verify the replacement against the expected target axis.", + ), + evidence_needed=("failing ZIP member", "replacement provenance"), + required_evidence_groups=(("bad_member",),), + priority=30, + ), + "HISTDATA_ZIP_FILENAME_INVALID": _OperationSpec( + category="rename_archive", + next_step=( + "Rename the archive to an accepted HistData filename after verifying its contents." + ), + preconditions=( + "Confirm the observed archive contains the reported symbol and period.", + "Confirm the destination filename does not already exist.", + ), + evidence_needed=( + "observed archive filename", + "accepted destination filename", + ), + required_evidence_groups=( + ("observed_filename",), + ("expected_filename", "accepted_filenames"), + ), + priority=40, + ), + "ZIP_MEMBER_MISSING": _OperationSpec( + category="restore_archive_member", + next_step=( + "Restore the expected data member from a trusted source, then rebuild and verify the archive." + ), + preconditions=( + "Confirm the expected member identity from the archive target axis.", + "Do not synthesize or copy data from a different period.", + ), + evidence_needed=("expected member name", "trusted replacement source"), + required_evidence_groups=(("expected_member",),), + priority=50, + ), + "ZIP_MEMBER_UNEXPECTED": _OperationSpec( + category="rebuild_archive_members", + next_step=( + "Inspect the observed members and rebuild the archive with the expected HistData data member." + ), + preconditions=( + "Compare every observed member axis with the expected archive axis.", + "Preserve the original archive until rebuilt contents pass inventory checks.", + ), + evidence_needed=("expected member name", "observed member names"), + required_evidence_groups=( + ("expected_member",), + ("observed_members",), + ), + priority=60, + ), + "HISTDATA_ZIP_MEMBER_FILENAME_INVALID": _OperationSpec( + category="rename_archive_member", + next_step=( + "Rename the data member to the expected HistData member name when its target identity is verified." + ), + preconditions=( + "Confirm the member contents match the archive target axis.", + "Rebuild the archive rather than editing an open ZIP in place.", + ), + evidence_needed=("observed member name", "expected member name"), + required_evidence_groups=( + ("observed_member",), + ("expected_member",), + ), + priority=70, + ), + "ZIP_EXTRA_MEMBER": _OperationSpec( + category="inspect_archive_members", + next_step=( + "Inspect unexpected members and rebuild the archive only after confirming they are not required data." + ), + preconditions=( + "Classify each extra member before removal.", + "Preserve the original archive until rebuilt contents pass inventory checks.", + ), + evidence_needed=("extra member names", "expected member name"), + required_evidence_groups=(("extra_members",),), + priority=80, + ), +} + +_EVIDENCE_KEYS = ( + "observed_filename", + "expected_filename", + "accepted_filenames", + "expected_pattern", + "observed_member", + "expected_member", + "observed_members", + "extra_members", + "bad_member", + "error_type", +) + + +@dataclass(frozen=True, slots=True) +class _PlanCandidate: + finding: QualityFinding + hint: QualityRemediationHint | None + operation: _OperationSpec | None + + @property + def sort_key( + self, + ) -> tuple[str, str, str, str, str, str, int, int, str, str, str]: + target = self.finding.target + priority = self.operation.priority if self.operation else 900 + return ( + target.symbol, + target.period, + target.data_format, + target.timeframe, + target.kind.value, + publish_safe_path(target.path), + priority, + -self.finding.severity.rank, + self.finding.rule_id, + self.finding.code, + _stable_finding_key(self.finding), + ) + + +def quality_repair_plan( + report: QualityReport, + *, + report_path: str = "", + item_limit: int | None = DEFAULT_QUALITY_REPAIR_PLAN_ITEM_LIMIT, + evidence_limit: int | None = DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, +) -> dict[str, JSONValue]: + """Return a bounded advisory repair plan without changing user data.""" + item_limit_state = bounded_report_limit( + item_limit, + default_limit=DEFAULT_QUALITY_REPAIR_PLAN_ITEM_LIMIT, + maximum_limit=MAXIMUM_QUALITY_REPAIR_PLAN_ITEM_LIMIT, + allow_unbounded=False, + ) + evidence_limit_state = bounded_report_limit( + evidence_limit, + default_limit=DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + maximum_limit=MAXIMUM_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + allow_unbounded=False, + ) + candidates = sorted( + (_candidate_for_finding(finding) for finding in report.findings), + key=lambda candidate: candidate.sort_key, + ) + all_items = [ + _repair_plan_item( + candidate, + rank=rank, + evidence_limit=evidence_limit_state, + ) + for rank, candidate in enumerate(candidates, start=1) + ] + included_items = item_limit_state.slice(all_items) + proposal_status_counts = Counter( + _string_value(_mapping(item.get("operation")).get("proposal_status")) + for item in all_items + ) + operation_category_counts = Counter( + _string_value(_mapping(item.get("operation")).get("category")) + for item in all_items + ) + count_payload = item_limit_state.count_payload(len(all_items)) + summary = report.summary() + payload: dict[str, JSONValue] = { + "schema_version": QUALITY_REPAIR_PLAN_SCHEMA_VERSION, + "mode": "non_mutating", + "apply_supported": False, + "mutating_operations_performed": False, + "input_report": { + "path": publish_safe_path(report_path), + "status": summary.status.value, + "target_count": summary.target_count, + "finding_count": summary.finding_count, + }, + "plan_item_count": len(all_items), + "included_plan_item_count": _int_value( + count_payload.get("included_count") + ), + "omitted_plan_item_count": _int_value( + count_payload.get("omitted_count") + ), + "truncated": bool(count_payload["truncated"]), + "proposal_status_counts": _sorted_count_payload(proposal_status_counts), + "operation_category_counts": _sorted_count_payload( + operation_category_counts + ), + "items": cast(JSONValue, included_items), + "payload_limits": { + "items": count_payload, + "evidence_per_item": evidence_limit_state.limit_payload(), + }, + "safety": { + "advisory_only": True, + "automatic_execution": "unsupported", + "network_access_performed": False, + "filesystem_mutation_performed": False, + "requires_user_verification_before_action": True, + }, + } + safe_payload: dict[str, JSONValue] = publish_safe_json_mapping(payload) + return safe_payload + + +def quality_repair_plan_to_json(payload: Mapping[str, JSONValue]) -> str: + """Return deterministic formatted JSON for a repair plan.""" + return json.dumps(payload, indent=2, sort_keys=True) + "\n" + + +def format_quality_repair_plan(payload: Mapping[str, JSONValue]) -> str: + """Return concise human-readable repair-plan output.""" + source = _mapping(payload.get("input_report")) + lines = [ + "Quality repair plan", + f"mode: {payload.get('mode', 'non_mutating')}", + f"report: {source.get('path', '') or 'in-memory'}", + ( + "items: " + f"{payload.get('plan_item_count', 0)} " + f"included: {payload.get('included_plan_item_count', 0)} " + f"omitted: {payload.get('omitted_plan_item_count', 0)}" + ), + "safety: advisory only; no files, archives, permissions, or network state changed", + ] + items = _mapping_sequence(payload.get("items")) + if not items: + plan_item_count = _int_value(payload.get("plan_item_count")) + if plan_item_count: + lines.append(f"- all {plan_item_count} plan items omitted by limit") + else: + lines.append("- no findings to plan") + return "\n".join(lines) + for item in items[:QUALITY_REPAIR_PLAN_DISPLAY_LIMIT]: + operation = _mapping(item.get("operation")) + target = _mapping(item.get("target")) + confidence = _mapping(item.get("confidence")) + lines.append( + "- " + f"#{item.get('rank', 0)} " + f"{item.get('severity', 'info')} " + f"{item.get('finding_code', 'unknown')} " + f"[{operation.get('proposal_status', 'unsupported')}/" + f"{operation.get('category', 'unsupported')}; " + f"confidence={confidence.get('level', 'low')}] " + f"{target.get('path', '') or 'unknown target'}: " + f"{operation.get('next_step', '')}" + ) + hidden = max( + 0, + _int_value(payload.get("plan_item_count")) + - min(len(items), QUALITY_REPAIR_PLAN_DISPLAY_LIMIT), + ) + if hidden: + lines.append(f"- additional plan items: {hidden}") + return "\n".join(lines) + + +def _candidate_for_finding(finding: QualityFinding) -> _PlanCandidate: + hints = remediation_hints_for_finding_code( + finding.code, + rule_id=finding.rule_id, + ) + hint = hints[0] if hints else None + operation = ( + _OPERATION_SPECS.get(finding.code) + if finding.rule_id == _ZIP_INVENTORY_RULE_ID + else None + ) + return _PlanCandidate(finding=finding, hint=hint, operation=operation) + + +def _repair_plan_item( + candidate: _PlanCandidate, + *, + rank: int, + evidence_limit: BoundedReportLimit, +) -> dict[str, JSONValue]: + finding = candidate.finding + evidence = _evidence_payload(finding.metadata, evidence_limit) + missing_evidence = _missing_evidence( + finding.metadata, + ( + candidate.operation.required_evidence_groups + if candidate.operation + else () + ), + ) + operation = _operation_payload(candidate, missing_evidence) + target = finding.target + hint = candidate.hint + return { + "rank": rank, + "finding_code": finding.code, + "rule_id": finding.rule_id, + "severity": finding.severity.value, + "remediation_hint_code": hint.code if hint else "", + "action_kind": hint.action_kind if hint else "", + "target": { + "path": publish_safe_path(target.path or finding.location.path), + "target_axis": { + "data_format": target.data_format, + "timeframe": target.timeframe, + "symbol": target.symbol, + "period": target.period, + "kind": target.kind.value, + }, + }, + "operation": operation, + "preconditions": ( + list(candidate.operation.preconditions) + if candidate.operation + else [ + "Obtain a supported remediation mapping and diagnostic context." + ] + ), + "evidence_needed": ( + list(candidate.operation.evidence_needed) + if candidate.operation + else ["supported operation contract"] + ), + "missing_evidence": list(missing_evidence), + "evidence": evidence, + "confidence": _confidence_payload(candidate, missing_evidence), + } + + +def _operation_payload( + candidate: _PlanCandidate, + missing_evidence: Sequence[str], +) -> dict[str, JSONValue]: + operation = candidate.operation + hint = candidate.hint + if operation is None: + reason = "unmapped_finding" if hint is None else "unsupported_action" + return { + "category": "unsupported", + "proposal_status": "unsupported", + "specificity": "advisory", + "automation_status": "unsupported", + "reason": reason, + "next_step": ( + "No concrete repair operation is supported for this finding; inspect it manually." + ), + } + contextual = bool(missing_evidence) + return { + "category": operation.category, + "proposal_status": "needs_context" if contextual else "proposed", + "specificity": "contextual" if contextual else "exact", + "automation_status": "manual_only", + "reason": ( + "required_evidence_missing" + if contextual + else "exact_rule_finding_operation_mapping" + ), + "next_step": _contextualized_next_step( + candidate.finding, + operation, + contextual=contextual, + ), + } + + +def _contextualized_next_step( + finding: QualityFinding, + operation: _OperationSpec, + *, + contextual: bool, +) -> str: + metadata = finding.metadata + if finding.code == "HISTDATA_ZIP_FILENAME_INVALID": + observed = _string_value(metadata.get("observed_filename")) + expected = _string_value(metadata.get("expected_filename")) + if not expected: + expected = _first_string(metadata.get("accepted_filenames")) + if observed and expected: + return f"Rename archive {observed!r} to {expected!r} after verifying its contents." + if finding.code == "HISTDATA_ZIP_MEMBER_FILENAME_INVALID": + observed = _string_value(metadata.get("observed_member")) + expected = _string_value(metadata.get("expected_member")) + if observed and expected: + return f"Rebuild the archive with member {observed!r} renamed to {expected!r}." + if contextual: + return f"{operation.next_step} Required evidence is still missing." + return operation.next_step + + +def _confidence_payload( + candidate: _PlanCandidate, + missing_evidence: Sequence[str], +) -> dict[str, JSONValue]: + if candidate.operation is None: + return { + "level": "low", + "basis": [ + "no supported exact operation mapping", + ( + "remediation hint exists" + if candidate.hint + else "remediation hint is unmapped" + ), + ], + } + if missing_evidence: + return { + "level": "medium", + "basis": [ + "exact rule and finding operation mapping", + "required diagnostic evidence is incomplete", + ], + } + return { + "level": "high", + "basis": [ + "exact rule and finding operation mapping", + "required diagnostic evidence is present", + "operation remains manual and non-mutating in the application", + ], + } + + +def _evidence_payload( + metadata: Mapping[str, JSONValue], + limit: BoundedReportLimit, +) -> dict[str, JSONValue]: + evidence_items: list[dict[str, JSONValue]] = [] + for key in _EVIDENCE_KEYS: + if key not in metadata: + continue + value = metadata.get(key) + if _is_scalar(value): + evidence_items.append({"kind": key, "value": value}) + elif isinstance(value, list): + evidence_items.extend( + {"kind": key, "value": item} + for item in value + if _is_scalar(item) + ) + evidence_items.extend(_topology_inspection_evidence(metadata)) + count_payload = limit.count_payload(len(evidence_items)) + return { + **count_payload, + "items": cast(JSONValue, limit.slice(evidence_items)), + } + + +def _topology_inspection_evidence( + metadata: Mapping[str, JSONValue], +) -> list[dict[str, JSONValue]]: + fingerprint = _mapping(metadata.get("time_series_fingerprint")) + topology = _mapping(fingerprint.get("temporal_topology")) + context = _mapping( + topology.get("inspection_context") or metadata.get("inspection_context") + ) + evidence: list[dict[str, JSONValue]] = [] + for name in sorted(context): + if name == "schema_version": + continue + section = _mapping(context.get(name)) + if not section: + continue + counts: dict[str, JSONValue] = {} + for key in ( + "total_count", + "included_count", + "omitted_count", + "truncated", + ): + value = section.get(key) + if value is not None: + counts[key] = value + if counts: + evidence.append( + { + "kind": f"inspection_context.{name}", + "value": counts, + } + ) + return evidence + + +def _missing_evidence( + metadata: Mapping[str, JSONValue], + required_groups: Sequence[Sequence[str]], +) -> tuple[str, ...]: + missing: list[str] = [] + for group in required_groups: + if not any(_has_evidence(metadata.get(key)) for key in group): + missing.append(" or ".join(group)) + return tuple(missing) + + +def _has_evidence(value: JSONValue | None) -> bool: + if isinstance(value, str): + return bool(value.strip()) + if isinstance(value, list): + return bool(value) + return value is not None + + +def _is_scalar(value: JSONValue | None) -> bool: + return isinstance(value, (str, int, float, bool)) + + +def _first_string(value: JSONValue | None) -> str: + if not isinstance(value, list): + return "" + for item in value: + if isinstance(item, str) and item: + return item + return "" + + +def _sorted_count_payload(counts: Counter[str]) -> dict[str, JSONValue]: + return { + key: count for key, count in sorted(counts.items()) if key and count > 0 + } + + +def _mapping(value: JSONValue | None) -> Mapping[str, JSONValue]: + return value if isinstance(value, Mapping) else {} + + +def _mapping_sequence(value: JSONValue | None) -> list[Mapping[str, JSONValue]]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, Mapping)] + + +def _string_value(value: JSONValue | None) -> str: + return value if isinstance(value, str) else "" + + +def _int_value(value: JSONValue | None) -> int: + return ( + value if isinstance(value, int) and not isinstance(value, bool) else 0 + ) + + +def _stable_finding_key(finding: QualityFinding) -> str: + payload: dict[str, JSONValue] = { + "message": finding.message, + "evidence": _evidence_payload( + finding.metadata, + bounded_report_limit( + MAXIMUM_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + default_limit=DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + maximum_limit=MAXIMUM_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + allow_unbounded=False, + ), + ), + "location": { + "path": publish_safe_path(finding.location.path), + "row_number": finding.location.row_number, + "timestamp_utc_ms": finding.location.timestamp_utc_ms, + "column": finding.location.column, + }, + } + return json.dumps( + publish_safe_json_mapping(payload), + sort_keys=True, + separators=(",", ":"), + ) diff --git a/src/histdatacom/data_quality/reporting.py b/src/histdatacom/data_quality/reporting.py index 8552741c..a084b414 100644 --- a/src/histdatacom/data_quality/reporting.py +++ b/src/histdatacom/data_quality/reporting.py @@ -19,21 +19,75 @@ QualityStatus, QualityTargetSummary, ) +from histdatacom.data_quality.classical_baselines import ( + CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY, + CLASSICAL_BASELINE_SUMMARY_METADATA_KEY, + classical_baseline_summary, + format_classical_baseline_summary_lines, +) +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY, + CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY, + classical_model_input_summary, + format_classical_model_input_summary_lines, +) +from histdatacom.data_quality.classical_model_comparison import ( + CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY, + CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY, + classical_model_comparison_summary, + format_classical_model_comparison_summary_lines, +) +from histdatacom.data_quality.autoregressive import ( + AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY, + AUTOREGRESSIVE_SUMMARY_METADATA_KEY, + autoregressive_summary, + format_autoregressive_summary_lines, +) +from histdatacom.data_quality.engine import QUALITY_ENGINE_METADATA_KEY +from histdatacom.data_quality.exponential_smoothing import ( + EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY, + EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY, + exponential_smoothing_summary, + format_exponential_smoothing_summary_lines, +) from histdatacom.data_quality.fingerprint_contracts import ( + FINGERPRINT_CROSS_SERIES_BOUNDED_PAYLOAD_KEY, FINGERPRINT_COVERAGE_BOUNDED_PAYLOAD_KEY, FINGERPRINT_DISTRIBUTION_ATTENTION_BOUNDED_PAYLOAD_KEY, FINGERPRINT_DISTRIBUTION_BOUNDED_PAYLOAD_KEY, + FINGERPRINT_PARITY_BOUNDED_PAYLOAD_KEY, FINGERPRINT_READINESS_BOUNDED_PAYLOAD_KEY, FINGERPRINT_READINESS_RISK_BOUNDED_PAYLOAD_KEY, FINGERPRINT_REGIME_BOUNDED_PAYLOAD_KEY, + FINGERPRINT_SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY, FINGERPRINT_TOPOLOGY_ATTENTION_BOUNDED_PAYLOAD_KEY, FINGERPRINT_TOPOLOGY_BOUNDED_PAYLOAD_KEY, ) +from histdatacom.data_quality.seasonal_exogenous import ( + SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY, + SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY, + format_seasonal_exogenous_summary_lines, + seasonal_exogenous_summary, +) +from histdatacom.data_quality.state_space import ( + STATE_SPACE_BOUNDED_PAYLOAD_KEY, + STATE_SPACE_SUMMARY_METADATA_KEY, + format_state_space_summary_lines, + state_space_summary, +) +from histdatacom.data_quality.volatility import ( + VOLATILITY_BOUNDED_PAYLOAD_KEY, + VOLATILITY_SUMMARY_METADATA_KEY, + format_volatility_summary_lines, + volatility_summary, +) from histdatacom.data_quality.fingerprints import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY, TIME_SERIES_FINGERPRINT_DISTRIBUTION_ATTENTION_METADATA_KEY, TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY, TIME_SERIES_FINGERPRINT_REGIME_SUMMARY_METADATA_KEY, @@ -42,6 +96,7 @@ series_fingerprint_coverage_summary, series_fingerprint_distribution_attention_summary, series_fingerprint_distribution_summary, + series_fingerprint_parity_summary, series_fingerprint_readiness_summary, series_fingerprint_readiness_risk_summary, series_fingerprint_regime_summary, @@ -56,9 +111,16 @@ bounded_report_limit, ) from histdatacom.data_quality.remediation import ( + RemediationActionability, + classify_remediation_actionability, remediation_hint_payloads_for_finding, ) from histdatacom.data_quality.profiles import QUALITY_REPORTING_METADATA_KEY +from histdatacom.data_quality.synthetic_constraints import ( + SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY, + format_synthetic_constraint_summary_lines, + synthetic_constraint_summary, +) from histdatacom.publication_safety import ( publish_safe_json_mapping, publish_safe_path, @@ -82,11 +144,23 @@ QUALITY_PAYLOAD_REMEDIATION_COVERAGE_TARGET_AXIS_LIMIT = 8 QUALITY_PAYLOAD_REMEDIATION_CATALOG_AUDIT_RULE_LIMIT = 16 QUALITY_PAYLOAD_REMEDIATION_CATALOG_AUDIT_SOURCE_LIMIT = 8 +QUALITY_REMEDIATION_PLAN_DISPLAY_LIMIT = 5 _FINGERPRINT_REPORT_SURFACE_EVIDENCE_ACTIVE = False _NEXT_ACTION_SEVERITY_RANK = {"info": 1, "warning": 2, "error": 3} _REMEDIATION_COVERAGE_SEVERITY_RANK = {"info": 1, "warning": 2, "error": 3} +_REMEDIATION_ACTIONABILITY_SORT = { + RemediationActionability.REMEDIABLE_DEFECT.value: 0, + RemediationActionability.NEEDS_DIAGNOSTIC_CONTEXT.value: 1, + RemediationActionability.NEEDS_RULE_ATTRIBUTION.value: 2, + RemediationActionability.UNSAFE_TO_AUTOMATE.value: 3, + RemediationActionability.POLICY_OR_PROFILE_DECISION.value: 4, + RemediationActionability.UNSUPPORTED_FORMAT_OR_CAPABILITY.value: 5, + RemediationActionability.EXPECTED_ARTIFACT_OR_CONTEXT.value: 6, + RemediationActionability.INFORMATIONAL_ONLY.value: 7, +} _NEXT_ACTION_ATTENTION_RANK = { + "contextual": 0, "session": 1, "sequence": 2, "unavailable": 3, @@ -109,6 +183,7 @@ class _NextActionAggregate: message: str action_kind: str rule_id: str + policy_context: dict[str, JSONValue] = field(default_factory=dict) occurrence_count: int = 0 severity_counts: Counter[str] = field(default_factory=Counter) attention_level_counts: Counter[str] = field(default_factory=Counter) @@ -125,6 +200,8 @@ class _RemediationCoverageAggregate: rule_id: str finding_code: str mapped: bool + actionability: str + actionability_reason: str occurrence_count: int = 0 severity_counts: Counter[str] = field(default_factory=Counter) target_axis_counts: Counter[tuple[str, str, str, str, str]] = field( @@ -294,6 +371,66 @@ def quality_report_payload( fingerprint_regimes ) payload["metadata"] = metadata + fingerprint_parity = _fingerprint_parity_summary(report) + if fingerprint_parity is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY] = ( + fingerprint_parity + ) + payload["metadata"] = metadata + synthetic_constraints = _synthetic_constraint_summary(report) + if synthetic_constraints is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY] = ( + synthetic_constraints + ) + payload["metadata"] = metadata + classical_baselines = _classical_baseline_summary(report) + if classical_baselines is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[CLASSICAL_BASELINE_SUMMARY_METADATA_KEY] = classical_baselines + payload["metadata"] = metadata + classical_model_input = _classical_model_input_summary(report) + if classical_model_input is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY] = ( + classical_model_input + ) + payload["metadata"] = metadata + exponential_smoothing = _exponential_smoothing_summary(report) + if exponential_smoothing is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY] = ( + exponential_smoothing + ) + payload["metadata"] = metadata + autoregressive = _autoregressive_summary(report) + if autoregressive is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[AUTOREGRESSIVE_SUMMARY_METADATA_KEY] = autoregressive + payload["metadata"] = metadata + seasonal_exogenous = _seasonal_exogenous_summary(report) + if seasonal_exogenous is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY] = seasonal_exogenous + payload["metadata"] = metadata + state_space = _state_space_summary(report) + if state_space is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[STATE_SPACE_SUMMARY_METADATA_KEY] = state_space + payload["metadata"] = metadata + volatility = _volatility_summary(report) + if volatility is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[VOLATILITY_SUMMARY_METADATA_KEY] = volatility + payload["metadata"] = metadata + classical_model_comparison = _classical_model_comparison_summary(report) + if classical_model_comparison is not None: + metadata = _mapping_payload(payload.get("metadata")) + metadata[CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY] = ( + classical_model_comparison + ) + payload["metadata"] = metadata fingerprint_topology = _fingerprint_topology_summary(report) if fingerprint_topology is not None: metadata = _mapping_payload(payload.get("metadata")) @@ -431,6 +568,9 @@ def format_quality_console_summary( lines.append(f"report: {report_path}") if summary.target_count == 0: lines.append("No data quality targets discovered.") + lines.extend( + format_quality_engine_skip_lines(_quality_engine_summary(report)) + ) lines.extend( format_quality_next_action_lines(quality_next_actions_summary(report)) ) @@ -465,6 +605,50 @@ def format_quality_console_summary( _fingerprint_regime_summary(report) ) ) + lines.extend( + format_fingerprint_parity_summary_lines( + _fingerprint_parity_summary(report) + ) + ) + lines.extend( + format_synthetic_constraint_summary_lines( + _synthetic_constraint_summary(report) + ) + ) + lines.extend( + format_classical_baseline_summary_lines( + _classical_baseline_summary(report) + ) + ) + lines.extend( + format_classical_model_input_summary_lines( + _classical_model_input_summary(report) + ) + ) + lines.extend( + format_exponential_smoothing_summary_lines( + _exponential_smoothing_summary(report) + ) + ) + lines.extend( + format_autoregressive_summary_lines(_autoregressive_summary(report)) + ) + lines.extend( + format_seasonal_exogenous_summary_lines( + _seasonal_exogenous_summary(report) + ) + ) + lines.extend( + format_state_space_summary_lines(_state_space_summary(report)) + ) + lines.extend( + format_volatility_summary_lines(_volatility_summary(report)) + ) + lines.extend( + format_classical_model_comparison_summary_lines( + _classical_model_comparison_summary(report) + ) + ) lines.extend( format_fingerprint_topology_attention_lines( _fingerprint_topology_attention_summary(report) @@ -485,6 +669,11 @@ def format_quality_console_summary( _fingerprint_readiness_risk_summary(report) ) ) + lines.extend( + format_cross_series_fingerprint_lines( + _cross_series_fingerprint_summary(report) + ) + ) sections = ( (QualityStatus.CLEAN, "Clean files"), @@ -541,17 +730,29 @@ def bounded_quality_payload( _fingerprint_distribution_attention_summary(report) ) fingerprint_regimes = _fingerprint_regime_summary(report) + fingerprint_parity = _fingerprint_parity_summary(report) + synthetic_constraints = _synthetic_constraint_summary(report) + classical_baselines = _classical_baseline_summary(report) + classical_model_input = _classical_model_input_summary(report) + exponential_smoothing = _exponential_smoothing_summary(report) + autoregressive = _autoregressive_summary(report) + seasonal_exogenous = _seasonal_exogenous_summary(report) + state_space = _state_space_summary(report) + volatility = _volatility_summary(report) + classical_model_comparison = _classical_model_comparison_summary(report) fingerprint_topology = _fingerprint_topology_summary(report) fingerprint_topology_attention = _fingerprint_topology_attention_summary( report ) fingerprint_readiness = _fingerprint_readiness_summary(report) fingerprint_readiness_risk = _fingerprint_readiness_risk_summary(report) + cross_series_fingerprint = _cross_series_fingerprint_summary(report) next_actions = quality_next_actions_summary(report) remediation_coverage = quality_remediation_coverage_summary(report) remediation_catalog_audit = quality_remediation_catalog_audit_summary( report ) + quality_engine = _quality_engine_summary(report) payload_limits: dict[str, JSONValue] = { "discovery_targets": _payload_limit_metadata( _sequence_count(discovery.get("targets")), @@ -605,6 +806,34 @@ def bounded_quality_payload( ) if fingerprint_regimes is not None: payload[FINGERPRINT_REGIME_BOUNDED_PAYLOAD_KEY] = fingerprint_regimes + if fingerprint_parity is not None: + payload[FINGERPRINT_PARITY_BOUNDED_PAYLOAD_KEY] = fingerprint_parity + if synthetic_constraints is not None: + payload[FINGERPRINT_SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY] = ( + synthetic_constraints + ) + if classical_baselines is not None: + payload[CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY] = classical_baselines + if classical_model_input is not None: + payload[CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY] = ( + classical_model_input + ) + if exponential_smoothing is not None: + payload[EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY] = ( + exponential_smoothing + ) + if autoregressive is not None: + payload[AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY] = autoregressive + if seasonal_exogenous is not None: + payload[SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY] = seasonal_exogenous + if state_space is not None: + payload[STATE_SPACE_BOUNDED_PAYLOAD_KEY] = state_space + if volatility is not None: + payload[VOLATILITY_BOUNDED_PAYLOAD_KEY] = volatility + if classical_model_comparison is not None: + payload[CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY] = ( + classical_model_comparison + ) if fingerprint_topology is not None: payload[FINGERPRINT_TOPOLOGY_BOUNDED_PAYLOAD_KEY] = fingerprint_topology if fingerprint_topology_attention is not None: @@ -619,6 +848,10 @@ def bounded_quality_payload( payload[FINGERPRINT_READINESS_RISK_BOUNDED_PAYLOAD_KEY] = ( fingerprint_readiness_risk ) + if cross_series_fingerprint is not None: + payload[FINGERPRINT_CROSS_SERIES_BOUNDED_PAYLOAD_KEY] = ( + cross_series_fingerprint + ) if next_actions is not None: payload["next_actions"] = next_actions if remediation_coverage is not None: @@ -630,6 +863,8 @@ def bounded_quality_payload( remediation_catalog_audit ) ) + if quality_engine is not None: + payload[QUALITY_ENGINE_METADATA_KEY] = quality_engine if not publish_safe: return payload return _publish_safe_mapping(payload) @@ -1038,10 +1273,14 @@ def quality_remediation_coverage_summary( finding_code_counts: Counter[str] = Counter() mapped_finding_code_counts: Counter[str] = Counter() unmapped_finding_code_counts: Counter[str] = Counter() + actionability_counts: Counter[str] = Counter() mapped_finding_count = 0 unmapped_finding_count = 0 for aggregate in aggregates.values(): + actionability_counts[ + aggregate.actionability + ] += aggregate.occurrence_count severity_counts.update(aggregate.severity_counts) rule_id_counts[aggregate.rule_id] += aggregate.occurrence_count finding_code_counts[ @@ -1085,6 +1324,20 @@ def quality_remediation_coverage_summary( for aggregate in included_unmapped_groups if _remediation_coverage_group_is_warning_or_error(aggregate) ) + unmapped_actionable_warning_error_groups = [ + aggregate + for aggregate in unmapped_groups + if _remediation_coverage_group_is_warning_or_error(aggregate) + and aggregate.actionability + == RemediationActionability.REMEDIABLE_DEFECT.value + ] + included_unmapped_actionable_warning_error_group_count = sum( + 1 + for aggregate in included_unmapped_groups + if _remediation_coverage_group_is_warning_or_error(aggregate) + and aggregate.actionability + == RemediationActionability.REMEDIABLE_DEFECT.value + ) count_limits = _remediation_coverage_count_limits( rule_id_counts=rule_id_counts, mapped_rule_id_counts=mapped_rule_id_counts, @@ -1108,6 +1361,37 @@ def quality_remediation_coverage_summary( for severity, count in unmapped_severity_counts.items() if severity in {"error", "warning"} ), + "actionability_counts": _counter_payload(actionability_counts), + "unmapped_actionable_warning_error_finding_count": sum( + aggregate.occurrence_count + for aggregate in unmapped_actionable_warning_error_groups + ), + "intentionally_unremediable_warning_error_finding_count": sum( + aggregate.occurrence_count + for aggregate in unmapped_groups + if _remediation_coverage_group_is_warning_or_error(aggregate) + and aggregate.actionability + in { + RemediationActionability.POLICY_OR_PROFILE_DECISION.value, + RemediationActionability.UNSUPPORTED_FORMAT_OR_CAPABILITY.value, + RemediationActionability.EXPECTED_ARTIFACT_OR_CONTEXT.value, + RemediationActionability.UNSAFE_TO_AUTOMATE.value, + } + ), + "blocked_by_attribution_warning_error_finding_count": sum( + aggregate.occurrence_count + for aggregate in unmapped_groups + if _remediation_coverage_group_is_warning_or_error(aggregate) + and aggregate.actionability + == RemediationActionability.NEEDS_RULE_ATTRIBUTION.value + ), + "blocked_by_missing_diagnostics_warning_error_finding_count": sum( + aggregate.occurrence_count + for aggregate in unmapped_groups + if _remediation_coverage_group_is_warning_or_error(aggregate) + and aggregate.actionability + == RemediationActionability.NEEDS_DIAGNOSTIC_CONTEXT.value + ), "severity_counts": _counter_payload(severity_counts), "mapped_severity_counts": _counter_payload(mapped_severity_counts), "unmapped_severity_counts": _counter_payload(unmapped_severity_counts), @@ -1161,6 +1445,17 @@ def quality_remediation_coverage_summary( unmapped_warning_error_group_count - included_unmapped_warning_error_group_count, ), + "unmapped_actionable_warning_error_group_count": len( + unmapped_actionable_warning_error_groups + ), + "included_unmapped_actionable_warning_error_group_count": ( + included_unmapped_actionable_warning_error_group_count + ), + "omitted_unmapped_actionable_warning_error_group_count": max( + 0, + len(unmapped_actionable_warning_error_groups) + - included_unmapped_actionable_warning_error_group_count, + ), "unmapped_groups": [ _remediation_coverage_group_payload( aggregate, @@ -1206,12 +1501,56 @@ def format_quality_remediation_coverage_lines( "omitted: " f"{_int_metadata(summary, 'omitted_unmapped_warning_error_group_count')}" ), + ( + "- actionability: actionable=" + f"{_int_metadata(summary, 'unmapped_actionable_warning_error_group_count')} " + "blocked_attribution=" + f"{_int_metadata(summary, 'blocked_by_attribution_warning_error_finding_count')} " + "blocked_diagnostics=" + f"{_int_metadata(summary, 'blocked_by_missing_diagnostics_warning_error_finding_count')} " + "intentional_boundary=" + f"{_int_metadata(summary, 'intentionally_unremediable_warning_error_finding_count')}" + ), ] for group in groups: lines.append(f"- {_format_quality_remediation_coverage_group(group)}") return lines +def format_quality_engine_skip_lines( + summary: Mapping[str, JSONValue] | None, +) -> list[str]: + """Return concise reconciliation lines for skipped rule evaluations.""" + if not summary: + return [] + skipped = _int_metadata(summary, "skipped_rule_evaluation_count") + skip_events = _mapping_payload(summary.get("skip_events")) + if not skipped or not skip_events: + return [] + lines = [ + "", + "Quality engine skips", + ( + "- evaluations: " + f"planned={_int_metadata(summary, 'planned_target_rule_evaluation_count')} " + f"executed={_int_metadata(summary, 'target_rule_evaluation_count')} " + f"skipped={skipped}" + ), + ( + "- events: " + f"included={_int_metadata(skip_events, 'included_event_count')} " + f"omitted={_int_metadata(skip_events, 'omitted_event_count')}" + ), + ] + reason_counts = _format_count_metadata(skip_events.get("reason_counts")) + if reason_counts: + lines.append(f"- reasons: {reason_counts}") + rule_id_counts = _format_count_metadata(skip_events.get("rule_id_counts")) + if rule_id_counts: + lines.append(f"- rules: {rule_id_counts}") + return lines + + def quality_remediation_catalog_audit_summary( report: QualityReport, *, @@ -1285,7 +1624,41 @@ def format_quality_remediation_catalog_audit_lines( "unmapped warning/error groups=" f"{report_gap_count}" ), + ( + "- attribution: " + "exact=" + f"{_int_metadata(audit_summary, 'exact_attribution_occurrence_count')} " + "inferred=" + f"{_int_metadata(audit_summary, 'inferred_attribution_occurrence_count')} " + "unresolved=" + f"{_int_metadata(audit_summary, 'unresolved_attribution_occurrence_count')}" + ), + ( + "- actionability: actionable=" + f"{_int_metadata(audit_summary, 'unmapped_actionable_warning_error_gap_count')} " + "blocked_attribution=" + f"{_int_metadata(audit_summary, 'blocked_by_attribution_warning_error_code_count')} " + "blocked_diagnostics=" + f"{_int_metadata(audit_summary, 'blocked_by_missing_diagnostics_warning_error_code_count')} " + "intentional_boundary=" + f"{_int_metadata(audit_summary, 'intentionally_unremediable_warning_error_code_count')}" + ), ] + remediation_plan = _mapping_payload(summary.get("remediation_plan")) + plan_items = _list_metadata(remediation_plan.get("items")) + if plan_items: + lines.append( + "- plan: candidates=" + f"{_int_metadata(remediation_plan, 'plan_item_count')} " + "included=" + f"{_int_metadata(remediation_plan, 'included_plan_item_count')} " + "truncated=" + f"{str(bool(remediation_plan.get('truncated'))).lower()}" + ) + lines.extend( + f"- plan {_format_quality_remediation_plan_item(item)}" + for item in plan_items[:QUALITY_REMEDIATION_PLAN_DISPLAY_LIMIT] + ) for group in _remediation_catalog_observed_gap_groups(summary): lines.append( "- observed " + _format_quality_remediation_coverage_group(group) @@ -1414,6 +1787,9 @@ def _collect_fingerprint_topology_next_actions( continue for hint in hints: if isinstance(hint, Mapping): + policy_context = _mapping_payload(hint.get("policy_context")) + if policy_context.get("actionable") is False: + continue _add_next_action_hint( aggregates, hint, @@ -1469,6 +1845,9 @@ def _add_next_action_hint( rule_id=rule_id, ), ) + policy_context = _mapping_payload(hint.get("policy_context")) + if policy_context and not aggregate.policy_context: + aggregate.policy_context = dict(policy_context) aggregate.occurrence_count += 1 aggregate.source_counts[source] += 1 aggregate.target_axis_counts[_target_axis_key(target_axis)] += 1 @@ -1509,7 +1888,7 @@ def _next_action_payload( aggregate.attention_level_counts, _NEXT_ACTION_ATTENTION_RANK, ) - return { + payload: dict[str, JSONValue] = { "code": aggregate.code, "message": aggregate.message, "action_kind": aggregate.action_kind, @@ -1539,6 +1918,9 @@ def _next_action_payload( "flag_counts": _counter_payload(aggregate.flag_counts), "target_axis_counts": target_axis_counts, } + if aggregate.policy_context: + payload["policy_context"] = dict(aggregate.policy_context) + return payload def _next_action_sort_key( @@ -1660,6 +2042,22 @@ def _format_quality_next_action_line( qualifiers.append(f"severity={severity}") if attention: qualifiers.append(f"attention={attention}") + policy = _mapping_payload(action.get("policy_context")) + weekend_policy = _optional_string_metadata( + policy, + "weekend_activity_policy", + ) + closure_policy = _optional_string_metadata( + policy, + "expected_session_closure_policy", + ) + profile_name = _optional_string_metadata(policy, "profile_name") + if weekend_policy: + qualifiers.append(f"weekend-policy={weekend_policy}") + if closure_policy == "unexpected": + qualifiers.append("closure-policy=unexpected") + if profile_name: + qualifiers.append(f"profile={profile_name}") qualifier_text = f", {', '.join(qualifiers)}" if qualifiers else "" return ( f"{_optional_string_metadata(action, 'urgency')} " @@ -1680,16 +2078,38 @@ def _remediation_coverage_aggregates( mapped = bool(remediation_hint_payloads_for_finding(finding)) rule_id = finding.rule_id or "unknown" finding_code = finding.code or "unknown" + decision = classify_remediation_actionability( + rule_id=rule_id, + finding_code=finding_code, + severity=finding.severity.value, + mapped=mapped, + attribution_status=("exact" if finding.rule_id else "unresolved"), + ) aggregate = aggregates.setdefault( (mapped, rule_id, finding_code), _RemediationCoverageAggregate( rule_id=rule_id, finding_code=finding_code, mapped=mapped, + actionability=decision.actionability.value, + actionability_reason=decision.reason, ), ) aggregate.occurrence_count += 1 aggregate.severity_counts[finding.severity.value] += 1 + max_severity = _ranked_counter_max( + aggregate.severity_counts, + _REMEDIATION_COVERAGE_SEVERITY_RANK, + ) + decision = classify_remediation_actionability( + rule_id=rule_id, + finding_code=finding_code, + severity=max_severity or finding.severity.value, + mapped=mapped, + attribution_status=("exact" if finding.rule_id else "unresolved"), + ) + aggregate.actionability = decision.actionability.value + aggregate.actionability_reason = decision.reason aggregate.target_axis_counts[ _target_axis_key(_target_axis_from_finding(finding)) ] += 1 @@ -1715,6 +2135,8 @@ def _remediation_coverage_group_payload( "rule_id": aggregate.rule_id, "finding_code": aggregate.finding_code, "mapped": aggregate.mapped, + "actionability": aggregate.actionability, + "actionability_reason": aggregate.actionability_reason, "max_severity": _ranked_counter_max( aggregate.severity_counts, _REMEDIATION_COVERAGE_SEVERITY_RANK, @@ -1734,12 +2156,13 @@ def _remediation_coverage_group_payload( def _remediation_coverage_group_sort_key( aggregate: _RemediationCoverageAggregate, -) -> tuple[int, int, int, str, str]: +) -> tuple[int, int, int, int, str, str]: max_severity = _ranked_counter_max( aggregate.severity_counts, _REMEDIATION_COVERAGE_SEVERITY_RANK, ) return ( + _REMEDIATION_ACTIONABILITY_SORT.get(aggregate.actionability, 9), -_REMEDIATION_COVERAGE_SEVERITY_RANK.get(max_severity or "", 0), -aggregate.occurrence_count, -len(aggregate.target_axis_counts), @@ -1822,13 +2245,34 @@ def _format_quality_remediation_coverage_group( f"{_optional_string_metadata(group, 'rule_id')}:" f"{_optional_string_metadata(group, 'finding_code')} " f"findings={_int_metadata(group, 'occurrence_count')} " - f"targets={_int_metadata(group, 'target_axis_count')}" + f"targets={_int_metadata(group, 'target_axis_count')} " + "actionability=" + f"{_optional_string_metadata(group, 'actionability')}" + f"({_optional_string_metadata(group, 'actionability_reason')})" ) def _format_quality_remediation_catalog_gap( gap: Mapping[str, JSONValue], ) -> str: + attribution = ( + _optional_string_metadata(gap, "attribution_status") or "unknown" + ) + attribution_reason = _optional_string_metadata( + gap, + "attribution_reason", + ) + attribution_text = f"attribution={attribution}" + if attribution_reason: + attribution_text += f"({attribution_reason})" + actionability = _optional_string_metadata(gap, "actionability") or "unknown" + actionability_reason = _optional_string_metadata( + gap, + "actionability_reason", + ) + actionability_text = f"actionability={actionability}" + if actionability_reason: + actionability_text += f"({actionability_reason})" return ( f"{_optional_string_metadata(gap, 'max_severity')} " f"rank={_int_metadata(gap, 'rank')} " @@ -1837,7 +2281,26 @@ def _format_quality_remediation_catalog_gap( "known=" f"{_int_metadata(gap, 'known_source_occurrence_count')} " "observed=" - f"{_int_metadata(gap, 'report_occurrence_count')}" + f"{_int_metadata(gap, 'report_occurrence_count')} " + f"{attribution_text} " + f"{actionability_text}" + ) + + +def _format_quality_remediation_plan_item( + item: Mapping[str, JSONValue], +) -> str: + selector = _mapping_payload(item.get("suggested_selector")) + action = _mapping_payload(item.get("suggested_action")) + fixability = _mapping_payload(item.get("fixability")) + return ( + f"rank={_int_metadata(item, 'rank')} " + f"{_optional_string_metadata(item, 'rule_id')}:" + f"{_optional_string_metadata(item, 'finding_code')} " + f"fixability={_optional_string_metadata(fixability, 'level')}" + f"/{_int_metadata(fixability, 'score')} " + f"selector={_optional_string_metadata(selector, 'shape')} " + f"action={_optional_string_metadata(action, 'action_kind')}" ) @@ -1888,6 +2351,24 @@ def _fingerprint_coverage_summary( ) +def _cross_series_fingerprint_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return run-scoped cross-series fingerprint report metadata.""" + return _optional_mapping_payload( + report.metadata.get(CROSS_SERIES_FINGERPRINT_METADATA_KEY) + ) + + +def _quality_engine_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return structured quality-engine metadata when evaluations were skipped.""" + return _optional_mapping_payload( + report.metadata.get(QUALITY_ENGINE_METADATA_KEY) + ) + + def _fingerprint_distribution_summary( report: QualityReport, ) -> dict[str, JSONValue] | None: @@ -1930,6 +2411,124 @@ def _fingerprint_regime_summary( ) +def _fingerprint_parity_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return cache/source parity metadata from report or findings.""" + summary = report.metadata.get( + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY + ) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload( + series_fingerprint_parity_summary(report.findings) + ) + + +def _synthetic_constraint_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return synthetic constraint metadata from report or findings.""" + summary = report.metadata.get(SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload( + synthetic_constraint_summary(report.findings) + ) + + +def _classical_baseline_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return classical baseline metadata from report or findings.""" + summary = report.metadata.get(CLASSICAL_BASELINE_SUMMARY_METADATA_KEY) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload( + classical_baseline_summary(report.findings) + ) + + +def _classical_model_input_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return classical model input metadata from report or findings.""" + summary = report.metadata.get(CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload( + classical_model_input_summary(report.findings) + ) + + +def _exponential_smoothing_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return exponential-smoothing metadata from report or findings.""" + summary = report.metadata.get(EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload( + exponential_smoothing_summary(report.findings) + ) + + +def _autoregressive_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return autoregressive-family metadata from report or findings.""" + summary = report.metadata.get(AUTOREGRESSIVE_SUMMARY_METADATA_KEY) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload(autoregressive_summary(report.findings)) + + +def _seasonal_exogenous_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return seasonal/exogenous metadata from report or findings.""" + summary = report.metadata.get(SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload( + seasonal_exogenous_summary(report.findings) + ) + + +def _state_space_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return state-space/Kalman metadata from report or findings.""" + summary = report.metadata.get(STATE_SPACE_SUMMARY_METADATA_KEY) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload(state_space_summary(report.findings)) + + +def _volatility_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return ARCH/GARCH metadata from report or findings.""" + summary = report.metadata.get(VOLATILITY_SUMMARY_METADATA_KEY) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload(volatility_summary(report.findings)) + + +def _classical_model_comparison_summary( + report: QualityReport, +) -> dict[str, JSONValue] | None: + """Return family-neutral comparison metadata from report or findings.""" + summary = report.metadata.get( + CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY + ) + if isinstance(summary, Mapping): + return dict(summary) + return _optional_mapping_payload( + classical_model_comparison_summary(report.findings) + ) + + def _fingerprint_topology_summary( report: QualityReport, ) -> dict[str, JSONValue] | None: @@ -2325,6 +2924,68 @@ def format_fingerprint_regime_summary_lines( return lines +def format_fingerprint_parity_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> list[str]: + """Return concise human-readable cache/source parity lines.""" + if not summary: + return [] + lines = [ + "", + "Fingerprint cache/source parity", + ( + "- targets: " + f"{_int_metadata(summary, 'target_count')} " + f"compared: {_int_metadata(summary, 'compared_target_count')} " + f"matching: {_int_metadata(summary, 'matching_target_count')} " + f"mismatched: {_int_metadata(summary, 'mismatched_target_count')} " + "not-compared: " + f"{_int_metadata(summary, 'not_compared_target_count')}" + ), + ] + for label, key in ( + ("statuses", "status_counts"), + ("mismatch codes", "mismatch_code_counts"), + ("computed from", "computed_from_counts"), + ("cache sources", "cache_source_counts"), + ("freshness", "freshness_counts"), + ): + counts = _format_count_metadata(summary.get(key)) + if counts: + lines.append(f"- {label}: {counts}") + targets = summary.get("target_summaries") + if isinstance(targets, list): + for item in targets: + if not isinstance(item, Mapping): + continue + axis = _mapping_payload(item.get("target_axis")) + codes = ",".join(_string_list_metadata(item.get("mismatch_codes"))) + suffix = f" codes={codes}" if codes else "" + lines.append( + "- " + f"{_format_target_axis(axis)}: " + f"{_string_metadata(item, 'status')} " + "sections=" + f"{_int_metadata(item, 'compared_section_count')} " + "mismatched=" + f"{_int_metadata(item, 'mismatched_section_count')}" + f"{suffix}" + ) + return lines + + +def _format_target_axis(axis: Mapping[str, JSONValue]) -> str: + return " ".join( + ( + _string_metadata(axis, "data_format"), + _string_metadata(axis, "symbol"), + _string_metadata(axis, "timeframe"), + _string_metadata(axis, "period"), + _string_metadata(axis, "kind"), + ) + ) + + def _format_fingerprint_regime_target_line( summary: Mapping[str, JSONValue], ) -> str: @@ -2461,6 +3122,9 @@ def format_fingerprint_topology_attention_lines( lines.append( f"- {_format_fingerprint_topology_attention_target_line(item)}" ) + lines.extend( + _format_fingerprint_topology_inspection_context_lines(item) + ) return lines @@ -2479,6 +3143,25 @@ def _format_fingerprint_topology_attention_target_line( cache_text = f", cache={cache_source}" if cache_source else "" hints = _remediation_hint_messages(summary.get("remediation_hints")) hint_text = f", next={'; '.join(hints)}" if hints else "" + policy = _mapping_payload(summary.get("calendar_policy")) + profile = _mapping_payload(policy.get("calendar_profile")) + weekend_policy = _optional_string_metadata( + policy, + "weekend_activity_policy", + ) + closure_policy = _optional_string_metadata( + policy, + "expected_session_closure_policy", + ) + profile_name = _optional_string_metadata(profile, "name") + policy_parts = [] + if weekend_policy and "weekend_activity" in flags: + policy_parts.append(f"weekend={weekend_policy}") + if closure_policy and "expected_session_closures" in flags: + policy_parts.append(f"closures={closure_policy}") + if profile_name and policy_parts: + policy_parts.append(f"profile={profile_name}") + policy_text = f", policy {' '.join(policy_parts)}" if policy_parts else "" return ( f"{data_format} {symbol} {timeframe} {period} {kind}: " f"{_string_metadata(summary, 'attention_level')}, " @@ -2492,10 +3175,89 @@ def _format_fingerprint_topology_attention_target_line( f"{_format_duration_ms(summary.get('max_gap_ms'))}, " f"computed_from={_string_metadata(summary, 'computed_from')}" f"{cache_text}" + f"{policy_text}" f"{hint_text}" ) +def _format_fingerprint_topology_inspection_context_lines( + summary: Mapping[str, JSONValue], +) -> list[str]: + context = _mapping_payload(summary.get("inspection_context")) + if not context: + return [] + lines: list[str] = [] + for section_name in ( + "invalid_timestamps", + "non_monotonic_timestamps", + "duplicate_timestamps", + "suspicious_gaps", + "expected_session_closures", + "weekend_activity", + ): + section = _mapping_payload(context.get(section_name)) + if not section: + continue + samples = _list_metadata(section.get("samples")) + evidence = "; ".join( + _format_fingerprint_topology_inspection_sample( + section_name, + sample, + ) + for sample in samples + ) + action = _mapping_payload(section.get("next_action")) + policy_note = _mapping_payload(section.get("policy_note")) + action_code = _optional_string_metadata(action, "code") + note_code = _optional_string_metadata(policy_note, "code") + if action_code: + action_text = f" action={action_code}" + elif note_code: + action_text = f" policy-note={note_code}" + else: + action_text = " contextual" + evidence_text = f" {evidence}" if evidence else "" + lines.append( + " - context " + f"{section_name}: " + f"samples={_int_metadata(section, 'included_count')}/" + f"{_int_metadata(section, 'total_count')} " + f"omitted={_int_metadata(section, 'omitted_count')}" + f"{action_text}{evidence_text}" + ) + return lines + + +def _format_fingerprint_topology_inspection_sample( + section_name: str, + sample: Mapping[str, JSONValue], +) -> str: + timestamp = _optional_string_metadata(sample, "timestamp_source") + row_number = _optional_int_text(sample.get("row_number")) + if section_name == "invalid_timestamps": + return f"row {row_number}={timestamp or 'unknown'}" + if section_name == "non_monotonic_timestamps": + metadata = _mapping_payload(sample.get("metadata")) + previous_row = _optional_int_text(metadata.get("previous_row_number")) + return f"rows {previous_row}->{row_number} at {timestamp or 'unknown'}" + if section_name == "duplicate_timestamps": + occurrences = _optional_int_text(sample.get("occurrence_count")) + return f"{timestamp or 'unknown'} x{occurrences}" + if section_name in {"suspicious_gaps", "expected_session_closures"}: + previous = _optional_string_metadata( + sample, + "previous_timestamp_source", + ) + expected = bool(sample.get("expected_session_related")) + session_text = "expected-session" if expected else "unexpected" + return ( + f"{previous or 'unknown'}->{timestamp or 'unknown'} " + f"({_format_duration_ms(sample.get('gap_ms'))}, {session_text})" + ) + session_state = _optional_string_metadata(sample, "session_state") + return f"{timestamp or 'unknown'} ({session_state or 'unknown'})" + + def format_fingerprint_topology_summary_lines( summary: Mapping[str, JSONValue] | None, ) -> list[str]: @@ -2665,11 +3427,53 @@ def format_fingerprint_readiness_summary_lines( f"{_int_metadata(summary, 'stationarity_skipped_window_count')}" ) lines.append(stationarity_line) + decomposition_counts = _format_count_metadata( + summary.get("decomposition_status_counts") + ) + if decomposition_counts: + decomposition_line = f"- decomposition: {decomposition_counts}" + decomposition_reasons = _format_count_metadata( + summary.get("decomposition_reason_counts") + ) + if decomposition_reasons: + decomposition_line += f" reasons: {decomposition_reasons}" + skipped_window_reasons = _format_count_metadata( + summary.get("decomposition_skipped_window_reason_counts") + ) + if skipped_window_reasons: + decomposition_line += ( + f" skipped-window reasons: {skipped_window_reasons}" + ) + basis_counts = _format_count_metadata( + summary.get("decomposition_basis_counts") + ) + if basis_counts: + decomposition_line += f" basis: {basis_counts}" + stationarity_counts = _format_count_metadata( + summary.get("decomposition_stationarity_status_counts") + ) + if stationarity_counts: + decomposition_line += f" stationarity: {stationarity_counts}" + structural_counts = _format_count_metadata( + summary.get("decomposition_structural_break_status_counts") + ) + if structural_counts: + decomposition_line += f" structural-breaks: {structural_counts}" + decomposition_line += ( + " computed_windows=" + f"{_int_metadata(summary, 'decomposition_computed_window_count')} " + "skipped_windows=" + f"{_int_metadata(summary, 'decomposition_skipped_window_count')} " + "structural_candidates=" + f"{_int_metadata(summary, 'decomposition_structural_break_candidate_count')}" + ) + lines.append(decomposition_line) count_lines = ( ("topology limitations", "topology_limitation_counts"), ("dynamics limitations", "dynamics_limitation_counts"), ("dependence limitations", "dependence_limitation_counts"), ("stationarity limitations", "stationarity_limitation_counts"), + ("decomposition limitations", "decomposition_limitation_counts"), ("row order", "row_order_counts"), ("computed from", "computed_from_counts"), ("cache sources", "cache_source_counts"), @@ -2751,6 +3555,50 @@ def format_fingerprint_readiness_risk_lines( return lines +def format_cross_series_fingerprint_lines( + summary: Mapping[str, JSONValue] | None, +) -> list[str]: + """Return concise human-readable cross-series fingerprint lines.""" + if not summary: + return [] + lines = [ + "", + "Cross-series fingerprint", + ( + "- status: " + f"{_string_metadata(summary, 'status')} " + f"series={_int_metadata(summary, 'fx_series_count')} " + f"groups={_int_metadata(summary, 'group_count')} " + f"included={_int_metadata(summary, 'included_group_count')} " + f"omitted={_int_metadata(summary, 'omitted_group_count')}" + ), + ] + triangular = _mapping_payload(summary.get("triangular_consistency")) + inverse = _mapping_payload(summary.get("inverse_consistency")) + stale = _mapping_payload(summary.get("stale_join_risk")) + lines.append( + "- consistency: " + f"triangles={_int_metadata(triangular, 'candidate_count')} " + f"triangle warnings={_int_metadata(triangular, 'warning_count')} " + f"triangle errors={_int_metadata(triangular, 'error_count')} " + f"inverse={_int_metadata(inverse, 'candidate_count')} " + f"stale joins={_int_metadata(stale, 'risk_count')}" + ) + for group in _list_metadata(summary.get("groups")): + grid = _mapping_payload(group.get("timestamp_grid")) + ranges = _mapping_payload(group.get("coverage_ranges")) + lines.append( + "- " + f"{_string_metadata(group, 'group_id')}: " + f"symbols={','.join(_string_list_metadata(group.get('symbols')))} " + f"common={_int_metadata(grid, 'common_timestamp_count')}/" + f"{_int_metadata(grid, 'union_timestamp_count')} " + f"ratio={_format_rate(grid.get('common_timestamp_ratio'))} " + f"unequal_ranges={_bool_text(ranges.get('unequal_ranges'))}" + ) + return lines + + def _format_fingerprint_readiness_risk_target( summary: Mapping[str, JSONValue], ) -> str: @@ -2804,6 +3652,12 @@ def _format_fingerprint_readiness_target_line( stationarity_text = ( f", {stationarity_details}" if stationarity_details else "" ) + decomposition_details = _format_fingerprint_readiness_decomposition_details( + _mapping_payload(summary.get("decomposition")) + ) + decomposition_text = ( + f", {decomposition_details}" if decomposition_details else "" + ) return ( f"{_string_metadata(axis, 'data_format')} " f"{_string_metadata(axis, 'symbol')} " @@ -2827,6 +3681,7 @@ def _format_fingerprint_readiness_target_line( f"{details_text}" f"{dependence_text}" f"{stationarity_text}" + f"{decomposition_text}" ) @@ -2941,6 +3796,44 @@ def _format_fingerprint_readiness_stationarity_details( ) +def _format_fingerprint_readiness_decomposition_details( + decomposition: Mapping[str, JSONValue], +) -> str: + if not decomposition: + return "" + status = _string_metadata(decomposition, "status") + reason = _optional_string_metadata(decomposition, "reason") + reason_text = f" reason={reason}" if reason else "" + skipped_reasons = _format_count_metadata( + decomposition.get("skipped_window_reason_counts") + ) + skipped_reason_text = ( + f" skipped_reasons={skipped_reasons}" if skipped_reasons else "" + ) + stationarity = _mapping_payload(decomposition.get("stationarity")) + structural = _mapping_payload(decomposition.get("structural_break")) + trend = _mapping_payload(decomposition.get("trend")) + projection = _mapping_payload(decomposition.get("training_projection")) + return ( + f"decomposition={status}{reason_text} " + f"basis={_string_metadata(decomposition, 'calculation_basis')} " + f"metric={_string_metadata(decomposition, 'metric')} " + f"samples={_int_metadata(decomposition, 'level_sample_count')}/" + f"{_int_metadata(decomposition, 'return_sample_count')} " + f"windows={_format_window_metadata(decomposition)} " + "computed_windows=" + f"{_int_metadata(decomposition, 'computed_window_count')} " + "skipped_windows=" + f"{_int_metadata(decomposition, 'skipped_window_count')} " + f"stationarity={_string_metadata(stationarity, 'status')} " + f"trend={_string_metadata(trend, 'direction')} " + f"structural={_string_metadata(structural, 'status')}/" + f"{_int_metadata(structural, 'candidate_count')} " + f"projection={_string_metadata(projection, 'grain')}" + f"{skipped_reason_text}" + ) + + def _format_window_metadata(stationarity: Mapping[str, JSONValue]) -> str: windows = _int_list_metadata(stationarity.get("windows")) return "[" + ",".join(str(window) for window in windows) + "]" diff --git a/src/histdatacom/data_quality/rules.py b/src/histdatacom/data_quality/rules.py index 8a8585c8..e03f584a 100644 --- a/src/histdatacom/data_quality/rules.py +++ b/src/histdatacom/data_quality/rules.py @@ -12,7 +12,10 @@ ) from histdatacom.data_quality.calendar import calendar_quality_rules from histdatacom.data_quality.discovery import normalize_quality_check_groups -from histdatacom.data_quality.fingerprints import HistDataSeriesFingerprintRule +from histdatacom.data_quality.fingerprints import ( + HistDataCrossSeriesFingerprintRule, + HistDataSeriesFingerprintRule, +) from histdatacom.data_quality.ingestion import ( HistDataAsciiRowCountIngestionRule, HistDataAsciiSchemaIngestionRule, @@ -28,6 +31,7 @@ ) from histdatacom.data_quality.provenance import provenance_quality_run_rules from histdatacom.data_quality.symbols import ( + CrossInstrumentScanProvider, HistDataCrossInstrumentConsistencyRule, domain_quality_rules, ) @@ -86,18 +90,33 @@ def quality_run_rules_for_groups( normalized = normalize_quality_check_groups(groups) quality_profile = quality_profile_from_value(profile) rules: list[QualityRunRule] = [] + shared_cross_instrument_scan = ( + CrossInstrumentScanProvider() if "all" in normalized else None + ) if "all" in normalized or "inventory" in normalized: rules.extend(manifest_quality_run_rules()) if "all" in normalized or "time" in normalized: rules.extend(_time_quality_run_rules(quality_profile)) if "all" in normalized or "domain" in normalized: - rules.extend(_domain_quality_run_rules(quality_profile)) + rules.extend( + _domain_quality_run_rules( + quality_profile, + scan_provider=shared_cross_instrument_scan, + ) + ) if "all" in normalized or "provenance" in normalized: rules.extend( provenance_quality_run_rules( explicit="provenance" in normalized and "all" not in normalized ) ) + if "all" in normalized or "fingerprint" in normalized: + rules.extend( + _fingerprint_quality_run_rules( + quality_profile, + scan_provider=shared_cross_instrument_scan, + ) + ) return tuple(rules) @@ -269,6 +288,8 @@ def _merged_thresholds( def _domain_quality_run_rules( profile: QualityProfile, + *, + scan_provider: CrossInstrumentScanProvider | None = None, ) -> tuple[QualityRunRule, ...]: return ( HistDataCrossInstrumentConsistencyRule( @@ -283,6 +304,7 @@ def _domain_quality_run_rules( "error_severity", QualitySeverity.ERROR, ), + scan_provider=scan_provider, ), ) @@ -310,3 +332,26 @@ def _fingerprint_quality_rules( profile=profile.fingerprint_profile(), ), ) + + +def _fingerprint_quality_run_rules( + profile: QualityProfile, + *, + scan_provider: CrossInstrumentScanProvider | None = None, +) -> tuple[QualityRunRule, ...]: + return ( + HistDataCrossSeriesFingerprintRule( + tolerance=profile.cross_instrument_tolerance(), + warning_severity=profile.severity( + "domain.cross_instrument_consistency", + "warning_severity", + QualitySeverity.WARNING, + ), + error_severity=profile.severity( + "domain.cross_instrument_consistency", + "error_severity", + QualitySeverity.ERROR, + ), + scan_provider=scan_provider, + ), + ) diff --git a/src/histdatacom/data_quality/seasonal_exogenous.py b/src/histdatacom/data_quality/seasonal_exogenous.py new file mode 100644 index 00000000..12f1ea7d --- /dev/null +++ b/src/histdatacom/data_quality/seasonal_exogenous.py @@ -0,0 +1,2267 @@ +"""Explicit SARIMA, ARIMAX, and SARIMAX fingerprint diagnostics. + +The family consumes the #421 regular-grid and rolling-origin contracts. All +exogenous values are deterministic calendar features known independently of +future market observations, and Statsmodels is imported lazily. +""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.metadata +import json +import math +import re +import time +import warnings +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from contextlib import nullcontext +from dataclasses import dataclass, field +from statistics import median +from typing import Any, cast + +from histdatacom.data_quality.autoregressive import ( + _covariance_condition_number, + _first_available_row_id, + _fit_converged, + _fitted_parameters, + _float, + _int, + _mapping, + _mapping_rows, + _optional_float, + _optional_int, + _rate, + _root_status, + _rounded, + _target_sort_key, + _text, + _warning_codes, +) +from histdatacom.data_quality.calendar import ( + SESSION_ASIA, + SESSION_LONDON, + SESSION_NEW_YORK, + SESSION_STATE_FRIDAY_CLOSE, + SESSION_STATE_MARKET_OPEN, + SESSION_STATE_SUNDAY_OPEN, + SESSION_STATE_WEEKEND_CLOSURE, + classify_histdata_timestamp, +) +from histdatacom.data_quality.calendar_profiles import ( + HistDataCalendarProfile, + default_calendar_profile, +) +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + ClassicalModelInputProfile, + ClassicalModelInputResult, + build_classical_model_input, +) +from histdatacom.data_quality.contracts import QualityFinding +from histdatacom.data_quality.exponential_smoothing import ( + _inverse_forecasts, + _reference_baseline_payloads, + _trailing_contiguous_indexes, +) +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.training_features import ( + SEASONAL_EXOGENOUS_COLUMNS, + SEASONAL_EXOGENOUS_FAMILY_COLUMN_SUFFIXES, + ensure_tick_training_features, +) +from histdatacom.runtime_contracts import JSONValue + +SEASONAL_EXOGENOUS_SCHEMA_VERSION = "histdatacom.seasonal-exogenous.v1" +SEASONAL_EXOGENOUS_CONFIGURATION_SCHEMA_VERSION = ( + "histdatacom.seasonal-exogenous-configuration.v1" +) +SEASONAL_EXOGENOUS_REGRESSOR_SCHEMA_VERSION = ( + "histdatacom.seasonal-exogenous-regressors.v1" +) +SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION = ( + "histdatacom.seasonal-exogenous-fit-result.v1" +) +SEASONAL_EXOGENOUS_FORECAST_SCHEMA_VERSION = ( + "histdatacom.seasonal-exogenous-forecast.v1" +) +SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION = ( + "histdatacom.seasonal-exogenous-evaluation.v1" +) +SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.seasonal-exogenous-training-projection.v1" +) +SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.seasonal-exogenous-summary.v1" +) +SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_seasonal_exogenous_summary" +) +SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY = "fingerprint_seasonal_exogenous" + +DEFAULT_SEASONAL_EXOGENOUS_SUMMARY_TARGET_LIMIT = 16 +DEFAULT_SEASONAL_EXOGENOUS_ROLLING_WINDOWS = (5, 20) +DEFAULT_SEASONAL_EXOGENOUS_ROUNDING_DIGITS = 12 +MAX_SEASONAL_EXOGENOUS_SPECIFICATIONS = 32 +MAX_SEASONAL_EXOGENOUS_ORDER = 64 +MAX_SEASONAL_PERIOD = 100_000 +MAX_SEASONAL_EXOGENOUS_REGRESSORS = 32 +MAX_SEASONAL_EXOGENOUS_FIXED_PARAMETERS = 32 + +SEASONAL_EXOGENOUS_FAMILIES = ("sarima", "arimax", "sarimax") +SEASONAL_EXOGENOUS_FAMILY_CODES = { + "sarima": 1, + "arimax": 2, + "sarimax": 3, +} +SEASONAL_EXOGENOUS_TREND_CODES = {"n": 0, "c": 1, "t": 2, "ct": 3} +SEASONAL_EXOGENOUS_INITIALIZATION_CODES = { + "default": 1, + "stationary": 2, + "approximate_diffuse": 3, +} +SEASONAL_EXOGENOUS_FIT_STATUS_CODES = { + "unavailable": 1, + "skipped": 2, + "failed": 3, + "limited": 4, + "fitted": 5, + "converged": 6, +} +SEASONAL_EXOGENOUS_REASON_CODES = { + "": 0, + "dependency_unavailable": 1, + "input_contract_unavailable": 2, + "insufficient_folds": 3, + "insufficient_history": 4, + "invalid_order": 5, + "invalid_configuration": 6, + "invalid_seasonality": 7, + "unknown_regressor": 8, + "future_regressor_unavailable": 9, + "partial_calendar_unavailable": 10, + "rank_deficient_regressors": 11, + "collinearity": 12, + "optimizer_failure": 13, + "singularity": 14, + "numerical_overflow": 15, + "numerical_failure": 16, + "resource_limit": 17, + "timeout": 18, + "backend_failure": 19, + "target_unavailable": 20, + "inverse_transform_unavailable": 21, + "zero_variance": 22, +} + +CALENDAR_REGRESSOR_NAMES = ( + "source_hour_sin", + "source_hour_cos", + "source_weekday_sin", + "source_weekday_cos", + "market_open", + "weekend_closure", + "sunday_open", + "friday_close", + "session_asia", + "session_london", + "session_new_york", + "overlap_asia_london", + "overlap_london_new_york", + "daily_rollover", + "london_fix", + "month_end", + "quarter_end", + "year_end", + "holiday_any", + "event_any", +) +PARTIAL_CALENDAR_REGRESSORS = {"holiday_any", "event_any"} + +_SPECIFICATION_ID = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") +_PARAMETER_NAME = re.compile(r"^[A-Za-z0-9_.()\[\]-]{1,96}$") +_TAG_REGRESSOR = re.compile(r"^tag:[a-z0-9][a-z0-9_.:-]{0,127}$") +_OPTIMIZERS = {"lbfgs", "bfgs", "powell", "nm", "cg", "ncg"} + + +@dataclass(frozen=True, slots=True) +class CalendarRegressorProfile: + """Bounded forecast-safe calendar-regressor controls.""" + + allow_partial_calendar: bool = True + require_complete_calendar_for: tuple[str, ...] = () + max_regressors: int = 16 + + def __post_init__(self) -> None: + if self.max_regressors < 1 or self.max_regressors > 64: + raise ValueError("max_regressors must be between 1 and 64") + unknown = set(self.require_complete_calendar_for) - ( + PARTIAL_CALENDAR_REGRESSORS + ) + if unknown: + raise ValueError("complete-calendar controls support holiday/event") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return stable JSON-compatible controls.""" + return { + "allow_partial_calendar": self.allow_partial_calendar, + "require_complete_calendar_for": list( + self.require_complete_calendar_for + ), + "max_regressors": self.max_regressors, + "market_observation_regressors_allowed": False, + "future_derived_regressors_allowed": False, + } + + +@dataclass(frozen=True, slots=True) +class SeasonalExogenousSpecification: + """One explicit SARIMA, ARIMAX, or SARIMAX configuration.""" + + specification_id: str + family: str + p: int + d: int = 0 + q: int = 0 + seasonal_p: int = 0 + seasonal_d: int = 0 + seasonal_q: int = 0 + seasonal_period: int = 0 + seasonal_cycle_ms: int = 0 + trend: str = "n" + initialization_method: str = "default" + optimizer: str = "lbfgs" + enforce_stationarity: bool = True + enforce_invertibility: bool = True + concentrate_scale: bool = False + use_exact_diffuse: bool = False + regressor_names: tuple[str, ...] = () + fixed_parameters: tuple[tuple[str, float], ...] = () + max_iterations: int = 200 + + def __post_init__(self) -> None: + if not _SPECIFICATION_ID.fullmatch(self.specification_id): + raise ValueError("invalid seasonal/exogenous specification_id") + if self.family not in SEASONAL_EXOGENOUS_FAMILIES: + raise ValueError("unsupported seasonal/exogenous family") + orders = { + "p": self.p, + "d": self.d, + "q": self.q, + "seasonal_p": self.seasonal_p, + "seasonal_d": self.seasonal_d, + "seasonal_q": self.seasonal_q, + } + for name, value in orders.items(): + if value < 0 or value > MAX_SEASONAL_EXOGENOUS_ORDER: + raise ValueError(f"{name} must be between 0 and 64") + has_seasonal = any((self.seasonal_p, self.seasonal_d, self.seasonal_q)) + has_exog = bool(self.regressor_names) + if self.family == "sarima" and (not has_seasonal or has_exog): + raise ValueError("SARIMA requires seasonality and no regressors") + if self.family == "arimax" and (has_seasonal or not has_exog): + raise ValueError("ARIMAX requires regressors and no seasonality") + if self.family == "sarimax" and (not has_seasonal or not has_exog): + raise ValueError("SARIMAX requires seasonality and regressors") + if has_seasonal: + if not 2 <= self.seasonal_period <= MAX_SEASONAL_PERIOD: + raise ValueError("seasonal_period must be between 2 and 100000") + if self.seasonal_cycle_ms < 1: + raise ValueError("seasonal_cycle_ms must be explicit") + elif self.seasonal_period or self.seasonal_cycle_ms: + raise ValueError("nonseasonal ARIMAX cannot set a seasonal cycle") + if self.trend not in SEASONAL_EXOGENOUS_TREND_CODES: + raise ValueError("trend must be n, c, t, or ct") + if self.d + self.seasonal_d > 0 and "c" in self.trend: + raise ValueError("integrated seasonal models cannot use constant") + if self.initialization_method not in ( + SEASONAL_EXOGENOUS_INITIALIZATION_CODES + ): + raise ValueError("unsupported initialization_method") + if ( + self.initialization_method == "stationary" + and self.d + self.seasonal_d > 0 + ): + raise ValueError( + "stationary initialization requires no differencing" + ) + if self.optimizer not in _OPTIMIZERS: + raise ValueError("unsupported seasonal/exogenous optimizer") + if not self.regressor_names and self.family in {"arimax", "sarimax"}: + raise ValueError("exogenous family requires regressors") + if len(self.regressor_names) > MAX_SEASONAL_EXOGENOUS_REGRESSORS: + raise ValueError("too many seasonal/exogenous regressors") + if len(set(self.regressor_names)) != len(self.regressor_names): + raise ValueError("regressor names must be unique") + for name in self.regressor_names: + if name not in CALENDAR_REGRESSOR_NAMES and not ( + _TAG_REGRESSOR.fullmatch(name) + ): + raise ValueError("unknown calendar regressor") + if len(self.fixed_parameters) > ( + MAX_SEASONAL_EXOGENOUS_FIXED_PARAMETERS + ): + raise ValueError("too many fixed seasonal/exogenous parameters") + seen: set[str] = set() + for name, fixed_value in self.fixed_parameters: + if ( + not _PARAMETER_NAME.fullmatch(name) + or name in seen + or not math.isfinite(fixed_value) + ): + raise ValueError( + "fixed parameters must be unique finite scalars" + ) + seen.add(name) + if self.max_iterations < 1: + raise ValueError("max_iterations must be positive") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return a stable serializable specification.""" + return { + "schema_version": SEASONAL_EXOGENOUS_CONFIGURATION_SCHEMA_VERSION, + "specification_id": self.specification_id, + "family": self.family, + "family_code": SEASONAL_EXOGENOUS_FAMILY_CODES[self.family], + "order": [self.p, self.d, self.q], + "seasonal_order": [ + self.seasonal_p, + self.seasonal_d, + self.seasonal_q, + self.seasonal_period, + ], + "seasonal_cycle_ms": self.seasonal_cycle_ms, + "trend": self.trend, + "trend_code": SEASONAL_EXOGENOUS_TREND_CODES[self.trend], + "initialization_method": self.initialization_method, + "optimizer": self.optimizer, + "enforce_stationarity": self.enforce_stationarity, + "enforce_invertibility": self.enforce_invertibility, + "concentrate_scale": self.concentrate_scale, + "use_exact_diffuse": self.use_exact_diffuse, + "regressor_names": list(self.regressor_names), + "fixed_parameters": cast( + JSONValue, + [ + {"parameter": name, "value": value} + for name, value in self.fixed_parameters + ], + ), + "max_iterations": self.max_iterations, + "automatic_order_selection": False, + "automatic_regressor_selection": False, + } + + +def _default_specifications() -> tuple[SeasonalExogenousSpecification, ...]: + return ( + SeasonalExogenousSpecification( + "sarima-1-0-0x1-0-0-60", + "sarima", + 1, + seasonal_p=1, + seasonal_period=60, + seasonal_cycle_ms=3_600_000, + trend="c", + ), + SeasonalExogenousSpecification( + "arimax-1-hour", + "arimax", + 1, + trend="c", + regressor_names=("source_hour_sin", "source_hour_cos"), + ), + SeasonalExogenousSpecification( + "sarimax-1-hour", + "sarimax", + 1, + seasonal_p=1, + seasonal_period=60, + seasonal_cycle_ms=3_600_000, + trend="c", + regressor_names=("source_hour_sin", "source_hour_cos"), + ), + ) + + +@dataclass(frozen=True, slots=True) +class SeasonalExogenousProfile: + """Explicit seasonal/exogenous controls; disabled by default.""" + + enabled: bool = False + specifications: tuple[SeasonalExogenousSpecification, ...] = field( + default_factory=_default_specifications + ) + projection_specification_ids: tuple[str, ...] = ( + "sarima-1-0-0x1-0-0-60", + "arimax-1-hour", + "sarimax-1-hour", + ) + projection_horizon: int = 1 + regressor_profile: CalendarRegressorProfile = field( + default_factory=CalendarRegressorProfile + ) + baseline_rolling_windows: tuple[int, ...] = ( + DEFAULT_SEASONAL_EXOGENOUS_ROLLING_WINDOWS + ) + compare_exponential_smoothing: bool = True + compare_autoregressive: bool = True + rounding_digits: int = DEFAULT_SEASONAL_EXOGENOUS_ROUNDING_DIGITS + + def __post_init__(self) -> None: + if not self.specifications: + raise ValueError( + "at least one seasonal/exogenous model is required" + ) + if len(self.specifications) > MAX_SEASONAL_EXOGENOUS_SPECIFICATIONS: + raise ValueError("too many seasonal/exogenous specifications") + identifiers = tuple( + item.specification_id for item in self.specifications + ) + if len(set(identifiers)) != len(identifiers): + raise ValueError( + "seasonal/exogenous specification IDs must be unique" + ) + selected = set(self.projection_specification_ids) + if not selected or not selected.issubset(identifiers): + raise ValueError("projection IDs must select configured models") + families = { + item.family + for item in self.specifications + if item.specification_id in selected + } + if len(families) != len(selected): + raise ValueError("select at most one projection per model family") + if self.projection_horizon < 1: + raise ValueError("projection_horizon must be positive") + if ( + not self.baseline_rolling_windows + or any(value < 1 for value in self.baseline_rolling_windows) + or tuple(sorted(set(self.baseline_rolling_windows))) + != self.baseline_rolling_windows + ): + raise ValueError("baseline windows must be sorted unique positives") + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between 0 and 16") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return stable profile metadata.""" + return { + "enabled": self.enabled, + "specifications": cast( + JSONValue, [item.to_metadata() for item in self.specifications] + ), + "projection_specification_ids": list( + self.projection_specification_ids + ), + "projection_horizon": self.projection_horizon, + "regressor_profile": self.regressor_profile.to_metadata(), + "baseline_rolling_windows": list(self.baseline_rolling_windows), + "compare_exponential_smoothing": self.compare_exponential_smoothing, + "compare_autoregressive": self.compare_autoregressive, + "rounding_digits": self.rounding_digits, + "automatic_order_selection": False, + "automatic_regressor_selection": False, + "automatic_winner": False, + } + + +@dataclass(frozen=True, slots=True) +class SeasonalExogenousResult: + """Bounded diagnostics plus durable row-key annotations.""" + + diagnostics: Mapping[str, JSONValue] + annotations: tuple[Mapping[str, Any], ...] + input_result: ClassicalModelInputResult + + +@dataclass(frozen=True, slots=True) +class _Backend: + version: str + sarimax: Any + + +@dataclass(frozen=True, slots=True) +class _FitOutcome: + status: str + reason: str + forecasts: tuple[float, ...] + parameters: Mapping[str, float] + warning_codes: tuple[str, ...] + converged: bool + stationary: bool | None + invertible: bool | None + ar_root_min_modulus: float | None + ma_root_min_modulus: float | None + covariance_condition_number: float | None + effective_observation_count: int + residual_summary: Mapping[str, JSONValue] + + +@dataclass(frozen=True, slots=True) +class _RegressorRow: + values: Mapping[str, float] + available: bool + availability: str + reason: str + active_sessions: tuple[str, ...] + special_tags: tuple[str, ...] + holiday_tags: tuple[str, ...] + event_tags: tuple[str, ...] + + +def seasonal_exogenous_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: SeasonalExogenousProfile | None = None, + calendar_profile: HistDataCalendarProfile | None = None, + exponential_smoothing: Mapping[str, JSONValue] | None = None, + autoregressive: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> SeasonalExogenousResult: + """Regularize an enriched tick frame and evaluate configured models.""" + selected_input = input_profile or ClassicalModelInputProfile(enabled=True) + input_result = build_classical_model_input( + frame, fingerprint, profile=selected_input, target=target + ) + return seasonal_exogenous_from_model_input( + frame, + input_result, + fingerprint, + input_profile=selected_input, + profile=profile, + calendar_profile=calendar_profile, + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + target=target, + ) + + +def seasonal_exogenous_from_model_input( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile, + profile: SeasonalExogenousProfile | None = None, + calendar_profile: HistDataCalendarProfile | None = None, + exponential_smoothing: Mapping[str, JSONValue] | None = None, + autoregressive: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> SeasonalExogenousResult: + """Fit explicit SARIMA, ARIMAX, and SARIMAX specifications.""" + selected = profile or SeasonalExogenousProfile(enabled=True) + selected_calendar = calendar_profile or default_calendar_profile() + rows = cast(list[dict[str, Any]], input_result.regularized_frame.to_dicts()) + regressors, regressor_contract = _calendar_regressors( + rows, selected, selected_calendar, target=target + ) + base = _base_payload( + input_result, + fingerprint, + selected, + regressor_contract, + selected_calendar, + ) + if selected.projection_horizon not in input_profile.horizons: + return _unavailable_result(input_result, base, "invalid_configuration") + if input_result.contract.get("status") == "unavailable": + return _unavailable_result( + input_result, base, "input_contract_unavailable" + ) + if not input_result.folds: + return _unavailable_result( + input_result, base, "insufficient_folds", status="limited" + ) + backend = _load_backend() + if backend is None: + return _unavailable_result(input_result, base, "dependency_unavailable") + return _evaluate_models( + frame, + input_result, + fingerprint, + input_profile, + selected, + selected_calendar, + regressors, + regressor_contract, + backend, + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + target=target, + ) + + +def seasonal_exogenous_diagnostics_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: SeasonalExogenousProfile | None = None, + calendar_profile: HistDataCalendarProfile | None = None, + target: Any | None = None, +) -> dict[str, JSONValue]: + """Return diagnostics without exposing the annotation wrapper.""" + return dict( + seasonal_exogenous_from_training_frame( + frame, + fingerprint, + input_profile=input_profile, + profile=profile, + calendar_profile=calendar_profile, + target=target, + ).diagnostics + ) + + +def project_seasonal_exogenous_onto_training_frame( + frame: Any, + result: SeasonalExogenousResult, + *, + target: Any | None = None, +) -> Any: + """Join bounded seasonal/exogenous annotations by durable row identity.""" + import polars as pl + + columns = set(getattr(frame, "columns", ())) + if not {"series_id", "period", "row_id"}.issubset(columns): + enriched = ensure_tick_training_features(frame, target=target) + else: + enriched = frame + left = enriched.drop( + [ + name + for name in SEASONAL_EXOGENOUS_COLUMNS + if name in enriched.columns + ] + ).with_row_index("__cm_seasonal_exogenous_original_order") + if result.annotations: + right = pl.DataFrame( + [dict(row) for row in result.annotations], infer_schema_length=None + ) + projected = left.join( + right, + on=["series_id", "period", "row_id"], + how="left", + validate="m:1", + ) + else: + projected = left + projected = _ensure_projection_columns(projected) + return projected.sort("__cm_seasonal_exogenous_original_order").drop( + "__cm_seasonal_exogenous_original_order" + ) + + +def seasonal_exogenous_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = DEFAULT_SEASONAL_EXOGENOUS_SUMMARY_TARGET_LIMIT, +) -> dict[str, JSONValue] | None: + """Return bounded report metadata for seasonal/exogenous results.""" + targets: list[dict[str, JSONValue]] = [] + statuses: Counter[str] = Counter() + for finding in findings: + fingerprint = _mapping(finding.metadata.get("time_series_fingerprint")) + payload = _mapping(fingerprint.get("seasonal_exogenous")) + if not payload: + continue + status = _text(payload.get("status")) or "unavailable" + evaluation = _mapping(payload.get("evaluation")) + fit_summary = _mapping(payload.get("fit_summary")) + statuses[status] += 1 + targets.append( + { + "target_axis": dict(_mapping(payload.get("target_axis"))), + "status": status, + "reason": payload.get("reason"), + "model_count": _int(evaluation.get("model_count")), + "fit_attempt_count": _int(fit_summary.get("fit_attempt_count")), + "evaluated_fold_count": _int( + evaluation.get("evaluated_fold_count") + ), + "failed_fit_count": _int(fit_summary.get("failed_fit_count")), + } + ) + if not targets: + return None + targets.sort(key=_target_sort_key) + limit = bounded_report_limit( + target_limit, + default_limit=DEFAULT_SEASONAL_EXOGENOUS_SUMMARY_TARGET_LIMIT, + allow_unbounded=True, + ) + included = limit.slice(targets) + omitted = len(targets) - len(included) + return { + "schema_version": SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION, + "advisory": True, + "target_count": len(targets), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "status_counts": dict(sorted(statuses.items())), + "target_summaries": cast(JSONValue, included), + "limit_metadata": {"targets": limit.limit_payload()}, + } + + +def format_seasonal_exogenous_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> tuple[str, ...]: + """Return concise human-readable seasonal/exogenous lines.""" + if not summary: + return () + statuses = _mapping(summary.get("status_counts")) + lines = [ + "", + "Seasonal and exogenous models", + ( + f"targets: {_int(summary.get('target_count'))} " + f"ready: {_int(statuses.get('ready'))} " + f"limited: {_int(statuses.get('limited'))} " + f"unavailable: {_int(statuses.get('unavailable'))}" + ), + ] + for item in _mapping_rows(summary.get("target_summaries")): + axis = _mapping(item.get("target_axis")) + label = "/".join( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + lines.append( + f"- {label}: {_text(item.get('status'))} " + f"models={_int(item.get('model_count'))} " + f"fits={_int(item.get('fit_attempt_count'))} " + f"folds={_int(item.get('evaluated_fold_count'))}" + ) + if summary.get("truncated") is True: + lines.append( + f"- {_int(summary.get('omitted_target_count'))} targets omitted" + ) + return tuple(lines) + + +def _calendar_regressors( + rows: Sequence[Mapping[str, Any]], + profile: SeasonalExogenousProfile, + calendar_profile: HistDataCalendarProfile, + *, + target: Any | None, +) -> tuple[tuple[_RegressorRow, ...], dict[str, JSONValue]]: + requested = tuple( + dict.fromkeys( + name + for specification in profile.specifications + for name in specification.regressor_names + ) + ) + tag_vocabulary = {item.tag for item in calendar_profile.date_tags} | { + item.tag for item in calendar_profile.window_tags + } + unknown = sorted( + name + for name in requested + if name.startswith("tag:") and name[4:] not in tag_vocabulary + ) + result: list[_RegressorRow] = [] + availability_counts: Counter[str] = Counter() + for row in rows: + timestamp = _int(row.get("cm_input_bin_start_utc_ms")) + classification = classify_histdata_timestamp( + timestamp, + calendar_profile=calendar_profile, + asset_class=( + _text(getattr(target, "metadata", {}).get("asset_class")) + if target is not None + else "" + ), + ) + source = classification.source_datetime + hour = source.hour + source.minute / 60.0 + weekday = float(source.weekday()) + values: dict[str, float] = { + "source_hour_sin": math.sin(2.0 * math.pi * hour / 24.0), + "source_hour_cos": math.cos(2.0 * math.pi * hour / 24.0), + "source_weekday_sin": math.sin(2.0 * math.pi * weekday / 7.0), + "source_weekday_cos": math.cos(2.0 * math.pi * weekday / 7.0), + "market_open": float( + classification.session_state == SESSION_STATE_MARKET_OPEN + ), + "weekend_closure": float( + classification.session_state == SESSION_STATE_WEEKEND_CLOSURE + ), + "sunday_open": float( + classification.session_state == SESSION_STATE_SUNDAY_OPEN + ), + "friday_close": float( + classification.session_state == SESSION_STATE_FRIDAY_CLOSE + ), + "session_asia": float( + SESSION_ASIA in classification.active_sessions + ), + "session_london": float( + SESSION_LONDON in classification.active_sessions + ), + "session_new_york": float( + SESSION_NEW_YORK in classification.active_sessions + ), + "overlap_asia_london": float( + "asia_london_overlap" in classification.overlaps + ), + "overlap_london_new_york": float( + "london_new_york_overlap" in classification.overlaps + ), + "daily_rollover": float( + "daily_rollover" in classification.special_tags + ), + "london_fix": float( + "london_4pm_fix_window" in classification.special_tags + ), + "month_end": float("month_end" in classification.special_tags), + "quarter_end": float("quarter_end" in classification.special_tags), + "year_end": float("year_end" in classification.special_tags), + "holiday_any": float(bool(classification.holiday_tags)), + "event_any": float(bool(classification.event_tags)), + } + for tag in tag_vocabulary: + values[f"tag:{tag}"] = float(tag in classification.calendar_tags) + reason = "" + availability = "complete" + if unknown: + availability, reason = "unavailable", "unknown_regressor" + elif ( + any(name in PARTIAL_CALENDAR_REGRESSORS for name in requested) + and not calendar_profile.complete + ): + availability = "partial" + required = set( + profile.regressor_profile.require_complete_calendar_for + ) + selected_partial = set(requested) & PARTIAL_CALENDAR_REGRESSORS + if required & selected_partial: + reason = "partial_calendar_unavailable" + elif not profile.regressor_profile.allow_partial_calendar: + reason = "partial_calendar_unavailable" + available = not reason + availability_counts[availability] += 1 + result.append( + _RegressorRow( + values=values, + available=available, + availability=availability, + reason=reason, + active_sessions=classification.active_sessions, + special_tags=classification.special_tags, + holiday_tags=classification.holiday_tags, + event_tags=classification.event_tags, + ) + ) + definitions = [ + { + "name": name, + "known_in_advance": True, + "market_observation": False, + "calculation_basis": ( + "configured_calendar_tag" + if name.startswith("tag:") + else "deterministic_calendar_classifier" + ), + } + for name in requested + ] + contract: dict[str, JSONValue] = { + "schema_version": SEASONAL_EXOGENOUS_REGRESSOR_SCHEMA_VERSION, + "status": "unavailable" if unknown else "ready", + "column_order": list(requested), + "regressor_count": len(requested), + "definitions": cast(JSONValue, definitions), + "supported_builtin_vocabulary": list(CALENDAR_REGRESSOR_NAMES), + "configured_tag_vocabulary": cast(JSONValue, sorted(tag_vocabulary)), + "unknown_regressors": cast(JSONValue, unknown), + "availability_counts": dict(sorted(availability_counts.items())), + "calendar_profile": calendar_profile.to_metadata(), + "calendar_profile_complete": calendar_profile.complete, + "future_values_derived_without_market_observations": True, + "future_observed_market_values_allowed": False, + "full_series_smoothed_state_allowed": False, + "provenance": { + "classifier": "histdatacom.data_quality.calendar", + "timestamp_basis": "regular_grid_bin_start_utc_ms", + "profile_source": calendar_profile.source, + "profile_version": calendar_profile.version, + }, + } + return tuple(result), contract + + +def _evaluate_models( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + input_profile: ClassicalModelInputProfile, + profile: SeasonalExogenousProfile, + calendar_profile: HistDataCalendarProfile, + regressors: Sequence[_RegressorRow], + regressor_contract: Mapping[str, JSONValue], + backend: _Backend, + *, + exponential_smoothing: Mapping[str, JSONValue] | None, + autoregressive: Mapping[str, JSONValue] | None, + target: Any | None, +) -> SeasonalExogenousResult: + rows = cast(list[dict[str, Any]], input_result.regularized_frame.to_dicts()) + folds = [dict(fold) for fold in input_result.folds] + origins = _folds_by_origin(folds) + resources = input_profile.resources + specifications = profile.specifications[: resources.max_candidate_orders] + limitations: list[str] = [] + estimated_memory = _estimated_memory(len(rows), len(folds), specifications) + if len(specifications) < len(profile.specifications): + limitations.append("resource_limit") + if estimated_memory > resources.max_memory_bytes: + limitations.append("resource_limit") + specifications = () + started = time.monotonic() + fit_attempt_count = 0 + all_evaluations: list[dict[str, Any]] = [] + model_payloads: list[dict[str, JSONValue]] = [] + all_fit_samples: list[dict[str, JSONValue]] = [] + fit_statuses: Counter[str] = Counter() + fit_reasons: Counter[str] = Counter() + warning_counts: Counter[str] = Counter() + for specification_code, specification in enumerate(specifications, start=1): + configuration_reason = _configuration_reason( + specification, input_profile, profile, regressor_contract + ) + model_id = _model_id( + specification, + _text(input_result.contract.get("derivation_id")), + backend.version, + calendar_profile, + ) + model_evaluations: list[dict[str, Any]] = [] + model_fit_samples: list[dict[str, JSONValue]] = [] + for _, origin_folds in origins: + if fit_attempt_count >= resources.max_fit_attempts: + limitations.append("resource_limit") + break + if time.monotonic() - started >= resources.max_wall_time_seconds: + limitations.append("timeout") + break + fit_attempt_count += 1 + origin = origin_folds[0] + training_start = _int(origin.get("training_start_index")) + training_end = _int(origin.get("training_end_index")) + indexes = _trailing_contiguous_indexes( + rows, + training_start, + training_end, + _text(origin.get("series_id")), + _text(origin.get("period")), + ) + values = tuple( + _float(rows[index].get("cm_input_value")) for index in indexes + ) + max_horizon = max( + _int(fold.get("horizon")) for fold in origin_folds + ) + future_indexes = tuple( + index + for index in range( + training_end + 1, training_end + max_horizon + 1 + ) + if index < len(rows) + ) + train_exog, exog_reason = _matrix_for_indexes( + regressors, indexes, specification.regressor_names + ) + future_exog, future_reason = _matrix_for_indexes( + regressors, future_indexes, specification.regressor_names + ) + reason = configuration_reason or exog_reason or future_reason + if ( + len(future_indexes) < max_horizon + and specification.regressor_names + ): + reason = reason or "future_regressor_unavailable" + if reason: + outcome = _empty_fit( + "skipped", + reason, + _effective_count(specification, len(values)), + ) + else: + outcome = _fit_specification( + specification, + values, + train_exog, + future_exog, + max_horizon, + backend, + ) + fit_statuses[outcome.status] += 1 + if outcome.reason: + fit_reasons[outcome.reason] += 1 + warning_counts.update(outcome.warning_codes) + fit_sample = _fit_sample( + outcome, + specification, + model_id, + origin, + indexes, + len(values), + ) + model_fit_samples.append(fit_sample) + all_fit_samples.append(fit_sample) + original_forecasts = _inverse_forecasts( + rows, + training_end, + outcome.forecasts, + input_profile, + profile.rounding_digits, + ) + for fold in origin_folds: + evaluation = _fold_evaluation( + rows, + regressors, + fold, + specification, + specification_code, + model_id, + outcome, + original_forecasts, + profile.rounding_digits, + ) + model_evaluations.append(evaluation) + all_evaluations.append(evaluation) + model_payloads.append( + _model_evaluation_payload( + specification, + specification_code, + model_id, + model_evaluations, + model_fit_samples, + resources.max_retained_diagnostics, + profile.rounding_digits, + ) + ) + if limitations and limitations[-1] in {"resource_limit", "timeout"}: + break + baselines = _reference_baseline_payloads( + rows, + folds, + profile.baseline_rolling_windows, + profile.rounding_digits, + ) + references = { + "exponential_smoothing": _model_references( + exponential_smoothing, + enabled=profile.compare_exponential_smoothing, + unavailable_reason="exponential_smoothing_not_enabled", + ), + "autoregressive": _model_references( + autoregressive, + enabled=profile.compare_autoregressive, + unavailable_reason="autoregressive_not_enabled", + ), + } + annotations, collisions = _build_annotations( + frame, all_evaluations, profile, input_result, target=target + ) + evaluated_count = sum( + item.get("status") == "evaluated" for item in all_evaluations + ) + failed_count = sum( + status in {"failed", "unavailable"} + for status in fit_statuses.elements() + ) + limited_count = fit_statuses["limited"] + fit_statuses["skipped"] + if not evaluated_count: + limitations.append("insufficient_history") + limitations = list(dict.fromkeys(limitations)) + status = ( + "ready" + if not limitations and not failed_count and not limited_count + else "limited" + ) + final_reason = ( + limitations[0] + if limitations + else sorted(fit_reasons)[0] if fit_reasons else None + ) + diagnostics: dict[str, JSONValue] = { + **_base_payload( + input_result, + fingerprint, + profile, + regressor_contract, + calendar_profile, + ), + "status": status, + "reason": final_reason, + "limitations": cast(JSONValue, limitations), + "backend": { + "provider": "statsmodels", + "version": backend.version, + "available": True, + "model_class": "statsmodels.tsa.statespace.SARIMAX", + "import_basis": "optional_models_extra", + }, + "fit_summary": { + "schema_version": SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION, + "fit_attempt_count": fit_attempt_count, + "status_counts": dict(sorted(fit_statuses.items())), + "reason_counts": dict(sorted(fit_reasons.items())), + "warning_counts": dict(sorted(warning_counts.items())), + "failed_fit_count": failed_count, + "limited_fit_count": limited_count, + "convergence_rate": _rate( + fit_statuses["converged"], + fit_attempt_count, + profile.rounding_digits, + ), + "failure_rate": _rate( + failed_count, fit_attempt_count, profile.rounding_digits + ), + "fit_samples": cast( + JSONValue, all_fit_samples[: resources.max_retained_diagnostics] + ), + "fit_samples_truncated": ( + len(all_fit_samples) > resources.max_retained_diagnostics + ), + }, + "resource_usage": { + "limits": resources.to_metadata(), + "estimated_working_memory_bytes": estimated_memory, + "memory_limit_exceeded": estimated_memory + > resources.max_memory_bytes, + "fit_attempt_count": fit_attempt_count, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "evaluation": { + "schema_version": SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin_calendar_exog", + "original_scale": True, + "model_count": len(model_payloads), + "fold_count": len(folds), + "evaluated_fold_count": evaluated_count, + "skipped_evaluation_count": len(all_evaluations) - evaluated_count, + "forecast_coverage_rate": _rate( + evaluated_count, len(all_evaluations), profile.rounding_digits + ), + "models": cast(JSONValue, model_payloads), + "reference_baselines": cast(JSONValue, baselines), + "reference_models": cast(JSONValue, references), + "regime_error_summary": _regime_error_summary( + all_evaluations, profile.rounding_digits + ), + "comparison_semantics": "descriptive_shared_folds_only", + "automatic_winner": False, + }, + "training_projection": _training_projection_metadata( + profile, input_result, len(annotations), collisions + ), + "fit_duration_included": False, + } + return SeasonalExogenousResult(diagnostics, annotations, input_result) + + +def _configuration_reason( + specification: SeasonalExogenousSpecification, + input_profile: ClassicalModelInputProfile, + profile: SeasonalExogenousProfile, + regressor_contract: Mapping[str, JSONValue], +) -> str: + if specification.seasonal_period: + expected = specification.seasonal_period * input_profile.frequency_ms + if expected != specification.seasonal_cycle_ms: + return "invalid_seasonality" + if ( + len(specification.regressor_names) + > profile.regressor_profile.max_regressors + ): + return "resource_limit" + unknown = set( + cast(list[str], regressor_contract.get("unknown_regressors", [])) + ) + if unknown & set(specification.regressor_names): + return "unknown_regressor" + return "" + + +def _matrix_for_indexes( + rows: Sequence[_RegressorRow], + indexes: Sequence[int], + names: Sequence[str], +) -> tuple[tuple[tuple[float, ...], ...] | None, str]: + if not names: + return None, "" + matrix: list[tuple[float, ...]] = [] + for index in indexes: + if index < 0 or index >= len(rows): + return None, "future_regressor_unavailable" + row = rows[index] + if not row.available: + return None, row.reason or "future_regressor_unavailable" + try: + matrix.append(tuple(float(row.values[name]) for name in names)) + except KeyError: + return None, "unknown_regressor" + return tuple(matrix), "" + + +def _fit_specification( + specification: SeasonalExogenousSpecification, + values: Sequence[float], + train_exog: Sequence[Sequence[float]] | None, + future_exog: Sequence[Sequence[float]] | None, + horizon: int, + backend: _Backend, +) -> _FitOutcome: + effective_count = _effective_count(specification, len(values)) + minimum = max( + 8, + specification.p + + specification.q + + specification.d + + ( + specification.seasonal_p + + specification.seasonal_q + + specification.seasonal_d + ) + * max(1, specification.seasonal_period) + + 3, + ) + if len(values) < minimum or effective_count < 3: + return _empty_fit("skipped", "insufficient_history", effective_count) + if max(values) - min(values) <= 1e-15: + return _empty_fit("skipped", "zero_variance", effective_count) + if train_exog is not None: + rank_reason = _regressor_rank_reason(train_exog) + if rank_reason: + return _empty_fit("skipped", rank_reason, effective_count) + try: + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + fitted = _statsmodels_fit( + specification, values, train_exog, backend + ) + raw_forecasts = fitted.forecast(steps=horizon, exog=future_exog) + stationary, ar_min = _root_status(getattr(fitted, "arroots", ())) + invertible, ma_min = _root_status(getattr(fitted, "maroots", ())) + condition = _covariance_condition_number(fitted) + residual_summary = _residual_summary( + getattr(fitted, "resid", ()), 12 + ) + forecasts = tuple(float(value) for value in raw_forecasts) + warning_codes = _warning_codes(captured) + if not forecasts or any( + not math.isfinite(value) for value in forecasts + ): + return _empty_fit( + "failed", + "numerical_failure", + effective_count, + warning_codes=warning_codes, + ) + converged = _fit_converged(fitted, warning_codes) + status = ( + "limited" + if "convergence_warning" in warning_codes or not converged + else "converged" + ) + return _FitOutcome( + status=status, + reason="optimizer_failure" if status == "limited" else "", + forecasts=forecasts, + parameters=_fitted_parameters(fitted), + warning_codes=warning_codes, + converged=converged, + stationary=stationary, + invertible=invertible, + ar_root_min_modulus=ar_min, + ma_root_min_modulus=ma_min, + covariance_condition_number=condition, + effective_observation_count=effective_count, + residual_summary=residual_summary, + ) + except (ArithmeticError, FloatingPointError, OverflowError): + return _empty_fit("failed", "numerical_overflow", effective_count) + except Exception as exc: + return _empty_fit( + "failed", _backend_failure_reason(exc), effective_count + ) + + +def _statsmodels_fit( + specification: SeasonalExogenousSpecification, + values: Sequence[float], + exog: Sequence[Sequence[float]] | None, + backend: _Backend, +) -> Any: + model = backend.sarimax( + values, + exog=exog, + order=(specification.p, specification.d, specification.q), + seasonal_order=( + specification.seasonal_p, + specification.seasonal_d, + specification.seasonal_q, + specification.seasonal_period, + ), + trend=specification.trend, + enforce_stationarity=specification.enforce_stationarity, + enforce_invertibility=specification.enforce_invertibility, + concentrate_scale=specification.concentrate_scale, + use_exact_diffuse=specification.use_exact_diffuse, + simple_differencing=False, + missing="raise", + validate_specification=True, + ) + if specification.initialization_method == "stationary": + model.initialize_stationary() + elif specification.initialization_method == "approximate_diffuse": + model.initialize_approximate_diffuse() + fixed = dict(specification.fixed_parameters) + unknown = sorted(set(fixed) - set(model.param_names)) + if unknown: + raise ValueError("fixed parameter is not present in model") + context = model.fix_params(fixed) if fixed else nullcontext() + with context: + return model.fit( + method=specification.optimizer, + maxiter=specification.max_iterations, + disp=False, + low_memory=False, + ) + + +def _regressor_rank_reason(matrix: Sequence[Sequence[float]]) -> str: + if not matrix or not matrix[0]: + return "" + try: + numpy = importlib.import_module("numpy") + array = numpy.asarray(matrix, dtype=float) + if not numpy.isfinite(array).all(): + return "future_regressor_unavailable" + rank = int(numpy.linalg.matrix_rank(array)) + if rank < array.shape[1]: + return "rank_deficient_regressors" + condition = float(numpy.linalg.cond(array)) + if not math.isfinite(condition) or condition > 1e12: + return "collinearity" + except (TypeError, ValueError): + return "future_regressor_unavailable" + return "" + + +def _empty_fit( + status: str, + reason: str, + effective_count: int, + *, + warning_codes: tuple[str, ...] = (), +) -> _FitOutcome: + return _FitOutcome( + status, + reason, + (), + {}, + warning_codes, + False, + None, + None, + None, + None, + None, + effective_count, + {}, + ) + + +def _effective_count( + specification: SeasonalExogenousSpecification, count: int +) -> int: + loss = ( + specification.d + + max(specification.p, specification.q) + + specification.seasonal_d * specification.seasonal_period + + max(specification.seasonal_p, specification.seasonal_q) + * specification.seasonal_period + ) + return max(0, count - loss) + + +def _fit_sample( + outcome: _FitOutcome, + specification: SeasonalExogenousSpecification, + model_id: str, + fold: Mapping[str, Any], + indexes: Sequence[int], + observation_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION, + "model_id": model_id, + "specification_id": specification.specification_id, + "family": specification.family, + "order": [specification.p, specification.d, specification.q], + "seasonal_order": [ + specification.seasonal_p, + specification.seasonal_d, + specification.seasonal_q, + specification.seasonal_period, + ], + "seasonal_cycle_ms": specification.seasonal_cycle_ms, + "regressor_names": list(specification.regressor_names), + "status": outcome.status, + "reason": outcome.reason or None, + "converged": outcome.converged, + "stationary": outcome.stationary, + "invertible": outcome.invertible, + "ar_root_min_modulus": _rounded(outcome.ar_root_min_modulus, 12), + "ma_root_min_modulus": _rounded(outcome.ma_root_min_modulus, 12), + "covariance_condition_number": _rounded( + outcome.covariance_condition_number, 12 + ), + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "observation_count": observation_count, + "effective_observation_count": outcome.effective_observation_count, + "nonseasonal_differencing_loss": specification.d, + "seasonal_differencing_loss": ( + specification.seasonal_d * specification.seasonal_period + ), + "segment_start_index": indexes[0] if indexes else None, + "segment_end_index": indexes[-1] if indexes else None, + "training_segment_policy": "trailing_contiguous_after_missing", + "parameters": dict(outcome.parameters), + "parameter_count": len(outcome.parameters), + "residual_summary": dict(outcome.residual_summary), + "warning_codes": cast(JSONValue, list(outcome.warning_codes)), + "fit_duration_included": False, + "backend_exception_text_included": False, + } + + +def _fold_evaluation( + rows: Sequence[Mapping[str, Any]], + regressors: Sequence[_RegressorRow], + fold: Mapping[str, Any], + specification: SeasonalExogenousSpecification, + specification_code: int, + model_id: str, + outcome: _FitOutcome, + original_forecasts: Sequence[float | None], + rounding_digits: int, +) -> dict[str, Any]: + horizon = _int(fold.get("horizon")) + forecast = ( + original_forecasts[horizon - 1] + if 0 < horizon <= len(original_forecasts) + else None + ) + transformed = ( + outcome.forecasts[horizon - 1] + if 0 < horizon <= len(outcome.forecasts) + else None + ) + target_index = _int(fold.get("target_index")) + actual = ( + _optional_float(rows[target_index].get("cm_input_observed_value")) + if 0 <= target_index < len(rows) + else None + ) + target_regressor = ( + regressors[target_index] + if 0 <= target_index < len(regressors) + else None + ) + if outcome.status in {"failed", "skipped", "unavailable"}: + status, reason = "not_evaluated", outcome.reason + elif fold.get("status") != "valid" or actual is None: + status, reason = "skipped", "target_unavailable" + elif forecast is None: + status, reason = "skipped", "inverse_transform_unavailable" + else: + status, reason = "evaluated", "" + error = ( + forecast - actual + if status == "evaluated" and forecast is not None and actual is not None + else None + ) + return { + "schema_version": SEASONAL_EXOGENOUS_FORECAST_SCHEMA_VERSION, + "status": status, + "reason": reason or None, + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "family_code": SEASONAL_EXOGENOUS_FAMILY_CODES[specification.family], + "p": specification.p, + "d": specification.d, + "q": specification.q, + "seasonal_p": specification.seasonal_p, + "seasonal_d": specification.seasonal_d, + "seasonal_q": specification.seasonal_q, + "seasonal_period": specification.seasonal_period, + "seasonal_cycle_ms": specification.seasonal_cycle_ms, + "trend": specification.trend, + "trend_code": SEASONAL_EXOGENOUS_TREND_CODES[specification.trend], + "regressor_names": list(specification.regressor_names), + "regressor_set_code": _regressor_set_code( + specification.regressor_names + ), + "regressor_count": len(specification.regressor_names), + "regressor_available": ( + target_regressor.available if target_regressor else False + ), + "regressor_availability": ( + target_regressor.availability if target_regressor else "unavailable" + ), + "target_active_sessions": list( + target_regressor.active_sessions if target_regressor else () + ), + "target_special_tags": list( + target_regressor.special_tags if target_regressor else () + ), + "fit_status": outcome.status, + "fit_reason": outcome.reason or None, + "converged": outcome.converged, + "stationary": outcome.stationary, + "invertible": outcome.invertible, + "ar_root_min_modulus": _rounded( + outcome.ar_root_min_modulus, rounding_digits + ), + "ma_root_min_modulus": _rounded( + outcome.ma_root_min_modulus, rounding_digits + ), + "covariance_condition_number": _rounded( + outcome.covariance_condition_number, rounding_digits + ), + "effective_observation_count": outcome.effective_observation_count, + "fold_id": _int(fold.get("fold_id")), + "origin_row_id": fold.get("origin_row_id"), + "target_row_id": fold.get("target_row_id"), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "target_bin_end_utc_ms": _int(fold.get("target_bin_end_utc_ms")), + "horizon": horizon, + "transformed_forecast": _rounded(transformed, rounding_digits), + "forecast": _rounded(forecast, rounding_digits), + "actual": _rounded(actual, rounding_digits), + "error": _rounded(error, rounding_digits), + "absolute_error": _rounded( + abs(error) if error is not None else None, rounding_digits + ), + "squared_error": _rounded( + error * error if error is not None else None, rounding_digits + ), + "original_scale": True, + "future_values_visible": False, + "future_market_values_used_as_regressors": False, + "full_series_smoothed_state_used": False, + "residual_state_reused_across_origins": False, + "automatic_winner": False, + } + + +def _model_evaluation_payload( + specification: SeasonalExogenousSpecification, + specification_code: int, + model_id: str, + evaluations: Sequence[Mapping[str, Any]], + fit_samples: Sequence[Mapping[str, JSONValue]], + retained_limit: int, + rounding_digits: int, +) -> dict[str, JSONValue]: + statuses = Counter(_text(row.get("status")) for row in evaluations) + fit_statuses = Counter(_text(row.get("status")) for row in fit_samples) + reasons = Counter( + _text(row.get("reason")) + for row in evaluations + if _text(row.get("reason")) + ) + horizons = sorted({_int(row.get("horizon")) for row in evaluations}) + return { + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "configuration": specification.to_metadata(), + "status": "ready" if statuses["evaluated"] else "limited", + "fit_status_counts": dict(sorted(fit_statuses.items())), + "evaluation_status_counts": dict(sorted(statuses.items())), + "reason_counts": dict(sorted(reasons.items())), + "forecast_coverage_rate": _rate( + statuses["evaluated"], len(evaluations), rounding_digits + ), + "convergence_rate": _rate( + fit_statuses["converged"], len(fit_samples), rounding_digits + ), + "horizon_metrics": cast( + JSONValue, + [ + _metrics_for_horizon(evaluations, horizon, rounding_digits) + for horizon in horizons + ], + ), + "parameter_stability": _parameter_stability( + fit_samples, rounding_digits + ), + "fold_results": cast( + JSONValue, [dict(row) for row in evaluations[:retained_limit]] + ), + "fold_results_truncated": len(evaluations) > retained_limit, + "fit_samples": cast( + JSONValue, [dict(row) for row in fit_samples[:retained_limit]] + ), + "fit_samples_truncated": len(fit_samples) > retained_limit, + "automatic_winner": False, + } + + +def _metrics_for_horizon( + evaluations: Sequence[Mapping[str, Any]], + horizon: int, + rounding_digits: int, +) -> dict[str, JSONValue]: + selected = [ + row + for row in evaluations + if _int(row.get("horizon")) == horizon + and row.get("status") == "evaluated" + ] + errors = [_float(row.get("error")) for row in selected] + absolute = [abs(value) for value in errors] + squared = [value * value for value in errors] + return { + "horizon": horizon, + "evaluation_count": len(selected), + "mae": _rounded( + sum(absolute) / len(absolute) if absolute else None, rounding_digits + ), + "rmse": _rounded( + math.sqrt(sum(squared) / len(squared)) if squared else None, + rounding_digits, + ), + "bias": _rounded( + sum(errors) / len(errors) if errors else None, rounding_digits + ), + "original_scale": True, + } + + +def _parameter_stability( + fit_samples: Sequence[Mapping[str, JSONValue]], rounding_digits: int +) -> dict[str, JSONValue]: + parameters: dict[str, list[float]] = {} + for sample in fit_samples: + for name, raw in _mapping(sample.get("parameters")).items(): + value = _optional_float(raw) + if value is not None: + parameters.setdefault(name, []).append(value) + summaries: dict[str, JSONValue] = {} + for name, values in sorted(parameters.items()): + center = median(values) + summaries[name] = { + "count": len(values), + "min": _rounded(min(values), rounding_digits), + "max": _rounded(max(values), rounding_digits), + "median": _rounded(center, rounding_digits), + "mad": _rounded( + median(abs(value - center) for value in values), rounding_digits + ), + } + return { + "parameter_count": len(summaries), + "parameters": summaries, + "bounded": True, + } + + +def _residual_summary(raw: Any, digits: int) -> dict[str, JSONValue]: + values = [ + value for item in raw if (value := _optional_float(item)) is not None + ] + if not values: + return {"count": 0, "mean": None, "std": None, "max_abs": None} + center = sum(values) / len(values) + variance = sum((value - center) ** 2 for value in values) / len(values) + return { + "count": len(values), + "mean": _rounded(center, digits), + "std": _rounded(math.sqrt(variance), digits), + "max_abs": _rounded(max(abs(value) for value in values), digits), + } + + +def _model_references( + payload: Mapping[str, JSONValue] | None, + *, + enabled: bool, + unavailable_reason: str, +) -> list[dict[str, JSONValue]]: + if not enabled: + return [{"status": "not_requested", "automatic_winner": False}] + selected = _mapping(payload) + models = _mapping_rows(_mapping(selected.get("evaluation")).get("models")) + if not models: + return [ + { + "status": "unavailable", + "reason": unavailable_reason, + "automatic_winner": False, + } + ] + return [ + { + "status": model.get("status"), + "family": model.get("family"), + "specification_id": model.get("specification_id"), + "model_id": model.get("model_id"), + "horizon_metrics": model.get("horizon_metrics"), + "calculation_basis": "shared_regular_grid_folds", + "automatic_winner": False, + } + for model in models[:MAX_SEASONAL_EXOGENOUS_SPECIFICATIONS] + ] + + +def _regime_error_summary( + evaluations: Sequence[Mapping[str, Any]], digits: int +) -> dict[str, JSONValue]: + sessions: dict[str, list[float]] = {} + tags: dict[str, list[float]] = {} + for row in evaluations: + error = _optional_float(row.get("error")) + if error is None or row.get("status") != "evaluated": + continue + for session in cast(list[str], row.get("target_active_sessions", [])): + sessions.setdefault(session, []).append(abs(error)) + for tag in cast(list[str], row.get("target_special_tags", [])): + tags.setdefault(tag, []).append(abs(error)) + return { + "by_active_session": cast( + JSONValue, _bounded_error_groups(sessions, digits) + ), + "by_special_tag": cast(JSONValue, _bounded_error_groups(tags, digits)), + "causal_interpretation": False, + } + + +def _bounded_error_groups( + groups: Mapping[str, Sequence[float]], digits: int +) -> list[dict[str, JSONValue]]: + return [ + { + "name": name, + "count": len(values), + "mae": _rounded(sum(values) / len(values), digits), + } + for name, values in sorted(groups.items())[:16] + if values + ] + + +def _build_annotations( + frame: Any | None, + evaluations: Sequence[Mapping[str, Any]], + profile: SeasonalExogenousProfile, + input_result: ClassicalModelInputResult, + *, + target: Any | None, +) -> tuple[tuple[Mapping[str, Any], ...], int]: + if frame is None: + return (), 0 + try: + enriched = ensure_tick_training_features(frame, target=target) + except (AttributeError, TypeError, ValueError): + return (), 0 + source_rows = cast(list[dict[str, Any]], enriched.to_dicts()) + availability: dict[tuple[str, str], list[tuple[int, int]]] = {} + for row in source_rows: + timestamp = _optional_int(row.get("timestamp_utc_ms")) + row_id = _optional_int(row.get("row_id")) + if timestamp is not None and row_id is not None: + availability.setdefault( + (_text(row.get("series_id")), _text(row.get("period"))), [] + ).append((timestamp, row_id)) + for values in availability.values(): + values.sort() + merged: dict[tuple[str, str, int], dict[str, Any]] = {} + family_folds: dict[tuple[str, str, int, str], int] = {} + collisions = 0 + selected = [ + row + for row in evaluations + if _text(row.get("specification_id")) + in profile.projection_specification_ids + and _int(row.get("horizon")) == profile.projection_horizon + ] + selected.sort( + key=lambda row: ( + _text(row.get("series_id")), + _text(row.get("period")), + _int(row.get("origin_bin_end_utc_ms")), + _int(row.get("fold_id")), + _text(row.get("family")), + ) + ) + for evaluation in selected: + group = ( + _text(evaluation.get("series_id")), + _text(evaluation.get("period")), + ) + family = _text(evaluation.get("family")) + for diagnostic, value_key, time_key in ( + (False, "forecast", "origin_bin_end_utc_ms"), + (True, "error", "target_bin_end_utc_ms"), + ): + if ( + diagnostic + and _optional_float(evaluation.get(value_key)) is None + ): + continue + row_id = _first_available_row_id( + availability.get(group, ()), _int(evaluation.get(time_key)) + ) + if row_id is None: + continue + key = (*group, row_id) + family_key = (*key, family) + current_fold = _int(evaluation.get("fold_id")) + annotation = _annotation_row( + evaluation, input_result, row_id, diagnostic=diagnostic + ) + existing = merged.setdefault( + key, + {"series_id": group[0], "period": group[1], "row_id": row_id}, + ) + existing_fold = family_folds.get(family_key) + if existing_fold is not None and existing_fold != current_fold: + collisions += 1 + if diagnostic and existing_fold == current_fold: + prefix = f"cm_{family}_" + for suffix in ( + "actual", + "error", + "diagnostic_available", + "diagnostic_available_at_utc_ms", + ): + existing[prefix + suffix] = annotation.get(prefix + suffix) + else: + existing.update(annotation) + family_folds[family_key] = current_fold + return tuple(merged[key] for key in sorted(merged)), collisions + + +def _annotation_row( + evaluation: Mapping[str, Any], + input_result: ClassicalModelInputResult, + row_id: int, + *, + diagnostic: bool, +) -> dict[str, Any]: + family = _text(evaluation.get("family")) + prefix = f"cm_{family}_" + fit_status = _text(evaluation.get("fit_status")) or "unavailable" + forecast_available = ( + not diagnostic + and _optional_float(evaluation.get("forecast")) is not None + ) + return { + "series_id": _text(evaluation.get("series_id")), + "period": _text(evaluation.get("period")), + "row_id": row_id, + prefix + + "schema_version": ( + SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION + ), + prefix + + "input_derivation_id": _text( + input_result.contract.get("derivation_id") + ), + prefix + "model_id": _text(evaluation.get("model_id")), + prefix + "family_code": _int(evaluation.get("family_code")), + prefix + + "specification_code": _int(evaluation.get("specification_code")), + prefix + "p": _int(evaluation.get("p")), + prefix + "d": _int(evaluation.get("d")), + prefix + "q": _int(evaluation.get("q")), + prefix + "seasonal_p": _int(evaluation.get("seasonal_p")), + prefix + "seasonal_d": _int(evaluation.get("seasonal_d")), + prefix + "seasonal_q": _int(evaluation.get("seasonal_q")), + prefix + "seasonal_period": _int(evaluation.get("seasonal_period")), + prefix + "seasonal_cycle_ms": _int(evaluation.get("seasonal_cycle_ms")), + prefix + "trend_code": _int(evaluation.get("trend_code")), + prefix + + "regressor_set_code": _int(evaluation.get("regressor_set_code")), + prefix + "regressor_count": _int(evaluation.get("regressor_count")), + prefix + + "regressor_available": bool( + evaluation.get("regressor_available", False) + ), + prefix + "calculation_basis_code": 1, + prefix + + "fit_status_code": SEASONAL_EXOGENOUS_FIT_STATUS_CODES.get( + fit_status, 1 + ), + prefix + + "failure_reason_code": SEASONAL_EXOGENOUS_REASON_CODES.get( + _text(evaluation.get("fit_reason")), 0 + ), + prefix + "converged": bool(evaluation.get("converged", False)), + prefix + "stationary": evaluation.get("stationary"), + prefix + "invertible": evaluation.get("invertible"), + prefix + + "ar_root_min_modulus": _optional_float( + evaluation.get("ar_root_min_modulus") + ), + prefix + + "ma_root_min_modulus": _optional_float( + evaluation.get("ma_root_min_modulus") + ), + prefix + + "covariance_condition_number": _optional_float( + evaluation.get("covariance_condition_number") + ), + prefix + + "effective_observation_count": _int( + evaluation.get("effective_observation_count") + ), + prefix + "fold_id": _int(evaluation.get("fold_id")), + prefix + "origin_row_id": evaluation.get("origin_row_id"), + prefix + "target_row_id": evaluation.get("target_row_id"), + prefix + "horizon": _int(evaluation.get("horizon")), + prefix + "forecast": _optional_float(evaluation.get("forecast")), + prefix + "forecast_available": forecast_available, + prefix + + "forecast_available_at_utc_ms": _int( + evaluation.get("origin_bin_end_utc_ms") + ), + prefix + + "actual": ( + _optional_float(evaluation.get("actual")) if diagnostic else None + ), + prefix + + "error": ( + _optional_float(evaluation.get("error")) if diagnostic else None + ), + prefix + "diagnostic_available": diagnostic, + prefix + + "diagnostic_available_at_utc_ms": ( + _int(evaluation.get("target_bin_end_utc_ms")) + if diagnostic + else None + ), + prefix + "diagnostic_only": diagnostic, + prefix + "original_scale": True, + prefix + "training_eligible": forecast_available, + } + + +def _training_projection_metadata( + profile: SeasonalExogenousProfile, + input_result: ClassicalModelInputResult, + annotation_count: int, + collision_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "row", + "identity_fields": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "mapping_policy": "first_source_row_at_or_after_availability", + "collision_policy": "latest_origin_wins_per_family", + "collision_count": collision_count, + "annotation_count": annotation_count, + "projection_specification_ids": list( + profile.projection_specification_ids + ), + "projection_horizon": profile.projection_horizon, + "input_derivation_id": input_result.contract.get("derivation_id"), + "column_names": list(SEASONAL_EXOGENOUS_COLUMNS), + "family_column_prefixes": [ + "cm_sarima_", + "cm_arimax_", + "cm_sarimax_", + ], + "forecast_time_values_use_future_market_observations": False, + "diagnostics_marked_post_observation": True, + "observed_columns_overwritten": False, + } + + +def _base_payload( + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + profile: SeasonalExogenousProfile, + regressor_contract: Mapping[str, JSONValue], + calendar_profile: HistDataCalendarProfile, +) -> dict[str, JSONValue]: + contract = input_result.contract + regularization = _mapping(contract.get("regularization")) + return { + "schema_version": SEASONAL_EXOGENOUS_SCHEMA_VERSION, + "advisory": True, + "target_axis": dict(_mapping(contract.get("target_axis"))), + "reference_fingerprint_id": contract.get("reference_fingerprint_id") + or fingerprint.get("fingerprint_id"), + "input_schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "input_derivation_id": contract.get("derivation_id"), + "input_status": contract.get("status"), + "calculation_basis": "regular_grid_rolling_origin_calendar_exog", + "configuration": profile.to_metadata(), + "regressors": dict(regressor_contract), + "calendar_profile": calendar_profile.to_metadata(), + "input_transform_policy": dict( + _mapping(contract.get("transform_policy")) + ), + "input_missingness_policy": { + "expected_closure_policy": regularization.get( + "expected_closure_policy" + ), + "unexpected_missing_policy": regularization.get( + "unexpected_missing_policy" + ), + "expected_closure_count": regularization.get( + "expected_closure_count" + ), + "unexpected_missing_count": regularization.get( + "unexpected_missing_count" + ), + "forward_fill_policy": regularization.get("forward_fill_policy"), + }, + "input_and_model_differencing_are_distinct": True, + "nonseasonal_and_seasonal_differencing_are_distinct": True, + "missing_observation_policy": "reset_to_trailing_contiguous_segment", + "forward_fill_policy": "never", + "original_scale_forecasts": True, + "future_calendar_values_known_independently": True, + "future_market_observation_regressors_allowed": False, + "full_series_smoothed_state_allowed": False, + "automatic_order_selection": False, + "automatic_regressor_selection": False, + "automatic_winner": False, + "hard_fail_quality_gate": False, + "fitted_objects_included": False, + "backend_exception_text_included": False, + } + + +def _unavailable_result( + input_result: ClassicalModelInputResult, + base: Mapping[str, JSONValue], + reason: str, + *, + status: str = "unavailable", +) -> SeasonalExogenousResult: + configuration = _mapping(base.get("configuration")) + diagnostics: dict[str, JSONValue] = { + **dict(base), + "status": status, + "reason": reason, + "limitations": [reason], + "backend": { + "provider": "statsmodels", + "version": None, + "available": reason != "dependency_unavailable", + "model_class": "statsmodels.tsa.statespace.SARIMAX", + }, + "fit_summary": { + "schema_version": SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION, + "fit_attempt_count": 0, + "status_counts": {}, + "reason_counts": {reason: 1}, + "warning_counts": {}, + "failed_fit_count": 0, + "limited_fit_count": 0, + "convergence_rate": None, + "failure_rate": None, + "fit_samples": [], + "fit_samples_truncated": False, + }, + "evaluation": { + "schema_version": SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin_calendar_exog", + "original_scale": True, + "model_count": 0, + "fold_count": len(input_result.folds), + "evaluated_fold_count": 0, + "skipped_evaluation_count": len(input_result.folds), + "forecast_coverage_rate": None, + "models": [], + "reference_baselines": [], + "reference_models": {}, + "regime_error_summary": {}, + "automatic_winner": False, + }, + "resource_usage": { + "estimated_working_memory_bytes": 0, + "memory_limit_exceeded": False, + "fit_attempt_count": 0, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "training_projection": { + "schema_version": ( + SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION + ), + "column_names": list(SEASONAL_EXOGENOUS_COLUMNS), + "projection_specification_ids": configuration.get( + "projection_specification_ids" + ) + or [], + "annotation_count": 0, + }, + "fit_duration_included": False, + } + return SeasonalExogenousResult(diagnostics, (), input_result) + + +def _load_backend() -> _Backend | None: + try: + module = importlib.import_module("statsmodels.tsa.statespace.sarimax") + version = importlib.metadata.version("statsmodels") + except (ImportError, importlib.metadata.PackageNotFoundError): + return None + return _Backend(version=version, sarimax=module.SARIMAX) + + +def _backend_failure_reason(exc: Exception) -> str: + message = str(exc).lower() + if "seasonal periodicity" in message or "seasonal_order" in message: + return "invalid_seasonality" + if "rank" in message: + return "rank_deficient_regressors" + if "collinear" in message: + return "collinearity" + if "exog" in message or "regressor" in message: + return "future_regressor_unavailable" + if "singular" in message or "linalg" in message: + return "singularity" + if "order" in message or "lag" in message: + return "invalid_order" + if "trend" in message or "parameter" in message or "initial" in message: + return "invalid_configuration" + if "overflow" in message or "finite" in message: + return "numerical_overflow" + return "backend_failure" + + +def _folds_by_origin( + folds: Sequence[Mapping[str, Any]], +) -> list[tuple[tuple[str, str, int], list[dict[str, Any]]]]: + grouped: dict[tuple[str, str, int], list[dict[str, Any]]] = {} + for fold in folds: + key = ( + _text(fold.get("series_id")), + _text(fold.get("period")), + _int(fold.get("origin_bin_end_utc_ms")), + ) + grouped.setdefault(key, []).append(dict(fold)) + return [ + (key, sorted(values, key=lambda item: _int(item.get("horizon")))) + for key, values in sorted(grouped.items()) + ] + + +def _estimated_memory( + row_count: int, + fold_count: int, + specifications: Sequence[SeasonalExogenousSpecification], +) -> int: + regressor_count = sum(len(item.regressor_names) for item in specifications) + return max( + 1, + row_count * max(1, regressor_count) * 8 + + fold_count * max(1, len(specifications)) * 4_096, + ) + + +def _model_id( + specification: SeasonalExogenousSpecification, + derivation_id: str, + backend_version: str, + calendar_profile: HistDataCalendarProfile, +) -> str: + payload = { + "backend": "statsmodels", + "backend_version": backend_version, + "configuration": specification.to_metadata(), + "input_derivation_id": derivation_id, + "calendar_profile": { + "name": calendar_profile.name, + "version": calendar_profile.version, + "source": calendar_profile.source, + }, + } + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("ascii") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _regressor_set_code(names: Sequence[str]) -> int: + encoded = "\n".join(names).encode("utf-8") + return int.from_bytes(hashlib.sha256(encoded).digest()[:4], "big") + + +def _ensure_projection_columns(frame: Any) -> Any: + import polars as pl + + expressions = [] + for name, dtype in _projection_dtypes().items(): + if name in frame.columns: + expressions.append(pl.col(name).cast(dtype).alias(name)) + else: + expressions.append(pl.lit(None).cast(dtype).alias(name)) + return frame.with_columns(expressions) + + +def _projection_dtypes() -> dict[str, Any]: + import polars as pl + + strings = {"schema_version", "input_derivation_id", "model_id"} + booleans = { + "regressor_available", + "converged", + "stationary", + "invertible", + "forecast_available", + "diagnostic_available", + "diagnostic_only", + "original_scale", + "training_eligible", + } + floats = { + "forecast", + "actual", + "error", + "ar_root_min_modulus", + "ma_root_min_modulus", + "covariance_condition_number", + } + return { + f"cm_{family}_{suffix}": ( + pl.Utf8 + if suffix in strings + else ( + pl.Boolean + if suffix in booleans + else pl.Float64 if suffix in floats else pl.Int64 + ) + ) + for family in SEASONAL_EXOGENOUS_FAMILIES + for suffix in SEASONAL_EXOGENOUS_FAMILY_COLUMN_SUFFIXES + } diff --git a/src/histdatacom/data_quality/state_space.py b/src/histdatacom/data_quality/state_space.py new file mode 100644 index 00000000..eceecb36 --- /dev/null +++ b/src/histdatacom/data_quality/state_space.py @@ -0,0 +1,2152 @@ +"""Explicit structural state-space and Kalman fingerprint diagnostics.""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.metadata +import json +import math +import re +import time +import warnings +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from contextlib import nullcontext +from dataclasses import dataclass, field +from statistics import median +from typing import Any, cast + +from histdatacom.data_quality.autoregressive import ( + _covariance_condition_number, + _first_available_row_id, + _fit_converged, + _fitted_parameters, + _float, + _int, + _mapping, + _mapping_rows, + _optional_float, + _optional_int, + _rate, + _rounded, + _target_sort_key, + _text, + _warning_codes, +) +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + ClassicalModelInputProfile, + ClassicalModelInputResult, + build_classical_model_input, +) +from histdatacom.data_quality.contracts import QualityFinding +from histdatacom.data_quality.exponential_smoothing import ( + _inverse_forecasts, + _reference_baseline_payloads, +) +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.seasonal_exogenous import _model_references +from histdatacom.data_quality.training_features import ( + KALMAN_COLUMNS, + STATE_SPACE_COLUMNS, + ensure_tick_training_features, +) +from histdatacom.runtime_contracts import JSONValue + +STATE_SPACE_SCHEMA_VERSION = "histdatacom.state-space.v1" +STATE_SPACE_CONFIGURATION_SCHEMA_VERSION = ( + "histdatacom.state-space-configuration.v1" +) +STATE_SPACE_FIT_SCHEMA_VERSION = "histdatacom.state-space-fit-result.v1" +STATE_SPACE_STATE_RESULT_SCHEMA_VERSION = "histdatacom.kalman-state-result.v1" +STATE_SPACE_FORECAST_SCHEMA_VERSION = "histdatacom.state-space-forecast.v1" +STATE_SPACE_EVALUATION_SCHEMA_VERSION = "histdatacom.state-space-evaluation.v1" +STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.state-space-training-projection.v1" +) +STATE_SPACE_SUMMARY_SCHEMA_VERSION = "histdatacom.state-space-summary.v1" +STATE_SPACE_SUMMARY_METADATA_KEY = "time_series_fingerprint_state_space_summary" +STATE_SPACE_BOUNDED_PAYLOAD_KEY = "fingerprint_state_space" + +DEFAULT_STATE_SPACE_SUMMARY_TARGET_LIMIT = 16 +DEFAULT_STATE_SPACE_ROLLING_WINDOWS = (5, 20) +DEFAULT_STATE_SPACE_ROUNDING_DIGITS = 12 +MAX_STATE_SPACE_SPECIFICATIONS = 32 +MAX_STATE_DIMENSION = 256 +MAX_COMPONENT_COUNT = 16 +MAX_RETAINED_STATES = 64 +MAX_FIXED_PARAMETERS = 32 + +STATE_SPACE_FAMILIES = ("local_level", "local_linear_trend", "structural") +STATE_SPACE_FAMILY_CODES = { + "local_level": 1, + "local_linear_trend": 2, + "structural": 3, +} +STATE_SPACE_INITIALIZATION_CODES = { + "default": 1, + "approximate_diffuse": 2, + "exact_diffuse": 3, +} +STATE_SPACE_FIT_STATUS_CODES = { + "unavailable": 1, + "skipped": 2, + "failed": 3, + "limited": 4, + "fitted": 5, + "converged": 6, +} +STATE_SPACE_REASON_CODES = { + "": 0, + "dependency_unavailable": 1, + "input_contract_unavailable": 2, + "insufficient_folds": 3, + "insufficient_history": 4, + "invalid_configuration": 5, + "invalid_time_basis": 6, + "unidentifiable_model": 7, + "singular_covariance": 8, + "non_positive_covariance": 9, + "diffuse_initialization_failure": 10, + "numerical_instability": 11, + "missingness_limitation": 12, + "long_missing_gap": 13, + "optimizer_failure": 14, + "resource_limit": 15, + "timeout": 16, + "backend_failure": 17, + "target_unavailable": 18, + "inverse_transform_unavailable": 19, + "zero_variance": 20, +} +KALMAN_FILTERED_CALCULATION_BASIS_CODE = 1 +KALMAN_SMOOTHED_CALCULATION_BASIS_CODE = 2 + +_SPECIFICATION_ID = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") +_PARAMETER_NAME = re.compile(r"^[A-Za-z0-9_.()\[\]-]{1,96}$") +_OPTIMIZERS = {"lbfgs", "bfgs", "powell", "nm", "cg", "ncg"} + + +@dataclass(frozen=True, slots=True) +class StateSpaceSpecification: + """One explicit local-level, local-trend, or structural configuration.""" + + specification_id: str + family: str + irregular: bool = True + stochastic_level: bool = True + stochastic_trend: bool = False + seasonal_period: int = 0 + seasonal_cycle_ms: int = 0 + stochastic_seasonal: bool = True + cycle: bool = False + stochastic_cycle: bool = False + damped_cycle: bool = False + autoregressive_order: int = 0 + initialization_method: str = "default" + approximate_diffuse_variance: float = 1_000_000.0 + optimizer: str = "lbfgs" + fixed_parameters: tuple[tuple[str, float], ...] = () + max_iterations: int = 200 + + def __post_init__(self) -> None: + if not _SPECIFICATION_ID.fullmatch(self.specification_id): + raise ValueError("invalid state-space specification_id") + if self.family not in STATE_SPACE_FAMILIES: + raise ValueError("unsupported state-space family") + if self.family == "local_level": + if self.stochastic_trend or self.seasonal_period or self.cycle: + raise ValueError( + "local-level models cannot add trend/seasonal/cycle" + ) + if self.autoregressive_order: + raise ValueError("local-level models cannot add autoregression") + if self.family == "local_linear_trend": + if not self.stochastic_trend: + raise ValueError("local-linear-trend requires stochastic_trend") + if self.seasonal_period or self.cycle or self.autoregressive_order: + raise ValueError( + "local-linear-trend is a dedicated trend model" + ) + if self.family == "structural" and not any( + ( + self.stochastic_level, + self.stochastic_trend, + self.seasonal_period, + self.cycle, + self.autoregressive_order, + ) + ): + raise ValueError("structural model requires an explicit component") + if self.seasonal_period: + if self.seasonal_period < 2 or self.seasonal_period > 100_000: + raise ValueError("seasonal_period must be between 2 and 100000") + if self.seasonal_cycle_ms < 1: + raise ValueError("seasonal_cycle_ms must be explicit") + elif self.seasonal_cycle_ms: + raise ValueError("seasonal_cycle_ms requires seasonal_period") + if self.autoregressive_order < 0 or self.autoregressive_order > 64: + raise ValueError("autoregressive_order must be between 0 and 64") + if self.initialization_method not in STATE_SPACE_INITIALIZATION_CODES: + raise ValueError("unsupported state-space initialization_method") + if ( + not math.isfinite(self.approximate_diffuse_variance) + or self.approximate_diffuse_variance <= 0 + ): + raise ValueError("approximate_diffuse_variance must be positive") + if self.optimizer not in _OPTIMIZERS: + raise ValueError("unsupported state-space optimizer") + if len(self.fixed_parameters) > MAX_FIXED_PARAMETERS: + raise ValueError("too many fixed state-space parameters") + seen: set[str] = set() + for name, value in self.fixed_parameters: + if ( + not _PARAMETER_NAME.fullmatch(name) + or name in seen + or not math.isfinite(value) + ): + raise ValueError( + "fixed parameters must be unique finite scalars" + ) + seen.add(name) + if self.max_iterations < 1: + raise ValueError("max_iterations must be positive") + + @property + def level(self) -> bool: + """Return whether the model includes a level state.""" + return True + + @property + def trend(self) -> bool: + """Return whether the model includes a trend state.""" + return self.family == "local_linear_trend" or self.stochastic_trend + + @property + def component_count(self) -> int: + """Return the explicitly configured structural component count.""" + return sum( + bool(value) + for value in ( + self.level, + self.trend, + self.irregular, + self.seasonal_period, + self.cycle, + self.autoregressive_order, + ) + ) + + def to_metadata(self) -> dict[str, JSONValue]: + """Return stable JSON-compatible configuration metadata.""" + latent_components = ["level"] + if self.trend: + latent_components.append("trend") + if self.seasonal_period: + latent_components.append("seasonal") + if self.cycle: + latent_components.append("cycle") + if self.autoregressive_order: + latent_components.append("autoregressive") + metadata: dict[str, JSONValue] = { + "schema_version": STATE_SPACE_CONFIGURATION_SCHEMA_VERSION, + "specification_id": self.specification_id, + "family": self.family, + "family_code": STATE_SPACE_FAMILY_CODES[self.family], + "observable": "cm_input_value", + "observable_contract": { + "name": "cm_input_value", + "scalar": True, + "missing_supported": True, + "time_basis": "regular_grid", + }, + "latent_state_contract": { + "components": cast(JSONValue, latent_components), + "bounded_by_profile_state_dimension": True, + }, + "observation_equation": "y_t = Z_t alpha_t + epsilon_t", + "transition_equation": "alpha_t = T_t alpha_(t-1) + eta_t", + "observation_noise": { + "enabled": self.irregular, + "estimated_variance": self.irregular, + }, + "process_noise": { + "stochastic_level": self.stochastic_level, + "stochastic_trend": self.stochastic_trend, + "stochastic_seasonal": ( + self.stochastic_seasonal and bool(self.seasonal_period) + ), + "stochastic_cycle": self.stochastic_cycle and self.cycle, + }, + "level": self.level, + "trend": self.trend, + "irregular": self.irregular, + "stochastic_level": self.stochastic_level, + "stochastic_trend": self.stochastic_trend, + "seasonal_period": self.seasonal_period, + "seasonal_cycle_ms": self.seasonal_cycle_ms, + "stochastic_seasonal": self.stochastic_seasonal, + "cycle": self.cycle, + "stochastic_cycle": self.stochastic_cycle, + "damped_cycle": self.damped_cycle, + "autoregressive_order": self.autoregressive_order, + "component_count": self.component_count, + "initialization_method": self.initialization_method, + "initialization_code": STATE_SPACE_INITIALIZATION_CODES[ + self.initialization_method + ], + "approximate_diffuse_variance": self.approximate_diffuse_variance, + "optimizer": self.optimizer, + "fixed_parameters": cast( + JSONValue, + [ + {"parameter": name, "value": value} + for name, value in self.fixed_parameters + ], + ), + "max_iterations": self.max_iterations, + "automatic_component_selection": False, + } + return metadata + + +def _default_specifications() -> tuple[StateSpaceSpecification, ...]: + return ( + StateSpaceSpecification("local-level", "local_level"), + StateSpaceSpecification( + "local-linear-trend", + "local_linear_trend", + stochastic_trend=True, + ), + StateSpaceSpecification( + "structural-hourly", + "structural", + seasonal_period=60, + seasonal_cycle_ms=3_600_000, + ), + ) + + +@dataclass(frozen=True, slots=True) +class StateSpaceProfile: + """Explicit state-space controls; disabled by default.""" + + enabled: bool = False + specifications: tuple[StateSpaceSpecification, ...] = field( + default_factory=_default_specifications + ) + projection_specification_id: str = "local-level" + projection_horizon: int = 1 + max_state_dimension: int = 64 + max_component_count: int = 8 + max_prediction_only_gap: int = 240 + max_retained_states: int = 16 + baseline_rolling_windows: tuple[int, ...] = ( + DEFAULT_STATE_SPACE_ROLLING_WINDOWS + ) + compare_exponential_smoothing: bool = True + compare_autoregressive: bool = True + compare_seasonal_exogenous: bool = True + rounding_digits: int = DEFAULT_STATE_SPACE_ROUNDING_DIGITS + + def __post_init__(self) -> None: + if not self.specifications: + raise ValueError( + "at least one state-space specification is required" + ) + if len(self.specifications) > MAX_STATE_SPACE_SPECIFICATIONS: + raise ValueError("too many state-space specifications") + identifiers = tuple( + item.specification_id for item in self.specifications + ) + if len(set(identifiers)) != len(identifiers): + raise ValueError("state-space specification IDs must be unique") + if self.projection_specification_id not in identifiers: + raise ValueError("projection ID must select a configured model") + if self.projection_horizon < 1: + raise ValueError("projection_horizon must be positive") + if not 1 <= self.max_state_dimension <= MAX_STATE_DIMENSION: + raise ValueError("max_state_dimension must be between 1 and 256") + if not 1 <= self.max_component_count <= MAX_COMPONENT_COUNT: + raise ValueError("max_component_count must be between 1 and 16") + if self.max_prediction_only_gap < 0: + raise ValueError("max_prediction_only_gap must be non-negative") + if not 1 <= self.max_retained_states <= MAX_RETAINED_STATES: + raise ValueError("max_retained_states must be between 1 and 64") + if ( + not self.baseline_rolling_windows + or any(value < 1 for value in self.baseline_rolling_windows) + or tuple(sorted(set(self.baseline_rolling_windows))) + != self.baseline_rolling_windows + ): + raise ValueError("baseline windows must be sorted unique positives") + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between 0 and 16") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return stable profile metadata.""" + return { + "enabled": self.enabled, + "specifications": cast( + JSONValue, [item.to_metadata() for item in self.specifications] + ), + "projection_specification_id": self.projection_specification_id, + "projection_horizon": self.projection_horizon, + "max_state_dimension": self.max_state_dimension, + "max_component_count": self.max_component_count, + "max_prediction_only_gap": self.max_prediction_only_gap, + "max_retained_states": self.max_retained_states, + "baseline_rolling_windows": list(self.baseline_rolling_windows), + "compare_exponential_smoothing": self.compare_exponential_smoothing, + "compare_autoregressive": self.compare_autoregressive, + "compare_seasonal_exogenous": self.compare_seasonal_exogenous, + "rounding_digits": self.rounding_digits, + "automatic_component_selection": False, + "automatic_winner": False, + "full_series_smoothing_projection": False, + } + + +@dataclass(frozen=True, slots=True) +class StateSpaceResult: + """Bounded diagnostics plus durable row-key annotations.""" + + diagnostics: Mapping[str, JSONValue] + annotations: tuple[Mapping[str, Any], ...] + input_result: ClassicalModelInputResult + + +@dataclass(frozen=True, slots=True) +class _Backend: + version: str + unobserved_components: Any + + +@dataclass(frozen=True, slots=True) +class _FitOutcome: + status: str + reason: str + forecasts: tuple[float, ...] + standard_errors: tuple[float, ...] + lower_bounds: tuple[float, ...] + upper_bounds: tuple[float, ...] + parameters: Mapping[str, float] + warning_codes: tuple[str, ...] + converged: bool + state_dimension: int + state_names: tuple[str, ...] + filtered_state: tuple[float | None, ...] + filtered_variance: tuple[float | None, ...] + smoothed_state: tuple[float | None, ...] + smoothed_variance: tuple[float | None, ...] + effective_observation_count: int + missing_observation_count: int + prediction_only_transition_count: int + max_prediction_only_gap: int + log_likelihood: float | None + aic: float | None + bic: float | None + covariance_condition_number: float | None + innovation_summary: Mapping[str, JSONValue] + + +def state_space_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: StateSpaceProfile | None = None, + exponential_smoothing: Mapping[str, JSONValue] | None = None, + autoregressive: Mapping[str, JSONValue] | None = None, + seasonal_exogenous: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> StateSpaceResult: + """Regularize an enriched tick frame and evaluate configured models.""" + selected_input = input_profile or ClassicalModelInputProfile(enabled=True) + input_result = build_classical_model_input( + frame, fingerprint, profile=selected_input, target=target + ) + return state_space_from_model_input( + frame, + input_result, + fingerprint, + input_profile=selected_input, + profile=profile, + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + seasonal_exogenous=seasonal_exogenous, + target=target, + ) + + +def state_space_from_model_input( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile, + profile: StateSpaceProfile | None = None, + exponential_smoothing: Mapping[str, JSONValue] | None = None, + autoregressive: Mapping[str, JSONValue] | None = None, + seasonal_exogenous: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> StateSpaceResult: + """Fit explicit structural models on shared rolling-origin folds.""" + selected = profile or StateSpaceProfile(enabled=True) + base = _base_payload(input_result, fingerprint, selected) + if selected.projection_horizon not in input_profile.horizons: + return _unavailable_result(input_result, base, "invalid_configuration") + if input_result.contract.get("status") == "unavailable": + return _unavailable_result( + input_result, base, "input_contract_unavailable" + ) + if not input_result.folds: + return _unavailable_result( + input_result, base, "insufficient_folds", status="limited" + ) + backend = _load_backend() + if backend is None: + return _unavailable_result(input_result, base, "dependency_unavailable") + return _evaluate_models( + frame, + input_result, + fingerprint, + input_profile, + selected, + backend, + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + seasonal_exogenous=seasonal_exogenous, + target=target, + ) + + +def state_space_diagnostics_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: StateSpaceProfile | None = None, + target: Any | None = None, +) -> dict[str, JSONValue]: + """Return diagnostics without exposing the annotation wrapper.""" + return dict( + state_space_from_training_frame( + frame, + fingerprint, + input_profile=input_profile, + profile=profile, + target=target, + ).diagnostics + ) + + +def project_state_space_onto_training_frame( + frame: Any, + result: StateSpaceResult, + *, + target: Any | None = None, +) -> Any: + """Join state-space/Kalman annotations by durable row identity.""" + import polars as pl + + columns = set(getattr(frame, "columns", ())) + if not {"series_id", "period", "row_id"}.issubset(columns): + enriched = ensure_tick_training_features(frame, target=target) + else: + enriched = frame + projection_columns = (*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS) + left = enriched.drop( + [name for name in projection_columns if name in enriched.columns] + ).with_row_index("__cm_state_space_original_order") + if result.annotations: + right = pl.DataFrame( + [dict(row) for row in result.annotations], infer_schema_length=None + ) + projected = left.join( + right, + on=["series_id", "period", "row_id"], + how="left", + validate="m:1", + ) + else: + projected = left + projected = _ensure_projection_columns(projected) + return projected.sort("__cm_state_space_original_order").drop( + "__cm_state_space_original_order" + ) + + +def state_space_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = DEFAULT_STATE_SPACE_SUMMARY_TARGET_LIMIT, +) -> dict[str, JSONValue] | None: + """Return bounded report metadata for structural-model results.""" + targets: list[dict[str, JSONValue]] = [] + statuses: Counter[str] = Counter() + for finding in findings: + fingerprint = _mapping(finding.metadata.get("time_series_fingerprint")) + payload = _mapping(fingerprint.get("state_space")) + if not payload: + continue + status = _text(payload.get("status")) or "unavailable" + evaluation = _mapping(payload.get("evaluation")) + fit_summary = _mapping(payload.get("fit_summary")) + statuses[status] += 1 + targets.append( + { + "target_axis": dict(_mapping(payload.get("target_axis"))), + "status": status, + "reason": payload.get("reason"), + "model_count": _int(evaluation.get("model_count")), + "fit_attempt_count": _int(fit_summary.get("fit_attempt_count")), + "evaluated_fold_count": _int( + evaluation.get("evaluated_fold_count") + ), + "failed_fit_count": _int(fit_summary.get("failed_fit_count")), + } + ) + if not targets: + return None + targets.sort(key=_target_sort_key) + limit = bounded_report_limit( + target_limit, + default_limit=DEFAULT_STATE_SPACE_SUMMARY_TARGET_LIMIT, + allow_unbounded=True, + ) + included = limit.slice(targets) + omitted = len(targets) - len(included) + return { + "schema_version": STATE_SPACE_SUMMARY_SCHEMA_VERSION, + "advisory": True, + "target_count": len(targets), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "status_counts": dict(sorted(statuses.items())), + "target_summaries": cast(JSONValue, included), + "limit_metadata": {"targets": limit.limit_payload()}, + } + + +def format_state_space_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> tuple[str, ...]: + """Return concise human-readable structural-model lines.""" + if not summary: + return () + statuses = _mapping(summary.get("status_counts")) + lines = [ + "", + "State-space and Kalman models", + ( + f"targets: {_int(summary.get('target_count'))} " + f"ready: {_int(statuses.get('ready'))} " + f"limited: {_int(statuses.get('limited'))} " + f"unavailable: {_int(statuses.get('unavailable'))}" + ), + ] + for item in _mapping_rows(summary.get("target_summaries")): + axis = _mapping(item.get("target_axis")) + label = "/".join( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + lines.append( + f"- {label}: {_text(item.get('status'))} " + f"models={_int(item.get('model_count'))} " + f"fits={_int(item.get('fit_attempt_count'))} " + f"folds={_int(item.get('evaluated_fold_count'))}" + ) + if summary.get("truncated") is True: + lines.append( + f"- {_int(summary.get('omitted_target_count'))} targets omitted" + ) + return tuple(lines) + + +def _evaluate_models( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + input_profile: ClassicalModelInputProfile, + profile: StateSpaceProfile, + backend: _Backend, + *, + exponential_smoothing: Mapping[str, JSONValue] | None, + autoregressive: Mapping[str, JSONValue] | None, + seasonal_exogenous: Mapping[str, JSONValue] | None, + target: Any | None, +) -> StateSpaceResult: + rows = cast(list[dict[str, Any]], input_result.regularized_frame.to_dicts()) + folds = [dict(fold) for fold in input_result.folds] + origins = _folds_by_origin(folds) + resources = input_profile.resources + specifications = profile.specifications[: resources.max_candidate_orders] + limitations: list[str] = [] + estimated_memory = _estimated_memory(len(rows), len(folds), specifications) + if len(specifications) < len(profile.specifications): + limitations.append("resource_limit") + if estimated_memory > resources.max_memory_bytes: + limitations.append("resource_limit") + specifications = () + started = time.monotonic() + fit_attempt_count = 0 + all_evaluations: list[dict[str, Any]] = [] + model_payloads: list[dict[str, JSONValue]] = [] + all_fit_samples: list[dict[str, JSONValue]] = [] + fit_statuses: Counter[str] = Counter() + fit_reasons: Counter[str] = Counter() + warning_counts: Counter[str] = Counter() + for specification_code, specification in enumerate(specifications, start=1): + configuration_reason = _configuration_reason( + specification, input_profile, profile + ) + model_id = _model_id( + specification, + _text(input_result.contract.get("derivation_id")), + backend.version, + ) + model_evaluations: list[dict[str, Any]] = [] + model_fit_samples: list[dict[str, JSONValue]] = [] + for _, origin_folds in origins: + if fit_attempt_count >= resources.max_fit_attempts: + limitations.append("resource_limit") + break + if time.monotonic() - started >= resources.max_wall_time_seconds: + limitations.append("timeout") + break + fit_attempt_count += 1 + origin = origin_folds[0] + start_index = _int(origin.get("training_start_index")) + end_index = _int(origin.get("training_end_index")) + indexes = tuple(range(start_index, end_index + 1)) + values = tuple( + _optional_float(rows[index].get("cm_input_value")) + for index in indexes + ) + max_horizon = max( + _int(fold.get("horizon")) for fold in origin_folds + ) + if configuration_reason: + outcome = _empty_fit( + "skipped", + configuration_reason, + observed_count=sum(value is not None for value in values), + missing_count=sum(value is None for value in values), + max_gap=_max_missing_run(values), + ) + else: + outcome = _fit_specification( + specification, + values, + max_horizon, + profile, + backend, + ) + fit_statuses[outcome.status] += 1 + if outcome.reason: + fit_reasons[outcome.reason] += 1 + warning_counts.update(outcome.warning_codes) + fit_sample = _fit_sample( + outcome, + specification, + model_id, + origin, + indexes, + len(values), + profile, + ) + model_fit_samples.append(fit_sample) + all_fit_samples.append(fit_sample) + original_forecasts = _inverse_forecasts( + rows, + end_index, + outcome.forecasts, + input_profile, + profile.rounding_digits, + ) + for fold in origin_folds: + evaluation = _fold_evaluation( + rows, + fold, + specification, + specification_code, + model_id, + outcome, + original_forecasts, + input_profile, + profile.rounding_digits, + profile.max_retained_states, + ) + model_evaluations.append(evaluation) + all_evaluations.append(evaluation) + model_payloads.append( + _model_evaluation_payload( + specification, + specification_code, + model_id, + model_evaluations, + model_fit_samples, + resources.max_retained_diagnostics, + profile.rounding_digits, + ) + ) + if limitations and limitations[-1] in {"resource_limit", "timeout"}: + break + baselines = _reference_baseline_payloads( + rows, + folds, + profile.baseline_rolling_windows, + profile.rounding_digits, + ) + references = { + "exponential_smoothing": _model_references( + exponential_smoothing, + enabled=profile.compare_exponential_smoothing, + unavailable_reason="exponential_smoothing_not_enabled", + ), + "autoregressive": _model_references( + autoregressive, + enabled=profile.compare_autoregressive, + unavailable_reason="autoregressive_not_enabled", + ), + "seasonal_exogenous": _model_references( + seasonal_exogenous, + enabled=profile.compare_seasonal_exogenous, + unavailable_reason="seasonal_exogenous_not_enabled", + ), + } + annotations, collisions = _build_annotations( + frame, all_evaluations, profile, input_result, target=target + ) + evaluated_count = sum( + item.get("status") == "evaluated" for item in all_evaluations + ) + failed_count = sum( + status in {"failed", "unavailable"} + for status in fit_statuses.elements() + ) + limited_count = fit_statuses["limited"] + fit_statuses["skipped"] + if not evaluated_count: + limitations.append("insufficient_history") + limitations = list(dict.fromkeys(limitations)) + status = ( + "ready" + if not limitations and not failed_count and not limited_count + else "limited" + ) + reason = ( + limitations[0] + if limitations + else sorted(fit_reasons)[0] if fit_reasons else None + ) + diagnostics: dict[str, JSONValue] = { + **_base_payload(input_result, fingerprint, profile), + "status": status, + "reason": reason, + "limitations": cast(JSONValue, limitations), + "backend": { + "provider": "statsmodels", + "version": backend.version, + "available": True, + "model_class": "statsmodels.tsa.statespace.UnobservedComponents", + "import_basis": "optional_models_extra", + }, + "fit_summary": { + "schema_version": STATE_SPACE_FIT_SCHEMA_VERSION, + "fit_attempt_count": fit_attempt_count, + "status_counts": dict(sorted(fit_statuses.items())), + "reason_counts": dict(sorted(fit_reasons.items())), + "warning_counts": dict(sorted(warning_counts.items())), + "failed_fit_count": failed_count, + "limited_fit_count": limited_count, + "convergence_rate": _rate( + fit_statuses["converged"], + fit_attempt_count, + profile.rounding_digits, + ), + "failure_rate": _rate( + failed_count, fit_attempt_count, profile.rounding_digits + ), + "fit_samples": cast( + JSONValue, + all_fit_samples[: resources.max_retained_diagnostics], + ), + "fit_samples_truncated": ( + len(all_fit_samples) > resources.max_retained_diagnostics + ), + }, + "resource_usage": { + "limits": resources.to_metadata(), + "state_space_limits": { + "max_state_dimension": profile.max_state_dimension, + "max_component_count": profile.max_component_count, + "max_prediction_only_gap": profile.max_prediction_only_gap, + "max_retained_states": profile.max_retained_states, + }, + "estimated_working_memory_bytes": estimated_memory, + "memory_limit_exceeded": ( + estimated_memory > resources.max_memory_bytes + ), + "fit_attempt_count": fit_attempt_count, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "evaluation": { + "schema_version": STATE_SPACE_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin_kalman_filter", + "original_scale": True, + "model_count": len(model_payloads), + "fold_count": len(folds), + "evaluated_fold_count": evaluated_count, + "skipped_evaluation_count": ( + len(all_evaluations) - evaluated_count + ), + "forecast_coverage_rate": _rate( + evaluated_count, + len(all_evaluations), + profile.rounding_digits, + ), + "models": cast(JSONValue, model_payloads), + "reference_baselines": cast(JSONValue, baselines), + "reference_models": cast(JSONValue, references), + "comparison_semantics": "descriptive_shared_folds_only", + "automatic_winner": False, + }, + "training_projection": _training_projection_metadata( + profile, input_result, len(annotations), collisions + ), + "fit_duration_included": False, + } + return StateSpaceResult(diagnostics, annotations, input_result) + + +def _configuration_reason( + specification: StateSpaceSpecification, + input_profile: ClassicalModelInputProfile, + profile: StateSpaceProfile, +) -> str: + if specification.seasonal_period: + expected = specification.seasonal_period * input_profile.frequency_ms + if expected != specification.seasonal_cycle_ms: + return "invalid_time_basis" + if specification.component_count > profile.max_component_count: + return "resource_limit" + return "" + + +def _fit_specification( + specification: StateSpaceSpecification, + values: Sequence[float | None], + horizon: int, + profile: StateSpaceProfile, + backend: _Backend, +) -> _FitOutcome: + observed_count = sum(value is not None for value in values) + missing_count = len(values) - observed_count + max_gap = _max_missing_run(values) + finite = [float(value) for value in values if value is not None] + minimum = max(8, specification.component_count * 3) + if observed_count < minimum: + return _empty_fit( + "skipped", + "insufficient_history", + observed_count=observed_count, + missing_count=missing_count, + max_gap=max_gap, + ) + if max(finite) - min(finite) <= 1e-15: + return _empty_fit( + "skipped", + "zero_variance", + observed_count=observed_count, + missing_count=missing_count, + max_gap=max_gap, + ) + if max_gap > profile.max_prediction_only_gap: + return _empty_fit( + "skipped", + "long_missing_gap", + observed_count=observed_count, + missing_count=missing_count, + max_gap=max_gap, + ) + try: + numpy = importlib.import_module("numpy") + endog = numpy.asarray( + [numpy.nan if value is None else float(value) for value in values], + dtype=float, + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + model = backend.unobserved_components( + endog, + level=True, + trend=specification.trend, + seasonal=specification.seasonal_period or None, + cycle=specification.cycle, + autoregressive=specification.autoregressive_order or None, + irregular=specification.irregular, + stochastic_level=specification.stochastic_level, + stochastic_trend=specification.stochastic_trend, + stochastic_seasonal=specification.stochastic_seasonal, + stochastic_cycle=specification.stochastic_cycle, + damped_cycle=specification.damped_cycle, + use_exact_diffuse=( + specification.initialization_method == "exact_diffuse" + ), + ) + if specification.initialization_method == "approximate_diffuse": + model.initialize_approximate_diffuse( + specification.approximate_diffuse_variance + ) + if model.k_states > profile.max_state_dimension: + return _empty_fit( + "skipped", + "resource_limit", + observed_count=observed_count, + missing_count=missing_count, + max_gap=max_gap, + state_dimension=int(model.k_states), + state_names=tuple(str(name) for name in model.state_names), + ) + fixed = dict(specification.fixed_parameters) + unknown = sorted(set(fixed) - set(model.param_names)) + if unknown: + return _empty_fit( + "skipped", + "invalid_configuration", + observed_count=observed_count, + missing_count=missing_count, + max_gap=max_gap, + state_dimension=int(model.k_states), + state_names=tuple(str(name) for name in model.state_names), + ) + context = model.fix_params(fixed) if fixed else nullcontext() + with context: + result = model.fit( + method=specification.optimizer, + maxiter=specification.max_iterations, + disp=False, + low_memory=False, + ) + forecast = result.get_forecast(steps=horizon) + predicted = tuple( + _finite_or_none(item) for item in forecast.predicted_mean + ) + standard_errors = tuple( + _finite_or_none(item) for item in forecast.se_mean + ) + intervals = forecast.conf_int(alpha=0.05) + lower = tuple(_finite_or_none(row[0]) for row in intervals) + upper = tuple(_finite_or_none(row[1]) for row in intervals) + filtered_state = _last_vector(result.filtered_state) + smoothed_state = _last_vector(result.smoothed_state) + filtered_variance = _last_covariance_diagonal(result.filtered_state_cov) + smoothed_variance = _last_covariance_diagonal(result.smoothed_state_cov) + covariance_condition = _covariance_condition_number(result) + if ( + any(value is None for value in predicted) + or any(value is None for value in standard_errors) + or any(value is None for value in lower) + or any(value is None for value in upper) + or not filtered_state + or all(value is None for value in filtered_state) + or not _states_are_valid( + ( + *filtered_state, + *smoothed_state, + *filtered_variance, + *smoothed_variance, + ) + ) + ): + return _empty_fit( + "failed", + "numerical_instability", + observed_count=observed_count, + missing_count=missing_count, + max_gap=max_gap, + state_dimension=int(model.k_states), + state_names=tuple(str(name) for name in model.state_names), + ) + if any( + value is not None and value < -1e-10 for value in filtered_variance + ): + return _empty_fit( + "failed", + "non_positive_covariance", + observed_count=observed_count, + missing_count=missing_count, + max_gap=max_gap, + state_dimension=int(model.k_states), + state_names=tuple(str(name) for name in model.state_names), + ) + warning_codes = _warning_codes(caught) + converged = _fit_converged(result, warning_codes) + status = "converged" if converged else "limited" + reason = "" if converged else "optimizer_failure" + return _FitOutcome( + status=status, + reason=reason, + forecasts=tuple(float(cast(float, value)) for value in predicted), + standard_errors=tuple( + float(cast(float, value)) for value in standard_errors + ), + lower_bounds=tuple(float(cast(float, value)) for value in lower), + upper_bounds=tuple(float(cast(float, value)) for value in upper), + parameters=_fitted_parameters(result), + warning_codes=warning_codes, + converged=converged, + state_dimension=int(model.k_states), + state_names=tuple(str(name) for name in model.state_names), + filtered_state=filtered_state, + filtered_variance=filtered_variance, + smoothed_state=smoothed_state, + smoothed_variance=smoothed_variance, + effective_observation_count=int( + getattr(result, "nobs_effective", observed_count) + ), + missing_observation_count=missing_count, + prediction_only_transition_count=missing_count, + max_prediction_only_gap=max_gap, + log_likelihood=_optional_float(getattr(result, "llf", None)), + aic=_optional_float(getattr(result, "aic", None)), + bic=_optional_float(getattr(result, "bic", None)), + covariance_condition_number=covariance_condition, + innovation_summary=_innovation_summary( + result, profile.rounding_digits + ), + ) + except Exception as exc: # backend failures are isolated and normalized + return _empty_fit( + "failed", + _backend_failure_reason(exc), + observed_count=observed_count, + missing_count=missing_count, + max_gap=max_gap, + ) + + +def _empty_fit( + status: str, + reason: str, + *, + observed_count: int, + missing_count: int, + max_gap: int, + state_dimension: int = 0, + state_names: tuple[str, ...] = (), +) -> _FitOutcome: + return _FitOutcome( + status=status, + reason=reason, + forecasts=(), + standard_errors=(), + lower_bounds=(), + upper_bounds=(), + parameters={}, + warning_codes=(), + converged=False, + state_dimension=state_dimension, + state_names=state_names, + filtered_state=(), + filtered_variance=(), + smoothed_state=(), + smoothed_variance=(), + effective_observation_count=observed_count, + missing_observation_count=missing_count, + prediction_only_transition_count=missing_count, + max_prediction_only_gap=max_gap, + log_likelihood=None, + aic=None, + bic=None, + covariance_condition_number=None, + innovation_summary={}, + ) + + +def _fit_sample( + outcome: _FitOutcome, + specification: StateSpaceSpecification, + model_id: str, + fold: Mapping[str, Any], + indexes: Sequence[int], + grid_observation_count: int, + profile: StateSpaceProfile, +) -> dict[str, JSONValue]: + retained = _retained_state_count(outcome, profile.max_retained_states) + states = [ + { + "name": outcome.state_names[index], + "filtered": outcome.filtered_state[index], + "filtered_variance": outcome.filtered_variance[index], + "smoothed": outcome.smoothed_state[index], + "smoothed_variance": outcome.smoothed_variance[index], + } + for index in range(retained) + ] + return { + "schema_version": STATE_SPACE_FIT_SCHEMA_VERSION, + "state_schema_version": STATE_SPACE_STATE_RESULT_SCHEMA_VERSION, + "model_id": model_id, + "specification_id": specification.specification_id, + "family": specification.family, + "status": outcome.status, + "reason": outcome.reason or None, + "converged": outcome.converged, + "state_dimension": outcome.state_dimension, + "state_names": list(outcome.state_names[:retained]), + "states": cast(JSONValue, states), + "states_truncated": len(outcome.state_names) > retained, + "filtered_calculation_basis": "kalman_filter_origin_training_segment", + "smoothed_calculation_basis": ( + "kalman_smoother_origin_training_segment_retrospective" + ), + "smoothing_used_for_forecast": False, + "full_series_smoothing_used": False, + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "grid_observation_count": grid_observation_count, + "effective_observation_count": outcome.effective_observation_count, + "missing_observation_count": outcome.missing_observation_count, + "prediction_only_transition_count": ( + outcome.prediction_only_transition_count + ), + "max_prediction_only_gap": outcome.max_prediction_only_gap, + "transition_elapsed_time_basis": "one_regular_grid_step", + "segment_start_index": indexes[0] if indexes else None, + "segment_end_index": indexes[-1] if indexes else None, + "parameters": dict(outcome.parameters), + "parameter_count": len(outcome.parameters), + "log_likelihood": _rounded( + outcome.log_likelihood, profile.rounding_digits + ), + "aic": _rounded(outcome.aic, profile.rounding_digits), + "bic": _rounded(outcome.bic, profile.rounding_digits), + "covariance_condition_number": _rounded( + outcome.covariance_condition_number, profile.rounding_digits + ), + "innovation_summary": dict(outcome.innovation_summary), + "warning_codes": list(outcome.warning_codes), + "fit_duration_included": False, + "backend_exception_text_included": False, + } + + +def _fold_evaluation( + rows: Sequence[Mapping[str, Any]], + fold: Mapping[str, Any], + specification: StateSpaceSpecification, + specification_code: int, + model_id: str, + outcome: _FitOutcome, + original_forecasts: Sequence[float | None], + input_profile: ClassicalModelInputProfile, + rounding_digits: int, + retained_state_limit: int, +) -> dict[str, Any]: + horizon = _int(fold.get("horizon")) + forecast = ( + original_forecasts[horizon - 1] + if 0 < horizon <= len(original_forecasts) + else None + ) + transformed = ( + outcome.forecasts[horizon - 1] + if 0 < horizon <= len(outcome.forecasts) + else None + ) + target_index = _int(fold.get("target_index")) + actual = ( + _optional_float(rows[target_index].get("cm_input_observed_value")) + if 0 <= target_index < len(rows) + else None + ) + if outcome.status in {"failed", "skipped", "unavailable"}: + status, reason = "not_evaluated", outcome.reason + elif fold.get("status") != "valid" or actual is None: + status, reason = "skipped", "target_unavailable" + elif forecast is None: + status, reason = "skipped", "inverse_transform_unavailable" + else: + status, reason = "evaluated", "" + error = ( + forecast - actual + if status == "evaluated" and forecast is not None and actual is not None + else None + ) + retained = _retained_state_count(outcome, retained_state_limit) + states_available = retained > 0 + return { + "schema_version": STATE_SPACE_FORECAST_SCHEMA_VERSION, + "status": status, + "reason": reason or None, + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "family_code": STATE_SPACE_FAMILY_CODES[specification.family], + "state_dimension": outcome.state_dimension, + "component_count": specification.component_count, + "initialization_code": STATE_SPACE_INITIALIZATION_CODES[ + specification.initialization_method + ], + "fit_status": outcome.status, + "fit_reason": outcome.reason or None, + "converged": outcome.converged, + "effective_observation_count": outcome.effective_observation_count, + "missing_observation_count": outcome.missing_observation_count, + "prediction_only_transition_count": ( + outcome.prediction_only_transition_count + ), + "max_prediction_only_gap": outcome.max_prediction_only_gap, + "fold_id": _int(fold.get("fold_id")), + "origin_row_id": fold.get("origin_row_id"), + "target_row_id": fold.get("target_row_id"), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "target_bin_end_utc_ms": _int(fold.get("target_bin_end_utc_ms")), + "horizon": horizon, + "transformed_forecast": _rounded(transformed, rounding_digits), + "forecast": _rounded(forecast, rounding_digits), + "forecast_standard_error": _sequence_value( + outcome.standard_errors, horizon, rounding_digits + ), + "forecast_lower": _sequence_value( + outcome.lower_bounds, horizon, rounding_digits + ), + "forecast_upper": _sequence_value( + outcome.upper_bounds, horizon, rounding_digits + ), + "uncertainty_scale": ( + "original" if input_profile.transform == "level" else "transformed" + ), + "actual": _rounded(actual, rounding_digits), + "error": _rounded(error, rounding_digits), + "absolute_error": _rounded( + abs(error) if error is not None else None, rounding_digits + ), + "squared_error": _rounded( + error * error if error is not None else None, rounding_digits + ), + "state_names": list(outcome.state_names[:retained]), + "filtered_state": list(outcome.filtered_state[:retained]), + "filtered_variance": list(outcome.filtered_variance[:retained]), + "smoothed_state": list(outcome.smoothed_state[:retained]), + "smoothed_variance": list(outcome.smoothed_variance[:retained]), + "states_truncated": len(outcome.state_names) > retained, + "filtered_state_available_at_origin": states_available, + "smoothed_state_retrospective": states_available, + "smoothed_state_used_for_forecast": False, + "full_series_smoothing_used": False, + "original_scale": True, + "future_values_visible": False, + "automatic_winner": False, + } + + +def _model_evaluation_payload( + specification: StateSpaceSpecification, + specification_code: int, + model_id: str, + evaluations: Sequence[Mapping[str, Any]], + fit_samples: Sequence[Mapping[str, JSONValue]], + retained_limit: int, + rounding_digits: int, +) -> dict[str, JSONValue]: + statuses = Counter(_text(row.get("status")) for row in evaluations) + fit_statuses = Counter(_text(row.get("status")) for row in fit_samples) + reasons = Counter( + _text(row.get("reason")) + for row in evaluations + if _text(row.get("reason")) + ) + horizons = sorted({_int(row.get("horizon")) for row in evaluations}) + return { + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "configuration": specification.to_metadata(), + "status": "ready" if statuses["evaluated"] else "limited", + "fit_status_counts": dict(sorted(fit_statuses.items())), + "evaluation_status_counts": dict(sorted(statuses.items())), + "reason_counts": dict(sorted(reasons.items())), + "forecast_coverage_rate": _rate( + statuses["evaluated"], len(evaluations), rounding_digits + ), + "convergence_rate": _rate( + fit_statuses["converged"], len(fit_samples), rounding_digits + ), + "horizon_metrics": cast( + JSONValue, + [ + _metrics_for_horizon(evaluations, horizon, rounding_digits) + for horizon in horizons + ], + ), + "parameter_stability": _parameter_stability( + fit_samples, rounding_digits + ), + "uncertainty_summary": _uncertainty_summary( + evaluations, rounding_digits + ), + "fold_results": cast( + JSONValue, [dict(row) for row in evaluations[:retained_limit]] + ), + "fold_results_truncated": len(evaluations) > retained_limit, + "fit_samples": cast( + JSONValue, [dict(row) for row in fit_samples[:retained_limit]] + ), + "fit_samples_truncated": len(fit_samples) > retained_limit, + "automatic_winner": False, + } + + +def _metrics_for_horizon( + evaluations: Sequence[Mapping[str, Any]], + horizon: int, + rounding_digits: int, +) -> dict[str, JSONValue]: + selected = [ + row + for row in evaluations + if _int(row.get("horizon")) == horizon + and row.get("status") == "evaluated" + ] + errors = [_float(row.get("error")) for row in selected] + absolute = [abs(value) for value in errors] + squared = [value * value for value in errors] + return { + "horizon": horizon, + "evaluation_count": len(selected), + "mae": _rounded( + sum(absolute) / len(absolute) if absolute else None, + rounding_digits, + ), + "rmse": _rounded( + math.sqrt(sum(squared) / len(squared)) if squared else None, + rounding_digits, + ), + "bias": _rounded( + sum(errors) / len(errors) if errors else None, + rounding_digits, + ), + "original_scale": True, + } + + +def _parameter_stability( + fit_samples: Sequence[Mapping[str, JSONValue]], rounding_digits: int +) -> dict[str, JSONValue]: + parameters: dict[str, list[float]] = {} + for sample in fit_samples: + for name, raw in _mapping(sample.get("parameters")).items(): + value = _optional_float(raw) + if value is not None: + parameters.setdefault(name, []).append(value) + summaries: dict[str, JSONValue] = {} + for name, values in sorted(parameters.items()): + center = median(values) + summaries[name] = { + "count": len(values), + "min": _rounded(min(values), rounding_digits), + "max": _rounded(max(values), rounding_digits), + "median": _rounded(center, rounding_digits), + "mad": _rounded( + median(abs(value - center) for value in values), + rounding_digits, + ), + } + return { + "parameter_count": len(summaries), + "parameters": summaries, + "bounded": True, + } + + +def _uncertainty_summary( + evaluations: Sequence[Mapping[str, Any]], rounding_digits: int +) -> dict[str, JSONValue]: + values = [ + value + for row in evaluations + if (value := _optional_float(row.get("forecast_standard_error"))) + is not None + ] + return _numeric_summary(values, rounding_digits) + + +def _innovation_summary(result: Any, digits: int) -> dict[str, JSONValue]: + try: + raw = result.filter_results.forecasts_error[0] + except (AttributeError, IndexError, TypeError): + return _numeric_summary((), digits) + values = [ + value for item in raw if (value := _optional_float(item)) is not None + ] + return _numeric_summary(values, digits) + + +def _numeric_summary( + values: Sequence[float], digits: int +) -> dict[str, JSONValue]: + if not values: + return {"count": 0, "mean": None, "std": None, "max_abs": None} + center = sum(values) / len(values) + variance = sum((value - center) ** 2 for value in values) / len(values) + return { + "count": len(values), + "mean": _rounded(center, digits), + "std": _rounded(math.sqrt(variance), digits), + "max_abs": _rounded(max(abs(value) for value in values), digits), + } + + +def _build_annotations( + frame: Any | None, + evaluations: Sequence[Mapping[str, Any]], + profile: StateSpaceProfile, + input_result: ClassicalModelInputResult, + *, + target: Any | None, +) -> tuple[tuple[Mapping[str, Any], ...], int]: + if frame is None: + return (), 0 + try: + enriched = ensure_tick_training_features(frame, target=target) + except (AttributeError, TypeError, ValueError): + return (), 0 + source_rows = cast(list[dict[str, Any]], enriched.to_dicts()) + availability: dict[tuple[str, str], list[tuple[int, int]]] = {} + for row in source_rows: + timestamp = _optional_int(row.get("timestamp_utc_ms")) + row_id = _optional_int(row.get("row_id")) + if timestamp is not None and row_id is not None: + availability.setdefault( + (_text(row.get("series_id")), _text(row.get("period"))), [] + ).append((timestamp, row_id)) + for values in availability.values(): + values.sort() + merged: dict[tuple[str, str, int], dict[str, Any]] = {} + selected = [ + row + for row in evaluations + if _text(row.get("specification_id")) + == profile.projection_specification_id + and _int(row.get("horizon")) == profile.projection_horizon + ] + selected.sort( + key=lambda row: ( + _text(row.get("series_id")), + _text(row.get("period")), + _int(row.get("origin_bin_end_utc_ms")), + _int(row.get("fold_id")), + ) + ) + collisions = 0 + for evaluation in selected: + group = ( + _text(evaluation.get("series_id")), + _text(evaluation.get("period")), + ) + for diagnostic, time_key in ( + (False, "origin_bin_end_utc_ms"), + (True, "target_bin_end_utc_ms"), + ): + if diagnostic and _optional_float(evaluation.get("error")) is None: + continue + row_id = _first_available_row_id( + availability.get(group, ()), _int(evaluation.get(time_key)) + ) + if row_id is None: + continue + key = (*group, row_id) + annotation = _annotation_row( + evaluation, input_result, row_id, diagnostic=diagnostic + ) + if key in merged: + collisions += 1 + merged[key] = _merge_annotation_rows(merged[key], annotation) + else: + merged[key] = annotation + return tuple(merged[key] for key in sorted(merged)), collisions + + +def _merge_annotation_rows( + current: Mapping[str, Any], incoming: Mapping[str, Any] +) -> dict[str, Any]: + """Preserve forecast and retrospective fields sharing one source row.""" + merged = dict(current) + availability_flags = { + "cm_state_space_forecast_available", + "cm_state_space_diagnostic_available", + "cm_state_space_diagnostic_only", + "cm_state_space_training_eligible", + "cm_kalman_filtered_available", + "cm_kalman_filtered_training_eligible", + "cm_kalman_smoothed_available", + "cm_kalman_smoothed_retrospective", + "cm_kalman_smoothed_diagnostic_only", + } + for name, value in incoming.items(): + if name in availability_flags: + merged[name] = bool(merged.get(name)) or bool(value) + elif value is not None: + merged[name] = value + merged["cm_kalman_smoothed_training_eligible"] = False + return merged + + +def _annotation_row( + evaluation: Mapping[str, Any], + input_result: ClassicalModelInputResult, + row_id: int, + *, + diagnostic: bool, +) -> dict[str, Any]: + fit_status = _text(evaluation.get("fit_status")) or "unavailable" + forecast_available = ( + not diagnostic + and _optional_float(evaluation.get("forecast")) is not None + ) + state_names = cast(list[str], evaluation.get("state_names", [])) + filtered = cast(list[Any], evaluation.get("filtered_state", [])) + filtered_variance = cast(list[Any], evaluation.get("filtered_variance", [])) + smoothed = cast(list[Any], evaluation.get("smoothed_state", [])) + smoothed_variance = cast(list[Any], evaluation.get("smoothed_variance", [])) + level_index = _state_index(state_names, "level") + trend_index = _state_index(state_names, "trend") + filtered_state_available = ( + forecast_available + and bool(evaluation.get("filtered_state_available_at_origin")) + and bool(filtered) + ) + smoothed_state_available = ( + diagnostic + and bool(evaluation.get("smoothed_state_retrospective")) + and bool(smoothed) + ) + origin_time = _int(evaluation.get("origin_bin_end_utc_ms")) + target_time = _int(evaluation.get("target_bin_end_utc_ms")) + return { + "series_id": _text(evaluation.get("series_id")), + "period": _text(evaluation.get("period")), + "row_id": row_id, + "cm_state_space_schema_version": ( + STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION + ), + "cm_state_space_input_derivation_id": _text( + input_result.contract.get("derivation_id") + ), + "cm_state_space_model_id": _text(evaluation.get("model_id")), + "cm_state_space_family_code": _int(evaluation.get("family_code")), + "cm_state_space_specification_code": _int( + evaluation.get("specification_code") + ), + "cm_state_space_state_dimension": _int( + evaluation.get("state_dimension") + ), + "cm_state_space_component_count": _int( + evaluation.get("component_count") + ), + "cm_state_space_initialization_code": _int( + evaluation.get("initialization_code") + ), + "cm_state_space_fit_status_code": STATE_SPACE_FIT_STATUS_CODES.get( + fit_status, 1 + ), + "cm_state_space_failure_reason_code": STATE_SPACE_REASON_CODES.get( + _text(evaluation.get("fit_reason")), 0 + ), + "cm_state_space_converged": bool(evaluation.get("converged", False)), + "cm_state_space_effective_observation_count": _int( + evaluation.get("effective_observation_count") + ), + "cm_state_space_missing_observation_count": _int( + evaluation.get("missing_observation_count") + ), + "cm_state_space_prediction_only_transition_count": _int( + evaluation.get("prediction_only_transition_count") + ), + "cm_state_space_max_prediction_only_gap": _int( + evaluation.get("max_prediction_only_gap") + ), + "cm_state_space_fold_id": _int(evaluation.get("fold_id")), + "cm_state_space_origin_row_id": evaluation.get("origin_row_id"), + "cm_state_space_target_row_id": evaluation.get("target_row_id"), + "cm_state_space_horizon": _int(evaluation.get("horizon")), + "cm_state_space_forecast": ( + _optional_float(evaluation.get("forecast")) + if not diagnostic + else None + ), + "cm_state_space_forecast_standard_error": ( + _optional_float(evaluation.get("forecast_standard_error")) + if not diagnostic + else None + ), + "cm_state_space_forecast_lower": ( + _optional_float(evaluation.get("forecast_lower")) + if not diagnostic + else None + ), + "cm_state_space_forecast_upper": ( + _optional_float(evaluation.get("forecast_upper")) + if not diagnostic + else None + ), + "cm_state_space_forecast_available": forecast_available, + "cm_state_space_forecast_available_at_utc_ms": ( + origin_time if not diagnostic else None + ), + "cm_state_space_actual": ( + _optional_float(evaluation.get("actual")) if diagnostic else None + ), + "cm_state_space_error": ( + _optional_float(evaluation.get("error")) if diagnostic else None + ), + "cm_state_space_diagnostic_available": diagnostic, + "cm_state_space_diagnostic_available_at_utc_ms": ( + target_time if diagnostic else None + ), + "cm_state_space_diagnostic_only": diagnostic, + "cm_state_space_original_scale": True, + "cm_state_space_training_eligible": forecast_available, + "cm_kalman_schema_version": STATE_SPACE_STATE_RESULT_SCHEMA_VERSION, + "cm_kalman_model_id": _text(evaluation.get("model_id")), + "cm_kalman_filtered_calculation_basis_code": ( + KALMAN_FILTERED_CALCULATION_BASIS_CODE + ), + "cm_kalman_filtered_level": ( + _vector_value(filtered, level_index) + if filtered_state_available + else None + ), + "cm_kalman_filtered_trend": ( + _vector_value(filtered, trend_index) + if filtered_state_available + else None + ), + "cm_kalman_filtered_level_variance": ( + _vector_value(filtered_variance, level_index) + if filtered_state_available + else None + ), + "cm_kalman_filtered_trend_variance": ( + _vector_value(filtered_variance, trend_index) + if filtered_state_available + else None + ), + "cm_kalman_filtered_available": filtered_state_available, + "cm_kalman_filtered_available_at_utc_ms": ( + origin_time if filtered_state_available else None + ), + "cm_kalman_filtered_training_eligible": filtered_state_available, + "cm_kalman_smoothed_calculation_basis_code": ( + KALMAN_SMOOTHED_CALCULATION_BASIS_CODE + ), + "cm_kalman_smoothed_level": ( + _vector_value(smoothed, level_index) + if smoothed_state_available + else None + ), + "cm_kalman_smoothed_trend": ( + _vector_value(smoothed, trend_index) + if smoothed_state_available + else None + ), + "cm_kalman_smoothed_level_variance": ( + _vector_value(smoothed_variance, level_index) + if smoothed_state_available + else None + ), + "cm_kalman_smoothed_trend_variance": ( + _vector_value(smoothed_variance, trend_index) + if smoothed_state_available + else None + ), + "cm_kalman_smoothed_available": smoothed_state_available, + "cm_kalman_smoothed_available_at_utc_ms": ( + target_time if smoothed_state_available else None + ), + "cm_kalman_smoothed_retrospective": smoothed_state_available, + "cm_kalman_smoothed_diagnostic_only": smoothed_state_available, + "cm_kalman_smoothed_training_eligible": False, + } + + +def _training_projection_metadata( + profile: StateSpaceProfile, + input_result: ClassicalModelInputResult, + annotation_count: int, + collision_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "row", + "identity_fields": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "mapping_policy": "first_source_row_at_or_after_availability", + "collision_policy": "merge_forecast_and_diagnostic_latest_origin_wins", + "collision_count": collision_count, + "annotation_count": annotation_count, + "projection_specification_id": profile.projection_specification_id, + "projection_horizon": profile.projection_horizon, + "input_derivation_id": input_result.contract.get("derivation_id"), + "column_names": list((*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS)), + "column_prefixes": ["cm_state_space_", "cm_kalman_"], + "filtered_state_forecast_safe": True, + "smoothed_state_retrospective_diagnostic_only": True, + "smoothed_state_training_eligible": False, + "full_series_smoothing_projected": False, + "observed_columns_overwritten": False, + } + + +def _base_payload( + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + profile: StateSpaceProfile, +) -> dict[str, JSONValue]: + contract = input_result.contract + regularization = _mapping(contract.get("regularization")) + return { + "schema_version": STATE_SPACE_SCHEMA_VERSION, + "advisory": True, + "target_axis": dict(_mapping(contract.get("target_axis"))), + "reference_fingerprint_id": contract.get("reference_fingerprint_id") + or fingerprint.get("fingerprint_id"), + "input_schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "input_derivation_id": contract.get("derivation_id"), + "input_status": contract.get("status"), + "calculation_basis": "regular_grid_rolling_origin_kalman_filter", + "configuration": profile.to_metadata(), + "input_transform_policy": dict( + _mapping(contract.get("transform_policy")) + ), + "input_missingness_policy": { + "expected_closure_policy": regularization.get( + "expected_closure_policy" + ), + "unexpected_missing_policy": regularization.get( + "unexpected_missing_policy" + ), + "expected_closure_count": regularization.get( + "expected_closure_count" + ), + "unexpected_missing_count": regularization.get( + "unexpected_missing_count" + ), + "forward_fill_policy": regularization.get("forward_fill_policy"), + "backend_missing_observation_behavior": ( + "prediction_only_kalman_transition" + ), + }, + "missing_observation_policy": { + "regular_time_basis": True, + "fill_policy": "none", + "transition_policy": "prediction_only", + "expected_closures_remain_missing": True, + "unexpected_missing_bins_remain_missing": True, + }, + "transition_time_basis": { + "grid": "regular", + "frequency_ms": regularization.get("frequency_ms"), + "irregular_elapsed_scaling_supported": False, + "prediction_only_transition_elapsed_ms": regularization.get( + "frequency_ms" + ), + }, + "forward_fill_policy": "never", + "original_scale_forecasts": True, + "filtering_basis": "origin_training_segment_only", + "smoothing_basis": ( + "origin_training_segment_retrospective_diagnostic_only" + ), + "smoothed_state_used_for_forecast": False, + "full_series_smoothing_used": False, + "automatic_component_selection": False, + "automatic_winner": False, + "hard_fail_quality_gate": False, + "fitted_objects_included": False, + "backend_exception_text_included": False, + } + + +def _unavailable_result( + input_result: ClassicalModelInputResult, + base: Mapping[str, JSONValue], + reason: str, + *, + status: str = "unavailable", +) -> StateSpaceResult: + configuration = _mapping(base.get("configuration")) + diagnostics: dict[str, JSONValue] = { + **dict(base), + "status": status, + "reason": reason, + "limitations": [reason], + "backend": { + "provider": "statsmodels", + "version": None, + "available": reason != "dependency_unavailable", + "model_class": "statsmodels.tsa.statespace.UnobservedComponents", + }, + "fit_summary": { + "schema_version": STATE_SPACE_FIT_SCHEMA_VERSION, + "fit_attempt_count": 0, + "status_counts": {}, + "reason_counts": {reason: 1}, + "warning_counts": {}, + "failed_fit_count": 0, + "limited_fit_count": 0, + "convergence_rate": None, + "failure_rate": None, + "fit_samples": [], + "fit_samples_truncated": False, + }, + "evaluation": { + "schema_version": STATE_SPACE_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin_kalman_filter", + "original_scale": True, + "model_count": 0, + "fold_count": len(input_result.folds), + "evaluated_fold_count": 0, + "skipped_evaluation_count": len(input_result.folds), + "forecast_coverage_rate": None, + "models": [], + "reference_baselines": [], + "reference_models": {}, + "automatic_winner": False, + }, + "resource_usage": { + "estimated_working_memory_bytes": 0, + "memory_limit_exceeded": False, + "fit_attempt_count": 0, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "training_projection": { + "schema_version": STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION, + "column_names": list((*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS)), + "projection_specification_id": configuration.get( + "projection_specification_id" + ), + "annotation_count": 0, + }, + "fit_duration_included": False, + } + return StateSpaceResult(diagnostics, (), input_result) + + +def _ensure_projection_columns(frame: Any) -> Any: + import polars as pl + + definitions = { + "schema_version": pl.Utf8, + "input_derivation_id": pl.Utf8, + "model_id": pl.Utf8, + "converged": pl.Boolean, + "forecast_available": pl.Boolean, + "diagnostic_available": pl.Boolean, + "diagnostic_only": pl.Boolean, + "original_scale": pl.Boolean, + "training_eligible": pl.Boolean, + "filtered_available": pl.Boolean, + "filtered_training_eligible": pl.Boolean, + "smoothed_available": pl.Boolean, + "smoothed_retrospective": pl.Boolean, + "smoothed_diagnostic_only": pl.Boolean, + "smoothed_training_eligible": pl.Boolean, + } + float_suffixes = { + "forecast", + "forecast_standard_error", + "forecast_lower", + "forecast_upper", + "actual", + "error", + "filtered_level", + "filtered_trend", + "filtered_level_variance", + "filtered_trend_variance", + "smoothed_level", + "smoothed_trend", + "smoothed_level_variance", + "smoothed_trend_variance", + } + result = frame + for name in (*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS): + if name in result.columns: + continue + prefix = ( + "cm_state_space_" + if name.startswith("cm_state_space_") + else "cm_kalman_" + ) + suffix = name.removeprefix(prefix) + dtype = definitions.get( + suffix, pl.Float64 if suffix in float_suffixes else pl.Int64 + ) + result = result.with_columns(pl.lit(None).cast(dtype).alias(name)) + return result + + +def _model_id( + specification: StateSpaceSpecification, + derivation_id: str, + backend_version: str, +) -> str: + payload = { + "schema_version": STATE_SPACE_CONFIGURATION_SCHEMA_VERSION, + "derivation_id": derivation_id, + "backend_version": backend_version, + "configuration": specification.to_metadata(), + } + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":") + ).encode() + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _estimated_memory( + row_count: int, + fold_count: int, + specifications: Sequence[StateSpaceSpecification], +) -> int: + component_weight = sum( + max(1, specification.component_count) + for specification in specifications + ) + return ( + max(1, row_count) * max(1, fold_count) * max(1, component_weight) * 64 + ) + + +def _load_backend() -> _Backend | None: + try: + module = importlib.import_module( + "statsmodels.tsa.statespace.structural" + ) + version = importlib.metadata.version("statsmodels") + except (ImportError, importlib.metadata.PackageNotFoundError): + return None + return _Backend( + version=version, unobserved_components=module.UnobservedComponents + ) + + +def _backend_failure_reason(exc: Exception) -> str: + message = str(exc).lower() + if "initial" in message or "diffuse" in message: + return "diffuse_initialization_failure" + if "positive" in message and "cov" in message: + return "non_positive_covariance" + if "singular" in message or "linalg" in message: + return "singular_covariance" + if "identif" in message: + return "unidentifiable_model" + if "seasonal" in message or "frequency" in message: + return "invalid_time_basis" + if "overflow" in message or "finite" in message or "nan" in message: + return "numerical_instability" + if "parameter" in message or "component" in message: + return "invalid_configuration" + return "backend_failure" + + +def _folds_by_origin( + folds: Sequence[Mapping[str, Any]], +) -> list[tuple[tuple[str, str, int], list[dict[str, Any]]]]: + grouped: dict[tuple[str, str, int], list[dict[str, Any]]] = {} + for fold in folds: + key = ( + _text(fold.get("series_id")), + _text(fold.get("period")), + _int(fold.get("origin_bin_end_utc_ms")), + ) + grouped.setdefault(key, []).append(dict(fold)) + return [ + (key, sorted(values, key=lambda item: _int(item.get("horizon")))) + for key, values in sorted(grouped.items()) + ] + + +def _max_missing_run(values: Sequence[float | None]) -> int: + longest = 0 + current = 0 + for value in values: + if value is None: + current += 1 + longest = max(longest, current) + else: + current = 0 + return longest + + +def _finite_or_none(value: Any) -> float | None: + parsed = _optional_float(value) + return parsed if parsed is not None and math.isfinite(parsed) else None + + +def _last_vector(raw: Any) -> tuple[float | None, ...]: + try: + return tuple(_finite_or_none(value) for value in raw[:, -1]) + except (IndexError, TypeError): + return () + + +def _last_covariance_diagonal(raw: Any) -> tuple[float | None, ...]: + try: + matrix = raw[:, :, -1] + return tuple( + _finite_or_none(matrix[index, index]) + for index in range(len(matrix)) + ) + except (IndexError, TypeError): + return () + + +def _states_are_valid(values: Sequence[float | None]) -> bool: + return all(value is None or math.isfinite(value) for value in values) + + +def _retained_state_count(outcome: _FitOutcome, limit: int) -> int: + return min( + limit, + len(outcome.state_names), + len(outcome.filtered_state), + len(outcome.filtered_variance), + len(outcome.smoothed_state), + len(outcome.smoothed_variance), + ) + + +def _sequence_value( + values: Sequence[float], horizon: int, digits: int +) -> float | None: + if 0 < horizon <= len(values): + value = _rounded(values[horizon - 1], digits) + return float(value) if value is not None else None + return None + + +def _state_index(names: Sequence[str], requested: str) -> int | None: + for index, name in enumerate(names): + if name == requested or name.startswith(f"{requested}."): + return index + return None + + +def _vector_value(values: Sequence[Any], index: int | None) -> float | None: + if index is None or index < 0 or index >= len(values): + return None + value = _optional_float(values[index]) + return float(value) if value is not None else None diff --git a/src/histdatacom/data_quality/symbols.py b/src/histdatacom/data_quality/symbols.py index 7c755b2f..af7e0caf 100644 --- a/src/histdatacom/data_quality/symbols.py +++ b/src/histdatacom/data_quality/symbols.py @@ -2,11 +2,13 @@ from __future__ import annotations +from collections import Counter from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field +from itertools import combinations import math from pathlib import Path -from typing import TypeVar +from typing import cast, TypeVar import zipfile from histdatacom.data_quality.contracts import ( @@ -23,6 +25,11 @@ from histdatacom.data_quality.polars_cache import ( read_quality_polars_cache, ) +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.training_features import ( + TRAINING_SCHEMA_VERSION, + ensure_tick_training_features, +) from histdatacom.fx_enums import Pairs from histdatacom.histdata_ascii import TICK, read_ascii_file from histdatacom.runtime_contracts import JSONValue @@ -30,6 +37,15 @@ DOMAIN_SYMBOL_METADATA_RULE_ID = "domain.symbol_metadata" DOMAIN_CROSS_INSTRUMENT_RULE_ID = "domain.cross_instrument_consistency" CROSS_INSTRUMENT_METADATA_KEY = "cross_instrument_consistency" +CROSS_SERIES_FINGERPRINT_RULE_ID = "fingerprint.cross_series" +CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION = ( + "histdatacom.cross-series-fingerprint.v1" +) +CROSS_SERIES_FINGERPRINT_METADATA_KEY = "cross_series_fingerprint" +CROSS_SERIES_FINGERPRINT_SUMMARY_CODE = "FINGERPRINT_CROSS_SERIES_SUMMARY" +TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_topology_summary" +) ASSET_CLASS_FX = "fx" ASSET_CLASS_METAL = "metal" @@ -86,6 +102,15 @@ ) MAX_CROSS_INSTRUMENT_SAMPLES = 5 +DEFAULT_CROSS_SERIES_GROUP_LIMIT = 32 +DEFAULT_CROSS_SERIES_CORRELATION_LIMIT = 32 +CROSS_SERIES_ROW_IDENTITY_COLUMNS = ( + "series_id", + "period", + "row_id", + "source_row_number", + "event_seq", +) _CrossSample = TypeVar("_CrossSample") @@ -216,11 +241,86 @@ def to_metadata(self) -> dict[str, JSONValue]: DEFAULT_CROSS_INSTRUMENT_TOLERANCE = HistDataCrossInstrumentTolerance() +@dataclass(frozen=True, slots=True) +class CrossInstrumentPointInput: + """One public event-time midpoint consumed by cross-series diagnostics.""" + + timestamp_utc_ms: int + price: float + row_id: int + source_row_number: int + event_seq: int + + def __post_init__(self) -> None: + if isinstance(self.timestamp_utc_ms, bool) or not isinstance( + self.timestamp_utc_ms, int + ): + raise ValueError("timestamp_utc_ms must be an integer") + price = float(self.price) + if not math.isfinite(price) or price <= 0.0: + raise ValueError("cross-instrument input price must be positive") + object.__setattr__(self, "price", price) + for name in ("row_id", "source_row_number", "event_seq"): + value = getattr(self, name) + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 0 + ): + raise ValueError(f"{name} must be a non-negative integer") + + +@dataclass(frozen=True, slots=True) +class CrossInstrumentSeriesInput: + """One reconstructed series supplied directly to the #331 rule surface.""" + + symbol: str + timeframe: str + period: str + series_id: str + points: tuple[CrossInstrumentPointInput, ...] + path: str = "" + computed_from: str = "reconstructed_event_stream" + + def __post_init__(self) -> None: + symbol = normalize_histdata_symbol(self.symbol) + if not symbol: + raise ValueError("cross-instrument input requires a symbol") + object.__setattr__(self, "symbol", symbol) + for name in ("timeframe", "period", "series_id", "computed_from"): + value = str(getattr(self, name) or "").strip() + if not value: + raise ValueError(f"cross-instrument input requires {name}") + object.__setattr__(self, name, value) + points = tuple( + sorted( + tuple(self.points), + key=lambda item: ( + item.timestamp_utc_ms, + item.event_seq, + item.row_id, + ), + ) + ) + if not points: + raise ValueError("cross-instrument input requires points") + object.__setattr__(self, "points", points) + path = str(self.path or "").strip() + object.__setattr__( + self, + "path", + path or f"reconstructed://{self.series_id}", + ) + + @dataclass(frozen=True, slots=True) class _CrossInstrumentPoint: timestamp_utc_ms: int price: float row_number: int + row_id: int + source_row_number: int + event_seq: int @dataclass(frozen=True, slots=True) @@ -231,6 +331,12 @@ class _CrossInstrumentSeries: period: str price_kind: str points: dict[int, _CrossInstrumentPoint] + row_count: int + duplicate_timestamp_count: int + computed_from: str + series_id: str + cache_source: str = "" + training_schema_version: str = "" @property def symbol(self) -> str: @@ -269,14 +375,21 @@ def to_metadata(self) -> dict[str, JSONValue]: return { "path": self.target.path, "symbol": self.symbol, + "series_id": self.series_id, "base": self.base, "quote": self.quote, "timeframe": self.timeframe, "period": self.period, "price_kind": self.price_kind, + "row_count": self.row_count, "timestamp_count": self.timestamp_count, + "duplicate_timestamp_count": self.duplicate_timestamp_count, "start_timestamp_utc_ms": self.start_utc_ms, "end_timestamp_utc_ms": self.end_utc_ms, + "computed_from": self.computed_from, + "cache_source": self.cache_source or None, + "training_schema_version": self.training_schema_version or None, + "identity_columns": list(CROSS_SERIES_ROW_IDENTITY_COLUMNS), } @@ -323,6 +436,15 @@ def to_metadata(self) -> dict[str, JSONValue]: "implied_price": self.implied_price, "relative_difference": self.relative_difference, "severity": self.severity.value, + "row_identity": { + "direct": _point_identity(self.direct, self.timestamp_utc_ms), + "numerator": _point_identity( + self.numerator, self.timestamp_utc_ms + ), + "denominator": _point_identity( + self.denominator, self.timestamp_utc_ms + ), + }, "relationship": ( f"{self.numerator.symbol} / {self.denominator.symbol} " f"~= {self.direct.symbol}" @@ -359,6 +481,10 @@ def to_metadata(self) -> dict[str, JSONValue]: "product": self.product, "relative_difference": self.relative_difference, "severity": self.severity.value, + "row_identity": { + "left": _point_identity(self.left, self.timestamp_utc_ms), + "right": _point_identity(self.right, self.timestamp_utc_ms), + }, "relationship": f"{self.left.symbol} * {self.right.symbol} ~= 1", "paths": { "left": self.left.target.path, @@ -410,6 +536,20 @@ def to_metadata(self) -> dict[str, JSONValue]: "start_timestamp_utc_ms": self.start_timestamp_utc_ms, "end_timestamp_utc_ms": self.end_timestamp_utc_ms, "affected_timestamp_count": self.affected_timestamp_count, + "row_identity": { + "stale": _point_identity( + self.stale_series, + self.stale_value_timestamp_utc_ms, + ), + "active_start": _point_identity( + self.active_series, + self.start_timestamp_utc_ms, + ), + "active_end": _point_identity( + self.active_series, + self.end_timestamp_utc_ms, + ), + }, "paths": { "stale": self.stale_series.target.path, "active": self.active_series.target.path, @@ -450,6 +590,44 @@ class _CrossInstrumentScan: stale_join_risks: list[_StaleJoinSample] = field(default_factory=list) +@dataclass(slots=True) +class CrossInstrumentScanProvider: + """Share one expensive cross-instrument scan across run-rule surfaces.""" + + _signature: tuple[object, ...] | None = field(default=None, init=False) + _scan: _CrossInstrumentScan | None = field(default=None, init=False) + + def scan( + self, + targets: tuple[QualityTarget, ...], + *, + tolerance: HistDataCrossInstrumentTolerance, + warning_severity: QualitySeverity, + error_severity: QualitySeverity, + ) -> _CrossInstrumentScan: + """Return a shared scan and release it after the second consumer.""" + signature = _cross_instrument_scan_signature( + targets, + tolerance=tolerance, + warning_severity=warning_severity, + error_severity=error_severity, + ) + if self._scan is not None and self._signature == signature: + scan = self._scan + self._scan = None + self._signature = None + return scan + scan = _scan_cross_instrument_consistency( + targets, + tolerance=tolerance, + warning_severity=warning_severity, + error_severity=error_severity, + ) + self._signature = signature + self._scan = scan + return scan + + FX_NON_JPY_PRECISION_RULE = HistDataSymbolPrecisionRule( name=FX_NON_JPY_PRECISION_RULE_NAME, expected_decimal_places=(6,), @@ -620,6 +798,11 @@ class HistDataCrossInstrumentConsistencyRule: ) warning_severity: QualitySeverity = QualitySeverity.WARNING error_severity: QualitySeverity = QualitySeverity.ERROR + scan_provider: CrossInstrumentScanProvider | None = field( + default=None, + repr=False, + compare=False, + ) rule_id: str = DOMAIN_CROSS_INSTRUMENT_RULE_ID description: str = ( "Compare related FX instruments across one quality run for " @@ -635,11 +818,12 @@ def evaluate_run( ) -> QualityReport: """Return cross-instrument consistency findings.""" target_tuple = tuple(targets) - scan = _scan_cross_instrument_consistency( + scan = _cross_instrument_scan_for_rule( target_tuple, tolerance=self.tolerance, warning_severity=self.warning_severity, error_severity=self.error_severity, + provider=self.scan_provider, ) payload = _cross_instrument_payload( scan, @@ -672,6 +856,121 @@ def evaluate_run( metadata={CROSS_INSTRUMENT_METADATA_KEY: payload}, ) + def evaluate_series( + self, + series: Iterable[CrossInstrumentSeriesInput], + *, + metadata: Mapping[str, JSONValue] | None = None, + ) -> QualityReport: + """Validate reconstructed event streams without a filesystem roundtrip.""" + inputs = tuple(series) + scan = _scan_cross_instrument_series_inputs( + inputs, + tolerance=self.tolerance, + warning_severity=self.warning_severity, + error_severity=self.error_severity, + ) + payload = _cross_instrument_payload(scan, tolerance=self.tolerance) + if not _has_cross_instrument_surface(scan): + return QualityReport( + metadata={CROSS_INSTRUMENT_METADATA_KEY: payload}, + ) + targets = tuple( + _quality_target_for_series_input(item) for item in inputs + ) + run_target = _cross_instrument_target( + targets, + metadata=metadata, + payload=payload, + ) + findings = _cross_instrument_findings( + run_target, + scan=scan, + tolerance=self.tolerance, + rule_id=self.rule_id, + ) + return QualityReport( + rule_results=( + QualityRuleResult( + rule_id=self.rule_id, + target=run_target, + findings=findings, + ), + ), + metadata={CROSS_INSTRUMENT_METADATA_KEY: payload}, + ) + + +@dataclass(slots=True) +class HistDataCrossSeriesFingerprintRule: + """Emit a bounded run-scoped fingerprint for related FX series.""" + + tolerance: HistDataCrossInstrumentTolerance = ( + DEFAULT_CROSS_INSTRUMENT_TOLERANCE + ) + warning_severity: QualitySeverity = QualitySeverity.WARNING + error_severity: QualitySeverity = QualitySeverity.ERROR + scan_provider: CrossInstrumentScanProvider | None = field( + default=None, + repr=False, + compare=False, + ) + group_limit: int = DEFAULT_CROSS_SERIES_GROUP_LIMIT + correlation_limit: int = DEFAULT_CROSS_SERIES_CORRELATION_LIMIT + rule_id: str = CROSS_SERIES_FINGERPRINT_RULE_ID + description: str = ( + "Summarize run-scoped FX panel coverage, topology, dependence, " + "inverse consistency, triangular consistency, and stale-join risk." + ) + + def evaluate_run( + self, + targets: Iterable[QualityTarget], + *, + metadata: Mapping[str, JSONValue] | None = None, + ) -> QualityReport: + """Return one deterministic INFO fingerprint for the quality run.""" + target_tuple = tuple(targets) + scan = _cross_instrument_scan_for_rule( + target_tuple, + tolerance=self.tolerance, + warning_severity=self.warning_severity, + error_severity=self.error_severity, + provider=self.scan_provider, + ) + payload = _cross_series_fingerprint_payload( + scan, + tolerance=self.tolerance, + metadata=metadata, + group_limit=self.group_limit, + correlation_limit=self.correlation_limit, + ) + run_target = _cross_instrument_target( + target_tuple, + metadata=metadata, + payload=payload, + rule_id=self.rule_id, + manifest="cross-series-fingerprint", + ) + finding = _cross_instrument_finding( + run_target, + code=CROSS_SERIES_FINGERPRINT_SUMMARY_CODE, + message="Run-scoped cross-series fingerprint summary.", + severity=QualitySeverity.INFO, + rule_id=self.rule_id, + metadata={CROSS_SERIES_FINGERPRINT_METADATA_KEY: payload}, + ) + return QualityReport( + rule_results=( + QualityRuleResult( + rule_id=self.rule_id, + target=run_target, + findings=(finding,), + ), + ), + metadata={CROSS_SERIES_FINGERPRINT_METADATA_KEY: payload}, + ) + def domain_quality_rules() -> tuple[QualityRule, ...]: """Return domain quality rules in deterministic execution order.""" @@ -746,6 +1045,35 @@ def _target_symbol(target: QualityTarget) -> str: return str(target.symbol or target.metadata.get("symbol", "") or "") +def _cross_series_id(target: QualityTarget) -> str: + symbol = normalize_histdata_symbol(_target_symbol(target)) + return f"ascii:{target.timeframe}:{symbol}:histdata.com" + + +def _duplicate_timestamp_row_count(counts: Counter[int]) -> int: + return sum(count for count in counts.values() if count > 1) + + +def _point_identity( + series: _CrossInstrumentSeries, + timestamp_utc_ms: int, +) -> dict[str, JSONValue]: + point = series.points.get(timestamp_utc_ms) + if point is None: + return { + "series_id": series.series_id, + "period": series.period, + "row_id": None, + } + return { + "series_id": series.series_id, + "period": series.period, + "row_id": point.row_id, + "source_row_number": point.source_row_number, + "event_seq": point.event_seq, + } + + def _domain_finding( target: QualityTarget, *, @@ -770,6 +1098,55 @@ def _domain_finding( ) +def _cross_instrument_scan_for_rule( + targets: tuple[QualityTarget, ...], + *, + tolerance: HistDataCrossInstrumentTolerance, + warning_severity: QualitySeverity, + error_severity: QualitySeverity, + provider: CrossInstrumentScanProvider | None, +) -> _CrossInstrumentScan: + if provider is not None: + return provider.scan( + targets, + tolerance=tolerance, + warning_severity=warning_severity, + error_severity=error_severity, + ) + return _scan_cross_instrument_consistency( + targets, + tolerance=tolerance, + warning_severity=warning_severity, + error_severity=error_severity, + ) + + +def _cross_instrument_scan_signature( + targets: tuple[QualityTarget, ...], + *, + tolerance: HistDataCrossInstrumentTolerance, + warning_severity: QualitySeverity, + error_severity: QualitySeverity, +) -> tuple[object, ...]: + target_axes = tuple( + ( + target.path, + target.kind.value, + target.data_format, + target.timeframe, + target.symbol, + target.period, + ) + for target in targets + ) + return ( + target_axes, + tolerance, + warning_severity.value, + error_severity.value, + ) + + def _scan_cross_instrument_consistency( targets: tuple[QualityTarget, ...], *, @@ -809,6 +1186,100 @@ def _scan_cross_instrument_consistency( continue scan.fx_series.append(series) + return _complete_cross_instrument_scan( + scan, + tolerance=tolerance, + warning_severity=warning_severity, + error_severity=error_severity, + ) + + +def _scan_cross_instrument_series_inputs( + inputs: tuple[CrossInstrumentSeriesInput, ...], + *, + tolerance: HistDataCrossInstrumentTolerance, + warning_severity: QualitySeverity, + error_severity: QualitySeverity, +) -> _CrossInstrumentScan: + scan = _CrossInstrumentScan( + target_count=len(inputs), + ascii_target_count=len(inputs), + ) + for item in inputs: + target = _quality_target_for_series_input(item) + metadata = symbol_metadata_for(item.symbol) + if metadata.asset_class != ASSET_CLASS_FX: + _append_unavailable( + scan, + _CrossInstrumentUnavailable( + reason="non_fx_series", + symbols=(item.symbol,), + timeframe=item.timeframe, + period=item.period, + ), + ) + continue + timestamp_counts = Counter( + point.timestamp_utc_ms for point in item.points + ) + points: dict[int, _CrossInstrumentPoint] = {} + for row_number, point in enumerate(item.points, start=1): + points.setdefault( + point.timestamp_utc_ms, + _CrossInstrumentPoint( + timestamp_utc_ms=point.timestamp_utc_ms, + price=point.price, + row_number=row_number, + row_id=point.row_id, + source_row_number=point.source_row_number, + event_seq=point.event_seq, + ), + ) + scan.fx_series.append( + _CrossInstrumentSeries( + target=target, + metadata=metadata, + timeframe=item.timeframe, + period=item.period, + price_kind="mid_bid_ask", + points=points, + row_count=len(item.points), + duplicate_timestamp_count=_duplicate_timestamp_row_count( + timestamp_counts + ), + computed_from=item.computed_from, + series_id=item.series_id, + ) + ) + return _complete_cross_instrument_scan( + scan, + tolerance=tolerance, + warning_severity=warning_severity, + error_severity=error_severity, + ) + + +def _quality_target_for_series_input( + item: CrossInstrumentSeriesInput, +) -> QualityTarget: + return QualityTarget( + path=item.path, + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=item.timeframe, + symbol=item.symbol, + period=item.period, + metadata={"computed_from": item.computed_from}, + ) + + +def _complete_cross_instrument_scan( + scan: _CrossInstrumentScan, + *, + tolerance: HistDataCrossInstrumentTolerance, + warning_severity: QualitySeverity, + error_severity: QualitySeverity, +) -> _CrossInstrumentScan: if not scan.fx_series: _append_unavailable( scan, @@ -929,6 +1400,7 @@ def _cross_instrument_series( ) points: dict[int, _CrossInstrumentPoint] = {} + timestamp_counts: Counter[int] = Counter() invalid_price_count = 0 for row_number, row in enumerate(batch.rows, start=1): timestamp_utc_ms = int(row[0]) @@ -936,11 +1408,15 @@ def _cross_instrument_series( if price is None: invalid_price_count += 1 continue + timestamp_counts[timestamp_utc_ms] += 1 if timestamp_utc_ms not in points: points[timestamp_utc_ms] = _CrossInstrumentPoint( timestamp_utc_ms=timestamp_utc_ms, price=price, row_number=row_number, + row_id=row_number, + source_row_number=row_number, + event_seq=row_number, ) if not points: @@ -968,6 +1444,12 @@ def _cross_instrument_series( period=target.period, price_kind=_cross_instrument_price_kind(target.timeframe), points=points, + row_count=len(batch.rows), + duplicate_timestamp_count=_duplicate_timestamp_row_count( + timestamp_counts + ), + computed_from="text_scan", + series_id=_cross_series_id(target), ), None, invalid_price_count, @@ -984,7 +1466,7 @@ def _cross_instrument_series_from_polars_cache( ] | None ): - columns = ("datetime", "bid", "ask") + columns = ("datetime", "bid", "ask", "vol") cache = read_quality_polars_cache( target, required_columns=columns, @@ -993,30 +1475,51 @@ def _cross_instrument_series_from_polars_cache( return None try: - frame = cache.frame.select(list(columns)) - rows = frame.iter_rows(named=False) + frame = ensure_tick_training_features(cache.frame, target=target) + selected_columns = ( + "datetime", + "bid", + "ask", + *CROSS_SERIES_ROW_IDENTITY_COLUMNS, + "training_schema_version", + ) + rows = frame.select(list(selected_columns)).iter_rows(named=True) except Exception: return None points: dict[int, _CrossInstrumentPoint] = {} + timestamp_counts: Counter[int] = Counter() invalid_price_count = 0 row_count = 0 + series_id = "" + training_schema_version = "" for row_number, row in enumerate(rows, start=1): row_count += 1 try: - timestamp_utc_ms = int(row[0]) - price = (float(row[1]) + float(row[2])) / 2.0 + timestamp_utc_ms = int(row["datetime"]) + price = (float(row["bid"]) + float(row["ask"])) / 2.0 + row_id = int(row["row_id"]) + source_row_number = int(row["source_row_number"]) + event_seq = int(row["event_seq"]) except (TypeError, ValueError): invalid_price_count += 1 continue if not math.isfinite(price) or price <= 0.0: invalid_price_count += 1 continue + timestamp_counts[timestamp_utc_ms] += 1 + series_id = str(row["series_id"] or series_id) + training_schema_version = str( + row["training_schema_version"] or training_schema_version + ) if timestamp_utc_ms not in points: points[timestamp_utc_ms] = _CrossInstrumentPoint( timestamp_utc_ms=timestamp_utc_ms, price=price, row_number=row_number, + row_id=row_id, + source_row_number=source_row_number, + event_seq=event_seq, ) if not points: @@ -1044,6 +1547,20 @@ def _cross_instrument_series_from_polars_cache( period=target.period, price_kind=_cross_instrument_price_kind(target.timeframe), points=points, + row_count=row_count, + duplicate_timestamp_count=_duplicate_timestamp_row_count( + timestamp_counts + ), + computed_from=( + "direct_cache" + if cache.source == "direct" + else "fresh_sibling_cache" + ), + series_id=series_id or _cross_series_id(target), + cache_source=cache.source, + training_schema_version=( + training_schema_version or TRAINING_SCHEMA_VERSION + ), ), None, invalid_price_count, @@ -1423,6 +1940,594 @@ def _relative_difference_severity( return None +def _cross_series_fingerprint_payload( + scan: _CrossInstrumentScan, + *, + tolerance: HistDataCrossInstrumentTolerance, + metadata: Mapping[str, JSONValue] | None, + group_limit: int, + correlation_limit: int, +) -> dict[str, JSONValue]: + """Return the bounded fingerprint projection of one shared domain scan.""" + group_limit_state = bounded_report_limit( + group_limit, + default_limit=DEFAULT_CROSS_SERIES_GROUP_LIMIT, + ) + correlation_limit_state = bounded_report_limit( + correlation_limit, + default_limit=DEFAULT_CROSS_SERIES_CORRELATION_LIMIT, + ) + topology_summary = _mapping_value( + (metadata or {}).get( + TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY + ) + ) + topology_rows = _mapping_rows(topology_summary.get("target_summaries")) + grouped = _series_by_timeframe_period(scan.fx_series) + unique_series = _unique_cross_series_rows(scan.fx_series) + expected_symbols = sorted({series.symbol for series in unique_series}) + group_payloads = [ + _cross_series_group_payload( + _unique_cross_series_group(group), + timeframe=timeframe, + period=period, + expected_symbols=expected_symbols, + topology_rows=topology_rows, + correlation_limit=correlation_limit_state.effective_limit, + ) + for (timeframe, period), group in sorted(grouped.items()) + if len(expected_symbols) >= 2 + ] + included_groups = list(group_limit_state.slice(group_payloads)) + omitted_group_count = max(0, len(group_payloads) - len(included_groups)) + source_basis_counts = Counter( + series.computed_from for series in unique_series + ) + cache_source_counts = Counter( + series.cache_source for series in unique_series if series.cache_source + ) + duplicate_timestamp_row_count = sum( + series.duplicate_timestamp_count for series in unique_series + ) + incomplete_group_count = sum( + 1 for group in group_payloads if group.get("complete") is not True + ) + correlation_status_counts: Counter[str] = Counter() + for group in group_payloads: + correlation = _mapping_value(group.get("return_correlation")) + for row in _mapping_rows(correlation.get("pairs")): + correlation_status_counts[str(row.get("status") or "unknown")] += 1 + + status = "valid" + if not group_payloads: + status = "unavailable" + elif ( + correlation_status_counts.get("unavailable", 0) + or incomplete_group_count + ): + status = "limited" + + return cast( + dict[str, JSONValue], + { + "schema_version": CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, + "rule_id": CROSS_SERIES_FINGERPRINT_RULE_ID, + "status": status, + "calculation_basis": "shared_cross_instrument_scan", + "data_format": "ascii", + "timeframe": TICK, + "target_count": scan.target_count, + "ascii_tick_target_count": scan.ascii_target_count, + "fx_series_count": len(unique_series), + "group_count": len(group_payloads), + "incomplete_group_count": incomplete_group_count, + "included_group_count": len(included_groups), + "omitted_group_count": omitted_group_count, + "truncated": omitted_group_count > 0, + "limit_metadata": { + "groups": group_limit_state.limit_payload(), + "correlations_per_group": ( + correlation_limit_state.limit_payload() + ), + }, + "source_basis_counts": _sorted_counter(source_basis_counts), + "cache_source_counts": _sorted_counter(cache_source_counts), + "row_identity": { + "columns": list(CROSS_SERIES_ROW_IDENTITY_COLUMNS), + "timestamp_is_durable_identity": False, + "duplicate_timestamp_row_count": ( + duplicate_timestamp_row_count + ), + "legacy_cache_enrichment_required": True, + "training_schema_version": TRAINING_SCHEMA_VERSION, + }, + "topology_basis": { + "schema_version": topology_summary.get("schema_version"), + "target_count": _json_int(topology_summary.get("target_count")), + "included_target_count": _json_int( + topology_summary.get("included_target_count") + ), + "truncated": topology_summary.get("truncated") is True, + }, + "panel_coverage": _cross_series_panel_coverage(unique_series), + "return_correlation_status_counts": _sorted_counter( + correlation_status_counts + ), + "triangular_consistency": _cross_series_consistency_payload( + candidate_count=scan.triangular_candidate_count, + compared_timestamp_count=( + scan.triangular_compared_timestamp_count + ), + warning_count=scan.triangular_warning_count, + error_count=scan.triangular_error_count, + warning_samples=scan.triangular_warnings, + error_samples=scan.triangular_errors, + ), + "inverse_consistency": _cross_series_consistency_payload( + candidate_count=scan.inverse_candidate_count, + compared_timestamp_count=( + scan.inverse_compared_timestamp_count + ), + warning_count=scan.inverse_warning_count, + error_count=scan.inverse_error_count, + warning_samples=scan.inverse_warnings, + error_samples=scan.inverse_errors, + ), + "stale_join_risk": { + "risk_count": scan.stale_join_risk_count, + "samples": _publish_safe_cross_samples(scan.stale_join_risks), + }, + "unavailable": { + "count": len(scan.unavailable), + "samples": _publish_safe_cross_samples(scan.unavailable), + }, + "tolerance": tolerance.to_metadata(), + "groups": included_groups, + }, + ) + + +def _cross_series_panel_coverage( + series_rows: Sequence[_CrossInstrumentSeries], +) -> list[JSONValue]: + by_timeframe: dict[str, list[_CrossInstrumentSeries]] = {} + for series in series_rows: + by_timeframe.setdefault(series.timeframe, []).append(series) + payloads: list[JSONValue] = [] + for timeframe, rows in sorted(by_timeframe.items()): + unique_rows = _unique_cross_series_rows(rows) + by_symbol: dict[str, list[_CrossInstrumentSeries]] = {} + for series in unique_rows: + by_symbol.setdefault(series.symbol, []).append(series) + if len(by_symbol) < 2: + continue + union_periods = sorted({series.period for series in unique_rows}) + common_periods = sorted( + set.intersection( + *( + {series.period for series in symbol_rows} + for symbol_rows in by_symbol.values() + ) + ) + ) + first_period_by_symbol = { + symbol: min(series.period for series in symbol_rows) + for symbol, symbol_rows in by_symbol.items() + } + last_period_by_symbol = { + symbol: max(series.period for series in symbol_rows) + for symbol, symbol_rows in by_symbol.items() + } + common_first_period = max(first_period_by_symbol.values()) + common_last_period = min(last_period_by_symbol.values()) + payloads.append( + cast( + JSONValue, + { + "timeframe": timeframe, + "symbols": sorted(by_symbol), + "union_period_count": len(union_periods), + "common_period_count": len(common_periods), + "common_first_period": common_first_period, + "common_last_period": common_last_period, + "unequal_period_ranges": ( + len(set(first_period_by_symbol.values())) > 1 + or len(set(last_period_by_symbol.values())) > 1 + ), + "limiting_start_symbols": sorted( + symbol + for symbol, period in first_period_by_symbol.items() + if period == common_first_period + ), + "limiting_end_symbols": sorted( + symbol + for symbol, period in last_period_by_symbol.items() + if period == common_last_period + ), + "first_period_by_symbol": dict( + sorted(first_period_by_symbol.items()) + ), + "last_period_by_symbol": dict( + sorted(last_period_by_symbol.items()) + ), + "missing_period_count_by_symbol": { + symbol: len( + set(union_periods).difference( + series.period for series in symbol_rows + ) + ) + for symbol, symbol_rows in sorted(by_symbol.items()) + }, + }, + ) + ) + return payloads + + +def _unique_cross_series_rows( + rows: Sequence[_CrossInstrumentSeries], +) -> list[_CrossInstrumentSeries]: + selected: dict[tuple[str, str, str], _CrossInstrumentSeries] = {} + for series in sorted(rows, key=_cross_series_source_preference): + selected.setdefault( + (series.timeframe, series.period, series.symbol), + series, + ) + return [selected[key] for key in sorted(selected)] + + +def _unique_cross_series_group( + group: Sequence[_CrossInstrumentSeries], +) -> list[_CrossInstrumentSeries]: + return _unique_cross_series_rows(group) + + +def _cross_series_source_preference( + series: _CrossInstrumentSeries, +) -> tuple[int, int, str]: + computation_rank = { + "direct_cache": 0, + "fresh_sibling_cache": 1, + "text_scan": 2, + } + kind_rank = { + QualityTargetKind.CACHE: 0, + QualityTargetKind.CSV: 1, + QualityTargetKind.ZIP: 2, + } + return ( + computation_rank.get(series.computed_from, 99), + kind_rank.get(series.target.kind, 99), + series.target.path, + ) + + +def _cross_series_group_payload( + group: list[_CrossInstrumentSeries], + *, + timeframe: str, + period: str, + expected_symbols: Sequence[str], + topology_rows: list[Mapping[str, JSONValue]], + correlation_limit: int, +) -> dict[str, JSONValue]: + union = set().union(*(series.timestamps for series in group)) + common = set.intersection(*(series.timestamps for series in group)) + starts = [ + start for series in group if (start := series.start_utc_ms) is not None + ] + ends = [end for series in group if (end := series.end_utc_ms) is not None] + common_start = max(starts) if starts else None + common_end = min(ends) if ends else None + symbols = [series.symbol for series in group] + group_topology_rows = [ + row + for row in topology_rows + if _topology_row_matches_group( + row, + timeframe=timeframe, + period=period, + symbols=set(symbols), + ) + ] + return cast( + dict[str, JSONValue], + { + "group_id": f"ascii:{timeframe}:{period}", + "target_axis": { + "data_format": "ascii", + "timeframe": timeframe, + "period": period, + }, + "symbols": symbols, + "expected_symbols": list(expected_symbols), + "missing_symbols": sorted( + set(expected_symbols).difference(symbols) + ), + "complete": set(symbols) == set(expected_symbols), + "series_count": len(group), + "series": [_cross_series_profile(series) for series in group], + "timestamp_grid": { + "union_timestamp_count": len(union), + "common_timestamp_count": len(common), + "common_timestamp_ratio": _rounded_ratio( + len(common), len(union) + ), + "missing_by_symbol": { + series.symbol: len(union.difference(series.timestamps)) + for series in group + }, + }, + "coverage_ranges": { + "common_start_timestamp_utc_ms": common_start, + "common_end_timestamp_utc_ms": common_end, + "unequal_ranges": (len(set(starts)) > 1 or len(set(ends)) > 1), + "limiting_start_symbols": sorted( + series.symbol + for series in group + if series.start_utc_ms == common_start + ), + "limiting_end_symbols": sorted( + series.symbol + for series in group + if series.end_utc_ms == common_end + ), + }, + "topology": _cross_series_topology_payload( + group_topology_rows, + group=group, + ), + "return_correlation": _cross_series_return_correlation_payload( + group, + limit=correlation_limit, + ), + }, + ) + + +def _cross_series_profile( + series: _CrossInstrumentSeries, +) -> dict[str, JSONValue]: + return cast( + dict[str, JSONValue], + { + "series_id": series.series_id, + "symbol": series.symbol, + "period": series.period, + "timeframe": series.timeframe, + "price_kind": series.price_kind, + "row_count": series.row_count, + "unique_timestamp_count": series.timestamp_count, + "duplicate_timestamp_row_count": series.duplicate_timestamp_count, + "start_timestamp_utc_ms": series.start_utc_ms, + "end_timestamp_utc_ms": series.end_utc_ms, + "computed_from": series.computed_from, + "cache_source": series.cache_source or None, + "training_schema_version": series.training_schema_version or None, + "identity_columns": list(CROSS_SERIES_ROW_IDENTITY_COLUMNS), + }, + ) + + +def _cross_series_topology_payload( + rows: list[Mapping[str, JSONValue]], + *, + group: Sequence[_CrossInstrumentSeries], +) -> dict[str, JSONValue]: + count_keys = ( + "row_count", + "parsed_row_count", + "invalid_timestamp_count", + "duplicate_timestamp_count", + "non_monotonic_count", + "suspicious_gap_count", + "expected_session_closure_count", + "weekend_activity_count", + ) + topology_computed_from_counts = Counter( + str(row.get("computed_from") or "unknown") for row in rows + ) + topology_cache_source_counts = Counter( + str(row.get("cache_source")) + for row in rows + if row.get("cache_source") is not None + ) + computed_from_counts = Counter(series.computed_from for series in group) + cache_source_counts = Counter( + series.cache_source for series in group if series.cache_source + ) + payload: dict[str, JSONValue] = { + key: sum(_json_int(row.get(key)) for row in rows) for key in count_keys + } + payload.update( + { + "target_count": len(rows), + "source_series_count": len(group), + "duplicate_timestamp_row_count": sum( + series.duplicate_timestamp_count for series in group + ), + "computed_from_counts": _sorted_counter(computed_from_counts), + "cache_source_counts": _sorted_counter(cache_source_counts), + "topology_computed_from_counts": _sorted_counter( + topology_computed_from_counts + ), + "topology_cache_source_counts": _sorted_counter( + topology_cache_source_counts + ), + "mixed_computation_basis": len(computed_from_counts) > 1, + "mixed_cache_source": len(cache_source_counts) > 1, + } + ) + return payload + + +def _cross_series_return_correlation_payload( + group: list[_CrossInstrumentSeries], + *, + limit: int, +) -> dict[str, JSONValue]: + rows = [ + _return_correlation_pair(left, right) + for left, right in combinations(group, 2) + ] + limit_state = bounded_report_limit( + limit, + default_limit=DEFAULT_CROSS_SERIES_CORRELATION_LIMIT, + ) + included = list(limit_state.slice(rows)) + omitted = max(0, len(rows) - len(included)) + return cast( + dict[str, JSONValue], + { + "pair_count": len(rows), + "included_pair_count": len(included), + "omitted_pair_count": omitted, + "truncated": omitted > 0, + "limit_metadata": {"pairs": limit_state.limit_payload()}, + "pairs": included, + }, + ) + + +def _return_correlation_pair( + left: _CrossInstrumentSeries, + right: _CrossInstrumentSeries, +) -> dict[str, JSONValue]: + left_returns = _log_returns_by_timestamp(left) + right_returns = _log_returns_by_timestamp(right) + common = sorted(set(left_returns).intersection(right_returns)) + payload: dict[str, JSONValue] = { + "left_symbol": left.symbol, + "right_symbol": right.symbol, + "overlap_return_count": len(common), + } + if len(common) < 2: + return { + **payload, + "status": "unavailable", + "reason": "insufficient_overlap", + } + left_values = [left_returns[timestamp] for timestamp in common] + right_values = [right_returns[timestamp] for timestamp in common] + correlation = _pearson_correlation(left_values, right_values) + if correlation is None: + return {**payload, "status": "unavailable", "reason": "zero_variance"} + return { + **payload, + "status": "valid", + "correlation": round(correlation, 12), + } + + +def _log_returns_by_timestamp( + series: _CrossInstrumentSeries, +) -> dict[int, float]: + returns: dict[int, float] = {} + previous_price: float | None = None + for timestamp, point in sorted(series.points.items()): + if ( + previous_price is not None + and previous_price > 0 + and point.price > 0 + ): + returns[timestamp] = math.log(point.price / previous_price) + previous_price = point.price + return returns + + +def _pearson_correlation( + left: Sequence[float], + right: Sequence[float], +) -> float | None: + count = len(left) + if count != len(right) or count < 2: + return None + left_mean = math.fsum(left) / count + right_mean = math.fsum(right) / count + left_delta = [value - left_mean for value in left] + right_delta = [value - right_mean for value in right] + numerator = math.fsum( + left_value * right_value + for left_value, right_value in zip(left_delta, right_delta, strict=True) + ) + left_scale = math.fsum(value * value for value in left_delta) + right_scale = math.fsum(value * value for value in right_delta) + denominator = math.sqrt(left_scale * right_scale) + if denominator == 0: + return None + return numerator / denominator + + +def _cross_series_consistency_payload( + *, + candidate_count: int, + compared_timestamp_count: int, + warning_count: int, + error_count: int, + warning_samples: Sequence[object], + error_samples: Sequence[object], +) -> dict[str, JSONValue]: + return { + "candidate_count": candidate_count, + "compared_timestamp_count": compared_timestamp_count, + "warning_count": warning_count, + "error_count": error_count, + "warning_samples": _publish_safe_cross_samples(warning_samples), + "error_samples": _publish_safe_cross_samples(error_samples), + } + + +def _publish_safe_cross_samples( + samples: Sequence[object], +) -> list[JSONValue]: + payloads: list[JSONValue] = [] + for sample in samples[:MAX_CROSS_INSTRUMENT_SAMPLES]: + converter = getattr(sample, "to_metadata", None) + if not callable(converter): + continue + payload = dict(converter()) + payload.pop("path", None) + payload.pop("paths", None) + payload.pop("error", None) + payloads.append(payload) + return payloads + + +def _topology_row_matches_group( + row: Mapping[str, JSONValue], + *, + timeframe: str, + period: str, + symbols: set[str], +) -> bool: + axis = _mapping_value(row.get("target_axis")) + return ( + str(axis.get("timeframe") or "").upper() == timeframe.upper() + and str(axis.get("period") or "") == period + and str(axis.get("symbol") or "").upper() in symbols + ) + + +def _mapping_value(value: object) -> Mapping[str, JSONValue]: + return value if isinstance(value, Mapping) else {} + + +def _mapping_rows(value: object) -> list[Mapping[str, JSONValue]]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, Mapping)] + + +def _sorted_counter(counter: Counter[str]) -> dict[str, JSONValue]: + return {key: counter[key] for key in sorted(counter)} + + +def _rounded_ratio(numerator: int, denominator: int) -> float | None: + if denominator <= 0: + return None + return round(numerator / denominator, 12) + + def _cross_instrument_payload( scan: _CrossInstrumentScan, *, @@ -1484,17 +2589,19 @@ def _cross_instrument_target( *, metadata: Mapping[str, JSONValue] | None, payload: Mapping[str, JSONValue], + rule_id: str = DOMAIN_CROSS_INSTRUMENT_RULE_ID, + manifest: str = "cross-instrument-consistency", ) -> QualityTarget: root = _quality_run_root(metadata) if not root and targets: root = str(Path(targets[0].path).parent) return QualityTarget( - path=root or "cross-instrument-consistency", + path=root or manifest, kind=QualityTargetKind.DIRECTORY, data_format="ascii", metadata={ - "manifest": "cross-instrument-consistency", - "rule_id": DOMAIN_CROSS_INSTRUMENT_RULE_ID, + "manifest": manifest, + "rule_id": rule_id, "target_count": _json_int(payload.get("target_count")), "fx_series_count": _json_int(payload.get("fx_series_count")), "triangular_candidate_count": _json_int( diff --git a/src/histdatacom/data_quality/synthetic_constraints.py b/src/histdatacom/data_quality/synthetic_constraints.py new file mode 100644 index 00000000..64637d11 --- /dev/null +++ b/src/histdatacom/data_quality/synthetic_constraints.py @@ -0,0 +1,1310 @@ +"""Generator-facing synthetic tick constraint and validation contracts.""" + +from __future__ import annotations + +import json +import math +from collections import Counter +from collections.abc import Iterable, Mapping +from typing import Any, cast + +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityReport, +) +from histdatacom.data_quality.limits import ( + BoundedReportLimit, + bounded_report_limit, +) +from histdatacom.data_quality.training_features import ( + IDENTITY_COLUMNS, + QUALITY_ISSUE_COLUMNS, + SYNTHETIC_PLACEHOLDER_COLUMNS, + TRAINING_REQUIRED_COLUMNS, + TRAINING_SCHEMA_VERSION, + ensure_tick_training_features, +) +from histdatacom.histdata_ascii import TICK +from histdatacom.runtime_contracts import JSONValue + +SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION = ( + "histdatacom.synthetic-fingerprint-constraints.v1" +) +SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.synthetic-fingerprint-constraint-summary.v1" +) +SYNTHETIC_VALIDATION_SCHEMA_VERSION = ( + "histdatacom.synthetic-fingerprint-validation.v1" +) +SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY = ( + "time_series_fingerprint_synthetic_constraint_summary" +) +SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY = "fingerprint_synthetic_constraints" +TIME_SERIES_FINGERPRINT_METADATA_KEY = "time_series_fingerprint" + +DEFAULT_SYNTHETIC_CONSTRAINT_CATEGORY_LIMIT = 32 +DEFAULT_SYNTHETIC_CONSTRAINT_HINT_LIMIT = 32 +DEFAULT_SYNTHETIC_CONSTRAINT_SUMMARY_LIMIT = 32 +DEFAULT_SYNTHETIC_VALIDATION_TARGET_LIMIT = 32 +DEFAULT_SYNTHETIC_VALIDATION_MISMATCH_LIMIT = 32 + +_DEFECT_SPECS = ( + ( + "avoid_negative_spread", + "dq_issue_negative_spread", + "negative_spread_count", + "hard", + ), + ( + "avoid_duplicate_timestamps", + "dq_issue_duplicate_timestamp", + "duplicate_timestamp_count", + "advisory", + ), + ( + "avoid_non_monotonic_timestamps", + "dq_issue_non_monotonic_timestamp", + "non_monotonic_count", + "hard", + ), + ( + "avoid_suspicious_non_session_gaps", + "dq_issue_suspicious_gap", + "suspicious_gap_count", + "advisory", + ), + ( + "avoid_invalid_rows", + "dq_issue_invalid_row", + "invalid_row_count", + "hard", + ), + ( + "avoid_partial_rows", + "dq_issue_partial_row", + "partial_row_count", + "hard", + ), + ( + "avoid_topology_unavailable", + "dq_issue_topology_unavailable", + "topology_unavailable_count", + "hard", + ), + ( + "avoid_fingerprint_unready", + "dq_issue_fingerprint_unready", + "fingerprint_unready_count", + "advisory", + ), +) + + +def synthetic_constraints_from_training_frame( + training_frame: Any, + *, + fingerprint: Mapping[str, JSONValue] | None = None, + target: Any | None = None, + category_limit: int | None = DEFAULT_SYNTHETIC_CONSTRAINT_CATEGORY_LIMIT, + hint_limit: int | None = DEFAULT_SYNTHETIC_CONSTRAINT_HINT_LIMIT, +) -> dict[str, JSONValue]: + """Derive constraints with the enriched tick frame as the primary input.""" + payload = dict(fingerprint or {}) + if "target_axis" not in payload: + payload["target_axis"] = _target_axis_from_target(target) + if "source" not in payload: + payload["source"] = { + "kind": "cache", + "cache_source": "direct", + } + return synthetic_constraints_from_fingerprint( + payload, + training_frame=training_frame, + target=target, + category_limit=category_limit, + hint_limit=hint_limit, + ) + + +def synthetic_constraints_from_fingerprint( + fingerprint: Mapping[str, JSONValue], + *, + training_frame: Any | None = None, + target: Any | None = None, + category_limit: int | None = DEFAULT_SYNTHETIC_CONSTRAINT_CATEGORY_LIMIT, + hint_limit: int | None = DEFAULT_SYNTHETIC_CONSTRAINT_HINT_LIMIT, +) -> dict[str, JSONValue]: + """Derive bounded ASCII-tick generator constraints from one fingerprint.""" + category_state = bounded_report_limit( + category_limit, + default_limit=DEFAULT_SYNTHETIC_CONSTRAINT_CATEGORY_LIMIT, + ) + hint_state = bounded_report_limit( + hint_limit, + default_limit=DEFAULT_SYNTHETIC_CONSTRAINT_HINT_LIMIT, + ) + target_axis = _mapping(fingerprint.get("target_axis")) + source = _mapping(fingerprint.get("source")) + supported = ( + _text(target_axis.get("data_format")) == "ascii" + and _text(target_axis.get("timeframe")) == TICK + ) + training, enriched = _training_substrate_payload( + training_frame, + target=target, + source=source, + ) + defects = _defect_constraints(fingerprint, enriched) + stylized = _stylized_fact_constraints(fingerprint, enriched) + artifacts = _source_artifact_constraints(fingerprint, training) + hints = _constraint_hints(fingerprint, defects, stylized) + limitations = _constraint_limitations( + fingerprint, + training, + supported=supported, + ) + included_defects = category_state.slice(defects) + included_stylized = category_state.slice(stylized) + included_artifacts = category_state.slice(artifacts) + included_hints = hint_state.slice(hints) + status = "ready" + if not supported: + status = "unavailable" + elif limitations: + status = "limited" + payload: dict[str, JSONValue] = { + "schema_version": SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION, + "status": status, + "advisory": True, + "base_grain": {"data_format": "ascii", "timeframe": TICK}, + "target_axis": dict(target_axis), + "training_substrate": training, + "output_contract": { + "observed_columns_preserved": ["bid", "ask"], + "synthetic_output_columns": list(SYNTHETIC_PLACEHOLDER_COLUMNS), + "durable_identity_columns": [ + "series_id", + "period", + "row_id", + ], + "timestamp_is_sole_identity": False, + "non_tick_input_constraints_supported": False, + "generation_in_scope": True, + "generation_issue": "#81", + "generation_method": "empirical_block_bootstrap", + }, + "defect_count": len(defects), + "stylized_fact_count": len(stylized), + "source_artifact_count": len(artifacts), + "hint_count": len(hints), + "included_defect_count": len(included_defects), + "included_stylized_fact_count": len(included_stylized), + "included_source_artifact_count": len(included_artifacts), + "included_hint_count": len(included_hints), + "omitted_defect_count": max(0, len(defects) - len(included_defects)), + "omitted_stylized_fact_count": max( + 0, len(stylized) - len(included_stylized) + ), + "omitted_source_artifact_count": max( + 0, len(artifacts) - len(included_artifacts) + ), + "omitted_hint_count": max(0, len(hints) - len(included_hints)), + "truncated": any( + ( + len(included_defects) < len(defects), + len(included_stylized) < len(stylized), + len(included_artifacts) < len(artifacts), + len(included_hints) < len(hints), + ) + ), + "limit_metadata": { + "categories": category_state.limit_payload(), + "hints": hint_state.limit_payload(), + }, + "defects_to_avoid": cast(JSONValue, included_defects), + "stylized_facts_to_preserve": cast(JSONValue, included_stylized), + "source_artifacts_to_parameterize": cast(JSONValue, included_artifacts), + "advisory_hints": cast(JSONValue, included_hints), + "limitation_codes": cast(JSONValue, limitations), + } + payload["constraint_id"] = _payload_id(payload) + return payload + + +def synthetic_constraint_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = DEFAULT_SYNTHETIC_CONSTRAINT_SUMMARY_LIMIT, +) -> dict[str, JSONValue] | None: + """Return a bounded report-level rollup of synthetic constraints.""" + limit_state = bounded_report_limit( + target_limit, + default_limit=DEFAULT_SYNTHETIC_CONSTRAINT_SUMMARY_LIMIT, + ) + targets: list[dict[str, JSONValue]] = [] + status_counts: Counter[str] = Counter() + defect_counts: Counter[str] = Counter() + hint_counts: Counter[str] = Counter() + substrate_counts: Counter[str] = Counter() + for finding in findings: + fingerprint = _mapping( + finding.metadata.get(TIME_SERIES_FINGERPRINT_METADATA_KEY) + ) + constraints = _mapping(fingerprint.get("synthetic_constraints")) + if not constraints: + continue + status = _text(constraints.get("status")) or "unknown" + training = _mapping(constraints.get("training_substrate")) + substrate_status = _text(training.get("status")) or "unknown" + status_counts[status] += 1 + substrate_counts[substrate_status] += 1 + defects = _mapping_rows(constraints.get("defects_to_avoid")) + for defect in defects: + if _int(defect.get("observed_count")) > 0: + defect_counts[_text(defect.get("code"))] += 1 + hints = _strings(constraints.get("advisory_hints")) + hint_counts.update(hints) + targets.append( + { + "target_axis": dict(_mapping(constraints.get("target_axis"))), + "status": status, + "constraint_id": constraints.get("constraint_id"), + "training_substrate_status": substrate_status, + "observed_defect_count": sum( + 1 + for defect in defects + if _int(defect.get("observed_count")) > 0 + ), + "stylized_fact_count": _int( + constraints.get("stylized_fact_count") + ), + "advisory_hints": cast(JSONValue, hints), + "limitation_codes": cast( + JSONValue, + _strings(constraints.get("limitation_codes")), + ), + } + ) + if not targets: + return None + targets.sort(key=_target_sort_key) + included = limit_state.slice(targets) + omitted = max(0, len(targets) - len(included)) + return { + "schema_version": SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION, + "target_count": len(targets), + "ready_target_count": status_counts.get("ready", 0), + "limited_target_count": status_counts.get("limited", 0), + "unavailable_target_count": status_counts.get("unavailable", 0), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "limit_metadata": {"targets": limit_state.limit_payload()}, + "status_counts": _counter_payload(status_counts), + "training_substrate_status_counts": _counter_payload(substrate_counts), + "observed_defect_target_counts": _counter_payload(defect_counts), + "advisory_hint_target_counts": _counter_payload(hint_counts), + "target_summaries": cast(JSONValue, included), + } + + +def validate_synthetic_constraint_reports( + reference: QualityReport, + candidate: QualityReport, + *, + target_limit: int | None = DEFAULT_SYNTHETIC_VALIDATION_TARGET_LIMIT, + mismatch_limit: int | None = DEFAULT_SYNTHETIC_VALIDATION_MISMATCH_LIMIT, +) -> dict[str, JSONValue]: + """Compare candidate fingerprint constraints with reference constraints.""" + target_state = bounded_report_limit( + target_limit, + default_limit=DEFAULT_SYNTHETIC_VALIDATION_TARGET_LIMIT, + ) + mismatch_state = bounded_report_limit( + mismatch_limit, + default_limit=DEFAULT_SYNTHETIC_VALIDATION_MISMATCH_LIMIT, + ) + references = _constraints_by_axis(reference.findings) + candidates = _constraints_by_axis(candidate.findings) + results: list[dict[str, JSONValue]] = [] + mismatch_code_counts: Counter[str] = Counter() + for axis_key, constraints in sorted(references.items()): + candidate_constraints = candidates.get(axis_key) + result = _validate_target_constraints( + constraints, + candidate_constraints, + mismatch_state=mismatch_state, + ) + results.append(result) + for code, count in _mapping(result.get("mismatch_code_counts")).items(): + mismatch_code_counts[code] += _int(count) + results.sort(key=_validation_target_sort_key) + included = target_state.slice(results) + omitted = max(0, len(results) - len(included)) + compared = sum( + 1 for item in results if item.get("status") != "not_compared" + ) + mismatched = sum(1 for item in results if item.get("status") == "mismatch") + status = "not_compared" + if compared: + status = "mismatch" if mismatched else "match" + return { + "schema_version": SYNTHETIC_VALIDATION_SCHEMA_VERSION, + "status": status, + "advisory": True, + "base_grain": {"data_format": "ascii", "timeframe": TICK}, + "reference_target_count": len(references), + "candidate_target_count": len(candidates), + "compared_target_count": compared, + "matching_target_count": sum( + 1 for item in results if item.get("status") == "match" + ), + "mismatched_target_count": mismatched, + "not_compared_target_count": sum( + 1 for item in results if item.get("status") == "not_compared" + ), + "mismatch_count": sum(mismatch_code_counts.values()), + "mismatch_code_counts": _counter_payload(mismatch_code_counts), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "limit_metadata": { + "targets": target_state.limit_payload(), + "mismatches": mismatch_state.limit_payload(), + }, + "target_results": cast(JSONValue, included), + } + + +def format_synthetic_constraint_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> list[str]: + """Return concise human-readable constraint summary lines.""" + if not summary: + return [] + lines = [ + "", + "Synthetic fingerprint constraints", + ( + "- targets: " + f"{_int(summary.get('target_count'))} " + f"ready: {_int(summary.get('ready_target_count'))} " + f"limited: {_int(summary.get('limited_target_count'))} " + f"unavailable: {_int(summary.get('unavailable_target_count'))}" + ), + ] + defect_counts = _mapping(summary.get("observed_defect_target_counts")) + if defect_counts: + lines.append("- observed defects: " + _format_counts(defect_counts)) + for target in _mapping_rows(summary.get("target_summaries")): + axis = _mapping(target.get("target_axis")) + lines.append( + "- " + f"{_format_axis(axis)}: {_text(target.get('status'))} " + f"facts={_int(target.get('stylized_fact_count'))} " + f"observed-defects={_int(target.get('observed_defect_count'))}" + ) + return lines + + +def format_synthetic_validation(payload: Mapping[str, JSONValue]) -> str: + """Return concise human-readable synthetic validation output.""" + lines = [ + "Synthetic fingerprint validation", + f"status: {_text(payload.get('status'))}", + ( + "targets: " + f"{_int(payload.get('reference_target_count'))} reference, " + f"{_int(payload.get('candidate_target_count'))} candidate, " + f"{_int(payload.get('mismatched_target_count'))} mismatched" + ), + ] + counts = _mapping(payload.get("mismatch_code_counts")) + if counts: + lines.append("mismatch codes: " + _format_counts(counts)) + for result in _mapping_rows(payload.get("target_results")): + axis = _mapping(result.get("target_axis")) + codes = ",".join(_strings(result.get("mismatch_codes"))) or "none" + lines.append( + f"- {_format_axis(axis)}: {_text(result.get('status'))} " + f"codes={codes}" + ) + return "\n".join(lines) + + +def _training_substrate_payload( + frame: Any | None, + *, + target: Any | None, + source: Mapping[str, JSONValue], +) -> tuple[dict[str, JSONValue], Any | None]: + unavailable: dict[str, JSONValue] = { + "status": "unavailable", + "reason": "training_frame_unavailable", + "training_schema_version": None, + "row_count": 0, + "required_column_count": len(TRAINING_REQUIRED_COLUMNS), + "missing_required_columns": list(TRAINING_REQUIRED_COLUMNS), + } + if frame is None: + return unavailable, None + raw_columns = set(getattr(frame, "columns", ())) + try: + enriched = ensure_tick_training_features(frame, target=target) + except (KeyError, OSError, TypeError, ValueError) as exc: + unavailable["reason"] = "training_enrichment_unavailable" + unavailable["error_type"] = type(exc).__name__ + return unavailable, None + columns = set(getattr(enriched, "columns", ())) + missing = sorted(set(TRAINING_REQUIRED_COLUMNS) - columns) + input_kind = _text(source.get("kind")) or "unknown" + cache_input = input_kind == "cache" + input_was_enriched = "training_schema_version" in raw_columns + return ( + { + "status": "available" if not missing else "limited", + "training_schema_version": TRAINING_SCHEMA_VERSION, + "row_count": int(getattr(enriched, "height", 0) or 0), + "required_column_count": len(TRAINING_REQUIRED_COLUMNS), + "column_count": len(columns), + "missing_required_columns": cast(JSONValue, missing), + "input_kind": input_kind, + "cache_source": source.get("cache_source"), + "cache_was_enriched": cache_input and input_was_enriched, + "legacy_cache_enriched_on_read": ( + cache_input and not input_was_enriched + ), + "source_rows_enriched_in_memory": ( + not cache_input and not input_was_enriched + ), + "identity_columns_present": set(IDENTITY_COLUMNS).issubset(columns), + "quality_issue_columns_present": set( + QUALITY_ISSUE_COLUMNS + ).issubset(columns), + "synthetic_output_columns_present": set( + SYNTHETIC_PLACEHOLDER_COLUMNS + ).issubset(columns), + }, + enriched, + ) + + +def _defect_constraints( + fingerprint: Mapping[str, JSONValue], + frame: Any | None, +) -> list[dict[str, JSONValue]]: + fallback = _fingerprint_defect_counts(fingerprint) + constraints: list[dict[str, JSONValue]] = [] + for code, column, fallback_key, severity in _DEFECT_SPECS: + count = _frame_true_count(frame, column) + source = "training_feature_column" + if count is None: + count = fallback.get(fallback_key, 0) + source = "fingerprint_fallback" + constraints.append( + { + "code": code, + "issue_column": column, + "severity": severity, + "requirement": "must_be_zero", + "observed_count": count, + "source": source, + } + ) + missing_columns = [] + if frame is not None: + missing_columns = sorted( + set(TRAINING_REQUIRED_COLUMNS) - set(frame.columns) + ) + constraints.append( + { + "code": "avoid_unsupported_schema", + "issue_column": None, + "severity": "hard", + "requirement": "canonical_training_schema_required", + "observed_count": 1 if missing_columns else 0, + "source": "training_schema_contract", + "missing_column_count": len(missing_columns), + } + ) + constraints.append( + { + "code": "avoid_structurally_invalid_timestamps", + "issue_column": None, + "severity": "hard", + "requirement": "must_be_zero", + "observed_count": fallback.get("invalid_timestamp_count", 0), + "source": "temporal_topology", + } + ) + return constraints + + +def _fingerprint_defect_counts( + fingerprint: Mapping[str, JSONValue], +) -> dict[str, int]: + topology = _mapping(fingerprint.get("temporal_topology")) + distribution = _mapping(fingerprint.get("tick_distribution")) + audit = _mapping(fingerprint.get("fingerprint_audit")) + statuses = _mapping(audit.get("section_statuses")) + unavailable_topology = ( + 1 + if _text(topology.get("computed_from")) == "unavailable" + or topology.get("parsed_row_count") is None + else 0 + ) + unready = sum( + 1 + for status in statuses.values() + if _text(status) in {"limited", "unavailable", "skipped"} + ) + return { + "negative_spread_count": _int( + distribution.get("negative_spread_count") + ), + "duplicate_timestamp_count": _int( + topology.get("duplicate_timestamp_count") + ), + "non_monotonic_count": _int(topology.get("non_monotonic_count")), + "suspicious_gap_count": _int(topology.get("suspicious_gap_count")), + "invalid_row_count": _int(distribution.get("invalid_row_count")), + "partial_row_count": _int(distribution.get("partial_row_count")), + "topology_unavailable_count": unavailable_topology, + "fingerprint_unready_count": unready, + "invalid_timestamp_count": _int( + topology.get("invalid_timestamp_count") + ), + } + + +def _stylized_fact_constraints( + fingerprint: Mapping[str, JSONValue], + training_frame: Any | None, +) -> list[dict[str, JSONValue]]: + topology = _mapping(fingerprint.get("temporal_topology")) + calendar = _mapping(fingerprint.get("calendar_regimes")) + distribution = _mapping(fingerprint.get("tick_distribution")) + dynamics = _mapping(fingerprint.get("microstructure_dynamics")) + dependence = _mapping(fingerprint.get("dependence")) + stationarity = _mapping(fingerprint.get("stationarity_diagnostics")) + decomposition = _mapping(fingerprint.get("decomposition")) + facts: list[dict[str, JSONValue]] = [] + _append_fact( + facts, + "session_activity_mix", + "calendar_regimes.session_state_counts", + calendar.get("session_state_counts"), + "distribution_l1", + 0.1, + ) + _append_fact( + facts, + "active_session_mix", + "calendar_regimes.active_session_counts", + calendar.get("active_session_counts"), + "distribution_l1", + 0.1, + ) + _append_fact( + facts, + "calendar_special_tag_mix", + "calendar_regimes.special_tag_counts", + calendar.get("special_tag_counts"), + "distribution_l1", + 0.1, + ) + _append_fact( + facts, + "gap_bucket_shape", + "temporal_topology.gap_bucket_counts", + topology.get("gap_bucket_counts"), + "distribution_l1", + 0.1, + ) + _append_fact( + facts, + "interval_topology", + "temporal_topology", + _selected( + topology, + ( + "sampling_basis", + "min_interval_ms", + "median_interval_ms", + "max_gap_ms", + "expected_session_closure_count", + "weekend_activity_count", + ), + ), + "numeric_relative_tolerance", + 0.1, + ) + _append_fact( + facts, + "spread_distribution", + "tick_distribution.spread", + _without_count(_mapping(distribution.get("spread"))), + "numeric_relative_tolerance", + 0.1, + ) + _append_fact( + facts, + "precision_regime", + "training_substrate.observed_float_decimal_places", + _training_precision_regime(training_frame), + "distribution_l1", + 0.0, + ) + for code, key in ( + ("spread_jump_behavior", "spread_jump"), + ("stale_quote_runs", "stale_quote"), + ("burst_behavior", "burst"), + ("one_sided_movement", "one_sided_movement"), + ): + _append_fact( + facts, + code, + f"microstructure_dynamics.{key}", + dynamics.get(key), + "numeric_relative_tolerance", + 0.1, + ) + absolute_acf = _mapping(dependence.get("absolute_spread_change_acf")) + _append_fact( + facts, + "volatility_clustering_proxy", + "dependence.absolute_spread_change_acf.lag_acf", + absolute_acf.get("lag_acf"), + "numeric_absolute_tolerance", + 0.2, + ) + shift = _mapping(stationarity.get("first_middle_last_distribution_shift")) + _append_fact( + facts, + "return_tail_shape", + "stationarity_diagnostics.first_middle_last_distribution_shift.return", + shift.get("return"), + "numeric_relative_tolerance", + 0.1, + ) + _append_fact( + facts, + "rolling_drift", + "stationarity_diagnostics.rolling_windows", + stationarity.get("rolling_windows"), + "numeric_relative_tolerance", + 0.1, + ) + _append_fact( + facts, + "stationarity_transform_policy", + "stationarity_diagnostics.recommended_transforms", + stationarity.get("recommended_transforms"), + "exact", + 0.0, + ) + _append_fact( + facts, + "structural_regime_proxy", + "decomposition.structural_break_proxy", + decomposition.get("structural_break_proxy"), + "numeric_relative_tolerance", + 0.1, + ) + return facts + + +def _append_fact( + facts: list[dict[str, JSONValue]], + code: str, + source: str, + value: JSONValue | None, + comparison: str, + tolerance: float, +) -> None: + if value in (None, {}, []): + return + facts.append( + { + "code": code, + "source": source, + "comparison": comparison, + "tolerance": tolerance, + "value": value, + } + ) + + +def _source_artifact_constraints( + fingerprint: Mapping[str, JSONValue], + training: Mapping[str, JSONValue], +) -> list[dict[str, JSONValue]]: + source = _mapping(fingerprint.get("source")) + coverage = _mapping(fingerprint.get("coverage")) + topology = _mapping(fingerprint.get("temporal_topology")) + calendar = _mapping(fingerprint.get("calendar_regimes")) + dynamics = _mapping(fingerprint.get("microstructure_dynamics")) + stationarity = _mapping(fingerprint.get("stationarity_diagnostics")) + artifacts = [ + _artifact( + "parameterize_source_provenance", + "source", + _selected(source, ("kind", "cache_source")), + ), + _artifact( + "parameterize_source_coverage", + "coverage", + _selected( + coverage, + ( + "row_count", + "parsed_row_count", + "start_timestamp_utc_ms", + "end_timestamp_utc_ms", + ), + ), + ), + _artifact( + "parameterize_gap_topology", + "temporal_topology", + _selected( + topology, + ( + "computed_from", + "cache_source", + "sampling_basis", + "expected_session_closure_count", + "suspicious_gap_count", + "weekend_activity_count", + ), + ), + ), + _artifact( + "parameterize_calendar_policy", + "calendar_regimes.calendar_policy.calendar_profile", + _mapping(calendar.get("calendar_policy")).get("calendar_profile"), + ), + _artifact( + "parameterize_dynamics_limitations", + "microstructure_dynamics.limitations", + dynamics.get("limitations"), + ), + _artifact( + "parameterize_stationarity_limitations", + "stationarity_diagnostics", + { + "limitations": stationarity.get("limitations"), + "zero_variance_metrics": stationarity.get( + "zero_variance_metrics" + ), + "skipped_window_reason_counts": stationarity.get( + "skipped_window_reason_counts" + ), + }, + ), + _artifact( + "parameterize_training_substrate", + "training_substrate", + _selected( + training, + ( + "status", + "training_schema_version", + "cache_was_enriched", + "legacy_cache_enriched_on_read", + "source_rows_enriched_in_memory", + "row_count", + ), + ), + ), + _artifact( + "defer_raw_m1_ohlc_constraints", + "protocol_scope", + { + "non_tick_input_constraints_supported": False, + "reason": "ascii_tick_only_protocol", + }, + ), + ] + return [ + artifact + for artifact in artifacts + if artifact["value"] not in (None, {}, []) + ] + + +def _artifact( + code: str, source: str, value: JSONValue | None +) -> dict[str, JSONValue]: + return {"code": code, "source": source, "value": value} + + +def _constraint_hints( + fingerprint: Mapping[str, JSONValue], + defects: list[dict[str, JSONValue]], + stylized: list[dict[str, JSONValue]], +) -> list[str]: + topology = _mapping(fingerprint.get("temporal_topology")) + stationarity = _mapping(fingerprint.get("stationarity_diagnostics")) + hints = [ + "preserve_session_activity_mix", + "preserve_spread_regime_mix", + "parameterize_gap_topology", + "parameterize_cache_provenance", + "write_only_synth_columns", + "preserve_observed_bid_ask", + "preserve_row_identity", + ] + if _int(topology.get("expected_session_closure_count")) > 0: + hints.append("preserve_expected_weekend_closures") + if ( + _int(topology.get("suspicious_gap_count")) > 0 + or _text(topology.get("sampling_basis")) != "observed_sequence" + ): + hints.append("do_not_train_on_irregular_grid_without_policy") + if _strings(stationarity.get("recommended_transforms")): + hints.append("apply_stationarity_transform_policy") + hints.extend( + _text(defect.get("code")) + for defect in defects + if _text(defect.get("code")) + ) + if not stylized: + hints.append("do_not_generate_without_reference_stylized_facts") + return _ordered_unique(hints) + + +def _constraint_limitations( + fingerprint: Mapping[str, JSONValue], + training: Mapping[str, JSONValue], + *, + supported: bool, +) -> list[str]: + limitations: list[str] = [] + if not supported: + limitations.append("unsupported_base_grain") + if training.get("status") != "available": + limitations.append("training_substrate_unavailable") + audit = _mapping(fingerprint.get("fingerprint_audit")) + statuses = _mapping(audit.get("section_statuses")) + if any(_text(value) == "unavailable" for value in statuses.values()): + limitations.append("fingerprint_sections_unavailable") + if not _mapping(fingerprint.get("stationarity_diagnostics")): + limitations.append("stationarity_diagnostics_unavailable") + if not _mapping(fingerprint.get("microstructure_dynamics")): + limitations.append("microstructure_dynamics_unavailable") + return _ordered_unique(limitations) + + +def _constraints_by_axis( + findings: Iterable[QualityFinding], +) -> dict[tuple[str, str, str, str], Mapping[str, JSONValue]]: + rows: dict[tuple[str, str, str, str], Mapping[str, JSONValue]] = {} + for finding in findings: + fingerprint = _mapping( + finding.metadata.get(TIME_SERIES_FINGERPRINT_METADATA_KEY) + ) + constraints = _mapping(fingerprint.get("synthetic_constraints")) + if not constraints: + continue + axis = _mapping(constraints.get("target_axis")) + rows[_axis_key(axis)] = constraints + return rows + + +def _validate_target_constraints( + reference: Mapping[str, JSONValue], + candidate: Mapping[str, JSONValue] | None, + *, + mismatch_state: BoundedReportLimit, +) -> dict[str, JSONValue]: + axis = dict(_mapping(reference.get("target_axis"))) + if not candidate: + code = "synthetic_candidate_target_missing" + detail: dict[str, JSONValue] = { + "code": code, + "category": "target", + "constraint_code": "target_axis", + } + included_codes = mismatch_state.slice([code]) + included_details = mismatch_state.slice([detail]) + return { + "target_axis": axis, + "status": "not_compared", + "reason": "candidate_target_missing", + "compared_fact_count": 0, + "mismatch_count": 1, + "mismatch_code_count": 1, + "included_mismatch_code_count": len(included_codes), + "omitted_mismatch_code_count": 1 - len(included_codes), + "included_mismatch_count": len(included_details), + "omitted_mismatch_count": 1 - len(included_details), + "truncated": not included_codes or not included_details, + "mismatch_code_counts": {code: 1}, + "mismatch_codes": cast(JSONValue, included_codes), + "mismatch_details": cast(JSONValue, included_details), + } + mismatches: list[dict[str, JSONValue]] = [] + for defect in _mapping_rows(candidate.get("defects_to_avoid")): + if ( + _text(defect.get("requirement")) + in {"must_be_zero", "canonical_training_schema_required"} + and _int(defect.get("observed_count")) > 0 + ): + code = _text(defect.get("code")) + mismatches.append( + { + "code": f"synthetic_candidate_{code}_present", + "category": "defect", + "constraint_code": code, + "observed_count": _int(defect.get("observed_count")), + } + ) + reference_facts = { + _text(item.get("code")): item + for item in _mapping_rows(reference.get("stylized_facts_to_preserve")) + } + candidate_facts = { + _text(item.get("code")): item + for item in _mapping_rows(candidate.get("stylized_facts_to_preserve")) + } + compared_fact_count = 0 + for code, fact in sorted(reference_facts.items()): + candidate_fact = candidate_facts.get(code) + if candidate_fact is None: + mismatches.append( + { + "code": "synthetic_candidate_stylized_fact_missing", + "category": "stylized_fact", + "constraint_code": code, + } + ) + continue + compared_fact_count += 1 + if not _fact_matches(fact, candidate_fact): + mismatches.append( + { + "code": f"synthetic_candidate_{code}_mismatch", + "category": "stylized_fact", + "constraint_code": code, + "comparison": fact.get("comparison"), + "tolerance": fact.get("tolerance"), + } + ) + reference_output = _mapping(reference.get("output_contract")) + candidate_output = _mapping(candidate.get("output_contract")) + for field, mismatch_code in ( + ( + "durable_identity_columns", + "synthetic_candidate_training_identity_mismatch", + ), + ( + "synthetic_output_columns", + "synthetic_candidate_output_contract_mismatch", + ), + ( + "observed_columns_preserved", + "synthetic_candidate_observed_column_contract_mismatch", + ), + ): + if reference_output.get(field) != candidate_output.get(field): + mismatches.append( + { + "code": mismatch_code, + "category": "output_contract", + "constraint_code": field, + } + ) + mismatch_codes = sorted( + {_text(item.get("code")) for item in mismatches if item.get("code")} + ) + included_details = mismatch_state.slice(mismatches) + included_codes = mismatch_state.slice(mismatch_codes) + code_counts = Counter( + _text(item.get("code")) for item in mismatches if item.get("code") + ) + return { + "target_axis": axis, + "status": "mismatch" if mismatches else "match", + "compared_fact_count": compared_fact_count, + "mismatch_count": len(mismatches), + "mismatch_code_count": len(mismatch_codes), + "included_mismatch_code_count": len(included_codes), + "omitted_mismatch_code_count": max( + 0, len(mismatch_codes) - len(included_codes) + ), + "included_mismatch_count": len(included_details), + "omitted_mismatch_count": max( + 0, len(mismatches) - len(included_details) + ), + "truncated": ( + len(included_details) < len(mismatches) + or len(included_codes) < len(mismatch_codes) + ), + "mismatch_code_counts": _counter_payload(code_counts), + "mismatch_codes": cast(JSONValue, included_codes), + "mismatch_details": cast(JSONValue, included_details), + } + + +def _fact_matches( + reference: Mapping[str, JSONValue], + candidate: Mapping[str, JSONValue], +) -> bool: + comparison = _text(reference.get("comparison")) + tolerance = _float(reference.get("tolerance")) + left = reference.get("value") + right = candidate.get("value") + if comparison == "exact": + return left == right + if comparison == "distribution_l1": + return _distribution_distance(left, right) <= tolerance + if comparison == "numeric_absolute_tolerance": + return _numeric_values_match(left, right, tolerance, relative=False) + if comparison == "numeric_relative_tolerance": + return _numeric_values_match(left, right, tolerance, relative=True) + return left == right + + +def _distribution_distance( + left: JSONValue | None, right: JSONValue | None +) -> float: + left_map = _numeric_leaf_map(left) + right_map = _numeric_leaf_map(right) + if not left_map and not right_map: + return 0.0 if left == right else math.inf + keys = set(left_map) | set(right_map) + left_total = sum(max(0.0, left_map.get(key, 0.0)) for key in keys) + right_total = sum(max(0.0, right_map.get(key, 0.0)) for key in keys) + if left_total <= 0.0 or right_total <= 0.0: + return 0.0 if left_map == right_map else math.inf + return ( + sum( + abs( + left_map.get(key, 0.0) / left_total + - right_map.get(key, 0.0) / right_total + ) + for key in keys + ) + / 2.0 + ) + + +def _numeric_values_match( + left: JSONValue | None, + right: JSONValue | None, + tolerance: float, + *, + relative: bool, +) -> bool: + left_map = _leaf_map(left) + right_map = _leaf_map(right) + if set(left_map) != set(right_map): + return False + for key, left_value in left_map.items(): + right_value = right_map[key] + if isinstance(left_value, bool) or isinstance(right_value, bool): + if left_value != right_value: + return False + continue + if isinstance(left_value, (int, float)) and isinstance( + right_value, (int, float) + ): + delta = abs(float(left_value) - float(right_value)) + allowed = tolerance + if relative: + allowed = max(1e-12, abs(float(left_value)) * tolerance) + if delta > allowed: + return False + continue + if left_value != right_value: + return False + return True + + +def _numeric_leaf_map(value: JSONValue | None) -> dict[str, float]: + return { + key: float(item) + for key, item in _leaf_map(value).items() + if not isinstance(item, bool) and isinstance(item, (int, float)) + } + + +def _leaf_map(value: JSONValue | None, prefix: str = "") -> dict[str, Any]: + if isinstance(value, Mapping): + rows: dict[str, Any] = {} + for key, item in sorted(value.items()): + path = f"{prefix}.{key}" if prefix else str(key) + rows.update(_leaf_map(item, path)) + return rows + if isinstance(value, list): + rows = {} + for index, item in enumerate(value): + path = f"{prefix}[{index}]" + rows.update(_leaf_map(item, path)) + return rows + return {prefix or "value": value} + + +def _frame_true_count(frame: Any | None, column: str) -> int | None: + if frame is None or column not in getattr(frame, "columns", ()): + return None + try: + return int(frame.get_column(column).fill_null(False).sum() or 0) + except (AttributeError, TypeError, ValueError): + return None + + +def _training_precision_regime(frame: Any | None) -> dict[str, JSONValue]: + if frame is None: + return {} + counts: Counter[str] = Counter() + for column in ("bid", "ask"): + if column not in getattr(frame, "columns", ()): + continue + try: + values = frame.get_column(column).to_list() + except (AttributeError, KeyError, TypeError, ValueError): + continue + for value in values: + if isinstance(value, bool) or not isinstance(value, (int, float)): + continue + if not math.isfinite(float(value)): + continue + text = f"{float(value):.12f}".rstrip("0").rstrip(".") + places = len(text.split(".", 1)[1]) if "." in text else 0 + counts[f"{column}:{places}"] += 1 + if not counts: + return {} + return { + "basis": "observed_float_value", + "decimal_place_counts": _counter_payload(counts), + "source_text_precision_preserved": False, + } + + +def _selected( + payload: Mapping[str, JSONValue], + fields: tuple[str, ...], +) -> dict[str, JSONValue]: + return {field: payload[field] for field in fields if field in payload} + + +def _without_count(payload: Mapping[str, JSONValue]) -> dict[str, JSONValue]: + return {key: value for key, value in payload.items() if key != "count"} + + +def _payload_id(payload: Mapping[str, JSONValue]) -> str: + import hashlib + + material = { + key: value for key, value in payload.items() if key != "constraint_id" + } + encoded = json.dumps( + material, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _axis_key(axis: Mapping[str, JSONValue]) -> tuple[str, str, str, str]: + return ( + _text(axis.get("data_format")), + _text(axis.get("timeframe")), + _text(axis.get("symbol")), + _text(axis.get("period")), + ) + + +def _target_axis_from_target(target: Any | None) -> dict[str, JSONValue]: + return { + "data_format": _text(getattr(target, "data_format", "")) or "ascii", + "timeframe": _text(getattr(target, "timeframe", "")) or TICK, + "symbol": _text(getattr(target, "symbol", "")), + "period": _text(getattr(target, "period", "")), + "kind": _text(getattr(getattr(target, "kind", None), "value", "")) + or "cache", + } + + +def _target_sort_key(target: Mapping[str, JSONValue]) -> tuple[str, ...]: + return _axis_key(_mapping(target.get("target_axis"))) + + +def _validation_target_sort_key( + target: Mapping[str, JSONValue], +) -> tuple[int, str, str, str, str]: + ranks = {"mismatch": 0, "not_compared": 1, "match": 2} + axis = _mapping(target.get("target_axis")) + return (ranks.get(_text(target.get("status")), 99), *_axis_key(axis)) + + +def _counter_payload(counter: Counter[str]) -> dict[str, JSONValue]: + return {key: counter[key] for key in sorted(counter) if key} + + +def _ordered_unique(values: Iterable[str]) -> list[str]: + seen: set[str] = set() + rows: list[str] = [] + for value in values: + if not value or value in seen: + continue + seen.add(value) + rows.append(value) + return rows + + +def _mapping(value: object) -> Mapping[str, JSONValue]: + return value if isinstance(value, Mapping) else {} + + +def _mapping_rows(value: object) -> list[Mapping[str, JSONValue]]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, Mapping)] + + +def _strings(value: object) -> list[str]: + if not isinstance(value, list): + return [] + return [_text(item) for item in value if _text(item)] + + +def _text(value: object) -> str: + return str(value).strip() if value is not None else "" + + +def _int(value: object) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int): + return value + if isinstance(value, float) and math.isfinite(value): + return int(value) + return 0 + + +def _float(value: object) -> float: + if isinstance(value, bool): + return 0.0 + if isinstance(value, (int, float)) and math.isfinite(float(value)): + return float(value) + return 0.0 + + +def _format_counts(counts: Mapping[str, JSONValue]) -> str: + return ", ".join( + f"{key}={_int(value)}" for key, value in sorted(counts.items()) + ) + + +def _format_axis(axis: Mapping[str, JSONValue]) -> str: + return ":".join( + ( + _text(axis.get("data_format")) or "unknown", + _text(axis.get("timeframe")) or "unknown", + _text(axis.get("symbol")) or "unknown", + _text(axis.get("period")) or "unknown", + ) + ) diff --git a/src/histdatacom/data_quality/synthetic_generation.py b/src/histdatacom/data_quality/synthetic_generation.py new file mode 100644 index 00000000..ad69b0db --- /dev/null +++ b/src/histdatacom/data_quality/synthetic_generation.py @@ -0,0 +1,1059 @@ +"""Deterministic synthetic ASCII tick generation from a reference set. + +The generator augments the canonical enriched tick frame in place. It never +overwrites observed bid/ask values or changes durable row identity. Generated +midpoint returns and spreads are sampled together in deterministic contiguous +blocks, then the candidate is always evaluated through the ordinary +fingerprint and synthetic-constraint validation path. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from statistics import median +from typing import Any, cast + +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprints import ( + SERIES_FINGERPRINT_RULE_ID, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_SCHEMA_VERSION, + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.synthetic_constraints import ( + synthetic_constraints_from_training_frame, + validate_synthetic_constraint_reports, +) +from histdatacom.data_quality.training_features import ( + HARD_ISSUE_COLUMNS, + SYNTHETIC_PLACEHOLDER_COLUMNS, + ensure_tick_training_features, +) +from histdatacom.histdata_ascii import ( + TICK, + format_influx_line, + read_polars_cache, + write_polars_cache, +) +from histdatacom.runtime_contracts import JSONValue + +SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION = ( + "histdatacom.synthetic-tick-generation.v1" +) +SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION = ( + "histdatacom.synthetic-tick-generation-configuration.v1" +) +SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION = ( + "histdatacom.synthetic-tick-generation-validation.v1" +) + +SYNTHETIC_TICK_METHOD_CODES = {"empirical_block_bootstrap": 1} +SYNTHETIC_TICK_STATUS_CODES = { + "unavailable": 1, + "limited": 2, + "ready": 3, +} +SYNTHETIC_TICK_REASON_CODES = { + "none": 0, + "reference_fingerprint_required": 1, + "unsupported_base_grain": 2, + "reference_constraints_unavailable": 3, + "reference_stylized_facts_unavailable": 4, + "insufficient_reference_rows": 5, + "existing_synthetic_values": 6, + "generation_row_limit": 7, + "unsupported_reference_fingerprint_schema": 8, + "reference_fingerprint_id_required": 9, + "reference_target_axis_mismatch": 10, +} + +DEFAULT_SYNTHETIC_TICK_SEED = 0 +DEFAULT_SYNTHETIC_TICK_BLOCK_SIZE = 32 +DEFAULT_SYNTHETIC_TICK_MINIMUM_REFERENCE_ROWS = 16 +DEFAULT_SYNTHETIC_TICK_MAX_REFERENCE_ROWS = 1_000_000 +DEFAULT_SYNTHETIC_TICK_MAX_GENERATED_ROWS = 1_000_000 +DEFAULT_SYNTHETIC_TICK_MAX_ABS_LOG_RETURN = 0.25 +DEFAULT_SYNTHETIC_TICK_ROUNDING_DIGITS = 8 +DEFAULT_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT = 12 + +MAX_SYNTHETIC_TICK_BLOCK_SIZE = 100_000 +MAX_SYNTHETIC_TICK_REFERENCE_ROWS = 10_000_000 +MAX_SYNTHETIC_TICK_GENERATED_ROWS = 10_000_000 +MAX_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT = 128 +MAX_SYNTHETIC_TICK_PRICE = 1_000_000_000_000.0 + + +@dataclass(frozen=True, slots=True) +class SyntheticTickGenerationProfile: + """Bounded deterministic generator configuration.""" + + method: str = "empirical_block_bootstrap" + seed: int = DEFAULT_SYNTHETIC_TICK_SEED + block_size: int = DEFAULT_SYNTHETIC_TICK_BLOCK_SIZE + minimum_reference_rows: int = DEFAULT_SYNTHETIC_TICK_MINIMUM_REFERENCE_ROWS + max_reference_rows: int = DEFAULT_SYNTHETIC_TICK_MAX_REFERENCE_ROWS + max_generated_rows: int = DEFAULT_SYNTHETIC_TICK_MAX_GENERATED_ROWS + max_abs_log_return: float = DEFAULT_SYNTHETIC_TICK_MAX_ABS_LOG_RETURN + rounding_digits: int = DEFAULT_SYNTHETIC_TICK_ROUNDING_DIGITS + diagnostic_sample_limit: int = ( + DEFAULT_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT + ) + anchor_mode: str = "first_valid_mid" + overwrite_existing: bool = False + + def __post_init__(self) -> None: + if self.method not in SYNTHETIC_TICK_METHOD_CODES: + raise ValueError(f"unsupported synthetic method: {self.method}") + _bounded_positive( + self.block_size, + MAX_SYNTHETIC_TICK_BLOCK_SIZE, + "block_size", + ) + if self.minimum_reference_rows < 2: + raise ValueError("minimum_reference_rows must be at least 2") + _bounded_positive( + self.max_reference_rows, + MAX_SYNTHETIC_TICK_REFERENCE_ROWS, + "max_reference_rows", + ) + if self.max_reference_rows < self.minimum_reference_rows: + raise ValueError( + "max_reference_rows must cover minimum_reference_rows" + ) + _bounded_positive( + self.max_generated_rows, + MAX_SYNTHETIC_TICK_GENERATED_ROWS, + "max_generated_rows", + ) + if not 0 < self.max_abs_log_return <= 1: + raise ValueError("max_abs_log_return must be in (0, 1]") + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between 0 and 16") + _bounded_positive( + self.diagnostic_sample_limit, + MAX_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT, + "diagnostic_sample_limit", + ) + if self.anchor_mode not in {"first_valid_mid", "median_valid_mid"}: + raise ValueError(f"unsupported anchor_mode: {self.anchor_mode}") + + def to_metadata(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible configuration metadata.""" + return { + "schema_version": ( + SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION + ), + "method": self.method, + "method_code": SYNTHETIC_TICK_METHOD_CODES[self.method], + "seed": self.seed, + "block_size": self.block_size, + "minimum_reference_rows": self.minimum_reference_rows, + "max_reference_rows": self.max_reference_rows, + "max_generated_rows": self.max_generated_rows, + "max_abs_log_return": self.max_abs_log_return, + "rounding_digits": self.rounding_digits, + "diagnostic_sample_limit": self.diagnostic_sample_limit, + "anchor_mode": self.anchor_mode, + "overwrite_existing": self.overwrite_existing, + } + + +@dataclass(frozen=True, slots=True) +class SyntheticTickGenerationResult: + """Generated enriched frame and bounded diagnostic evidence.""" + + frame: Any + diagnostics: Mapping[str, JSONValue] + candidate_report: QualityReport | None = None + + +def generate_synthetic_ticks_from_reference( + reference_frame: Any, + reference_fingerprint: Mapping[str, JSONValue], + *, + profile: SyntheticTickGenerationProfile | None = None, + target: QualityTarget | None = None, +) -> SyntheticTickGenerationResult: + """Populate synthetic columns from one fingerprint-backed reference set.""" + selected = profile or SyntheticTickGenerationProfile() + _require_reference_fingerprint(reference_fingerprint) + reference_axis = _mapping(reference_fingerprint.get("target_axis")) + selected_target = target or _target_from_axis(reference_axis) + _require_tick_axis(reference_axis, selected_target) + if not _axis_matches_target(reference_axis, selected_target): + raise ValueError("reference_target_axis_mismatch") + enriched = ensure_tick_training_features( + reference_frame, target=selected_target + ) + if ( + _has_existing_synthetic_values(enriched) + and not selected.overwrite_existing + ): + raise ValueError("existing_synthetic_values") + constraints = _mapping(reference_fingerprint.get("synthetic_constraints")) + if not constraints: + constraints = synthetic_constraints_from_training_frame( + enriched, + fingerprint=reference_fingerprint, + target=selected_target, + ) + constraint_status = _text(constraints.get("status")) + stylized_facts = _mapping_rows( + constraints.get("stylized_facts_to_preserve") + ) + base = _base_diagnostics( + selected, + reference_fingerprint, + constraints, + selected_target, + input_row_count=int(getattr(enriched, "height", 0) or 0), + ) + if constraint_status == "unavailable": + return _unavailable_result( + enriched, + base, + "reference_constraints_unavailable", + ) + if not stylized_facts: + return _unavailable_result( + enriched, + base, + "reference_stylized_facts_unavailable", + ) + + reference_rows = _reference_rows(enriched, selected) + transitions = _reference_transitions(reference_rows) + if len(reference_rows) < selected.minimum_reference_rows or not transitions: + base["reference_evidence"] = _reference_evidence( + enriched, + reference_rows, + transitions, + selected, + ) + return _unavailable_result( + enriched, + base, + "insufficient_reference_rows", + ) + + output, generation = _generate_frame( + enriched, + reference_rows, + transitions, + constraint_id=_text(constraints.get("constraint_id")), + constraint_status=constraint_status, + profile=selected, + ) + base["reference_evidence"] = _reference_evidence( + enriched, + reference_rows, + transitions, + selected, + ) + base["generation"] = generation + generated_count = _int(generation.get("generated_row_count")) + input_count = int(getattr(enriched, "height", 0) or 0) + status = "ready" if generated_count == input_count else "limited" + reason = None if status == "ready" else "generation_row_limit" + base["status"] = status + base["status_code"] = SYNTHETIC_TICK_STATUS_CODES[status] + base["reason"] = reason + base["reason_code"] = SYNTHETIC_TICK_REASON_CODES[reason or "none"] + + candidate_report, validation = validate_synthetic_tick_frame( + output, + reference_fingerprint, + target=selected_target, + ) + base["validation"] = validation + base["generation_id"] = _stable_id("synthetic-generation", base) + return SyntheticTickGenerationResult(output, base, candidate_report) + + +def validate_synthetic_tick_frame( + generated_frame: Any, + reference_fingerprint: Mapping[str, JSONValue], + *, + target: QualityTarget | None = None, +) -> tuple[QualityReport, dict[str, JSONValue]]: + """Validate a generated frame through the ordinary fingerprint rule.""" + _require_reference_fingerprint(reference_fingerprint) + reference_axis = _mapping(reference_fingerprint.get("target_axis")) + selected_target = target or _target_from_axis(reference_axis) + _require_tick_axis(reference_axis, selected_target) + if not _axis_matches_target(reference_axis, selected_target): + raise ValueError("reference_target_axis_mismatch") + symbol = _text(reference_axis.get("symbol")) or selected_target.symbol + period = _text(reference_axis.get("period")) or selected_target.period + with tempfile.TemporaryDirectory(prefix="histdatacom-synthetic-") as root: + cache_path = Path(root) / f"DAT_ASCII_{symbol}_T_{period}.data" + _write_candidate_market_cache(generated_frame, cache_path) + candidate_target = QualityTarget( + path=str(cache_path), + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period=period, + ) + report, validation = validate_synthetic_tick_cache( + cache_path, + reference_fingerprint, + target=candidate_target, + ) + validation["same_point_influx_projection"] = ( + _influx_projection_evidence(generated_frame, selected_target) + ) + return report, validation + + +def validate_synthetic_tick_cache( + cache_path: str | Path, + reference_fingerprint: Mapping[str, JSONValue], + *, + target: QualityTarget | None = None, +) -> tuple[QualityReport, dict[str, JSONValue]]: + """Fingerprint and validate a generated candidate cache.""" + _require_reference_fingerprint(reference_fingerprint) + source = Path(cache_path) + reference_axis = _mapping(reference_fingerprint.get("target_axis")) + candidate_target = target or QualityTarget( + path=str(source), + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol=_text(reference_axis.get("symbol")), + period=_text(reference_axis.get("period")), + ) + _require_tick_axis(reference_axis, candidate_target) + if not _axis_matches_target(reference_axis, candidate_target): + raise ValueError("reference_target_axis_mismatch") + [candidate_finding] = HistDataSeriesFingerprintRule( + profile=HistDataFingerprintProfile() + ).evaluate(candidate_target) + candidate_report = QualityReport( + targets=(candidate_target,), + rule_results=( + QualityRuleResult( + rule_id=candidate_finding.rule_id, + target=candidate_target, + findings=(candidate_finding,), + ), + ), + ) + reference_report = _fingerprint_report( + reference_fingerprint, + target=_target_from_axis(reference_axis), + ) + comparison = validate_synthetic_constraint_reports( + reference_report, + candidate_report, + ) + candidate_fingerprint = _mapping( + candidate_finding.metadata.get(TIME_SERIES_FINGERPRINT_METADATA_KEY) + ) + validation: dict[str, JSONValue] = { + "schema_version": SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION, + "status": comparison.get("status"), + "same_fingerprint_path_used": True, + "candidate_fingerprint_schema_version": candidate_fingerprint.get( + "schema_version" + ), + "candidate_fingerprint_id": candidate_fingerprint.get("fingerprint_id"), + "constraint_validation": comparison, + "same_point_influx_projection": _influx_projection_evidence( + read_polars_cache(source), candidate_target + ), + "hard_quality_gate": False, + } + return candidate_report, validation + + +def reference_fingerprint_from_report( + report: QualityReport, + *, + target: QualityTarget | None = None, +) -> dict[str, JSONValue]: + """Select one fingerprint from a saved report by durable target axis.""" + candidates: list[dict[str, JSONValue]] = [] + for finding in report.findings: + payload = _mapping( + finding.metadata.get(TIME_SERIES_FINGERPRINT_METADATA_KEY) + ) + if not payload: + continue + if target is not None and not _axis_matches_target( + _mapping(payload.get("target_axis")), target + ): + continue + candidates.append(payload) + if not candidates: + raise ValueError("reference_fingerprint_required") + if len(candidates) > 1 and target is None: + raise ValueError("multiple_reference_fingerprints_require_target") + candidates.sort(key=_fingerprint_sort_key) + return candidates[0] + + +def format_synthetic_tick_generation( + diagnostics: Mapping[str, JSONValue], +) -> str: + """Return concise human-readable generator diagnostics.""" + generation = _mapping(diagnostics.get("generation")) + validation = _mapping(diagnostics.get("validation")) + axis = _mapping(diagnostics.get("target_axis")) + return "\n".join( + ( + "Synthetic tick generation", + ( + "target: " + f"{_text(axis.get('symbol'))} " + f"{_text(axis.get('timeframe'))} " + f"{_text(axis.get('period'))}" + ), + ( + f"status: {_text(diagnostics.get('status'))} " + f"reason: {_text(diagnostics.get('reason')) or 'none'}" + ), + ( + "rows: " + f"{_int(generation.get('generated_row_count'))} generated, " + f"{_int(generation.get('omitted_row_count'))} omitted" + ), + ( + "method: " + f"{_text(generation.get('method'))} " + f"seed={_int(generation.get('seed'))}" + ), + f"validation: {_text(validation.get('status')) or 'not_run'}", + "observed bid/ask preserved: yes", + ) + ) + + +def _generate_frame( + frame: Any, + reference_rows: Sequence[Mapping[str, Any]], + transitions: Sequence[tuple[float, float, int]], + *, + constraint_id: str, + constraint_status: str, + profile: SyntheticTickGenerationProfile, +) -> tuple[Any, dict[str, JSONValue]]: + import polars as pl + + row_count = int(getattr(frame, "height", 0) or 0) + generated_count = min(row_count, profile.max_generated_rows) + indices = _sample_transition_indices( + len(transitions), + generated_count, + block_size=profile.block_size, + seed=profile.seed, + constraint_id=constraint_id, + ) + valid_mids = [_float(row.get("mid")) for row in reference_rows] + anchor_mid = ( + median(valid_mids) + if profile.anchor_mode == "median_valid_mid" + else valid_mids[0] + ) + minimum_price = 10 ** (-profile.rounding_digits) + previous_mid = max( + minimum_price, + min(MAX_SYNTHETIC_TICK_PRICE, anchor_mid), + ) + confidence = _confidence( + valid_count=len(reference_rows), + inspected_count=min(row_count, profile.max_reference_rows), + constraint_status=constraint_status, + digits=profile.rounding_digits, + ) + bids: list[float | None] = [] + asks: list[float | None] = [] + spreads: list[float | None] = [] + mids: list[float | None] = [] + method_codes: list[int | None] = [] + confidences: list[float | None] = [] + usable: list[bool | None] = [] + samples: list[dict[str, JSONValue]] = [] + clipped_count = 0 + price_clamp_count = int(previous_mid != anchor_mid) + for output_index in range(generated_count): + transition_index = indices[output_index] + log_return, sampled_spread, source_row_id = transitions[ + transition_index + ] + bounded_return = max( + -profile.max_abs_log_return, + min(profile.max_abs_log_return, log_return), + ) + clipped_count += bounded_return != log_return + if output_index: + candidate_mid = previous_mid * math.exp(bounded_return) + previous_mid = max( + minimum_price, + min(MAX_SYNTHETIC_TICK_PRICE, candidate_mid), + ) + price_clamp_count += previous_mid != candidate_mid + spread = max(0.0, sampled_spread) + mid = previous_mid + bid = mid - spread / 2.0 + ask = mid + spread / 2.0 + if bid <= 0: + bid = max(mid * 0.5, minimum_price) + ask = bid + spread + mid = (bid + ask) / 2.0 + rounded_bid = round(bid, profile.rounding_digits) + bid = max( + minimum_price, + min(MAX_SYNTHETIC_TICK_PRICE, rounded_bid), + ) + rounded_ask = round(max(ask, bid), profile.rounding_digits) + ask = max(bid, min(MAX_SYNTHETIC_TICK_PRICE, rounded_ask)) + price_clamp_count += bid != rounded_bid or ask != rounded_ask + # Derive these fields from the persisted synthetic quotes so the row + # contract remains exact for downstream scalar consumers. + spread = ask - bid + mid = round((ask + bid) / 2.0, profile.rounding_digits) + bids.append(bid) + asks.append(ask) + spreads.append(spread) + mids.append(mid) + method_codes.append(SYNTHETIC_TICK_METHOD_CODES[profile.method]) + confidences.append(confidence) + usable.append(True) + if len(samples) < profile.diagnostic_sample_limit: + samples.append( + { + "output_row_offset": output_index, + "reference_transition_index": transition_index, + "reference_source_row_id": source_row_id, + } + ) + omitted = row_count - generated_count + bids.extend([None] * omitted) + asks.extend([None] * omitted) + spreads.extend([None] * omitted) + mids.extend([None] * omitted) + method_codes.extend([None] * omitted) + confidences.extend([None] * omitted) + usable.extend([False] * omitted) + output = frame.drop( + [ + name + for name in SYNTHETIC_PLACEHOLDER_COLUMNS + if name in frame.columns + ] + ).with_columns( + [ + pl.Series("synth_bid", bids, dtype=pl.Float64), + pl.Series("synth_ask", asks, dtype=pl.Float64), + pl.Series("synth_spread", spreads, dtype=pl.Float64), + pl.Series("synth_mid", mids, dtype=pl.Float64), + pl.Series("synth_method_code", method_codes, dtype=pl.Int32), + pl.Series("synth_confidence", confidences, dtype=pl.Float64), + pl.Series("synth_usable", usable, dtype=pl.Boolean), + ] + ) + return output, { + "method": profile.method, + "method_code": SYNTHETIC_TICK_METHOD_CODES[profile.method], + "seed": profile.seed, + "block_size": profile.block_size, + "calculation_basis": "paired_mid_log_return_and_spread_blocks", + "input_row_count": row_count, + "generated_row_count": generated_count, + "omitted_row_count": omitted, + "return_clip_count": clipped_count, + "price_clamp_count": price_clamp_count, + "price_bounds": { + "minimum": minimum_price, + "maximum": MAX_SYNTHETIC_TICK_PRICE, + }, + "confidence": confidence, + "sample_count": len(samples), + "samples": cast(JSONValue, samples), + "samples_truncated": generated_count > len(samples), + "same_row_grain": True, + "timestamp_generation": False, + "observed_columns_overwritten": False, + "synthetic_columns_only": True, + } + + +def _reference_rows( + frame: Any, + profile: SyntheticTickGenerationProfile, +) -> list[dict[str, Any]]: + columns = ["row_id", "mid", "spread", *HARD_ISSUE_COLUMNS] + rows = cast( + list[dict[str, Any]], + frame.select(columns).head(profile.max_reference_rows).to_dicts(), + ) + return [row for row in rows if _valid_reference_row(row)] + + +def _valid_reference_row(row: Mapping[str, Any]) -> bool: + mid = _optional_float(row.get("mid")) + spread = _optional_float(row.get("spread")) + return ( + mid is not None + and mid > 0 + and spread is not None + and spread >= 0 + and not any(bool(row.get(column)) for column in HARD_ISSUE_COLUMNS) + ) + + +def _reference_transitions( + rows: Sequence[Mapping[str, Any]], +) -> list[tuple[float, float, int]]: + transitions: list[tuple[float, float, int]] = [] + previous: Mapping[str, Any] | None = None + for row in rows: + if previous is None: + previous = row + continue + current_row_id = _int(row.get("row_id")) + previous_row_id = _int(previous.get("row_id")) + if current_row_id != previous_row_id + 1: + previous = row + continue + current_mid = _float(row.get("mid")) + previous_mid = _float(previous.get("mid")) + log_return = math.log(current_mid / previous_mid) + if math.isfinite(log_return): + transitions.append( + ( + log_return, + max(0.0, _float(row.get("spread"))), + current_row_id, + ) + ) + previous = row + return transitions + + +def _sample_transition_indices( + transition_count: int, + output_count: int, + *, + block_size: int, + seed: int, + constraint_id: str, +) -> list[int]: + indices: list[int] = [] + block_number = 0 + while len(indices) < output_count: + material = ( + f"{seed}:{constraint_id}:{block_number}:{transition_count}" + ).encode("utf-8") + start = int.from_bytes(hashlib.sha256(material).digest()[:8], "big") + start %= transition_count + for offset in range(block_size): + indices.append((start + offset) % transition_count) + if len(indices) >= output_count: + break + block_number += 1 + return indices + + +def _reference_evidence( + frame: Any, + rows: Sequence[Mapping[str, Any]], + transitions: Sequence[tuple[float, float, int]], + profile: SyntheticTickGenerationProfile, +) -> dict[str, JSONValue]: + inspected = min( + int(getattr(frame, "height", 0) or 0), profile.max_reference_rows + ) + return { + "inspected_row_count": inspected, + "valid_reference_row_count": len(rows), + "filtered_reference_row_count": max(0, inspected - len(rows)), + "transition_count": len(transitions), + "reference_rows_truncated": int(getattr(frame, "height", 0) or 0) + > inspected, + "hard_issue_columns": list(HARD_ISSUE_COLUMNS), + "defective_rows_used": False, + } + + +def _base_diagnostics( + profile: SyntheticTickGenerationProfile, + fingerprint: Mapping[str, JSONValue], + constraints: Mapping[str, JSONValue], + target: QualityTarget, + *, + input_row_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION, + "advisory": True, + "status": "unavailable", + "status_code": SYNTHETIC_TICK_STATUS_CODES["unavailable"], + "reason": None, + "reason_code": SYNTHETIC_TICK_REASON_CODES["none"], + "target_axis": { + "data_format": target.data_format, + "timeframe": target.timeframe, + "symbol": target.symbol, + "period": target.period, + "kind": target.kind.value, + }, + "reference_fingerprint_id": fingerprint.get("fingerprint_id"), + "reference_fingerprint_count": 1, + "reference_fingerprint_ids": [fingerprint.get("fingerprint_id")], + "reference_constraint_id": constraints.get("constraint_id"), + "reference_constraint_status": constraints.get("status"), + "constraint_application": _constraint_application(constraints), + "reference_fingerprint_required": True, + "configuration": profile.to_metadata(), + "input_row_count": input_row_count, + "base_grain": {"data_format": "ascii", "timeframe": TICK}, + "output_contract": { + "columns": list(SYNTHETIC_PLACEHOLDER_COLUMNS), + "durable_identity": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "observed_bid_ask_preserved": True, + "same_frame_projection": True, + "volume_generation": False, + "raw_m1_generation": False, + }, + "resource_policy": { + "max_reference_rows": profile.max_reference_rows, + "max_generated_rows": profile.max_generated_rows, + "bounded": True, + }, + "quality_gate": False, + } + + +def _unavailable_result( + frame: Any, + diagnostics: dict[str, JSONValue], + reason: str, +) -> SyntheticTickGenerationResult: + diagnostics["status"] = "unavailable" + diagnostics["status_code"] = SYNTHETIC_TICK_STATUS_CODES["unavailable"] + diagnostics["reason"] = reason + diagnostics["reason_code"] = SYNTHETIC_TICK_REASON_CODES[reason] + diagnostics["validation"] = { + "schema_version": SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION, + "status": "not_run", + "reason": reason, + "same_fingerprint_path_used": False, + } + diagnostics["generation_id"] = _stable_id( + "synthetic-generation", diagnostics + ) + return SyntheticTickGenerationResult(frame, diagnostics) + + +def _write_candidate_market_cache(frame: Any, path: Path) -> None: + import polars as pl + + columns = set(getattr(frame, "columns", ())) + required = {"datetime", "synth_bid", "synth_ask"} + if not required.issubset(columns): + raise ValueError("generated frame is missing candidate market columns") + expressions = [ + pl.col("datetime").cast(pl.Int64), + pl.col("synth_bid").cast(pl.Float64).alias("bid"), + pl.col("synth_ask").cast(pl.Float64).alias("ask"), + ( + pl.col("vol").cast(pl.Int64) + if "vol" in columns + else pl.lit(0).cast(pl.Int64).alias("vol") + ), + ] + candidate = frame.filter(pl.col("synth_usable").fill_null(False)).select( + expressions + ) + if not candidate.height: + raise ValueError("generated frame has no usable synthetic rows") + write_polars_cache(candidate, path) + + +def _constraint_application( + constraints: Mapping[str, JSONValue], +) -> dict[str, JSONValue]: + defects = _mapping_rows(constraints.get("defects_to_avoid")) + stylized = _mapping_rows(constraints.get("stylized_facts_to_preserve")) + artifacts = _mapping_rows( + constraints.get("source_artifacts_to_parameterize") + ) + return { + "defects_to_avoid": { + "count": len(defects), + "codes": [_text(item.get("code")) for item in defects], + "application": "filter_flagged_reference_rows_and_enforce_quote_invariants", + }, + "stylized_facts_to_preserve": { + "count": len(stylized), + "codes": [_text(item.get("code")) for item in stylized], + "application": "paired_empirical_blocks_then_fingerprint_validation", + }, + "source_artifacts_to_parameterize": { + "count": len(artifacts), + "codes": [_text(item.get("code")) for item in artifacts], + "application": "preserve_reference_row_topology_and_expose_configuration", + }, + } + + +def _influx_projection_evidence( + frame: Any, + target: QualityTarget, +) -> dict[str, JSONValue]: + columns = set(getattr(frame, "columns", ())) + required = {"datetime", "bid", "ask", "synth_bid", "synth_ask"} + if not required.issubset(columns) or not int( + getattr(frame, "height", 0) or 0 + ): + return { + "status": "not_applicable", + "reason": "enriched_synthetic_columns_unavailable", + "same_measurement": False, + } + usable = ( + frame.filter(frame.get_column("synth_usable").fill_null(False)) + if "synth_usable" in columns + else frame + ) + if not int(getattr(usable, "height", 0) or 0): + return { + "status": "unavailable", + "reason": "no_usable_synthetic_row", + "same_measurement": False, + } + line = format_influx_line( + target.symbol, + target.data_format, + target.timeframe, + usable.row(0), + columns=usable.columns, + ) + observed = "bidquote=" in line and "askquote=" in line + synthetic_fields = all( + f"{name}=" in line for name in SYNTHETIC_PLACEHOLDER_COLUMNS + ) + training_fields = all( + marker in line + for marker in ( + ",row_id=", + "mid=", + "dq_issue_negative_spread=", + "class_training_action_code=", + ) + ) + same_measurement = ( + observed + and synthetic_fields + and training_fields + and line.count(" ") == 2 + ) + return { + "status": "valid" if same_measurement else "invalid", + "reason": ( + None if same_measurement else "same_point_projection_missing_fields" + ), + "same_measurement": same_measurement, + "observed_fields_present": observed, + "synthetic_fields_present": synthetic_fields, + "training_fields_present": training_fields, + "line_count": 1, + "raw_line_included": False, + } + + +def _fingerprint_report( + fingerprint: Mapping[str, JSONValue], + *, + target: QualityTarget, +) -> QualityReport: + finding = QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id=SERIES_FINGERPRINT_RULE_ID, + target=target, + metadata={TIME_SERIES_FINGERPRINT_METADATA_KEY: dict(fingerprint)}, + ) + return QualityReport( + targets=(target,), + rule_results=( + QualityRuleResult( + rule_id=SERIES_FINGERPRINT_RULE_ID, + target=target, + findings=(finding,), + ), + ), + ) + + +def _target_from_axis(axis: Mapping[str, JSONValue]) -> QualityTarget: + return QualityTarget( + path=( + f"DAT_ASCII_{_text(axis.get('symbol'))}_T_" + f"{_text(axis.get('period'))}.data" + ), + kind=QualityTargetKind.CACHE, + data_format=_text(axis.get("data_format")) or "ascii", + timeframe=_text(axis.get("timeframe")) or TICK, + symbol=_text(axis.get("symbol")), + period=_text(axis.get("period")), + ) + + +def _require_tick_axis( + axis: Mapping[str, JSONValue], target: QualityTarget +) -> None: + data_format = _text(axis.get("data_format")) or target.data_format + timeframe = _text(axis.get("timeframe")) or target.timeframe + if data_format.lower() != "ascii" or timeframe != TICK: + raise ValueError("unsupported_base_grain") + + +def _require_reference_fingerprint( + fingerprint: Mapping[str, JSONValue], +) -> None: + if not fingerprint: + raise ValueError("reference_fingerprint_required") + if ( + fingerprint.get("schema_version") + != TIME_SERIES_FINGERPRINT_SCHEMA_VERSION + ): + raise ValueError("unsupported_reference_fingerprint_schema") + if not _text(fingerprint.get("fingerprint_id")): + raise ValueError("reference_fingerprint_id_required") + + +def _axis_matches_target( + axis: Mapping[str, JSONValue], target: QualityTarget +) -> bool: + return bool( + _text(axis.get("data_format")).lower() == target.data_format.lower() + and _text(axis.get("timeframe")) == target.timeframe + and _text(axis.get("symbol")).upper() == target.symbol.upper() + and _text(axis.get("period")) == target.period + ) + + +def _fingerprint_sort_key( + fingerprint: Mapping[str, JSONValue], +) -> tuple[str, str, str, str]: + axis = _mapping(fingerprint.get("target_axis")) + return ( + _text(axis.get("data_format")), + _text(axis.get("timeframe")), + _text(axis.get("symbol")), + _text(axis.get("period")), + ) + + +def _has_existing_synthetic_values(frame: Any) -> bool: + columns = set(getattr(frame, "columns", ())) + return any( + name in columns and frame.get_column(name).null_count() < frame.height + for name in SYNTHETIC_PLACEHOLDER_COLUMNS + ) + + +def _confidence( + *, + valid_count: int, + inspected_count: int, + constraint_status: str, + digits: int, +) -> float: + validity = valid_count / inspected_count if inspected_count else 0.0 + constraint_weight = 1.0 if constraint_status == "ready" else 0.75 + return round(min(1.0, validity * constraint_weight), digits) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _bounded_positive(value: int, maximum: int, name: str) -> None: + if value < 1 or value > maximum: + raise ValueError(f"{name} must be between 1 and {maximum}") + + +def _mapping(value: Any) -> dict[str, JSONValue]: + return dict(value) if isinstance(value, Mapping) else {} + + +def _mapping_rows(value: Any) -> list[dict[str, JSONValue]]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + return [] + return [_mapping(item) for item in value if isinstance(item, Mapping)] + + +def _text(value: Any) -> str: + return "" if value is None else str(value) + + +def _int(value: Any) -> int: + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +def _float(value: Any) -> float: + parsed = _optional_float(value) + return parsed if parsed is not None else 0.0 + + +def _optional_float(value: Any) -> float | None: + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if math.isfinite(parsed) else None + + +__all__ = [ + "DEFAULT_SYNTHETIC_TICK_BLOCK_SIZE", + "DEFAULT_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT", + "DEFAULT_SYNTHETIC_TICK_MAX_ABS_LOG_RETURN", + "DEFAULT_SYNTHETIC_TICK_MAX_GENERATED_ROWS", + "DEFAULT_SYNTHETIC_TICK_MAX_REFERENCE_ROWS", + "DEFAULT_SYNTHETIC_TICK_MINIMUM_REFERENCE_ROWS", + "DEFAULT_SYNTHETIC_TICK_ROUNDING_DIGITS", + "DEFAULT_SYNTHETIC_TICK_SEED", + "SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION", + "SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION", + "SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION", + "SYNTHETIC_TICK_METHOD_CODES", + "SYNTHETIC_TICK_REASON_CODES", + "SYNTHETIC_TICK_STATUS_CODES", + "SyntheticTickGenerationProfile", + "SyntheticTickGenerationResult", + "format_synthetic_tick_generation", + "generate_synthetic_ticks_from_reference", + "reference_fingerprint_from_report", + "validate_synthetic_tick_cache", + "validate_synthetic_tick_frame", +] diff --git a/src/histdatacom/data_quality/time.py b/src/histdatacom/data_quality/time.py index 4c85e4a4..5451f8d2 100644 --- a/src/histdatacom/data_quality/time.py +++ b/src/histdatacom/data_quality/time.py @@ -2,12 +2,12 @@ from __future__ import annotations -from collections import OrderedDict, defaultdict +from collections import Counter, OrderedDict, defaultdict from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any +from typing import Any, cast import zipfile from histdatacom.data_quality.contracts import ( @@ -21,11 +21,16 @@ QualityTarget, QualityTargetKind, ) +from histdatacom.data_quality.limits import ( + BoundedReportLimit, + bounded_report_limit, +) from histdatacom.data_quality.polars_cache import ( read_quality_polars_cache, ) from histdatacom.histdata_ascii import ( EST_NO_DST_OFFSET_MS, + TICK, columns_for_timeframe, delimiter_for_timeframe, parse_histdata_datetime_to_utc_ms, @@ -54,6 +59,11 @@ FX_CLOSE_OPEN_MINUTE = 17 * 60 FX_CLOSE_OPEN_TIME_OF_DAY_MS = FX_CLOSE_OPEN_MINUTE * ONE_MINUTE_MS MAX_TIMESTAMP_SAMPLES = 5 +MAX_TIMESTAMP_INSPECTION_TEXT_CHARS = 80 +DEFAULT_TIMESTAMP_INSPECTION_SAMPLE_LIMIT = MAX_TIMESTAMP_SAMPLES +TIMESTAMP_TOPOLOGY_INSPECTION_SCHEMA_VERSION = ( + "histdatacom.timestamp-topology-inspection.v1" +) TIMESTAMP_SCAN_CACHE_MAX_ENTRIES = 4 TIMESTAMP_BOUNDARY_CACHE_MAX_ENTRIES = 2_048 UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) @@ -80,6 +90,7 @@ class _TimestampScanCacheKey: period: str size_bytes: int mtime_ns: int + prefer_cache: bool @dataclass(frozen=True, slots=True) @@ -134,6 +145,9 @@ class _TimestampScan: row_count: int = 0 parsed_row_count: int = 0 invalid_timestamp_count: int = 0 + invalid_timestamps: list["_TimestampIssueSample"] = field( + default_factory=list + ) field_count_error_count: int = 0 header_row_count: int = 0 valid_rows: list[_TimestampSample] = field(default_factory=list) @@ -195,6 +209,32 @@ class _TimestampSequenceScan: tick_duplicate_rows: list[_TimestampIssueSample] = field( default_factory=list ) + tick_duplicate_timestamp_count: int = 0 + tick_duplicate_timestamps: list["_DuplicateTimestampSample"] = field( + default_factory=list + ) + + +@dataclass(frozen=True, slots=True) +class _DuplicateTimestampSample: + row_number: int + timestamp_source: str + timestamp_utc_ms: int + occurrence_count: int + exact_row_group_count: int + source_member: str = "" + + def to_dict(self) -> dict[str, JSONValue]: + """Return publish-safe duplicate timestamp evidence.""" + return { + "row_number": self.row_number, + "timestamp_source": self.timestamp_source, + "timestamp_utc_ms": self.timestamp_utc_ms, + "utc_timestamp": _utc_iso_from_ms(self.timestamp_utc_ms), + "occurrence_count": self.occurrence_count, + "exact_row_group_count": self.exact_row_group_count, + "source_member": self.source_member, + } @dataclass(frozen=True, slots=True) @@ -675,6 +715,10 @@ def timestamp_topology_payload_for_target( target: QualityTarget, *, tolerance: HistDataGapTolerance | None = None, + inspection_sample_limit: int | None = ( + DEFAULT_TIMESTAMP_INSPECTION_SAMPLE_LIMIT + ), + prefer_cache: bool = True, ) -> dict[str, JSONValue]: """Return a bounded timestamp topology payload for one quality target.""" resolved_tolerance = tolerance or HistDataGapTolerance() @@ -685,7 +729,10 @@ def timestamp_topology_payload_for_target( ) try: - scan = _timestamp_scan_for_target(target) + scan = _timestamp_scan_for_target( + target, + prefer_cache=prefer_cache, + ) except ValueError as exc: return _empty_timestamp_topology_payload( reason="unsupported_timeframe", @@ -712,7 +759,7 @@ def timestamp_topology_payload_for_target( interval_summary = _timestamp_interval_summary(scan) tick_duplicate_count = sequence.tick_duplicate_row_count - return { + payload: dict[str, JSONValue] = { "row_count": scan.row_count, "parsed_row_count": scan.parsed_row_count, "invalid_timestamp_count": scan.invalid_timestamp_count, @@ -740,14 +787,28 @@ def timestamp_topology_payload_for_target( "cache_source": scan.cache_source or None, "gap_tolerance": resolved_tolerance.to_metadata(), } + inspection_context = _timestamp_inspection_context( + scan, + sequence, + gap_scan, + sample_limit=inspection_sample_limit, + ) + if inspection_context: + payload["inspection_context"] = inspection_context + return payload def _is_ascii_text_target(target: QualityTarget) -> bool: - return target.data_format == "ascii" and target.kind in { - QualityTargetKind.CSV, - QualityTargetKind.ZIP, - QualityTargetKind.CACHE, - } + return ( + target.data_format == "ascii" + and target.timeframe == TICK + and target.kind + in { + QualityTargetKind.CSV, + QualityTargetKind.ZIP, + QualityTargetKind.CACHE, + } + ) def _read_text_payload(target: QualityTarget) -> _TextPayload: @@ -810,14 +871,20 @@ def _source_error( ) -def _timestamp_scan_for_target(target: QualityTarget) -> _TimestampScan: - key = _timestamp_scan_cache_key(target) +def _timestamp_scan_for_target( + target: QualityTarget, + *, + prefer_cache: bool = True, +) -> _TimestampScan: + key = _timestamp_scan_cache_key(target, prefer_cache=prefer_cache) cached = _TIMESTAMP_SCAN_CACHE.get(key) if cached is not None: _TIMESTAMP_SCAN_CACHE.move_to_end(key) return cached - cache_scan = _timestamp_scan_from_polars_cache(target) + cache_scan = ( + _timestamp_scan_from_polars_cache(target) if prefer_cache else None + ) if cache_scan is not None: _cache_timestamp_scan(key, cache_scan) boundary = _continuity_boundary_from_scan( @@ -1129,11 +1196,63 @@ def _scan_timestamp_sequence_polars( }, ), ) + duplicate_groups = ( + frame.group_by(duplicate_columns, maintain_order=True) + .agg( + [ + pl.len().alias("_occurrence_count"), + pl.col("_row_number").first().alias("_first_row_number"), + pl.col("datetime").first().alias("_first_datetime"), + ] + ) + .filter(pl.col("_occurrence_count") > 1) + ) + grouped_by_timestamp: dict[str, dict[str, int]] = {} + for row in duplicate_groups.iter_rows(named=True): + timestamp_source = str(row["_source_timestamp"]) + aggregate = grouped_by_timestamp.setdefault( + timestamp_source, + { + "row_number": int(row["_first_row_number"]), + "timestamp_utc_ms": int(row["_first_datetime"]), + "occurrence_count": 0, + "exact_row_group_count": 0, + }, + ) + aggregate["row_number"] = min( + aggregate["row_number"], + int(row["_first_row_number"]), + ) + aggregate["occurrence_count"] += int(row["_occurrence_count"]) + aggregate["exact_row_group_count"] += 1 + duplicate_timestamp_samples = [ + _DuplicateTimestampSample( + row_number=values["row_number"], + timestamp_source=timestamp_source, + timestamp_utc_ms=values["timestamp_utc_ms"], + occurrence_count=values["occurrence_count"], + exact_row_group_count=values["exact_row_group_count"], + ) + for timestamp_source, values in grouped_by_timestamp.items() + ] + duplicate_timestamp_samples.sort( + key=lambda sample: ( + sample.row_number, + sample.timestamp_utc_ms, + sample.timestamp_source, + ) + ) + scan.tick_duplicate_timestamp_count = len(duplicate_timestamp_samples) + scan.tick_duplicate_timestamps.extend( + duplicate_timestamp_samples[:MAX_TIMESTAMP_SAMPLES] + ) return scan def _timestamp_scan_cache_key( target: QualityTarget, + *, + prefer_cache: bool, ) -> _TimestampScanCacheKey: path = Path(target.path) try: @@ -1151,6 +1270,7 @@ def _timestamp_scan_cache_key( period=target.period, size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, + prefer_cache=prefer_cache, ) @@ -1238,6 +1358,18 @@ def _scan_timestamp_rows( ) except ValueError: scan.invalid_timestamp_count += 1 + _append_issue_sample( + scan.invalid_timestamps, + _TimestampIssueSample( + row_number=row_number, + timestamp_source=timestamp_source, + timestamp_utc_ms=None, + source_member=source_member, + metadata={ + "reason_code": "timestamp_parse_failure", + }, + ), + ) _record_raw_timestamp_shape( scan, @@ -1357,6 +1489,7 @@ def _scan_timestamp_sequence( scan = _TimestampSequenceScan() previous: _TimestampSample | None = None seen_tick_rows: dict[tuple[str, ...], _TimestampSample] = {} + tick_row_counts: Counter[tuple[str, ...]] = Counter() for row in rows: if ( @@ -1382,6 +1515,7 @@ def _scan_timestamp_sequence( previous = row if target.timeframe == "T": + tick_row_counts[row.row_values] += 1 duplicate = seen_tick_rows.get(row.row_values) if duplicate is not None: scan.tick_duplicate_row_count += 1 @@ -1402,6 +1536,16 @@ def _scan_timestamp_sequence( else: seen_tick_rows[row.row_values] = row + if target.timeframe == "T": + duplicate_timestamps = _duplicate_timestamp_samples( + seen_tick_rows, + tick_row_counts, + ) + scan.tick_duplicate_timestamp_count = len(duplicate_timestamps) + scan.tick_duplicate_timestamps.extend( + duplicate_timestamps[:MAX_TIMESTAMP_SAMPLES] + ) + return scan @@ -2149,7 +2293,7 @@ def _timestamp_boundary_for_target( target: QualityTarget, ) -> _ContinuityBoundary | None: try: - key = _timestamp_scan_cache_key(target) + key = _timestamp_scan_cache_key(target, prefer_cache=True) except _SourceReadError: return None cached_boundary = _cached_timestamp_boundary(key) @@ -3193,8 +3337,16 @@ def _append_gap_sample( samples: list[_TimestampGapSample], sample: _TimestampGapSample, ) -> None: - if len(samples) < MAX_TIMESTAMP_SAMPLES: - samples.append(sample) + samples.append(sample) + samples.sort( + key=lambda item: ( + -item.gap_ms, + item.current.row_number, + item.previous.row_number, + item.current.timestamp_utc_ms, + ) + ) + del samples[MAX_TIMESTAMP_SAMPLES:] def _append_weekend_activity_sample( @@ -3231,6 +3383,280 @@ def _weekend_activity_samples( return [sample.to_dict() for sample in samples] +def _duplicate_timestamp_samples( + first_rows: Mapping[tuple[str, ...], _TimestampSample], + row_counts: Mapping[tuple[str, ...], int], +) -> list[_DuplicateTimestampSample]: + grouped: dict[str, dict[str, int | str]] = {} + for row_values, occurrence_count in row_counts.items(): + if occurrence_count <= 1: + continue + first = first_rows[row_values] + aggregate = grouped.setdefault( + first.timestamp_source, + { + "row_number": first.row_number, + "timestamp_utc_ms": first.timestamp_utc_ms, + "occurrence_count": 0, + "exact_row_group_count": 0, + "source_member": first.source_member, + }, + ) + aggregate["row_number"] = min( + int(aggregate["row_number"]), + first.row_number, + ) + aggregate["occurrence_count"] = int( + aggregate["occurrence_count"] + ) + int(occurrence_count) + aggregate["exact_row_group_count"] = ( + int(aggregate["exact_row_group_count"]) + 1 + ) + samples = [ + _DuplicateTimestampSample( + row_number=int(values["row_number"]), + timestamp_source=timestamp_source, + timestamp_utc_ms=int(values["timestamp_utc_ms"]), + occurrence_count=int(values["occurrence_count"]), + exact_row_group_count=int(values["exact_row_group_count"]), + source_member=str(values["source_member"]), + ) + for timestamp_source, values in grouped.items() + ] + samples.sort( + key=lambda sample: ( + sample.row_number, + sample.timestamp_utc_ms, + sample.timestamp_source, + ) + ) + return samples + + +def _timestamp_inspection_context( + scan: _TimestampScan, + sequence: _TimestampSequenceScan, + gap_scan: _TimestampGapScan, + *, + sample_limit: int | None, +) -> dict[str, JSONValue]: + limit = bounded_report_limit( + sample_limit, + default_limit=DEFAULT_TIMESTAMP_INSPECTION_SAMPLE_LIMIT, + minimum_limit=0, + maximum_limit=MAX_TIMESTAMP_SAMPLES, + allow_unbounded=False, + ) + context: dict[str, JSONValue] = { + "schema_version": TIMESTAMP_TOPOLOGY_INSPECTION_SCHEMA_VERSION, + } + if scan.invalid_timestamp_count: + context["invalid_timestamps"] = _bounded_inspection_section( + total_count=scan.invalid_timestamp_count, + samples=[ + _inspection_issue_sample(sample) + for sample in scan.invalid_timestamps + ], + limit=limit, + extra={"parse_failure_count": scan.invalid_timestamp_count}, + ) + if sequence.non_monotonic_count: + context["non_monotonic_timestamps"] = _bounded_inspection_section( + total_count=sequence.non_monotonic_count, + samples=[ + _inspection_issue_sample(sample) + for sample in sequence.non_monotonic_rows + ], + limit=limit, + ) + if sequence.tick_duplicate_row_count: + context["duplicate_timestamps"] = _bounded_inspection_section( + total_count=sequence.tick_duplicate_timestamp_count, + samples=[ + _inspection_duplicate_timestamp_sample(sample) + for sample in sequence.tick_duplicate_timestamps + ], + limit=limit, + extra={ + "duplicate_row_count": sequence.tick_duplicate_row_count, + }, + ) + if gap_scan.suspicious_gap_count: + context["suspicious_gaps"] = _bounded_inspection_section( + total_count=gap_scan.suspicious_gap_count, + samples=[ + _inspection_gap_sample( + sample, + expected_session_related=False, + ) + for sample in gap_scan.suspicious_gaps + ], + limit=limit, + ) + if gap_scan.expected_session_closure_count: + context["expected_session_closures"] = _bounded_inspection_section( + total_count=gap_scan.expected_session_closure_count, + samples=[ + _inspection_gap_sample( + sample, + expected_session_related=True, + ) + for sample in gap_scan.expected_session_closures + ], + limit=limit, + extra={"actionable": False}, + ) + if gap_scan.weekend_activity_count: + context["weekend_activity"] = _bounded_inspection_section( + total_count=gap_scan.weekend_activity_count, + samples=[ + _inspection_weekend_sample(sample) + for sample in gap_scan.weekend_activity + ], + limit=limit, + extra={ + "session_bucket_counts": { + "weekend_closure": gap_scan.weekend_activity_count, + } + }, + ) + if len(context) == 1: + return {} + return context + + +def _inspection_issue_sample( + sample: _TimestampIssueSample, +) -> dict[str, JSONValue]: + timestamp_source, truncated = _inspection_timestamp_text( + sample.timestamp_source + ) + metadata: dict[str, JSONValue] = {} + reason_code = sample.metadata.get("reason_code") + if reason_code is not None: + metadata["reason_code"] = str(reason_code) + previous_row_number = sample.metadata.get("previous_row_number") + if previous_row_number is not None: + metadata["previous_row_number"] = _metadata_int(previous_row_number) + previous_timestamp = sample.metadata.get("previous_timestamp_source") + if previous_timestamp is not None: + previous_text, previous_truncated = _inspection_timestamp_text( + str(previous_timestamp) + ) + metadata["previous_timestamp_source"] = previous_text + metadata["previous_timestamp_source_truncated"] = previous_truncated + for key in ("previous_timestamp_utc_ms", "previous_utc_timestamp"): + value = sample.metadata.get(key) + if value is not None: + metadata[key] = value + return { + "row_number": sample.row_number, + "timestamp_source": timestamp_source, + "timestamp_source_truncated": truncated, + "timestamp_utc_ms": sample.timestamp_utc_ms, + "utc_timestamp": sample.utc_timestamp, + "metadata": metadata, + } + + +def _inspection_duplicate_timestamp_sample( + sample: _DuplicateTimestampSample, +) -> dict[str, JSONValue]: + timestamp_source, truncated = _inspection_timestamp_text( + sample.timestamp_source + ) + return { + "row_number": sample.row_number, + "timestamp_source": timestamp_source, + "timestamp_source_truncated": truncated, + "timestamp_utc_ms": sample.timestamp_utc_ms, + "utc_timestamp": _utc_iso_from_ms(sample.timestamp_utc_ms), + "occurrence_count": sample.occurrence_count, + "exact_row_group_count": sample.exact_row_group_count, + } + + +def _inspection_gap_sample( + sample: _TimestampGapSample, + *, + expected_session_related: bool, +) -> dict[str, JSONValue]: + previous_timestamp, previous_truncated = _inspection_timestamp_text( + sample.previous.timestamp_source + ) + timestamp_source, truncated = _inspection_timestamp_text( + sample.current.timestamp_source + ) + return { + "previous_row_number": sample.previous.row_number, + "previous_timestamp_source": previous_timestamp, + "previous_timestamp_source_truncated": previous_truncated, + "previous_timestamp_utc_ms": sample.previous.timestamp_utc_ms, + "previous_utc_timestamp": sample.previous.utc_timestamp, + "row_number": sample.current.row_number, + "timestamp_source": timestamp_source, + "timestamp_source_truncated": truncated, + "timestamp_utc_ms": sample.current.timestamp_utc_ms, + "utc_timestamp": sample.current.utc_timestamp, + "gap_ms": sample.gap_ms, + "classification": sample.classification, + "expected_session_related": expected_session_related, + } + + +def _inspection_weekend_sample( + sample: _WeekendActivitySample, +) -> dict[str, JSONValue]: + timestamp_source, truncated = _inspection_timestamp_text( + sample.row.timestamp_source + ) + return { + "row_number": sample.row.row_number, + "timestamp_source": timestamp_source, + "timestamp_source_truncated": truncated, + "timestamp_utc_ms": sample.row.timestamp_utc_ms, + "utc_timestamp": sample.row.utc_timestamp, + "session_state": sample.session_state, + } + + +def _inspection_timestamp_text(value: str) -> tuple[str, bool]: + normalized = str(value).replace("\r", " ").replace("\n", " ") + truncated = len(normalized) > MAX_TIMESTAMP_INSPECTION_TEXT_CHARS + return normalized[:MAX_TIMESTAMP_INSPECTION_TEXT_CHARS], truncated + + +def _bounded_inspection_section( + *, + total_count: int, + samples: list[dict[str, JSONValue]], + limit: BoundedReportLimit, + extra: Mapping[str, JSONValue] | None = None, +) -> dict[str, JSONValue]: + included = limit.slice(samples) + included_count = len(included) + normalized_total = max(0, int(total_count)) + omitted_count = max(0, normalized_total - included_count) + payload: dict[str, JSONValue] = { + "total_count": normalized_total, + "included_count": included_count, + "omitted_count": omitted_count, + "truncated": omitted_count > 0, + "limit_metadata": { + "samples": { + **limit.limit_payload(), + "total_count": normalized_total, + "included_count": included_count, + "omitted_count": omitted_count, + "truncated": omitted_count > 0, + } + }, + "samples": cast(JSONValue, included), + } + payload.update(dict(extra or {})) + return payload + + def _continuity_samples( samples: list[_ContinuityComparison], ) -> list[JSONValue]: diff --git a/src/histdatacom/data_quality/training_features.py b/src/histdatacom/data_quality/training_features.py new file mode 100644 index 00000000..874a262b --- /dev/null +++ b/src/histdatacom/data_quality/training_features.py @@ -0,0 +1,1854 @@ +"""Row-aligned ASCII tick training feature substrate. + +The canonical training surface is a single flat Polars frame: observed tick +market data plus deterministic row identity, issue flags, classification codes, +training controls, and synthetic placeholders. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityLocation, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.histdata_ascii import TICK +from histdatacom.runtime_contracts import JSONValue + +TRAINING_SCHEMA_VERSION = "histdatacom.ascii-tick-training-features.v1" +TRAINING_FEATURE_REPORT_SCHEMA_VERSION = ( + "histdatacom.training-feature-report.v1" +) +TRAINING_FEATURE_REPORT_RULE_ID = "training_features.row_issues" + +DEFAULT_SUSPICIOUS_TICK_GAP_MS = 172_800_000 + +IDENTITY_COLUMNS = ( + "training_schema_version", + "series_id", + "period", + "row_id", + "source_row_number", + "event_seq", + "symbol", + "format", + "timeframe", + "source", + "timestamp_utc_ms", +) + +QUALITY_ISSUE_COLUMNS = ( + "dq_issue_duplicate_timestamp", + "dq_issue_non_monotonic_timestamp", + "dq_issue_gap_after_previous", + "dq_issue_suspicious_gap", + "dq_issue_weekend_activity", + "dq_issue_session_closed", + "dq_issue_negative_spread", + "dq_issue_zero_spread", + "dq_issue_wide_spread", + "dq_issue_invalid_row", + "dq_issue_partial_row", + "dq_issue_source_unavailable", + "dq_issue_topology_unavailable", + "dq_issue_distribution_missing", + "dq_issue_precision_warning", + "dq_issue_cache_float_precision", + "dq_issue_fingerprint_unready", +) + +HARD_ISSUE_COLUMNS = ( + "dq_issue_non_monotonic_timestamp", + "dq_issue_negative_spread", + "dq_issue_invalid_row", + "dq_issue_source_unavailable", + "dq_issue_topology_unavailable", +) + +WARNING_ISSUE_COLUMNS = tuple( + column + for column in QUALITY_ISSUE_COLUMNS + if column not in HARD_ISSUE_COLUMNS +) + +SYNTHETIC_PLACEHOLDER_COLUMNS = ( + "synth_bid", + "synth_ask", + "synth_spread", + "synth_mid", + "synth_method_code", + "synth_confidence", + "synth_usable", +) + +CLASSICAL_MODEL_CONTRACT_COLUMNS = ( + "cm_input_schema_version", + "cm_input_derivation_id", + "cm_input_status_code", + "cm_input_ready", + "cm_input_exclusion_reason_code", + "cm_input_frequency_ms", + "cm_input_bin_start_utc_ms", + "cm_input_bin_end_utc_ms", + "cm_input_available_at_utc_ms", + "cm_input_observation_count", + "cm_input_source_first_row_id", + "cm_input_source_last_row_id", + "cm_input_observed_value", + "cm_input_value", + "cm_input_spread", + "cm_input_transform_code", + "cm_input_calculation_basis_code", + "cm_input_available", + "cm_input_expected_closure", + "cm_input_unexpected_missing", + "cm_fold_schema_version", + "cm_fold_id", + "cm_fold_kind_code", + "cm_fold_origin_row_id", + "cm_fold_target_row_id", + "cm_fold_horizon", + "cm_fold_training_start_row_id", + "cm_fold_training_end_row_id", + "cm_fold_evaluation_start_row_id", + "cm_fold_evaluation_end_row_id", + "cm_evaluation_schema_version", + "cm_evaluation_status_code", + "cm_evaluation_target_available", + "cm_evaluation_forecast", + "cm_evaluation_actual", + "cm_evaluation_error", + "cm_evaluation_diagnostic_only", +) + +EXPONENTIAL_SMOOTHING_COLUMNS = ( + "cm_ets_schema_version", + "cm_ets_input_derivation_id", + "cm_ets_model_id", + "cm_ets_family_code", + "cm_ets_specification_code", + "cm_ets_error_code", + "cm_ets_trend_code", + "cm_ets_seasonal_code", + "cm_ets_damped_trend", + "cm_ets_initialization_code", + "cm_ets_fit_status_code", + "cm_ets_converged", + "cm_ets_fold_id", + "cm_ets_origin_row_id", + "cm_ets_target_row_id", + "cm_ets_horizon", + "cm_ets_forecast", + "cm_ets_forecast_available", + "cm_ets_forecast_available_at_utc_ms", + "cm_ets_actual", + "cm_ets_error", + "cm_ets_diagnostic_available", + "cm_ets_diagnostic_available_at_utc_ms", + "cm_ets_diagnostic_only", + "cm_ets_original_scale", + "cm_ets_training_eligible", +) + +AUTOREGRESSIVE_FAMILY_COLUMN_SUFFIXES = ( + "schema_version", + "input_derivation_id", + "model_id", + "family_code", + "specification_code", + "p", + "d", + "q", + "trend_code", + "calculation_basis_code", + "fit_status_code", + "failure_reason_code", + "converged", + "stationary", + "invertible", + "ar_root_min_modulus", + "ma_root_min_modulus", + "covariance_condition_number", + "effective_observation_count", + "fold_id", + "origin_row_id", + "target_row_id", + "horizon", + "forecast", + "forecast_available", + "forecast_available_at_utc_ms", + "actual", + "error", + "diagnostic_available", + "diagnostic_available_at_utc_ms", + "diagnostic_only", + "original_scale", + "training_eligible", +) +AUTOREGRESSIVE_COLUMNS = tuple( + f"cm_{family}_{suffix}" + for family in ("ar", "arma", "arima") + for suffix in AUTOREGRESSIVE_FAMILY_COLUMN_SUFFIXES +) + +SEASONAL_EXOGENOUS_FAMILY_COLUMN_SUFFIXES = ( + "schema_version", + "input_derivation_id", + "model_id", + "family_code", + "specification_code", + "p", + "d", + "q", + "seasonal_p", + "seasonal_d", + "seasonal_q", + "seasonal_period", + "seasonal_cycle_ms", + "trend_code", + "regressor_set_code", + "regressor_count", + "regressor_available", + "calculation_basis_code", + "fit_status_code", + "failure_reason_code", + "converged", + "stationary", + "invertible", + "ar_root_min_modulus", + "ma_root_min_modulus", + "covariance_condition_number", + "effective_observation_count", + "fold_id", + "origin_row_id", + "target_row_id", + "horizon", + "forecast", + "forecast_available", + "forecast_available_at_utc_ms", + "actual", + "error", + "diagnostic_available", + "diagnostic_available_at_utc_ms", + "diagnostic_only", + "original_scale", + "training_eligible", +) +SEASONAL_EXOGENOUS_COLUMNS = tuple( + f"cm_{family}_{suffix}" + for family in ("sarima", "arimax", "sarimax") + for suffix in SEASONAL_EXOGENOUS_FAMILY_COLUMN_SUFFIXES +) + +STATE_SPACE_COLUMN_SUFFIXES = ( + "schema_version", + "input_derivation_id", + "model_id", + "family_code", + "specification_code", + "state_dimension", + "component_count", + "initialization_code", + "fit_status_code", + "failure_reason_code", + "converged", + "effective_observation_count", + "missing_observation_count", + "prediction_only_transition_count", + "max_prediction_only_gap", + "fold_id", + "origin_row_id", + "target_row_id", + "horizon", + "forecast", + "forecast_standard_error", + "forecast_lower", + "forecast_upper", + "forecast_available", + "forecast_available_at_utc_ms", + "actual", + "error", + "diagnostic_available", + "diagnostic_available_at_utc_ms", + "diagnostic_only", + "original_scale", + "training_eligible", +) +STATE_SPACE_COLUMNS = tuple( + f"cm_state_space_{suffix}" for suffix in STATE_SPACE_COLUMN_SUFFIXES +) + +KALMAN_COLUMN_SUFFIXES = ( + "schema_version", + "model_id", + "filtered_calculation_basis_code", + "filtered_level", + "filtered_trend", + "filtered_level_variance", + "filtered_trend_variance", + "filtered_available", + "filtered_available_at_utc_ms", + "filtered_training_eligible", + "smoothed_calculation_basis_code", + "smoothed_level", + "smoothed_trend", + "smoothed_level_variance", + "smoothed_trend_variance", + "smoothed_available", + "smoothed_available_at_utc_ms", + "smoothed_retrospective", + "smoothed_diagnostic_only", + "smoothed_training_eligible", +) +KALMAN_COLUMNS = tuple( + f"cm_kalman_{suffix}" for suffix in KALMAN_COLUMN_SUFFIXES +) + +VOLATILITY_FAMILY_COLUMN_SUFFIXES = ( + "schema_version", + "input_derivation_id", + "model_id", + "specification_code", + "input_definition_code", + "mean_model_code", + "distribution_code", + "innovation_order", + "variance_order", + "scale_factor", + "fit_status_code", + "failure_reason_code", + "converged", + "effective_observation_count", + "missing_reset_count", + "fold_id", + "origin_row_id", + "target_row_id", + "horizon", + "mean_forecast", + "variance_forecast", + "volatility_forecast", + "annualized_variance_forecast", + "annualized_volatility_forecast", + "forecast_available", + "forecast_available_at_utc_ms", + "actual_return", + "realized_variance_proxy", + "mean_error", + "variance_error", + "volatility_error", + "qlike_loss", + "diagnostic_available", + "diagnostic_available_at_utc_ms", + "diagnostic_only", + "persistence", + "unconditional_variance", + "boundary_parameter", + "training_eligible", +) +VOLATILITY_COLUMNS = tuple( + f"cm_{family}_{suffix}" + for family in ("arch", "garch") + for suffix in VOLATILITY_FAMILY_COLUMN_SUFFIXES +) + +CLASSICAL_MODEL_COMPARISON_COLUMNS = ( + "cm_comparison_schema_version", + "cm_comparison_id", + "cm_comparison_dataset_id", + "cm_comparison_regularization_contract_id", + "cm_comparison_fold_set_id", + "cm_comparison_family_code", + "cm_comparison_model_id", + "cm_comparison_specification_code", + "cm_comparison_target_metric_code", + "cm_comparison_scale_code", + "cm_comparison_metric_code", + "cm_comparison_horizon", + "cm_comparison_fold_id", + "cm_comparison_origin_row_id", + "cm_comparison_target_row_id", + "cm_comparison_eligible", + "cm_comparison_eligibility_code", + "cm_comparison_reason_code", + "cm_comparison_metric_value", + "cm_comparison_reference_metric_value", + "cm_comparison_diagnostic_available", + "cm_comparison_diagnostic_available_at_utc_ms", + "cm_comparison_diagnostic_only", + "cm_comparison_training_eligible", + "cm_skill_schema_version", + "cm_skill_reference_baseline_code", + "cm_skill_status_code", + "cm_skill_value", + "cm_skill_negative", + "cm_skill_support_count", + "cm_skill_available", + "cm_skill_diagnostic_only", + "cm_stability_schema_version", + "cm_stability_status_code", + "cm_stability_fold_count", + "cm_stability_error_drift", + "cm_stability_skill_drift", + "cm_stability_parameter_drift", + "cm_stability_fit_duration_drift", + "cm_stability_convergence_rate", + "cm_stability_failure_rate", + "cm_stability_available", + "cm_stability_diagnostic_only", +) + +TRAINING_REQUIRED_COLUMNS = ( + *IDENTITY_COLUMNS, + "spread", + "mid", + "quality_status_code", + "quality_severity_code", + "quality_finding_count", + "quality_warning_count", + "quality_error_count", + *QUALITY_ISSUE_COLUMNS, + "gap_from_previous_ms", + "expected_gap_ms", + "period_invalid_row_rate", + "period_zero_spread_rate", + "period_negative_spread_rate", + "period_duplicate_timestamp_count", + "period_suspicious_gap_count", + "period_quality_issue_count", + "training_usable", + "training_weight", + "training_exclusion_reason_code", + "class_quality_state_code", + "class_session_state_code", + "class_spread_regime_code", + "class_gap_state_code", + "class_volatility_regime_code", + "class_training_action_code", + *SYNTHETIC_PLACEHOLDER_COLUMNS, + *CLASSICAL_MODEL_CONTRACT_COLUMNS, + *EXPONENTIAL_SMOOTHING_COLUMNS, + *AUTOREGRESSIVE_COLUMNS, + *SEASONAL_EXOGENOUS_COLUMNS, + *STATE_SPACE_COLUMNS, + *KALMAN_COLUMNS, + *VOLATILITY_COLUMNS, + *CLASSICAL_MODEL_COMPARISON_COLUMNS, +) + +ISSUE_CODE_BY_COLUMN = { + "dq_issue_duplicate_timestamp": "DQ_ISSUE_DUPLICATE_TIMESTAMP", + "dq_issue_non_monotonic_timestamp": "DQ_ISSUE_NON_MONOTONIC_TIMESTAMP", + "dq_issue_gap_after_previous": "DQ_ISSUE_GAP_AFTER_PREVIOUS", + "dq_issue_suspicious_gap": "DQ_ISSUE_SUSPICIOUS_GAP", + "dq_issue_weekend_activity": "DQ_ISSUE_WEEKEND_ACTIVITY", + "dq_issue_session_closed": "DQ_ISSUE_SESSION_CLOSED", + "dq_issue_negative_spread": "DQ_ISSUE_NEGATIVE_SPREAD", + "dq_issue_zero_spread": "DQ_ISSUE_ZERO_SPREAD", + "dq_issue_wide_spread": "DQ_ISSUE_WIDE_SPREAD", + "dq_issue_invalid_row": "DQ_ISSUE_INVALID_ROW", + "dq_issue_partial_row": "DQ_ISSUE_PARTIAL_ROW", + "dq_issue_source_unavailable": "DQ_ISSUE_SOURCE_UNAVAILABLE", + "dq_issue_topology_unavailable": "DQ_ISSUE_TOPOLOGY_UNAVAILABLE", + "dq_issue_distribution_missing": "DQ_ISSUE_DISTRIBUTION_MISSING", + "dq_issue_precision_warning": "DQ_ISSUE_PRECISION_WARNING", + "dq_issue_cache_float_precision": "DQ_ISSUE_CACHE_FLOAT_PRECISION", + "dq_issue_fingerprint_unready": "DQ_ISSUE_FINGERPRINT_UNREADY", +} + + +@dataclass(frozen=True, slots=True) +class TrainingFeatureDefinition: + """A scalar column in the row-aligned training feature schema.""" + + name: str + dtype: str + default: JSONValue | None + description: str + source: str + grain: str + nullable: bool = True + + +def training_feature_definitions() -> tuple[TrainingFeatureDefinition, ...]: + """Return the canonical ASCII tick training feature catalog.""" + definitions = [ + TrainingFeatureDefinition( + "training_schema_version", + "Utf8", + TRAINING_SCHEMA_VERSION, + "Version marker for the enriched ASCII tick training schema.", + "training_features", + "row", + False, + ), + TrainingFeatureDefinition( + "series_id", + "Utf8", + "", + "Deterministic series identity derived from stable dimensions.", + "training_features", + "series", + False, + ), + TrainingFeatureDefinition( + "period", + "Utf8", + "", + "Source archive/cache period for the row.", + "source", + "period", + False, + ), + TrainingFeatureDefinition( + "row_id", + "Int64", + None, + "Deterministic row identity within the series and period.", + "training_features", + "row", + False, + ), + TrainingFeatureDefinition( + "source_row_number", + "Int64", + None, + "One-based row number from the source order.", + "source", + "row", + False, + ), + TrainingFeatureDefinition( + "event_seq", + "Int64", + None, + "Deterministic event sequence preserving duplicate timestamps.", + "training_features", + "row", + False, + ), + TrainingFeatureDefinition( + "symbol", + "Utf8", + "", + "FX symbol or pair for the observed tick series.", + "source", + "series", + False, + ), + TrainingFeatureDefinition( + "format", + "Utf8", + "ascii", + "HistData source format. Initial training support is ASCII only.", + "source", + "series", + False, + ), + TrainingFeatureDefinition( + "timeframe", + "Utf8", + TICK, + "HistData source timeframe. Initial training support is tick only.", + "source", + "series", + False, + ), + TrainingFeatureDefinition( + "source", + "Utf8", + "histdata.com", + "Source system for the observed row.", + "source", + "series", + False, + ), + TrainingFeatureDefinition( + "timestamp_utc_ms", + "Int64", + None, + "Observed UTC epoch millisecond timestamp as a feature.", + "source", + "row", + True, + ), + TrainingFeatureDefinition( + "spread", + "Float64", + None, + "Observed ask minus bid.", + "market", + "row", + True, + ), + TrainingFeatureDefinition( + "mid", + "Float64", + None, + "Observed midpoint between bid and ask.", + "market", + "row", + True, + ), + TrainingFeatureDefinition( + "gap_from_previous_ms", + "Int64", + None, + "Observed timestamp gap from the prior source row.", + "time", + "row", + True, + ), + TrainingFeatureDefinition( + "expected_gap_ms", + "Int64", + None, + "Expected tick gap when a deterministic expectation is available.", + "time", + "row", + True, + ), + ] + definitions.extend(_quality_definition_rows()) + definitions.extend(_period_metric_definition_rows()) + definitions.extend(_classification_definition_rows()) + definitions.extend(_synthetic_definition_rows()) + definitions.extend(_classical_model_contract_definition_rows()) + definitions.extend(_exponential_smoothing_definition_rows()) + definitions.extend(_autoregressive_definition_rows()) + definitions.extend(_seasonal_exogenous_definition_rows()) + definitions.extend(_state_space_definition_rows()) + definitions.extend(_volatility_definition_rows()) + definitions.extend(_classical_model_comparison_definition_rows()) + return tuple(definitions) + + +def required_training_feature_columns() -> tuple[str, ...]: + """Return the required row-aligned training feature column names.""" + return TRAINING_REQUIRED_COLUMNS + + +def enrich_tick_cache_with_training_features( + frame: Any, + *, + target: Any | None = None, + symbol: str = "", + data_format: str = "", + timeframe: str = "", + period: str = "", + source: str = "histdata.com", + quality_report: QualityReport | None = None, + quality_payload: Mapping[str, JSONValue] | None = None, + fingerprint_payload: Mapping[str, JSONValue] | None = None, + classification_profile: Mapping[str, JSONValue] | None = None, +) -> Any: + """Return a flat enriched ASCII tick training feature frame. + + The output keeps observed market columns intact and adds deterministic + identity, quality issue, classification, training-control, period-metric, + and synthetic placeholder columns at the same row grain. + """ + import polars as pl + + _ = ( + quality_report, + quality_payload, + fingerprint_payload, + classification_profile, + ) + context = _training_context( + target, + symbol=symbol, + data_format=data_format, + timeframe=timeframe, + period=period, + source=source, + ) + _require_ascii_tick_context(context) + if "training_schema_version" in getattr(frame, "columns", ()): + return _ensure_registered_columns(frame) + + _require_observed_tick_columns(frame) + observed_columns = set(frame.columns) + series_id = _series_id(context) + + enriched = frame.with_row_index("__training_row_index", offset=1) + partial_exprs = [ + pl.col(column).is_null() if column in observed_columns else pl.lit(True) + for column in ("datetime", "bid", "ask", "vol") + ] + suspicious_gap_threshold = _suspicious_gap_threshold(context) + enriched = enriched.with_columns( + [ + pl.lit(TRAINING_SCHEMA_VERSION).alias("training_schema_version"), + pl.lit(series_id).alias("series_id"), + pl.lit(context["period"]).alias("period"), + pl.col("__training_row_index").cast(pl.Int64).alias("row_id"), + pl.col("__training_row_index") + .cast(pl.Int64) + .alias("source_row_number"), + pl.col("__training_row_index").cast(pl.Int64).alias("event_seq"), + pl.lit(context["symbol"]).alias("symbol"), + pl.lit(context["format"]).alias("format"), + pl.lit(context["timeframe"]).alias("timeframe"), + pl.lit(context["source"]).alias("source"), + pl.col("datetime").cast(pl.Int64).alias("timestamp_utc_ms"), + (pl.col("ask") - pl.col("bid")).alias("spread"), + ((pl.col("ask") + pl.col("bid")) / 2).alias("mid"), + pl.col("datetime") + .cast(pl.Int64) + .diff() + .alias("gap_from_previous_ms"), + pl.lit(None).cast(pl.Int64).alias("expected_gap_ms"), + pl.sum_horizontal(partial_exprs) + .gt(0) + .alias("dq_issue_partial_row"), + ] + ) + enriched = enriched.with_columns( + [ + pl.col("datetime") + .is_duplicated() + .fill_null(False) + .alias("dq_issue_duplicate_timestamp"), + pl.col("gap_from_previous_ms") + .lt(0) + .fill_null(False) + .alias("dq_issue_non_monotonic_timestamp"), + pl.col("gap_from_previous_ms") + .gt(suspicious_gap_threshold) + .fill_null(False) + .alias("dq_issue_suspicious_gap"), + pl.lit(False).alias("dq_issue_weekend_activity"), + pl.lit(False).alias("dq_issue_session_closed"), + pl.col("spread") + .lt(0) + .fill_null(False) + .alias("dq_issue_negative_spread"), + pl.col("spread") + .eq(0) + .fill_null(False) + .alias("dq_issue_zero_spread"), + pl.col("spread") + .gt(0.01) + .fill_null(False) + .alias("dq_issue_wide_spread"), + _invalid_row_expr().alias("dq_issue_invalid_row"), + pl.lit(False).alias("dq_issue_source_unavailable"), + pl.lit(False).alias("dq_issue_topology_unavailable"), + pl.lit(False).alias("dq_issue_distribution_missing"), + pl.lit(False).alias("dq_issue_precision_warning"), + pl.lit(False).alias("dq_issue_cache_float_precision"), + pl.lit(False).alias("dq_issue_fingerprint_unready"), + ] + ) + enriched = enriched.with_columns( + pl.col("dq_issue_suspicious_gap").alias("dq_issue_gap_after_previous") + ) + enriched = _with_period_metrics(enriched) + enriched = _with_quality_counts(enriched) + enriched = _with_classification(enriched) + enriched = _with_synthetic_placeholders(enriched) + enriched = enriched.drop("__training_row_index") + return _ensure_registered_columns(enriched) + + +def ensure_tick_training_features( + frame: Any, + *, + target: Any | None = None, + symbol: str = "", + data_format: str = "", + timeframe: str = "", + period: str = "", + source: str = "histdata.com", +) -> Any: + """Return an enriched ASCII tick frame, preserving already-enriched caches.""" + return enrich_tick_cache_with_training_features( + frame, + target=target, + symbol=symbol, + data_format=data_format, + timeframe=timeframe, + period=period, + source=source, + ) + + +def quality_report_from_training_features( + frame: Any, + *, + target: Any | None = None, +) -> QualityReport: + """Derive an audit QualityReport from row-aligned training issue columns.""" + _require_ascii_tick_context( + _training_context( + target, + symbol="", + data_format="", + timeframe="", + period="", + source="histdata.com", + ) + ) + quality_target = _quality_target_from_context(target) + issue_counts = _issue_counts(frame) + findings = tuple( + _finding_for_issue_column(quality_target, column, count) + for column, count in issue_counts.items() + if count > 0 + ) + rule_result = QualityRuleResult( + rule_id=TRAINING_FEATURE_REPORT_RULE_ID, + target=quality_target, + findings=findings, + ) + return QualityReport( + targets=(quality_target,), + rule_results=(rule_result,), + metadata={ + "schema_version": TRAINING_FEATURE_REPORT_SCHEMA_VERSION, + "training_schema_version": TRAINING_SCHEMA_VERSION, + "row_count": int(getattr(frame, "height", 0) or 0), + "issue_counts": _json_int_counts(issue_counts), + "training_action_counts": _json_int_counts( + _value_counts(frame, "class_training_action_code") + ), + "quality_state_counts": _json_int_counts( + _value_counts(frame, "class_quality_state_code") + ), + }, + ) + + +def _quality_definition_rows() -> list[TrainingFeatureDefinition]: + definitions = [ + TrainingFeatureDefinition( + "quality_status_code", + "Int32", + 0, + "Row quality status code: 0 clean, 1 warning, 2 failed.", + "training_features", + "row", + False, + ), + TrainingFeatureDefinition( + "quality_severity_code", + "Int32", + 0, + "Compact row severity code aligned to quality status.", + "training_features", + "row", + False, + ), + TrainingFeatureDefinition( + "quality_finding_count", + "Int32", + 0, + "Count of row-aligned quality issues.", + "training_features", + "row", + False, + ), + TrainingFeatureDefinition( + "quality_warning_count", + "Int32", + 0, + "Count of warning issue flags on this row.", + "training_features", + "row", + False, + ), + TrainingFeatureDefinition( + "quality_error_count", + "Int32", + 0, + "Count of hard error issue flags on this row.", + "training_features", + "row", + False, + ), + ] + for column in QUALITY_ISSUE_COLUMNS: + definitions.append( + TrainingFeatureDefinition( + column, + "Boolean", + False, + f"Row-aligned data-quality issue indicator: {column}.", + "training_features", + "row", + False, + ) + ) + return definitions + + +def _period_metric_definition_rows() -> list[TrainingFeatureDefinition]: + return [ + TrainingFeatureDefinition( + "period_invalid_row_rate", + "Float64", + 0.0, + "Period-level invalid row rate projected onto each row.", + "training_features", + "period", + False, + ), + TrainingFeatureDefinition( + "period_zero_spread_rate", + "Float64", + 0.0, + "Period-level zero-spread row rate projected onto each row.", + "training_features", + "period", + False, + ), + TrainingFeatureDefinition( + "period_negative_spread_rate", + "Float64", + 0.0, + "Period-level negative-spread row rate projected onto each row.", + "training_features", + "period", + False, + ), + TrainingFeatureDefinition( + "period_duplicate_timestamp_count", + "Int64", + 0, + "Rows in the period flagged with duplicate timestamps.", + "training_features", + "period", + False, + ), + TrainingFeatureDefinition( + "period_suspicious_gap_count", + "Int64", + 0, + "Rows in the period flagged with suspicious timestamp gaps.", + "training_features", + "period", + False, + ), + TrainingFeatureDefinition( + "period_quality_issue_count", + "Int64", + 0, + "Total row issue flags in the period projected onto each row.", + "training_features", + "period", + False, + ), + ] + + +def _classification_definition_rows() -> list[TrainingFeatureDefinition]: + return [ + TrainingFeatureDefinition( + "training_usable", + "Boolean", + True, + "Whether the observed row is usable for training as-is.", + "classification", + "row", + False, + ), + TrainingFeatureDefinition( + "training_weight", + "Float64", + 1.0, + "Default deterministic training weight for the row.", + "classification", + "row", + False, + ), + TrainingFeatureDefinition( + "training_exclusion_reason_code", + "Int32", + 0, + "Compact exclusion reason code; 0 means not excluded.", + "classification", + "row", + False, + ), + TrainingFeatureDefinition( + "class_quality_state_code", + "Int32", + 0, + "Quality class: 0 clean, 1 warning, 2 failed, 3 unusable.", + "classification", + "row", + False, + ), + TrainingFeatureDefinition( + "class_session_state_code", + "Int32", + 0, + "Session class placeholder for later calendar conditioning.", + "classification", + "row", + False, + ), + TrainingFeatureDefinition( + "class_spread_regime_code", + "Int32", + 0, + "Spread class: 0 normal, 1 zero, 2 negative, 3 wide, 4 unknown.", + "classification", + "row", + False, + ), + TrainingFeatureDefinition( + "class_gap_state_code", + "Int32", + 0, + "Gap class: 0 continuous/unknown, 2 suspicious, 3 nonmonotonic, 4 duplicate.", + "classification", + "row", + False, + ), + TrainingFeatureDefinition( + "class_volatility_regime_code", + "Int32", + 0, + "Volatility regime placeholder for later tick-return features.", + "classification", + "row", + False, + ), + TrainingFeatureDefinition( + "class_training_action_code", + "Int32", + 0, + "Training action: 0 use, 1 augment, 2 repair, 3 replace, 4 exclude.", + "classification", + "row", + False, + ), + ] + + +def _synthetic_definition_rows() -> list[TrainingFeatureDefinition]: + return [ + TrainingFeatureDefinition( + "synth_bid", + "Float64", + None, + "Generated synthetic bid; observed bid is never overwritten.", + "synthetic", + "row", + True, + ), + TrainingFeatureDefinition( + "synth_ask", + "Float64", + None, + "Generated synthetic ask; observed ask is never overwritten.", + "synthetic", + "row", + True, + ), + TrainingFeatureDefinition( + "synth_spread", + "Float64", + None, + "Generated spread derived from synthetic bid and ask.", + "synthetic", + "row", + True, + ), + TrainingFeatureDefinition( + "synth_mid", + "Float64", + None, + "Generated midpoint derived from synthetic bid and ask.", + "synthetic", + "row", + True, + ), + TrainingFeatureDefinition( + "synth_method_code", + "Int32", + None, + "Deterministic synthetic-generation method code.", + "synthetic", + "row", + True, + ), + TrainingFeatureDefinition( + "synth_confidence", + "Float64", + None, + "Reference-evidence confidence for the generated row.", + "synthetic", + "row", + True, + ), + TrainingFeatureDefinition( + "synth_usable", + "Boolean", + None, + "Whether generated synthetic values are usable on this row.", + "synthetic", + "row", + True, + ), + ] + + +def _classical_model_contract_definition_rows() -> ( + list[TrainingFeatureDefinition] +): + strings = { + "cm_input_schema_version", + "cm_input_derivation_id", + "cm_fold_schema_version", + "cm_evaluation_schema_version", + } + booleans = { + "cm_input_ready", + "cm_input_available", + "cm_input_expected_closure", + "cm_input_unexpected_missing", + "cm_evaluation_target_available", + "cm_evaluation_diagnostic_only", + } + floats = { + "cm_input_observed_value", + "cm_input_value", + "cm_input_spread", + "cm_evaluation_forecast", + "cm_evaluation_actual", + "cm_evaluation_error", + } + definitions: list[TrainingFeatureDefinition] = [] + for name in CLASSICAL_MODEL_CONTRACT_COLUMNS: + dtype = ( + "Utf8" + if name in strings + else ( + "Boolean" + if name in booleans + else "Float64" if name in floats else "Int64" + ) + ) + definitions.append( + TrainingFeatureDefinition( + name, + dtype, + None, + ( + "Point-in-time-safe classical model contract scalar: " + f"{name}." + ), + "classical_model_contracts", + "row", + True, + ) + ) + return definitions + + +def _exponential_smoothing_definition_rows() -> list[TrainingFeatureDefinition]: + strings = { + "cm_ets_schema_version", + "cm_ets_input_derivation_id", + "cm_ets_model_id", + } + booleans = { + "cm_ets_damped_trend", + "cm_ets_converged", + "cm_ets_forecast_available", + "cm_ets_diagnostic_available", + "cm_ets_diagnostic_only", + "cm_ets_original_scale", + "cm_ets_training_eligible", + } + floats = {"cm_ets_forecast", "cm_ets_actual", "cm_ets_error"} + definitions: list[TrainingFeatureDefinition] = [] + for name in EXPONENTIAL_SMOOTHING_COLUMNS: + dtype = ( + "Utf8" + if name in strings + else ( + "Boolean" + if name in booleans + else "Float64" if name in floats else "Int64" + ) + ) + definitions.append( + TrainingFeatureDefinition( + name, + dtype, + None, + f"Point-in-time-safe exponential-smoothing scalar: {name}.", + "exponential_smoothing", + "row", + True, + ) + ) + return definitions + + +def _autoregressive_definition_rows() -> list[TrainingFeatureDefinition]: + strings = { + "schema_version", + "input_derivation_id", + "model_id", + } + booleans = { + "converged", + "stationary", + "invertible", + "forecast_available", + "diagnostic_available", + "diagnostic_only", + "original_scale", + "training_eligible", + } + floats = { + "forecast", + "actual", + "error", + "ar_root_min_modulus", + "ma_root_min_modulus", + "covariance_condition_number", + } + definitions: list[TrainingFeatureDefinition] = [] + for name in AUTOREGRESSIVE_COLUMNS: + suffix = name.split("_", maxsplit=2)[-1] + dtype = ( + "Utf8" + if suffix in strings + else ( + "Boolean" + if suffix in booleans + else "Float64" if suffix in floats else "Int64" + ) + ) + definitions.append( + TrainingFeatureDefinition( + name, + dtype, + None, + f"Point-in-time-safe autoregressive scalar: {name}.", + "autoregressive", + "row", + True, + ) + ) + return definitions + + +def _seasonal_exogenous_definition_rows() -> list[TrainingFeatureDefinition]: + strings = {"schema_version", "input_derivation_id", "model_id"} + booleans = { + "regressor_available", + "converged", + "stationary", + "invertible", + "forecast_available", + "diagnostic_available", + "diagnostic_only", + "original_scale", + "training_eligible", + } + floats = { + "forecast", + "actual", + "error", + "ar_root_min_modulus", + "ma_root_min_modulus", + "covariance_condition_number", + } + definitions: list[TrainingFeatureDefinition] = [] + for name in SEASONAL_EXOGENOUS_COLUMNS: + suffix = name.split("_", maxsplit=2)[-1] + dtype = ( + "Utf8" + if suffix in strings + else ( + "Boolean" + if suffix in booleans + else "Float64" if suffix in floats else "Int64" + ) + ) + definitions.append( + TrainingFeatureDefinition( + name, + dtype, + None, + f"Point-in-time-safe seasonal/exogenous scalar: {name}.", + "seasonal_exogenous", + "row", + True, + ) + ) + return definitions + + +def _state_space_definition_rows() -> list[TrainingFeatureDefinition]: + strings = {"schema_version", "input_derivation_id", "model_id"} + booleans = { + "converged", + "forecast_available", + "diagnostic_available", + "diagnostic_only", + "original_scale", + "training_eligible", + "filtered_available", + "filtered_training_eligible", + "smoothed_available", + "smoothed_retrospective", + "smoothed_diagnostic_only", + "smoothed_training_eligible", + } + floats = { + "forecast", + "forecast_standard_error", + "forecast_lower", + "forecast_upper", + "actual", + "error", + "filtered_level", + "filtered_trend", + "filtered_level_variance", + "filtered_trend_variance", + "smoothed_level", + "smoothed_trend", + "smoothed_level_variance", + "smoothed_trend_variance", + } + definitions: list[TrainingFeatureDefinition] = [] + for name in (*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS): + if name.startswith("cm_state_space_"): + suffix = name.removeprefix("cm_state_space_") + else: + suffix = name.removeprefix("cm_kalman_") + dtype = ( + "Utf8" + if suffix in strings + else ( + "Boolean" + if suffix in booleans + else "Float64" if suffix in floats else "Int64" + ) + ) + definitions.append( + TrainingFeatureDefinition( + name, + dtype, + None, + f"Point-in-time state-space/Kalman scalar: {name}.", + "state_space", + "row", + True, + ) + ) + return definitions + + +def _volatility_definition_rows() -> list[TrainingFeatureDefinition]: + strings = {"schema_version", "input_derivation_id", "model_id"} + booleans = { + "converged", + "forecast_available", + "diagnostic_available", + "diagnostic_only", + "boundary_parameter", + "training_eligible", + } + floats = { + "scale_factor", + "mean_forecast", + "variance_forecast", + "volatility_forecast", + "annualized_variance_forecast", + "annualized_volatility_forecast", + "actual_return", + "realized_variance_proxy", + "mean_error", + "variance_error", + "volatility_error", + "qlike_loss", + "persistence", + "unconditional_variance", + } + definitions: list[TrainingFeatureDefinition] = [] + for name in VOLATILITY_COLUMNS: + suffix = name.removeprefix("cm_arch_").removeprefix("cm_garch_") + dtype = ( + "Utf8" + if suffix in strings + else ( + "Boolean" + if suffix in booleans + else "Float64" if suffix in floats else "Int64" + ) + ) + definitions.append( + TrainingFeatureDefinition( + name, + dtype, + None, + f"Point-in-time ARCH/GARCH scalar: {name}.", + "volatility", + "row", + True, + ) + ) + return definitions + + +def _classical_model_comparison_definition_rows() -> ( + list[TrainingFeatureDefinition] +): + strings = { + "schema_version", + "id", + "dataset_id", + "regularization_contract_id", + "fold_set_id", + "model_id", + } + booleans = { + "eligible", + "diagnostic_available", + "diagnostic_only", + "training_eligible", + "negative", + "available", + } + floats = { + "metric_value", + "reference_metric_value", + "value", + "error_drift", + "skill_drift", + "parameter_drift", + "fit_duration_drift", + "convergence_rate", + "failure_rate", + } + definitions: list[TrainingFeatureDefinition] = [] + for name in CLASSICAL_MODEL_COMPARISON_COLUMNS: + suffix = name + for prefix in ("cm_comparison_", "cm_skill_", "cm_stability_"): + if name.startswith(prefix): + suffix = name.removeprefix(prefix) + break + dtype = ( + "Utf8" + if suffix in strings + else ( + "Boolean" + if suffix in booleans + else "Float64" if suffix in floats else "Int64" + ) + ) + definitions.append( + TrainingFeatureDefinition( + name, + dtype, + None, + ("Retrospective family-neutral comparison scalar: " f"{name}."), + "classical_model_comparison", + "row", + True, + ) + ) + return definitions + + +def _training_context( + target: Any | None, + *, + symbol: str, + data_format: str, + timeframe: str, + period: str, + source: str, +) -> dict[str, str]: + context = { + "symbol": _context_value(target, "symbol") + or _context_value(target, "data_fxpair") + or symbol, + "format": _context_value(target, "data_format") + or _context_value(target, "format") + or data_format + or "ascii", + "timeframe": _context_value(target, "timeframe") + or _context_value(target, "data_timeframe") + or timeframe + or TICK, + "period": _context_value(target, "period") + or _context_value(target, "data_datemonth") + or period, + "source": _context_value(target, "source") or source or "histdata.com", + } + context["symbol"] = context["symbol"].upper() + context["format"] = context["format"].lower() + context["timeframe"] = _normalize_timeframe(context["timeframe"]) + context["source"] = context["source"] or "histdata.com" + return context + + +def _context_value(target: Any | None, name: str) -> str: + if target is None: + return "" + if isinstance(target, Mapping): + return str(target.get(name, "") or "") + return str(getattr(target, name, "") or "") + + +def _normalize_timeframe(value: str) -> str: + normalized = value.strip() + aliases = { + "tick": TICK, + "ticks": TICK, + "tick-data-quotes": TICK, + "tick_data_quotes": TICK, + "t": TICK, + } + return str(aliases.get(normalized.lower(), normalized)) + + +def _require_ascii_tick_context(context: Mapping[str, str]) -> None: + if context["format"] != "ascii" or context["timeframe"] != TICK: + raise ValueError("training features support ASCII tick inputs only") + + +def _require_observed_tick_columns(frame: Any) -> None: + missing = sorted({"datetime", "bid", "ask"} - set(frame.columns)) + if missing: + raise ValueError( + "ASCII tick training enrichment requires columns: " + + ", ".join(missing) + ) + + +def _series_id(context: Mapping[str, str]) -> str: + return ":".join( + ( + context["format"], + context["timeframe"], + context["symbol"], + context["source"].lower(), + ) + ) + + +def _suspicious_gap_threshold(context: Mapping[str, str]) -> int: + value = context.get("suspicious_gap_ms", "") + try: + return int(value) + except (TypeError, ValueError): + return DEFAULT_SUSPICIOUS_TICK_GAP_MS + + +def _invalid_row_expr() -> Any: + import polars as pl + + return ( + pl.col("datetime").is_null() + | pl.col("bid").is_null() + | pl.col("ask").is_null() + | pl.col("bid").le(0) + | pl.col("ask").le(0) + ).fill_null(True) + + +def _with_period_metrics(frame: Any) -> Any: + import polars as pl + + issue_sum_expr = _issue_sum_expr(QUALITY_ISSUE_COLUMNS) + return frame.with_columns( + [ + pl.col("dq_issue_invalid_row") + .cast(pl.Float64) + .mean() + .fill_null(0.0) + .alias("period_invalid_row_rate"), + pl.col("dq_issue_zero_spread") + .cast(pl.Float64) + .mean() + .fill_null(0.0) + .alias("period_zero_spread_rate"), + pl.col("dq_issue_negative_spread") + .cast(pl.Float64) + .mean() + .fill_null(0.0) + .alias("period_negative_spread_rate"), + pl.col("dq_issue_duplicate_timestamp") + .cast(pl.Int64) + .sum() + .alias("period_duplicate_timestamp_count"), + pl.col("dq_issue_suspicious_gap") + .cast(pl.Int64) + .sum() + .alias("period_suspicious_gap_count"), + issue_sum_expr.sum() + .cast(pl.Int64) + .alias("period_quality_issue_count"), + ] + ) + + +def _with_quality_counts(frame: Any) -> Any: + import polars as pl + + hard_count = _issue_sum_expr(HARD_ISSUE_COLUMNS) + warning_count = _issue_sum_expr(WARNING_ISSUE_COLUMNS) + return frame.with_columns( + [ + hard_count.cast(pl.Int32).alias("quality_error_count"), + warning_count.cast(pl.Int32).alias("quality_warning_count"), + ] + ).with_columns( + [ + (pl.col("quality_error_count") + pl.col("quality_warning_count")) + .cast(pl.Int32) + .alias("quality_finding_count"), + pl.when(pl.col("quality_error_count") > 0) + .then(2) + .when(pl.col("quality_warning_count") > 0) + .then(1) + .otherwise(0) + .cast(pl.Int32) + .alias("quality_status_code"), + pl.when(pl.col("quality_error_count") > 0) + .then(2) + .when(pl.col("quality_warning_count") > 0) + .then(1) + .otherwise(0) + .cast(pl.Int32) + .alias("quality_severity_code"), + ] + ) + + +def _with_classification(frame: Any) -> Any: + import polars as pl + + has_error = pl.col("quality_error_count") > 0 + has_warning = pl.col("quality_warning_count") > 0 + return frame.with_columns( + [ + has_error.not_().alias("training_usable"), + pl.when(has_error) + .then(0.0) + .when(has_warning) + .then(0.5) + .otherwise(1.0) + .alias("training_weight"), + pl.when(pl.col("dq_issue_invalid_row")) + .then(1) + .when(pl.col("dq_issue_negative_spread")) + .then(2) + .when(pl.col("dq_issue_non_monotonic_timestamp")) + .then(3) + .when(pl.col("dq_issue_source_unavailable")) + .then(4) + .when(pl.col("dq_issue_topology_unavailable")) + .then(5) + .otherwise(0) + .cast(pl.Int32) + .alias("training_exclusion_reason_code"), + pl.when( + pl.col("dq_issue_source_unavailable") + | pl.col("dq_issue_topology_unavailable") + | pl.col("dq_issue_invalid_row") + ) + .then(3) + .when(has_error) + .then(2) + .when(has_warning) + .then(1) + .otherwise(0) + .cast(pl.Int32) + .alias("class_quality_state_code"), + pl.lit(0).cast(pl.Int32).alias("class_session_state_code"), + pl.when(pl.col("dq_issue_negative_spread")) + .then(2) + .when(pl.col("dq_issue_zero_spread")) + .then(1) + .when(pl.col("dq_issue_wide_spread")) + .then(3) + .when(pl.col("dq_issue_distribution_missing")) + .then(4) + .otherwise(0) + .cast(pl.Int32) + .alias("class_spread_regime_code"), + pl.when(pl.col("dq_issue_non_monotonic_timestamp")) + .then(3) + .when(pl.col("dq_issue_suspicious_gap")) + .then(2) + .when(pl.col("dq_issue_duplicate_timestamp")) + .then(4) + .otherwise(0) + .cast(pl.Int32) + .alias("class_gap_state_code"), + pl.lit(0).cast(pl.Int32).alias("class_volatility_regime_code"), + pl.when(has_error) + .then(4) + .when(has_warning) + .then(1) + .otherwise(0) + .cast(pl.Int32) + .alias("class_training_action_code"), + ] + ) + + +def _with_synthetic_placeholders(frame: Any) -> Any: + import polars as pl + + return frame.with_columns( + [ + pl.lit(None).cast(pl.Float64).alias("synth_bid"), + pl.lit(None).cast(pl.Float64).alias("synth_ask"), + pl.lit(None).cast(pl.Float64).alias("synth_spread"), + pl.lit(None).cast(pl.Float64).alias("synth_mid"), + pl.lit(None).cast(pl.Int32).alias("synth_method_code"), + pl.lit(None).cast(pl.Float64).alias("synth_confidence"), + pl.lit(None).cast(pl.Boolean).alias("synth_usable"), + ] + ) + + +def _issue_sum_expr(columns: tuple[str, ...]) -> Any: + import polars as pl + + return pl.sum_horizontal( + [pl.col(column).cast(pl.Int32) for column in columns] + ) + + +def _ensure_registered_columns(frame: Any) -> Any: + missing = [ + definition + for definition in training_feature_definitions() + if definition.name not in frame.columns + ] + if not missing: + return frame + + return frame.with_columns( + [_default_expr(definition) for definition in missing] + ) + + +def _default_expr(definition: TrainingFeatureDefinition) -> Any: + import polars as pl + + return ( + pl.lit(definition.default) + .cast(_polars_dtype(definition.dtype)) + .alias(definition.name) + ) + + +def _polars_dtype(dtype: str) -> Any: + import polars as pl + + return { + "Boolean": pl.Boolean, + "Float64": pl.Float64, + "Int32": pl.Int32, + "Int64": pl.Int64, + "Utf8": pl.Utf8, + }[dtype] + + +def _quality_target_from_context(target: Any | None) -> QualityTarget: + if isinstance(target, QualityTarget): + return target + kind = QualityTargetKind.UNKNOWN + path = _context_value(target, "path") + if path and Path(path).name == ".data": + kind = QualityTargetKind.CACHE + return QualityTarget( + path=path, + kind=kind, + data_format=_context_value(target, "data_format") + or _context_value(target, "format"), + timeframe=_context_value(target, "timeframe") + or _context_value(target, "data_timeframe"), + symbol=_context_value(target, "symbol") + or _context_value(target, "data_fxpair"), + period=_context_value(target, "period") + or _context_value(target, "data_datemonth"), + ) + + +def _issue_counts(frame: Any) -> dict[str, int]: + import polars as pl + + if getattr(frame, "height", 0) < 1: + return {column: 0 for column in QUALITY_ISSUE_COLUMNS} + + expressions = [ + pl.col(column).cast(pl.Int64).sum().alias(column) + for column in QUALITY_ISSUE_COLUMNS + if column in frame.columns + ] + if not expressions: + return {column: 0 for column in QUALITY_ISSUE_COLUMNS} + + values = frame.select(expressions).to_dicts()[0] + return { + column: int(values.get(column, 0) or 0) + for column in QUALITY_ISSUE_COLUMNS + } + + +def _finding_for_issue_column( + target: QualityTarget, + column: str, + count: int, +) -> QualityFinding: + severity = ( + QualitySeverity.ERROR + if column in HARD_ISSUE_COLUMNS + else QualitySeverity.WARNING + ) + code = ISSUE_CODE_BY_COLUMN[column] + return QualityFinding( + severity=severity, + code=code, + message=f"{count} row(s) flagged {column}.", + rule_id=TRAINING_FEATURE_REPORT_RULE_ID, + target=target, + location=QualityLocation(path=target.path, column=column), + metadata={ + "schema_version": TRAINING_FEATURE_REPORT_SCHEMA_VERSION, + "training_schema_version": TRAINING_SCHEMA_VERSION, + "issue_column": column, + "row_count": count, + }, + ) + + +def _value_counts(frame: Any, column: str) -> dict[str, int]: + if column not in getattr(frame, "columns", ()): + return {} + rows = frame.get_column(column).value_counts().to_dicts() + counts: dict[str, int] = {} + for row in rows: + value = row[column] + count = row["count"] + counts[str(value)] = int(count) + return counts + + +def _json_int_counts(values: Mapping[str, int]) -> dict[str, JSONValue]: + return {str(key): int(value) for key, value in values.items()} diff --git a/src/histdatacom/data_quality/volatility.py b/src/histdatacom/data_quality/volatility.py new file mode 100644 index 00000000..fdf73d67 --- /dev/null +++ b/src/histdatacom/data_quality/volatility.py @@ -0,0 +1,2179 @@ +"""Explicit ARCH/GARCH volatility diagnostics and training projections.""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.metadata +import json +import math +import re +import time +import warnings +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from statistics import median +from typing import Any, cast + +from histdatacom.data_quality.autoregressive import ( + _first_available_row_id, + _int, + _mapping, + _mapping_rows, + _optional_float, + _optional_int, + _rate, + _rounded, + _target_sort_key, + _text, + _warning_codes, +) +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + ClassicalModelInputProfile, + ClassicalModelInputResult, + build_classical_model_input, +) +from histdatacom.data_quality.contracts import QualityFinding +from histdatacom.data_quality.limits import bounded_report_limit +from histdatacom.data_quality.seasonal_exogenous import _model_references +from histdatacom.data_quality.training_features import ( + VOLATILITY_COLUMNS, + ensure_tick_training_features, +) +from histdatacom.runtime_contracts import JSONValue + +VOLATILITY_SCHEMA_VERSION = "histdatacom.volatility.v1" +VOLATILITY_CONFIGURATION_SCHEMA_VERSION = ( + "histdatacom.volatility-configuration.v1" +) +VOLATILITY_FIT_SCHEMA_VERSION = "histdatacom.volatility-fit-result.v1" +VOLATILITY_FORECAST_SCHEMA_VERSION = "histdatacom.volatility-forecast.v1" +VOLATILITY_EVALUATION_SCHEMA_VERSION = "histdatacom.volatility-evaluation.v1" +VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION = ( + "histdatacom.volatility-training-projection.v1" +) +VOLATILITY_SUMMARY_SCHEMA_VERSION = "histdatacom.volatility-summary.v1" +VOLATILITY_SUMMARY_METADATA_KEY = "time_series_fingerprint_volatility_summary" +VOLATILITY_BOUNDED_PAYLOAD_KEY = "fingerprint_volatility" + +DEFAULT_VOLATILITY_SUMMARY_TARGET_LIMIT = 16 +DEFAULT_VOLATILITY_ROUNDING_DIGITS = 12 +MAX_VOLATILITY_SPECIFICATIONS = 32 +MAX_VOLATILITY_ORDER = 64 +MAX_PARAMETER_BOUNDS = 64 + +VOLATILITY_FAMILIES = ("arch", "garch") +VOLATILITY_FAMILY_CODES = {"arch": 1, "garch": 2} +VOLATILITY_INPUT_DEFINITIONS = ( + "raw_return", + "log_return", + "demeaned_return", + "mean_model_residual", +) +VOLATILITY_INPUT_DEFINITION_CODES = { + name: index + for index, name in enumerate(VOLATILITY_INPUT_DEFINITIONS, start=1) +} +VOLATILITY_MEAN_MODELS = ("zero", "constant") +VOLATILITY_MEAN_MODEL_CODES = {"zero": 1, "constant": 2} +VOLATILITY_DISTRIBUTIONS = ("normal", "students_t") +VOLATILITY_DISTRIBUTION_CODES = {"normal": 1, "students_t": 2} +VOLATILITY_INITIALIZATIONS = ("backend_default", "sample_variance", "fixed") +VOLATILITY_REALIZED_VARIANCE_PROXIES = ("squared_return",) +VOLATILITY_FIT_STATUS_CODES = { + "unavailable": 1, + "skipped": 2, + "failed": 3, + "limited": 4, + "fitted": 5, + "converged": 6, +} +VOLATILITY_REASON_CODES = { + "": 0, + "dependency_unavailable": 1, + "input_contract_unavailable": 2, + "insufficient_folds": 3, + "insufficient_history": 4, + "invalid_configuration": 5, + "invalid_transform": 6, + "residual_series_unavailable": 7, + "residual_reference_mismatch": 8, + "non_finite_input": 9, + "zero_variance": 10, + "scaling_failure": 11, + "invalid_order": 12, + "non_positive_variance": 13, + "non_stationary_variance": 14, + "parameter_bound_violation": 15, + "optimizer_failure": 16, + "singular_covariance": 17, + "numerical_instability": 18, + "resource_limit": 19, + "timeout": 20, + "backend_failure": 21, + "target_unavailable": 22, +} + +ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY: tuple[ + Mapping[str, JSONValue], ... +] = ( + { + "family": "gjr_garch", + "status": "registered_not_enabled", + "backend_volatility": "GARCH", + "asymmetric_order_parameter": "o", + }, + { + "family": "egarch", + "status": "registered_not_enabled", + "backend_volatility": "EGARCH", + }, +) + +_SPECIFICATION_ID = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") +_PARAMETER_NAME = re.compile(r"^[A-Za-z0-9_.()\[\]-]{1,96}$") + + +@dataclass(frozen=True, slots=True) +class VolatilitySpecification: + """One explicit symmetric ARCH(q) or GARCH(p,q) configuration.""" + + specification_id: str + family: str + input_definition: str = "raw_return" + mean_model: str = "zero" + mean_model_reference_id: str = "" + distribution: str = "normal" + innovation_order: int = 1 + variance_order: int = 0 + scale_factor: float = 100.0 + variance_initialization: str = "backend_default" + initial_variance: float | None = None + covariance_type: str = "robust" + optimizer_tolerance: float | None = None + parameter_bounds: tuple[tuple[str, float | None, float | None], ...] = () + max_iterations: int = 200 + + def __post_init__(self) -> None: + if not _SPECIFICATION_ID.fullmatch(self.specification_id): + raise ValueError("invalid volatility specification_id") + if self.family not in VOLATILITY_FAMILIES: + raise ValueError("unsupported volatility family") + if self.input_definition not in VOLATILITY_INPUT_DEFINITIONS: + raise ValueError("unsupported volatility input_definition") + if self.mean_model not in VOLATILITY_MEAN_MODELS: + raise ValueError("unsupported volatility mean_model") + if self.input_definition == "mean_model_residual": + if not self.mean_model_reference_id: + raise ValueError("mean-model residuals require a reference ID") + if self.mean_model != "zero": + raise ValueError("residual inputs require a zero mean model") + elif self.mean_model_reference_id: + raise ValueError("mean_model_reference_id requires residual input") + if ( + self.input_definition == "demeaned_return" + and self.mean_model != "zero" + ): + raise ValueError("demeaned inputs require a zero mean model") + if self.distribution not in VOLATILITY_DISTRIBUTIONS: + raise ValueError("unsupported volatility distribution") + if not 1 <= self.innovation_order <= MAX_VOLATILITY_ORDER: + raise ValueError("innovation_order must be between 1 and 64") + if self.family == "arch" and self.variance_order != 0: + raise ValueError("ARCH requires variance_order=0") + if ( + self.family == "garch" + and not 1 <= self.variance_order <= MAX_VOLATILITY_ORDER + ): + raise ValueError("GARCH variance_order must be between 1 and 64") + if not math.isfinite(self.scale_factor) or self.scale_factor <= 0: + raise ValueError("scale_factor must be positive and finite") + if self.variance_initialization not in VOLATILITY_INITIALIZATIONS: + raise ValueError("unsupported variance_initialization") + if self.variance_initialization == "fixed": + if ( + self.initial_variance is None + or not math.isfinite(self.initial_variance) + or self.initial_variance <= 0 + ): + raise ValueError( + "fixed initialization requires positive initial_variance" + ) + elif self.initial_variance is not None: + raise ValueError("initial_variance requires fixed initialization") + if self.covariance_type not in {"robust", "classic"}: + raise ValueError("unsupported covariance_type") + if self.optimizer_tolerance is not None and ( + not math.isfinite(self.optimizer_tolerance) + or self.optimizer_tolerance <= 0 + ): + raise ValueError("optimizer_tolerance must be positive") + if len(self.parameter_bounds) > MAX_PARAMETER_BOUNDS: + raise ValueError("too many parameter bounds") + seen: set[str] = set() + for name, lower, upper in self.parameter_bounds: + if not _PARAMETER_NAME.fullmatch(name) or name in seen: + raise ValueError("parameter bounds require unique valid names") + if lower is not None and not math.isfinite(lower): + raise ValueError("parameter lower bounds must be finite") + if upper is not None and not math.isfinite(upper): + raise ValueError("parameter upper bounds must be finite") + if lower is not None and upper is not None and lower > upper: + raise ValueError("parameter lower bound exceeds upper bound") + seen.add(name) + if self.max_iterations < 1: + raise ValueError("max_iterations must be positive") + + def to_metadata(self) -> dict[str, JSONValue]: + return { + "schema_version": VOLATILITY_CONFIGURATION_SCHEMA_VERSION, + "specification_id": self.specification_id, + "family": self.family, + "family_code": VOLATILITY_FAMILY_CODES[self.family], + "input_definition": self.input_definition, + "input_definition_code": VOLATILITY_INPUT_DEFINITION_CODES[ + self.input_definition + ], + "mean_model": self.mean_model, + "mean_model_code": VOLATILITY_MEAN_MODEL_CODES[self.mean_model], + "mean_model_reference_id": self.mean_model_reference_id or None, + "distribution": self.distribution, + "distribution_code": VOLATILITY_DISTRIBUTION_CODES[ + self.distribution + ], + "innovation_order": self.innovation_order, + "variance_order": self.variance_order, + "scale_factor": self.scale_factor, + "variance_initialization": self.variance_initialization, + "initial_variance": self.initial_variance, + "covariance_type": self.covariance_type, + "optimizer_tolerance": self.optimizer_tolerance, + "parameter_bounds": cast( + JSONValue, + [ + {"parameter": name, "lower": lower, "upper": upper} + for name, lower, upper in self.parameter_bounds + ], + ), + "max_iterations": self.max_iterations, + "symmetric": True, + "asymmetric_order": 0, + "power": 2.0, + "automatic_order_selection": False, + } + + +def _default_specifications() -> tuple[VolatilitySpecification, ...]: + return ( + VolatilitySpecification("arch-5", "arch", innovation_order=5), + VolatilitySpecification( + "garch-1-1", "garch", mean_model="constant", variance_order=1 + ), + ) + + +@dataclass(frozen=True, slots=True) +class VolatilityProfile: + """Explicit volatility controls; disabled by default.""" + + enabled: bool = False + specifications: tuple[VolatilitySpecification, ...] = field( + default_factory=_default_specifications + ) + projection_specification_ids: tuple[str, ...] = ("arch-5", "garch-1-1") + projection_horizon: int = 1 + realized_variance_proxy: str = "squared_return" + annualization_periods: int = 0 + baseline_rolling_windows: tuple[int, ...] = (5, 20) + ewma_decay: float = 0.94 + maximum_persistence: float = 0.999999 + maximum_covariance_condition_number: float = 1e30 + boundary_tolerance: float = 1e-6 + compare_exponential_smoothing: bool = True + compare_autoregressive: bool = True + compare_seasonal_exogenous: bool = True + compare_state_space: bool = True + rounding_digits: int = DEFAULT_VOLATILITY_ROUNDING_DIGITS + + def __post_init__(self) -> None: + if ( + not self.specifications + or len(self.specifications) > MAX_VOLATILITY_SPECIFICATIONS + ): + raise ValueError( + "volatility specifications must contain 1 to 32 items" + ) + identifiers = tuple( + item.specification_id for item in self.specifications + ) + if len(set(identifiers)) != len(identifiers): + raise ValueError("volatility specification IDs must be unique") + if not self.projection_specification_ids or not set( + self.projection_specification_ids + ).issubset(identifiers): + raise ValueError("projection IDs must select configured models") + projected_families = { + item.family + for item in self.specifications + if item.specification_id in self.projection_specification_ids + } + if len(projected_families) != len(self.projection_specification_ids): + raise ValueError( + "only one projection specification per family is supported" + ) + if self.projection_horizon < 1: + raise ValueError("projection_horizon must be positive") + if ( + self.realized_variance_proxy + not in VOLATILITY_REALIZED_VARIANCE_PROXIES + ): + raise ValueError("unsupported realized_variance_proxy") + if self.annualization_periods < 0: + raise ValueError("annualization_periods must be non-negative") + if ( + not self.baseline_rolling_windows + or any(value < 2 for value in self.baseline_rolling_windows) + or tuple(sorted(set(self.baseline_rolling_windows))) + != self.baseline_rolling_windows + ): + raise ValueError( + "baseline windows must be sorted unique values >=2" + ) + if not 0 < self.ewma_decay < 1: + raise ValueError("ewma_decay must be between zero and one") + if not 0 < self.maximum_persistence <= 1: + raise ValueError("maximum_persistence must be in (0,1]") + if ( + not math.isfinite(self.maximum_covariance_condition_number) + or self.maximum_covariance_condition_number <= 1 + ): + raise ValueError( + "maximum_covariance_condition_number must exceed one" + ) + if not 0 < self.boundary_tolerance < 1: + raise ValueError("boundary_tolerance must be in (0,1)") + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between 0 and 16") + + def to_metadata(self) -> dict[str, JSONValue]: + return { + "enabled": self.enabled, + "specifications": cast( + JSONValue, [item.to_metadata() for item in self.specifications] + ), + "projection_specification_ids": list( + self.projection_specification_ids + ), + "projection_horizon": self.projection_horizon, + "realized_variance_proxy": self.realized_variance_proxy, + "annualization_periods": self.annualization_periods, + "baseline_rolling_windows": list(self.baseline_rolling_windows), + "ewma_decay": self.ewma_decay, + "maximum_persistence": self.maximum_persistence, + "maximum_covariance_condition_number": ( + self.maximum_covariance_condition_number + ), + "boundary_tolerance": self.boundary_tolerance, + "compare_exponential_smoothing": self.compare_exponential_smoothing, + "compare_autoregressive": self.compare_autoregressive, + "compare_seasonal_exogenous": self.compare_seasonal_exogenous, + "compare_state_space": self.compare_state_space, + "rounding_digits": self.rounding_digits, + "automatic_order_selection": False, + "automatic_winner": False, + "asymmetric_extensions": cast( + JSONValue, list(ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY) + ), + } + + +@dataclass(frozen=True, slots=True) +class VolatilityResult: + diagnostics: Mapping[str, JSONValue] + annotations: tuple[Mapping[str, Any], ...] + input_result: ClassicalModelInputResult + + +@dataclass(frozen=True, slots=True) +class _Backend: + version: str + arch_model: Any + + +@dataclass(frozen=True, slots=True) +class _FitOutcome: + status: str + reason: str + mean_forecasts: tuple[float, ...] + variance_forecasts: tuple[float, ...] + parameters: Mapping[str, float] + warning_codes: tuple[str, ...] + converged: bool + effective_observation_count: int + missing_reset_count: int + persistence: float | None + unconditional_variance: float | None + covariance_condition_number: float | None + boundary_parameter: bool + standardized_residual_summary: Mapping[str, JSONValue] + log_likelihood: float | None + aic: float | None + bic: float | None + + +def volatility_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: VolatilityProfile | None = None, + residual_series: Sequence[float | None] | None = None, + residual_series_reference_id: str = "", + exponential_smoothing: Mapping[str, JSONValue] | None = None, + autoregressive: Mapping[str, JSONValue] | None = None, + seasonal_exogenous: Mapping[str, JSONValue] | None = None, + state_space: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> VolatilityResult: + selected_input = input_profile or ClassicalModelInputProfile(enabled=True) + input_result = build_classical_model_input( + frame, fingerprint, profile=selected_input, target=target + ) + return volatility_from_model_input( + frame, + input_result, + fingerprint, + input_profile=selected_input, + profile=profile, + residual_series=residual_series, + residual_series_reference_id=residual_series_reference_id, + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + seasonal_exogenous=seasonal_exogenous, + state_space=state_space, + target=target, + ) + + +def volatility_from_model_input( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile, + profile: VolatilityProfile | None = None, + residual_series: Sequence[float | None] | None = None, + residual_series_reference_id: str = "", + exponential_smoothing: Mapping[str, JSONValue] | None = None, + autoregressive: Mapping[str, JSONValue] | None = None, + seasonal_exogenous: Mapping[str, JSONValue] | None = None, + state_space: Mapping[str, JSONValue] | None = None, + target: Any | None = None, +) -> VolatilityResult: + selected = profile or VolatilityProfile(enabled=True) + base = _base_payload(input_result, fingerprint, selected) + if selected.projection_horizon not in input_profile.horizons: + return _unavailable_result(input_result, base, "invalid_configuration") + if input_result.contract.get("status") == "unavailable": + return _unavailable_result( + input_result, base, "input_contract_unavailable" + ) + if not input_result.folds: + return _unavailable_result( + input_result, base, "insufficient_folds", status="limited" + ) + backend = _load_backend() + if backend is None: + return _unavailable_result(input_result, base, "dependency_unavailable") + return _evaluate_models( + frame, + input_result, + fingerprint, + input_profile, + selected, + backend, + residual_series=residual_series, + residual_series_reference_id=residual_series_reference_id, + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + seasonal_exogenous=seasonal_exogenous, + state_space=state_space, + target=target, + ) + + +def volatility_diagnostics_from_training_frame( + frame: Any | None, + fingerprint: Mapping[str, JSONValue], + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: VolatilityProfile | None = None, + target: Any | None = None, +) -> dict[str, JSONValue]: + return dict( + volatility_from_training_frame( + frame, + fingerprint, + input_profile=input_profile, + profile=profile, + target=target, + ).diagnostics + ) + + +def project_volatility_onto_training_frame( + frame: Any, result: VolatilityResult, *, target: Any | None = None +) -> Any: + import polars as pl + + columns = set(getattr(frame, "columns", ())) + enriched = ( + frame + if {"series_id", "period", "row_id"}.issubset(columns) + else ensure_tick_training_features(frame, target=target) + ) + left = enriched.drop( + [name for name in VOLATILITY_COLUMNS if name in enriched.columns] + ).with_row_index("__cm_volatility_original_order") + if result.annotations: + right = pl.DataFrame( + [dict(row) for row in result.annotations], infer_schema_length=None + ) + projected = left.join( + right, + on=["series_id", "period", "row_id"], + how="left", + validate="m:1", + ) + else: + projected = left + projected = _ensure_projection_columns(projected) + return projected.sort("__cm_volatility_original_order").drop( + "__cm_volatility_original_order" + ) + + +def volatility_summary( + findings: Iterable[QualityFinding], + *, + target_limit: int | None = DEFAULT_VOLATILITY_SUMMARY_TARGET_LIMIT, +) -> dict[str, JSONValue] | None: + targets: list[dict[str, JSONValue]] = [] + statuses: Counter[str] = Counter() + for finding in findings: + fingerprint = _mapping(finding.metadata.get("time_series_fingerprint")) + payload = _mapping(fingerprint.get("volatility")) + if not payload: + continue + status = _text(payload.get("status")) or "unavailable" + evaluation = _mapping(payload.get("evaluation")) + fit_summary = _mapping(payload.get("fit_summary")) + statuses[status] += 1 + targets.append( + { + "target_axis": dict(_mapping(payload.get("target_axis"))), + "status": status, + "reason": payload.get("reason"), + "model_count": _int(evaluation.get("model_count")), + "fit_attempt_count": _int(fit_summary.get("fit_attempt_count")), + "evaluated_fold_count": _int( + evaluation.get("evaluated_fold_count") + ), + "failed_fit_count": _int(fit_summary.get("failed_fit_count")), + } + ) + if not targets: + return None + targets.sort(key=_target_sort_key) + limit = bounded_report_limit( + target_limit, + default_limit=DEFAULT_VOLATILITY_SUMMARY_TARGET_LIMIT, + allow_unbounded=True, + ) + included = limit.slice(targets) + omitted = len(targets) - len(included) + return { + "schema_version": VOLATILITY_SUMMARY_SCHEMA_VERSION, + "advisory": True, + "target_count": len(targets), + "included_target_count": len(included), + "omitted_target_count": omitted, + "truncated": omitted > 0, + "status_counts": dict(sorted(statuses.items())), + "target_summaries": cast(JSONValue, included), + "limit_metadata": {"targets": limit.limit_payload()}, + } + + +def format_volatility_summary_lines( + summary: Mapping[str, JSONValue] | None, +) -> tuple[str, ...]: + if not summary: + return () + statuses = _mapping(summary.get("status_counts")) + lines = [ + "", + "ARCH and GARCH volatility models", + f"targets: {_int(summary.get('target_count'))} ready: {_int(statuses.get('ready'))} limited: {_int(statuses.get('limited'))} unavailable: {_int(statuses.get('unavailable'))}", + ] + for item in _mapping_rows(summary.get("target_summaries")): + axis = _mapping(item.get("target_axis")) + label = "/".join( + _text(axis.get(key)) + for key in ("data_format", "timeframe", "symbol", "period") + ) + lines.append( + f"- {label}: {_text(item.get('status'))} models={_int(item.get('model_count'))} fits={_int(item.get('fit_attempt_count'))} folds={_int(item.get('evaluated_fold_count'))}" + ) + if summary.get("truncated") is True: + lines.append( + f"- {_int(summary.get('omitted_target_count'))} targets omitted" + ) + return tuple(lines) + + +def _evaluate_models( + frame: Any | None, + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + input_profile: ClassicalModelInputProfile, + profile: VolatilityProfile, + backend: _Backend, + *, + residual_series: Sequence[float | None] | None, + residual_series_reference_id: str, + exponential_smoothing: Mapping[str, JSONValue] | None, + autoregressive: Mapping[str, JSONValue] | None, + seasonal_exogenous: Mapping[str, JSONValue] | None, + state_space: Mapping[str, JSONValue] | None, + target: Any | None, +) -> VolatilityResult: + rows = cast(list[dict[str, Any]], input_result.regularized_frame.to_dicts()) + folds = [dict(fold) for fold in input_result.folds] + origins = _folds_by_origin(folds) + resources = input_profile.resources + specifications = profile.specifications[: resources.max_candidate_orders] + limitations: list[str] = [] + estimated_memory = len(rows) * max(1, len(specifications)) * 8 * 10 + if ( + len(specifications) < len(profile.specifications) + or estimated_memory > resources.max_memory_bytes + ): + limitations.append("resource_limit") + if estimated_memory > resources.max_memory_bytes: + specifications = () + if residual_series is not None and len(residual_series) != len(rows): + return _unavailable_result( + input_result, + _base_payload(input_result, fingerprint, profile), + "invalid_configuration", + ) + started = time.monotonic() + fit_attempt_count = 0 + all_evaluations: list[dict[str, Any]] = [] + all_fit_samples: list[dict[str, JSONValue]] = [] + model_payloads: list[dict[str, JSONValue]] = [] + fit_statuses: Counter[str] = Counter() + fit_reasons: Counter[str] = Counter() + warning_counts: Counter[str] = Counter() + for specification_code, specification in enumerate(specifications, start=1): + model_id = _model_id( + specification, + _text(input_result.contract.get("derivation_id")), + backend.version, + ) + model_evaluations: list[dict[str, Any]] = [] + model_fit_samples: list[dict[str, JSONValue]] = [] + configuration_reason = _configuration_reason( + specification, + input_profile, + residual_series, + residual_series_reference_id, + ) + for _, origin_folds in origins: + if fit_attempt_count >= resources.max_fit_attempts: + limitations.append("resource_limit") + break + if time.monotonic() - started >= resources.max_wall_time_seconds: + limitations.append("timeout") + break + fit_attempt_count += 1 + origin = origin_folds[0] + start = _int(origin.get("training_start_index")) + end = _int(origin.get("training_end_index")) + source = ( + residual_series + if specification.input_definition == "mean_model_residual" + else tuple( + _optional_float(row.get("cm_input_value")) for row in rows + ) + ) + values = ( + tuple(source[index] for index in range(start, end + 1)) + if source is not None + else () + ) + if specification.input_definition == "demeaned_return": + finite = [float(value) for value in values if value is not None] + center = sum(finite) / len(finite) if finite else 0.0 + values = tuple( + None if value is None else float(value) - center + for value in values + ) + max_horizon = max( + _int(fold.get("horizon")) for fold in origin_folds + ) + outcome = ( + _empty_fit("skipped", configuration_reason) + if configuration_reason + else _fit_specification( + specification, values, max_horizon, profile, backend + ) + ) + fit_statuses[outcome.status] += 1 + if outcome.reason: + fit_reasons[outcome.reason] += 1 + warning_counts.update(outcome.warning_codes) + fit_sample = _fit_sample( + outcome, + specification, + model_id, + origin, + profile.rounding_digits, + ) + model_fit_samples.append(fit_sample) + all_fit_samples.append(fit_sample) + for fold in origin_folds: + evaluation = _fold_evaluation( + rows, + residual_series, + fold, + specification, + specification_code, + model_id, + outcome, + profile, + ) + model_evaluations.append(evaluation) + all_evaluations.append(evaluation) + model_payloads.append( + _model_evaluation_payload( + specification, + specification_code, + model_id, + model_evaluations, + model_fit_samples, + resources.max_retained_diagnostics, + profile.rounding_digits, + ) + ) + if limitations and limitations[-1] in {"resource_limit", "timeout"}: + break + baselines = _variance_baselines(rows, folds, profile) + references = { + "exponential_smoothing": _model_references( + exponential_smoothing, + enabled=profile.compare_exponential_smoothing, + unavailable_reason="exponential_smoothing_not_enabled", + ), + "autoregressive": _model_references( + autoregressive, + enabled=profile.compare_autoregressive, + unavailable_reason="autoregressive_not_enabled", + ), + "seasonal_exogenous": _model_references( + seasonal_exogenous, + enabled=profile.compare_seasonal_exogenous, + unavailable_reason="seasonal_exogenous_not_enabled", + ), + "state_space": _model_references( + state_space, + enabled=profile.compare_state_space, + unavailable_reason="state_space_not_enabled", + ), + } + annotations, collisions = _build_annotations( + frame, all_evaluations, profile, input_result, target=target + ) + evaluated_count = sum( + item.get("status") == "evaluated" for item in all_evaluations + ) + failed_count = sum( + status in {"failed", "unavailable"} + for status in fit_statuses.elements() + ) + limited_count = fit_statuses["limited"] + fit_statuses["skipped"] + if not evaluated_count: + limitations.append("insufficient_history") + limitations = list(dict.fromkeys(limitations)) + status = ( + "ready" + if not limitations and not failed_count and not limited_count + else "limited" + ) + reason = ( + limitations[0] + if limitations + else sorted(fit_reasons)[0] if fit_reasons else None + ) + diagnostics: dict[str, JSONValue] = { + **_base_payload(input_result, fingerprint, profile), + "status": status, + "reason": reason, + "limitations": cast(JSONValue, limitations), + "backend": { + "provider": "arch", + "version": backend.version, + "available": True, + "model_factory": "arch.arch_model", + "import_basis": "optional_models_extra", + }, + "fit_summary": { + "schema_version": VOLATILITY_FIT_SCHEMA_VERSION, + "fit_attempt_count": fit_attempt_count, + "status_counts": dict(sorted(fit_statuses.items())), + "reason_counts": dict(sorted(fit_reasons.items())), + "warning_counts": dict(sorted(warning_counts.items())), + "failed_fit_count": failed_count, + "limited_fit_count": limited_count, + "convergence_rate": _rate( + fit_statuses["converged"], + fit_attempt_count, + profile.rounding_digits, + ), + "failure_rate": _rate( + failed_count, fit_attempt_count, profile.rounding_digits + ), + "fit_samples": cast( + JSONValue, all_fit_samples[: resources.max_retained_diagnostics] + ), + "fit_samples_truncated": len(all_fit_samples) + > resources.max_retained_diagnostics, + }, + "resource_usage": { + "limits": resources.to_metadata(), + "estimated_working_memory_bytes": estimated_memory, + "memory_limit_exceeded": estimated_memory + > resources.max_memory_bytes, + "fit_attempt_count": fit_attempt_count, + "wall_time_limit_enforced": True, + "wall_time_observed_in_payload": False, + }, + "evaluation": { + "schema_version": VOLATILITY_EVALUATION_SCHEMA_VERSION, + "calculation_basis": "regular_grid_rolling_origin_conditional_variance", + "variance_scale": "unscaled_return_squared", + "mean_scale": "unscaled_return", + "realized_variance_proxy": profile.realized_variance_proxy, + "model_count": len(model_payloads), + "fold_count": len(folds), + "evaluated_fold_count": evaluated_count, + "skipped_evaluation_count": len(all_evaluations) - evaluated_count, + "forecast_coverage_rate": _rate( + evaluated_count, len(all_evaluations), profile.rounding_digits + ), + "models": cast(JSONValue, model_payloads), + "reference_variance_baselines": cast(JSONValue, baselines), + "baseline_relative_skill": cast( + JSONValue, + _baseline_relative_skill( + model_payloads, baselines, profile.rounding_digits + ), + ), + "preceding_mean_model_references": cast(JSONValue, references), + "comparison_semantics": "descriptive_shared_folds_separate_mean_and_variance_metrics", + "automatic_winner": False, + }, + "training_projection": _training_projection_metadata( + profile, input_result, len(annotations), collisions + ), + "fit_duration_included": False, + } + return VolatilityResult(diagnostics, annotations, input_result) + + +def _configuration_reason( + specification: VolatilitySpecification, + input_profile: ClassicalModelInputProfile, + residual_series: Sequence[float | None] | None, + residual_reference: str, +) -> str: + expected = ( + "return" + if specification.input_definition in {"raw_return", "demeaned_return"} + else ( + "log_return" + if specification.input_definition == "log_return" + else input_profile.transform + ) + ) + if ( + specification.input_definition != "mean_model_residual" + and input_profile.transform != expected + ): + return "invalid_transform" + if specification.input_definition == "mean_model_residual": + if residual_series is None: + return "residual_series_unavailable" + if residual_reference != specification.mean_model_reference_id: + return "residual_reference_mismatch" + if ( + specification.innovation_order + specification.variance_order + > input_profile.resources.max_candidate_orders + ): + return "invalid_order" + return "" + + +def _fit_specification( + specification: VolatilitySpecification, + values: Sequence[float | None], + horizon: int, + profile: VolatilityProfile, + backend: _Backend, +) -> _FitOutcome: + trailing, resets = _trailing_contiguous(values) + minimum = max( + 12, + 4 * (specification.innovation_order + specification.variance_order + 1), + ) + if len(trailing) < minimum: + return _empty_fit( + "skipped", + "insufficient_history", + observed_count=len(trailing), + missing_reset_count=resets, + ) + if any(not math.isfinite(value) for value in trailing): + return _empty_fit( + "failed", + "non_finite_input", + observed_count=len(trailing), + missing_reset_count=resets, + ) + center = sum(trailing) / len(trailing) + sample_variance = sum((value - center) ** 2 for value in trailing) / len( + trailing + ) + if not math.isfinite(sample_variance) or sample_variance <= 1e-24: + return _empty_fit( + "skipped", + "zero_variance", + observed_count=len(trailing), + missing_reset_count=resets, + ) + scaled = [value * specification.scale_factor for value in trailing] + if any(not math.isfinite(value) for value in scaled): + return _empty_fit( + "failed", + "scaling_failure", + observed_count=len(trailing), + missing_reset_count=resets, + ) + backcast = None + if specification.variance_initialization == "sample_variance": + backcast = sample_variance * specification.scale_factor**2 + elif specification.variance_initialization == "fixed": + backcast = ( + cast(float, specification.initial_variance) + * specification.scale_factor**2 + ) + try: + numpy = importlib.import_module("numpy") + model = backend.arch_model( + numpy.asarray(scaled, dtype=float), + mean="Zero" if specification.mean_model == "zero" else "Constant", + vol="ARCH" if specification.family == "arch" else "GARCH", + p=specification.innovation_order, + o=0, + q=specification.variance_order, + power=2.0, + dist="normal" if specification.distribution == "normal" else "t", + rescale=False, + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = model.fit( + update_freq=0, + disp="off", + show_warning=False, + cov_type=specification.covariance_type, + tol=specification.optimizer_tolerance, + options={"maxiter": specification.max_iterations}, + backcast=backcast, + ) + forecast = result.forecast( + horizon=horizon, method="analytic", reindex=False + ) + scale2 = specification.scale_factor**2 + means = tuple( + float(value) / specification.scale_factor + for value in forecast.mean.values[-1] + ) + variances = tuple( + float(value) / scale2 for value in forecast.variance.values[-1] + ) + if ( + not means + or not variances + or any(not math.isfinite(value) for value in (*means, *variances)) + ): + return _empty_fit( + "failed", + "numerical_instability", + observed_count=len(trailing), + missing_reset_count=resets, + ) + if any(value <= 0 for value in variances): + return _empty_fit( + "failed", + "non_positive_variance", + observed_count=len(trailing), + missing_reset_count=resets, + ) + parameters = { + str(name): float(value) + for name, value in result.params.items() + if math.isfinite(float(value)) + } + covariance = numpy.asarray(result.param_cov, dtype=float) + if covariance.size and not numpy.isfinite(covariance).all(): + return _empty_fit( + "failed", + "singular_covariance", + observed_count=len(trailing), + missing_reset_count=resets, + ) + covariance_condition = ( + float(numpy.linalg.cond(covariance)) if covariance.size else None + ) + if covariance_condition is not None and ( + not math.isfinite(covariance_condition) + or covariance_condition + > profile.maximum_covariance_condition_number + ): + return _empty_fit( + "failed", + "singular_covariance", + observed_count=len(trailing), + missing_reset_count=resets, + ) + persistence = sum( + value + for name, value in parameters.items() + if name.startswith("alpha[") or name.startswith("beta[") + ) + omega = parameters.get("omega") + unconditional = ( + omega / scale2 / (1.0 - persistence) + if omega is not None and 0 <= persistence < 1 + else None + ) + boundary = _boundary_parameter( + parameters, specification, profile.boundary_tolerance + ) + reason = "" + status = "converged" + converged = int(getattr(result, "convergence_flag", 1)) == 0 and bool( + getattr(result.optimization_result, "success", False) + ) + if not converged: + status, reason = "limited", "optimizer_failure" + elif persistence >= profile.maximum_persistence: + status, reason = "limited", "non_stationary_variance" + elif _violates_parameter_bounds( + parameters, specification.parameter_bounds + ): + status, reason = "limited", "parameter_bound_violation" + residuals = [ + float(value) + for value in result.std_resid + if math.isfinite(float(value)) + ] + return _FitOutcome( + status, + reason, + means, + variances, + parameters, + _warning_codes(caught), + converged, + int(getattr(result, "nobs", len(trailing))), + resets, + persistence, + unconditional, + covariance_condition, + boundary, + _numeric_summary(residuals, profile.rounding_digits), + _finite_float(getattr(result, "loglikelihood", None)), + _finite_float(getattr(result, "aic", None)), + _finite_float(getattr(result, "bic", None)), + ) + except ( + ArithmeticError, + ImportError, + RuntimeError, + TypeError, + ValueError, + ZeroDivisionError, + ): + return _empty_fit( + "failed", + "backend_failure", + observed_count=len(trailing), + missing_reset_count=resets, + ) + + +def _fold_evaluation( + rows: Sequence[Mapping[str, Any]], + residual_series: Sequence[float | None] | None, + fold: Mapping[str, Any], + specification: VolatilitySpecification, + specification_code: int, + model_id: str, + outcome: _FitOutcome, + profile: VolatilityProfile, +) -> dict[str, Any]: + horizon = _int(fold.get("horizon")) + target_index = _int(fold.get("target_index")) + actual = None + if 0 <= target_index < len(rows): + actual = ( + _optional_float(residual_series[target_index]) + if specification.input_definition == "mean_model_residual" + and residual_series is not None + else _optional_float(rows[target_index].get("cm_input_value")) + ) + mean_forecast = ( + outcome.mean_forecasts[horizon - 1] + if 0 < horizon <= len(outcome.mean_forecasts) + else None + ) + variance_forecast = ( + outcome.variance_forecasts[horizon - 1] + if 0 < horizon <= len(outcome.variance_forecasts) + else None + ) + if outcome.status in {"failed", "skipped", "unavailable"}: + status, reason = "not_evaluated", outcome.reason + elif fold.get("status") != "valid" or actual is None: + status, reason = "skipped", "target_unavailable" + elif variance_forecast is None or mean_forecast is None: + status, reason = "skipped", "numerical_instability" + else: + status, reason = "evaluated", "" + realized = ( + actual**2 if status == "evaluated" and actual is not None else None + ) + volatility = ( + math.sqrt(variance_forecast) + if variance_forecast is not None and variance_forecast > 0 + else None + ) + annualized_variance = ( + variance_forecast * profile.annualization_periods + if variance_forecast is not None and profile.annualization_periods + else None + ) + annualized_volatility = ( + math.sqrt(annualized_variance) + if annualized_variance is not None + else None + ) + return { + "schema_version": VOLATILITY_FORECAST_SCHEMA_VERSION, + "status": status, + "reason": reason or None, + "series_id": _text(fold.get("series_id")), + "period": _text(fold.get("period")), + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "family_code": VOLATILITY_FAMILY_CODES[specification.family], + "input_definition": specification.input_definition, + "input_definition_code": VOLATILITY_INPUT_DEFINITION_CODES[ + specification.input_definition + ], + "mean_model": specification.mean_model, + "mean_model_code": VOLATILITY_MEAN_MODEL_CODES[ + specification.mean_model + ], + "distribution": specification.distribution, + "distribution_code": VOLATILITY_DISTRIBUTION_CODES[ + specification.distribution + ], + "innovation_order": specification.innovation_order, + "variance_order": specification.variance_order, + "scale_factor": specification.scale_factor, + "fit_status": outcome.status, + "fit_reason": outcome.reason or None, + "converged": outcome.converged, + "effective_observation_count": outcome.effective_observation_count, + "missing_reset_count": outcome.missing_reset_count, + "persistence": _rounded(outcome.persistence, profile.rounding_digits), + "unconditional_variance": _rounded( + outcome.unconditional_variance, profile.rounding_digits + ), + "boundary_parameter": outcome.boundary_parameter, + "fold_id": _int(fold.get("fold_id")), + "origin_row_id": fold.get("origin_row_id"), + "target_row_id": fold.get("target_row_id"), + "origin_bin_end_utc_ms": _int(fold.get("origin_bin_end_utc_ms")), + "target_bin_end_utc_ms": _int(fold.get("target_bin_end_utc_ms")), + "horizon": horizon, + "mean_forecast": _rounded(mean_forecast, profile.rounding_digits), + "variance_forecast": _rounded( + variance_forecast, profile.rounding_digits + ), + "volatility_forecast": _rounded(volatility, profile.rounding_digits), + "annualized_variance_forecast": _rounded( + annualized_variance, profile.rounding_digits + ), + "annualized_volatility_forecast": _rounded( + annualized_volatility, profile.rounding_digits + ), + "actual_return": _rounded(actual, profile.rounding_digits), + "realized_variance_proxy": _rounded(realized, profile.rounding_digits), + "mean_error": _rounded( + ( + mean_forecast - actual + if status == "evaluated" + and mean_forecast is not None + and actual is not None + else None + ), + profile.rounding_digits, + ), + "variance_error": _rounded( + ( + variance_forecast - realized + if status == "evaluated" + and variance_forecast is not None + and realized is not None + else None + ), + profile.rounding_digits, + ), + "volatility_error": _rounded( + ( + volatility - abs(actual) + if status == "evaluated" + and volatility is not None + and actual is not None + else None + ), + profile.rounding_digits, + ), + "qlike_loss": _rounded( + ( + math.log(variance_forecast) + realized / variance_forecast + if status == "evaluated" + and variance_forecast is not None + and variance_forecast > 0 + and realized is not None + else None + ), + profile.rounding_digits, + ), + } + + +def _fit_sample( + outcome: _FitOutcome, + specification: VolatilitySpecification, + model_id: str, + origin: Mapping[str, Any], + digits: int, +) -> dict[str, JSONValue]: + return { + "schema_version": VOLATILITY_FIT_SCHEMA_VERSION, + "model_id": model_id, + "specification_id": specification.specification_id, + "family": specification.family, + "fold_id": _int(origin.get("fold_id")), + "origin_row_id": origin.get("origin_row_id"), + "status": outcome.status, + "reason": outcome.reason or None, + "converged": outcome.converged, + "effective_observation_count": outcome.effective_observation_count, + "missing_reset_count": outcome.missing_reset_count, + "parameters": cast( + JSONValue, + { + name: _rounded(value, digits) + for name, value in sorted(outcome.parameters.items()) + }, + ), + "persistence": _rounded(outcome.persistence, digits), + "unconditional_variance": _rounded( + outcome.unconditional_variance, digits + ), + "covariance_condition_number": _rounded( + outcome.covariance_condition_number, digits + ), + "boundary_parameter": outcome.boundary_parameter, + "standardized_residual_summary": dict( + outcome.standardized_residual_summary + ), + "log_likelihood": _rounded(outcome.log_likelihood, digits), + "aic": _rounded(outcome.aic, digits), + "bic": _rounded(outcome.bic, digits), + "warning_codes": list(outcome.warning_codes), + "backend_exception_text_included": False, + } + + +def _model_evaluation_payload( + specification: VolatilitySpecification, + specification_code: int, + model_id: str, + evaluations: Sequence[Mapping[str, Any]], + fit_samples: Sequence[Mapping[str, JSONValue]], + diagnostic_limit: int, + digits: int, +) -> dict[str, JSONValue]: + evaluated = [row for row in evaluations if row.get("status") == "evaluated"] + horizons: dict[str, JSONValue] = {} + for horizon in sorted({_int(row.get("horizon")) for row in evaluated}): + rows = [row for row in evaluated if _int(row.get("horizon")) == horizon] + horizons[str(horizon)] = _metric_summary(rows, digits) + return { + "schema_version": VOLATILITY_EVALUATION_SCHEMA_VERSION, + "model_id": model_id, + "specification_id": specification.specification_id, + "specification_code": specification_code, + "family": specification.family, + "configuration": specification.to_metadata(), + "status": "evaluated" if evaluated else "not_evaluated", + "evaluated_count": len(evaluated), + "horizon_metrics": horizons, + "rolling_window_stability": _rolling_stability(evaluated, digits), + "parameter_stability": _parameter_stability(fit_samples, digits), + "fit_samples": cast(JSONValue, list(fit_samples[:diagnostic_limit])), + "fit_samples_truncated": len(fit_samples) > diagnostic_limit, + "automatic_winner": False, + } + + +def _metric_summary( + rows: Sequence[Mapping[str, Any]], digits: int +) -> dict[str, JSONValue]: + def values(name: str) -> list[float]: + return [ + value + for row in rows + if (value := _optional_float(row.get(name))) is not None + ] + + mean_errors, variance_errors, volatility_errors, qlike = ( + values("mean_error"), + values("variance_error"), + values("volatility_error"), + values("qlike_loss"), + ) + return { + "count": len(rows), + "mean_metrics": _errors(mean_errors, digits), + "variance_metrics": { + **_errors(variance_errors, digits), + "mean_qlike": _rounded( + sum(qlike) / len(qlike) if qlike else None, digits + ), + }, + "volatility_metrics": _errors(volatility_errors, digits), + "metrics_are_not_interchangeable": True, + } + + +def _errors(errors: Sequence[float], digits: int) -> dict[str, JSONValue]: + absolute = [abs(value) for value in errors] + return { + "count": len(errors), + "mae": _rounded( + sum(absolute) / len(absolute) if absolute else None, digits + ), + "rmse": _rounded( + ( + math.sqrt(sum(value * value for value in errors) / len(errors)) + if errors + else None + ), + digits, + ), + "bias": _rounded(sum(errors) / len(errors) if errors else None, digits), + } + + +def _rolling_stability( + rows: Sequence[Mapping[str, Any]], digits: int +) -> dict[str, JSONValue]: + width = max(1, math.ceil(len(rows) / 3)) + chunks = [ + rows[index : index + width] for index in range(0, len(rows), width) + ] + return { + "segments": cast( + JSONValue, + [_metric_summary(chunk, digits) for chunk in chunks if chunk], + ), + "automatic_winner": False, + } + + +def _parameter_stability( + samples: Sequence[Mapping[str, JSONValue]], digits: int +) -> dict[str, JSONValue]: + grouped: dict[str, list[float]] = {} + for sample in samples: + for name, raw in _mapping(sample.get("parameters")).items(): + value = _optional_float(raw) + if value is not None: + grouped.setdefault(name, []).append(value) + return { + "parameters": cast( + JSONValue, + { + name: _numeric_summary(values, digits) + for name, values in sorted(grouped.items()) + }, + ), + "bounded": True, + } + + +def _variance_baselines( + rows: Sequence[Mapping[str, Any]], + folds: Sequence[Mapping[str, Any]], + profile: VolatilityProfile, +) -> list[dict[str, JSONValue]]: + predictions: dict[str, list[dict[str, Any]]] = { + f"rolling_variance_{window}": [] + for window in profile.baseline_rolling_windows + } + predictions[f"ewma_variance_{profile.ewma_decay}"] = [] + for fold in folds: + if fold.get("status") != "valid": + continue + start, end, target = ( + _int(fold.get("training_start_index")), + _int(fold.get("training_end_index")), + _int(fold.get("target_index")), + ) + if not 0 <= target < len(rows): + continue + actual = _optional_float(rows[target].get("cm_input_value")) + training = [ + _optional_float(rows[index].get("cm_input_value")) + for index in range(start, end + 1) + ] + trailing, _ = _trailing_contiguous(training) + if actual is None: + continue + realized = actual**2 + for window in profile.baseline_rolling_windows: + if len(trailing) >= window: + sample = trailing[-window:] + center = sum(sample) / len(sample) + prediction = sum( + (value - center) ** 2 for value in sample + ) / len(sample) + predictions[f"rolling_variance_{window}"].append( + { + "horizon": _int(fold.get("horizon")), + "error": prediction - realized, + "qlike": ( + math.log(prediction) + realized / prediction + if prediction > 0 + else None + ), + } + ) + if trailing: + variance = trailing[0] ** 2 + for value in trailing[1:]: + variance = ( + profile.ewma_decay * variance + + (1 - profile.ewma_decay) * value**2 + ) + predictions[f"ewma_variance_{profile.ewma_decay}"].append( + { + "horizon": _int(fold.get("horizon")), + "error": variance - realized, + "qlike": ( + math.log(variance) + realized / variance + if variance > 0 + else None + ), + } + ) + output: list[dict[str, JSONValue]] = [] + for name, evaluations in predictions.items(): + errors = [cast(float, row["error"]) for row in evaluations] + qlike = [ + cast(float, row["qlike"]) + for row in evaluations + if row["qlike"] is not None + ] + horizon_metrics: dict[str, JSONValue] = {} + for horizon in sorted( + {_int(row.get("horizon")) for row in evaluations} + ): + horizon_rows = [ + row + for row in evaluations + if _int(row.get("horizon")) == horizon + ] + horizon_errors = [cast(float, row["error"]) for row in horizon_rows] + horizon_qlike = [ + cast(float, row["qlike"]) + for row in horizon_rows + if row["qlike"] is not None + ] + horizon_metrics[str(horizon)] = { + **_errors(horizon_errors, profile.rounding_digits), + "mean_qlike": _rounded( + ( + sum(horizon_qlike) / len(horizon_qlike) + if horizon_qlike + else None + ), + profile.rounding_digits, + ), + } + output.append( + { + "name": name, + "realized_variance_proxy": profile.realized_variance_proxy, + "metrics": { + **_errors(errors, profile.rounding_digits), + "mean_qlike": _rounded( + sum(qlike) / len(qlike) if qlike else None, + profile.rounding_digits, + ), + }, + "horizon_metrics": horizon_metrics, + "automatic_winner": False, + } + ) + return output + + +def _baseline_relative_skill( + models: Sequence[Mapping[str, JSONValue]], + baselines: Sequence[Mapping[str, JSONValue]], + digits: int, +) -> list[dict[str, JSONValue]]: + output: list[dict[str, JSONValue]] = [] + for model in models: + model_horizons = _mapping(model.get("horizon_metrics")) + comparisons: list[dict[str, JSONValue]] = [] + for baseline in baselines: + baseline_horizons = _mapping(baseline.get("horizon_metrics")) + horizon_skill: dict[str, JSONValue] = {} + for horizon, raw_model_metrics in sorted(model_horizons.items()): + model_variance = _mapping( + _mapping(raw_model_metrics).get("variance_metrics") + ) + baseline_metrics = _mapping(baseline_horizons.get(horizon)) + model_mae = _optional_float(model_variance.get("mae")) + baseline_mae = _optional_float(baseline_metrics.get("mae")) + model_qlike = _optional_float(model_variance.get("mean_qlike")) + baseline_qlike = _optional_float( + baseline_metrics.get("mean_qlike") + ) + horizon_skill[horizon] = { + "variance_mae_skill": _rounded( + ( + 1.0 - model_mae / baseline_mae + if model_mae is not None + and baseline_mae is not None + and baseline_mae > 0 + else None + ), + digits, + ), + "qlike_difference": _rounded( + ( + model_qlike - baseline_qlike + if model_qlike is not None + and baseline_qlike is not None + else None + ), + digits, + ), + "positive_mae_skill_means_model_improvement": True, + "negative_qlike_difference_means_model_improvement": True, + } + comparisons.append( + { + "baseline": baseline.get("name"), + "horizon_skill": horizon_skill, + } + ) + output.append( + { + "model_id": model.get("model_id"), + "specification_id": model.get("specification_id"), + "comparisons": cast(JSONValue, comparisons), + "descriptive_only": True, + "automatic_winner": False, + } + ) + return output + + +def _build_annotations( + frame: Any | None, + evaluations: Sequence[Mapping[str, Any]], + profile: VolatilityProfile, + input_result: ClassicalModelInputResult, + *, + target: Any | None, +) -> tuple[tuple[Mapping[str, Any], ...], int]: + if frame is None: + return (), 0 + try: + enriched = ensure_tick_training_features(frame, target=target) + except (AttributeError, TypeError, ValueError): + return (), 0 + availability: dict[tuple[str, str], list[tuple[int, int]]] = {} + for row in cast(list[dict[str, Any]], enriched.to_dicts()): + timestamp, row_id = _optional_int( + row.get("timestamp_utc_ms") + ), _optional_int(row.get("row_id")) + if timestamp is not None and row_id is not None: + availability.setdefault( + (_text(row.get("series_id")), _text(row.get("period"))), [] + ).append((timestamp, row_id)) + for values in availability.values(): + values.sort() + selected = [ + row + for row in evaluations + if _text(row.get("specification_id")) + in profile.projection_specification_ids + and _int(row.get("horizon")) == profile.projection_horizon + ] + selected.sort( + key=lambda row: ( + _text(row.get("series_id")), + _text(row.get("period")), + _int(row.get("origin_bin_end_utc_ms")), + _int(row.get("fold_id")), + ) + ) + merged: dict[tuple[str, str, int], dict[str, Any]] = {} + collisions = 0 + for evaluation in selected: + group = ( + _text(evaluation.get("series_id")), + _text(evaluation.get("period")), + ) + for diagnostic, key_name in ( + (False, "origin_bin_end_utc_ms"), + (True, "target_bin_end_utc_ms"), + ): + if ( + diagnostic + and _optional_float(evaluation.get("variance_error")) is None + ): + continue + row_id = _first_available_row_id( + availability.get(group, ()), _int(evaluation.get(key_name)) + ) + if row_id is None: + continue + key = (*group, row_id) + annotation = _annotation_row( + evaluation, input_result, row_id, diagnostic=diagnostic + ) + if key in merged: + collisions += 1 + merged[key] = _merge_annotation_rows(merged[key], annotation) + else: + merged[key] = annotation + return tuple(merged[key] for key in sorted(merged)), collisions + + +def _annotation_row( + evaluation: Mapping[str, Any], + input_result: ClassicalModelInputResult, + row_id: int, + *, + diagnostic: bool, +) -> dict[str, Any]: + family = _text(evaluation.get("family")) + prefix = f"cm_{family}_" + fit_status = _text(evaluation.get("fit_status")) or "unavailable" + forecast_available = ( + not diagnostic + and _optional_float(evaluation.get("variance_forecast")) is not None + ) + origin_time, target_time = _int( + evaluation.get("origin_bin_end_utc_ms") + ), _int(evaluation.get("target_bin_end_utc_ms")) + row: dict[str, Any] = { + "series_id": _text(evaluation.get("series_id")), + "period": _text(evaluation.get("period")), + "row_id": row_id, + } + values: dict[str, Any] = { + "schema_version": VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION, + "input_derivation_id": _text( + input_result.contract.get("derivation_id") + ), + "model_id": _text(evaluation.get("model_id")), + "specification_code": _int(evaluation.get("specification_code")), + "input_definition_code": _int(evaluation.get("input_definition_code")), + "mean_model_code": _int(evaluation.get("mean_model_code")), + "distribution_code": _int(evaluation.get("distribution_code")), + "innovation_order": _int(evaluation.get("innovation_order")), + "variance_order": _int(evaluation.get("variance_order")), + "scale_factor": _optional_float(evaluation.get("scale_factor")), + "fit_status_code": VOLATILITY_FIT_STATUS_CODES.get(fit_status, 1), + "failure_reason_code": VOLATILITY_REASON_CODES.get( + _text(evaluation.get("fit_reason")), 0 + ), + "converged": bool(evaluation.get("converged", False)), + "effective_observation_count": _int( + evaluation.get("effective_observation_count") + ), + "missing_reset_count": _int(evaluation.get("missing_reset_count")), + "fold_id": _int(evaluation.get("fold_id")), + "origin_row_id": evaluation.get("origin_row_id"), + "target_row_id": evaluation.get("target_row_id"), + "horizon": _int(evaluation.get("horizon")), + "mean_forecast": ( + None + if diagnostic + else _optional_float(evaluation.get("mean_forecast")) + ), + "variance_forecast": ( + None + if diagnostic + else _optional_float(evaluation.get("variance_forecast")) + ), + "volatility_forecast": ( + None + if diagnostic + else _optional_float(evaluation.get("volatility_forecast")) + ), + "annualized_variance_forecast": ( + None + if diagnostic + else _optional_float(evaluation.get("annualized_variance_forecast")) + ), + "annualized_volatility_forecast": ( + None + if diagnostic + else _optional_float( + evaluation.get("annualized_volatility_forecast") + ) + ), + "forecast_available": forecast_available, + "forecast_available_at_utc_ms": ( + origin_time if forecast_available else None + ), + "actual_return": ( + _optional_float(evaluation.get("actual_return")) + if diagnostic + else None + ), + "realized_variance_proxy": ( + _optional_float(evaluation.get("realized_variance_proxy")) + if diagnostic + else None + ), + "mean_error": ( + _optional_float(evaluation.get("mean_error")) + if diagnostic + else None + ), + "variance_error": ( + _optional_float(evaluation.get("variance_error")) + if diagnostic + else None + ), + "volatility_error": ( + _optional_float(evaluation.get("volatility_error")) + if diagnostic + else None + ), + "qlike_loss": ( + _optional_float(evaluation.get("qlike_loss")) + if diagnostic + else None + ), + "diagnostic_available": diagnostic, + "diagnostic_available_at_utc_ms": target_time if diagnostic else None, + "diagnostic_only": diagnostic, + "persistence": _optional_float(evaluation.get("persistence")), + "unconditional_variance": _optional_float( + evaluation.get("unconditional_variance") + ), + "boundary_parameter": bool(evaluation.get("boundary_parameter", False)), + "training_eligible": forecast_available, + } + row.update({prefix + suffix: value for suffix, value in values.items()}) + return row + + +def _merge_annotation_rows( + current: Mapping[str, Any], incoming: Mapping[str, Any] +) -> dict[str, Any]: + merged = dict(current) + flags = { + name + for name in VOLATILITY_COLUMNS + if name.endswith( + ( + "forecast_available", + "diagnostic_available", + "diagnostic_only", + "training_eligible", + ) + ) + } + for name, value in incoming.items(): + if name in flags: + merged[name] = bool(merged.get(name)) or bool(value) + elif value is not None: + merged[name] = value + return merged + + +def _training_projection_metadata( + profile: VolatilityProfile, + input_result: ClassicalModelInputResult, + annotation_count: int, + collision_count: int, +) -> dict[str, JSONValue]: + return { + "schema_version": VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION, + "grain": "row", + "identity_fields": ["series_id", "period", "row_id"], + "timestamp_is_sole_identity": False, + "mapping_policy": "first_source_row_at_or_after_availability", + "collision_policy": "merge_forecast_and_diagnostic_latest_origin_wins", + "collision_count": collision_count, + "annotation_count": annotation_count, + "projection_specification_ids": list( + profile.projection_specification_ids + ), + "projection_horizon": profile.projection_horizon, + "input_derivation_id": input_result.contract.get("derivation_id"), + "column_names": list(VOLATILITY_COLUMNS), + "column_prefixes": ["cm_arch_", "cm_garch_"], + "forecast_rows_point_in_time_safe": True, + "diagnostic_rows_retrospective": True, + "observed_columns_overwritten": False, + } + + +def _base_payload( + input_result: ClassicalModelInputResult, + fingerprint: Mapping[str, JSONValue], + profile: VolatilityProfile, +) -> dict[str, JSONValue]: + contract = input_result.contract + regularization = _mapping(contract.get("regularization")) + return { + "schema_version": VOLATILITY_SCHEMA_VERSION, + "advisory": True, + "target_axis": dict(_mapping(contract.get("target_axis"))), + "reference_fingerprint_id": contract.get("reference_fingerprint_id") + or fingerprint.get("fingerprint_id"), + "input_schema_version": CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + "input_derivation_id": contract.get("derivation_id"), + "input_status": contract.get("status"), + "calculation_basis": "regular_grid_rolling_origin_conditional_variance", + "configuration": profile.to_metadata(), + "input_transform_policy": dict( + _mapping(contract.get("transform_policy")) + ), + "input_missingness_policy": { + "expected_closure_policy": regularization.get( + "expected_closure_policy" + ), + "unexpected_missing_policy": regularization.get( + "unexpected_missing_policy" + ), + "fit_policy": "trailing_contiguous_observations_after_last_missing_bin", + "missing_values_filled": False, + }, + "variance_contract": { + "conditional_variance": True, + "realized_proxy": profile.realized_variance_proxy, + "variance_scale": "unscaled_input_squared", + "volatility_scale": "unscaled_input", + "mean_metrics_separate": True, + }, + "asymmetric_extension_registry": cast( + JSONValue, list(ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY) + ), + } + + +def _unavailable_result( + input_result: ClassicalModelInputResult, + base: Mapping[str, JSONValue], + reason: str, + *, + status: str = "unavailable", +) -> VolatilityResult: + diagnostics: dict[str, JSONValue] = { + **dict(base), + "status": status, + "reason": reason, + "limitations": [reason], + "backend": { + "provider": "arch", + "available": False, + "import_basis": "optional_models_extra", + }, + "fit_summary": { + "schema_version": VOLATILITY_FIT_SCHEMA_VERSION, + "fit_attempt_count": 0, + "status_counts": {}, + "reason_counts": {reason: 1}, + "failed_fit_count": 0, + "limited_fit_count": 0, + "fit_samples": [], + "backend_exception_text_included": False, + }, + "evaluation": { + "schema_version": VOLATILITY_EVALUATION_SCHEMA_VERSION, + "model_count": 0, + "fold_count": len(input_result.folds), + "evaluated_fold_count": 0, + "models": [], + "reference_variance_baselines": [], + "comparison_semantics": "descriptive_shared_folds_separate_mean_and_variance_metrics", + "automatic_winner": False, + }, + "training_projection": { + "schema_version": VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION, + "column_names": list(VOLATILITY_COLUMNS), + "column_prefixes": ["cm_arch_", "cm_garch_"], + "annotation_count": 0, + "observed_columns_overwritten": False, + }, + "fit_duration_included": False, + } + return VolatilityResult(diagnostics, (), input_result) + + +def _empty_fit( + status: str, + reason: str, + *, + observed_count: int = 0, + missing_reset_count: int = 0, +) -> _FitOutcome: + return _FitOutcome( + status, + reason, + (), + (), + {}, + (), + False, + observed_count, + missing_reset_count, + None, + None, + None, + False, + {"count": 0, "mean": None, "std": None, "max_abs": None}, + None, + None, + None, + ) + + +def _trailing_contiguous( + values: Sequence[float | None], +) -> tuple[list[float], int]: + last_missing = max( + (index for index, value in enumerate(values) if value is None), + default=-1, + ) + trailing = [ + float(value) + for value in values[last_missing + 1 :] + if value is not None + ] + return trailing, sum(value is None for value in values) + + +def _boundary_parameter( + parameters: Mapping[str, float], + specification: VolatilitySpecification, + tolerance: float, +) -> bool: + if any( + name == "omega" and value <= tolerance + for name, value in parameters.items() + ): + return True + if any( + (name.startswith("alpha[") or name.startswith("beta[")) + and value <= tolerance + for name, value in parameters.items() + ): + return True + return any( + ( + lower is not None + and abs(parameters.get(name, math.inf) - lower) <= tolerance + ) + or ( + upper is not None + and abs(parameters.get(name, -math.inf) - upper) <= tolerance + ) + for name, lower, upper in specification.parameter_bounds + ) + + +def _violates_parameter_bounds( + parameters: Mapping[str, float], + bounds: Sequence[tuple[str, float | None, float | None]], +) -> bool: + return any( + name not in parameters + or (lower is not None and parameters[name] < lower) + or (upper is not None and parameters[name] > upper) + for name, lower, upper in bounds + ) + + +def _numeric_summary( + values: Sequence[float], digits: int +) -> dict[str, JSONValue]: + if not values: + return { + "count": 0, + "mean": None, + "std": None, + "median": None, + "mad": None, + "max_abs": None, + } + center = sum(values) / len(values) + med = median(values) + return { + "count": len(values), + "mean": _rounded(center, digits), + "std": _rounded( + math.sqrt( + sum((value - center) ** 2 for value in values) / len(values) + ), + digits, + ), + "median": _rounded(med, digits), + "mad": _rounded(median(abs(value - med) for value in values), digits), + "max_abs": _rounded(max(abs(value) for value in values), digits), + } + + +def _finite_float(value: Any) -> float | None: + try: + result = float(value) + except (TypeError, ValueError): + return None + return result if math.isfinite(result) else None + + +def _model_id( + specification: VolatilitySpecification, + derivation_id: str, + backend_version: str, +) -> str: + payload = { + "schema_version": VOLATILITY_SCHEMA_VERSION, + "input_derivation_id": derivation_id, + "backend": {"provider": "arch", "version": backend_version}, + "configuration": specification.to_metadata(), + } + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _folds_by_origin( + folds: Sequence[Mapping[str, Any]], +) -> list[tuple[tuple[Any, ...], list[dict[str, Any]]]]: + grouped: dict[tuple[Any, ...], list[dict[str, Any]]] = {} + for fold in folds: + key = ( + _text(fold.get("series_id")), + _text(fold.get("period")), + _int(fold.get("training_start_index")), + _int(fold.get("training_end_index")), + _int(fold.get("origin_bin_end_utc_ms")), + ) + grouped.setdefault(key, []).append(dict(fold)) + return [ + (key, sorted(values, key=lambda item: _int(item.get("horizon")))) + for key, values in sorted(grouped.items()) + ] + + +def _load_backend() -> _Backend | None: + try: + module = importlib.import_module("arch") + version = importlib.metadata.version("arch") + except (ImportError, importlib.metadata.PackageNotFoundError): + return None + return _Backend(version, module.arch_model) + + +def _ensure_projection_columns(frame: Any) -> Any: + import polars as pl + + strings = {"schema_version", "input_derivation_id", "model_id"} + booleans = { + "converged", + "forecast_available", + "diagnostic_available", + "diagnostic_only", + "boundary_parameter", + "training_eligible", + } + floats = { + "scale_factor", + "mean_forecast", + "variance_forecast", + "volatility_forecast", + "annualized_variance_forecast", + "annualized_volatility_forecast", + "actual_return", + "realized_variance_proxy", + "mean_error", + "variance_error", + "volatility_error", + "qlike_loss", + "persistence", + "unconditional_variance", + } + expressions = [] + for name in VOLATILITY_COLUMNS: + if name in frame.columns: + continue + suffix = name.removeprefix("cm_arch_").removeprefix("cm_garch_") + dtype = ( + pl.Utf8 + if suffix in strings + else ( + pl.Boolean + if suffix in booleans + else pl.Float64 if suffix in floats else pl.Int64 + ) + ) + expressions.append(pl.lit(None, dtype=dtype).alias(name)) + return frame.with_columns(expressions) if expressions else frame diff --git a/src/histdatacom/helper_args.py b/src/histdatacom/helper_args.py index 4ccd3be7..ad5a177e 100644 --- a/src/histdatacom/helper_args.py +++ b/src/histdatacom/helper_args.py @@ -21,9 +21,12 @@ "timeframes": set(), "start_yearmonth": "", "end_yearmonth": "", + "random_window": "", + "random_seed": None, "default_download_dir": f"data{os.sep}", "from_api": False, "api_return_type": "polars", + "output_timezone": "", "batch_size": "5000", "delete_after_influx": False, "zip_persist": False, @@ -45,6 +48,7 @@ "quality_preflight_run_validation": False, "quality_preflight_sample_size": 4, "quality_preflight_validation_report_path": "", + "quality_preflight_validation_evidence_path": "", "quality_profile_preview": False, "quality_profile_preview_format": "json", "quality_profile_preview_output_path": "", diff --git a/src/histdatacom/histdata_ascii.py b/src/histdatacom/histdata_ascii.py index d5825721..8cce8259 100644 --- a/src/histdatacom/histdata_ascii.py +++ b/src/histdatacom/histdata_ascii.py @@ -7,12 +7,13 @@ from __future__ import annotations import csv +import math import zipfile from dataclasses import dataclass from datetime import datetime, timezone from io import BytesIO from pathlib import Path -from typing import Any, Iterable, Sequence +from typing import Any, Iterable, Mapping, Sequence EST_NO_DST_OFFSET_MS = 18_000_000 UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) @@ -148,6 +149,8 @@ def parse_ascii_lines(timeframe: str, lines: Iterable[str]) -> ParsedAsciiBatch: def read_ascii_file(path: Path, timeframe: str) -> ParsedAsciiBatch: """Parse a plain CSV file or a ZIP containing one HistData CSV file.""" + if filename_has_unsupported_raw_dimensions(path): + raise ValueError("raw import supports ASCII tick inputs only") if path.suffix == ".zip": with zipfile.ZipFile(path) as archive: names = tuple( @@ -155,6 +158,8 @@ def read_ascii_file(path: Path, timeframe: str) -> ParsedAsciiBatch: ) if len(names) != 1: raise ValueError("expected ZIP archive to contain one CSV file") + if filename_has_unsupported_raw_dimensions(names[0]): + raise ValueError("raw import supports ASCII tick inputs only") with archive.open(names[0]) as source: text = source.read().decode("utf-8").splitlines() return parse_ascii_lines(timeframe, text) @@ -297,11 +302,50 @@ def _single_csv_member_from_zip(path: Path) -> bytes: ) if len(names) != 1: raise ValueError("expected ZIP archive to contain one CSV file") + if filename_has_unsupported_raw_dimensions(names[0]): + raise ValueError("raw import supports ASCII tick inputs only") return archive.read(names[0]) +def filename_has_unsupported_raw_dimensions(filename: str | Path) -> bool: + """Return whether a HistData filename declares a retired raw axis.""" + name = Path(str(filename)).name.upper() + stem = name + # Live HistData ASCII archives include a same-stem ``.txt`` status report + # beside the CSV data member. It declares the same supported raw axes and + # must not be mistaken for a retired platform or timeframe. + for suffix in (".ZIP", ".CSV", ".TXT"): + if stem.endswith(suffix): + stem = stem[: -len(suffix)] + parts = stem.split("_") + if stem.startswith("DAT_"): + if len(parts) < 5: + return True + period = parts[4] + return ( + parts[1] != "ASCII" + or parts[3] != TICK + or len(period) not in {4, 6} + or not period.isdigit() + ) + if stem.startswith("HISTDATA_COM_"): + if len(parts) != 5: + return True + timeframe_period = parts[4] + period = timeframe_period[1:] + is_tick = ( + len(period) in {4, 6} + and timeframe_period.startswith(TICK) + and period.isdigit() + ) + return parts[2] != "ASCII" or not is_tick + return False + + def read_ascii_file_to_polars(path: Path, timeframe: str) -> Any: """Read a plain CSV file or ZIP archive into a raw Polars dataframe.""" + if filename_has_unsupported_raw_dimensions(path): + raise ValueError("raw import supports ASCII tick inputs only") if path.suffix.lower() == ".zip": return _read_csv_to_polars( BytesIO(_single_csv_member_from_zip(path)), @@ -313,6 +357,7 @@ def read_ascii_file_to_polars(path: Path, timeframe: str) -> Any: def write_polars_cache(frame: Any, path: Path) -> None: """Write a Polars dataframe cache using Arrow IPC payloads.""" + _validate_cache_dimensions(frame) frame.write_ipc(path) @@ -321,9 +366,30 @@ def read_polars_cache(path: Path) -> Any: import polars as pl try: - return pl.read_ipc(path) + frame = pl.read_ipc(path) except Exception as err: raise ValueError(LEGACY_CACHE_ERROR) from err + _validate_cache_dimensions(frame) + return frame + + +def _validate_cache_dimensions(frame: Any) -> None: + """Reject enriched caches that declare retired raw dimensions.""" + columns = set(getattr(frame, "columns", ())) + if "format" in columns: + formats = { + str(value).lower() + for value in frame.get_column("format").drop_nulls().unique() + } + if formats != {"ascii"}: + raise ValueError("cache supports ASCII tick inputs only") + if "timeframe" in columns: + timeframes = { + str(value) + for value in frame.get_column("timeframe").drop_nulls().unique() + } + if timeframes != {TICK}: + raise ValueError("cache supports ASCII tick inputs only") def to_arrow_table(batch: ParsedAsciiBatch) -> Any: @@ -390,15 +456,131 @@ def merge_batches( def format_influx_line( - pair: str, data_format: str, timeframe: str, row: Sequence[Any] + pair: str, + data_format: str, + timeframe: str, + row: Sequence[Any], + *, + columns: Sequence[str] | None = None, ) -> str: - """Return the line protocol string currently emitted for a parsed row.""" - tags = ( - f"source=histdata.com,format={data_format},timeframe={timeframe}" - ).replace(" ", "") + """Return line protocol for a raw or enriched ASCII tick cache row.""" + _validate_influx_dimensions(data_format, timeframe) + + if columns is None: + tags = ( + f"source=histdata.com,format={data_format},timeframe={timeframe}" + ).replace(" ", "") + fields = f"bidquote={row[1]},askquote={row[2]}".replace(" ", "") + return f"{pair},{tags} {fields} {row[0]}" + + values = _row_values(row, columns) + _validate_influx_dimensions( + str(values.get("format") or data_format), + str(values.get("timeframe") or timeframe), + ) + tags = _influx_tags(values, data_format, timeframe) + fields = _influx_fields(values) + timestamp = values.get("datetime", row[0]) + return f"{_escape_influx_key(pair)},{tags} {fields} {timestamp}" + +def _validate_influx_dimensions(data_format: str, timeframe: str) -> None: + if str(data_format).lower() != "ascii": + raise ValueError("Influx projection supports ASCII tick inputs only") if timeframe != TICK: raise ValueError(f"unsupported ASCII timeframe: {timeframe}") - fields = f"bidquote={row[1]},askquote={row[2]}".replace(" ", "") - return f"{pair},{tags} {fields} {row[0]}" + +def _row_values( + row: Sequence[Any], + columns: Sequence[str], +) -> dict[str, Any]: + return dict(zip(columns, row, strict=False)) + + +def _influx_tags( + values: Mapping[str, Any], + data_format: str, + timeframe: str, +) -> str: + source = str(values.get("source") or "histdata.com") + tags = { + "source": source, + "format": str(values.get("format") or data_format), + "timeframe": str(values.get("timeframe") or timeframe), + } + period = values.get("period") + if period not in (None, ""): + tags["period"] = str(period) + row_id = values.get("row_id") + if row_id not in (None, ""): + tags["row_id"] = str(row_id) + return ",".join( + f"{_escape_influx_key(key)}={_escape_influx_key(value)}" + for key, value in tags.items() + ) + + +def _influx_fields(values: Mapping[str, Any]) -> str: + fields: list[str] = [] + _append_influx_field(fields, "bidquote", values.get("bid")) + _append_influx_field(fields, "askquote", values.get("ask")) + excluded = { + "datetime", + "bid", + "ask", + "vol", + "training_schema_version", + "series_id", + "row_id", + "symbol", + "format", + "timeframe", + "source", + "period", + } + for name, value in values.items(): + if name in excluded: + continue + _append_influx_field(fields, name, value) + return ",".join(fields) + + +def _append_influx_field( + fields: list[str], + name: str, + value: Any, +) -> None: + formatted = _format_influx_field_value(value) + if formatted is None: + return + fields.append(f"{_escape_influx_key(name)}={formatted}") + + +def _format_influx_field_value(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return f"{value}i" + if isinstance(value, float): + if not math.isfinite(value): + return None + return str(value) + return _format_influx_string_field(str(value)) + + +def _format_influx_string_field(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def _escape_influx_key(value: Any) -> str: + return ( + str(value) + .replace("\\", "\\\\") + .replace(" ", "\\ ") + .replace(",", "\\,") + .replace("=", "\\=") + ) diff --git a/src/histdatacom/histdata_com.py b/src/histdatacom/histdata_com.py index 3f44cdf0..854acd4f 100644 --- a/src/histdatacom/histdata_com.py +++ b/src/histdatacom/histdata_com.py @@ -39,14 +39,18 @@ routed_command_from_cli_args, ) from histdatacom.exceptions import ( + ConfigurationError, format_exception_for_cli, format_failure_info_for_cli, InfluxConfigurationError, ) from histdatacom.data_quality.preflight import ( + QUALITY_PREFLIGHT_VALIDATION_EVIDENCE_SCHEMA_VERSION, format_quality_preflight_console_summary, format_quality_run_preflight_warning, quality_preflight_to_markdown, + quality_preflight_validation_evidence_payload, + quality_preflight_validation_evidence_to_json, quality_run_preflight_warning, register_quality_preflight_evidence_artifact, run_cache_quality_preflight, @@ -57,13 +61,17 @@ from histdatacom.data_quality.profiles import ( QualityProfile, quality_profile_from_value, + quality_profile_resolution_from_value, + quality_profile_source_kind, ) from histdatacom.data_quality.reporting import ( + format_cross_series_fingerprint_lines, format_fingerprint_distribution_attention_lines, format_fingerprint_distribution_summary_lines, format_fingerprint_readiness_risk_lines, format_fingerprint_topology_attention_lines, format_fingerprint_topology_summary_lines, + format_quality_engine_skip_lines, format_quality_next_action_lines, format_quality_remediation_coverage_lines, ) @@ -76,6 +84,11 @@ from histdatacom.publication_safety import publish_safe_path from histdatacom.publication_safety import publish_safe_json_mapping from histdatacom.records import Record +from histdatacom.random_windows import ( + RANDOM_WINDOW_SELECTION_METADATA_KEY, + RandomWindowSelectionV1, + RandomWindowError, +) from histdatacom.runtime_contracts import ( FailureInfo, JSONValue, @@ -99,6 +112,7 @@ ) from histdatacom.utils import ( load_influx_yaml, + normalize_output_timezone, set_working_data_dir, normalize_api_return_type, ) @@ -139,6 +153,7 @@ class RuntimeContext: orchestration_keep_runtime: bool orchestration_wait_result: bool api_return_type: str | None + output_timezone: str data_quality: bool quality_paths: tuple[str, ...] quality_check_groups: tuple[str, ...] @@ -158,8 +173,10 @@ class RuntimeContext: quality_preflight_run_validation: bool quality_preflight_sample_size: int quality_preflight_validation_report_path: str | None + quality_preflight_validation_evidence_path: str quality_profile_path: str quality_profile: Mapping[str, Any] + quality_profile_resolution: Mapping[str, Any] quality_profile_preview: bool quality_profile_preview_format: str quality_profile_preview_output_path: str @@ -305,6 +322,7 @@ def _run_quality_preflight(self) -> dict[str, Any]: ) ) _attach_quality_preflight_profile_preview(payload, self.context) + _attach_quality_preflight_validation_evidence(payload, self.context) if self.context.quality_preflight_report_path: report_path = Path( self.context.quality_preflight_report_path @@ -429,7 +447,18 @@ def _run_orchestration_job( if self._should_materialize_orchestration_api_return(payload): records = _cache_records_from_orchestration_payload(payload) if records: - return self._materialize_orchestration_api_return(records) + selection = _random_window_selection_from_orchestration_payload( + payload + ) + if self.context.request.random_window and selection is None: + raise RandomWindowError( + "completed random-window run is missing its resolved " + "selection" + ) + return self._materialize_orchestration_api_return( + records, + random_selection=selection, + ) if not self.context.from_api: print(json.dumps(payload, indent=2, sort_keys=True)) # noqa:T201 return payload @@ -516,6 +545,8 @@ def _should_materialize_orchestration_api_return( def _materialize_orchestration_api_return( self, records: list[Record], + *, + random_selection: RandomWindowSelectionV1 | None = None, ) -> list | PolarsDataFrame | DataFrame | Table: """Rebuild the legacy API dataframe return from cache artifacts.""" from histdatacom.api import Api @@ -523,6 +554,8 @@ def _materialize_orchestration_api_return( return Api().merge_records( records, return_type=str(self.context.api_return_type or ""), + output_timezone=self.context.output_timezone, + random_selection=random_selection, ) @@ -540,6 +573,17 @@ def _resolve_runtime_context(options: Options) -> RuntimeContext: args["default_download_dir"] = set_working_data_dir(args["data_directory"]) args["api_return_type"] = normalize_api_return_type(args["api_return_type"]) options.api_return_type = args["api_return_type"] + configured_output_timezone = str(args.get("output_timezone", "") or "") + try: + args["output_timezone"] = normalize_output_timezone( + configured_output_timezone + ) + except ValueError as err: + raise ConfigurationError( + str(err), + detail={"output_timezone": configured_output_timezone.strip()}, + ) from err + options.output_timezone = args["output_timezone"] _attach_influx_config_metadata(options, args) try: should_submit_to_orchestration(args) @@ -558,6 +602,7 @@ def _resolve_runtime_context(options: Options) -> RuntimeContext: orchestration_keep_runtime=bool(args["orchestration_keep_runtime"]), orchestration_wait_result=bool(args["orchestration_wait_result"]), api_return_type=args["api_return_type"], + output_timezone=args["output_timezone"], data_quality=bool(args["data_quality"]), quality_paths=tuple( str(path) for path in (args.get("quality_paths") or ()) @@ -615,8 +660,14 @@ def _resolve_runtime_context(options: Options) -> RuntimeContext: if args.get("quality_preflight_validation_report_path") is None else str(args["quality_preflight_validation_report_path"]) ), + quality_preflight_validation_evidence_path=str( + args.get("quality_preflight_validation_evidence_path") or "" + ), quality_profile_path=str(args.get("quality_profile_path") or ""), quality_profile=dict(args.get("quality_profile") or {}), + quality_profile_resolution=dict( + args.get("quality_profile_resolution") or {} + ), quality_profile_preview=bool(args["quality_profile_preview"]), quality_profile_preview_format=str( args.get("quality_profile_preview_format") or "json" @@ -707,6 +758,36 @@ def _attach_quality_preflight_profile_preview( ) +def _attach_quality_preflight_validation_evidence( + payload: dict[str, Any], + context: RuntimeContext, +) -> None: + """Write optional bounded validation evidence into the artifact map.""" + destination = context.quality_preflight_validation_evidence_path.strip() + if not destination: + return + validation_payload = quality_preflight_validation_evidence_payload(payload) + artifact = write_quality_preflight_evidence_artifact( + quality_preflight_validation_evidence_to_json(validation_payload), + destination, + kind="quality-preflight-validation-evidence", + output_format="json", + schema_version=(QUALITY_PREFLIGHT_VALIDATION_EVIDENCE_SCHEMA_VERSION), + label="Validation evidence", + console_label="validation evidence", + ) + artifact["generated_at_utc"] = str( + validation_payload.get("generated_at_utc", "") + ) + artifact["validation_state"] = str(validation_payload.get("state", "")) + register_quality_preflight_evidence_artifact( + payload, + "validation_evidence", + artifact, + legacy_key="validation_evidence", + ) + + def _format_quality_profile_preview( payload: Mapping[str, JSONValue], *, @@ -907,6 +988,7 @@ def _preview_channel_detail_text(channel: Mapping[str, JSONValue]) -> str: "profile_name", "profile_source", "source_path", + "selected_by", "paths", ): value = channel.get(key) @@ -947,7 +1029,12 @@ def _preview_value_source_text(row: Mapping[str, JSONValue]) -> str: source = str(row.get("source") or "unknown") value = _preview_display_value(row.get("value")) override = " override" if row.get("override") else "" - return f"{path} [{source}{override}]: {value}" + previous = "" + if row.get("override"): + previous_source = str(row.get("previous_source") or "unknown") + previous_value = _preview_display_value(row.get("previous_value")) + previous = f"; previous={previous_source}:{previous_value}" + return f"{path} [{source}{override}]: {value}{previous}" def _preview_display_value( @@ -993,11 +1080,12 @@ def _quality_profile_preview_payload( ) -> dict[str, JSONValue]: """Return a deterministic preview of the resolved quality profile.""" profile = quality_profile_from_value(context.quality_profile) - resolved_profile = profile.to_request_payload() - if "reporting" not in resolved_profile: - resolved_profile["reporting"] = ( - profile.reporting_profile().to_metadata() - ) + resolution = dict(context.quality_profile_resolution) + if not resolution: + resolution = quality_profile_resolution_from_value(profile).to_payload() + resolved_profile = dict( + _preview_mapping(resolution.get("resolved_profile")) + ) profile_metadata = profile.to_metadata() profile_metadata.setdefault("configured_reporting_keys", []) @@ -1009,9 +1097,13 @@ def _quality_profile_preview_payload( audit_override_enabled = bool( context.args.get("quality_remediation_catalog_audit") ) - cli_overrides: dict[str, JSONValue] = {} - if audit_override_enabled: - cli_overrides["reporting.remediation_catalog_audit.enabled"] = True + effective_sources = [ + dict(item) + for item in _preview_mapping_rows( + resolution.get("effective_value_sources") + ) + ] + cli_overrides = _quality_profile_cli_overrides(effective_sources) profile_inputs: dict[str, JSONValue] = { "from_api": context.from_api, "config_path": str(context.args.get("config_path") or ""), @@ -1039,8 +1131,7 @@ def _quality_profile_preview_payload( "profile_explanation": _quality_profile_preview_explanation( profile, resolved_profile=resolved_profile, - profile_inputs=profile_inputs, - cli_overrides=cli_overrides, + resolution=resolution, ), "resolved_profile": resolved_profile, } @@ -1050,17 +1141,17 @@ def _quality_profile_preview_explanation( profile: QualityProfile, *, resolved_profile: Mapping[str, JSONValue], - profile_inputs: Mapping[str, JSONValue], - cli_overrides: Mapping[str, JSONValue], + resolution: Mapping[str, JSONValue], ) -> dict[str, JSONValue]: - """Return deterministic provenance and default-diff metadata.""" + """Render provenance retained by the profile resolver.""" default_profile = _resolved_default_quality_profile_payload() source_kind = _quality_profile_source_kind(profile) - effective_sources = _quality_profile_value_sources( - resolved_profile, - profile=profile, - cli_overrides=cli_overrides, - ) + effective_sources = [ + dict(item) + for item in _preview_mapping_rows( + resolution.get("effective_value_sources") + ) + ] effective_diff = _quality_profile_effective_diff( default_profile, resolved_profile, @@ -1075,10 +1166,14 @@ def _quality_profile_preview_explanation( "source_path": profile.source_path, "is_default": profile.is_default, }, - "input_channels": _quality_profile_input_channels( - profile, - profile_inputs=profile_inputs, - cli_overrides=cli_overrides, + "input_channels": cast( + JSONValue, + [ + dict(item) + for item in _preview_mapping_rows( + resolution.get("input_channels") + ) + ], ), "effective_value_sources": _bounded_source_items( effective_sources, @@ -1096,75 +1191,19 @@ def _resolved_default_quality_profile_payload() -> dict[str, JSONValue]: return payload -def _quality_profile_input_channels( - profile: QualityProfile, - *, - profile_inputs: Mapping[str, JSONValue], - cli_overrides: Mapping[str, JSONValue], -) -> list[JSONValue]: - """Return the input channels that shaped the resolved profile.""" - config_path = str(profile_inputs.get("config_path") or "") - profile_path = str(profile_inputs.get("quality_profile_path") or "") - source_kind = _quality_profile_source_kind(profile) - channels: list[dict[str, JSONValue]] = [ - { - "kind": "built_in_default", - "active": True, - "description": "Built-in quality profile defaults.", - }, - { - "kind": "yaml_config", - "active": bool(config_path), - "path": config_path, - }, - { - "kind": source_kind, - "active": not profile.is_default, - "profile_name": profile.name, - "profile_source": profile.source, - "source_path": profile.source_path or profile_path, - }, - { - "kind": "api_options", - "active": ( - bool(profile_inputs.get("from_api")) - and source_kind != "api_options" - ), - }, - { - "kind": "cli_override", - "active": bool(cli_overrides), - "paths": cast(JSONValue, sorted(cli_overrides)), - }, - ] - return [channel for channel in channels if channel["active"]] - - -def _quality_profile_value_sources( - resolved_profile: Mapping[str, JSONValue], - *, - profile: QualityProfile, - cli_overrides: Mapping[str, JSONValue], -) -> list[dict[str, JSONValue]]: - """Return per-value provenance for the resolved preview profile.""" - source_by_path: dict[str, dict[str, JSONValue]] = {} - for path, value in _flatten_json_mapping(resolved_profile): - source_by_path[path] = { - "path": path, - "value": value, - "source": _quality_profile_path_source(path, profile=profile), - } - for dotted_path, value in sorted(cli_overrides.items()): - pointer = _json_pointer_from_dotted_path(dotted_path) - previous = source_by_path.get(pointer, {}) - source_by_path[pointer] = { - "path": pointer, - "value": value, - "source": "cli_override", - "overridden_source": str(previous.get("source") or "unknown"), - "override": True, - } - return [source_by_path[path] for path in sorted(source_by_path)] +def _quality_profile_cli_overrides( + effective_sources: Sequence[Mapping[str, JSONValue]], +) -> dict[str, JSONValue]: + """Return compatibility CLI overrides from resolver provenance.""" + overrides: dict[str, JSONValue] = {} + for item in effective_sources: + if item.get("source") != "cli_override" or not item.get("override"): + continue + path = str(item.get("path") or "") + if not path: + continue + overrides[_dotted_path_from_json_pointer(path)] = item.get("value") + return overrides def _quality_profile_effective_diff( @@ -1224,42 +1263,9 @@ def _prune_expanded_empty_mapping_values( del values[path] -def _quality_profile_path_source( - path: str, - *, - profile: QualityProfile, -) -> str: - """Return the source label for one resolved profile value path.""" - if profile.is_default: - return "built_in_default" - if path.startswith("/rules/") or path.startswith("/modeling_assumptions/"): - return _quality_profile_source_kind(profile) - if path.startswith("/reporting/"): - reporting = profile.reporting - if _profile_has_json_pointer( - reporting, path.removeprefix("/reporting") - ): - return _quality_profile_source_kind(profile) - return "built_in_default" - if path in {"/name", "/source", "/source_path", "/schema_version"}: - return _quality_profile_source_kind(profile) - return "built_in_default" - - def _quality_profile_source_kind(profile: QualityProfile) -> str: """Return a stable public source kind for a quality profile.""" - source = str(getattr(profile, "source", "") or "") - if source == "default": - return "built_in_default" - if source == "file": - return "profile_file" - if source == "api-options": - return "api_options" - if source == "cli-options": - return "cli_options" - if source == "operator-config": - return "operator_config" - return source.replace("-", "_") or "unknown" + return str(quality_profile_source_kind(profile)) def _bounded_source_items( @@ -1299,25 +1305,13 @@ def _flatten_json_mapping( return flattened -def _profile_has_json_pointer( - value: Mapping[str, JSONValue], - pointer: str, -) -> bool: - """Return whether a profile mapping explicitly contains a pointer path.""" - if not pointer: - return bool(value) - current: object = value - for raw_token in pointer.strip("/").split("/"): - token = _json_pointer_token_unescape(raw_token) - if not isinstance(current, Mapping) or token not in current: - return False - current = current[token] - return True - - -def _json_pointer_from_dotted_path(path: str) -> str: - """Translate legacy dotted override paths into JSON pointers.""" - return "/" + "/".join(_json_pointer_token(part) for part in path.split(".")) +def _dotted_path_from_json_pointer(path: str) -> str: + """Translate a JSON pointer into the compatibility dotted path form.""" + return ".".join( + _json_pointer_token_unescape(token) + for token in path.strip("/").split("/") + if token + ) def _json_pointer_token(value: str) -> str: @@ -1419,7 +1413,15 @@ def main( cli_args = sys.argv[1:] if not options else [] routed_command = routed_command_from_cli_args( cli_args, - {"analytics", "cleanup", "groups", "jobs", "quality", "runtime"}, + { + "analytics", + "cleanup", + "groups", + "jobs", + "quality", + "reconstruction", + "runtime", + }, ) if not options and routed_command == "cleanup": from histdatacom.cleanup_cli import main as cleanup_main @@ -1443,6 +1445,12 @@ def main( return quality_main( remove_routed_command_from_cli_args(cli_args, "quality") ) + if not options and routed_command == "reconstruction": + from histdatacom.reconstruction_cli import main as reconstruction_main + + return reconstruction_main( + remove_routed_command_from_cli_args(cli_args, "reconstruction") + ) if not options and routed_command == "runtime": reexec_code = _maybe_reexec_windows_runtime_cli(cli_args) if reexec_code is not None: @@ -1462,7 +1470,17 @@ def main( if not options: options = Options() - _HistDataCom(options).run() + try: + _HistDataCom(options).run() + except ConfigurationError as err: + print( # noqa:T201 + format_exception_for_cli( + err, + title="HistData configuration invalid", + ), + file=sys.stderr, + ) + raise SystemExit(1) from err return None options.from_api = True return _HistDataCom(options).run() @@ -1546,6 +1564,24 @@ def _cache_records_from_orchestration_payload(payload: dict) -> list[Record]: return records +def _random_window_selection_from_orchestration_payload( + payload: Mapping[str, Any], +) -> RandomWindowSelectionV1 | None: + """Recover and verify one compact selection from a completed run.""" + selections: dict[str, RandomWindowSelectionV1] = {} + for item in _iter_mapping_payloads(payload): + candidate = item.get(RANDOM_WINDOW_SELECTION_METADATA_KEY) + if not isinstance(candidate, Mapping): + continue + selection = RandomWindowSelectionV1.from_dict(candidate) + selections[selection.selection_id] = selection + if len(selections) > 1: + raise ValueError( + "orchestration payload contains conflicting random windows" + ) + return next(iter(selections.values()), None) + + def _repository_available_data_from_orchestration_payload( payload: Mapping[str, Any], ) -> dict[str, Any] | None: @@ -1768,6 +1804,11 @@ def _format_orchestration_quality_console_summary( lines.append(f"decision: {decision['reason']}") if int(summary.get("target_count", 0) or 0) == 0: lines.append("No data quality targets discovered.") + lines.extend( + format_quality_engine_skip_lines( + _mapping_from_payload(quality_payload.get("quality_engine")) + ) + ) lines.extend( format_quality_next_action_lines( _mapping_from_payload(quality_payload.get("next_actions")) @@ -1811,6 +1852,13 @@ def _format_orchestration_quality_console_summary( ) ) ) + lines.extend( + format_cross_series_fingerprint_lines( + _mapping_from_payload( + quality_payload.get("fingerprint_cross_series") + ) + ) + ) lines.extend(_format_quality_target_sections(quality_payload)) return "\n".join(lines) diff --git a/src/histdatacom/manifest_store.py b/src/histdatacom/manifest_store.py index b744ad14..340c0571 100644 --- a/src/histdatacom/manifest_store.py +++ b/src/histdatacom/manifest_store.py @@ -730,6 +730,97 @@ def get_job_snapshot(self, job_id: str) -> dict[str, Any] | None: return None return dict(json.loads(str(row["payload_json"]))) + def compare_and_swap_job_snapshot( + self, + snapshot: Any, + *, + expected_snapshot_id: str | None, + ) -> bool: + """Atomically replace a bounded job snapshot when its ID is current. + + This is the manifest-store primitive used by resumable workflows whose + checkpoint chain must reject stale workers. Repeating the exact same + write succeeds idempotently; a different current snapshot fails closed. + """ + payload = _mapping_payload(snapshot) + job_id = str( + payload.get("job_id", "") or payload.get("workflow_id", "") + ) + snapshot_id = str(payload.get("snapshot_id", "") or "").strip() + if not job_id: + raise ValueError("Job snapshot requires job_id or workflow_id.") + if not snapshot_id: + raise ValueError("CAS job snapshot requires snapshot_id.") + normalized_expected = ( + str(expected_snapshot_id).strip() + if expected_snapshot_id is not None + else None + ) + now = _utc_now() + payload["updated_at_utc"] = str( + payload.get("updated_at_utc", "") or now + ) + with self._connect() as conn: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + "SELECT payload_json FROM jobs WHERE job_id = ?", + (job_id,), + ).fetchone() + if row is None: + if normalized_expected is not None: + conn.rollback() + return False + else: + current = json.loads(str(row["payload_json"])) + current_id = str(current.get("snapshot_id", "") or "") + if current_id == snapshot_id: + conn.rollback() + return True + if current_id != (normalized_expected or ""): + conn.rollback() + return False + conn.execute( + """ + INSERT INTO jobs ( + job_id, + request_id, + workflow_id, + run_id, + lifecycle, + status, + task_queue, + payload_json, + updated_at_utc, + schema_version + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_id) DO UPDATE SET + request_id=excluded.request_id, + workflow_id=excluded.workflow_id, + run_id=excluded.run_id, + lifecycle=excluded.lifecycle, + status=excluded.status, + task_queue=excluded.task_queue, + payload_json=excluded.payload_json, + updated_at_utc=excluded.updated_at_utc, + schema_version=excluded.schema_version + """, + ( + job_id, + str(payload.get("request_id", "") or ""), + str(payload.get("workflow_id", "") or job_id), + str(payload.get("run_id", "") or ""), + str(payload.get("lifecycle", "") or ""), + str(payload.get("status", "") or ""), + str(payload.get("task_queue", "") or ""), + _json_dumps(payload), + now, + int(payload.get("schema_version", 1) or 1), + ), + ) + conn.commit() + return True + def list_job_snapshots( self, *, diff --git a/src/histdatacom/market_context/__init__.py b/src/histdatacom/market_context/__init__.py new file mode 100644 index 00000000..ab828905 --- /dev/null +++ b/src/histdatacom/market_context/__init__.py @@ -0,0 +1,233 @@ +"""Point-in-time market-context contracts and bounded query helpers.""" + +from histdatacom.market_context.contracts import ( + MARKET_CONTEXT_CALENDAR_STATE_SCHEMA_VERSION, + MARKET_CONTEXT_EVENT_SCHEMA_VERSION, + MARKET_CONTEXT_QUERY_SCHEMA_VERSION, + MARKET_CONTEXT_SOURCE_SCHEMA_VERSION, + MARKET_CONTEXT_TIMELINE_SCHEMA_VERSION, + MAX_MARKET_CONTEXT_ADAPTERS, + MAX_MARKET_CONTEXT_EVENTS, + MAX_MARKET_CONTEXT_QUERY_EVENTS, + MarketContextCalendarStateV1, + MarketContextEventV1, + MarketContextKind, + MarketContextMissingReason, + MarketContextPrecision, + MarketContextQueryLimitError, + MarketContextQueryStatus, + MarketContextQueryV1, + MarketContextSourceAdapterV1, + MarketContextSourceV1, + MarketContextTimelineV1, + MarketContextView, + StaticMarketContextSourceAdapterV1, + build_market_context_timeline, + market_context_calendar_state, + market_context_information_inputs, + normalize_market_context_datetime, + query_market_context, + query_market_context_window, +) +from histdatacom.market_context.corpus import ( + DEFAULT_MARKET_CONTEXT_SOURCES, + DEFAULT_ONS_QUERIES, + MARKET_CONTEXT_CORPUS_SCHEMA_VERSION, + MARKET_CONTEXT_COVERAGE_SCHEMA_VERSION, + MARKET_CONTEXT_PREFLIGHT_SCHEMA_VERSION, + MARKET_CONTEXT_SOURCE_EVIDENCE_SCHEMA_VERSION, + BankOfEnglandBankRateAdapterV1, + EcbPolicyRateAdapterV1, + FederalReserveFomcCalendarAdapterV1, + FederalReserveFomcHistoricalAdapterV1, + MarketContextCorpusBuildV1, + MarketContextCorpusPreflightError, + MarketContextCorpusPreflightV1, + MarketContextCorpusV1, + MarketContextCoverageSliceV1, + MarketContextFetchProfileV1, + MarketContextSourceEvidenceV1, + MarketContextSourceSnapshotV1, + OnsReleaseCalendarAdapterV1, + OperatorMarketContextCatalogAdapterV1, + build_live_market_context_corpus, + build_market_context_corpus_from_snapshots, + market_context_benchmark_event_state, + packaged_operator_catalog_path, + preflight_market_context_corpus, + query_market_context_corpus, + read_market_context_corpus, + replay_market_context_corpus, + require_market_context_corpus, + write_market_context_corpus, +) +from histdatacom.market_context.positioning import ( + CFTC_POSITIONING_ARCHIVE_EVIDENCE_SCHEMA_VERSION, + CFTC_POSITIONING_BINDING_SCHEMA_VERSION, + CFTC_POSITIONING_MAPPING_SCHEMA_VERSION, + CFTC_POSITIONING_CORPUS_SCHEMA_VERSION, + CFTC_POSITIONING_COVERAGE_SCHEMA_VERSION, + CFTC_POSITIONING_DIFF_SCHEMA_VERSION, + CFTC_POSITIONING_PREFLIGHT_SCHEMA_VERSION, + CFTC_POSITIONING_PROFILE_SCHEMA_VERSION, + CFTC_POSITIONING_QUERY_SCHEMA_VERSION, + CFTC_POSITIONING_RELEASE_SCHEMA_VERSION, + CFTC_POSITIONING_SMOKE_SCHEMA_VERSION, + CFTC_POSITIONING_SNAPSHOT_SCHEMA_VERSION, + CFTC_POSITIONING_SOURCE_SCHEMA_VERSION, + CftcArchiveConsistencyV1, + CftcAvailabilityConfidence, + CftcMappingKind, + CftcPositioningBenchmarkSmokeV1, + CftcPositioningConsumer, + CftcPositioningConsumerBindingV1, + CftcPositioningCorpusBuildV1, + CftcPositioningCorpusV1, + CftcPositioningCoverageSliceV1, + CftcPositioningDiffV1, + CftcPositioningFetchProfileV1, + CftcPositioningPreflightError, + CftcPositioningPreflightV1, + CftcPositioningQueryStatus, + CftcPositioningQueryV1, + CftcPositioningRawSourceV1, + CftcPositioningSnapshotV1, + CftcPositioningSymbolMappingV1, + CftcReleaseEvidenceV1, + CftcReportFamily, + CftcReportScope, + CftcRestatementStatus, + apply_cftc_positioning_to_benchmark_events, + apply_cftc_positioning_to_motif_condition, + bind_cftc_positioning_query, + build_cftc_positioning_benchmark_smoke, + build_cftc_positioning_corpus_from_sources, + build_live_cftc_positioning_corpus, + cftc_positioning_information_inputs, + cftc_positioning_state_label, + compare_cftc_positioning_corpora, + default_cftc_positioning_symbol_mappings, + preflight_cftc_positioning_corpus, + query_cftc_positioning_corpus, + read_cftc_positioning_corpus, + replay_cftc_positioning_corpus, + require_cftc_positioning_corpus, + validate_cftc_positioning_consumer_binding, + write_cftc_positioning_benchmark_smoke, + write_cftc_positioning_consumer_binding, + write_cftc_positioning_corpus, +) + +__all__ = [ + "CFTC_POSITIONING_ARCHIVE_EVIDENCE_SCHEMA_VERSION", + "CFTC_POSITIONING_BINDING_SCHEMA_VERSION", + "CFTC_POSITIONING_MAPPING_SCHEMA_VERSION", + "CFTC_POSITIONING_CORPUS_SCHEMA_VERSION", + "CFTC_POSITIONING_COVERAGE_SCHEMA_VERSION", + "CFTC_POSITIONING_DIFF_SCHEMA_VERSION", + "CFTC_POSITIONING_PREFLIGHT_SCHEMA_VERSION", + "CFTC_POSITIONING_PROFILE_SCHEMA_VERSION", + "CFTC_POSITIONING_QUERY_SCHEMA_VERSION", + "CFTC_POSITIONING_RELEASE_SCHEMA_VERSION", + "CFTC_POSITIONING_SMOKE_SCHEMA_VERSION", + "CFTC_POSITIONING_SNAPSHOT_SCHEMA_VERSION", + "CFTC_POSITIONING_SOURCE_SCHEMA_VERSION", + "CftcArchiveConsistencyV1", + "CftcAvailabilityConfidence", + "CftcMappingKind", + "CftcPositioningBenchmarkSmokeV1", + "CftcPositioningConsumer", + "CftcPositioningConsumerBindingV1", + "CftcPositioningCorpusBuildV1", + "CftcPositioningCorpusV1", + "CftcPositioningCoverageSliceV1", + "CftcPositioningDiffV1", + "CftcPositioningFetchProfileV1", + "CftcPositioningPreflightError", + "CftcPositioningPreflightV1", + "CftcPositioningQueryStatus", + "CftcPositioningQueryV1", + "CftcPositioningRawSourceV1", + "CftcPositioningSnapshotV1", + "CftcPositioningSymbolMappingV1", + "CftcReleaseEvidenceV1", + "CftcReportFamily", + "CftcReportScope", + "CftcRestatementStatus", + "DEFAULT_MARKET_CONTEXT_SOURCES", + "DEFAULT_ONS_QUERIES", + "MARKET_CONTEXT_CALENDAR_STATE_SCHEMA_VERSION", + "MARKET_CONTEXT_CORPUS_SCHEMA_VERSION", + "MARKET_CONTEXT_COVERAGE_SCHEMA_VERSION", + "MARKET_CONTEXT_EVENT_SCHEMA_VERSION", + "MARKET_CONTEXT_QUERY_SCHEMA_VERSION", + "MARKET_CONTEXT_PREFLIGHT_SCHEMA_VERSION", + "MARKET_CONTEXT_SOURCE_SCHEMA_VERSION", + "MARKET_CONTEXT_SOURCE_EVIDENCE_SCHEMA_VERSION", + "MARKET_CONTEXT_TIMELINE_SCHEMA_VERSION", + "MAX_MARKET_CONTEXT_ADAPTERS", + "MAX_MARKET_CONTEXT_EVENTS", + "MAX_MARKET_CONTEXT_QUERY_EVENTS", + "MarketContextCalendarStateV1", + "BankOfEnglandBankRateAdapterV1", + "EcbPolicyRateAdapterV1", + "FederalReserveFomcCalendarAdapterV1", + "FederalReserveFomcHistoricalAdapterV1", + "MarketContextCorpusBuildV1", + "MarketContextCorpusPreflightError", + "MarketContextCorpusPreflightV1", + "MarketContextCorpusV1", + "MarketContextCoverageSliceV1", + "MarketContextEventV1", + "MarketContextFetchProfileV1", + "MarketContextKind", + "MarketContextMissingReason", + "MarketContextPrecision", + "MarketContextQueryLimitError", + "MarketContextQueryStatus", + "MarketContextQueryV1", + "MarketContextSourceAdapterV1", + "MarketContextSourceEvidenceV1", + "MarketContextSourceSnapshotV1", + "MarketContextSourceV1", + "MarketContextTimelineV1", + "MarketContextView", + "StaticMarketContextSourceAdapterV1", + "OnsReleaseCalendarAdapterV1", + "OperatorMarketContextCatalogAdapterV1", + "build_live_market_context_corpus", + "build_market_context_corpus_from_snapshots", + "build_market_context_timeline", + "market_context_calendar_state", + "market_context_benchmark_event_state", + "market_context_information_inputs", + "normalize_market_context_datetime", + "packaged_operator_catalog_path", + "preflight_market_context_corpus", + "query_market_context", + "query_market_context_corpus", + "query_market_context_window", + "read_market_context_corpus", + "replay_market_context_corpus", + "require_market_context_corpus", + "write_market_context_corpus", + "apply_cftc_positioning_to_benchmark_events", + "apply_cftc_positioning_to_motif_condition", + "bind_cftc_positioning_query", + "build_cftc_positioning_benchmark_smoke", + "build_cftc_positioning_corpus_from_sources", + "build_live_cftc_positioning_corpus", + "cftc_positioning_information_inputs", + "cftc_positioning_state_label", + "compare_cftc_positioning_corpora", + "default_cftc_positioning_symbol_mappings", + "preflight_cftc_positioning_corpus", + "query_cftc_positioning_corpus", + "read_cftc_positioning_corpus", + "replay_cftc_positioning_corpus", + "require_cftc_positioning_corpus", + "validate_cftc_positioning_consumer_binding", + "write_cftc_positioning_benchmark_smoke", + "write_cftc_positioning_consumer_binding", + "write_cftc_positioning_corpus", +] diff --git a/src/histdatacom/market_context/assets/operator_shocks_v1.json b/src/histdatacom/market_context/assets/operator_shocks_v1.json new file mode 100644 index 00000000..a67c1c4a --- /dev/null +++ b/src/histdatacom/market_context/assets/operator_shocks_v1.json @@ -0,0 +1,221 @@ +{ + "catalog_version": "2026-07-15.1", + "events": [ + { + "affected_currencies": [ + "EUR", + "GBP", + "USD" + ], + "affected_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "ambiguity_reason": "The retained record conditions the UTC calendar day; it does not claim one exact market-impact instant.", + "available_at": "2001-09-12T00:00:00+00:00", + "canonical_key": "operator.shock.september-11-attacks.2001-09-11", + "confidence": 1.0, + "first_known_at": "2001-09-12T00:00:00+00:00", + "kind": "unscheduled_shock", + "limitations": [ + "The full-day window is conservative and does not infer the first tradable reaction timestamp; ex-ante availability is delayed until the following UTC day.", + "Public record: National Commission on Terrorist Attacks Upon the United States, https://www.9-11commission.gov/report/911Report.pdf" + ], + "post_event_ns": 86400000000000, + "pre_event_ns": 0, + "precision": "window_only", + "revision_sequence": 0, + "source_event_time": "2001-09-11T00:00:00+00:00", + "source_timezone": "UTC", + "tags": [ + "geopolitical", + "public_record", + "terrorism" + ], + "title": "September 11 attacks", + "vintage_id": "operator-public-record-2001-09-11-v1" + }, + { + "affected_currencies": [ + "EUR", + "GBP", + "USD" + ], + "affected_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "ambiguity_reason": "The retained record conditions the filing day and does not claim one exact market-impact instant.", + "available_at": "2008-09-16T00:00:00+00:00", + "canonical_key": "operator.shock.lehman-bankruptcy.2008-09-15", + "confidence": 1.0, + "first_known_at": "2008-09-16T00:00:00+00:00", + "kind": "unscheduled_shock", + "limitations": [ + "The filing-day window is not a claim that all market effects began at midnight UTC; ex-ante availability is delayed until the following UTC day.", + "Public record: Lehman Brothers Holdings Inc. Form 8-K, https://www.sec.gov/Archives/edgar/data/806085/000119312508198257/d8k.htm" + ], + "post_event_ns": 86400000000000, + "pre_event_ns": 0, + "precision": "window_only", + "revision_sequence": 0, + "source_event_time": "2008-09-15T00:00:00+00:00", + "source_timezone": "UTC", + "tags": [ + "bankruptcy", + "financial_crisis", + "public_record" + ], + "title": "Lehman Brothers bankruptcy filing", + "vintage_id": "operator-public-record-2008-09-15-v1" + }, + { + "affected_currencies": [ + "EUR", + "GBP", + "USD" + ], + "affected_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "ambiguity_reason": "The official announcement time is retained as a narrow source window rather than an exact universal market-impact boundary.", + "available_at": "2015-01-15T10:30:00+01:00", + "canonical_key": "operator.shock.snb-minimum-exchange-rate.2015-01-15", + "confidence": 1.0, + "first_known_at": "2015-01-15T10:30:00+01:00", + "kind": "unscheduled_shock", + "limitations": [ + "This record conditions the announcement window and makes no causal claim for each FX instrument.", + "Public record: Swiss National Bank press release, https://www.snb.ch/en/publications/communication/press-releases-restricted/pre_20150115" + ], + "post_event_ns": 7200000000000, + "pre_event_ns": 1800000000000, + "precision": "approximate", + "revision_sequence": 0, + "source_event_time": "2015-01-15T10:30:00+01:00", + "source_timezone": "Europe/Zurich", + "tags": [ + "central_bank", + "public_record", + "snb" + ], + "title": "SNB discontinues minimum exchange rate", + "vintage_id": "operator-public-record-2015-01-15-v1" + }, + { + "affected_currencies": [ + "EUR", + "GBP", + "USD" + ], + "affected_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "ambiguity_reason": "The retained record conditions the referendum-result day and does not invent an exact completion timestamp for the nationwide count.", + "available_at": "2016-06-25T00:00:00+00:00", + "canonical_key": "operator.shock.eu-referendum-result.2016-06-24", + "confidence": 1.0, + "first_known_at": "2016-06-25T00:00:00+00:00", + "kind": "unscheduled_shock", + "limitations": [ + "The full-day result window does not expose the result before its retained knowledge boundary; ex-ante availability is delayed until the following UTC day.", + "Public record: UK Electoral Commission referendum results, https://www.electoralcommission.org.uk/who-we-are-and-what-we-do/elections-and-referendums/past-elections-and-referendums/eu-referendum/results-and-turnout-eu-referendum" + ], + "post_event_ns": 86400000000000, + "pre_event_ns": 0, + "precision": "window_only", + "revision_sequence": 0, + "source_event_time": "2016-06-24T00:00:00+00:00", + "source_timezone": "UTC", + "tags": [ + "brexit", + "public_record", + "referendum" + ], + "title": "United Kingdom votes to leave the European Union", + "vintage_id": "operator-public-record-2016-06-24-v1" + }, + { + "affected_currencies": [ + "EUR", + "GBP", + "USD" + ], + "affected_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "ambiguity_reason": "The retained record conditions the WHO declaration day and does not claim a single market-impact instant.", + "available_at": "2020-03-12T00:00:00+00:00", + "canonical_key": "operator.shock.who-pandemic-characterization.2020-03-11", + "confidence": 1.0, + "first_known_at": "2020-03-12T00:00:00+00:00", + "kind": "unscheduled_shock", + "limitations": [ + "The declaration-day window is distinct from the broader pandemic market regime; ex-ante availability is delayed until the following UTC day.", + "Public record: World Health Organization opening remarks, https://www.who.int/director-general/speeches/detail/who-director-general-s-opening-remarks-at-the-media-briefing-on-covid-19---11-march-2020" + ], + "post_event_ns": 86400000000000, + "pre_event_ns": 0, + "precision": "window_only", + "revision_sequence": 0, + "source_event_time": "2020-03-11T00:00:00+00:00", + "source_timezone": "UTC", + "tags": [ + "covid-19", + "pandemic", + "public_record" + ], + "title": "WHO characterizes COVID-19 as a pandemic", + "vintage_id": "operator-public-record-2020-03-11-v1" + }, + { + "affected_currencies": [ + "EUR", + "GBP", + "USD" + ], + "affected_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "ambiguity_reason": "The retained record conditions the invasion day and does not claim one exact market-impact instant.", + "available_at": "2022-02-25T00:00:00+00:00", + "canonical_key": "operator.shock.russia-invasion-ukraine.2022-02-24", + "confidence": 1.0, + "first_known_at": "2022-02-25T00:00:00+00:00", + "kind": "unscheduled_shock", + "limitations": [ + "The full-day window does not infer the timing or causality of every market response; ex-ante availability is delayed until the following UTC day.", + "Public record: United Nations General Assembly ES-11 record, https://www.un.org/en/ga/emergency-special-session-ukraine/" + ], + "post_event_ns": 86400000000000, + "pre_event_ns": 0, + "precision": "window_only", + "revision_sequence": 0, + "source_event_time": "2022-02-24T00:00:00+00:00", + "source_timezone": "UTC", + "tags": [ + "geopolitical", + "public_record", + "war" + ], + "title": "Russian Federation invasion of Ukraine", + "vintage_id": "operator-public-record-2022-02-24-v1" + } + ], + "limitations": [ + "This catalog is deliberately selective; an absent date is not a no-shock label.", + "Events are public-record conditioning windows, not causal market labels or exact historical tick reconstructions.", + "Upstream citations establish factual provenance; upstream reuse terms remain authoritative." + ], + "schema_version": "histdatacom.operator-market-context-catalog.v1" +} diff --git a/src/histdatacom/market_context/contracts.py b/src/histdatacom/market_context/contracts.py new file mode 100644 index 00000000..10be7006 --- /dev/null +++ b/src/histdatacom/market_context/contracts.py @@ -0,0 +1,1695 @@ +"""Versioned point-in-time market-context contracts and bounded queries. + +This module keeps macro, central-bank, news, and calendar context outside the +row-aligned analytical surface. Context is stored once as an immutable, +provenance-rich timeline and joined to bounded reconstruction windows through +small query sidecars. Version-one contracts never fetch or license a source; +adapters normalize operator-approved evidence into these contracts. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Protocol, + TypeVar, + cast, + runtime_checkable, +) +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from histdatacom.data_quality.calendar import ( + calendar_policy_metadata, + classify_histdata_timestamp, +) +from histdatacom.data_quality.calendar_profiles import HistDataCalendarProfile +from histdatacom.runtime_contracts import JSONValue + +if TYPE_CHECKING: + from histdatacom.synthetic.information import ( + InformationMode, + InformationSplitKind, + ReconstructionInformationInputV1, + ) + from histdatacom.synthetic.streaming import ReconstructionWindowV1 + +MARKET_CONTEXT_SOURCE_SCHEMA_VERSION = "histdatacom.market-context-source.v1" +MARKET_CONTEXT_EVENT_SCHEMA_VERSION = "histdatacom.market-context-event.v1" +MARKET_CONTEXT_TIMELINE_SCHEMA_VERSION = ( + "histdatacom.market-context-timeline.v1" +) +MARKET_CONTEXT_CALENDAR_STATE_SCHEMA_VERSION = ( + "histdatacom.market-context-calendar-state.v1" +) +MARKET_CONTEXT_QUERY_SCHEMA_VERSION = "histdatacom.market-context-query.v1" + +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 +MAX_MARKET_CONTEXT_EVENTS = 4096 +MAX_MARKET_CONTEXT_QUERY_EVENTS = 512 +MAX_MARKET_CONTEXT_ADAPTERS = 64 +MAX_MARKET_CONTEXT_ITEMS = 64 +MAX_MARKET_CONTEXT_TEXT = 1024 +MAX_MARKET_CONTEXT_METADATA_BYTES = 16_384 +MAX_MARKET_CONTEXT_TIMELINE_BYTES = 16_777_216 + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_CANONICAL_KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._:-]*$") +_SYMBOL_RE = re.compile(r"^[A-Z0-9._:-]{3,32}$") +_CURRENCY_RE = re.compile(r"^[A-Z]{3}$") +_EnumT = TypeVar("_EnumT", bound=Enum) + + +def canonical_contract_json(value: Any) -> str: + """Encode contracts without importing the synthetic package facade. + + Importing ``histdatacom.synthetic.contracts`` executes the synthetic + package ``__init__`` first. That facade imports broker fingerprint code, + which imports this module and creates a market-context-first circular + import. Keeping the shared byte-for-byte canonical encoding local makes + this domain independently importable while preserving every existing ID. + """ + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + + +class MarketContextKind(str, Enum): + """Stable semantic kinds for external market-context records.""" + + MACRO_RELEASE = "macro_release" + CENTRAL_BANK_DECISION = "central_bank_decision" + POLICY_RATE_CHANGE = "policy_rate_change" + CENTRAL_BANK_STATEMENT = "central_bank_statement" + SCHEDULED_COMMUNICATION = "scheduled_communication" + UNSCHEDULED_SHOCK = "unscheduled_shock" + NEWS_WINDOW = "news_window" + + @classmethod + def from_value( + cls, value: str | "MarketContextKind" + ) -> "MarketContextKind": + """Return a strict normalized context kind.""" + return _enum_value(cls, value, "market context kind") + + +class MarketContextPrecision(str, Enum): + """How precisely the event time and semantic boundary are known.""" + + EXACT = "exact" + APPROXIMATE = "approximate" + WINDOW_ONLY = "window_only" + + @classmethod + def from_value( + cls, value: str | "MarketContextPrecision" + ) -> "MarketContextPrecision": + """Return a strict normalized precision value.""" + return _enum_value(cls, value, "market context precision") + + +class MarketContextView(str, Enum): + """Whether a query exposes all vintages or only point-in-time knowledge.""" + + EX_POST = "ex_post" + EX_ANTE = "ex_ante" + + @classmethod + def from_value( + cls, value: str | "MarketContextView" + ) -> "MarketContextView": + """Return a strict normalized query view.""" + return _enum_value(cls, value, "market context view") + + +class MarketContextQueryStatus(str, Enum): + """Whether an event query matched explicit context.""" + + MATCHED = "matched" + MISSING = "missing" + + @classmethod + def from_value( + cls, value: str | "MarketContextQueryStatus" + ) -> "MarketContextQueryStatus": + """Return a strict normalized query status.""" + return _enum_value(cls, value, "market context query status") + + +class MarketContextMissingReason(str, Enum): + """Stable explanations for an empty context query.""" + + NO_MATCHING_EVENT = "no_matching_event" + NOT_AVAILABLE_AS_OF = "not_available_as_of" + OUTSIDE_TIMELINE_COVERAGE = "outside_timeline_coverage" + TIMELINE_INCOMPLETE = "timeline_incomplete" + + @classmethod + def from_value( + cls, value: str | "MarketContextMissingReason" + ) -> "MarketContextMissingReason": + """Return a strict normalized missing-context reason.""" + return _enum_value(cls, value, "market context missing reason") + + +class MarketContextQueryLimitError(ValueError): + """Raised when a bounded context query would retain too many events.""" + + +@dataclass(frozen=True, slots=True) +class MarketContextSourceV1: + """Immutable provenance and redistribution policy for one source object.""" + + name: str + source_version: str + retrieved_at_ns: int + content_sha256: str + adapter_name: str + adapter_version: str + license_name: str + redistribution_allowed: bool + redistribution_constraints: tuple[str, ...] + limitations: tuple[str, ...] + source_uri: str | None = None + metadata: Mapping[str, JSONValue] | None = None + source_id: str = "" + schema_version: str = MARKET_CONTEXT_SOURCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_SOURCE_SCHEMA_VERSION: + raise ValueError("unsupported market context source schema") + for name in ( + "name", + "source_version", + "adapter_name", + "adapter_version", + "license_name", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "retrieved_at_ns", + _bounded_int64(self.retrieved_at_ns, "retrieved_at_ns"), + ) + object.__setattr__( + self, + "content_sha256", + _required_sha256(self.content_sha256, "content_sha256"), + ) + if not isinstance(self.redistribution_allowed, bool): + raise ValueError("redistribution_allowed must be a boolean") + constraints = _bounded_text_tuple( + self.redistribution_constraints, + "redistribution_constraints", + ) + if not self.redistribution_allowed and not constraints: + raise ValueError( + "non-redistributable sources require explicit constraints" + ) + limitations = _bounded_text_tuple(self.limitations, "limitations") + if not limitations: + raise ValueError("market context sources require limitations") + object.__setattr__(self, "redistribution_constraints", constraints) + object.__setattr__(self, "limitations", limitations) + object.__setattr__(self, "source_uri", _optional_text(self.source_uri)) + metadata = dict(self.metadata or {}) + _validate_json_value(metadata, "source.metadata") + _ensure_payload_size(metadata, MAX_MARKET_CONTEXT_METADATA_BYTES) + object.__setattr__(self, "metadata", metadata) + expected = _stable_id("market-context-source", self.identity_payload()) + supplied = _optional_text(self.source_id) + if supplied is not None and supplied != expected: + raise ValueError("source_id does not match deterministic identity") + object.__setattr__(self, "source_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the complete deterministic source identity payload.""" + return { + "schema_version": self.schema_version, + "name": self.name, + "source_version": self.source_version, + "retrieved_at_ns": self.retrieved_at_ns, + "content_sha256": self.content_sha256, + "adapter_name": self.adapter_name, + "adapter_version": self.adapter_version, + "license_name": self.license_name, + "redistribution_allowed": self.redistribution_allowed, + "redistribution_constraints": list(self.redistribution_constraints), + "limitations": list(self.limitations), + "source_uri": self.source_uri, + "metadata": dict(self.metadata or {}), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible source metadata.""" + return {**self.identity_payload(), "source_id": self.source_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "MarketContextSourceV1": + """Restore and verify one source contract.""" + _require_schema(data, MARKET_CONTEXT_SOURCE_SCHEMA_VERSION) + return cls( + name=str(data.get("name", "")), + source_version=str(data.get("source_version", "")), + retrieved_at_ns=cast(int, data.get("retrieved_at_ns")), + content_sha256=str(data.get("content_sha256", "")), + adapter_name=str(data.get("adapter_name", "")), + adapter_version=str(data.get("adapter_version", "")), + license_name=str(data.get("license_name", "")), + redistribution_allowed=_strict_bool( + data.get("redistribution_allowed"), + "redistribution_allowed", + ), + redistribution_constraints=_string_tuple( + data.get("redistribution_constraints") + ), + limitations=_string_tuple(data.get("limitations")), + source_uri=_optional_text(data.get("source_uri")), + metadata=_mapping(data.get("metadata")), + source_id=str(data.get("source_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "MarketContextSourceV1": + """Restore a source from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class MarketContextEventV1: + """One immutable event vintage with explicit knowledge-time semantics.""" + + canonical_key: str + kind: MarketContextKind + title: str + source: MarketContextSourceV1 + source_event_time: str + source_timezone: str + event_time_ns: int + first_known_at_ns: int + available_at_ns: int + pre_event_ns: int + post_event_ns: int + affected_currencies: tuple[str, ...] + affected_symbols: tuple[str, ...] + confidence: float + precision: MarketContextPrecision + limitations: tuple[str, ...] + vintage_id: str + revision_sequence: int = 0 + supersedes_event_id: str | None = None + source_time_fold: int | None = None + ambiguity_reason: str | None = None + expected_value: float | None = None + actual_value: float | None = None + previous_value: float | None = None + revised_previous_value: float | None = None + surprise_value: float | None = None + value_unit: str | None = None + content_sha256: str | None = None + tags: tuple[str, ...] = () + event_id: str = "" + schema_version: str = MARKET_CONTEXT_EVENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_EVENT_SCHEMA_VERSION: + raise ValueError("unsupported market context event schema") + if not isinstance(self.source, MarketContextSourceV1): + raise ValueError("source must be a market context source") + object.__setattr__( + self, + "canonical_key", + _canonical_key(self.canonical_key), + ) + object.__setattr__( + self, "kind", MarketContextKind.from_value(self.kind) + ) + object.__setattr__(self, "title", _required_text(self.title)) + object.__setattr__( + self, "source_event_time", _required_text(self.source_event_time) + ) + object.__setattr__( + self, "source_timezone", _required_text(self.source_timezone) + ) + fold = _optional_fold(self.source_time_fold) + object.__setattr__(self, "source_time_fold", fold) + event_time = _bounded_int64(self.event_time_ns, "event_time_ns") + normalized_time = normalize_market_context_datetime( + self.source_event_time, + self.source_timezone, + fold=fold, + ) + if normalized_time != event_time: + raise ValueError( + "event_time_ns does not match normalized source event time" + ) + object.__setattr__(self, "event_time_ns", event_time) + first_known = _bounded_int64( + self.first_known_at_ns, "first_known_at_ns" + ) + available = _bounded_int64(self.available_at_ns, "available_at_ns") + if first_known > available: + raise ValueError( + "first_known_at_ns must not follow available_at_ns" + ) + if self.source.retrieved_at_ns < available: + raise ValueError( + "source retrieval cannot precede record availability" + ) + object.__setattr__(self, "first_known_at_ns", first_known) + object.__setattr__(self, "available_at_ns", available) + pre = _nonnegative_int64(self.pre_event_ns, "pre_event_ns") + post = _positive_int64(self.post_event_ns, "post_event_ns") + _bounded_int64(event_time - pre, "window_start_ns") + _bounded_int64(event_time + post, "window_end_ns") + object.__setattr__(self, "pre_event_ns", pre) + object.__setattr__(self, "post_event_ns", post) + currencies = _normalized_currencies(self.affected_currencies) + symbols = _normalized_symbols(self.affected_symbols) + if not currencies and not symbols: + raise ValueError( + "market context events require an affected currency or symbol" + ) + object.__setattr__(self, "affected_currencies", currencies) + object.__setattr__(self, "affected_symbols", symbols) + confidence = _finite_float(self.confidence, "confidence") + if not 0.0 <= confidence <= 1.0: + raise ValueError("confidence must be between zero and one") + object.__setattr__(self, "confidence", confidence) + precision = MarketContextPrecision.from_value(self.precision) + object.__setattr__(self, "precision", precision) + ambiguity = _optional_text(self.ambiguity_reason) + if self.kind in { + MarketContextKind.UNSCHEDULED_SHOCK, + MarketContextKind.NEWS_WINDOW, + }: + if precision is MarketContextPrecision.EXACT or ambiguity is None: + raise ValueError( + "unscheduled/news context requires non-exact precision " + "and an ambiguity reason" + ) + object.__setattr__(self, "ambiguity_reason", ambiguity) + limitations = _bounded_text_tuple(self.limitations, "limitations") + if not limitations: + raise ValueError("market context events require limitations") + object.__setattr__(self, "limitations", limitations) + object.__setattr__(self, "vintage_id", _required_text(self.vintage_id)) + revision = _nonnegative_int(self.revision_sequence, "revision_sequence") + supersedes = _optional_text(self.supersedes_event_id) + if revision == 0 and supersedes is not None: + raise ValueError("an initial event cannot supersede another event") + if revision > 0 and supersedes is None: + raise ValueError("a revised event requires supersedes_event_id") + object.__setattr__(self, "revision_sequence", revision) + object.__setattr__(self, "supersedes_event_id", supersedes) + for name in ( + "expected_value", + "actual_value", + "previous_value", + "revised_previous_value", + "surprise_value", + ): + object.__setattr__( + self, name, _optional_finite(getattr(self, name), name) + ) + if self.expected_value is not None and self.actual_value is not None: + expected_surprise = self.actual_value - self.expected_value + if self.surprise_value is None: + object.__setattr__(self, "surprise_value", expected_surprise) + elif not math.isclose( + self.surprise_value, + expected_surprise, + rel_tol=1e-12, + abs_tol=1e-12, + ): + raise ValueError( + "surprise_value must equal actual minus expected" + ) + elif self.surprise_value is not None: + raise ValueError( + "surprise_value requires expected_value and actual_value" + ) + object.__setattr__(self, "value_unit", _optional_text(self.value_unit)) + content_hash = _optional_text(self.content_sha256) + if content_hash is not None: + content_hash = _required_sha256(content_hash, "content_sha256") + object.__setattr__(self, "content_sha256", content_hash) + object.__setattr__( + self, + "tags", + tuple(sorted(_bounded_text_tuple(self.tags, "tags"))), + ) + expected = _stable_id("market-context-event", self.identity_payload()) + supplied = _optional_text(self.event_id) + if supplied is not None and supplied != expected: + raise ValueError("event_id does not match deterministic identity") + object.__setattr__(self, "event_id", expected) + + @property + def window_start_ns(self) -> int: + """Return the inclusive conditioned-window start.""" + return self.event_time_ns - self.pre_event_ns + + @property + def window_end_ns(self) -> int: + """Return the exclusive conditioned-window end.""" + return self.event_time_ns + self.post_event_ns + + def overlaps(self, start_ns: int, end_ns: int) -> bool: + """Return whether this event's conditioned window overlaps a query.""" + start = _bounded_int64(start_ns, "start_ns") + end = _bounded_int64(end_ns, "end_ns") + if end <= start: + raise ValueError("query end_ns must be greater than start_ns") + return self.window_start_ns < end and self.window_end_ns > start + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the complete deterministic event identity payload.""" + return { + "schema_version": self.schema_version, + "canonical_key": self.canonical_key, + "kind": self.kind.value, + "title": self.title, + "source": self.source.to_dict(), + "source_event_time": self.source_event_time, + "source_timezone": self.source_timezone, + "source_time_fold": self.source_time_fold, + "event_time_ns": self.event_time_ns, + "first_known_at_ns": self.first_known_at_ns, + "available_at_ns": self.available_at_ns, + "pre_event_ns": self.pre_event_ns, + "post_event_ns": self.post_event_ns, + "window_start_ns": self.window_start_ns, + "window_end_ns": self.window_end_ns, + "window_semantics": "[event-pre,event+post)", + "affected_currencies": list(self.affected_currencies), + "affected_symbols": list(self.affected_symbols), + "confidence": self.confidence, + "precision": self.precision.value, + "ambiguity_reason": self.ambiguity_reason, + "limitations": list(self.limitations), + "vintage_id": self.vintage_id, + "revision_sequence": self.revision_sequence, + "supersedes_event_id": self.supersedes_event_id, + "expected_value": self.expected_value, + "actual_value": self.actual_value, + "previous_value": self.previous_value, + "revised_previous_value": self.revised_previous_value, + "surprise_value": self.surprise_value, + "value_unit": self.value_unit, + "content_sha256": self.content_sha256, + "tags": list(self.tags), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible event metadata.""" + return {**self.identity_payload(), "event_id": self.event_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "MarketContextEventV1": + """Restore and verify one market-context event.""" + _require_schema(data, MARKET_CONTEXT_EVENT_SCHEMA_VERSION) + return cls( + canonical_key=str(data.get("canonical_key", "")), + kind=MarketContextKind.from_value(str(data.get("kind", ""))), + title=str(data.get("title", "")), + source=MarketContextSourceV1.from_dict( + _mapping(data.get("source")) + ), + source_event_time=str(data.get("source_event_time", "")), + source_timezone=str(data.get("source_timezone", "")), + source_time_fold=_optional_int(data.get("source_time_fold")), + event_time_ns=cast(int, data.get("event_time_ns")), + first_known_at_ns=cast(int, data.get("first_known_at_ns")), + available_at_ns=cast(int, data.get("available_at_ns")), + pre_event_ns=cast(int, data.get("pre_event_ns", 0)), + post_event_ns=cast(int, data.get("post_event_ns", 1)), + affected_currencies=_string_tuple(data.get("affected_currencies")), + affected_symbols=_string_tuple(data.get("affected_symbols")), + confidence=cast(float, data.get("confidence")), + precision=MarketContextPrecision.from_value( + str(data.get("precision", "")) + ), + ambiguity_reason=_optional_text(data.get("ambiguity_reason")), + limitations=_string_tuple(data.get("limitations")), + vintage_id=str(data.get("vintage_id", "")), + revision_sequence=cast(int, data.get("revision_sequence", 0)), + supersedes_event_id=_optional_text(data.get("supersedes_event_id")), + expected_value=_optional_number(data.get("expected_value")), + actual_value=_optional_number(data.get("actual_value")), + previous_value=_optional_number(data.get("previous_value")), + revised_previous_value=_optional_number( + data.get("revised_previous_value") + ), + surprise_value=_optional_number(data.get("surprise_value")), + value_unit=_optional_text(data.get("value_unit")), + content_sha256=_optional_text(data.get("content_sha256")), + tags=_string_tuple(data.get("tags")), + event_id=str(data.get("event_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "MarketContextEventV1": + """Restore an event from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class MarketContextTimelineV1: + """Immutable ordered context vintages over one declared coverage range.""" + + timeline_version: str + coverage_start_ns: int + coverage_end_ns: int + complete: bool + events: tuple[MarketContextEventV1, ...] + limitations: tuple[str, ...] + timeline_id: str = "" + schema_version: str = MARKET_CONTEXT_TIMELINE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_TIMELINE_SCHEMA_VERSION: + raise ValueError("unsupported market context timeline schema") + object.__setattr__( + self, "timeline_version", _required_text(self.timeline_version) + ) + start = _bounded_int64(self.coverage_start_ns, "coverage_start_ns") + end = _bounded_int64(self.coverage_end_ns, "coverage_end_ns") + if end <= start: + raise ValueError( + "coverage_end_ns must be greater than coverage_start_ns" + ) + if not isinstance(self.complete, bool): + raise ValueError("complete must be a boolean") + values = tuple(self.events) + if len(values) > MAX_MARKET_CONTEXT_EVENTS: + raise ValueError("market context timeline exceeds event limit") + if any(not isinstance(item, MarketContextEventV1) for item in values): + raise ValueError("events must contain market context events") + ordered = tuple( + sorted( + values, + key=lambda item: ( + item.event_time_ns, + item.canonical_key, + item.revision_sequence, + item.event_id, + ), + ) + ) + _validate_timeline_events(ordered, start, end) + limitations = _bounded_text_tuple(self.limitations, "limitations") + if not limitations: + raise ValueError("market context timelines require limitations") + object.__setattr__(self, "coverage_start_ns", start) + object.__setattr__(self, "coverage_end_ns", end) + object.__setattr__(self, "events", ordered) + object.__setattr__(self, "limitations", limitations) + expected = _stable_id( + "market-context-timeline", self.identity_payload() + ) + supplied = _optional_text(self.timeline_id) + if supplied is not None and supplied != expected: + raise ValueError( + "timeline_id does not match deterministic identity" + ) + object.__setattr__(self, "timeline_id", expected) + _ensure_payload_size(self.to_dict(), MAX_MARKET_CONTEXT_TIMELINE_BYTES) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the complete deterministic timeline identity payload.""" + return { + "schema_version": self.schema_version, + "timeline_version": self.timeline_version, + "coverage_start_ns": self.coverage_start_ns, + "coverage_end_ns": self.coverage_end_ns, + "coverage_semantics": "[coverage_start_ns,coverage_end_ns)", + "complete": self.complete, + "events": [item.to_dict() for item in self.events], + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible timeline metadata.""" + return {**self.identity_payload(), "timeline_id": self.timeline_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "MarketContextTimelineV1": + """Restore and verify one immutable timeline.""" + _require_schema(data, MARKET_CONTEXT_TIMELINE_SCHEMA_VERSION) + return cls( + timeline_version=str(data.get("timeline_version", "")), + coverage_start_ns=cast(int, data.get("coverage_start_ns")), + coverage_end_ns=cast(int, data.get("coverage_end_ns")), + complete=_strict_bool(data.get("complete"), "complete"), + events=tuple( + MarketContextEventV1.from_dict(item) + for item in _mapping_sequence(data.get("events")) + ), + limitations=_string_tuple(data.get("limitations")), + timeline_id=str(data.get("timeline_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "MarketContextTimelineV1": + """Restore a timeline from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class MarketContextCalendarStateV1: + """Compact calendar/session classification for one window timestamp.""" + + timestamp_utc_ns: int + session_state: str + clock_sessions: tuple[str, ...] + active_sessions: tuple[str, ...] + overlaps: tuple[str, ...] + special_tags: tuple[str, ...] + holiday_tags: tuple[str, ...] + event_tags: tuple[str, ...] + calendar_tags: tuple[str, ...] + profile_source: str + profile_version: str + profile_complete: bool + limitations: tuple[str, ...] + state_id: str = "" + schema_version: str = MARKET_CONTEXT_CALENDAR_STATE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_CALENDAR_STATE_SCHEMA_VERSION: + raise ValueError("unsupported market context calendar-state schema") + object.__setattr__( + self, + "timestamp_utc_ns", + _bounded_int64(self.timestamp_utc_ns, "timestamp_utc_ns"), + ) + for name in ("session_state", "profile_source", "profile_version"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if not isinstance(self.profile_complete, bool): + raise ValueError("profile_complete must be a boolean") + for name in ( + "clock_sessions", + "active_sessions", + "overlaps", + "special_tags", + "holiday_tags", + "event_tags", + "calendar_tags", + "limitations", + ): + object.__setattr__( + self, + name, + _bounded_text_tuple(getattr(self, name), name), + ) + if not self.limitations: + raise ValueError("calendar states require explicit limitations") + expected = _stable_id( + "market-context-calendar", self.identity_payload() + ) + supplied = _optional_text(self.state_id) + if supplied is not None and supplied != expected: + raise ValueError("state_id does not match deterministic identity") + object.__setattr__(self, "state_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the complete deterministic calendar-state payload.""" + return { + "schema_version": self.schema_version, + "timestamp_utc_ns": self.timestamp_utc_ns, + "session_state": self.session_state, + "clock_sessions": list(self.clock_sessions), + "active_sessions": list(self.active_sessions), + "overlaps": list(self.overlaps), + "special_tags": list(self.special_tags), + "holiday_tags": list(self.holiday_tags), + "event_tags": list(self.event_tags), + "calendar_tags": list(self.calendar_tags), + "profile_source": self.profile_source, + "profile_version": self.profile_version, + "profile_complete": self.profile_complete, + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible calendar metadata.""" + return {**self.identity_payload(), "state_id": self.state_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "MarketContextCalendarStateV1": + """Restore and verify one calendar state.""" + _require_schema(data, MARKET_CONTEXT_CALENDAR_STATE_SCHEMA_VERSION) + return cls( + timestamp_utc_ns=cast(int, data.get("timestamp_utc_ns")), + session_state=str(data.get("session_state", "")), + clock_sessions=_string_tuple(data.get("clock_sessions")), + active_sessions=_string_tuple(data.get("active_sessions")), + overlaps=_string_tuple(data.get("overlaps")), + special_tags=_string_tuple(data.get("special_tags")), + holiday_tags=_string_tuple(data.get("holiday_tags")), + event_tags=_string_tuple(data.get("event_tags")), + calendar_tags=_string_tuple(data.get("calendar_tags")), + profile_source=str(data.get("profile_source", "")), + profile_version=str(data.get("profile_version", "")), + profile_complete=_strict_bool( + data.get("profile_complete"), "profile_complete" + ), + limitations=_string_tuple(data.get("limitations")), + state_id=str(data.get("state_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "MarketContextCalendarStateV1": + """Restore a calendar state from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class MarketContextQueryV1: + """Bounded event/calendar sidecar for one reconstruction window.""" + + timeline_id: str + view: MarketContextView + start_ns: int + end_ns: int + as_of_ns: int | None + events: tuple[MarketContextEventV1, ...] + status: MarketContextQueryStatus + missing_reason: MarketContextMissingReason | None + calendar_state: MarketContextCalendarStateV1 | None + requested_currencies: tuple[str, ...] = () + requested_symbols: tuple[str, ...] = () + requested_kinds: tuple[MarketContextKind, ...] = () + window_id: str | None = None + limitations: tuple[str, ...] = () + query_id: str = "" + schema_version: str = MARKET_CONTEXT_QUERY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_QUERY_SCHEMA_VERSION: + raise ValueError("unsupported market context query schema") + object.__setattr__( + self, "timeline_id", _required_text(self.timeline_id) + ) + object.__setattr__( + self, "view", MarketContextView.from_value(self.view) + ) + start = _bounded_int64(self.start_ns, "start_ns") + end = _bounded_int64(self.end_ns, "end_ns") + if end <= start: + raise ValueError("query end_ns must be greater than start_ns") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + as_of = ( + None + if self.as_of_ns is None + else _bounded_int64(self.as_of_ns, "as_of_ns") + ) + if self.view is MarketContextView.EX_ANTE and as_of is None: + raise ValueError("ex-ante context queries require as_of_ns") + object.__setattr__(self, "as_of_ns", as_of) + events = tuple(self.events) + if len(events) > MAX_MARKET_CONTEXT_QUERY_EVENTS: + raise ValueError("market context query exceeds event limit") + if any(not isinstance(item, MarketContextEventV1) for item in events): + raise ValueError("query events must contain market context events") + object.__setattr__(self, "events", events) + status = MarketContextQueryStatus.from_value(self.status) + reason = self.missing_reason + if reason is not None: + reason = MarketContextMissingReason.from_value(reason) + if events and ( + status is not MarketContextQueryStatus.MATCHED or reason + ): + raise ValueError("matched context cannot carry a missing reason") + if not events and ( + status is not MarketContextQueryStatus.MISSING or reason is None + ): + raise ValueError("empty context requires a missing reason") + object.__setattr__(self, "status", status) + object.__setattr__(self, "missing_reason", reason) + if self.calendar_state is not None and not isinstance( + self.calendar_state, MarketContextCalendarStateV1 + ): + raise ValueError("calendar_state must use the v1 contract") + object.__setattr__( + self, + "requested_currencies", + _normalized_currencies(self.requested_currencies), + ) + object.__setattr__( + self, + "requested_symbols", + _normalized_symbols(self.requested_symbols), + ) + kinds = tuple( + sorted( + { + MarketContextKind.from_value(item) + for item in self.requested_kinds + }, + key=lambda item: item.value, + ) + ) + object.__setattr__(self, "requested_kinds", kinds) + object.__setattr__(self, "window_id", _optional_text(self.window_id)) + object.__setattr__( + self, + "limitations", + _bounded_text_tuple(self.limitations, "limitations"), + ) + expected = _stable_id("market-context-query", self.identity_payload()) + supplied = _optional_text(self.query_id) + if supplied is not None and supplied != expected: + raise ValueError("query_id does not match deterministic identity") + object.__setattr__(self, "query_id", expected) + + @property + def information_mode(self) -> InformationMode: + """Return the reconstruction information mode implied by this view.""" + from histdatacom.synthetic.information import ( # pylint: disable=import-outside-toplevel + InformationMode, + ) + + if self.view is MarketContextView.EX_ANTE: + return InformationMode.EX_ANTE_SIMULATION + return InformationMode.EX_POST_RECONSTRUCTION + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the complete deterministic query identity payload.""" + return { + "schema_version": self.schema_version, + "timeline_id": self.timeline_id, + "view": self.view.value, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "interval_semantics": "[start_ns,end_ns)", + "as_of_ns": self.as_of_ns, + "events": [item.to_dict() for item in self.events], + "status": self.status.value, + "missing_reason": ( + self.missing_reason.value if self.missing_reason else None + ), + "calendar_state": ( + self.calendar_state.to_dict() if self.calendar_state else None + ), + "requested_currencies": list(self.requested_currencies), + "requested_symbols": list(self.requested_symbols), + "requested_kinds": [item.value for item in self.requested_kinds], + "window_id": self.window_id, + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible query metadata.""" + return {**self.identity_payload(), "query_id": self.query_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "MarketContextQueryV1": + """Restore and verify one query sidecar.""" + _require_schema(data, MARKET_CONTEXT_QUERY_SCHEMA_VERSION) + reason = data.get("missing_reason") + calendar = data.get("calendar_state") + return cls( + timeline_id=str(data.get("timeline_id", "")), + view=MarketContextView.from_value(str(data.get("view", ""))), + start_ns=cast(int, data.get("start_ns")), + end_ns=cast(int, data.get("end_ns")), + as_of_ns=_optional_int(data.get("as_of_ns")), + events=tuple( + MarketContextEventV1.from_dict(item) + for item in _mapping_sequence(data.get("events")) + ), + status=MarketContextQueryStatus.from_value( + str(data.get("status", "")) + ), + missing_reason=( + MarketContextMissingReason.from_value(str(reason)) + if reason is not None + else None + ), + calendar_state=( + MarketContextCalendarStateV1.from_dict(_mapping(calendar)) + if calendar is not None + else None + ), + requested_currencies=_string_tuple( + data.get("requested_currencies") + ), + requested_symbols=_string_tuple(data.get("requested_symbols")), + requested_kinds=tuple( + MarketContextKind.from_value(str(item)) + for item in _sequence(data.get("requested_kinds")) + ), + window_id=_optional_text(data.get("window_id")), + limitations=_string_tuple(data.get("limitations")), + query_id=str(data.get("query_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "MarketContextQueryV1": + """Restore a query from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@runtime_checkable +class MarketContextSourceAdapterV1(Protocol): + """Shared adapter seam for operator-approved context sources.""" + + @property + def adapter_name(self) -> str: + """Return the immutable adapter family name.""" + + @property + def adapter_version(self) -> str: + """Return the immutable adapter implementation version.""" + + def load_events(self) -> Iterable[MarketContextEventV1]: + """Yield normalized immutable event vintages.""" + + +@dataclass(frozen=True, slots=True) +class StaticMarketContextSourceAdapterV1: + """Deterministic in-memory adapter for fixtures and normalized imports.""" + + adapter_name: str + adapter_version: str + events: tuple[MarketContextEventV1, ...] + + def __post_init__(self) -> None: + object.__setattr__( + self, "adapter_name", _required_text(self.adapter_name) + ) + object.__setattr__( + self, "adapter_version", _required_text(self.adapter_version) + ) + events = tuple(self.events) + if len(events) > MAX_MARKET_CONTEXT_EVENTS: + raise ValueError("static adapter exceeds event limit") + for event in events: + if not isinstance(event, MarketContextEventV1): + raise ValueError("static adapter events must use v1 contracts") + _verify_adapter_event(self, event) + object.__setattr__(self, "events", events) + + def load_events(self) -> Iterable[MarketContextEventV1]: + """Yield the immutable configured event vintages.""" + return iter(self.events) + + +def build_market_context_timeline( + adapters: Sequence[MarketContextSourceAdapterV1], + *, + timeline_version: str, + coverage_start_ns: int, + coverage_end_ns: int, + complete: bool, + limitations: Sequence[str], +) -> MarketContextTimelineV1: + """Collect bounded adapter output into one deterministic timeline.""" + values = tuple(adapters) + if len(values) > MAX_MARKET_CONTEXT_ADAPTERS: + raise ValueError("market context adapter count exceeds v1 limit") + events: list[MarketContextEventV1] = [] + for adapter in values: + if not isinstance(adapter, MarketContextSourceAdapterV1): + raise ValueError("adapter does not implement the v1 context seam") + _required_text(adapter.adapter_name) + _required_text(adapter.adapter_version) + for event in adapter.load_events(): + if not isinstance(event, MarketContextEventV1): + raise ValueError("adapter emitted a non-v1 context event") + _verify_adapter_event(adapter, event) + events.append(event) + if len(events) > MAX_MARKET_CONTEXT_EVENTS: + raise ValueError("market context adapters exceeded event limit") + return MarketContextTimelineV1( + timeline_version=timeline_version, + coverage_start_ns=coverage_start_ns, + coverage_end_ns=coverage_end_ns, + complete=complete, + events=tuple(events), + limitations=tuple(limitations), + ) + + +def normalize_market_context_datetime( + value: str, + timezone_name: str, + *, + fold: int | None = None, +) -> int: + """Normalize one ISO source time to exact UTC nanoseconds. + + Naive timestamps require an IANA timezone. Ambiguous daylight-saving + folds require an explicit ``fold`` value and nonexistent wall times fail + closed. Offset-aware source strings are checked against the named zone. + """ + text = _required_text(value) + zone_name = _required_text(timezone_name) + try: + zone = ZoneInfo(zone_name) + except ZoneInfoNotFoundError as err: + raise ValueError("unsupported source timezone") from err + try: + parsed = datetime.fromisoformat( + text.removesuffix("Z") + ("+00:00" if text.endswith("Z") else "") + ) + except ValueError as err: + raise ValueError("source_event_time must be ISO-8601") from err + fold_value = _optional_fold(fold) + if parsed.tzinfo is not None: + if fold_value is not None: + raise ValueError("source_time_fold is only valid for naive times") + projected = parsed.astimezone(zone) + if ( + projected.replace(tzinfo=None) != parsed.replace(tzinfo=None) + or projected.utcoffset() != parsed.utcoffset() + ): + raise ValueError("source offset does not match source timezone") + aware = parsed + else: + candidates: dict[int, datetime] = {} + for candidate_fold in (0, 1): + candidate = parsed.replace(tzinfo=zone, fold=candidate_fold) + round_trip = ( + candidate.astimezone(timezone.utc) + .astimezone(zone) + .replace(tzinfo=None) + ) + if round_trip == parsed: + offset = candidate.utcoffset() + if offset is not None: + candidates[int(offset.total_seconds())] = candidate + if not candidates: + raise ValueError("source event time is nonexistent in its timezone") + if len(candidates) > 1 and fold_value is None: + raise ValueError( + "ambiguous source event time requires source_time_fold" + ) + if fold_value is None: + aware = next(iter(candidates.values())) + else: + aware = parsed.replace(tzinfo=zone, fold=fold_value) + round_trip = ( + aware.astimezone(timezone.utc) + .astimezone(zone) + .replace(tzinfo=None) + ) + if round_trip != parsed: + raise ValueError( + "source_time_fold does not resolve the wall time" + ) + utc = aware.astimezone(timezone.utc) + epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) + delta = utc - epoch + value_ns = ( + delta.days * 86_400_000_000_000 + + delta.seconds * 1_000_000_000 + + delta.microseconds * 1_000 + ) + return _bounded_int64(value_ns, "normalized event time") + + +def market_context_calendar_state( + timestamp_utc_ns: int, + *, + calendar_profile: HistDataCalendarProfile | None = None, + asset_class: str = "fx", +) -> MarketContextCalendarStateV1: + """Reuse the canonical HistData calendar classifier for one UTC instant.""" + timestamp_ns = _bounded_int64(timestamp_utc_ns, "timestamp_utc_ns") + classification = classify_histdata_timestamp( + timestamp_ns // 1_000_000, + calendar_profile=calendar_profile, + asset_class=asset_class, + ) + policy = calendar_policy_metadata(calendar_profile) + profile = _mapping(policy.get("calendar_profile")) + limitation = str(policy.get("holiday_calendar_limitations") or "").strip() + limitations = ( + limitation + or "Calendar state is a deterministic classifier, not an event corpus.", + ) + return MarketContextCalendarStateV1( + timestamp_utc_ns=timestamp_ns, + session_state=classification.session_state, + clock_sessions=classification.clock_sessions, + active_sessions=classification.active_sessions, + overlaps=classification.overlaps, + special_tags=classification.special_tags, + holiday_tags=classification.holiday_tags, + event_tags=classification.event_tags, + calendar_tags=classification.calendar_tags, + profile_source=str(policy.get("holiday_calendar_source") or "unknown"), + profile_version=str(profile.get("version") or "unversioned"), + profile_complete=bool(policy.get("holiday_calendar_complete")), + limitations=limitations, + ) + + +def query_market_context( + timeline: MarketContextTimelineV1, + *, + start_ns: int, + end_ns: int, + view: MarketContextView, + as_of_ns: int | None = None, + currencies: Sequence[str] = (), + symbols: Sequence[str] = (), + kinds: Sequence[MarketContextKind] = (), + include_calendar: bool = True, + calendar_at_ns: int | None = None, + calendar_profile: HistDataCalendarProfile | None = None, + max_events: int = MAX_MARKET_CONTEXT_QUERY_EVENTS, + window_id: str | None = None, +) -> MarketContextQueryV1: + """Return a bounded ex-post or point-in-time-safe context sidecar.""" + if not isinstance(timeline, MarketContextTimelineV1): + raise ValueError("timeline must use the v1 market-context contract") + start = _bounded_int64(start_ns, "start_ns") + end = _bounded_int64(end_ns, "end_ns") + if end <= start: + raise ValueError("query end_ns must be greater than start_ns") + selected_view = MarketContextView.from_value(view) + as_of = None if as_of_ns is None else _bounded_int64(as_of_ns, "as_of_ns") + if selected_view is MarketContextView.EX_ANTE and as_of is None: + raise ValueError("ex-ante context queries require as_of_ns") + requested_currencies = _normalized_currencies(currencies) + requested_symbols = _normalized_symbols(symbols) + requested_kinds = tuple( + sorted( + {MarketContextKind.from_value(item) for item in kinds}, + key=lambda item: item.value, + ) + ) + limit = _positive_int(max_events, "max_events") + if limit > MAX_MARKET_CONTEXT_QUERY_EVENTS: + raise ValueError("max_events exceeds the v1 query limit") + matched: list[MarketContextEventV1] = [] + hidden_by_availability = False + for event in timeline.events: + if not event.overlaps(start, end): + continue + if requested_currencies or requested_symbols: + currency_match = bool( + set(requested_currencies).intersection( + event.affected_currencies + ) + ) + symbol_match = bool( + set(requested_symbols).intersection(event.affected_symbols) + ) + if not (currency_match or symbol_match): + continue + if requested_kinds and event.kind not in requested_kinds: + continue + if selected_view is MarketContextView.EX_ANTE and ( + event.first_known_at_ns > cast(int, as_of) + or event.available_at_ns > cast(int, as_of) + ): + hidden_by_availability = True + continue + matched.append(event) + if len(matched) > limit: + raise MarketContextQueryLimitError( + "market context query exceeds configured event limit" + ) + if matched: + status = MarketContextQueryStatus.MATCHED + missing_reason = None + else: + status = MarketContextQueryStatus.MISSING + if ( + end <= timeline.coverage_start_ns + or start >= timeline.coverage_end_ns + ): + missing_reason = ( + MarketContextMissingReason.OUTSIDE_TIMELINE_COVERAGE + ) + elif hidden_by_availability: + missing_reason = MarketContextMissingReason.NOT_AVAILABLE_AS_OF + elif not timeline.complete: + missing_reason = MarketContextMissingReason.TIMELINE_INCOMPLETE + else: + missing_reason = MarketContextMissingReason.NO_MATCHING_EVENT + state = None + if include_calendar: + calendar_time = start if calendar_at_ns is None else calendar_at_ns + if not start <= calendar_time < end: + raise ValueError( + "calendar_at_ns must lie inside the query interval" + ) + state = market_context_calendar_state( + calendar_time, + calendar_profile=calendar_profile, + ) + return MarketContextQueryV1( + timeline_id=timeline.timeline_id, + view=selected_view, + start_ns=start, + end_ns=end, + as_of_ns=as_of, + events=tuple(matched), + status=status, + missing_reason=missing_reason, + calendar_state=state, + requested_currencies=requested_currencies, + requested_symbols=requested_symbols, + requested_kinds=requested_kinds, + window_id=window_id, + limitations=timeline.limitations, + ) + + +def query_market_context_window( + timeline: MarketContextTimelineV1, + window: ReconstructionWindowV1, + *, + view: MarketContextView, + as_of_ns: int | None = None, + currencies: Sequence[str] = (), + kinds: Sequence[MarketContextKind] = (), + include_calendar: bool = True, + calendar_profile: HistDataCalendarProfile | None = None, + max_events: int = MAX_MARKET_CONTEXT_QUERY_EVENTS, +) -> MarketContextQueryV1: + """Join context to one streaming window without materializing row columns.""" + from histdatacom.synthetic.streaming import ( # pylint: disable=import-outside-toplevel + ReconstructionWindowV1, + ) + + if not isinstance(window, ReconstructionWindowV1): + raise ValueError("window must use the v1 reconstruction contract") + selected_view = MarketContextView.from_value(view) + effective_as_of = as_of_ns + if selected_view is MarketContextView.EX_ANTE and effective_as_of is None: + effective_as_of = window.core_start_ns + return query_market_context( + timeline, + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + view=selected_view, + as_of_ns=effective_as_of, + currencies=tuple(currencies) + _currencies_for_symbols(window.symbols), + symbols=window.symbols, + kinds=kinds, + include_calendar=include_calendar, + calendar_at_ns=window.core_start_ns, + calendar_profile=calendar_profile, + max_events=max_events, + window_id=window.window_id, + ) + + +def market_context_information_inputs( + query: MarketContextQueryV1, + *, + run_id: str, + used_at_ns: int, + split_kind: InformationSplitKind | None = None, +) -> tuple[ReconstructionInformationInputV1, ...]: + """Bind queried event vintages into the existing leakage-audit graph.""" + from histdatacom.synthetic.information import ( # pylint: disable=import-outside-toplevel + InformationInputKind, + InformationScope, + InformationStage, + ReconstructionInformationInputV1, + ) + + if not isinstance(query, MarketContextQueryV1): + raise ValueError("query must use the v1 market-context contract") + run = _required_text(run_id) + used = _bounded_int64(used_at_ns, "used_at_ns") + if query.view is MarketContextView.EX_ANTE and query.as_of_ns != used: + raise ValueError( + "ex-ante information use must equal the query as_of_ns" + ) + by_event_id: dict[str, ReconstructionInformationInputV1] = {} + results: list[ReconstructionInformationInputV1] = [] + for event in sorted( + query.events, + key=lambda item: (item.revision_sequence, item.event_id), + ): + supersedes_input_id = None + if event.supersedes_event_id is not None: + predecessor = by_event_id.get(event.supersedes_event_id) + if predecessor is None: + raise ValueError("query omits a required revision predecessor") + supersedes_input_id = predecessor.input_id + # The information graph records when this vintage became a usable + # fact. The target market-event time remains immutable in the context + # event itself. This distinction lets a published schedule be used + # before the scheduled release without mislabeling it as realized + # future information. + information_event_time = event.available_at_ns + allowed_lookahead = 0 + if query.view is MarketContextView.EX_POST: + allowed_lookahead = max(0, information_event_time - used) + value = ReconstructionInformationInputV1( + run_id=run, + artifact_id=f"{query.timeline_id}:{event.event_id}", + information_mode=query.information_mode, + input_kind=InformationInputKind.EXTERNAL, + stage=InformationStage.NEWS_CONTEXT, + scope=( + InformationScope.REVISION + if event.revision_sequence + else InformationScope.POINT_IN_TIME + ), + event_time_ns=information_event_time, + available_at_ns=event.available_at_ns, + used_at_ns=used, + observation_start_ns=information_event_time, + observation_end_ns=information_event_time, + vintage_id=event.vintage_id, + reason=( + "point-in-time market context " + f"{event.canonical_key} revision {event.revision_sequence}; " + f"target_event_time_ns={event.event_time_ns}" + ), + revision_sequence=event.revision_sequence, + supersedes_input_id=supersedes_input_id, + allowed_lookahead_ns=allowed_lookahead, + split_kind=split_kind, + ) + by_event_id[event.event_id] = value + results.append(value) + return tuple(results) + + +def _validate_timeline_events( + events: Sequence[MarketContextEventV1], + coverage_start_ns: int, + coverage_end_ns: int, +) -> None: + ids: set[str] = set() + logical: set[tuple[str, int]] = set() + by_id: dict[str, MarketContextEventV1] = {} + for event in events: + if not coverage_start_ns <= event.event_time_ns < coverage_end_ns: + raise ValueError("event time lies outside timeline coverage") + if event.event_id in ids: + raise ValueError("duplicate market context event_id") + key = (event.canonical_key, event.revision_sequence) + if key in logical: + raise ValueError("duplicate logical market context event") + ids.add(event.event_id) + logical.add(key) + by_id[event.event_id] = event + for event in events: + if event.revision_sequence == 0: + continue + predecessor = by_id.get(cast(str, event.supersedes_event_id)) + if predecessor is None: + raise ValueError( + "revised event predecessor is absent from timeline" + ) + if predecessor.canonical_key != event.canonical_key: + raise ValueError("revision changes canonical event identity") + if predecessor.revision_sequence + 1 != event.revision_sequence: + raise ValueError("revision sequence must advance by exactly one") + if predecessor.event_time_ns != event.event_time_ns: + raise ValueError("revision changes semantic event time") + if predecessor.first_known_at_ns != event.first_known_at_ns: + raise ValueError("revision changes first-known time") + if predecessor.available_at_ns >= event.available_at_ns: + raise ValueError("revision availability must advance") + + +def _verify_adapter_event( + adapter: MarketContextSourceAdapterV1, + event: MarketContextEventV1, +) -> None: + if event.source.adapter_name != adapter.adapter_name: + raise ValueError("event source adapter_name does not match adapter") + if event.source.adapter_version != adapter.adapter_version: + raise ValueError("event source adapter_version does not match adapter") + + +def _enum_value( + enum_type: type[_EnumT], + value: str | _EnumT, + label: str, +) -> _EnumT: + if isinstance(value, enum_type): + return value + try: + return enum_type(str(value).strip().lower()) + except ValueError as err: + raise ValueError(f"unsupported {label}") from err + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = str(canonical_contract_json(payload)).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _canonical_key(value: Any) -> str: + normalized = _required_text(value).lower() + if not _CANONICAL_KEY_RE.fullmatch(normalized): + raise ValueError("canonical_key contains unsupported characters") + return normalized + + +def _required_text(value: Any) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("required text is empty") + if len(text) > MAX_MARKET_CONTEXT_TEXT: + raise ValueError("text exceeds market-context limit") + return text + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return _required_text(text) if text else None + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a boolean") + return value + + +def _strict_int(value: Any, name: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{name} must be an integer") + return value + + +def _optional_int(value: Any) -> int | None: + if value is None: + return None + return _strict_int(value, "optional integer") + + +def _optional_fold(value: Any) -> int | None: + if value is None: + return None + fold = _strict_int(value, "source_time_fold") + if fold not in (0, 1): + raise ValueError("source_time_fold must be zero or one") + return fold + + +def _bounded_int64(value: Any, name: str) -> int: + result = _strict_int(value, name) + if not INT64_MIN <= result <= INT64_MAX: + raise ValueError(f"{name} exceeds signed int64") + return result + + +def _nonnegative_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result < 0: + raise ValueError(f"{name} must be nonnegative") + return result + + +def _positive_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _nonnegative_int64(value: Any, name: str) -> int: + result = _bounded_int64(value, name) + if result < 0: + raise ValueError(f"{name} must be nonnegative") + return result + + +def _positive_int64(value: Any, name: str) -> int: + result = _bounded_int64(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be numeric") + try: + result = float(value) + except (TypeError, ValueError) as err: + raise ValueError(f"{name} must be numeric") from err + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _optional_finite(value: Any, name: str) -> float | None: + if value is None: + return None + return _finite_float(value, name) + + +def _optional_number(value: Any) -> float | None: + return None if value is None else _finite_float(value, "optional number") + + +def _bounded_text_tuple(values: Iterable[Any], name: str) -> tuple[str, ...]: + result = tuple(dict.fromkeys(_required_text(value) for value in values)) + if len(result) > MAX_MARKET_CONTEXT_ITEMS: + raise ValueError(f"{name} exceeds market-context item limit") + return result + + +def _normalized_currencies(values: Iterable[Any]) -> tuple[str, ...]: + result = tuple(sorted({_required_text(value).upper() for value in values})) + if len(result) > MAX_MARKET_CONTEXT_ITEMS: + raise ValueError("affected currencies exceed v1 limit") + if any(not _CURRENCY_RE.fullmatch(value) for value in result): + raise ValueError("affected currency must be a three-letter code") + return result + + +def _normalized_symbols(values: Iterable[Any]) -> tuple[str, ...]: + result = tuple(sorted({_required_text(value).upper() for value in values})) + if len(result) > MAX_MARKET_CONTEXT_ITEMS: + raise ValueError("affected symbols exceed v1 limit") + if any(not _SYMBOL_RE.fullmatch(value) for value in result): + raise ValueError("affected symbol contains unsupported characters") + return result + + +def _currencies_for_symbols(values: Iterable[str]) -> tuple[str, ...]: + currencies: set[str] = set() + for value in values: + symbol = str(value).strip().upper() + if len(symbol) == 6 and symbol.isalpha(): + currencies.update((symbol[:3], symbol[3:])) + return tuple(sorted(currencies)) + + +def _required_sha256(value: Any, name: str) -> str: + text = _required_text(value).lower() + if not _SHA256_RE.fullmatch(text): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return text + + +def _validate_json_value(value: Any, path: str) -> None: + if value is None or isinstance(value, (str, int, bool)): + return + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError(f"{path} contains a non-finite float") + return + if isinstance(value, list): + for index, item in enumerate(value): + _validate_json_value(item, f"{path}[{index}]") + return + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"{path} contains a non-string key") + _validate_json_value(item, f"{path}.{key}") + return + raise ValueError(f"{path} is not JSON-compatible") + + +def _ensure_payload_size(value: Mapping[str, JSONValue], maximum: int) -> None: + encoded = str(canonical_contract_json(value)).encode("utf-8") + if len(encoded) > maximum: + raise ValueError("market-context payload exceeds v1 size limit") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError("unsupported market-context schema version") + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected mapping") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if value is None: + return () + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError("expected sequence") + return value + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + return tuple(_mapping(item) for item in _sequence(value)) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + try: + value = json.loads(text) + except json.JSONDecodeError as err: + raise ValueError("invalid market-context JSON") from err + return _mapping(value) diff --git a/src/histdatacom/market_context/corpus.py b/src/histdatacom/market_context/corpus.py new file mode 100644 index 00000000..8bc160e3 --- /dev/null +++ b/src/histdatacom/market_context/corpus.py @@ -0,0 +1,2960 @@ +"""Production market-context sources and immutable corpus artifacts. + +The contracts in :mod:`histdatacom.market_context.contracts` define the +point-in-time event and query boundary. This module supplies legally usable +official-source adapters, content-addressed acquisition evidence, deterministic +replay, coverage diagnostics, and installed-code loading seams for carving and +benchmark work. + +Source snapshots are intentionally retained outside row-aligned market data. +Reconstruction windows consume the existing bounded ``MarketContextQueryV1`` +sidecar, never repeated macro/news columns on every tick. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import re +import tempfile +import time +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, replace +from datetime import date, datetime, timedelta, timezone +from html.parser import HTMLParser +from pathlib import Path +from typing import Any, cast +from urllib.parse import urlencode + +import requests + +from histdatacom.market_context.contracts import ( + MAX_MARKET_CONTEXT_EVENTS, + MarketContextEventV1, + MarketContextKind, + MarketContextPrecision, + MarketContextQueryV1, + MarketContextSourceV1, + MarketContextTimelineV1, + MarketContextView, + normalize_market_context_datetime, + query_market_context, +) +from histdatacom.resource_usage import peak_rss_bytes +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.market_context.contracts import canonical_contract_json + +MARKET_CONTEXT_CORPUS_SCHEMA_VERSION = "histdatacom.market-context-corpus.v1" +MARKET_CONTEXT_SOURCE_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.market-context-source-evidence.v1" +) +MARKET_CONTEXT_COVERAGE_SCHEMA_VERSION = ( + "histdatacom.market-context-coverage.v1" +) +MARKET_CONTEXT_PREFLIGHT_SCHEMA_VERSION = ( + "histdatacom.market-context-preflight.v1" +) +OPERATOR_MARKET_CONTEXT_CATALOG_SCHEMA_VERSION = ( + "histdatacom.operator-market-context-catalog.v1" +) + +ONS_RELEASES_URI = "https://api.beta.ons.gov.uk/v1/search/releases" +ECB_POLICY_RATE_SERIES_KEY = "FM.D.U2.EUR.4F.KR.MRR_RT.LEV" +ECB_POLICY_RATE_URI = ( + "https://data-api.ecb.europa.eu/service/data/" + "FM/D.U2.EUR.4F.KR.MRR_RT.LEV" +) +BOE_BANK_RATE_URI = ( + "https://www.bankofengland.co.uk/boeapps/database/" "Bank-Rate.asp?hl=en-GB" +) +FED_FOMC_CALENDAR_URI = ( + "https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm" +) +FED_FOMC_HISTORICAL_URI = ( + "https://www.federalreserve.gov/monetarypolicy/fomchistorical{year}.htm" +) + +DEFAULT_ONS_QUERIES = ( + "consumer price inflation", + "gross domestic product", + "labour market", + "retail sales", + "public sector finances", +) +DEFAULT_MARKET_CONTEXT_SOURCES = ( + "ons", + "ecb", + "boe", + "fed", + "operator", +) +DEFAULT_USER_AGENT = ( + "histdatacom-market-context/2.1.0 " + "(+https://github.com/dmidlo/histdata.com-tools)" +) + +DAY_NS = 86_400_000_000_000 +HOUR_NS = 3_600_000_000_000 +MAX_CORPUS_BYTES = 64 * 1024 * 1024 +MAX_SOURCE_EVIDENCE = 64 +MAX_COVERAGE_SLICES = 128 +MAX_DIAGNOSTICS = 256 +_SOURCE_KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._:-]{0,127}$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_ONS_TITLE_PATTERNS = ( + re.compile(r"^consumer price inflation, uk:", re.IGNORECASE), + re.compile( + r"^gdp (?:monthly|first quarterly|quarterly national accounts)", + re.IGNORECASE, + ), + re.compile(r"^labour market overview, uk:", re.IGNORECASE), + re.compile(r"^retail sales, great britain:", re.IGNORECASE), + re.compile(r"^public sector finances, uk:", re.IGNORECASE), +) + + +@dataclass(frozen=True, slots=True) +class MarketContextFetchProfileV1: + """Bounded official-source acquisition and corpus-build policy.""" + + start_date: str + end_date: str + sources: tuple[str, ...] = DEFAULT_MARKET_CONTEXT_SOURCES + ons_queries: tuple[str, ...] = DEFAULT_ONS_QUERIES + timeout_seconds: float = 30.0 + max_response_bytes: int = 16 * 1024 * 1024 + max_total_source_bytes: int = 64 * 1024 * 1024 + max_ons_pages_per_query: int = 8 + max_events: int = MAX_MARKET_CONTEXT_EVENTS + max_runtime_seconds: float = 300.0 + + def __post_init__(self) -> None: + start = _parse_date(self.start_date, "start_date") + end = _parse_date(self.end_date, "end_date") + if end < start: + raise ValueError("market-context end_date precedes start_date") + sources = tuple(sorted({_source_key(item) for item in self.sources})) + unsupported = set(sources).difference(DEFAULT_MARKET_CONTEXT_SOURCES) + if unsupported: + raise ValueError( + "unsupported market-context source: " + + ", ".join(sorted(unsupported)) + ) + queries = tuple( + dict.fromkeys(_required_text(item) for item in self.ons_queries) + ) + if len(queries) > 16: + raise ValueError("ONS query count exceeds sixteen") + if "ons" in sources and not queries: + raise ValueError("ONS acquisition requires at least one query") + _positive_float(self.timeout_seconds, "timeout_seconds") + _bounded_int( + self.max_response_bytes, "max_response_bytes", 1, MAX_CORPUS_BYTES + ) + _bounded_int( + self.max_total_source_bytes, + "max_total_source_bytes", + 1, + 4 * MAX_CORPUS_BYTES, + ) + _bounded_int( + self.max_ons_pages_per_query, "max_ons_pages_per_query", 1, 64 + ) + _bounded_int( + self.max_events, "max_events", 1, MAX_MARKET_CONTEXT_EVENTS + ) + _positive_float(self.max_runtime_seconds, "max_runtime_seconds") + object.__setattr__(self, "start_date", start.isoformat()) + object.__setattr__(self, "end_date", end.isoformat()) + object.__setattr__(self, "sources", sources) + object.__setattr__(self, "ons_queries", queries) + + @property + def coverage_start_ns(self) -> int: + return _date_start_ns(_parse_date(self.start_date, "start_date")) + + @property + def coverage_end_ns(self) -> int: + return _date_start_ns( + _parse_date(self.end_date, "end_date") + timedelta(days=1) + ) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "start_date": self.start_date, + "end_date": self.end_date, + "sources": list(self.sources), + "ons_queries": list(self.ons_queries), + "timeout_seconds": self.timeout_seconds, + "max_response_bytes": self.max_response_bytes, + "max_total_source_bytes": self.max_total_source_bytes, + "max_ons_pages_per_query": self.max_ons_pages_per_query, + "max_events": self.max_events, + "max_runtime_seconds": self.max_runtime_seconds, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "MarketContextFetchProfileV1": + return cls( + start_date=str(data.get("start_date", "")), + end_date=str(data.get("end_date", "")), + sources=_string_tuple(data.get("sources")), + ons_queries=_string_tuple(data.get("ons_queries")), + timeout_seconds=_number(data.get("timeout_seconds")), + max_response_bytes=_strict_int( + data.get("max_response_bytes"), "max_response_bytes" + ), + max_total_source_bytes=_strict_int( + data.get("max_total_source_bytes"), "max_total_source_bytes" + ), + max_ons_pages_per_query=_strict_int( + data.get("max_ons_pages_per_query"), "max_ons_pages_per_query" + ), + max_events=_strict_int(data.get("max_events"), "max_events"), + max_runtime_seconds=_number(data.get("max_runtime_seconds")), + ) + + +@dataclass(slots=True) +class _AcquisitionBudget: + profile: MarketContextFetchProfileV1 + started: float + consumed_bytes: int = 0 + + def check(self) -> None: + if ( + time.perf_counter() - self.started + > self.profile.max_runtime_seconds + ): + raise ValueError( + "market-context acquisition exceeded runtime limit" + ) + if self.consumed_bytes > self.profile.max_total_source_bytes: + raise ValueError( + "market-context acquisition exceeded total source-byte limit" + ) + + @property + def remaining_bytes(self) -> int: + self.check() + return self.profile.max_total_source_bytes - self.consumed_bytes + + @property + def remaining_seconds(self) -> float: + self.check() + return max( + 0.001, + self.profile.max_runtime_seconds + - (time.perf_counter() - self.started), + ) + + def consume(self, size: int) -> None: + self.consumed_bytes += size + self.check() + + +@dataclass(frozen=True, slots=True) +class MarketContextSourceSnapshotV1: + """One bounded source body with exact retrieval and reuse metadata.""" + + source_key: str + source_name: str + source_uri: str + retrieved_at_ns: int + content: bytes + content_type: str + adapter_name: str + adapter_version: str + license_name: str + redistribution_allowed: bool + redistribution_constraints: tuple[str, ...] + limitations: tuple[str, ...] + metadata: Mapping[str, JSONValue] + + def __post_init__(self) -> None: + object.__setattr__(self, "source_key", _source_key(self.source_key)) + for name in ( + "source_name", + "source_uri", + "content_type", + "adapter_name", + "adapter_version", + "license_name", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + _bounded_int(self.retrieved_at_ns, "retrieved_at_ns", 0, 2**63 - 1) + if not isinstance(self.content, bytes) or not self.content: + raise ValueError("source snapshot content must be non-empty bytes") + if len(self.content) > MAX_CORPUS_BYTES: + raise ValueError("source snapshot exceeds corpus byte bound") + if not isinstance(self.redistribution_allowed, bool): + raise ValueError("redistribution_allowed must be a boolean") + constraints = tuple( + _required_text(item) for item in self.redistribution_constraints + ) + limitations = tuple(_required_text(item) for item in self.limitations) + if not limitations: + raise ValueError("source snapshot requires limitations") + if not self.redistribution_allowed and not constraints: + raise ValueError("restricted source snapshot requires constraints") + _validate_json_mapping(self.metadata, "snapshot metadata") + object.__setattr__(self, "redistribution_constraints", constraints) + object.__setattr__(self, "limitations", limitations) + object.__setattr__(self, "metadata", dict(self.metadata)) + + @property + def content_sha256(self) -> str: + return hashlib.sha256(self.content).hexdigest() + + def source_contract(self) -> MarketContextSourceV1: + """Return the contract embedded in every event from this snapshot.""" + return MarketContextSourceV1( + name=self.source_name, + source_version=f"sha256:{self.content_sha256}", + retrieved_at_ns=self.retrieved_at_ns, + content_sha256=self.content_sha256, + adapter_name=self.adapter_name, + adapter_version=self.adapter_version, + license_name=self.license_name, + redistribution_allowed=self.redistribution_allowed, + redistribution_constraints=self.redistribution_constraints, + limitations=self.limitations, + source_uri=self.source_uri, + metadata={**dict(self.metadata), "source_key": self.source_key}, + ) + + +@dataclass(frozen=True, slots=True) +class MarketContextSourceEvidenceV1: + """Publishable evidence for one retained raw source snapshot.""" + + source_key: str + source_name: str + source_uri: str + retrieved_at_ns: int + content_sha256: str + size_bytes: int + content_type: str + adapter_name: str + adapter_version: str + license_name: str + redistribution_allowed: bool + redistribution_constraints: tuple[str, ...] + limitations: tuple[str, ...] + metadata: Mapping[str, JSONValue] + emitted_event_count: int + diagnostics: tuple[str, ...] = () + schema_version: str = MARKET_CONTEXT_SOURCE_EVIDENCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_SOURCE_EVIDENCE_SCHEMA_VERSION: + raise ValueError( + "unsupported market-context source evidence schema" + ) + object.__setattr__(self, "source_key", _source_key(self.source_key)) + for name in ( + "source_name", + "source_uri", + "content_type", + "adapter_name", + "adapter_version", + "license_name", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + _bounded_int(self.retrieved_at_ns, "retrieved_at_ns", 0, 2**63 - 1) + _sha256(self.content_sha256, "content_sha256") + _bounded_int(self.size_bytes, "size_bytes", 1, MAX_CORPUS_BYTES) + _bounded_int( + self.emitted_event_count, + "emitted_event_count", + 0, + MAX_MARKET_CONTEXT_EVENTS, + ) + if not isinstance(self.redistribution_allowed, bool): + raise ValueError("redistribution_allowed must be a boolean") + constraints = tuple( + _required_text(item) for item in self.redistribution_constraints + ) + limitations = tuple(_required_text(item) for item in self.limitations) + if not limitations: + raise ValueError("source evidence requires limitations") + if not self.redistribution_allowed and not constraints: + raise ValueError("restricted source evidence requires constraints") + _validate_json_mapping(self.metadata, "source evidence metadata") + object.__setattr__( + self, + "redistribution_constraints", + constraints, + ) + object.__setattr__(self, "limitations", limitations) + object.__setattr__( + self, + "diagnostics", + tuple( + _required_text(item) + for item in self.diagnostics[:MAX_DIAGNOSTICS] + ), + ) + object.__setattr__(self, "metadata", dict(self.metadata)) + + @classmethod + def from_snapshot( + cls, + snapshot: MarketContextSourceSnapshotV1, + *, + event_count: int, + diagnostics: Sequence[str] = (), + ) -> "MarketContextSourceEvidenceV1": + return cls( + source_key=snapshot.source_key, + source_name=snapshot.source_name, + source_uri=snapshot.source_uri, + retrieved_at_ns=snapshot.retrieved_at_ns, + content_sha256=snapshot.content_sha256, + size_bytes=len(snapshot.content), + content_type=snapshot.content_type, + adapter_name=snapshot.adapter_name, + adapter_version=snapshot.adapter_version, + license_name=snapshot.license_name, + redistribution_allowed=snapshot.redistribution_allowed, + redistribution_constraints=snapshot.redistribution_constraints, + limitations=snapshot.limitations, + metadata=snapshot.metadata, + emitted_event_count=event_count, + diagnostics=tuple(diagnostics), + ) + + def source_contract(self) -> MarketContextSourceV1: + """Return the event provenance contract represented by this evidence.""" + return MarketContextSourceV1( + name=self.source_name, + source_version=f"sha256:{self.content_sha256}", + retrieved_at_ns=self.retrieved_at_ns, + content_sha256=self.content_sha256, + adapter_name=self.adapter_name, + adapter_version=self.adapter_version, + license_name=self.license_name, + redistribution_allowed=self.redistribution_allowed, + redistribution_constraints=self.redistribution_constraints, + limitations=self.limitations, + source_uri=self.source_uri, + metadata={**dict(self.metadata), "source_key": self.source_key}, + ) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "source_key": self.source_key, + "source_name": self.source_name, + "source_uri": self.source_uri, + "retrieved_at_ns": self.retrieved_at_ns, + "content_sha256": self.content_sha256, + "size_bytes": self.size_bytes, + "content_type": self.content_type, + "adapter_name": self.adapter_name, + "adapter_version": self.adapter_version, + "license_name": self.license_name, + "redistribution_allowed": self.redistribution_allowed, + "redistribution_constraints": list(self.redistribution_constraints), + "limitations": list(self.limitations), + "metadata": dict(self.metadata), + "emitted_event_count": self.emitted_event_count, + "diagnostics": list(self.diagnostics), + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "MarketContextSourceEvidenceV1": + return cls( + source_key=str(data.get("source_key", "")), + source_name=str(data.get("source_name", "")), + source_uri=str(data.get("source_uri", "")), + retrieved_at_ns=_strict_int( + data.get("retrieved_at_ns"), "retrieved_at_ns" + ), + content_sha256=str(data.get("content_sha256", "")), + size_bytes=_strict_int(data.get("size_bytes"), "size_bytes"), + content_type=str(data.get("content_type", "")), + adapter_name=str(data.get("adapter_name", "")), + adapter_version=str(data.get("adapter_version", "")), + license_name=str(data.get("license_name", "")), + redistribution_allowed=_strict_bool( + data.get("redistribution_allowed"), "redistribution_allowed" + ), + redistribution_constraints=_string_tuple( + data.get("redistribution_constraints") + ), + limitations=_string_tuple(data.get("limitations")), + metadata=_mapping(data.get("metadata")), + emitted_event_count=_strict_int( + data.get("emitted_event_count"), "emitted_event_count" + ), + diagnostics=_string_tuple(data.get("diagnostics")), + schema_version=str(data.get("schema_version", "")), + ) + + def restore_snapshot(self, content: bytes) -> MarketContextSourceSnapshotV1: + """Restore and hash-check a raw snapshot for deterministic replay.""" + if len(content) != self.size_bytes: + raise ValueError("source snapshot size differs from evidence") + if hashlib.sha256(content).hexdigest() != self.content_sha256: + raise ValueError("source snapshot hash differs from evidence") + return MarketContextSourceSnapshotV1( + source_key=self.source_key, + source_name=self.source_name, + source_uri=self.source_uri, + retrieved_at_ns=self.retrieved_at_ns, + content=content, + content_type=self.content_type, + adapter_name=self.adapter_name, + adapter_version=self.adapter_version, + license_name=self.license_name, + redistribution_allowed=self.redistribution_allowed, + redistribution_constraints=self.redistribution_constraints, + limitations=self.limitations, + metadata=self.metadata, + ) + + +@dataclass(frozen=True, slots=True) +class MarketContextCoverageSliceV1: + """One explicit source/currency/kind support interval.""" + + source_family: str + currency: str + kind: MarketContextKind + coverage_start_ns: int + coverage_end_ns: int + complete: bool + event_count: int + missingness_reason: str + schema_version: str = MARKET_CONTEXT_COVERAGE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_COVERAGE_SCHEMA_VERSION: + raise ValueError("unsupported market-context coverage schema") + object.__setattr__( + self, "source_family", _source_key(self.source_family) + ) + currency = _required_text(self.currency).upper() + if not re.fullmatch(r"[A-Z]{3}", currency): + raise ValueError("coverage currency must use ISO-4217 form") + object.__setattr__(self, "currency", currency) + object.__setattr__( + self, "kind", MarketContextKind.from_value(self.kind) + ) + start = _strict_int(self.coverage_start_ns, "coverage_start_ns") + end = _strict_int(self.coverage_end_ns, "coverage_end_ns") + if end <= start: + raise ValueError("coverage slice end must follow start") + if not isinstance(self.complete, bool): + raise ValueError("coverage complete must be a boolean") + _bounded_int( + self.event_count, "event_count", 0, MAX_MARKET_CONTEXT_EVENTS + ) + object.__setattr__( + self, "missingness_reason", _required_text(self.missingness_reason) + ) + + def supports(self, start_ns: int, end_ns: int) -> bool: + return ( + self.complete + and self.coverage_start_ns <= start_ns + and self.coverage_end_ns >= end_ns + ) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "source_family": self.source_family, + "currency": self.currency, + "kind": self.kind.value, + "coverage_start_ns": self.coverage_start_ns, + "coverage_end_ns": self.coverage_end_ns, + "complete": self.complete, + "event_count": self.event_count, + "missingness_reason": self.missingness_reason, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "MarketContextCoverageSliceV1": + return cls( + source_family=str(data.get("source_family", "")), + currency=str(data.get("currency", "")), + kind=MarketContextKind.from_value(str(data.get("kind", ""))), + coverage_start_ns=_strict_int( + data.get("coverage_start_ns"), "coverage_start_ns" + ), + coverage_end_ns=_strict_int( + data.get("coverage_end_ns"), "coverage_end_ns" + ), + complete=_strict_bool(data.get("complete"), "complete"), + event_count=_strict_int(data.get("event_count"), "event_count"), + missingness_reason=str(data.get("missingness_reason", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class MarketContextCorpusV1: + """Self-contained immutable corpus plus acquisition and coverage evidence.""" + + profile: MarketContextFetchProfileV1 + timeline: MarketContextTimelineV1 + sources: tuple[MarketContextSourceEvidenceV1, ...] + coverage: tuple[MarketContextCoverageSliceV1, ...] + duplicate_event_count: int + counts_by_year_currency_kind: Mapping[str, int] + runtime_seconds: float + peak_memory_bytes: int + limitations: tuple[str, ...] + corpus_id: str = "" + schema_version: str = MARKET_CONTEXT_CORPUS_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_CORPUS_SCHEMA_VERSION: + raise ValueError("unsupported market-context corpus schema") + if not isinstance(self.profile, MarketContextFetchProfileV1): + raise ValueError("corpus requires a v1 fetch profile") + if not isinstance(self.timeline, MarketContextTimelineV1): + raise ValueError("corpus requires a v1 market-context timeline") + sources = tuple(sorted(self.sources, key=lambda item: item.source_key)) + if not sources or len(sources) > MAX_SOURCE_EVIDENCE: + raise ValueError( + "market-context source evidence is empty or unbounded" + ) + if len({item.source_key for item in sources}) != len(sources): + raise ValueError("market-context source keys must be unique") + actual_families = { + _source_family_for_adapter(item.adapter_name) for item in sources + } + if actual_families != set(self.profile.sources): + raise ValueError( + "market-context source evidence differs from fetch profile" + ) + if ( + self.timeline.coverage_start_ns != self.profile.coverage_start_ns + or self.timeline.coverage_end_ns != self.profile.coverage_end_ns + ): + raise ValueError( + "market-context timeline coverage differs from fetch profile" + ) + evidence_by_key = {item.source_key: item for item in sources} + for event in self.timeline.events: + source_key = (event.source.metadata or {}).get("source_key") + evidence = evidence_by_key.get(str(source_key)) + if ( + evidence is None + or event.source.source_id + != evidence.source_contract().source_id + ): + raise ValueError( + "market-context event provenance differs from source evidence" + ) + coverage = tuple( + sorted( + self.coverage, + key=lambda item: ( + item.currency, + item.kind.value, + item.source_family, + item.coverage_start_ns, + ), + ) + ) + if not coverage or len(coverage) > MAX_COVERAGE_SLICES: + raise ValueError("market-context coverage is empty or unbounded") + _bounded_int( + self.duplicate_event_count, + "duplicate_event_count", + 0, + MAX_MARKET_CONTEXT_EVENTS, + ) + counts = { + _required_text(key): _bounded_int( + value, f"count {key}", 0, MAX_MARKET_CONTEXT_EVENTS + ) + for key, value in sorted(self.counts_by_year_currency_kind.items()) + } + expected_counts = _event_counts_by_year_currency_kind( + self.timeline.events + ) + if counts != expected_counts: + raise ValueError( + "market-context corpus counts differ from timeline events" + ) + for item in coverage: + if item.source_family not in actual_families: + raise ValueError( + "market-context coverage has no source evidence" + ) + if ( + item.coverage_start_ns < self.timeline.coverage_start_ns + or item.coverage_end_ns > self.timeline.coverage_end_ns + ): + raise ValueError( + "market-context coverage exceeds timeline coverage" + ) + event_count = sum( + event.kind is item.kind + and item.currency in event.affected_currencies + and _source_family_for_adapter(event.source.adapter_name) + == item.source_family + for event in self.timeline.events + ) + if item.event_count != event_count: + raise ValueError( + "market-context coverage count differs from timeline events" + ) + runtime = _finite_nonnegative(self.runtime_seconds, "runtime_seconds") + peak = _bounded_int( + self.peak_memory_bytes, "peak_memory_bytes", 0, 2**63 - 1 + ) + limitations = tuple(_required_text(item) for item in self.limitations) + if not limitations: + raise ValueError("market-context corpus requires limitations") + object.__setattr__(self, "sources", sources) + object.__setattr__(self, "coverage", coverage) + object.__setattr__(self, "counts_by_year_currency_kind", counts) + object.__setattr__(self, "runtime_seconds", runtime) + object.__setattr__(self, "peak_memory_bytes", peak) + object.__setattr__(self, "limitations", limitations) + expected = _stable_id("market-context-corpus", self.identity_payload()) + if self.corpus_id and self.corpus_id != expected: + raise ValueError("corpus_id does not match deterministic identity") + object.__setattr__(self, "corpus_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "profile": self.profile.to_dict(), + "timeline": self.timeline.to_dict(), + "sources": [item.to_dict() for item in self.sources], + "coverage": [item.to_dict() for item in self.coverage], + "duplicate_event_count": self.duplicate_event_count, + "counts_by_year_currency_kind": dict( + self.counts_by_year_currency_kind + ), + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "event_count": len(self.timeline.events), + "source_count": len(self.sources), + "source_bytes": sum(item.size_bytes for item in self.sources), + "runtime_seconds": self.runtime_seconds, + "peak_memory_bytes": self.peak_memory_bytes, + "corpus_id": self.corpus_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "MarketContextCorpusV1": + if data.get("schema_version") != MARKET_CONTEXT_CORPUS_SCHEMA_VERSION: + raise ValueError("unsupported market-context corpus schema") + corpus = cls( + profile=MarketContextFetchProfileV1.from_dict( + _mapping(data.get("profile")) + ), + timeline=MarketContextTimelineV1.from_dict( + _mapping(data.get("timeline")) + ), + sources=tuple( + MarketContextSourceEvidenceV1.from_dict(item) + for item in _mapping_sequence(data.get("sources")) + ), + coverage=tuple( + MarketContextCoverageSliceV1.from_dict(item) + for item in _mapping_sequence(data.get("coverage")) + ), + duplicate_event_count=_strict_int( + data.get("duplicate_event_count"), "duplicate_event_count" + ), + counts_by_year_currency_kind={ + str(key): _strict_int(value, f"count {key}") + for key, value in _mapping( + data.get("counts_by_year_currency_kind") + ).items() + }, + runtime_seconds=_number(data.get("runtime_seconds")), + peak_memory_bytes=_strict_int( + data.get("peak_memory_bytes"), "peak_memory_bytes" + ), + limitations=_string_tuple(data.get("limitations")), + corpus_id=str(data.get("corpus_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + if _strict_int(data.get("event_count"), "event_count") != len( + corpus.timeline.events + ): + raise ValueError("market-context corpus event count differs") + if _strict_int(data.get("source_count"), "source_count") != len( + corpus.sources + ): + raise ValueError("market-context corpus source count differs") + if _strict_int(data.get("source_bytes"), "source_bytes") != sum( + item.size_bytes for item in corpus.sources + ): + raise ValueError("market-context corpus source bytes differ") + return corpus + + +@dataclass(frozen=True, slots=True) +class MarketContextCorpusBuildV1: + """In-memory build with raw snapshots retained for artifact writing.""" + + corpus: MarketContextCorpusV1 + snapshots: tuple[MarketContextSourceSnapshotV1, ...] + + def __post_init__(self) -> None: + if not isinstance(self.corpus, MarketContextCorpusV1): + raise ValueError("market-context build requires a v1 corpus") + snapshots = tuple( + sorted(self.snapshots, key=lambda item: item.source_key) + ) + if not snapshots or len(snapshots) > MAX_SOURCE_EVIDENCE: + raise ValueError("market-context build snapshots are unbounded") + if len({item.source_key for item in snapshots}) != len(snapshots): + raise ValueError( + "market-context build snapshot keys must be unique" + ) + evidence_by_key = { + item.source_key: item for item in self.corpus.sources + } + if set(evidence_by_key) != {item.source_key for item in snapshots}: + raise ValueError( + "market-context build snapshots differ from source evidence" + ) + for snapshot in snapshots: + evidence = evidence_by_key[snapshot.source_key] + expected = MarketContextSourceEvidenceV1.from_snapshot( + snapshot, + event_count=evidence.emitted_event_count, + diagnostics=evidence.diagnostics, + ) + if expected != evidence: + raise ValueError( + "market-context build snapshot differs from source evidence" + ) + object.__setattr__(self, "snapshots", snapshots) + + +@dataclass(frozen=True, slots=True) +class MarketContextCorpusPreflightV1: + """Explicit support decision for a requested reconstruction interval.""" + + corpus_id: str + start_ns: int + end_ns: int + currencies: tuple[str, ...] + kinds: tuple[MarketContextKind, ...] + ready: bool + reasons: tuple[str, ...] + matched_coverage: tuple[str, ...] + schema_version: str = MARKET_CONTEXT_PREFLIGHT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MARKET_CONTEXT_PREFLIGHT_SCHEMA_VERSION: + raise ValueError("unsupported market-context preflight schema") + _required_text(self.corpus_id) + if self.end_ns <= self.start_ns: + raise ValueError("market-context preflight end must follow start") + currencies = tuple( + sorted({_required_text(item).upper() for item in self.currencies}) + ) + kinds = tuple( + sorted( + {MarketContextKind.from_value(item) for item in self.kinds}, + key=lambda item: item.value, + ) + ) + if not isinstance(self.ready, bool): + raise ValueError("preflight ready must be a boolean") + reasons = tuple(_required_text(item) for item in self.reasons) + if self.ready == bool(reasons): + raise ValueError("preflight readiness and reasons contradict") + object.__setattr__(self, "currencies", currencies) + object.__setattr__(self, "kinds", kinds) + object.__setattr__(self, "reasons", reasons) + object.__setattr__( + self, + "matched_coverage", + tuple(_required_text(item) for item in self.matched_coverage), + ) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "corpus_id": self.corpus_id, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "currencies": list(self.currencies), + "kinds": [item.value for item in self.kinds], + "ready": self.ready, + "reasons": list(self.reasons), + "matched_coverage": list(self.matched_coverage), + } + + +class MarketContextCorpusPreflightError(ValueError): + """Raised when required market-context support is absent.""" + + def __init__(self, decision: MarketContextCorpusPreflightV1) -> None: + self.decision = decision + super().__init__( + "market-context corpus preflight failed: " + + "; ".join(decision.reasons) + ) + + +class OnsReleaseCalendarAdapterV1: + """Normalize one official ONS release-search response page.""" + + adapter_name = "ons-release-calendar" + adapter_version = "1.0" + + def __init__(self, snapshot: MarketContextSourceSnapshotV1) -> None: + _verify_snapshot_adapter( + snapshot, self.adapter_name, self.adapter_version + ) + self.snapshot = snapshot + self.diagnostics: tuple[str, ...] = () + + def load_events(self) -> Iterable[MarketContextEventV1]: + try: + payload = json.loads(self.snapshot.content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError( + "ONS release page is not valid UTF-8 JSON" + ) from exc + releases = _sequence(_mapping(payload).get("releases")) + source = self.snapshot.source_contract() + events: list[MarketContextEventV1] = [] + diagnostics: list[str] = [] + for index, raw in enumerate(releases): + item = _mapping(raw) + description = _mapping(item.get("description")) + title = str(description.get("title") or "").strip() + if not any( + pattern.search(title) for pattern in _ONS_TITLE_PATTERNS + ): + continue + if bool(description.get("cancelled")): + diagnostics.append(f"cancelled_release:{index}") + continue + release_time = str(description.get("release_date") or "").strip() + uri = str(item.get("uri") or "").strip() + if not title or not release_time or not uri: + diagnostics.append(f"missing_release_field:{index}") + continue + try: + event_time = normalize_market_context_datetime( + release_time, "UTC" + ) + except ValueError: + diagnostics.append(f"invalid_release_time:{index}") + continue + canonical = "ons." + re.sub( + r"[^a-z0-9._:-]+", "-", uri.strip("/").lower() + ) + changes = _sequence_or_empty(item.get("date_changes")) + tags = ["ons", "scheduled", "macro_release"] + if changes: + tags.append("schedule_changed_without_change_timestamp") + event_content = canonical_contract_json( + cast( + Mapping[str, JSONValue], + _json_value_mapping( + { + "uri": uri, + "description": { + "title": title, + "release_date": release_time, + "cancelled": False, + }, + "date_changes": changes, + } + ), + ) + ).encode("utf-8") + events.append( + MarketContextEventV1( + canonical_key=canonical, + kind=MarketContextKind.MACRO_RELEASE, + title=title, + source=source, + source_event_time=release_time, + source_timezone="UTC", + event_time_ns=event_time, + first_known_at_ns=event_time, + available_at_ns=event_time, + pre_event_ns=30 * 60 * 1_000_000_000, + post_event_ns=2 * HOUR_NS, + affected_currencies=("GBP",), + affected_symbols=("EURGBP", "GBPUSD"), + confidence=1.0, + precision=MarketContextPrecision.EXACT, + limitations=( + "The current ONS release-search endpoint does not retain when a schedule was first published.", + "Date changes lack change-known timestamps and are not exposed as ex-ante revisions.", + "This calendar record contains no consensus expectation or realized statistic value.", + ), + vintage_id=f"ons-release:{uri}:{release_time}", + content_sha256=hashlib.sha256(event_content).hexdigest(), + tags=tuple(tags), + ) + ) + self.diagnostics = tuple(diagnostics[:MAX_DIAGNOSTICS]) + return tuple(events) + + +class EcbPolicyRateAdapterV1: + """Collapse the official daily MRO level series to effective changes.""" + + adapter_name = "ecb-policy-rate" + adapter_version = "1.0" + + def __init__(self, snapshot: MarketContextSourceSnapshotV1) -> None: + _verify_snapshot_adapter( + snapshot, self.adapter_name, self.adapter_version + ) + self.snapshot = snapshot + self.diagnostics: tuple[str, ...] = () + self.coverage_start_ns: int | None = None + self.coverage_end_ns: int | None = None + self.coverage_complete = False + + def load_events(self) -> Iterable[MarketContextEventV1]: + try: + text = self.snapshot.content.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise ValueError("ECB policy-rate CSV is not UTF-8") from exc + source = self.snapshot.source_contract() + events: list[MarketContextEventV1] = [] + diagnostics: list[str] = [] + observations: list[tuple[date, float, Mapping[str | None, Any]]] = [] + for index, row in enumerate(csv.DictReader(io.StringIO(text))): + series_key = str(row.get("KEY") or "").strip() + if series_key and series_key != ECB_POLICY_RATE_SERIES_KEY: + raise ValueError( + "ECB policy-rate response contains an unexpected series" + ) + period = str(row.get("TIME_PERIOD") or "").strip() + raw_value = str(row.get("OBS_VALUE") or "").strip() + if not period or not raw_value: + diagnostics.append(f"missing_rate_field:{index}") + continue + try: + value = float(raw_value) + observed_date = _parse_date(period, "ECB TIME_PERIOD") + except ValueError: + diagnostics.append(f"invalid_rate_row:{index}") + continue + observations.append((observed_date, value, dict(row))) + + observations.sort(key=lambda item: item[0]) + if not observations: + self.diagnostics = tuple(diagnostics[:MAX_DIAGNOSTICS]) + return () + for previous, current in zip(observations, observations[1:]): + if current[0] == previous[0]: + if current[1] != previous[1]: + raise ValueError( + "ECB policy-rate response has conflicting daily values" + ) + diagnostics.append( + f"duplicate_rate_date:{current[0].isoformat()}" + ) + continue + if current[0] != previous[0] + timedelta(days=1): + diagnostics.append( + "non_contiguous_rate_date:" + f"{previous[0].isoformat()}:{current[0].isoformat()}" + ) + self.coverage_start_ns = _date_start_ns(observations[0][0]) + self.coverage_end_ns = _date_start_ns( + observations[-1][0] + timedelta(days=1) + ) + self.coverage_complete = not diagnostics + + previous_value: float | None = None + emitted_dates: set[date] = set() + for observed_date, value, observation_row in observations: + if observed_date in emitted_dates: + continue + emitted_dates.add(observed_date) + if previous_value is not None and value == previous_value: + continue + period = observed_date.isoformat() + event_time = _date_start_ns(observed_date) + conservative_available = event_time + DAY_NS + title = str( + observation_row.get("TITLE") + or "ECB main refinancing operations rate" + ).strip() + initial_level = previous_value is None + events.append( + MarketContextEventV1( + canonical_key=f"ecb.fm.mrr.{period}", + kind=MarketContextKind.POLICY_RATE_CHANGE, + title=title, + source=source, + source_event_time=f"{period}T00:00:00+00:00", + source_timezone="UTC", + event_time_ns=event_time, + first_known_at_ns=conservative_available, + available_at_ns=conservative_available, + pre_event_ns=DAY_NS, + post_event_ns=DAY_NS, + affected_currencies=("EUR",), + affected_symbols=("EURGBP", "EURUSD"), + confidence=0.9, + precision=MarketContextPrecision.WINDOW_ONLY, + ambiguity_reason=( + "The SDMX observation is the rate-change effective date, not the policy announcement timestamp." + ), + limitations=( + "Effective-date observations do not prove the announcement time or original schedule vintage.", + "Ex-ante availability is delayed until the next UTC day instead of inferring an intraday publication time.", + "No historical consensus expectation is supplied.", + ), + vintage_id=f"ecb-fm-mrr:{period}:{value:g}", + actual_value=value, + previous_value=previous_value, + value_unit="percent", + content_sha256=_row_sha256(observation_row), + tags=( + "ecb", + "effective_date", + "main_refinancing_rate", + ( + "series_initial_level" + if initial_level + else "effective_rate_change" + ), + "daily_level_collapsed_to_change", + ), + ) + ) + previous_value = value + self.diagnostics = tuple(diagnostics[:MAX_DIAGNOSTICS]) + return tuple(events) + + +class BankOfEnglandBankRateAdapterV1: + """Normalize the official Bank of England Bank Rate history table.""" + + adapter_name = "boe-bank-rate" + adapter_version = "1.0" + + def __init__(self, snapshot: MarketContextSourceSnapshotV1) -> None: + _verify_snapshot_adapter( + snapshot, self.adapter_name, self.adapter_version + ) + self.snapshot = snapshot + self.diagnostics: tuple[str, ...] = () + self.coverage_start_ns: int | None = None + self.coverage_end_ns: int | None = None + self.coverage_complete = False + + def load_events(self) -> Iterable[MarketContextEventV1]: + try: + text = self.snapshot.content.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise ValueError("Bank Rate history is not UTF-8 HTML") from exc + parser = _BankRateTableParser() + parser.feed(text) + source = self.snapshot.source_contract() + events: list[MarketContextEventV1] = [] + diagnostics: list[str] = [] + for index, row in enumerate(parser.rows): + if len(row) < 2: + diagnostics.append(f"missing_bank_rate_field:{index}") + continue + try: + changed = datetime.strptime(row[0], "%d %b %y").date() + value = float(row[1]) + except ValueError: + diagnostics.append(f"invalid_bank_rate_row:{index}") + continue + source_time = f"{changed.isoformat()}T00:00:00+00:00" + event_time = normalize_market_context_datetime(source_time, "UTC") + conservative_available = event_time + DAY_NS + events.append( + MarketContextEventV1( + canonical_key=f"boe.bank-rate.{changed.isoformat()}", + kind=MarketContextKind.POLICY_RATE_CHANGE, + title="Bank of England official Bank Rate change", + source=source, + source_event_time=source_time, + source_timezone="UTC", + event_time_ns=event_time, + first_known_at_ns=conservative_available, + available_at_ns=conservative_available, + pre_event_ns=DAY_NS, + post_event_ns=DAY_NS, + affected_currencies=("GBP",), + affected_symbols=("EURGBP", "GBPUSD"), + confidence=0.9, + precision=MarketContextPrecision.WINDOW_ONLY, + ambiguity_reason=( + "The official table provides a change date but not the announcement timestamp." + ), + limitations=( + "The date-only history does not retain the original decision schedule or announcement time.", + "Ex-ante availability is delayed until the next UTC day instead of inferring an intraday publication time.", + "No historical consensus expectation is supplied.", + ), + vintage_id=f"boe-bank-rate:{changed.isoformat()}:{value:g}", + actual_value=value, + value_unit="percent", + content_sha256=hashlib.sha256( + f"{row[0]}|{row[1]}".encode("utf-8") + ).hexdigest(), + tags=("bank_of_england", "bank_rate", "effective_date"), + ) + ) + self.diagnostics = tuple(diagnostics[:MAX_DIAGNOSTICS]) + if events: + self.coverage_start_ns = min(item.event_time_ns for item in events) + self.coverage_end_ns = ( + self.snapshot.retrieved_at_ns // DAY_NS + 1 + ) * DAY_NS + self.coverage_complete = not diagnostics + return tuple(events) + + +class FederalReserveFomcCalendarAdapterV1: + """Normalize official FOMC meeting-end policy-decision timestamps.""" + + adapter_name = "federal-reserve-fomc-calendar" + adapter_version = "1.0" + + def __init__(self, snapshot: MarketContextSourceSnapshotV1) -> None: + _verify_snapshot_adapter( + snapshot, self.adapter_name, self.adapter_version + ) + self.snapshot = snapshot + self.diagnostics: tuple[str, ...] = () + + def load_events(self) -> Iterable[MarketContextEventV1]: + try: + text = self.snapshot.content.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise ValueError("FOMC calendar is not UTF-8 HTML") from exc + source = self.snapshot.source_contract() + events: list[MarketContextEventV1] = [] + diagnostics: list[str] = [] + for year_text, section in re.findall( + r"(20\d{2}) FOMC Meetings(.*?)(?=(?:20\d{2}) FOMC Meetings|$)", + text, + re.DOTALL, + ): + year = int(year_text) + pairs = re.findall( + r"fomc-meeting__month[^>]*>\s*([^<]+)" + r".*?fomc-meeting__date[^>]*>([^<]+)", + section, + re.DOTALL, + ) + for month_text, date_text in pairs: + notation_vote = "notation" in date_text.lower() + try: + month = _fomc_end_month(month_text) + day = int(re.findall(r"\d+", date_text)[-1]) + event_date = date(year, month, day) + except (IndexError, ValueError): + diagnostics.append( + f"invalid_fomc_date:{year_text}:{month_text}:{date_text}" + ) + continue + if notation_vote: + source_time = f"{event_date.isoformat()}T00:00:00" + precision = MarketContextPrecision.WINDOW_ONLY + ambiguity = "The calendar identifies a notation-vote date but not its release time." + pre_ns = 0 + post_ns = DAY_NS + else: + source_time = f"{event_date.isoformat()}T14:00:00" + precision = MarketContextPrecision.EXACT + ambiguity = None + pre_ns = 30 * 60 * 1_000_000_000 + post_ns = 2 * HOUR_NS + event_time = normalize_market_context_datetime( + source_time, "America/New_York" + ) + available = min(event_time, self.snapshot.retrieved_at_ns) + events.append( + MarketContextEventV1( + canonical_key=f"federal-reserve.fomc.{event_date.isoformat()}", + kind=MarketContextKind.CENTRAL_BANK_DECISION, + title=( + "Federal Reserve FOMC notation vote" + if notation_vote + else "Federal Reserve FOMC policy decision" + ), + source=source, + source_event_time=source_time, + source_timezone="America/New_York", + event_time_ns=event_time, + first_known_at_ns=available, + available_at_ns=available, + pre_event_ns=pre_ns, + post_event_ns=post_ns, + affected_currencies=("USD",), + affected_symbols=("EURUSD", "GBPUSD"), + confidence=0.95, + precision=precision, + ambiguity_reason=ambiguity, + limitations=( + "The live calendar does not prove when each schedule was first published.", + "Regular decisions use the Federal Reserve's documented 2 p.m. Eastern announcement time for this supported 2021+ calendar.", + "Unexpected timing exceptions require a later authoritative vintage rather than inference.", + ), + vintage_id=f"fomc-calendar:{event_date.isoformat()}:{date_text.strip()}", + content_sha256=hashlib.sha256( + f"{year_text}|{month_text}|{date_text}".encode( + "utf-8" + ) + ).hexdigest(), + tags=( + "federal_reserve", + "fomc", + "notation_vote" if notation_vote else "scheduled", + ), + ) + ) + self.diagnostics = tuple(diagnostics[:MAX_DIAGNOSTICS]) + return tuple(events) + + +class FederalReserveFomcHistoricalAdapterV1: + """Normalize official 2000-2020 FOMC historical meeting pages.""" + + adapter_name = "federal-reserve-fomc-historical" + adapter_version = "1.0" + + def __init__(self, snapshot: MarketContextSourceSnapshotV1) -> None: + _verify_snapshot_adapter( + snapshot, self.adapter_name, self.adapter_version + ) + self.snapshot = snapshot + self.diagnostics: tuple[str, ...] = () + + def load_events(self) -> Iterable[MarketContextEventV1]: + try: + text = self.snapshot.content.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise ValueError("historical FOMC page is not UTF-8 HTML") from exc + source = self.snapshot.source_contract() + events: list[MarketContextEventV1] = [] + diagnostics: list[str] = [] + headings = re.findall(r"]*>(.*?)", text, re.DOTALL) + for index, raw_heading in enumerate(headings): + heading = " ".join(re.sub(r"<[^>]+>", " ", raw_heading).split()) + if not ( + " Meeting" in heading or "notation vote" in heading.lower() + ): + continue + match = re.search( + r"([A-Za-z]+(?:/[A-Za-z]+)?)\s+" + r"([0-9]+(?:[-/][0-9]+)?)" + r"(?:\s+\([^)]*\))?\s+" + r"(?:Meeting|\(notation vote\)|-).*?(20[0-2][0-9])", + heading, + re.IGNORECASE, + ) + if match is None: + diagnostics.append(f"unparsed_historical_heading:{index}") + continue + try: + month = _fomc_end_month(match.group(1)) + day = int(re.findall(r"\d+", match.group(2))[-1]) + event_date = date(int(match.group(3)), month, day) + except (IndexError, ValueError): + diagnostics.append(f"invalid_historical_heading:{index}") + continue + if "cancelled" in heading.lower(): + diagnostics.append( + f"cancelled_historical_meeting:{event_date.isoformat()}" + ) + continue + notation_vote = "notation vote" in heading.lower() + unscheduled = "unscheduled" in heading.lower() + source_time = f"{event_date.isoformat()}T00:00:00" + event_time = normalize_market_context_datetime( + source_time, "America/New_York" + ) + available = normalize_market_context_datetime( + f"{event_date.isoformat()}T23:59:59", "America/New_York" + ) + tags = ["federal_reserve", "fomc", "historical_page"] + tags.append("unscheduled" if unscheduled else "scheduled") + if notation_vote: + tags.append("notation_vote") + events.append( + MarketContextEventV1( + canonical_key=( + f"federal-reserve.fomc.{event_date.isoformat()}" + ), + kind=MarketContextKind.CENTRAL_BANK_DECISION, + title=( + "Federal Reserve FOMC notation vote" + if notation_vote + else "Federal Reserve FOMC historical meeting" + ), + source=source, + source_event_time=source_time, + source_timezone="America/New_York", + event_time_ns=event_time, + first_known_at_ns=event_time, + available_at_ns=available, + pre_event_ns=0, + post_event_ns=DAY_NS, + affected_currencies=("USD",), + affected_symbols=("EURUSD", "GBPUSD"), + confidence=0.9, + precision=MarketContextPrecision.WINDOW_ONLY, + ambiguity_reason=( + "The historical page proves the meeting date but this adapter does not infer a release timestamp." + ), + limitations=( + "The date-only event becomes ex-ante eligible only after the local calendar day, preventing intraday look-ahead.", + "The current historical page does not prove the original schedule-publication time.", + "Conference calls without a retained meeting or notation-vote heading are excluded.", + ), + vintage_id=( + f"fomc-historical:{event_date.isoformat()}:{heading}" + ), + content_sha256=hashlib.sha256( + heading.encode("utf-8") + ).hexdigest(), + tags=tuple(tags), + ) + ) + self.diagnostics = tuple(diagnostics[:MAX_DIAGNOSTICS]) + return tuple(events) + + +class OperatorMarketContextCatalogAdapterV1: + """Normalize an operator-maintained shock/revision catalog.""" + + adapter_name = "operator-market-context-catalog" + adapter_version = "1.0" + + def __init__(self, snapshot: MarketContextSourceSnapshotV1) -> None: + _verify_snapshot_adapter( + snapshot, self.adapter_name, self.adapter_version + ) + self.snapshot = snapshot + self.diagnostics: tuple[str, ...] = () + + def load_events(self) -> Iterable[MarketContextEventV1]: + try: + payload = _mapping( + json.loads(self.snapshot.content.decode("utf-8")) + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError( + "operator catalog is not valid UTF-8 JSON" + ) from exc + if ( + payload.get("schema_version") + != OPERATOR_MARKET_CONTEXT_CATALOG_SCHEMA_VERSION + ): + raise ValueError( + "unsupported operator market-context catalog schema" + ) + raw_events = sorted( + _mapping_sequence(payload.get("events")), + key=lambda item: ( + str(item.get("canonical_key", "")), + _strict_int( + item.get("revision_sequence", 0), "revision_sequence" + ), + ), + ) + source = self.snapshot.source_contract() + predecessors: dict[str, MarketContextEventV1] = {} + events: list[MarketContextEventV1] = [] + for item in raw_events: + canonical = str(item.get("canonical_key", "")) + revision = _strict_int( + item.get("revision_sequence", 0), "revision_sequence" + ) + predecessor = predecessors.get(canonical) + if revision != ( + 0 if predecessor is None else predecessor.revision_sequence + 1 + ): + raise ValueError( + "operator catalog revision sequence is not contiguous" + ) + timezone_name = str(item.get("source_timezone", "")) + source_time = str(item.get("source_event_time", "")) + source_time_fold = _optional_int(item.get("source_time_fold")) + event_time = _normalize_catalog_datetime( + source_time, timezone_name, fold=source_time_fold + ) + first_known = _normalize_catalog_datetime( + str(item.get("first_known_at", "")), + timezone_name, + fold=source_time_fold, + ) + available = _normalize_catalog_datetime( + str(item.get("available_at", "")), + timezone_name, + fold=source_time_fold, + ) + event = MarketContextEventV1( + canonical_key=canonical, + kind=MarketContextKind.from_value(str(item.get("kind", ""))), + title=str(item.get("title", "")), + source=source, + source_event_time=source_time, + source_timezone=timezone_name, + source_time_fold=source_time_fold, + event_time_ns=event_time, + first_known_at_ns=first_known, + available_at_ns=available, + pre_event_ns=_strict_int( + item.get("pre_event_ns"), "pre_event_ns" + ), + post_event_ns=_strict_int( + item.get("post_event_ns"), "post_event_ns" + ), + affected_currencies=_string_tuple( + item.get("affected_currencies") + ), + affected_symbols=_string_tuple(item.get("affected_symbols")), + confidence=_number(item.get("confidence")), + precision=MarketContextPrecision.from_value( + str(item.get("precision", "")) + ), + ambiguity_reason=_optional_text(item.get("ambiguity_reason")), + limitations=_string_tuple(item.get("limitations")), + vintage_id=str(item.get("vintage_id", "")), + revision_sequence=revision, + supersedes_event_id=( + predecessor.event_id if predecessor else None + ), + expected_value=_optional_number(item.get("expected_value")), + actual_value=_optional_number(item.get("actual_value")), + previous_value=_optional_number(item.get("previous_value")), + revised_previous_value=_optional_number( + item.get("revised_previous_value") + ), + value_unit=_optional_text(item.get("value_unit")), + content_sha256=hashlib.sha256( + canonical_contract_json( + cast(Mapping[str, JSONValue], _json_value_mapping(item)) + ).encode("utf-8") + ).hexdigest(), + tags=_string_tuple(item.get("tags")), + ) + predecessors[canonical] = event + events.append(event) + return tuple(events) + + +def packaged_operator_catalog_path() -> Path: + """Return the installed operator-maintained shock catalog path.""" + return Path(__file__).with_name("assets") / "operator_shocks_v1.json" + + +def build_live_market_context_corpus( + profile: MarketContextFetchProfileV1, + *, + operator_catalog_path: str | Path | None = None, +) -> MarketContextCorpusBuildV1: + """Acquire approved official sources and build one replayable corpus.""" + started = time.perf_counter() + budget = _AcquisitionBudget(profile=profile, started=started) + snapshots: list[MarketContextSourceSnapshotV1] = [] + if "ons" in profile.sources: + snapshots.extend(_fetch_ons_snapshots(profile, budget)) + if "ecb" in profile.sources: + snapshots.append(_fetch_ecb_snapshot(profile, budget)) + if "boe" in profile.sources: + snapshots.append(_fetch_boe_snapshot(profile, budget)) + if "fed" in profile.sources: + snapshots.extend(_fetch_fed_snapshots(profile, budget)) + if "operator" in profile.sources: + operator_snapshot = _read_operator_snapshot( + operator_catalog_path or packaged_operator_catalog_path() + ) + budget.consume(len(operator_snapshot.content)) + snapshots.append(operator_snapshot) + budget.check() + return build_market_context_corpus_from_snapshots( + snapshots, + profile=profile, + runtime_started=started, + ) + + +def build_market_context_corpus_from_snapshots( + snapshots: Sequence[MarketContextSourceSnapshotV1], + *, + profile: MarketContextFetchProfileV1, + runtime_started: float | None = None, +) -> MarketContextCorpusBuildV1: + """Build the corpus deterministically from already retained snapshots.""" + started = ( + time.perf_counter() if runtime_started is None else runtime_started + ) + values = tuple(sorted(snapshots, key=lambda item: item.source_key)) + if not values or len(values) > MAX_SOURCE_EVIDENCE: + raise ValueError("market-context snapshot count is empty or unbounded") + if len({item.source_key for item in values}) != len(values): + raise ValueError("market-context snapshot keys must be unique") + actual_families = { + _source_family_for_adapter(item.adapter_name) for item in values + } + if actual_families != set(profile.sources): + raise ValueError( + "market-context snapshot families differ from fetch profile: " + f"expected {sorted(profile.sources)}, got {sorted(actual_families)}" + ) + if ( + sum(len(item.content) for item in values) + > profile.max_total_source_bytes + ): + raise ValueError( + "market-context snapshots exceed total source-byte limit" + ) + all_events: list[MarketContextEventV1] = [] + evidence: list[MarketContextSourceEvidenceV1] = [] + source_events: dict[str, tuple[MarketContextEventV1, ...]] = {} + source_support: dict[str, tuple[int, int, bool]] = {} + source_diagnostics: dict[str, tuple[str, ...]] = {} + for snapshot in values: + adapter = _adapter_for_snapshot(snapshot) + events = tuple(adapter.load_events()) + diagnostics = tuple(getattr(adapter, "diagnostics", ())) + source_diagnostics[snapshot.source_key] = diagnostics + support_start = getattr(adapter, "coverage_start_ns", None) + support_end = getattr(adapter, "coverage_end_ns", None) + support_complete = bool(getattr(adapter, "coverage_complete", False)) + if support_start is not None and support_end is not None: + source_support[snapshot.source_key] = ( + support_start, + support_end, + support_complete, + ) + source_events[snapshot.source_key] = events + all_events.extend(events) + evidence.append( + MarketContextSourceEvidenceV1.from_snapshot( + snapshot, + event_count=len(events), + diagnostics=diagnostics, + ) + ) + deduplicated, duplicate_count = _deduplicate_events(all_events) + retained = tuple( + event + for event in deduplicated + if profile.coverage_start_ns + <= event.event_time_ns + < profile.coverage_end_ns + ) + if not retained: + raise ValueError("market-context corpus contains no in-range events") + if len(retained) > profile.max_events: + raise ValueError("market-context corpus exceeds configured event limit") + timeline = MarketContextTimelineV1( + timeline_version="official-source-corpus-v1", + coverage_start_ns=profile.coverage_start_ns, + coverage_end_ns=profile.coverage_end_ns, + complete=True, + events=retained, + limitations=( + "Completeness means every configured source response was acquired and parsed; it does not assert that every real-world event is represented.", + "Absence of a matching event means no matching retained record, never proof that no event occurred.", + "Current-source archives may omit historical schedule vintages and later corrections; source-level limitations remain authoritative.", + "CFTC Commitments of Traders is persistent positioning state and is intentionally outside this event corpus (issue #468).", + ), + ) + coverage = _coverage_slices( + profile, + values, + source_events, + source_support, + source_diagnostics, + retained, + ) + counts = _event_counts_by_year_currency_kind(retained) + elapsed = time.perf_counter() - started + if elapsed > profile.max_runtime_seconds: + raise ValueError("market-context corpus build exceeded runtime limit") + corpus = MarketContextCorpusV1( + profile=profile, + timeline=timeline, + sources=tuple(evidence), + coverage=coverage, + duplicate_event_count=duplicate_count, + counts_by_year_currency_kind=counts, + runtime_seconds=round(elapsed, 6), + peak_memory_bytes=peak_rss_bytes(), + limitations=timeline.limitations, + ) + return MarketContextCorpusBuildV1(corpus=corpus, snapshots=values) + + +def write_market_context_corpus( + build: MarketContextCorpusBuildV1, + directory: str | Path, +) -> Mapping[str, ArtifactRef]: + """Write content-addressed raw, timeline, and corpus artifacts once.""" + root = Path(directory).expanduser().resolve() + raw_root = root / "sources" + raw_root.mkdir(parents=True, exist_ok=True) + artifacts: dict[str, ArtifactRef] = {} + for snapshot in build.snapshots: + suffix = _content_suffix(snapshot.content_type) + path = ( + raw_root + / f"{snapshot.source_key}-{snapshot.content_sha256}{suffix}" + ) + _write_once(path, snapshot.content) + artifacts[f"source:{snapshot.source_key}"] = ArtifactRef( + kind="market_context_source_snapshot_v1", + path=str(path), + size_bytes=len(snapshot.content), + sha256=snapshot.content_sha256, + metadata={ + "adapter_name": snapshot.adapter_name, + "retrieved_at_ns": snapshot.retrieved_at_ns, + "source_uri": snapshot.source_uri, + }, + ) + timeline_bytes = build.corpus.timeline.to_json().encode("utf-8") + b"\n" + timeline_hash = hashlib.sha256(timeline_bytes).hexdigest() + timeline_path = root / f"market-context-timeline-{timeline_hash}.json" + _write_once(timeline_path, timeline_bytes) + artifacts["timeline"] = ArtifactRef( + kind="market_context_timeline_v1", + path=str(timeline_path), + size_bytes=len(timeline_bytes), + sha256=timeline_hash, + metadata={"timeline_id": build.corpus.timeline.timeline_id}, + ) + corpus_bytes = ( + canonical_contract_json(build.corpus.to_dict()).encode("utf-8") + b"\n" + ) + corpus_hash = hashlib.sha256(corpus_bytes).hexdigest() + corpus_path = root / f"market-context-corpus-{corpus_hash}.json" + _write_once(corpus_path, corpus_bytes) + artifacts["corpus"] = ArtifactRef( + kind="market_context_corpus_v1", + path=str(corpus_path), + size_bytes=len(corpus_bytes), + sha256=corpus_hash, + metadata={ + "corpus_id": build.corpus.corpus_id, + "timeline_id": build.corpus.timeline.timeline_id, + "event_count": len(build.corpus.timeline.events), + }, + ) + return artifacts + + +def read_market_context_corpus(path: str | Path) -> MarketContextCorpusV1: + """Load and strictly verify one self-contained corpus artifact.""" + source = Path(path).expanduser().resolve() + if source.stat().st_size > MAX_CORPUS_BYTES: + raise ValueError("market-context corpus artifact exceeds size bound") + match = re.fullmatch( + r"market-context-corpus-([0-9a-f]{64})\.json", source.name + ) + if match is None: + raise ValueError( + "market-context corpus artifact name is not content addressed" + ) + content = source.read_bytes() + if hashlib.sha256(content).hexdigest() != match.group(1): + raise ValueError( + "market-context corpus artifact hash differs from name" + ) + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError( + "market-context corpus artifact is invalid JSON" + ) from exc + return MarketContextCorpusV1.from_dict(_mapping(payload)) + + +def replay_market_context_corpus( + corpus_path: str | Path, + *, + source_directory: str | Path | None = None, +) -> MarketContextCorpusBuildV1: + """Rebuild a corpus exactly from its retained content-addressed sources.""" + corpus = read_market_context_corpus(corpus_path) + root = ( + Path(source_directory).expanduser().resolve() + if source_directory is not None + else Path(corpus_path).expanduser().resolve().parent / "sources" + ) + snapshots: list[MarketContextSourceSnapshotV1] = [] + for item in corpus.sources: + matches = sorted( + root.glob(f"{item.source_key}-{item.content_sha256}.*") + ) + if len(matches) != 1: + raise ValueError( + f"expected one retained snapshot for source {item.source_key}" + ) + content = matches[0].read_bytes() + snapshots.append(item.restore_snapshot(content)) + rebuilt = build_market_context_corpus_from_snapshots( + snapshots, + profile=corpus.profile, + ) + if rebuilt.corpus.corpus_id != corpus.corpus_id: + raise ValueError("replayed market-context corpus identity differs") + return rebuilt + + +def preflight_market_context_corpus( + corpus: MarketContextCorpusV1, + *, + start_ns: int, + end_ns: int, + currencies: Sequence[str], + kinds: Sequence[MarketContextKind], +) -> MarketContextCorpusPreflightV1: + """Return support evidence without treating an ordinary empty window as failure.""" + if end_ns <= start_ns: + raise ValueError("market-context preflight end must follow start") + selected_currencies = tuple( + sorted({_required_text(item).upper() for item in currencies}) + ) + selected_kinds = tuple( + sorted( + {MarketContextKind.from_value(item) for item in kinds}, + key=lambda item: item.value, + ) + ) + reasons: list[str] = [] + matched: list[str] = [] + if ( + start_ns < corpus.timeline.coverage_start_ns + or end_ns > corpus.timeline.coverage_end_ns + ): + reasons.append( + "requested interval lies outside corpus timeline coverage" + ) + if not corpus.timeline.complete: + reasons.append("corpus timeline is incomplete") + for currency in selected_currencies: + for kind in selected_kinds: + candidates = [ + item + for item in corpus.coverage + if item.currency == currency and item.kind is kind + ] + supporting = [ + item for item in candidates if item.supports(start_ns, end_ns) + ] + if supporting: + matched.extend( + f"{item.source_family}:{currency}:{kind.value}" + for item in supporting + ) + else: + reasons.append( + f"unsupported context coverage for {currency}/{kind.value}" + ) + return MarketContextCorpusPreflightV1( + corpus_id=corpus.corpus_id, + start_ns=start_ns, + end_ns=end_ns, + currencies=selected_currencies, + kinds=selected_kinds, + ready=not reasons, + reasons=tuple(dict.fromkeys(reasons)), + matched_coverage=tuple(sorted(set(matched))), + ) + + +def require_market_context_corpus( + corpus: MarketContextCorpusV1, + *, + start_ns: int, + end_ns: int, + currencies: Sequence[str], + kinds: Sequence[MarketContextKind], +) -> MarketContextCorpusPreflightV1: + """Fail closed when a reconstruction requires unsupported context.""" + decision = preflight_market_context_corpus( + corpus, + start_ns=start_ns, + end_ns=end_ns, + currencies=currencies, + kinds=kinds, + ) + if not decision.ready: + raise MarketContextCorpusPreflightError(decision) + return decision + + +def query_market_context_corpus( + corpus: MarketContextCorpusV1, + *, + start_ns: int, + end_ns: int, + view: MarketContextView, + as_of_ns: int | None = None, + currencies: Sequence[str] = (), + symbols: Sequence[str] = (), + kinds: Sequence[MarketContextKind] = (), + include_calendar: bool = True, + max_events: int = 512, + window_id: str | None = None, + require_supported: bool = True, +) -> MarketContextQueryV1: + """Query a corpus and preflight every declared currency/kind requirement.""" + required_currencies = {_required_text(item).upper() for item in currencies} + for symbol in symbols: + normalized = _required_text(symbol).upper() + if re.fullmatch(r"[A-Z]{6}", normalized): + required_currencies.update((normalized[:3], normalized[3:])) + if require_supported and kinds and required_currencies: + require_market_context_corpus( + corpus, + start_ns=start_ns, + end_ns=end_ns, + currencies=tuple(sorted(required_currencies)), + kinds=kinds, + ) + return query_market_context( + corpus.timeline, + start_ns=start_ns, + end_ns=end_ns, + view=view, + as_of_ns=as_of_ns, + currencies=currencies, + symbols=symbols, + kinds=kinds, + include_calendar=include_calendar, + max_events=max_events, + window_id=window_id, + ) + + +def market_context_benchmark_event_state(query: MarketContextQueryV1) -> str: + """Project a bounded context query onto the benchmark event-state seam.""" + if query.events: + kinds = "+".join(sorted({item.kind.value for item in query.events})) + tags = sorted({tag for item in query.events for tag in item.tags})[:4] + suffix = ":" + "+".join(tags) if tags else "" + return f"market_context:{kinds}{suffix}" + reason = ( + query.missing_reason.value + if query.missing_reason is not None + else "unknown" + ) + return f"market_context:none:{reason}" + + +def _fetch_ons_snapshots( + profile: MarketContextFetchProfileV1, + budget: _AcquisitionBudget, +) -> list[MarketContextSourceSnapshotV1]: + snapshots: list[MarketContextSourceSnapshotV1] = [] + for query_number, query in enumerate(profile.ons_queries): + offset = 0 + for page in range(profile.max_ons_pages_per_query): + params = { + "limit": "1000", + "offset": str(offset), + "sort": "release_date_asc", + "fromDate": profile.start_date, + "toDate": profile.end_date, + "release-type": "type-published", + "query": query, + } + uri = ONS_RELEASES_URI + "?" + urlencode(params) + content, content_type, retrieved = _fetch_bytes( + uri, profile, budget=budget + ) + key = f"ons.q{query_number:02d}.p{page:02d}" + snapshots.append( + MarketContextSourceSnapshotV1( + source_key=key, + source_name="Office for National Statistics release calendar", + source_uri=uri, + retrieved_at_ns=retrieved, + content=content, + content_type=content_type, + adapter_name=OnsReleaseCalendarAdapterV1.adapter_name, + adapter_version=OnsReleaseCalendarAdapterV1.adapter_version, + license_name="Open Government Licence v3.0", + redistribution_allowed=True, + redistribution_constraints=( + "Attribute the Office for National Statistics and the OGL v3.0.", + ), + limitations=( + "The search endpoint exposes the current release record, not a complete schedule-vintage archive.", + "Search relevance can return unrelated titles; the adapter applies a documented allowlist.", + ), + metadata={ + "provider": "ONS", + "query": query, + "offset": offset, + "limit": 1000, + "license_uri": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", + "documentation_uri": "https://developer.ons.gov.uk/search/search-releases/", + }, + ) + ) + payload = _mapping(json.loads(content.decode("utf-8"))) + releases = _sequence(payload.get("releases")) + breakdown = _mapping(payload.get("breakdown")) + total = int(breakdown.get("total") or len(releases)) + offset += len(releases) + if not releases or offset >= total: + break + budget.check() + else: + raise ValueError(f"ONS query exceeded page limit: {query}") + return snapshots + + +def _fetch_ecb_snapshot( + profile: MarketContextFetchProfileV1, + budget: _AcquisitionBudget, +) -> MarketContextSourceSnapshotV1: + params = { + "format": "csvdata", + "startPeriod": "1999-01-01", + "endPeriod": profile.end_date, + "includeHistory": "true", + } + uri = ECB_POLICY_RATE_URI + "?" + urlencode(params) + content, content_type, retrieved = _fetch_bytes(uri, profile, budget=budget) + return MarketContextSourceSnapshotV1( + source_key="ecb.policy-rate", + source_name="European Central Bank key interest rate SDMX series", + source_uri=uri, + retrieved_at_ns=retrieved, + content=content, + content_type=content_type, + adapter_name=EcbPolicyRateAdapterV1.adapter_name, + adapter_version=EcbPolicyRateAdapterV1.adapter_version, + license_name="ECB statistics reuse policy", + redistribution_allowed=True, + redistribution_constraints=( + "Attribute reused statistics as Source: ECB statistics.", + "Preserve raw statistics and metadata unchanged; identify normalized corpus events as derived transformations.", + ), + limitations=( + "The selected daily level series is collapsed to effective level transitions; these are not announcement timestamps.", + "Historical revision availability is limited to the history exposed by this SDMX response.", + ), + metadata={ + "provider": "ECB", + "series_key": ECB_POLICY_RATE_SERIES_KEY, + "series_start_period": "1999-01-01", + "license_uri": "https://www.ecb.europa.eu/stats/ecb_statistics/governance_and_quality_framework/html/usage_policy.en.html", + "documentation_uri": "https://data.ecb.europa.eu/help/api/data", + }, + ) + + +def _fetch_boe_snapshot( + profile: MarketContextFetchProfileV1, + budget: _AcquisitionBudget, +) -> MarketContextSourceSnapshotV1: + content, content_type, retrieved = _fetch_bytes( + BOE_BANK_RATE_URI, profile, budget=budget + ) + return MarketContextSourceSnapshotV1( + source_key="boe.bank-rate", + source_name="Bank of England official Bank Rate history", + source_uri=BOE_BANK_RATE_URI, + retrieved_at_ns=retrieved, + content=content, + content_type=content_type, + adapter_name=BankOfEnglandBankRateAdapterV1.adapter_name, + adapter_version=BankOfEnglandBankRateAdapterV1.adapter_version, + license_name="Open Government Licence v3.0", + redistribution_allowed=True, + redistribution_constraints=( + "Attribute the Bank of England and OGL v3.0; third-party database series remain excluded.", + ), + limitations=( + "Only the official Bank Rate table is selected; no third-party exchange-rate series are reused.", + "The table supplies date changes and levels but not historical publication vintages.", + ), + metadata={ + "provider": "Bank of England", + "series": "IUDBEDR", + "license_uri": "https://www.bankofengland.co.uk/legal", + }, + ) + + +def _fetch_fed_snapshots( + profile: MarketContextFetchProfileV1, + budget: _AcquisitionBudget, +) -> list[MarketContextSourceSnapshotV1]: + snapshots: list[MarketContextSourceSnapshotV1] = [] + start_year = _parse_date(profile.start_date, "start_date").year + end_year = _parse_date(profile.end_date, "end_date").year + for year in range(max(2000, start_year), min(2020, end_year) + 1): + uri = FED_FOMC_HISTORICAL_URI.format(year=year) + content, content_type, retrieved = _fetch_bytes( + uri, profile, budget=budget + ) + snapshots.append( + MarketContextSourceSnapshotV1( + source_key=f"fed.fomc-historical.{year}", + source_name=( + f"Federal Reserve FOMC historical materials {year}" + ), + source_uri=uri, + retrieved_at_ns=retrieved, + content=content, + content_type=content_type, + adapter_name=( + FederalReserveFomcHistoricalAdapterV1.adapter_name + ), + adapter_version=( + FederalReserveFomcHistoricalAdapterV1.adapter_version + ), + license_name="United States public domain", + redistribution_allowed=True, + redistribution_constraints=( + "Cite the Federal Reserve Board as source.", + ), + limitations=( + "Historical pages prove meeting-day records but not original schedule-publication times.", + "Date-only records are exposed ex-ante only after the local meeting day.", + ), + metadata={ + "provider": "Federal Reserve Board", + "year": year, + "license_uri": "https://www.federalreserve.gov/disclaimer.htm", + "historical_index_uri": "https://www.federalreserve.gov/monetarypolicy/fomc_historical.htm", + }, + ) + ) + content, content_type, retrieved = _fetch_bytes( + FED_FOMC_CALENDAR_URI, profile, budget=budget + ) + snapshots.append( + MarketContextSourceSnapshotV1( + source_key="fed.fomc-calendar", + source_name="Federal Reserve FOMC meeting calendar", + source_uri=FED_FOMC_CALENDAR_URI, + retrieved_at_ns=retrieved, + content=content, + content_type=content_type, + adapter_name=FederalReserveFomcCalendarAdapterV1.adapter_name, + adapter_version=FederalReserveFomcCalendarAdapterV1.adapter_version, + license_name="United States public domain", + redistribution_allowed=True, + redistribution_constraints=( + "Cite the Federal Reserve Board as source.", + ), + limitations=( + "The live calendar currently covers 2021 onward and is not a historical schedule-vintage archive.", + "First-known times are conservative lower-information bounds derived from event or retrieval time.", + ), + metadata={ + "provider": "Federal Reserve Board", + "license_uri": "https://www.federalreserve.gov/disclaimer.htm", + "announcement_time_evidence": "https://www.federalreserve.gov/economy-at-a-glance-policy-rate.htm", + }, + ) + ) + return snapshots + + +def _read_operator_snapshot(path: str | Path) -> MarketContextSourceSnapshotV1: + source = Path(path).expanduser().resolve() + content = source.read_bytes() + if not content or len(content) > MAX_CORPUS_BYTES: + raise ValueError( + "operator market-context catalog is empty or unbounded" + ) + return MarketContextSourceSnapshotV1( + source_key="operator.shock-catalog", + source_name="HistDataCom operator-maintained public shock catalog", + source_uri=source.as_uri(), + retrieved_at_ns=time.time_ns(), + content=content, + content_type="application/json", + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + adapter_version=OperatorMarketContextCatalogAdapterV1.adapter_version, + license_name="MIT-licensed normalized factual metadata; upstream terms cited per record", + redistribution_allowed=True, + redistribution_constraints=( + "Retain the catalog's upstream public-record citations and limitations.", + ), + limitations=( + "The catalog is deliberately selective and cannot establish that an unlisted shock did not occur.", + "Window-only dates do not claim an exact market-impact timestamp or causal boundary.", + ), + metadata={"provider": "HistDataCom operator catalog"}, + ) + + +def _fetch_bytes( + uri: str, + profile: MarketContextFetchProfileV1, + *, + budget: _AcquisitionBudget | None = None, +) -> tuple[bytes, str, int]: + if budget is not None: + budget.check() + response_limit = profile.max_response_bytes + request_timeout = profile.timeout_seconds + if budget is not None: + response_limit = min(response_limit, budget.remaining_bytes) + request_timeout = min(request_timeout, budget.remaining_seconds) + if response_limit <= 0: + raise ValueError( + "market-context acquisition exhausted total source-byte limit" + ) + try: + response = requests.get( + uri, + headers={"User-Agent": DEFAULT_USER_AGENT, "Accept": "*/*"}, + timeout=request_timeout, + stream=True, + ) + try: + response.raise_for_status() + declared = response.headers.get("Content-Length") + if declared and int(declared) > response_limit: + raise ValueError( + "market-context response exceeds declared byte limit" + ) + chunks: list[bytes] = [] + size = 0 + for chunk in response.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + size += len(chunk) + if size > response_limit: + raise ValueError( + "market-context response exceeds byte limit" + ) + chunks.append(chunk) + if budget is not None: + budget.check() + finally: + response.close() + except requests.RequestException as exc: + raise ValueError( + f"market-context source request failed: {uri}" + ) from exc + content = b"".join(chunks) + if not content: + raise ValueError(f"market-context source returned an empty body: {uri}") + if budget is not None: + budget.consume(len(content)) + return ( + content, + str( + response.headers.get("Content-Type") or "application/octet-stream" + ).split(";", 1)[0], + time.time_ns(), + ) + + +def _adapter_for_snapshot(snapshot: MarketContextSourceSnapshotV1) -> Any: + adapters = { + OnsReleaseCalendarAdapterV1.adapter_name: OnsReleaseCalendarAdapterV1, + EcbPolicyRateAdapterV1.adapter_name: EcbPolicyRateAdapterV1, + BankOfEnglandBankRateAdapterV1.adapter_name: BankOfEnglandBankRateAdapterV1, + FederalReserveFomcCalendarAdapterV1.adapter_name: FederalReserveFomcCalendarAdapterV1, + FederalReserveFomcHistoricalAdapterV1.adapter_name: FederalReserveFomcHistoricalAdapterV1, + OperatorMarketContextCatalogAdapterV1.adapter_name: OperatorMarketContextCatalogAdapterV1, + } + try: + adapter_type = adapters[snapshot.adapter_name] + except KeyError as exc: + raise ValueError( + f"unsupported market-context adapter: {snapshot.adapter_name}" + ) from exc + return adapter_type(snapshot) + + +def _source_family_for_adapter(adapter_name: str) -> str: + families = { + OnsReleaseCalendarAdapterV1.adapter_name: "ons", + EcbPolicyRateAdapterV1.adapter_name: "ecb", + BankOfEnglandBankRateAdapterV1.adapter_name: "boe", + FederalReserveFomcCalendarAdapterV1.adapter_name: "fed", + FederalReserveFomcHistoricalAdapterV1.adapter_name: "fed", + OperatorMarketContextCatalogAdapterV1.adapter_name: "operator", + } + try: + return families[adapter_name] + except KeyError as exc: + raise ValueError( + f"unsupported market-context adapter: {adapter_name}" + ) from exc + + +def _event_counts_by_year_currency_kind( + events: Sequence[MarketContextEventV1], +) -> dict[str, int]: + counts: Counter[str] = Counter() + for event in events: + year = datetime.fromtimestamp( + event.event_time_ns / 1_000_000_000, tz=timezone.utc + ).year + for currency in event.affected_currencies: + counts[f"{year}|{currency}|{event.kind.value}"] += 1 + return dict(sorted(counts.items())) + + +def _deduplicate_events( + events: Sequence[MarketContextEventV1], +) -> tuple[tuple[MarketContextEventV1, ...], int]: + grouped: dict[str, dict[int, list[MarketContextEventV1]]] = {} + for event in events: + grouped.setdefault(event.canonical_key, {}).setdefault( + event.revision_sequence, [] + ).append(event) + retained: list[MarketContextEventV1] = [] + duplicate_count = 0 + for canonical in sorted(grouped): + revisions = grouped[canonical] + expected_revisions = list(range(max(revisions) + 1)) + if sorted(revisions) != expected_revisions: + raise ValueError( + f"market-context revisions are not contiguous: {canonical}" + ) + predecessor: MarketContextEventV1 | None = None + for revision in expected_revisions: + candidates = sorted( + revisions[revision], key=lambda item: item.event_id + ) + reference = candidates[0] + for candidate in candidates[1:]: + if _event_semantic_key(candidate) != _event_semantic_key( + reference + ): + raise ValueError( + "conflicting duplicate market-context event: " + + canonical + ) + duplicate_count += 1 + expected_predecessor = ( + predecessor.event_id if predecessor is not None else None + ) + if reference.supersedes_event_id != expected_predecessor: + reference = replace( + reference, + supersedes_event_id=expected_predecessor, + event_id="", + ) + retained.append(reference) + predecessor = reference + return ( + tuple( + sorted( + retained, + key=lambda item: ( + item.event_time_ns, + item.canonical_key, + item.revision_sequence, + item.event_id, + ), + ) + ), + duplicate_count, + ) + + +def _event_semantic_key(event: MarketContextEventV1) -> tuple[Any, ...]: + """Return duplicate equivalence excluding only acquisition provenance. + + The same upstream logical record can arrive through multiple ONS query + pages, so source snapshot identity and the derived event ID are excluded. + Every event semantic, window, value, precision, and normalized row hash is + compared; disagreement fails closed instead of choosing one candidate. + """ + return ( + event.schema_version, + event.canonical_key, + event.kind.value, + event.title, + event.source_event_time, + event.source_timezone, + event.source_time_fold, + event.event_time_ns, + event.first_known_at_ns, + event.available_at_ns, + event.pre_event_ns, + event.post_event_ns, + event.affected_currencies, + event.affected_symbols, + event.confidence, + event.precision.value, + event.ambiguity_reason, + event.limitations, + event.vintage_id, + event.revision_sequence, + event.expected_value, + event.actual_value, + event.previous_value, + event.revised_previous_value, + event.surprise_value, + event.value_unit, + event.content_sha256, + event.tags, + ) + + +def _coverage_slices( + profile: MarketContextFetchProfileV1, + snapshots: Sequence[MarketContextSourceSnapshotV1], + source_events: Mapping[str, tuple[MarketContextEventV1, ...]], + source_support: Mapping[str, tuple[int, int, bool]], + source_diagnostics: Mapping[str, tuple[str, ...]], + retained: Sequence[MarketContextEventV1], +) -> tuple[MarketContextCoverageSliceV1, ...]: + families = {item.adapter_name for item in snapshots} + result: list[MarketContextCoverageSliceV1] = [] + official_specs = ( + ( + EcbPolicyRateAdapterV1.adapter_name, + "ecb", + "EUR", + MarketContextKind.POLICY_RATE_CHANGE, + ), + ( + BankOfEnglandBankRateAdapterV1.adapter_name, + "boe", + "GBP", + MarketContextKind.POLICY_RATE_CHANGE, + ), + ) + for adapter_name, family, currency, kind in official_specs: + if adapter_name not in families: + continue + supports = [ + source_support[item.source_key] + for item in snapshots + if item.adapter_name == adapter_name + and item.source_key in source_support + ] + if len(supports) != 1: + continue + support_start, support_end, support_complete = supports[0] + start = max(profile.coverage_start_ns, support_start) + end = min(profile.coverage_end_ns, support_end) + if end <= start: + continue + count = sum( + 1 + for item in retained + if item.kind is kind + and currency in item.affected_currencies + and item.source.adapter_name == adapter_name + ) + result.append( + MarketContextCoverageSliceV1( + source_family=family, + currency=currency, + kind=kind, + coverage_start_ns=start, + coverage_end_ns=end, + complete=support_complete, + event_count=count, + missingness_reason=( + "coverage is bounded by the parsed official source history; zero transitions inside a supported interval means no effective rate change" + ), + ) + ) + bounded_specs = ( + ( + OnsReleaseCalendarAdapterV1.adapter_name, + "ons", + "GBP", + MarketContextKind.MACRO_RELEASE, + ), + ) + for adapter_name, family, currency, kind in bounded_specs: + events = [ + item + for item in retained + if item.source.adapter_name == adapter_name + if profile.coverage_start_ns + <= item.event_time_ns + < profile.coverage_end_ns + ] + if not events: + continue + start = min(item.event_time_ns for item in events) + end = min( + profile.coverage_end_ns, + max(item.event_time_ns for item in events) + DAY_NS, + ) + result.append( + MarketContextCoverageSliceV1( + source_family=family, + currency=currency, + kind=kind, + coverage_start_ns=start, + coverage_end_ns=end, + complete=not _has_structural_diagnostics( + snapshots, + source_diagnostics, + adapter_name, + ignored_prefixes=("cancelled_release:",), + ), + event_count=len(events), + missingness_reason=( + "coverage is bounded by the first and last retained live-archive records" + ), + ) + ) + retained_fed_events = [ + item + for item in retained + if item.source.adapter_name + in { + FederalReserveFomcHistoricalAdapterV1.adapter_name, + FederalReserveFomcCalendarAdapterV1.adapter_name, + } + if profile.coverage_start_ns + <= item.event_time_ns + < profile.coverage_end_ns + ] + raw_fed_events = [ + event + for snapshot in snapshots + if snapshot.adapter_name + in { + FederalReserveFomcHistoricalAdapterV1.adapter_name, + FederalReserveFomcCalendarAdapterV1.adapter_name, + } + for event in source_events.get(snapshot.source_key, ()) + ] + if retained_fed_events and raw_fed_events: + result.append( + MarketContextCoverageSliceV1( + source_family="fed", + currency="USD", + kind=MarketContextKind.CENTRAL_BANK_DECISION, + coverage_start_ns=profile.coverage_start_ns, + coverage_end_ns=min( + profile.coverage_end_ns, + max(item.event_time_ns for item in raw_fed_events) + DAY_NS, + ), + complete=not _has_structural_diagnostics( + snapshots, + source_diagnostics, + FederalReserveFomcHistoricalAdapterV1.adapter_name, + ignored_prefixes=("cancelled_historical_meeting:",), + ) + and not _has_structural_diagnostics( + snapshots, + source_diagnostics, + FederalReserveFomcCalendarAdapterV1.adapter_name, + ignored_prefixes=(), + ), + event_count=len(retained_fed_events), + missingness_reason=( + "coverage is bounded by retained official historical and current FOMC pages" + ), + ) + ) + operator_events = [ + item + for item in retained + if item.source.adapter_name + == OperatorMarketContextCatalogAdapterV1.adapter_name + ] + for currency in ("EUR", "GBP", "USD"): + count = sum( + currency in item.affected_currencies + and item.kind is MarketContextKind.UNSCHEDULED_SHOCK + for item in operator_events + ) + if count: + result.append( + MarketContextCoverageSliceV1( + source_family="operator", + currency=currency, + kind=MarketContextKind.UNSCHEDULED_SHOCK, + coverage_start_ns=profile.coverage_start_ns, + coverage_end_ns=profile.coverage_end_ns, + complete=False, + event_count=count, + missingness_reason=( + "the curated shock list is selective; absence cannot establish a no-shock state" + ), + ) + ) + return tuple(result) + + +def _has_structural_diagnostics( + snapshots: Sequence[MarketContextSourceSnapshotV1], + diagnostics: Mapping[str, tuple[str, ...]], + adapter_name: str, + *, + ignored_prefixes: tuple[str, ...], +) -> bool: + return any( + not item.startswith(ignored_prefixes) + for snapshot in snapshots + if snapshot.adapter_name == adapter_name + for item in diagnostics.get(snapshot.source_key, ()) + ) + + +class _BankRateTableParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.in_table = False + self.in_cell = False + self.cell_parts: list[str] = [] + self.current_row: list[str] = [] + self.rows: list[tuple[str, ...]] = [] + + def handle_starttag( + self, tag: str, attrs: list[tuple[str, str | None]] + ) -> None: + attributes = dict(attrs) + if tag.lower() == "table" and attributes.get("id") == "stats-table": + self.in_table = True + elif self.in_table and tag.lower() == "tr": + self.current_row = [] + elif self.in_table and tag.lower() == "td": + self.in_cell = True + self.cell_parts = [] + + def handle_data(self, data: str) -> None: + if self.in_cell: + self.cell_parts.append(data) + + def handle_endtag(self, tag: str) -> None: + if self.in_table and tag.lower() == "td" and self.in_cell: + self.current_row.append(" ".join("".join(self.cell_parts).split())) + self.in_cell = False + elif self.in_table and tag.lower() == "tr": + if self.current_row: + self.rows.append(tuple(self.current_row)) + self.current_row = [] + elif self.in_table and tag.lower() == "table": + self.in_table = False + + +def _fomc_end_month(value: str) -> int: + key = value.strip().split("/")[-1].lower() + aliases = { + "jan": 1, + "january": 1, + "feb": 2, + "february": 2, + "mar": 3, + "march": 3, + "apr": 4, + "april": 4, + "may": 5, + "jun": 6, + "june": 6, + "jul": 7, + "july": 7, + "aug": 8, + "august": 8, + "sep": 9, + "september": 9, + "oct": 10, + "october": 10, + "nov": 11, + "november": 11, + "dec": 12, + "december": 12, + } + try: + return aliases[key] + except KeyError as exc: + raise ValueError("unsupported FOMC month") from exc + + +def _verify_snapshot_adapter( + snapshot: MarketContextSourceSnapshotV1, + adapter_name: str, + adapter_version: str, +) -> None: + if ( + snapshot.adapter_name != adapter_name + or snapshot.adapter_version != adapter_version + ): + raise ValueError("source snapshot adapter identity differs") + + +def _write_once(path: Path, content: bytes) -> None: + if path.exists(): + if path.read_bytes() != content: + raise ValueError( + f"immutable artifact already exists with different content: {path}" + ) + return + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, path) + except FileExistsError: + if path.read_bytes() != content: + raise ValueError( + "immutable artifact already exists with different content: " + f"{path}" + ) from None + finally: + temporary.unlink(missing_ok=True) + + +def _content_suffix(content_type: str) -> str: + lowered = content_type.lower() + if "json" in lowered: + return ".json" + if "csv" in lowered: + return ".csv" + if "html" in lowered: + return ".html" + return ".bin" + + +def _row_sha256(row: Mapping[str | None, Any]) -> str: + normalized = { + str(key): str(value) + for key, value in sorted(row.items(), key=lambda item: str(item[0])) + } + return hashlib.sha256( + json.dumps(normalized, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + ).hexdigest() + + +def _parse_date(value: str, name: str) -> date: + try: + return date.fromisoformat(str(value)) + except ValueError as exc: + raise ValueError(f"{name} must be YYYY-MM-DD") from exc + + +def _normalize_catalog_datetime( + value: str, + timezone_name: str, + *, + fold: int | None, +) -> int: + text = str(value) + try: + parsed = datetime.fromisoformat( + text.removesuffix("Z") + ("+00:00" if text.endswith("Z") else "") + ) + except ValueError as exc: + raise ValueError("operator catalog time must be ISO-8601") from exc + return int( + normalize_market_context_datetime( + text, + timezone_name, + fold=fold if parsed.tzinfo is None else None, + ) + ) + + +def _date_start_ns(value: date) -> int: + return ( + int( + datetime.combine( + value, datetime.min.time(), tzinfo=timezone.utc + ).timestamp() + ) + * 1_000_000_000 + ) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _source_key(value: Any) -> str: + text = str(value or "").strip().lower() + if not _SOURCE_KEY_RE.fullmatch(text): + raise ValueError("invalid market-context source key") + return text + + +def _required_text(value: Any) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("required text is empty") + if len(text) > 2048: + raise ValueError("text exceeds market-context corpus limit") + return text + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return _required_text(text) if text else None + + +def _sha256(value: str, name: str) -> str: + if not _SHA256_RE.fullmatch(str(value)): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return str(value) + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _optional_int(value: Any) -> int | None: + if value is None: + return None + return _strict_int(value, "optional integer") + + +def _bounded_int(value: Any, name: str, lower: int, upper: int) -> int: + result = _strict_int(value, name) + if not lower <= result <= upper: + raise ValueError(f"{name} is outside its bound") + return result + + +def _number(value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("expected a numeric value") + return float(value) + + +def _optional_number(value: Any) -> float | None: + return None if value is None else _number(value) + + +def _positive_float(value: Any, name: str) -> float: + result = _number(value) + if not 0.0 < result < float("inf"): + raise ValueError(f"{name} must be finite and positive") + return result + + +def _finite_nonnegative(value: Any, name: str) -> float: + result = _number(value) + if not 0.0 <= result < float("inf"): + raise ValueError(f"{name} must be finite and non-negative") + return result + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a boolean") + return value + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return cast(Mapping[str, Any], value) + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError("expected a sequence") + return value + + +def _sequence_or_empty(value: Any) -> Sequence[Any]: + return () if value is None else _sequence(value) + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + return tuple(_mapping(item) for item in _sequence(value)) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _json_value_mapping(value: Mapping[str, Any]) -> dict[str, JSONValue]: + encoded = json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ) + restored = json.loads(encoded) + return cast(dict[str, JSONValue], restored) + + +def _validate_json_mapping(value: Mapping[str, JSONValue], name: str) -> None: + try: + encoded = canonical_contract_json(dict(value)).encode("utf-8") + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} is not JSON-compatible") from exc + if len(encoded) > 16_384: + raise ValueError(f"{name} exceeds 16 KiB") + + +__all__ = [ + "BankOfEnglandBankRateAdapterV1", + "DEFAULT_MARKET_CONTEXT_SOURCES", + "DEFAULT_ONS_QUERIES", + "EcbPolicyRateAdapterV1", + "FederalReserveFomcCalendarAdapterV1", + "FederalReserveFomcHistoricalAdapterV1", + "MARKET_CONTEXT_CORPUS_SCHEMA_VERSION", + "MARKET_CONTEXT_COVERAGE_SCHEMA_VERSION", + "MARKET_CONTEXT_PREFLIGHT_SCHEMA_VERSION", + "MARKET_CONTEXT_SOURCE_EVIDENCE_SCHEMA_VERSION", + "MarketContextCorpusBuildV1", + "MarketContextCorpusPreflightError", + "MarketContextCorpusPreflightV1", + "MarketContextCorpusV1", + "MarketContextCoverageSliceV1", + "MarketContextFetchProfileV1", + "MarketContextSourceEvidenceV1", + "MarketContextSourceSnapshotV1", + "OnsReleaseCalendarAdapterV1", + "OperatorMarketContextCatalogAdapterV1", + "build_live_market_context_corpus", + "build_market_context_corpus_from_snapshots", + "market_context_benchmark_event_state", + "packaged_operator_catalog_path", + "preflight_market_context_corpus", + "query_market_context_corpus", + "read_market_context_corpus", + "replay_market_context_corpus", + "require_market_context_corpus", + "write_market_context_corpus", +] diff --git a/src/histdatacom/market_context/positioning.py b/src/histdatacom/market_context/positioning.py new file mode 100644 index 00000000..481e0cc6 --- /dev/null +++ b/src/histdatacom/market_context/positioning.py @@ -0,0 +1,4084 @@ +"""Point-in-time-safe CFTC Commitments of Traders positioning state. + +Commitments of Traders is persistent weekly futures positioning, not a market +event and not spot-FX volume. This module deliberately keeps COT outside the +``MarketContextEventV1`` timeline while reusing the same bounded acquisition, +content-addressed artifact, replay, preflight, and information-audit patterns. + +The CFTC Public Reporting Environment (PRE) exposes current historical rows, +not a complete archive of every value as originally published. Consequently, +PRE-derived rows remain ``current_state_only`` and cannot pass strict ex-ante +queries merely because an official publication time is known. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import math +import os +import re +import statistics +import time +import tempfile +import zipfile +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from datetime import date, datetime, time as datetime_time, timedelta, timezone +from enum import Enum +from pathlib import Path +from typing import Any, Protocol, cast +from zoneinfo import ZoneInfo + +import requests + +from histdatacom.market_context.contracts import canonical_contract_json +from histdatacom.resource_usage import peak_rss_bytes +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.information import ( + InformationInputKind, + InformationMode, + InformationScope, + InformationSplitKind, + InformationStage, + ReconstructionInformationInputV1, +) + +CFTC_POSITIONING_PROFILE_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-fetch-profile.v1" +) +CFTC_POSITIONING_SOURCE_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-source.v1" +) +CFTC_POSITIONING_MAPPING_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-symbol-mapping.v1" +) +CFTC_POSITIONING_RELEASE_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-release-evidence.v1" +) +CFTC_POSITIONING_SNAPSHOT_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-snapshot.v1" +) +CFTC_POSITIONING_COVERAGE_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-coverage.v1" +) +CFTC_POSITIONING_ARCHIVE_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-archive-consistency.v1" +) +CFTC_POSITIONING_CORPUS_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-corpus.v1" +) +CFTC_POSITIONING_DIFF_SCHEMA_VERSION = "histdatacom.cftc-positioning-diff.v1" +CFTC_POSITIONING_PREFLIGHT_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-preflight.v1" +) +CFTC_POSITIONING_QUERY_SCHEMA_VERSION = "histdatacom.cftc-positioning-query.v1" +CFTC_POSITIONING_BINDING_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-consumer-binding.v1" +) +CFTC_POSITIONING_SMOKE_SCHEMA_VERSION = ( + "histdatacom.cftc-positioning-benchmark-smoke.v1" +) + +CFTC_LEGACY_DATASET_ID = "srt6-5q2f" +CFTC_TFF_DATASET_ID = "udgc-27he" +CFTC_PRE_RESOURCE_TEMPLATE = ( + "https://publicreporting.cftc.gov/resource/{dataset_id}.json" +) +CFTC_PRE_METADATA_TEMPLATE = ( + "https://publicreporting.cftc.gov/api/views/{dataset_id}.json" +) +CFTC_RELEASE_SCHEDULE_URI = "https://www.cftc.gov/MarketReports/CommitmentsofTraders/ReleaseSchedule/index.htm" +CFTC_SPECIAL_ANNOUNCEMENTS_URI = ( + "https://www.cftc.gov/MarketReports/CommitmentsofTraders/" + "HistoricalSpecialAnnouncements/index.htm" +) +CFTC_HISTORICAL_COMPRESSED_URI = ( + "https://www.cftc.gov/MarketReports/CommitmentsofTraders/" + "HistoricalCompressed/index.htm" +) +CFTC_2025_BACKLOG_URI = "https://www.cftc.gov/PressRoom/PressReleases/9147-25" +CFTC_WEB_POLICY_URI = "https://www.cftc.gov/WebPolicy/index.htm" +CME_FX_QUOTE_URI = ( + "https://www.cmegroup.com/education/courses/introduction-to-fx/" + "understanding-fx-quote-conventions" +) +CME_EURGBP_RULE_URI = ( + "https://www.cmegroup.com/rulebook/CME/III/300/301/301.pdf" +) + +CFTC_CONTRACT_CODES = ("096742", "099741", "299741") +CFTC_CONTRACT_SYMBOLS: Mapping[str, tuple[str, ...]] = { + "096742": ("GBPUSD",), + "099741": ("EURUSD",), + "299741": ("EURGBP",), +} +CFTC_DIRECT_EURGBP_START = date(2014, 6, 10) +CFTC_TFF_START = date(2006, 6, 13) +CFTC_HISTORICAL_ARCHIVES = ( + ("legacy", "futures_only", "deacot1986_2016.zip"), + ("legacy", "combined", "deahistfo_1995_2016.zip"), + ("tff", "futures_only", "fin_fut_txt_2006_2016.zip"), + ("tff", "combined", "fin_com_txt_2006_2016.zip"), +) +CFTC_HISTORICAL_ARCHIVE_TEMPLATE = ( + "https://www.cftc.gov/files/dea/history/{archive_name}" +) +DEFAULT_CFTC_USER_AGENT = "histdatacom-cftc-positioning/2.1.0 (+https://github.com/dmidlo/histdata.com-tools)" +CFTC_POSITIONING_ADAPTER_VERSION = "cftc-pre-positioning-adapter-v1" + +DAY_NS = 86_400_000_000_000 +MAX_CFTC_ROWS = 100_000 +MAX_CFTC_SOURCES = 4096 +MAX_CFTC_COVERAGE_SLICES = 2048 +MAX_CFTC_VALUES = 160 +MAX_CFTC_QUERY_SNAPSHOTS = 24 +MAX_CFTC_QUERY_BYTES = 2 * 1024 * 1024 +MAX_CFTC_CORPUS_BYTES = 128 * 1024 * 1024 +MAX_CFTC_SOURCE_BYTES = 128 * 1024 * 1024 +MAX_CFTC_TOTAL_SOURCE_BYTES = 384 * 1024 * 1024 +MAX_CFTC_TEXT = 2048 +MAX_CFTC_BINDING_VALUES = 256 +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_SOURCE_KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._:-]{0,191}$") +_CONTRACT_CODE_RE = re.compile(r"^[0-9A-Z+]{6}$") + + +class CftcReportFamily(str, Enum): + """CFTC classification schema family.""" + + LEGACY = "legacy" + TFF = "tff" + + @classmethod + def from_value(cls, value: str | "CftcReportFamily") -> "CftcReportFamily": + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as exc: + raise ValueError("unsupported CFTC report family") from exc + + +class CftcReportScope(str, Enum): + """Keep futures-only and futures/options-combined reports distinct.""" + + FUTURES_ONLY = "futures_only" + COMBINED = "combined" + + @classmethod + def from_value(cls, value: str | "CftcReportScope") -> "CftcReportScope": + if isinstance(value, cls): + return value + normalized = str(value).strip().lower().replace(" ", "_") + if normalized in {"futonly", "futures", "futuresonly"}: + normalized = cls.FUTURES_ONLY.value + try: + return cls(normalized) + except ValueError as exc: + raise ValueError("unsupported CFTC report scope") from exc + + +class CftcAvailabilityConfidence(str, Enum): + """Strength of publication/knowledge-time evidence.""" + + VERIFIED = "verified" + NOMINAL = "nominal" + UNKNOWN = "unknown" + CORRECTION_QUALIFIED = "correction_qualified" + RESTATEMENT_QUALIFIED = "restatement_qualified" + + @classmethod + def from_value( + cls, value: str | "CftcAvailabilityConfidence" + ) -> "CftcAvailabilityConfidence": + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as exc: + raise ValueError( + "unsupported CFTC availability confidence" + ) from exc + + +class CftcRestatementStatus(str, Enum): + """Whether a row is an original publication vintage or current state.""" + + CURRENT_STATE_ONLY = "current_state_only" + ORIGINAL_VERIFIED = "original_verified" + RESTATED_CURRENT_STATE = "restated_current_state" + RESTATEMENT_INCOMPLETE = "restatement_incomplete" + + @classmethod + def from_value( + cls, value: str | "CftcRestatementStatus" + ) -> "CftcRestatementStatus": + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as exc: + raise ValueError("unsupported CFTC restatement status") from exc + + +class CftcMappingKind(str, Enum): + """Direct contract state or explicit two-leg EURGBP composite.""" + + DIRECT = "direct" + DERIVED_TWO_LEG = "derived_two_leg" + + +class CftcPositioningQueryStatus(str, Enum): + """Fail-closed latest-known-state query outcome.""" + + READY = "ready" + MISSING = "missing" + STALE = "stale" + NOT_AVAILABLE = "not_available_as_of" + UNSUPPORTED = "unsupported" + RESTATEMENT_INCOMPLETE = "restatement_incomplete" + + +class CftcPositioningConsumer(str, Enum): + """Production seams that retain a positioning query identity.""" + + BENCHMARK = "benchmark" + MOTIF_SELECTION = "motif_selection" + PLANNING = "planning" + CARVING = "carving" + + +@dataclass(frozen=True, slots=True) +class CftcPositioningFetchProfileV1: + """Bounded deterministic CFTC acquisition policy.""" + + start_date: str + end_date: str + dataset_ids: tuple[str, ...] = ( + CFTC_LEGACY_DATASET_ID, + CFTC_TFF_DATASET_ID, + ) + contract_codes: tuple[str, ...] = CFTC_CONTRACT_CODES + historical_archives: tuple[str, ...] = tuple( + item[2] for item in CFTC_HISTORICAL_ARCHIVES + ) + page_size: int = 1000 + max_pages_per_dataset: int = 128 + timeout_seconds: float = 45.0 + max_response_bytes: int = 32 * 1024 * 1024 + max_total_source_bytes: int = 256 * 1024 * 1024 + max_rows: int = MAX_CFTC_ROWS + max_runtime_seconds: float = 600.0 + max_peak_memory_bytes: int = 2 * 1024**3 + max_staleness_days: int = 14 + user_agent: str = DEFAULT_CFTC_USER_AGENT + schema_version: str = CFTC_POSITIONING_PROFILE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_PROFILE_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning profile schema") + start = _parse_date(self.start_date, "start_date") + end = _parse_date(self.end_date, "end_date") + if end < start: + raise ValueError("CFTC positioning end_date precedes start_date") + datasets = tuple( + sorted({_required_text(item) for item in self.dataset_ids}) + ) + if set(datasets) != {CFTC_LEGACY_DATASET_ID, CFTC_TFF_DATASET_ID}: + raise ValueError( + "CFTC positioning requires Legacy and TFF datasets" + ) + codes = tuple( + sorted({_contract_code(item) for item in self.contract_codes}) + ) + if not set(CFTC_CONTRACT_CODES).issubset(codes): + raise ValueError( + "CFTC positioning requires EUR, GBP, and EURGBP codes" + ) + archives = tuple( + dict.fromkeys( + _required_text(item) for item in self.historical_archives + ) + ) + allowed_archives = {item[2] for item in CFTC_HISTORICAL_ARCHIVES} + if not set(archives).issubset(allowed_archives): + raise ValueError("unsupported CFTC historical archive") + _bounded_int(self.page_size, "page_size", 1, 50_000) + _bounded_int( + self.max_pages_per_dataset, "max_pages_per_dataset", 1, 1024 + ) + _positive_float(self.timeout_seconds, "timeout_seconds") + _bounded_int( + self.max_response_bytes, + "max_response_bytes", + 1, + MAX_CFTC_SOURCE_BYTES, + ) + _bounded_int( + self.max_total_source_bytes, + "max_total_source_bytes", + 1, + MAX_CFTC_TOTAL_SOURCE_BYTES, + ) + _bounded_int(self.max_rows, "max_rows", 1, MAX_CFTC_ROWS) + _positive_float(self.max_runtime_seconds, "max_runtime_seconds") + _bounded_int( + self.max_peak_memory_bytes, + "max_peak_memory_bytes", + 1, + 16 * 1024**3, + ) + _bounded_int(self.max_staleness_days, "max_staleness_days", 1, 365) + object.__setattr__(self, "start_date", start.isoformat()) + object.__setattr__(self, "end_date", end.isoformat()) + object.__setattr__(self, "dataset_ids", datasets) + object.__setattr__(self, "contract_codes", codes) + object.__setattr__(self, "historical_archives", archives) + object.__setattr__(self, "user_agent", _required_text(self.user_agent)) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "start_date": self.start_date, + "end_date": self.end_date, + "dataset_ids": list(self.dataset_ids), + "contract_codes": list(self.contract_codes), + "historical_archives": list(self.historical_archives), + "page_size": self.page_size, + "max_pages_per_dataset": self.max_pages_per_dataset, + "timeout_seconds": self.timeout_seconds, + "max_response_bytes": self.max_response_bytes, + "max_total_source_bytes": self.max_total_source_bytes, + "max_rows": self.max_rows, + "max_runtime_seconds": self.max_runtime_seconds, + "max_peak_memory_bytes": self.max_peak_memory_bytes, + "max_staleness_days": self.max_staleness_days, + "user_agent": self.user_agent, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CftcPositioningFetchProfileV1": + _require_schema(data, CFTC_POSITIONING_PROFILE_SCHEMA_VERSION) + return cls( + start_date=str(data.get("start_date", "")), + end_date=str(data.get("end_date", "")), + dataset_ids=_string_tuple(data.get("dataset_ids")), + contract_codes=_string_tuple(data.get("contract_codes")), + historical_archives=_string_tuple(data.get("historical_archives")), + page_size=_strict_int(data.get("page_size"), "page_size"), + max_pages_per_dataset=_strict_int( + data.get("max_pages_per_dataset"), "max_pages_per_dataset" + ), + timeout_seconds=_finite_float( + data.get("timeout_seconds"), "timeout_seconds" + ), + max_response_bytes=_strict_int( + data.get("max_response_bytes"), "max_response_bytes" + ), + max_total_source_bytes=_strict_int( + data.get("max_total_source_bytes"), "max_total_source_bytes" + ), + max_rows=_strict_int(data.get("max_rows"), "max_rows"), + max_runtime_seconds=_finite_float( + data.get("max_runtime_seconds"), "max_runtime_seconds" + ), + max_peak_memory_bytes=_strict_int( + data.get("max_peak_memory_bytes"), "max_peak_memory_bytes" + ), + max_staleness_days=_strict_int( + data.get("max_staleness_days"), "max_staleness_days" + ), + user_agent=str(data.get("user_agent", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CftcPositioningRawSourceV1: + """One retained official response plus deterministic query evidence.""" + + source_key: str + source_kind: str + source_uri: str + retrieved_at_ns: int + content: bytes + content_type: str + query_parameters: Mapping[str, str] = field(default_factory=dict) + dataset_id: str | None = None + report_family: CftcReportFamily | None = None + report_scope: CftcReportScope | None = None + redistribution_allowed: bool = True + limitations: tuple[str, ...] = () + schema_version: str = CFTC_POSITIONING_SOURCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_SOURCE_SCHEMA_VERSION: + raise ValueError("unsupported CFTC source schema") + object.__setattr__(self, "source_key", _source_key(self.source_key)) + object.__setattr__(self, "source_kind", _source_key(self.source_kind)) + object.__setattr__(self, "source_uri", _required_text(self.source_uri)) + _bounded_int64(self.retrieved_at_ns, "retrieved_at_ns") + if not isinstance(self.content, bytes) or not self.content: + raise ValueError("CFTC source content must be non-empty bytes") + if len(self.content) > MAX_CFTC_SOURCE_BYTES: + raise ValueError("CFTC source exceeds per-response byte bound") + object.__setattr__( + self, "content_type", _required_text(self.content_type) + ) + parameters = { + _required_text(str(key)): _required_text(str(value)) + for key, value in sorted(self.query_parameters.items()) + } + object.__setattr__(self, "query_parameters", parameters) + dataset = _optional_text(self.dataset_id) + if dataset is not None and dataset not in { + CFTC_LEGACY_DATASET_ID, + CFTC_TFF_DATASET_ID, + }: + raise ValueError("unsupported CFTC dataset ID") + object.__setattr__(self, "dataset_id", dataset) + if self.report_family is not None: + object.__setattr__( + self, + "report_family", + CftcReportFamily.from_value(self.report_family), + ) + if self.report_scope is not None: + object.__setattr__( + self, + "report_scope", + CftcReportScope.from_value(self.report_scope), + ) + if not isinstance(self.redistribution_allowed, bool): + raise ValueError("redistribution_allowed must be boolean") + object.__setattr__( + self, + "limitations", + tuple(_required_text(item) for item in self.limitations), + ) + + @property + def content_sha256(self) -> str: + return hashlib.sha256(self.content).hexdigest() + + @property + def source_id(self) -> str: + return _stable_id("cftc-positioning-source", self.identity_payload()) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "adapter_version": CFTC_POSITIONING_ADAPTER_VERSION, + "source_key": self.source_key, + "source_kind": self.source_kind, + "source_uri": self.source_uri, + "content_sha256": self.content_sha256, + "size_bytes": len(self.content), + "content_type": self.content_type, + "query_parameters": dict(self.query_parameters), + "dataset_id": self.dataset_id, + "report_family": ( + self.report_family.value + if self.report_family is not None + else None + ), + "report_scope": ( + self.report_scope.value + if self.report_scope is not None + else None + ), + "redistribution_allowed": self.redistribution_allowed, + "limitations": list(self.limitations), + } + + def evidence_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "retrieved_at_ns": self.retrieved_at_ns, + "source_id": self.source_id, + } + + @classmethod + def restore( + cls, data: Mapping[str, Any], content: bytes + ) -> "CftcPositioningRawSourceV1": + _require_schema(data, CFTC_POSITIONING_SOURCE_SCHEMA_VERSION) + if data.get("adapter_version") != CFTC_POSITIONING_ADAPTER_VERSION: + raise ValueError("CFTC restored source adapter version differs") + family = data.get("report_family") + scope = data.get("report_scope") + source = cls( + source_key=str(data.get("source_key", "")), + source_kind=str(data.get("source_kind", "")), + source_uri=str(data.get("source_uri", "")), + retrieved_at_ns=_strict_int( + data.get("retrieved_at_ns"), "retrieved_at_ns" + ), + content=content, + content_type=str(data.get("content_type", "")), + query_parameters={ + str(key): str(value) + for key, value in _mapping(data.get("query_parameters")).items() + }, + dataset_id=_optional_text(data.get("dataset_id")), + report_family=( + CftcReportFamily.from_value(str(family)) + if family is not None + else None + ), + report_scope=( + CftcReportScope.from_value(str(scope)) + if scope is not None + else None + ), + redistribution_allowed=_strict_bool( + data.get("redistribution_allowed"), "redistribution_allowed" + ), + limitations=_string_tuple(data.get("limitations")), + schema_version=str(data.get("schema_version", "")), + ) + if source.content_sha256 != str(data.get("content_sha256", "")): + raise ValueError("CFTC restored source hash differs") + if len(content) != _strict_int(data.get("size_bytes"), "size_bytes"): + raise ValueError("CFTC restored source size differs") + if source.source_id != str(data.get("source_id", "")): + raise ValueError("CFTC restored source identity differs") + return source + + +@dataclass(frozen=True, slots=True) +class CftcPositioningSymbolMappingV1: + """Versioned contract-code and quote-orientation evidence.""" + + symbol: str + contract_codes: tuple[str, ...] + mapping_kind: CftcMappingKind + quote_convention: str + source_uris: tuple[str, ...] + valid_from_date: str | None = None + notes: tuple[str, ...] = () + mapping_id: str = "" + schema_version: str = CFTC_POSITIONING_MAPPING_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_MAPPING_SCHEMA_VERSION: + raise ValueError("unsupported CFTC symbol mapping schema") + symbol = _normalized_symbol(self.symbol) + codes = tuple(_contract_code(item) for item in self.contract_codes) + kind = CftcMappingKind(self.mapping_kind) + if kind is CftcMappingKind.DIRECT and len(codes) != 1: + raise ValueError("direct CFTC mapping requires one contract") + if kind is CftcMappingKind.DERIVED_TWO_LEG and ( + symbol != "EURGBP" or codes != ("099741", "096742") + ): + raise ValueError( + "derived CFTC mapping must retain EUR and GBP legs" + ) + valid_from = ( + _parse_date(self.valid_from_date, "valid_from_date").isoformat() + if self.valid_from_date is not None + else None + ) + if symbol == "EURGBP" and kind is CftcMappingKind.DIRECT: + if valid_from != CFTC_DIRECT_EURGBP_START.isoformat(): + raise ValueError("direct EURGBP mapping start date differs") + uris = tuple( + sorted({_required_text(item) for item in self.source_uris}) + ) + if not uris: + raise ValueError("CFTC symbol mapping requires source URIs") + notes = tuple(_required_text(item) for item in self.notes) + object.__setattr__(self, "symbol", symbol) + object.__setattr__(self, "contract_codes", codes) + object.__setattr__(self, "mapping_kind", kind) + object.__setattr__( + self, "quote_convention", _required_text(self.quote_convention) + ) + object.__setattr__(self, "source_uris", uris) + object.__setattr__(self, "valid_from_date", valid_from) + object.__setattr__(self, "notes", notes) + expected = _stable_id("cftc-symbol-mapping", self.identity_payload()) + if self.mapping_id and self.mapping_id != expected: + raise ValueError("CFTC symbol mapping identity differs") + object.__setattr__(self, "mapping_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "contract_codes": list(self.contract_codes), + "mapping_kind": self.mapping_kind.value, + "quote_convention": self.quote_convention, + "source_uris": list(self.source_uris), + "valid_from_date": self.valid_from_date, + "notes": list(self.notes), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "mapping_id": self.mapping_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CftcPositioningSymbolMappingV1": + _require_schema(data, CFTC_POSITIONING_MAPPING_SCHEMA_VERSION) + return cls( + symbol=str(data.get("symbol", "")), + contract_codes=_string_tuple(data.get("contract_codes")), + mapping_kind=CftcMappingKind(str(data.get("mapping_kind", ""))), + quote_convention=str(data.get("quote_convention", "")), + source_uris=_string_tuple(data.get("source_uris")), + valid_from_date=_optional_text(data.get("valid_from_date")), + notes=_string_tuple(data.get("notes")), + mapping_id=str(data.get("mapping_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +def default_cftc_positioning_symbol_mappings() -> ( + tuple[CftcPositioningSymbolMappingV1, ...] +): + """Return audited direction and direct/leg mapping contracts.""" + legacy = CFTC_PRE_METADATA_TEMPLATE.format( + dataset_id=CFTC_LEGACY_DATASET_ID + ) + tff = CFTC_PRE_METADATA_TEMPLATE.format(dataset_id=CFTC_TFF_DATASET_ID) + datasets = (legacy, tff) + return ( + CftcPositioningSymbolMappingV1( + symbol="EURUSD", + contract_codes=("099741",), + mapping_kind=CftcMappingKind.DIRECT, + quote_convention="USD per EUR", + source_uris=(*datasets, CME_FX_QUOTE_URI), + notes=("CFTC EUR FX futures positioning maps to EURUSD.",), + ), + CftcPositioningSymbolMappingV1( + symbol="GBPUSD", + contract_codes=("096742",), + mapping_kind=CftcMappingKind.DIRECT, + quote_convention="USD per GBP", + source_uris=(*datasets, CME_FX_QUOTE_URI), + notes=("CFTC British Pound futures positioning maps to GBPUSD.",), + ), + CftcPositioningSymbolMappingV1( + symbol="EURGBP", + contract_codes=("099741", "096742"), + mapping_kind=CftcMappingKind.DERIVED_TWO_LEG, + quote_convention=( + "EURUSD and GBPUSD leg identities; no pooled composite position" + ), + source_uris=(*datasets, CME_FX_QUOTE_URI), + notes=("Before direct support, retain both source-leg IDs.",), + ), + CftcPositioningSymbolMappingV1( + symbol="EURGBP", + contract_codes=("299741",), + mapping_kind=CftcMappingKind.DIRECT, + quote_convention="GBP per EUR", + source_uris=(*datasets, CME_EURGBP_RULE_URI), + valid_from_date=CFTC_DIRECT_EURGBP_START.isoformat(), + notes=("Direct EURGBP CFTC state begins on 2014-06-10.",), + ), + ) + + +@dataclass(frozen=True, slots=True) +class CftcReleaseEvidenceV1: + """Publication/knowledge evidence for one report measurement date.""" + + report_date: str + confidence: CftcAvailabilityConfidence + source_id: str + publication_at_ns: int | None = None + knowledge_at_ns: int | None = None + restatement_detected_at_ns: int | None = None + notes: tuple[str, ...] = () + evidence_id: str = "" + schema_version: str = CFTC_POSITIONING_RELEASE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_RELEASE_SCHEMA_VERSION: + raise ValueError("unsupported CFTC release evidence schema") + report = _parse_date(self.report_date, "report_date") + object.__setattr__(self, "report_date", report.isoformat()) + object.__setattr__( + self, + "confidence", + CftcAvailabilityConfidence.from_value(self.confidence), + ) + object.__setattr__(self, "source_id", _required_text(self.source_id)) + for name in ( + "publication_at_ns", + "knowledge_at_ns", + "restatement_detected_at_ns", + ): + value = getattr(self, name) + if value is not None: + object.__setattr__(self, name, _bounded_int64(value, name)) + if self.publication_at_ns is not None and self.knowledge_at_ns is None: + object.__setattr__(self, "knowledge_at_ns", self.publication_at_ns) + if ( + self.knowledge_at_ns is not None + and self.publication_at_ns is not None + and self.knowledge_at_ns < self.publication_at_ns + ): + raise ValueError("CFTC knowledge time precedes publication") + if self.confidence is CftcAvailabilityConfidence.UNKNOWN and ( + self.publication_at_ns is not None + or self.knowledge_at_ns is not None + ): + raise ValueError( + "unknown CFTC availability cannot invent timestamps" + ) + notes = tuple(_required_text(item) for item in self.notes) + object.__setattr__(self, "notes", notes) + expected = _stable_id("cftc-release-evidence", self.identity_payload()) + if self.evidence_id and self.evidence_id != expected: + raise ValueError("CFTC release evidence identity differs") + object.__setattr__(self, "evidence_id", expected) + + @property + def strict_ex_ante_time_eligible(self) -> bool: + return ( + self.confidence is CftcAvailabilityConfidence.VERIFIED + and self.publication_at_ns is not None + and self.knowledge_at_ns is not None + ) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "report_date": self.report_date, + "confidence": self.confidence.value, + "source_id": self.source_id, + "publication_at_ns": self.publication_at_ns, + "knowledge_at_ns": self.knowledge_at_ns, + "restatement_detected_at_ns": self.restatement_detected_at_ns, + "notes": list(self.notes), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CftcReleaseEvidenceV1": + _require_schema(data, CFTC_POSITIONING_RELEASE_SCHEMA_VERSION) + return cls( + report_date=str(data.get("report_date", "")), + confidence=CftcAvailabilityConfidence.from_value( + str(data.get("confidence", "")) + ), + source_id=str(data.get("source_id", "")), + publication_at_ns=_optional_int(data.get("publication_at_ns")), + knowledge_at_ns=_optional_int(data.get("knowledge_at_ns")), + restatement_detected_at_ns=_optional_int( + data.get("restatement_detected_at_ns") + ), + notes=_string_tuple(data.get("notes")), + evidence_id=str(data.get("evidence_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CftcPositioningSnapshotV1: + """One immutable family/scope/contract/report-date positioning vector.""" + + report_family: CftcReportFamily + report_scope: CftcReportScope + contract_code: str + contract_name: str + market_name: str + report_date: str + dataset_id: str + source_id: str + source_row_sha256: str + pre_row_id: str + release_evidence: CftcReleaseEvidenceV1 + restatement_status: CftcRestatementStatus + values: Mapping[str, float] + snapshot_id: str = "" + schema_version: str = CFTC_POSITIONING_SNAPSHOT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_SNAPSHOT_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning snapshot schema") + family = CftcReportFamily.from_value(self.report_family) + scope = CftcReportScope.from_value(self.report_scope) + code = _contract_code(self.contract_code) + object.__setattr__(self, "report_family", family) + object.__setattr__(self, "report_scope", scope) + object.__setattr__(self, "contract_code", code) + object.__setattr__( + self, "contract_name", _required_text(self.contract_name) + ) + object.__setattr__( + self, "market_name", _required_text(self.market_name) + ) + report = _parse_date(self.report_date, "report_date") + object.__setattr__(self, "report_date", report.isoformat()) + expected_dataset = ( + CFTC_LEGACY_DATASET_ID + if family is CftcReportFamily.LEGACY + else CFTC_TFF_DATASET_ID + ) + if self.dataset_id != expected_dataset: + raise ValueError("CFTC snapshot dataset differs from report family") + object.__setattr__(self, "source_id", _required_text(self.source_id)) + object.__setattr__( + self, + "source_row_sha256", + _required_sha256(self.source_row_sha256, "source_row_sha256"), + ) + object.__setattr__(self, "pre_row_id", _required_text(self.pre_row_id)) + if not isinstance(self.release_evidence, CftcReleaseEvidenceV1): + raise ValueError("CFTC snapshot requires release evidence") + if self.release_evidence.report_date != self.report_date: + raise ValueError("CFTC release evidence report date differs") + object.__setattr__( + self, + "restatement_status", + CftcRestatementStatus.from_value(self.restatement_status), + ) + values = { + _source_key(str(key)): _finite_float(value, str(key)) + for key, value in sorted(self.values.items()) + } + if not values or len(values) > MAX_CFTC_VALUES: + raise ValueError("CFTC snapshot values are empty or unbounded") + if "open_interest_all" not in values or values["open_interest_all"] < 0: + raise ValueError("CFTC snapshot requires nonnegative open interest") + object.__setattr__(self, "values", values) + expected = _stable_id( + "cftc-positioning-snapshot", self.identity_payload() + ) + if self.snapshot_id and self.snapshot_id != expected: + raise ValueError("CFTC snapshot identity differs") + object.__setattr__(self, "snapshot_id", expected) + + @property + def logical_key(self) -> str: + return ":".join( + ( + self.report_family.value, + self.report_scope.value, + self.contract_code, + self.report_date, + ) + ) + + @property + def measurement_start_ns(self) -> int: + return _date_start_ns(_parse_date(self.report_date, "report_date")) + + @property + def valid_from_ns(self) -> int | None: + """Knowledge time when this state may begin to govern a query.""" + return self.release_evidence.knowledge_at_ns + + @property + def strict_ex_ante_eligible(self) -> bool: + return ( + self.release_evidence.strict_ex_ante_time_eligible + and self.restatement_status + is CftcRestatementStatus.ORIGINAL_VERIFIED + ) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "report_family": self.report_family.value, + "report_scope": self.report_scope.value, + "contract_code": self.contract_code, + "contract_name": self.contract_name, + "market_name": self.market_name, + "report_date": self.report_date, + "measurement_start_ns": self.measurement_start_ns, + "valid_from_ns": self.valid_from_ns, + "dataset_id": self.dataset_id, + "source_id": self.source_id, + "source_row_sha256": self.source_row_sha256, + "pre_row_id": self.pre_row_id, + "release_evidence": self.release_evidence.to_dict(), + "restatement_status": self.restatement_status.value, + "values": dict(self.values), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "logical_key": self.logical_key, + "measurement_date_only": True, + "snapshot_id": self.snapshot_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CftcPositioningSnapshotV1": + _require_schema(data, CFTC_POSITIONING_SNAPSHOT_SCHEMA_VERSION) + snapshot = cls( + report_family=CftcReportFamily.from_value( + str(data.get("report_family", "")) + ), + report_scope=CftcReportScope.from_value( + str(data.get("report_scope", "")) + ), + contract_code=str(data.get("contract_code", "")), + contract_name=str(data.get("contract_name", "")), + market_name=str(data.get("market_name", "")), + report_date=str(data.get("report_date", "")), + dataset_id=str(data.get("dataset_id", "")), + source_id=str(data.get("source_id", "")), + source_row_sha256=str(data.get("source_row_sha256", "")), + pre_row_id=str(data.get("pre_row_id", "")), + release_evidence=CftcReleaseEvidenceV1.from_dict( + _mapping(data.get("release_evidence")) + ), + restatement_status=CftcRestatementStatus.from_value( + str(data.get("restatement_status", "")) + ), + values={ + str(key): _finite_float(value, str(key)) + for key, value in _mapping(data.get("values")).items() + }, + snapshot_id=str(data.get("snapshot_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + if data.get("logical_key") != snapshot.logical_key: + raise ValueError("CFTC snapshot logical key differs") + if data.get("measurement_date_only") is not True: + raise ValueError("CFTC measurement date must remain date-only") + if data.get("measurement_start_ns") != snapshot.measurement_start_ns: + raise ValueError("CFTC measurement start differs from report date") + if data.get("valid_from_ns") != snapshot.valid_from_ns: + raise ValueError( + "CFTC valid-from time differs from release evidence" + ) + return snapshot + + +@dataclass(frozen=True, slots=True) +class CftcPositioningCoverageSliceV1: + """Year/family/scope/contract coverage and missingness evidence.""" + + year: int + report_family: CftcReportFamily + report_scope: CftcReportScope + contract_code: str + row_count: int + first_report_date: str + last_report_date: str + missing_week_count: int + duplicate_key_count: int + contract_names: tuple[str, ...] + market_names: tuple[str, ...] + availability_counts: Mapping[str, int] + restatement_counts: Mapping[str, int] + source_hashes: tuple[str, ...] + source_bytes: int + processing_seconds: float + schema_version: str = CFTC_POSITIONING_COVERAGE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_COVERAGE_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning coverage schema") + _bounded_int(self.year, "year", 1900, 2200) + object.__setattr__( + self, + "report_family", + CftcReportFamily.from_value(self.report_family), + ) + object.__setattr__( + self, "report_scope", CftcReportScope.from_value(self.report_scope) + ) + object.__setattr__( + self, "contract_code", _contract_code(self.contract_code) + ) + _bounded_int(self.row_count, "row_count", 1, MAX_CFTC_ROWS) + first = _parse_date(self.first_report_date, "first_report_date") + last = _parse_date(self.last_report_date, "last_report_date") + if first.year != self.year or last.year != self.year or last < first: + raise ValueError("CFTC coverage dates differ from year") + object.__setattr__(self, "first_report_date", first.isoformat()) + object.__setattr__(self, "last_report_date", last.isoformat()) + _bounded_int( + self.missing_week_count, "missing_week_count", 0, MAX_CFTC_ROWS + ) + _bounded_int( + self.duplicate_key_count, "duplicate_key_count", 0, MAX_CFTC_ROWS + ) + names = tuple( + sorted({_required_text(item) for item in self.contract_names}) + ) + markets = tuple( + sorted({_required_text(item) for item in self.market_names}) + ) + if not names or not markets: + raise ValueError("CFTC coverage requires contract and market names") + object.__setattr__(self, "contract_names", names) + object.__setattr__(self, "market_names", markets) + availability = _count_mapping( + self.availability_counts, "availability_counts" + ) + restatements = _count_mapping( + self.restatement_counts, "restatement_counts" + ) + if sum(availability.values()) != self.row_count: + raise ValueError("CFTC availability counts differ from rows") + if sum(restatements.values()) != self.row_count: + raise ValueError("CFTC restatement counts differ from rows") + hashes = tuple( + sorted( + { + _required_sha256(item, "source_hash") + for item in self.source_hashes + } + ) + ) + if not hashes: + raise ValueError("CFTC coverage requires source hashes") + _bounded_int( + self.source_bytes, "source_bytes", 0, MAX_CFTC_TOTAL_SOURCE_BYTES + ) + _finite_nonnegative(self.processing_seconds, "processing_seconds") + object.__setattr__(self, "availability_counts", availability) + object.__setattr__(self, "restatement_counts", restatements) + object.__setattr__(self, "source_hashes", hashes) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "year": self.year, + "report_family": self.report_family.value, + "report_scope": self.report_scope.value, + "contract_code": self.contract_code, + "row_count": self.row_count, + "first_report_date": self.first_report_date, + "last_report_date": self.last_report_date, + "missing_week_count": self.missing_week_count, + "duplicate_key_count": self.duplicate_key_count, + "contract_names": list(self.contract_names), + "market_names": list(self.market_names), + "availability_counts": dict(self.availability_counts), + "restatement_counts": dict(self.restatement_counts), + "source_hashes": list(self.source_hashes), + "source_bytes": self.source_bytes, + "processing_seconds": self.processing_seconds, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CftcPositioningCoverageSliceV1": + _require_schema(data, CFTC_POSITIONING_COVERAGE_SCHEMA_VERSION) + return cls( + year=_strict_int(data.get("year"), "year"), + report_family=CftcReportFamily.from_value( + str(data.get("report_family", "")) + ), + report_scope=CftcReportScope.from_value( + str(data.get("report_scope", "")) + ), + contract_code=str(data.get("contract_code", "")), + row_count=_strict_int(data.get("row_count"), "row_count"), + first_report_date=str(data.get("first_report_date", "")), + last_report_date=str(data.get("last_report_date", "")), + missing_week_count=_strict_int( + data.get("missing_week_count"), "missing_week_count" + ), + duplicate_key_count=_strict_int( + data.get("duplicate_key_count"), "duplicate_key_count" + ), + contract_names=_string_tuple(data.get("contract_names")), + market_names=_string_tuple(data.get("market_names")), + availability_counts={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("availability_counts") + ).items() + }, + restatement_counts={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("restatement_counts") + ).items() + }, + source_hashes=_string_tuple(data.get("source_hashes")), + source_bytes=_strict_int(data.get("source_bytes"), "source_bytes"), + processing_seconds=_finite_float( + data.get("processing_seconds"), "processing_seconds" + ), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CftcArchiveConsistencyV1: + """Comparison of PRE current state with official compressed history.""" + + source_id: str + report_family: CftcReportFamily + report_scope: CftcReportScope + selected_row_count: int + matched_pre_rows: int + missing_pre_rows: int + open_interest_mismatch_count: int + contract_name_change_count: int + limitations: tuple[str, ...] + evidence_id: str = "" + schema_version: str = CFTC_POSITIONING_ARCHIVE_EVIDENCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != CFTC_POSITIONING_ARCHIVE_EVIDENCE_SCHEMA_VERSION + ): + raise ValueError("unsupported CFTC archive evidence schema") + object.__setattr__(self, "source_id", _required_text(self.source_id)) + object.__setattr__( + self, + "report_family", + CftcReportFamily.from_value(self.report_family), + ) + object.__setattr__( + self, "report_scope", CftcReportScope.from_value(self.report_scope) + ) + for name in ( + "selected_row_count", + "matched_pre_rows", + "missing_pre_rows", + "open_interest_mismatch_count", + "contract_name_change_count", + ): + _bounded_int(getattr(self, name), name, 0, MAX_CFTC_ROWS) + if ( + self.matched_pre_rows + self.missing_pre_rows + != self.selected_row_count + ): + raise ValueError("CFTC archive evidence counts do not reconcile") + limitations = tuple(_required_text(item) for item in self.limitations) + if not limitations: + raise ValueError("CFTC archive evidence requires limitations") + object.__setattr__(self, "limitations", limitations) + expected = _stable_id( + "cftc-archive-consistency", self.identity_payload() + ) + if self.evidence_id and self.evidence_id != expected: + raise ValueError("CFTC archive evidence identity differs") + object.__setattr__(self, "evidence_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "source_id": self.source_id, + "report_family": self.report_family.value, + "report_scope": self.report_scope.value, + "selected_row_count": self.selected_row_count, + "matched_pre_rows": self.matched_pre_rows, + "missing_pre_rows": self.missing_pre_rows, + "open_interest_mismatch_count": self.open_interest_mismatch_count, + "contract_name_change_count": self.contract_name_change_count, + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CftcArchiveConsistencyV1": + _require_schema(data, CFTC_POSITIONING_ARCHIVE_EVIDENCE_SCHEMA_VERSION) + return cls( + source_id=str(data.get("source_id", "")), + report_family=CftcReportFamily.from_value( + str(data.get("report_family", "")) + ), + report_scope=CftcReportScope.from_value( + str(data.get("report_scope", "")) + ), + selected_row_count=_strict_int( + data.get("selected_row_count"), "selected_row_count" + ), + matched_pre_rows=_strict_int( + data.get("matched_pre_rows"), "matched_pre_rows" + ), + missing_pre_rows=_strict_int( + data.get("missing_pre_rows"), "missing_pre_rows" + ), + open_interest_mismatch_count=_strict_int( + data.get("open_interest_mismatch_count"), + "open_interest_mismatch_count", + ), + contract_name_change_count=_strict_int( + data.get("contract_name_change_count"), + "contract_name_change_count", + ), + limitations=_string_tuple(data.get("limitations")), + evidence_id=str(data.get("evidence_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CftcPositioningCorpusV1: + """Self-contained current-state corpus with immutable source lineage.""" + + profile: CftcPositioningFetchProfileV1 + sources: tuple[Mapping[str, JSONValue], ...] + symbol_mappings: tuple[CftcPositioningSymbolMappingV1, ...] + snapshots: tuple[CftcPositioningSnapshotV1, ...] + coverage: tuple[CftcPositioningCoverageSliceV1, ...] + archive_consistency: tuple[CftcArchiveConsistencyV1, ...] + duplicate_key_count: int + runtime_seconds: float + peak_memory_bytes: int + limitations: tuple[str, ...] + corpus_id: str = "" + schema_version: str = CFTC_POSITIONING_CORPUS_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_CORPUS_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning corpus schema") + if not isinstance(self.profile, CftcPositioningFetchProfileV1): + raise ValueError("CFTC corpus requires a v1 profile") + sources = tuple( + sorted( + (_json_value_mapping(item) for item in self.sources), + key=lambda item: str(item.get("source_key", "")), + ) + ) + if not sources or len(sources) > MAX_CFTC_SOURCES: + raise ValueError("CFTC corpus sources are empty or unbounded") + source_ids = {str(item.get("source_id", "")) for item in sources} + if "" in source_ids or len(source_ids) != len(sources): + raise ValueError("CFTC corpus source identities are invalid") + mappings = tuple( + sorted( + self.symbol_mappings, + key=lambda item: (item.symbol, item.mapping_kind.value), + ) + ) + expected_mappings = { + ("EURUSD", CftcMappingKind.DIRECT), + ("GBPUSD", CftcMappingKind.DIRECT), + ("EURGBP", CftcMappingKind.DIRECT), + ("EURGBP", CftcMappingKind.DERIVED_TWO_LEG), + } + if { + (item.symbol, item.mapping_kind) for item in mappings + } != expected_mappings: + raise ValueError("CFTC corpus symbol mappings are incomplete") + if any( + not set(item.contract_codes).issubset(self.profile.contract_codes) + for item in mappings + ): + raise ValueError("CFTC symbol mapping exceeds profile contracts") + snapshots = tuple( + sorted( + self.snapshots, + key=lambda item: ( + item.report_date, + item.report_family.value, + item.report_scope.value, + item.contract_code, + ), + ) + ) + if not snapshots or len(snapshots) > self.profile.max_rows: + raise ValueError("CFTC corpus snapshots are empty or unbounded") + keys = [item.logical_key for item in snapshots] + if len(set(keys)) != len(keys): + raise ValueError("CFTC corpus contains duplicate logical keys") + if any(item.source_id not in source_ids for item in snapshots): + raise ValueError("CFTC snapshot source is absent from corpus") + coverage = tuple( + sorted( + self.coverage, + key=lambda item: ( + item.year, + item.report_family.value, + item.report_scope.value, + item.contract_code, + ), + ) + ) + if not coverage or len(coverage) > MAX_CFTC_COVERAGE_SLICES: + raise ValueError("CFTC corpus coverage is empty or unbounded") + expected_counts = Counter( + ( + _parse_date(item.report_date, "report_date").year, + item.report_family, + item.report_scope, + item.contract_code, + ) + for item in snapshots + ) + actual_counts = Counter( + { + ( + item.year, + item.report_family, + item.report_scope, + item.contract_code, + ): item.row_count + for item in coverage + } + ) + if actual_counts != expected_counts: + raise ValueError( + "CFTC corpus coverage counts differ from snapshots" + ) + archive = tuple( + sorted( + self.archive_consistency, + key=lambda item: ( + item.report_family.value, + item.report_scope.value, + ), + ) + ) + if len(archive) != len(self.profile.historical_archives): + raise ValueError("CFTC archive evidence differs from profile") + _bounded_int( + self.duplicate_key_count, + "duplicate_key_count", + 0, + MAX_CFTC_ROWS, + ) + _finite_nonnegative(self.runtime_seconds, "runtime_seconds") + _bounded_int( + self.peak_memory_bytes, + "peak_memory_bytes", + 0, + self.profile.max_peak_memory_bytes, + ) + limitations = tuple(_required_text(item) for item in self.limitations) + if not limitations: + raise ValueError("CFTC corpus requires limitations") + object.__setattr__(self, "sources", sources) + object.__setattr__(self, "symbol_mappings", mappings) + object.__setattr__(self, "snapshots", snapshots) + object.__setattr__(self, "coverage", coverage) + object.__setattr__(self, "archive_consistency", archive) + object.__setattr__(self, "limitations", limitations) + expected = _stable_id( + "cftc-positioning-corpus", self.identity_payload() + ) + if self.corpus_id and self.corpus_id != expected: + raise ValueError("CFTC corpus identity differs") + object.__setattr__(self, "corpus_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "profile": self.profile.to_dict(), + "sources": [dict(item) for item in self.sources], + "symbol_mappings": [ + item.to_dict() for item in self.symbol_mappings + ], + "snapshots": [item.to_dict() for item in self.snapshots], + "coverage": [item.to_dict() for item in self.coverage], + "archive_consistency": [ + item.to_dict() for item in self.archive_consistency + ], + "duplicate_key_count": self.duplicate_key_count, + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "snapshot_count": len(self.snapshots), + "source_count": len(self.sources), + "source_bytes": sum( + _strict_int(item.get("size_bytes"), "size_bytes") + for item in self.sources + ), + "runtime_seconds": self.runtime_seconds, + "peak_memory_bytes": self.peak_memory_bytes, + "corpus_id": self.corpus_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CftcPositioningCorpusV1": + _require_schema(data, CFTC_POSITIONING_CORPUS_SCHEMA_VERSION) + corpus = cls( + profile=CftcPositioningFetchProfileV1.from_dict( + _mapping(data.get("profile")) + ), + sources=tuple( + _json_value_mapping(item) + for item in _mapping_sequence(data.get("sources")) + ), + symbol_mappings=tuple( + CftcPositioningSymbolMappingV1.from_dict(item) + for item in _mapping_sequence(data.get("symbol_mappings")) + ), + snapshots=tuple( + CftcPositioningSnapshotV1.from_dict(item) + for item in _mapping_sequence(data.get("snapshots")) + ), + coverage=tuple( + CftcPositioningCoverageSliceV1.from_dict(item) + for item in _mapping_sequence(data.get("coverage")) + ), + archive_consistency=tuple( + CftcArchiveConsistencyV1.from_dict(item) + for item in _mapping_sequence(data.get("archive_consistency")) + ), + duplicate_key_count=_strict_int( + data.get("duplicate_key_count"), "duplicate_key_count" + ), + runtime_seconds=_finite_float( + data.get("runtime_seconds"), "runtime_seconds" + ), + peak_memory_bytes=_strict_int( + data.get("peak_memory_bytes"), "peak_memory_bytes" + ), + limitations=_string_tuple(data.get("limitations")), + corpus_id=str(data.get("corpus_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + if _strict_int(data.get("snapshot_count"), "snapshot_count") != len( + corpus.snapshots + ): + raise ValueError("CFTC corpus snapshot count differs") + if _strict_int(data.get("source_count"), "source_count") != len( + corpus.sources + ): + raise ValueError("CFTC corpus source count differs") + return corpus + + +@dataclass(frozen=True, slots=True) +class CftcPositioningCorpusBuildV1: + """In-memory corpus plus retained official responses for replay.""" + + corpus: CftcPositioningCorpusV1 + raw_sources: tuple[CftcPositioningRawSourceV1, ...] + + def __post_init__(self) -> None: + if not isinstance(self.corpus, CftcPositioningCorpusV1): + raise ValueError("CFTC build requires a v1 corpus") + sources = tuple( + sorted(self.raw_sources, key=lambda item: item.source_key) + ) + evidence_ids = { + str(item.get("source_id", "")) for item in self.corpus.sources + } + if {item.source_id for item in sources} != evidence_ids: + raise ValueError("CFTC build sources differ from corpus evidence") + if len({item.source_key for item in sources}) != len(sources): + raise ValueError("CFTC build source keys must be unique") + object.__setattr__(self, "raw_sources", sources) + + +@dataclass(frozen=True, slots=True) +class CftcPositioningDiffV1: + """Bounded immutable refresh/restatement comparison.""" + + previous_corpus_id: str + current_corpus_id: str + added_keys: tuple[str, ...] + removed_keys: tuple[str, ...] + changed_keys: tuple[str, ...] + previous_snapshot_ids: Mapping[str, str] + current_snapshot_ids: Mapping[str, str] + diff_id: str = "" + schema_version: str = CFTC_POSITIONING_DIFF_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_DIFF_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning diff schema") + object.__setattr__( + self, "previous_corpus_id", _required_text(self.previous_corpus_id) + ) + object.__setattr__( + self, "current_corpus_id", _required_text(self.current_corpus_id) + ) + for name in ("added_keys", "removed_keys", "changed_keys"): + values = tuple( + sorted({_required_text(item) for item in getattr(self, name)}) + ) + if len(values) > MAX_CFTC_ROWS: + raise ValueError("CFTC diff exceeds row bound") + object.__setattr__(self, name, values) + previous = _id_mapping(self.previous_snapshot_ids) + current = _id_mapping(self.current_snapshot_ids) + expected_keys = set(self.removed_keys) | set(self.changed_keys) + if set(previous) != expected_keys: + raise ValueError("CFTC diff previous snapshot IDs do not reconcile") + expected_current = set(self.added_keys) | set(self.changed_keys) + if set(current) != expected_current: + raise ValueError("CFTC diff current snapshot IDs do not reconcile") + object.__setattr__(self, "previous_snapshot_ids", previous) + object.__setattr__(self, "current_snapshot_ids", current) + expected = _stable_id("cftc-positioning-diff", self.identity_payload()) + if self.diff_id and self.diff_id != expected: + raise ValueError("CFTC positioning diff identity differs") + object.__setattr__(self, "diff_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "previous_corpus_id": self.previous_corpus_id, + "current_corpus_id": self.current_corpus_id, + "added_keys": list(self.added_keys), + "removed_keys": list(self.removed_keys), + "changed_keys": list(self.changed_keys), + "previous_snapshot_ids": dict(self.previous_snapshot_ids), + "current_snapshot_ids": dict(self.current_snapshot_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "diff_id": self.diff_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CftcPositioningDiffV1": + _require_schema(data, CFTC_POSITIONING_DIFF_SCHEMA_VERSION) + return cls( + previous_corpus_id=str(data.get("previous_corpus_id", "")), + current_corpus_id=str(data.get("current_corpus_id", "")), + added_keys=_string_tuple(data.get("added_keys")), + removed_keys=_string_tuple(data.get("removed_keys")), + changed_keys=_string_tuple(data.get("changed_keys")), + previous_snapshot_ids={ + str(key): str(value) + for key, value in _mapping( + data.get("previous_snapshot_ids") + ).items() + }, + current_snapshot_ids={ + str(key): str(value) + for key, value in _mapping( + data.get("current_snapshot_ids") + ).items() + }, + diff_id=str(data.get("diff_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CftcPositioningPreflightV1: + """Explicit readiness decision for required window positioning state.""" + + corpus_id: str + start_ns: int + end_ns: int + information_mode: InformationMode + as_of_ns: int | None + symbols: tuple[str, ...] + report_families: tuple[CftcReportFamily, ...] + report_scopes: tuple[CftcReportScope, ...] + ready: bool + reasons: tuple[str, ...] + schema_version: str = CFTC_POSITIONING_PREFLIGHT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_PREFLIGHT_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning preflight schema") + object.__setattr__(self, "corpus_id", _required_text(self.corpus_id)) + start = _bounded_int64(self.start_ns, "start_ns") + end = _bounded_int64(self.end_ns, "end_ns") + if end <= start: + raise ValueError("CFTC preflight end must follow start") + mode = InformationMode.from_value(self.information_mode) + object.__setattr__(self, "information_mode", mode) + if mode is InformationMode.EX_ANTE_SIMULATION: + if self.as_of_ns is None: + raise ValueError("ex-ante CFTC preflight requires as_of_ns") + as_of = _bounded_int64(self.as_of_ns, "as_of_ns") + if as_of > start: + raise ValueError( + "CFTC preflight as_of cannot follow window start" + ) + object.__setattr__(self, "as_of_ns", as_of) + elif self.as_of_ns is not None: + raise ValueError("ex-post CFTC preflight does not accept as_of_ns") + symbols = _normalized_symbols(self.symbols) + families = tuple( + sorted( + { + CftcReportFamily.from_value(item) + for item in self.report_families + }, + key=lambda item: item.value, + ) + ) + scopes = tuple( + sorted( + { + CftcReportScope.from_value(item) + for item in self.report_scopes + }, + key=lambda item: item.value, + ) + ) + if not symbols or not families or not scopes: + raise ValueError("CFTC preflight scope cannot be empty") + if not isinstance(self.ready, bool): + raise ValueError("CFTC preflight ready must be boolean") + reasons = tuple(_required_text(item) for item in self.reasons) + if self.ready == bool(reasons): + raise ValueError("CFTC preflight readiness and reasons contradict") + object.__setattr__(self, "symbols", symbols) + object.__setattr__(self, "report_families", families) + object.__setattr__(self, "report_scopes", scopes) + object.__setattr__(self, "reasons", reasons) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "corpus_id": self.corpus_id, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "information_mode": self.information_mode.value, + "as_of_ns": self.as_of_ns, + "symbols": list(self.symbols), + "report_families": [item.value for item in self.report_families], + "report_scopes": [item.value for item in self.report_scopes], + "ready": self.ready, + "reasons": list(self.reasons), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CftcPositioningPreflightV1": + _require_schema(data, CFTC_POSITIONING_PREFLIGHT_SCHEMA_VERSION) + return cls( + corpus_id=str(data.get("corpus_id", "")), + start_ns=_strict_int(data.get("start_ns"), "start_ns"), + end_ns=_strict_int(data.get("end_ns"), "end_ns"), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + as_of_ns=_optional_int(data.get("as_of_ns")), + symbols=_string_tuple(data.get("symbols")), + report_families=tuple( + CftcReportFamily.from_value(str(item)) + for item in _sequence(data.get("report_families")) + ), + report_scopes=tuple( + CftcReportScope.from_value(str(item)) + for item in _sequence(data.get("report_scopes")) + ), + ready=_strict_bool(data.get("ready"), "ready"), + reasons=_string_tuple(data.get("reasons")), + schema_version=str(data.get("schema_version", "")), + ) + + +class CftcPositioningPreflightError(ValueError): + """Raised when required positioning context is unsupported.""" + + def __init__(self, decision: CftcPositioningPreflightV1) -> None: + self.decision = decision + super().__init__( + "CFTC positioning preflight failed: " + "; ".join(decision.reasons) + ) + + +@dataclass(frozen=True, slots=True) +class CftcPositioningQueryV1: + """Bounded latest-known positioning sidecar for one reconstruction window.""" + + corpus_id: str + information_mode: InformationMode + start_ns: int + end_ns: int + as_of_ns: int | None + symbols: tuple[str, ...] + report_families: tuple[CftcReportFamily, ...] + report_scopes: tuple[CftcReportScope, ...] + snapshots: tuple[CftcPositioningSnapshotV1, ...] + symbol_snapshot_ids: Mapping[str, tuple[str, ...]] + mapping_kinds: Mapping[str, str] + derived_values: Mapping[str, float] + status: CftcPositioningQueryStatus + reason: str + age_seconds: Mapping[str, int] + query_id: str = "" + schema_version: str = CFTC_POSITIONING_QUERY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_QUERY_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning query schema") + object.__setattr__(self, "corpus_id", _required_text(self.corpus_id)) + mode = InformationMode.from_value(self.information_mode) + object.__setattr__(self, "information_mode", mode) + if self.end_ns <= self.start_ns: + raise ValueError("CFTC query end must follow start") + _bounded_int64(self.start_ns, "start_ns") + _bounded_int64(self.end_ns, "end_ns") + if mode is InformationMode.EX_ANTE_SIMULATION: + if self.as_of_ns is None: + raise ValueError("ex-ante CFTC query requires as_of_ns") + if self.as_of_ns > self.start_ns: + raise ValueError("CFTC query as_of cannot follow window start") + _bounded_int64(self.as_of_ns, "as_of_ns") + elif self.as_of_ns is not None: + raise ValueError("ex-post CFTC query does not accept as_of_ns") + status = CftcPositioningQueryStatus(self.status) + symbols = _normalized_symbols(self.symbols) + families = tuple( + sorted( + { + CftcReportFamily.from_value(item) + for item in self.report_families + }, + key=lambda item: item.value, + ) + ) + scopes = tuple( + sorted( + { + CftcReportScope.from_value(item) + for item in self.report_scopes + }, + key=lambda item: item.value, + ) + ) + snapshots = tuple( + sorted(self.snapshots, key=lambda item: item.snapshot_id) + ) + if len(snapshots) > MAX_CFTC_QUERY_SNAPSHOTS: + raise ValueError("CFTC query snapshot limit exceeded") + if len({item.snapshot_id for item in snapshots}) != len(snapshots): + raise ValueError("CFTC query snapshots must be unique") + snapshot_ids = {item.snapshot_id for item in snapshots} + symbol_ids: dict[str, tuple[str, ...]] = {} + for symbol, values in sorted(self.symbol_snapshot_ids.items()): + normalized = _normalized_symbol(symbol) + ids = tuple(sorted({_required_text(item) for item in values})) + if not ids or not set(ids).issubset(snapshot_ids): + raise ValueError("CFTC symbol snapshot mapping is invalid") + symbol_ids[normalized] = ids + if set(symbol_ids) != set(symbols): + if status is CftcPositioningQueryStatus.READY: + raise ValueError("ready CFTC query lacks symbol snapshots") + mapping_kinds = { + _normalized_symbol(key): CftcMappingKind(value).value + for key, value in sorted(self.mapping_kinds.items()) + } + if set(mapping_kinds) != set(symbol_ids): + raise ValueError("CFTC mapping kinds differ from symbol snapshots") + derived = _metric_mapping(self.derived_values) + reason = _required_text(self.reason) + ages = { + _required_text(key): _bounded_int(value, str(key), 0, 2**31 - 1) + for key, value in sorted(self.age_seconds.items()) + } + if not set(ages).issubset(snapshot_ids): + raise ValueError("CFTC age evidence references unknown snapshots") + if status is CftcPositioningQueryStatus.READY: + if not snapshots or set(symbol_ids) != set(symbols): + raise ValueError("ready CFTC query is incomplete") + if reason != "ready": + raise ValueError("ready CFTC query reason differs") + else: + if reason == "ready": + raise ValueError("non-ready CFTC query requires refusal reason") + object.__setattr__(self, "symbols", symbols) + object.__setattr__(self, "report_families", families) + object.__setattr__(self, "report_scopes", scopes) + object.__setattr__(self, "snapshots", snapshots) + object.__setattr__(self, "symbol_snapshot_ids", symbol_ids) + object.__setattr__(self, "mapping_kinds", mapping_kinds) + object.__setattr__(self, "derived_values", derived) + object.__setattr__(self, "status", status) + object.__setattr__(self, "reason", reason) + object.__setattr__(self, "age_seconds", ages) + _ensure_payload_size(self.identity_payload(), MAX_CFTC_QUERY_BYTES) + expected = _stable_id("cftc-positioning-query", self.identity_payload()) + if self.query_id and self.query_id != expected: + raise ValueError("CFTC positioning query identity differs") + object.__setattr__(self, "query_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "corpus_id": self.corpus_id, + "information_mode": self.information_mode.value, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "as_of_ns": self.as_of_ns, + "symbols": list(self.symbols), + "report_families": [item.value for item in self.report_families], + "report_scopes": [item.value for item in self.report_scopes], + "snapshots": [item.to_dict() for item in self.snapshots], + "symbol_snapshot_ids": { + key: list(value) + for key, value in self.symbol_snapshot_ids.items() + }, + "mapping_kinds": dict(self.mapping_kinds), + "derived_values": dict(self.derived_values), + "status": self.status.value, + "reason": self.reason, + "age_seconds": dict(self.age_seconds), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "query_id": self.query_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CftcPositioningQueryV1": + _require_schema(data, CFTC_POSITIONING_QUERY_SCHEMA_VERSION) + return cls( + corpus_id=str(data.get("corpus_id", "")), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + start_ns=_strict_int(data.get("start_ns"), "start_ns"), + end_ns=_strict_int(data.get("end_ns"), "end_ns"), + as_of_ns=_optional_int(data.get("as_of_ns")), + symbols=_string_tuple(data.get("symbols")), + report_families=tuple( + CftcReportFamily.from_value(str(item)) + for item in _sequence(data.get("report_families")) + ), + report_scopes=tuple( + CftcReportScope.from_value(str(item)) + for item in _sequence(data.get("report_scopes")) + ), + snapshots=tuple( + CftcPositioningSnapshotV1.from_dict(item) + for item in _mapping_sequence(data.get("snapshots")) + ), + symbol_snapshot_ids={ + str(key): _string_tuple(value) + for key, value in _mapping( + data.get("symbol_snapshot_ids") + ).items() + }, + mapping_kinds={ + str(key): str(value) + for key, value in _mapping(data.get("mapping_kinds")).items() + }, + derived_values={ + str(key): _finite_float(value, str(key)) + for key, value in _mapping(data.get("derived_values")).items() + }, + status=CftcPositioningQueryStatus(str(data.get("status", ""))), + reason=str(data.get("reason", "")), + age_seconds={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping(data.get("age_seconds")).items() + }, + query_id=str(data.get("query_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CftcPositioningConsumerBindingV1: + """Companion receipt binding an immutable v1 consumer to one query.""" + + consumer: CftcPositioningConsumer + consumer_artifact_id: str + run_id: str + window_id: str + corpus_id: str + query_id: str + snapshot_ids: tuple[str, ...] + information_input_ids: tuple[str, ...] + state_label: str + metrics: Mapping[str, float] + binding_id: str = "" + schema_version: str = CFTC_POSITIONING_BINDING_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_BINDING_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning binding schema") + object.__setattr__( + self, "consumer", CftcPositioningConsumer(self.consumer) + ) + for name in ( + "consumer_artifact_id", + "run_id", + "window_id", + "corpus_id", + "query_id", + "state_label", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + snapshots = tuple( + sorted({_required_text(item) for item in self.snapshot_ids}) + ) + inputs = tuple( + sorted( + {_required_text(item) for item in self.information_input_ids} + ) + ) + if not snapshots or not inputs: + raise ValueError("CFTC positioning binding requires lineage IDs") + metrics = _metric_mapping(self.metrics) + if len(metrics) > MAX_CFTC_BINDING_VALUES: + raise ValueError("CFTC positioning binding metrics exceed limit") + object.__setattr__(self, "snapshot_ids", snapshots) + object.__setattr__(self, "information_input_ids", inputs) + object.__setattr__(self, "metrics", metrics) + expected = _stable_id( + "cftc-positioning-binding", self.identity_payload() + ) + if self.binding_id and self.binding_id != expected: + raise ValueError("CFTC positioning binding identity differs") + object.__setattr__(self, "binding_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "consumer": self.consumer.value, + "consumer_artifact_id": self.consumer_artifact_id, + "run_id": self.run_id, + "window_id": self.window_id, + "corpus_id": self.corpus_id, + "query_id": self.query_id, + "snapshot_ids": list(self.snapshot_ids), + "information_input_ids": list(self.information_input_ids), + "state_label": self.state_label, + "metrics": dict(self.metrics), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "binding_id": self.binding_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CftcPositioningConsumerBindingV1": + _require_schema(data, CFTC_POSITIONING_BINDING_SCHEMA_VERSION) + return cls( + consumer=CftcPositioningConsumer(str(data.get("consumer", ""))), + consumer_artifact_id=str(data.get("consumer_artifact_id", "")), + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + corpus_id=str(data.get("corpus_id", "")), + query_id=str(data.get("query_id", "")), + snapshot_ids=_string_tuple(data.get("snapshot_ids")), + information_input_ids=_string_tuple( + data.get("information_input_ids") + ), + state_label=str(data.get("state_label", "")), + metrics={ + str(key): _finite_float(value, str(key)) + for key, value in _mapping(data.get("metrics")).items() + }, + binding_id=str(data.get("binding_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CftcPositioningBenchmarkSmokeV1: + """Real-window deterministic benchmark-consumption evidence.""" + + corpus_id: str + query_id: str + binding_id: str + source_artifact_id: str + source_sha256: str + source_row_count: int + benchmark_event_ids: tuple[str, ...] + logical_output_sha256: str + reload_output_sha256: str + deterministic_reload: bool + smoke_id: str = "" + schema_version: str = CFTC_POSITIONING_SMOKE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CFTC_POSITIONING_SMOKE_SCHEMA_VERSION: + raise ValueError("unsupported CFTC positioning smoke schema") + for name in ( + "corpus_id", + "query_id", + "binding_id", + "source_artifact_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "source_sha256", + _required_sha256(self.source_sha256, "source_sha256"), + ) + _bounded_int(self.source_row_count, "source_row_count", 1, 1_000_000) + event_ids = tuple( + _required_text(item) for item in self.benchmark_event_ids + ) + if not event_ids or len(event_ids) != self.source_row_count: + raise ValueError("CFTC smoke event IDs do not reconcile") + object.__setattr__(self, "benchmark_event_ids", event_ids) + for name in ("logical_output_sha256", "reload_output_sha256"): + object.__setattr__( + self, + name, + _required_sha256(getattr(self, name), name), + ) + if not isinstance(self.deterministic_reload, bool): + raise ValueError("deterministic_reload must be boolean") + if self.deterministic_reload != ( + self.logical_output_sha256 == self.reload_output_sha256 + ): + raise ValueError("CFTC smoke deterministic flag contradicts hashes") + expected = _stable_id("cftc-positioning-smoke", self.identity_payload()) + if self.smoke_id and self.smoke_id != expected: + raise ValueError("CFTC positioning smoke identity differs") + object.__setattr__(self, "smoke_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "corpus_id": self.corpus_id, + "query_id": self.query_id, + "binding_id": self.binding_id, + "source_artifact_id": self.source_artifact_id, + "source_sha256": self.source_sha256, + "source_row_count": self.source_row_count, + "benchmark_event_ids": list(self.benchmark_event_ids), + "logical_output_sha256": self.logical_output_sha256, + "reload_output_sha256": self.reload_output_sha256, + "deterministic_reload": self.deterministic_reload, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "smoke_id": self.smoke_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CftcPositioningBenchmarkSmokeV1": + _require_schema(data, CFTC_POSITIONING_SMOKE_SCHEMA_VERSION) + return cls( + corpus_id=str(data.get("corpus_id", "")), + query_id=str(data.get("query_id", "")), + binding_id=str(data.get("binding_id", "")), + source_artifact_id=str(data.get("source_artifact_id", "")), + source_sha256=str(data.get("source_sha256", "")), + source_row_count=_strict_int( + data.get("source_row_count"), "source_row_count" + ), + benchmark_event_ids=_string_tuple(data.get("benchmark_event_ids")), + logical_output_sha256=str(data.get("logical_output_sha256", "")), + reload_output_sha256=str(data.get("reload_output_sha256", "")), + deterministic_reload=_strict_bool( + data.get("deterministic_reload"), "deterministic_reload" + ), + smoke_id=str(data.get("smoke_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +class CftcSourceFetcherV1(Protocol): + """Injectable bounded source fetch seam used by production and tests.""" + + def __call__( + self, + url: str, + *, + parameters: Mapping[str, str], + timeout_seconds: float, + max_bytes: int, + user_agent: str, + ) -> tuple[bytes, str, str]: ... + + +def build_live_cftc_positioning_corpus( + profile: CftcPositioningFetchProfileV1, + *, + fetcher: CftcSourceFetcherV1 | None = None, + retrieved_at_ns: int | None = None, +) -> CftcPositioningCorpusBuildV1: + """Acquire bounded official sources and build one replayable corpus.""" + if not isinstance(profile, CftcPositioningFetchProfileV1): + raise ValueError("CFTC live build requires a v1 fetch profile") + started = time.perf_counter() + retrieved = ( + _bounded_int64(retrieved_at_ns, "retrieved_at_ns") + if retrieved_at_ns is not None + else int(datetime.now(timezone.utc).timestamp() * 1_000_000_000) + ) + get = fetcher or _fetch_http_source + sources: list[CftcPositioningRawSourceV1] = [] + source_bytes = 0 + + def acquire( + *, + source_key: str, + source_kind: str, + url: str, + parameters: Mapping[str, str] | None = None, + dataset_id: str | None = None, + report_family: CftcReportFamily | None = None, + report_scope: CftcReportScope | None = None, + redistribution_allowed: bool = True, + limitations: tuple[str, ...] = (), + ) -> CftcPositioningRawSourceV1: + nonlocal source_bytes + params = dict(parameters or {}) + content, content_type, resolved_uri = get( + url, + parameters=params, + timeout_seconds=profile.timeout_seconds, + max_bytes=profile.max_response_bytes, + user_agent=profile.user_agent, + ) + source_bytes += len(content) + if source_bytes > profile.max_total_source_bytes: + raise ValueError("CFTC acquisition exceeds total source-byte limit") + if time.perf_counter() - started > profile.max_runtime_seconds: + raise ValueError("CFTC acquisition exceeds runtime limit") + if _peak_memory_bytes() > profile.max_peak_memory_bytes: + raise ValueError("CFTC acquisition exceeds peak-memory limit") + source = CftcPositioningRawSourceV1( + source_key=source_key, + source_kind=source_kind, + source_uri=resolved_uri, + retrieved_at_ns=retrieved, + content=content, + content_type=content_type, + query_parameters=params, + dataset_id=dataset_id, + report_family=report_family, + report_scope=report_scope, + redistribution_allowed=redistribution_allowed, + limitations=limitations, + ) + sources.append(source) + return source + + family_by_dataset = { + CFTC_LEGACY_DATASET_ID: CftcReportFamily.LEGACY, + CFTC_TFF_DATASET_ID: CftcReportFamily.TFF, + } + for dataset_id in profile.dataset_ids: + family = family_by_dataset[dataset_id] + acquire( + source_key=f"pre.{dataset_id}.metadata", + source_kind="pre_metadata", + url=CFTC_PRE_METADATA_TEMPLATE.format(dataset_id=dataset_id), + dataset_id=dataset_id, + report_family=family, + limitations=( + "PRE metadata describes the current dataset, not historical schema vintages.", + ), + ) + offset = 0 + for page_number in range(profile.max_pages_per_dataset): + parameters = _pre_query_parameters(profile, offset=offset) + source = acquire( + source_key=f"pre.{dataset_id}.page-{page_number:04d}", + source_kind="pre_data", + url=CFTC_PRE_RESOURCE_TEMPLATE.format(dataset_id=dataset_id), + parameters=parameters, + dataset_id=dataset_id, + report_family=family, + limitations=( + "PRE history is current state and is not a complete original-vintage archive.", + "The PRE ID field is retained but is not used as the corpus primary key.", + ), + ) + rows = _json_rows(source.content, "PRE response") + if len(rows) < profile.page_size: + break + offset += profile.page_size + else: + raise ValueError( + "CFTC PRE pagination exceeded configured page limit" + ) + + official_pages = ( + ( + "official.release-schedule", + "release_schedule", + CFTC_RELEASE_SCHEDULE_URI, + True, + ( + "The schedule is tentative and usually maps Friday publication to the prior Tuesday.", + ), + ), + ( + "official.special-announcements", + "special_announcements", + CFTC_SPECIAL_ANNOUNCEMENTS_URI, + True, + ( + "Special announcements are sparse and do not form a complete publication-vintage archive.", + ), + ), + ( + "official.historical-compressed-index", + "historical_index", + CFTC_HISTORICAL_COMPRESSED_URI, + True, + ( + "Historical compressed files may be corrected after original publication.", + ), + ), + ( + "official.2025-backlog", + "backlog_schedule", + CFTC_2025_BACKLOG_URI, + True, + ( + "The 2025 shutdown schedule supplies verified nonstandard publication dates.", + ), + ), + ( + "official.web-policy", + "license_policy", + CFTC_WEB_POLICY_URI, + True, + ( + "CFTC requests acknowledgement when public-domain government information is reused.", + ), + ), + ) + for ( + source_key, + source_kind, + uri, + redistributable, + limitations, + ) in official_pages: + acquire( + source_key=source_key, + source_kind=source_kind, + url=uri, + redistribution_allowed=redistributable, + limitations=limitations, + ) + + archive_contracts = {item[2]: item[:2] for item in CFTC_HISTORICAL_ARCHIVES} + for archive_name in profile.historical_archives: + family_name, scope_name = archive_contracts[archive_name] + acquire( + source_key=f"official.archive.{archive_name.lower()}", + source_kind="historical_archive", + url=CFTC_HISTORICAL_ARCHIVE_TEMPLATE.format( + archive_name=archive_name + ), + report_family=CftcReportFamily.from_value(family_name), + report_scope=CftcReportScope.from_value(scope_name), + limitations=( + "Compressed history is current corrected state, not proof of the original publication vintage.", + ), + ) + + return build_cftc_positioning_corpus_from_sources( + sources, + profile=profile, + runtime_started=started, + ) + + +def build_cftc_positioning_corpus_from_sources( + raw_sources: Sequence[CftcPositioningRawSourceV1], + *, + profile: CftcPositioningFetchProfileV1, + runtime_started: float | None = None, +) -> CftcPositioningCorpusBuildV1: + """Build a corpus deterministically from already-retained responses.""" + started = ( + time.perf_counter() if runtime_started is None else runtime_started + ) + sources = tuple(sorted(raw_sources, key=lambda item: item.source_key)) + if not sources or len(sources) > MAX_CFTC_SOURCES: + raise ValueError("CFTC source set is empty or unbounded") + if len({item.source_key for item in sources}) != len(sources): + raise ValueError("CFTC source keys must be unique") + if ( + sum(len(item.content) for item in sources) + > profile.max_total_source_bytes + ): + raise ValueError("CFTC source set exceeds total byte limit") + _validate_required_sources(sources, profile) + releases = _build_release_evidence(sources) + duplicate_counts: Counter[tuple[str, str, str, str]] = Counter() + candidates: dict[str, CftcPositioningSnapshotV1] = {} + for source in sources: + if source.source_kind != "pre_data": + continue + if source.dataset_id is None or source.report_family is None: + raise ValueError("CFTC PRE page lacks dataset/family metadata") + for row in _json_rows(source.content, "PRE response"): + report_date = _parse_pre_report_date( + str(row.get("report_date_as_yyyy_mm_dd", "")) + ) + if not ( + _parse_date(profile.start_date, "start_date") + <= report_date + <= _parse_date(profile.end_date, "end_date") + ): + continue + code = _contract_code(str(row.get("cftc_contract_market_code", ""))) + if code not in profile.contract_codes: + continue + scope = CftcReportScope.from_value( + str(row.get("futonly_or_combined", "")) + ) + key_tuple = ( + source.report_family.value, + scope.value, + code, + report_date.isoformat(), + ) + release = releases.get(report_date.isoformat()) + if release is None: + release = _nominal_release_evidence(report_date, sources) + row_hash = hashlib.sha256( + canonical_contract_json( + cast(Mapping[str, JSONValue], _json_value_mapping(row)) + ).encode("utf-8") + ).hexdigest() + snapshot = CftcPositioningSnapshotV1( + report_family=source.report_family, + report_scope=scope, + contract_code=code, + contract_name=str(row.get("contract_market_name", "")).strip(), + market_name=str( + row.get("market_and_exchange_names", "") + ).strip(), + report_date=report_date.isoformat(), + dataset_id=source.dataset_id, + source_id=source.source_id, + source_row_sha256=row_hash, + pre_row_id=str(row.get("id", "")), + release_evidence=release, + restatement_status=_restatement_status( + source.report_family, report_date + ), + values=_positioning_values(row), + ) + existing = candidates.get(snapshot.logical_key) + if existing is None: + candidates[snapshot.logical_key] = snapshot + continue + duplicate_counts[key_tuple] += 1 + if ( + existing.values != snapshot.values + or existing.contract_name != snapshot.contract_name + or existing.market_name != snapshot.market_name + ): + raise ValueError( + "conflicting CFTC rows share family/scope/contract/report date" + ) + snapshots = tuple(candidates.values()) + if not snapshots: + raise ValueError("CFTC PRE sources contain no selected rows") + if len(snapshots) > profile.max_rows: + raise ValueError("CFTC selected rows exceed configured limit") + source_evidence = tuple(source.evidence_dict() for source in sources) + evidence_by_id = {source.source_id: source for source in sources} + coverage = _build_cftc_coverage( + snapshots, + duplicate_counts=duplicate_counts, + evidence_by_id=evidence_by_id, + ) + archive_evidence = tuple( + _archive_consistency(source, snapshots, profile) + for source in sources + if source.source_kind == "historical_archive" + ) + elapsed = time.perf_counter() - started + if elapsed > profile.max_runtime_seconds: + raise ValueError("CFTC corpus build exceeds runtime limit") + peak = _peak_memory_bytes() + if peak > profile.max_peak_memory_bytes: + raise ValueError("CFTC corpus build exceeds peak-memory limit") + corpus = CftcPositioningCorpusV1( + profile=profile, + sources=source_evidence, + symbol_mappings=default_cftc_positioning_symbol_mappings(), + snapshots=snapshots, + coverage=coverage, + archive_consistency=archive_evidence, + duplicate_key_count=sum(duplicate_counts.values()), + runtime_seconds=round(elapsed, 6), + peak_memory_bytes=int(peak), + limitations=( + "COT is futures positioning/open-interest context, not decentralized spot-FX volume, sentiment truth, or a causal event label.", + "PRE and compressed history expose current corrected state and do not form a complete historical vintage archive.", + "Nominal Friday publication estimates are excluded from strict ex-ante use.", + "Legacy and TFF classifications and futures-only/combined scopes remain separate and cannot be pooled.", + "Direct EURGBP state begins with contract 299741; earlier EURGBP state retains both EUR and GBP leg identities.", + "No individual-trader inference, weekly interpolation, or automatic trading recommendation is supported.", + ), + ) + return CftcPositioningCorpusBuildV1(corpus=corpus, raw_sources=sources) + + +def _fetch_http_source( + url: str, + *, + parameters: Mapping[str, str], + timeout_seconds: float, + max_bytes: int, + user_agent: str, +) -> tuple[bytes, str, str]: + """Fetch one official response with a strict byte cap.""" + last_error: requests.RequestException | None = None + for _attempt in range(3): + try: + with requests.get( + url, + params=dict(parameters), + headers={"User-Agent": user_agent, "Accept": "*/*"}, + timeout=timeout_seconds, + stream=True, + ) as response: + response.raise_for_status() + declared = response.headers.get("Content-Length") + if declared is not None and int(declared) > max_bytes: + raise ValueError( + "CFTC response exceeds declared byte limit" + ) + chunks: list[bytes] = [] + total = 0 + for chunk in response.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + total += len(chunk) + if total > max_bytes: + raise ValueError("CFTC response exceeds byte limit") + chunks.append(chunk) + content = b"".join(chunks) + if not content: + raise ValueError("CFTC response is empty") + content_type = response.headers.get( + "Content-Type", "application/octet-stream" + ) + return ( + content, + content_type.split(";", 1)[0].strip(), + response.url, + ) + except requests.RequestException as exc: + last_error = exc + if last_error is None: + raise RuntimeError("CFTC fetch retry state is invalid") + raise last_error + + +def _pre_query_parameters( + profile: CftcPositioningFetchProfileV1, *, offset: int +) -> dict[str, str]: + quoted_codes = ",".join(f"'{item}'" for item in profile.contract_codes) + return { + "$where": ( + f"cftc_contract_market_code in ({quoted_codes}) " + "AND report_date_as_yyyy_mm_dd between " + f"'{profile.start_date}T00:00:00.000' and " + f"'{profile.end_date}T23:59:59.999'" + ), + "$order": ( + "report_date_as_yyyy_mm_dd,cftc_contract_market_code,futonly_or_combined,id" + ), + "$limit": str(profile.page_size), + "$offset": str(offset), + } + + +def compare_cftc_positioning_corpora( + previous: CftcPositioningCorpusV1, + current: CftcPositioningCorpusV1, +) -> CftcPositioningDiffV1: + """Compare logical rows without confusing a refresh with a restatement.""" + previous_by_key = {item.logical_key: item for item in previous.snapshots} + current_by_key = {item.logical_key: item for item in current.snapshots} + previous_keys = set(previous_by_key) + current_keys = set(current_by_key) + added = tuple(sorted(current_keys - previous_keys)) + removed = tuple(sorted(previous_keys - current_keys)) + changed = tuple( + sorted( + key + for key in previous_keys & current_keys + if previous_by_key[key].source_row_sha256 + != current_by_key[key].source_row_sha256 + ) + ) + return CftcPositioningDiffV1( + previous_corpus_id=previous.corpus_id, + current_corpus_id=current.corpus_id, + added_keys=added, + removed_keys=removed, + changed_keys=changed, + previous_snapshot_ids={ + key: previous_by_key[key].snapshot_id for key in removed + changed + }, + current_snapshot_ids={ + key: current_by_key[key].snapshot_id for key in added + changed + }, + ) + + +def write_cftc_positioning_corpus( + build: CftcPositioningCorpusBuildV1, + directory: str | Path, + *, + previous_corpus: CftcPositioningCorpusV1 | None = None, +) -> Mapping[str, ArtifactRef]: + """Write raw, corpus, coverage, archive, and optional diff artifacts once.""" + if not isinstance(build, CftcPositioningCorpusBuildV1): + raise ValueError("CFTC artifact writer requires a v1 corpus build") + root = Path(directory).expanduser().resolve() + raw_root = root / "sources" + raw_root.mkdir(parents=True, exist_ok=True) + artifacts: dict[str, ArtifactRef] = {} + for source in build.raw_sources: + suffix = _content_suffix(source.content_type, source.source_uri) + source_path = ( + raw_root / f"{source.source_key}-{source.content_sha256}{suffix}" + ) + _write_once(source_path, source.content) + artifacts[f"source:{source.source_key}"] = ArtifactRef( + kind="cftc_positioning_source_v1", + path=str(source_path), + size_bytes=len(source.content), + sha256=source.content_sha256, + metadata={ + "source_id": source.source_id, + "source_uri": source.source_uri, + "retrieved_at_ns": source.retrieved_at_ns, + "redistribution_allowed": source.redistribution_allowed, + }, + ) + + artifacts["coverage"] = _write_json_artifact( + root, + "cftc-positioning-coverage", + "cftc_positioning_coverage_v1", + { + "schema_version": CFTC_POSITIONING_COVERAGE_SCHEMA_VERSION, + "corpus_id": build.corpus.corpus_id, + "coverage": [item.to_dict() for item in build.corpus.coverage], + }, + metadata={"corpus_id": build.corpus.corpus_id}, + ) + artifacts["archive_consistency"] = _write_json_artifact( + root, + "cftc-positioning-archive-consistency", + "cftc_positioning_archive_consistency_v1", + { + "schema_version": CFTC_POSITIONING_ARCHIVE_EVIDENCE_SCHEMA_VERSION, + "corpus_id": build.corpus.corpus_id, + "archive_consistency": [ + item.to_dict() for item in build.corpus.archive_consistency + ], + }, + metadata={"corpus_id": build.corpus.corpus_id}, + ) + artifacts["corpus"] = _write_json_artifact( + root, + "cftc-positioning-corpus", + "cftc_positioning_corpus_v1", + build.corpus.to_dict(), + metadata={ + "corpus_id": build.corpus.corpus_id, + "snapshot_count": len(build.corpus.snapshots), + }, + ) + if previous_corpus is not None: + diff = compare_cftc_positioning_corpora(previous_corpus, build.corpus) + artifacts["diff"] = _write_json_artifact( + root, + "cftc-positioning-diff", + "cftc_positioning_diff_v1", + diff.to_dict(), + metadata={"diff_id": diff.diff_id}, + ) + return artifacts + + +def read_cftc_positioning_corpus(path: str | Path) -> CftcPositioningCorpusV1: + """Load and hash-verify one self-contained corpus artifact.""" + source = Path(path).expanduser().resolve() + if source.stat().st_size > MAX_CFTC_CORPUS_BYTES: + raise ValueError("CFTC corpus artifact exceeds size bound") + match = re.fullmatch( + r"cftc-positioning-corpus-([0-9a-f]{64})\.json", source.name + ) + if match is None: + raise ValueError("CFTC corpus artifact name is not content addressed") + content = source.read_bytes() + if hashlib.sha256(content).hexdigest() != match.group(1): + raise ValueError("CFTC corpus artifact hash differs from name") + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("CFTC corpus artifact is invalid JSON") from exc + return CftcPositioningCorpusV1.from_dict(_mapping(payload)) + + +def replay_cftc_positioning_corpus( + corpus_path: str | Path, + *, + source_directory: str | Path | None = None, +) -> CftcPositioningCorpusBuildV1: + """Rebuild exactly from retained responses and verify corpus identity.""" + corpus = read_cftc_positioning_corpus(corpus_path) + root = ( + Path(source_directory).expanduser().resolve() + if source_directory is not None + else Path(corpus_path).expanduser().resolve().parent / "sources" + ) + restored: list[CftcPositioningRawSourceV1] = [] + for item in corpus.sources: + source_key = str(item.get("source_key", "")) + content_hash = str(item.get("content_sha256", "")) + matches = sorted(root.glob(f"{source_key}-{content_hash}.*")) + if len(matches) != 1: + raise ValueError( + f"expected one retained CFTC source for {source_key}" + ) + content = matches[0].read_bytes() + if len(content) > MAX_CFTC_SOURCE_BYTES: + raise ValueError("retained CFTC source exceeds size bound") + restored.append(CftcPositioningRawSourceV1.restore(item, content)) + rebuilt = build_cftc_positioning_corpus_from_sources( + restored, profile=corpus.profile + ) + if rebuilt.corpus.corpus_id != corpus.corpus_id: + raise ValueError("replayed CFTC positioning corpus identity differs") + return rebuilt + + +def query_cftc_positioning_corpus( + corpus: CftcPositioningCorpusV1, + *, + start_ns: int, + end_ns: int, + information_mode: InformationMode, + as_of_ns: int | None = None, + symbols: Sequence[str] = ("EURUSD", "GBPUSD", "EURGBP"), + report_families: Sequence[CftcReportFamily] = ( + CftcReportFamily.LEGACY, + CftcReportFamily.TFF, + ), + report_scopes: Sequence[CftcReportScope] = ( + CftcReportScope.FUTURES_ONLY, + CftcReportScope.COMBINED, + ), + max_staleness_days: int | None = None, +) -> CftcPositioningQueryV1: + """Select latest known weekly state, keeping all report schemas separate.""" + mode = InformationMode.from_value(information_mode) + selected_symbols = _normalized_symbols(symbols) + families = tuple( + sorted( + {CftcReportFamily.from_value(item) for item in report_families}, + key=lambda item: item.value, + ) + ) + scopes = tuple( + sorted( + {CftcReportScope.from_value(item) for item in report_scopes}, + key=lambda item: item.value, + ) + ) + _validate_query_interval(start_ns, end_ns, mode, as_of_ns) + if not selected_symbols or not families or not scopes: + raise ValueError("CFTC query scope cannot be empty") + supported_symbols = {item.symbol for item in corpus.symbol_mappings} + unsupported_symbols = sorted(set(selected_symbols) - supported_symbols) + if unsupported_symbols: + return CftcPositioningQueryV1( + corpus_id=corpus.corpus_id, + information_mode=mode, + start_ns=start_ns, + end_ns=end_ns, + as_of_ns=as_of_ns, + symbols=selected_symbols, + report_families=families, + report_scopes=scopes, + snapshots=(), + symbol_snapshot_ids={}, + mapping_kinds={}, + derived_values={}, + status=CftcPositioningQueryStatus.UNSUPPORTED, + reason="unsupported positioning symbols: " + + ", ".join(unsupported_symbols), + age_seconds={}, + ) + stale_days = ( + corpus.profile.max_staleness_days + if max_staleness_days is None + else _bounded_int(max_staleness_days, "max_staleness_days", 1, 365) + ) + window_date = datetime.fromtimestamp( + start_ns / 1_000_000_000, timezone.utc + ).date() + cutoff_ns = as_of_ns if mode is InformationMode.EX_ANTE_SIMULATION else None + selected: dict[str, CftcPositioningSnapshotV1] = {} + symbol_ids: dict[str, set[str]] = defaultdict(set) + mapping_kinds: dict[str, str] = {} + ages: dict[str, int] = {} + missing: list[str] = [] + unavailable: list[str] = [] + restatement_incomplete: list[str] = [] + stale: list[str] = [] + + for symbol in selected_symbols: + symbol_mapping = _select_symbol_mapping(corpus, symbol, window_date) + codes = symbol_mapping.contract_codes + mapping_kinds[symbol] = symbol_mapping.mapping_kind.value + for family in families: + if family is CftcReportFamily.TFF and window_date < CFTC_TFF_START: + missing.append(f"{symbol}/{family.value}:not-yet-published") + continue + for scope in scopes: + for code in codes: + candidates = sorted( + ( + item + for item in corpus.snapshots + if item.report_family is family + and item.report_scope is scope + and item.contract_code == code + and item.measurement_start_ns <= start_ns + ), + key=lambda item: item.measurement_start_ns, + reverse=True, + ) + seam = f"{symbol}/{family.value}/{scope.value}/{code}" + if not candidates: + missing.append(seam) + continue + if mode is InformationMode.EX_ANTE_SIMULATION: + time_eligible = [ + item + for item in candidates + if item.release_evidence.strict_ex_ante_time_eligible + and item.release_evidence.knowledge_at_ns + is not None + and cutoff_ns is not None + and item.release_evidence.knowledge_at_ns + <= cutoff_ns + ] + if not time_eligible: + unavailable.append(seam) + continue + vintage_eligible = [ + item + for item in time_eligible + if item.strict_ex_ante_eligible + ] + if not vintage_eligible: + restatement_incomplete.append(seam) + continue + candidate = vintage_eligible[0] + else: + candidate = candidates[0] + age_seconds = max( + 0, + (start_ns - candidate.measurement_start_ns) + // 1_000_000_000, + ) + selected[candidate.snapshot_id] = candidate + symbol_ids[symbol].add(candidate.snapshot_id) + ages[candidate.snapshot_id] = int(age_seconds) + if age_seconds > stale_days * 86_400: + stale.append( + f"{seam}:age_seconds={age_seconds}:" + f"limit_seconds={stale_days * 86_400}" + ) + continue + + complete_symbols = set(symbol_ids) == set(selected_symbols) + if restatement_incomplete: + status = CftcPositioningQueryStatus.RESTATEMENT_INCOMPLETE + reason = "original publication vintage unavailable: " + ", ".join( + sorted(restatement_incomplete) + ) + elif unavailable: + status = CftcPositioningQueryStatus.NOT_AVAILABLE + reason = "positioning state unavailable as of cutoff: " + ", ".join( + sorted(unavailable) + ) + elif stale: + status = CftcPositioningQueryStatus.STALE + reason = "positioning state exceeds staleness limit: " + ", ".join( + sorted(stale) + ) + elif missing or not complete_symbols: + status = CftcPositioningQueryStatus.MISSING + reason = "positioning state missing: " + ", ".join(sorted(missing)) + else: + status = CftcPositioningQueryStatus.READY + reason = "ready" + retain_state = status in { + CftcPositioningQueryStatus.READY, + CftcPositioningQueryStatus.STALE, + } + snapshots = tuple(selected.values()) if retain_state else () + ready_symbol_ids = ( + {key: tuple(value) for key, value in symbol_ids.items()} + if retain_state + else {} + ) + ready_mappings = mapping_kinds if retain_state else {} + return CftcPositioningQueryV1( + corpus_id=corpus.corpus_id, + information_mode=mode, + start_ns=start_ns, + end_ns=end_ns, + as_of_ns=as_of_ns, + symbols=selected_symbols, + report_families=families, + report_scopes=scopes, + snapshots=snapshots, + symbol_snapshot_ids=ready_symbol_ids, + mapping_kinds=ready_mappings, + derived_values=( + _derive_query_values( + corpus, + snapshots, + ready_symbol_ids, + information_mode=mode, + as_of_ns=as_of_ns, + ) + if status is CftcPositioningQueryStatus.READY + else {} + ), + status=status, + reason=reason, + age_seconds=(ages if retain_state else {}), + ) + + +def preflight_cftc_positioning_corpus( + corpus: CftcPositioningCorpusV1, + **query_parameters: Any, +) -> CftcPositioningPreflightV1: + """Return a fail-closed readiness decision for one window.""" + query = query_cftc_positioning_corpus(corpus, **query_parameters) + return CftcPositioningPreflightV1( + corpus_id=corpus.corpus_id, + start_ns=query.start_ns, + end_ns=query.end_ns, + information_mode=query.information_mode, + as_of_ns=query.as_of_ns, + symbols=query.symbols, + report_families=query.report_families, + report_scopes=query.report_scopes, + ready=query.status is CftcPositioningQueryStatus.READY, + reasons=( + () + if query.status is CftcPositioningQueryStatus.READY + else (query.reason,) + ), + ) + + +def require_cftc_positioning_corpus( + corpus: CftcPositioningCorpusV1, + **query_parameters: Any, +) -> CftcPositioningPreflightV1: + """Raise with structured evidence when positioning is not supported.""" + decision = preflight_cftc_positioning_corpus(corpus, **query_parameters) + if not decision.ready: + raise CftcPositioningPreflightError(decision) + return decision + + +def cftc_positioning_information_inputs( + query: CftcPositioningQueryV1, + *, + run_id: str, + used_at_ns: int, + split_kind: InformationSplitKind | None = None, +) -> tuple[ReconstructionInformationInputV1, ...]: + """Declare every selected current-state row to the information audit.""" + if query.status is not CftcPositioningQueryStatus.READY: + raise ValueError("CFTC information inputs require a ready query") + used = _bounded_int64(used_at_ns, "used_at_ns") + inputs: list[ReconstructionInformationInputV1] = [] + for snapshot in query.snapshots: + evidence = snapshot.release_evidence + available = ( + evidence.restatement_detected_at_ns + or evidence.knowledge_at_ns + or evidence.publication_at_ns + or used + ) + revision = snapshot.restatement_status in { + CftcRestatementStatus.CURRENT_STATE_ONLY, + CftcRestatementStatus.RESTATED_CURRENT_STATE, + CftcRestatementStatus.RESTATEMENT_INCOMPLETE, + } + scope = ( + InformationScope.FULL_PERIOD_SUMMARY + if revision + and query.information_mode is InformationMode.EX_POST_RECONSTRUCTION + and available > used + else ( + InformationScope.REVISION + if revision + else InformationScope.POINT_IN_TIME + ) + ) + inputs.append( + ReconstructionInformationInputV1( + run_id=run_id, + artifact_id=f"{query.corpus_id}:{snapshot.snapshot_id}", + information_mode=query.information_mode, + input_kind=InformationInputKind.EXTERNAL, + stage=InformationStage.FEATURE, + scope=scope, + event_time_ns=snapshot.measurement_start_ns, + available_at_ns=available, + used_at_ns=used, + observation_start_ns=snapshot.measurement_start_ns, + observation_end_ns=snapshot.measurement_start_ns, + vintage_id=snapshot.snapshot_id, + reason=( + "CFTC current corrected-state positioning sidecar" + if revision + else "CFTC verified original-vintage positioning sidecar" + ), + allowed_lookahead_ns=( + max(0, available - used) + if query.information_mode + is InformationMode.EX_POST_RECONSTRUCTION + else 0 + ), + split_kind=split_kind, + ) + ) + return tuple(inputs) + + +def bind_cftc_positioning_query( + query: CftcPositioningQueryV1, + *, + consumer: CftcPositioningConsumer, + consumer_artifact_id: str, + run_id: str, + window_id: str, + information_inputs: Sequence[ReconstructionInformationInputV1], +) -> CftcPositioningConsumerBindingV1: + """Bind immutable v1 consumers through a companion lineage receipt.""" + if query.status is not CftcPositioningQueryStatus.READY: + raise ValueError("CFTC consumer binding requires a ready query") + inputs = tuple(information_inputs) + expected_artifacts = { + f"{query.corpus_id}:{item.snapshot_id}" for item in query.snapshots + } + if {item.artifact_id for item in inputs} != expected_artifacts: + raise ValueError("CFTC information inputs differ from query snapshots") + if any(item.run_id != run_id for item in inputs): + raise ValueError("CFTC information input run differs from binding") + return CftcPositioningConsumerBindingV1( + consumer=consumer, + consumer_artifact_id=consumer_artifact_id, + run_id=run_id, + window_id=window_id, + corpus_id=query.corpus_id, + query_id=query.query_id, + snapshot_ids=tuple(item.snapshot_id for item in query.snapshots), + information_input_ids=tuple(item.input_id for item in inputs), + state_label=cftc_positioning_state_label(query), + metrics=query.derived_values, + ) + + +def cftc_positioning_state_label(query: CftcPositioningQueryV1) -> str: + """Return a compact non-causal benchmark/motif state label.""" + if query.status is not CftcPositioningQueryStatus.READY: + return f"cftc_positioning:none:{query.status.value}" + dates = sorted({item.report_date for item in query.snapshots}) + return f"cftc_positioning:weekly:{dates[-1]}" + + +def apply_cftc_positioning_to_benchmark_events( + events: Sequence[Any], + binding: CftcPositioningConsumerBindingV1, +) -> tuple[Any, ...]: + """Project a receipt onto the existing benchmark event-state seam.""" + if binding.consumer is not CftcPositioningConsumer.BENCHMARK: + raise ValueError("CFTC benchmark projection requires benchmark binding") + projected: list[Any] = [] + for event in events: + if not hasattr(event, "event_state") or not hasattr( + event, "benchmark_event_id" + ): + raise ValueError( + "CFTC benchmark projection requires benchmark events" + ) + projected.append( + replace( + event, + event_state=f"{event.event_state}|{binding.state_label}", + benchmark_event_id="", + ) + ) + return tuple(projected) + + +def apply_cftc_positioning_to_motif_condition( + condition: Any, + binding: CftcPositioningConsumerBindingV1, +) -> Any: + """Project a bounded label while the companion receipt retains metrics.""" + if binding.consumer is not CftcPositioningConsumer.MOTIF_SELECTION: + raise ValueError("CFTC motif projection requires motif binding") + if not hasattr(condition, "event_tags") or not hasattr( + condition, "metrics" + ): + raise ValueError("CFTC motif projection requires a motif condition") + tags = tuple(condition.event_tags) + (binding.state_label,) + return replace(condition, event_tags=tags) + + +def validate_cftc_positioning_consumer_binding( + consumer_artifact: Any, + binding: CftcPositioningConsumerBindingV1, +) -> None: + """Verify run/window/identity continuity for planning or carving sidecars.""" + artifact_id = _consumer_artifact_id(consumer_artifact) + if artifact_id != binding.consumer_artifact_id: + raise ValueError("CFTC consumer artifact identity differs from binding") + if ( + hasattr(consumer_artifact, "run_id") + and consumer_artifact.run_id != binding.run_id + ): + raise ValueError("CFTC consumer run differs from binding") + if ( + hasattr(consumer_artifact, "window_id") + and consumer_artifact.window_id != binding.window_id + ): + raise ValueError("CFTC consumer window differs from binding") + + +def build_cftc_positioning_benchmark_smoke( + query: CftcPositioningQueryV1, + binding: CftcPositioningConsumerBindingV1, + events: Sequence[Any], + *, + source_artifact_id: str, + source_sha256: str, + reload_events: Sequence[Any] | None = None, +) -> CftcPositioningBenchmarkSmokeV1: + """Record deterministic held-out benchmark consumption evidence.""" + if binding.consumer is not CftcPositioningConsumer.BENCHMARK: + raise ValueError("CFTC smoke requires benchmark binding") + if binding.query_id != query.query_id: + raise ValueError("CFTC smoke query differs from binding") + projected = apply_cftc_positioning_to_benchmark_events(events, binding) + reloaded = apply_cftc_positioning_to_benchmark_events( + events if reload_events is None else reload_events, binding + ) + logical_hash = _event_output_hash(projected) + reload_hash = _event_output_hash(reloaded) + return CftcPositioningBenchmarkSmokeV1( + corpus_id=query.corpus_id, + query_id=query.query_id, + binding_id=binding.binding_id, + source_artifact_id=source_artifact_id, + source_sha256=source_sha256, + source_row_count=len(projected), + benchmark_event_ids=tuple( + item.benchmark_event_id for item in projected + ), + logical_output_sha256=logical_hash, + reload_output_sha256=reload_hash, + deterministic_reload=logical_hash == reload_hash, + ) + + +def write_cftc_positioning_benchmark_smoke( + smoke: CftcPositioningBenchmarkSmokeV1, + directory: str | Path, +) -> ArtifactRef: + """Write content-addressed real-consumption smoke evidence.""" + return _write_json_artifact( + Path(directory).expanduser().resolve(), + "cftc-positioning-benchmark-smoke", + "cftc_positioning_benchmark_smoke_v1", + smoke.to_dict(), + metadata={"smoke_id": smoke.smoke_id}, + ) + + +def write_cftc_positioning_consumer_binding( + binding: CftcPositioningConsumerBindingV1, + directory: str | Path, +) -> ArtifactRef: + """Write one content-addressed consumer-lineage receipt.""" + return _write_json_artifact( + Path(directory).expanduser().resolve(), + "cftc-positioning-consumer-binding", + "cftc_positioning_consumer_binding_v1", + binding.to_dict(), + metadata={ + "binding_id": binding.binding_id, + "consumer": binding.consumer.value, + }, + ) + + +def _validate_required_sources( + sources: Sequence[CftcPositioningRawSourceV1], + profile: CftcPositioningFetchProfileV1, +) -> None: + by_key = {item.source_key: item for item in sources} + required_keys = { + "official.release-schedule", + "official.special-announcements", + "official.historical-compressed-index", + "official.2025-backlog", + "official.web-policy", + } + missing_keys = required_keys - set(by_key) + if missing_keys: + raise ValueError( + "CFTC source set lacks required evidence: " + + ", ".join(sorted(missing_keys)) + ) + for dataset_id in profile.dataset_ids: + metadata = by_key.get(f"pre.{dataset_id}.metadata") + pages = sorted( + ( + item + for item in sources + if item.dataset_id == dataset_id + and item.source_kind == "pre_data" + ), + key=lambda item: item.source_key, + ) + if metadata is None or not pages: + raise ValueError(f"CFTC source set is incomplete for {dataset_id}") + expected_family = ( + CftcReportFamily.LEGACY + if dataset_id == CFTC_LEGACY_DATASET_ID + else CftcReportFamily.TFF + ) + if metadata.report_family is not expected_family: + raise ValueError("CFTC PRE metadata family differs from dataset") + offsets: list[int] = [] + for page in pages: + if page.report_family is not expected_family: + raise ValueError("CFTC PRE page family differs from dataset") + parameters = page.query_parameters + expected = _pre_query_parameters( + profile, offset=int(parameters.get("$offset", "-1")) + ) + if dict(parameters) != expected: + raise ValueError("CFTC PRE page query differs from profile") + offsets.append(int(parameters["$offset"])) + expected_offsets = list( + range(0, len(pages) * profile.page_size, profile.page_size) + ) + if sorted(offsets) != expected_offsets: + raise ValueError("CFTC PRE pages are missing or non-contiguous") + archive_names = { + Path(item.source_uri.split("?", 1)[0]).name + for item in sources + if item.source_kind == "historical_archive" + } + if archive_names != set(profile.historical_archives): + raise ValueError("CFTC historical archive sources differ from profile") + if any( + not isinstance(item, CftcPositioningRawSourceV1) for item in sources + ): + raise ValueError("CFTC source set contains unsupported values") + + +def _build_release_evidence( + sources: Sequence[CftcPositioningRawSourceV1], +) -> dict[str, CftcReleaseEvidenceV1]: + by_key = {item.source_key: item for item in sources} + schedule = by_key["official.release-schedule"] + special = by_key["official.special-announcements"] + backlog = by_key["official.2025-backlog"] + _require_source_terms(schedule, ("commitments", "release")) + _require_source_terms(special, ("commitments",)) + _require_source_terms(backlog, ("2025",)) + evidence: dict[str, CftcReleaseEvidenceV1] = {} + + backlog_dates = { + "2025-09-30": "2025-11-19", + "2025-10-07": "2025-11-21", + "2025-10-14": "2025-11-25", + "2025-10-21": "2025-12-02", + "2025-10-28": "2025-12-05", + "2025-11-04": "2025-12-09", + "2025-11-10": "2025-12-10", + "2025-11-18": "2025-12-12", + "2025-11-25": "2025-12-15", + "2025-12-02": "2025-12-17", + "2025-12-09": "2025-12-19", + "2025-12-16": "2025-12-23", + "2025-12-23": "2025-12-29", + } + for report_date, publication_date in backlog_dates.items(): + publication = _publication_ns( + _parse_date(publication_date, "publication_date") + ) + evidence[report_date] = CftcReleaseEvidenceV1( + report_date=report_date, + confidence=CftcAvailabilityConfidence.VERIFIED, + source_id=backlog.source_id, + publication_at_ns=publication, + knowledge_at_ns=publication, + notes=( + "Actual delayed publication date verified by CFTC release 9147-25.", + "The PRE row remains current corrected state rather than a captured original vintage.", + ), + ) + + qualified = ( + ( + "2010-05-18", + "2010-05-27", + CftcAvailabilityConfidence.RESTATEMENT_QUALIFIED, + "CFTC later corrected GBP and EUR rows.", + ), + ( + "2015-06-16", + "2015-06-23", + CftcAvailabilityConfidence.CORRECTION_QUALIFIED, + "CFTC identified an incomplete or premature publication.", + ), + ( + "2015-06-30", + "2015-07-06", + CftcAvailabilityConfidence.CORRECTION_QUALIFIED, + "CFTC special-announcement timing supersedes a nominal Friday estimate.", + ), + ( + "2018-09-18", + "2018-09-26", + CftcAvailabilityConfidence.RESTATEMENT_QUALIFIED, + "CFTC later revised the published report.", + ), + ) + for report_text, detected_text, confidence, note in qualified: + detected = _publication_ns(_parse_date(detected_text, "detected_date")) + evidence[report_text] = CftcReleaseEvidenceV1( + report_date=report_text, + confidence=confidence, + source_id=special.source_id, + publication_at_ns=None, + knowledge_at_ns=None, + restatement_detected_at_ns=detected, + notes=(note, "Original row vintage is not retained in PRE."), + ) + return evidence + + +def _nominal_release_evidence( + report_date: date, + sources: Sequence[CftcPositioningRawSourceV1], +) -> CftcReleaseEvidenceV1: + schedule = next( + item + for item in sources + if item.source_key == "official.release-schedule" + ) + days_to_friday = (4 - report_date.weekday()) % 7 + nominal_date = report_date + timedelta(days=days_to_friday) + publication = _publication_ns(nominal_date) + return CftcReleaseEvidenceV1( + report_date=report_date.isoformat(), + confidence=CftcAvailabilityConfidence.NOMINAL, + source_id=schedule.source_id, + publication_at_ns=publication, + knowledge_at_ns=publication, + notes=( + "Nominal Friday 15:30 America/New_York estimate; not verified for strict ex-ante use.", + "Holiday, shutdown, cyber-event, premature-release, and correction exceptions may apply.", + ), + ) + + +def _restatement_status( + report_family: CftcReportFamily, report_date: date +) -> CftcRestatementStatus: + del report_family + if report_date in { + date(2010, 5, 18), + date(2015, 6, 16), + date(2015, 6, 30), + date(2018, 9, 18), + }: + return CftcRestatementStatus.RESTATED_CURRENT_STATE + return CftcRestatementStatus.CURRENT_STATE_ONLY + + +def _positioning_values(row: Mapping[str, Any]) -> dict[str, float]: + values: dict[str, float] = {} + allowed_fragments = ( + "open_interest", + "positions_", + "_positions_", + "change_in_", + "pct_of_oi", + "traders_", + "conc_", + ) + for raw_key, raw_value in row.items(): + key = _source_key(str(raw_key).strip().lower()) + if not any(fragment in key for fragment in allowed_fragments): + continue + if raw_value is None or str(raw_value).strip() in {"", "."}: + continue + try: + value = float(str(raw_value).replace(",", "")) + except ValueError: + continue + if math.isfinite(value): + values[key] = value + if "open_interest_all" not in values: + raise ValueError("CFTC PRE row lacks open_interest_all") + return values + + +def _build_cftc_coverage( + snapshots: Sequence[CftcPositioningSnapshotV1], + *, + duplicate_counts: Mapping[tuple[str, str, str, str], int], + evidence_by_id: Mapping[str, CftcPositioningRawSourceV1], +) -> tuple[CftcPositioningCoverageSliceV1, ...]: + grouped: dict[ + tuple[int, CftcReportFamily, CftcReportScope, str], + list[CftcPositioningSnapshotV1], + ] = defaultdict(list) + for snapshot in snapshots: + report = _parse_date(snapshot.report_date, "report_date") + grouped[ + ( + report.year, + snapshot.report_family, + snapshot.report_scope, + snapshot.contract_code, + ) + ].append(snapshot) + result: list[CftcPositioningCoverageSliceV1] = [] + for (year, family, scope, code), values in sorted( + grouped.items(), + key=lambda item: ( + item[0][0], + item[0][1].value, + item[0][2].value, + item[0][3], + ), + ): + ordered = sorted(values, key=lambda item: item.report_date) + dates = [ + _parse_date(item.report_date, "report_date") for item in ordered + ] + missing_weeks = sum( + max(0, ((right - left).days // 7) - 1) + for left, right in zip(dates, dates[1:]) + ) + source_ids = {item.source_id for item in ordered} + sources = [evidence_by_id[item] for item in source_ids] + duplicate_count = sum( + count + for ( + family_name, + scope_name, + duplicate_code, + report_date, + ), count in duplicate_counts.items() + if family_name == family.value + and scope_name == scope.value + and duplicate_code == code + and _parse_date(report_date, "report_date").year == year + ) + result.append( + CftcPositioningCoverageSliceV1( + year=year, + report_family=family, + report_scope=scope, + contract_code=code, + row_count=len(ordered), + first_report_date=ordered[0].report_date, + last_report_date=ordered[-1].report_date, + missing_week_count=missing_weeks, + duplicate_key_count=duplicate_count, + contract_names=tuple(item.contract_name for item in ordered), + market_names=tuple(item.market_name for item in ordered), + availability_counts=dict( + Counter( + item.release_evidence.confidence.value + for item in ordered + ) + ), + restatement_counts=dict( + Counter(item.restatement_status.value for item in ordered) + ), + source_hashes=tuple(item.content_sha256 for item in sources), + source_bytes=sum(len(item.content) for item in sources), + processing_seconds=0.0, + ) + ) + return tuple(result) + + +def _archive_consistency( + source: CftcPositioningRawSourceV1, + snapshots: Sequence[CftcPositioningSnapshotV1], + profile: CftcPositioningFetchProfileV1, +) -> CftcArchiveConsistencyV1: + if source.report_family is None or source.report_scope is None: + raise ValueError("CFTC historical archive lacks family/scope metadata") + pre_by_key = { + (item.contract_code, item.report_date): item + for item in snapshots + if item.report_family is source.report_family + and item.report_scope is source.report_scope + } + selected = 0 + matched = 0 + missing = 0 + oi_mismatch = 0 + name_change = 0 + for row in _archive_rows(source.content): + normalized = { + _archive_field(key): value + for key, value in row.items() + if key is not None + } + code_text = _first_archive_value( + normalized, + "cftc_contract_market_code", + "cftc_contract_market_code_quotes", + ) + if not code_text: + continue + code = str(code_text).strip().strip('"').zfill(6) + if code not in profile.contract_codes: + continue + report_text = _first_archive_value( + normalized, + "as_of_date_in_form_yyyymmdd", + "as_of_date_in_form_yyyy_mm_dd", + "as_of_date_in_form_yymmdd", + "report_date_as_yyyy_mm_dd", + "report_date_as_yyyymmdd", + ) + report = _parse_archive_date(str(report_text or "")) + if report is None or not ( + _parse_date(profile.start_date, "start_date") + <= report + <= _parse_date(profile.end_date, "end_date") + ): + continue + selected += 1 + pre = pre_by_key.get((code, report.isoformat())) + if pre is None: + missing += 1 + continue + matched += 1 + oi_text = _first_archive_value(normalized, "open_interest_all") + if oi_text not in {None, ""}: + try: + archive_oi = float(str(oi_text).replace(",", "")) + except ValueError: + archive_oi = math.nan + if ( + math.isfinite(archive_oi) + and archive_oi != pre.values["open_interest_all"] + ): + oi_mismatch += 1 + archive_name = str( + _first_archive_value( + normalized, "market_and_exchange_names", "contract_market_name" + ) + or "" + ).strip() + if archive_name and archive_name not in { + pre.market_name, + pre.contract_name, + }: + name_change += 1 + return CftcArchiveConsistencyV1( + source_id=source.source_id, + report_family=source.report_family, + report_scope=source.report_scope, + selected_row_count=selected, + matched_pre_rows=matched, + missing_pre_rows=missing, + open_interest_mismatch_count=oi_mismatch, + contract_name_change_count=name_change, + limitations=( + "Consistency compares current PRE rows with current compressed-history rows and does not prove original-vintage equality.", + "Name changes are diagnostic metadata drift; numeric open-interest mismatches remain explicit.", + ), + ) + + +def _archive_rows(content: bytes) -> Iterable[Mapping[str, str]]: + try: + archive = zipfile.ZipFile(io.BytesIO(content)) + except zipfile.BadZipFile as exc: + raise ValueError("CFTC historical archive is not a valid ZIP") from exc + members = sorted( + (item for item in archive.infolist() if not item.is_dir()), + key=lambda item: item.filename, + ) + if not members: + raise ValueError("CFTC historical archive is empty") + total_uncompressed = sum(item.file_size for item in members) + if total_uncompressed > MAX_CFTC_SOURCE_BYTES * 4: + raise ValueError("CFTC historical archive expands beyond safety limit") + selected = next( + ( + item + for item in members + if item.filename.lower().endswith((".txt", ".csv")) + ), + members[0], + ) + if selected.file_size > MAX_CFTC_SOURCE_BYTES * 4: + raise ValueError("CFTC historical member exceeds safety limit") + try: + with archive.open(selected) as binary: + with io.TextIOWrapper( + binary, encoding="utf-8-sig", errors="replace", newline="" + ) as text_content: + yield from csv.DictReader(text_content) + finally: + archive.close() + + +def _peak_memory_bytes() -> int: + return peak_rss_bytes() + + +def _derive_query_values( + corpus: CftcPositioningCorpusV1, + selected: Sequence[CftcPositioningSnapshotV1], + symbol_snapshot_ids: Mapping[str, tuple[str, ...]], + *, + information_mode: InformationMode, + as_of_ns: int | None, +) -> dict[str, float]: + symbol_by_snapshot = { + snapshot_id: symbol + for symbol, snapshot_ids in symbol_snapshot_ids.items() + for snapshot_id in snapshot_ids + } + metrics: dict[str, float] = {} + for snapshot in selected: + pair = _primary_position_pair(snapshot.values, snapshot.report_family) + if pair is None: + continue + long_key, short_key = pair + net = snapshot.values[long_key] - snapshot.values[short_key] + oi = snapshot.values["open_interest_all"] + prefix = ".".join( + ( + symbol_by_snapshot[snapshot.snapshot_id].lower(), + snapshot.report_family.value, + snapshot.report_scope.value, + snapshot.contract_code, + ) + ) + metrics[f"{prefix}.net"] = net + metrics[f"{prefix}.net_open_interest"] = net / oi if oi else 0.0 + history = sorted( + ( + item + for item in corpus.snapshots + if item.report_family is snapshot.report_family + and item.report_scope is snapshot.report_scope + and item.contract_code == snapshot.contract_code + and item.report_date <= snapshot.report_date + and ( + information_mode is InformationMode.EX_POST_RECONSTRUCTION + or ( + item.strict_ex_ante_eligible + and item.release_evidence.knowledge_at_ns is not None + and as_of_ns is not None + and item.release_evidence.knowledge_at_ns <= as_of_ns + ) + ) + ), + key=lambda item: item.report_date, + )[-52:] + historical_nets: list[float] = [] + for item in history: + historical_pair = _primary_position_pair( + item.values, item.report_family + ) + if historical_pair is not None: + historical_nets.append( + item.values[historical_pair[0]] + - item.values[historical_pair[1]] + ) + deviation = ( + statistics.pstdev(historical_nets) + if len(historical_nets) > 1 + else 0.0 + ) + metrics[f"{prefix}.net_change"] = ( + net - historical_nets[-2] if len(historical_nets) > 1 else 0.0 + ) + metrics[f"{prefix}.rolling_52w_zscore"] = ( + (net - statistics.fmean(historical_nets)) / deviation + if deviation > 0 + else 0.0 + ) + return metrics + + +def _primary_position_pair( + values: Mapping[str, float], family: CftcReportFamily +) -> tuple[str, str] | None: + preferred = ( + ("noncomm_positions_long_all", "noncomm_positions_short_all") + if family is CftcReportFamily.LEGACY + else ("lev_money_positions_long_all", "lev_money_positions_short_all") + ) + if preferred[0] in values and preferred[1] in values: + return preferred + for long_key in sorted( + key for key in values if "positions_long_all" in key + ): + short_key = long_key.replace( + "positions_long_all", "positions_short_all" + ) + if short_key in values: + return long_key, short_key + return None + + +def _write_json_artifact( + root: Path, + stem: str, + kind: str, + payload: Mapping[str, JSONValue], + *, + metadata: Mapping[str, JSONValue], +) -> ArtifactRef: + root.mkdir(parents=True, exist_ok=True) + content = canonical_contract_json(payload).encode("utf-8") + b"\n" + digest = hashlib.sha256(content).hexdigest() + artifact_path = root / f"{stem}-{digest}.json" + _write_once(artifact_path, content) + return ArtifactRef( + kind=kind, + path=str(artifact_path), + size_bytes=len(content), + sha256=digest, + metadata=dict(metadata), + ) + + +def _write_once(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + if path.read_bytes() != content: + raise ValueError(f"content-addressed artifact differs: {path}") + return + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.tmp-", dir=path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + if path.exists(): + if path.read_bytes() != content: + raise ValueError(f"content-addressed artifact differs: {path}") + else: + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def _content_suffix(content_type: str, source_uri: str) -> str: + media_type = content_type.lower() + if "json" in media_type: + return ".json" + if "zip" in media_type or source_uri.lower().split("?", 1)[0].endswith( + ".zip" + ): + return ".zip" + if "pdf" in media_type or source_uri.lower().split("?", 1)[0].endswith( + ".pdf" + ): + return ".pdf" + if "html" in media_type: + return ".html" + if "csv" in media_type: + return ".csv" + return ".bin" + + +def _json_rows(content: bytes, label: str) -> tuple[Mapping[str, Any], ...]: + if len(content) > MAX_CFTC_SOURCE_BYTES: + raise ValueError(f"{label} exceeds size bound") + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"{label} is invalid JSON") from exc + return _mapping_sequence(payload) + + +def _parse_pre_report_date(value: str) -> date: + text = value.strip() + if not text: + raise ValueError("CFTC PRE row lacks report date") + return _parse_date(text[:10], "report_date") + + +def _publication_ns(value: date) -> int: + eastern = ZoneInfo("America/New_York") + published = datetime.combine(value, datetime_time(15, 30), tzinfo=eastern) + return int(published.timestamp() * 1_000_000_000) + + +def _date_start_ns(value: date) -> int: + return int( + datetime.combine( + value, datetime_time(), tzinfo=timezone.utc + ).timestamp() + * 1_000_000_000 + ) + + +def _validate_query_interval( + start_ns: int, + end_ns: int, + mode: InformationMode, + as_of_ns: int | None, +) -> None: + start = _bounded_int64(start_ns, "start_ns") + end = _bounded_int64(end_ns, "end_ns") + if end <= start: + raise ValueError("CFTC query end must follow start") + if mode is InformationMode.EX_ANTE_SIMULATION: + if as_of_ns is None: + raise ValueError("ex-ante CFTC query requires as_of_ns") + cutoff = _bounded_int64(as_of_ns, "as_of_ns") + if cutoff > start: + raise ValueError("CFTC query as_of cannot follow start") + elif as_of_ns is not None: + raise ValueError("ex-post CFTC query does not accept as_of_ns") + + +def _select_symbol_mapping( + corpus: CftcPositioningCorpusV1, symbol: str, window_date: date +) -> CftcPositioningSymbolMappingV1: + candidates = [ + item for item in corpus.symbol_mappings if item.symbol == symbol + ] + direct = [ + item + for item in candidates + if item.mapping_kind is CftcMappingKind.DIRECT + and ( + item.valid_from_date is None + or _parse_date(item.valid_from_date, "valid_from_date") + <= window_date + ) + ] + if direct: + return direct[-1] + derived = [ + item + for item in candidates + if item.mapping_kind is CftcMappingKind.DERIVED_TWO_LEG + ] + if derived: + return derived[0] + raise ValueError(f"unsupported CFTC positioning symbol: {symbol}") + + +def _normalized_symbol(value: str) -> str: + normalized = re.sub(r"[^A-Za-z]", "", _required_text(value)).upper() + if re.fullmatch(r"[A-Z]{6}", normalized) is None: + raise ValueError("CFTC positioning symbol must be a six-letter pair") + return normalized + + +def _normalized_symbols(values: Iterable[str]) -> tuple[str, ...]: + return tuple(sorted({_normalized_symbol(item) for item in values})) + + +def _consumer_artifact_id(value: Any) -> str: + if isinstance(value, str): + return _required_text(value) + for name in ( + "batch_id", + "plan_id", + "benchmark_id", + "artifact_id", + "condition_id", + ): + candidate = getattr(value, name, None) + if candidate: + return _required_text(str(candidate)) + raise ValueError("CFTC consumer artifact lacks a stable identity") + + +def _event_output_hash(events: Sequence[Any]) -> str: + payload: list[JSONValue] = [] + for event in events: + if not hasattr(event, "to_dict"): + raise ValueError( + "CFTC smoke event lacks deterministic serialization" + ) + payload.append(_json_value_mapping(event.to_dict())) + return hashlib.sha256( + canonical_contract_json({"events": payload}).encode("utf-8") + ).hexdigest() + + +def _require_source_terms( + source: CftcPositioningRawSourceV1, terms: Sequence[str] +) -> None: + content = source.content.decode("utf-8", errors="ignore").lower() + if any(term.lower() not in content for term in terms): + raise ValueError( + f"CFTC source {source.source_key} lacks expected content" + ) + + +def _archive_field(value: str) -> str: + normalized = re.sub(r"[^a-z0-9]+", "_", str(value).strip().lower()).strip( + "_" + ) + return normalized + + +def _first_archive_value(row: Mapping[str, str], *names: str) -> str | None: + for name in names: + value = row.get(name) + if value is not None: + return value + return None + + +def _parse_archive_date(value: str) -> date | None: + text = value.strip().strip('"') + if not text: + return None + formats = ( + "%Y-%m-%d", + "%Y%m%d", + "%m/%d/%Y", + "%m/%d/%Y %I:%M:%S %p", + "%y%m%d", + ) + for format_string in formats: + try: + return datetime.strptime(text, format_string).date() + except ValueError: + continue + if len(text) >= 10: + try: + return date.fromisoformat(text[:10]) + except ValueError: + pass + return None + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + return f"{prefix}-{digest}" + + +def _required_text(value: Any) -> str: + if not isinstance(value, str): + raise ValueError("value must be text") + normalized = value.strip() + if not normalized or len(normalized) > MAX_CFTC_TEXT: + raise ValueError("text is empty or exceeds limit") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + return _required_text(value) + + +def _source_key(value: str) -> str: + normalized = value.strip().lower() + if _SOURCE_KEY_RE.fullmatch(normalized) is None: + raise ValueError("invalid CFTC source key") + return normalized + + +def _contract_code(value: str) -> str: + normalized = value.strip().upper().zfill(6) + if _CONTRACT_CODE_RE.fullmatch(normalized) is None: + raise ValueError("invalid CFTC contract code") + return normalized + + +def _parse_date(value: str, name: str) -> date: + try: + return date.fromisoformat(_required_text(value)) + except ValueError as exc: + raise ValueError(f"{name} must be an ISO date") from exc + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be finite") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be finite") from exc + if not math.isfinite(normalized): + raise ValueError(f"{name} must be finite") + return normalized + + +def _positive_float(value: Any, name: str) -> float: + normalized = _finite_float(value, name) + if normalized <= 0: + raise ValueError(f"{name} must be positive") + return normalized + + +def _finite_nonnegative(value: Any, name: str) -> float: + normalized = _finite_float(value, name) + if normalized < 0: + raise ValueError(f"{name} must be nonnegative") + return normalized + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _optional_int(value: Any) -> int | None: + if value is None: + return None + return _strict_int(value, "optional integer") + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + normalized = _strict_int(value, name) + if normalized < minimum or normalized > maximum: + raise ValueError(f"{name} is outside bounds") + return normalized + + +def _bounded_int64(value: Any, name: str) -> int: + return _bounded_int(value, name, -(2**63), 2**63 - 1) + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be boolean") + return value + + +def _required_sha256(value: Any, name: str) -> str: + normalized = _required_text(value) + if _SHA256_RE.fullmatch(normalized) is None: + raise ValueError(f"{name} must be lowercase SHA-256") + return normalized + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("value must be an object") + return cast(Mapping[str, Any], value) + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError("value must be an array") + return value + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + result: list[Mapping[str, Any]] = [] + for item in _sequence(value): + result.append(_mapping(item)) + return tuple(result) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(_required_text(item) for item in _sequence(value)) + + +def _metric_mapping(value: Mapping[str, Any]) -> dict[str, float]: + result = { + _source_key(str(key)): _finite_float(item, str(key)) + for key, item in sorted(value.items()) + } + if len(result) > MAX_CFTC_BINDING_VALUES: + raise ValueError("CFTC metric mapping exceeds limit") + return result + + +def _count_mapping(value: Mapping[str, Any], name: str) -> dict[str, int]: + result = { + _source_key(str(key)): _bounded_int(item, str(key), 0, MAX_CFTC_ROWS) + for key, item in sorted(value.items()) + } + if not result: + raise ValueError(f"{name} cannot be empty") + return result + + +def _id_mapping(value: Mapping[str, Any]) -> dict[str, str]: + return { + _required_text(str(key)): _required_text(str(item)) + for key, item in sorted(value.items()) + } + + +def _json_value_mapping(value: Mapping[str, Any]) -> dict[str, JSONValue]: + result: dict[str, JSONValue] = {} + for key, item in value.items(): + normalized_key = _required_text(str(key)) + result[normalized_key] = _validate_json_value(item, depth=0) + return result + + +def _validate_json_value(value: Any, *, depth: int) -> JSONValue: + if depth > 16: + raise ValueError("JSON value nesting exceeds limit") + if value is None or isinstance(value, (str, bool, int)): + return cast(JSONValue, value) + if isinstance(value, float): + return cast(JSONValue, _finite_float(value, "JSON float")) + if isinstance(value, Mapping): + return cast( + JSONValue, + { + _required_text(str(key)): _validate_json_value( + item, depth=depth + 1 + ) + for key, item in value.items() + }, + ) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + return cast( + JSONValue, + [_validate_json_value(item, depth=depth + 1) for item in value], + ) + raise ValueError("unsupported JSON value") + + +def _ensure_payload_size( + payload: Mapping[str, JSONValue], maximum: int +) -> None: + if len(canonical_contract_json(payload).encode("utf-8")) > maximum: + raise ValueError("CFTC payload exceeds size bound") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if data.get("schema_version") != expected: + raise ValueError("unsupported CFTC artifact schema") diff --git a/src/histdatacom/options.py b/src/histdatacom/options.py index b6d1af1c..071972aa 100644 --- a/src/histdatacom/options.py +++ b/src/histdatacom/options.py @@ -30,6 +30,7 @@ class Options: "orchestration_start", "orchestration_keep_runtime", "orchestration_wait_result", + "output_timezone", "pair_groups", "pairs", "quality_check_groups", @@ -48,9 +49,11 @@ class Options: "quality_preflight_run_validation", "quality_preflight_sample_size", "quality_preflight_validation_report_path", + "quality_preflight_validation_evidence_path", "quality_paths", "quality_profile", "quality_profile_path", + "quality_profile_resolution", "quality_profile_preview", "quality_profile_preview_format", "quality_profile_preview_output_path", @@ -58,6 +61,8 @@ class Options: "quality_report_path", "repo_quality_columns", "repo_quality_refresh", + "random_seed", + "random_window", "request_bundle_out", "request_json_out", "schedule_key", @@ -93,6 +98,8 @@ def __init__(self) -> None: self.timeframes: set = Timeframe.list_keys() self.start_yearmonth: str | None = "" self.end_yearmonth: str | None = "" + self.random_window: str = "" + self.random_seed: int | None = None self.data_directory: str = "data" self.from_api: bool = False self.api_return_type: str | None = None @@ -120,8 +127,10 @@ def __init__(self) -> None: self.quality_preflight_report_path: str | None = None self.quality_preflight_run_validation: bool = False self.quality_preflight_validation_report_path: str | None = None + self.quality_preflight_validation_evidence_path: str | None = None self.quality_profile_path: str | None = None self.quality_profile: dict = {} + self.quality_profile_resolution: dict = {} self.quality_profile_preview: bool = False self.quality_profile_preview_format: str = "json" self.quality_profile_preview_output_path: str | None = None @@ -134,6 +143,7 @@ def __init__(self) -> None: self.orchestration_start: bool = True self.orchestration_keep_runtime: bool = False self.orchestration_wait_result: bool = True + self.output_timezone: str = "" self.no_overlap: bool = False self.schedule_key: str = "" self.verbosity: int = 0 diff --git a/src/histdatacom/orchestration/__init__.py b/src/histdatacom/orchestration/__init__.py index d1f9154e..2ea5c83d 100644 --- a/src/histdatacom/orchestration/__init__.py +++ b/src/histdatacom/orchestration/__init__.py @@ -107,6 +107,10 @@ "histdatacom.orchestration.client", "DEFAULT_RUN_WORKFLOW_NAME", ), + "DEFAULT_RECONSTRUCTION_WORKFLOW_NAME": ( + "histdatacom.orchestration.client", + "DEFAULT_RECONSTRUCTION_WORKFLOW_NAME", + ), "DEFAULT_TASK_QUEUE_PREFIX": ( "histdatacom.orchestration.queues", "DEFAULT_TASK_QUEUE_PREFIX", @@ -119,6 +123,66 @@ "histdatacom.orchestration.workflows", "DEFAULT_WORKFLOWS", ), + "ReconstructionCheckpointStore": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionCheckpointStore", + ), + "ReconstructionRunReportV1": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionRunReportV1", + ), + "ReconstructionRunWorkflow": ( + "histdatacom.orchestration.workflows", + "ReconstructionRunWorkflow", + ), + "ReconstructionStage": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionStage", + ), + "ReconstructionStageCommandV1": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionStageCommandV1", + ), + "ReconstructionStageInvocationV1": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionStageInvocationV1", + ), + "ReconstructionStageOutcomeV1": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionStageOutcomeV1", + ), + "ReconstructionStageStatus": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionStageStatus", + ), + "ReconstructionWindowStateV1": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionWindowStateV1", + ), + "ReconstructionWindowTaskV1": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionWindowTaskV1", + ), + "ReconstructionWindowWorkflow": ( + "histdatacom.orchestration.workflows", + "ReconstructionWindowWorkflow", + ), + "ReconstructionWorkflowRequestV1": ( + "histdatacom.orchestration.reconstruction", + "ReconstructionWorkflowRequestV1", + ), + "register_reconstruction_stage_handler": ( + "histdatacom.orchestration.reconstruction", + "register_reconstruction_stage_handler", + ), + "registered_reconstruction_stage_handlers": ( + "histdatacom.orchestration.reconstruction", + "registered_reconstruction_stage_handlers", + ), + "submit_reconstruction_request": ( + "histdatacom.orchestration.client", + "submit_reconstruction_request", + ), "FANOUT_METADATA_KEY": ( "histdatacom.orchestration.workflows", "FANOUT_METADATA_KEY", diff --git a/src/histdatacom/orchestration/activities.py b/src/histdatacom/orchestration/activities.py index e52cc747..37aa67e9 100644 --- a/src/histdatacom/orchestration/activities.py +++ b/src/histdatacom/orchestration/activities.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from dataclasses import replace from importlib import import_module import logging @@ -76,9 +77,22 @@ WorkStatus, derive_work_id, ) +from histdatacom.random_windows import random_window_selection_from_metadata from histdatacom.source_cleanup import source_artifact_cleanliness_payload from histdatacom.verbosity import safe_log_extra from histdatacom.utils import set_working_data_dir +from histdatacom.orchestration.reconstruction import ( + ReconstructionCheckpointStore, + ReconstructionCommitPhase, + ReconstructionReportMismatch, + ReconstructionWorkflowRequestV1, + ReconstructionWindowStateV1, + RegisteredReconstructionStageExecutor, + cleanup_reconstruction_window_scratch, + reconcile_reconstruction_report, + run_reconstruction_window, + write_reconstruction_report, +) from histdatacom.orchestration.workflow_metadata import TASK_QUEUE_METADATA_KEY @@ -193,10 +207,10 @@ def dataset_plan_activity(payload: dict[str, Any]) -> dict[str, Any]: repository_ranges = ( read_repository_data_file(repo_path) if ( - not request.start_yearmonth - and not request.end_yearmonth - and repo_path.exists() + (not request.start_yearmonth and not request.end_yearmonth) + or bool(request.random_window) ) + and repo_path.exists() else {} ) repository_range_count = len(repository_pair_data(repository_ranges)) @@ -211,6 +225,8 @@ def dataset_plan_activity(payload: dict[str, Any]) -> dict[str, Any]: zip_persist=request.zip_persist, cache_only=request.build_cache, repository_ranges=repository_ranges, + random_window=request.random_window, + random_seed=request.random_seed, ) plan_id = derive_work_id( request.request_id, @@ -225,6 +241,8 @@ def dataset_plan_activity(payload: dict[str, Any]) -> dict[str, Any]: metadata={ "start_yearmonth": request.start_yearmonth, "end_yearmonth": request.end_yearmonth, + "random_window": request.random_window, + "random_seed": request.random_seed, "formats": list(request.formats), "pairs": list(request.pairs), "timeframes": list(request.timeframes), @@ -540,6 +558,99 @@ def validate_urls_activity(payload: dict[str, Any]) -> dict[str, Any]: return _activity_batch_payload(outputs, aggregate) +@activity_defn(name="reconstruction_window") +async def reconstruction_window_activity( + payload: dict[str, Any], +) -> dict[str, Any]: + """Run or resume one artifact-backed synthetic reconstruction window.""" + request = ReconstructionWorkflowRequestV1.from_dict( + _mapping(payload.get("request", {})) + ) + if len(request.tasks) != 1: + raise ValueError("reconstruction window activity requires one task") + task = request.tasks[0] + store = ReconstructionCheckpointStore(request.manifest_store_root) + + def emit(heartbeat: Any) -> None: + _activity_heartbeat(heartbeat.to_dict()) + + try: + state = await run_reconstruction_window( + request, + task, + checkpoint_store=store, + stage_executor=RegisteredReconstructionStageExecutor(), + heartbeat=emit, + cancellation_requested=_activity_cancelled, + ) + except (Exception, asyncio.CancelledError) as err: + if not _reconstruction_cancellation_error(err): + raise + stored_state = store.load(task.window) + if stored_state is not None and stored_state.checkpoint.phase not in { + ReconstructionCommitPhase.COMMITTED, + ReconstructionCommitPhase.CANCELLED, + }: + cancelled = stored_state.interrupted( + ReconstructionCommitPhase.CANCELLED, + "Temporal cancelled reconstruction window activity", + ) + store.save(cancelled, expected_state_id=stored_state.state_id) + cleanup_reconstruction_window_scratch(task.scratch_directory) + raise + return cast(dict[str, Any], state.to_dict()) + + +@activity_defn(name="reconstruction_report") +def reconstruction_report_activity( + payload: dict[str, Any], +) -> dict[str, Any]: + """Reconcile workflow checkpoints with committed storage manifests.""" + request = ReconstructionWorkflowRequestV1.from_dict( + _mapping(payload.get("request", {})) + ) + checkpoint_store = ReconstructionCheckpointStore( + request.manifest_store_root + ) + loaded_states = tuple( + checkpoint_store.load(task.window) for task in request.tasks + ) + if any(state is None for state in loaded_states): + raise ReconstructionReportMismatch( + "reconstruction report cannot find every durable window checkpoint" + ) + states = tuple( + cast(ReconstructionWindowStateV1, state) for state in loaded_states + ) + report = reconcile_reconstruction_report( + request, + states, + progress=lambda metadata: _activity_heartbeat(dict(metadata)), + ) + report_ref = write_reconstruction_report(report, request.report_root) + ManifestStatusStore(request.manifest_store_root).write_job_snapshot( + { + "schema_version": 1, + "job_id": f"reconstruction-run-{request.request_id}", + "request_id": request.request_id, + "workflow_id": f"reconstruction-run-{request.request_id}", + "run_id": request.run.run_id, + "lifecycle": report.status, + "status": ( + WorkStatus.COMPLETED.value + if report.status == "committed" + else WorkStatus.FAILED.value + ), + "artifacts": [report_ref.to_dict()], + "reconstruction_report": report.to_dict(), + } + ) + return { + "report": report.to_dict(), + "report_ref": report_ref.to_dict(), + } + + @activity_defn(name="download_archives") def download_archives_activity( payload: dict[str, Any], @@ -694,6 +805,7 @@ def merge_cache_activity( dict[str, Any], result.to_dict(), ) + _require_resolved_random_window(request, work_items) if _activity_cancelled(): result = _observe_and_persist_stage_result( @@ -752,6 +864,7 @@ def import_to_influx_activity( dict[str, Any], result.to_dict(), ) + _require_resolved_random_window(request, work_items) total = len(work_items) args = _influx_args(request) @@ -803,9 +916,17 @@ def default_activities() -> tuple[Callable[..., Any], ...]: build_cache_activity, merge_cache_activity, import_to_influx_activity, + reconstruction_window_activity, + reconstruction_report_activity, ) +def _reconstruction_cancellation_error(err: BaseException) -> bool: + """Return whether Temporal or asyncio delivered cancellation.""" + normalized = type(err).__name__.strip().lower() + return normalized in {"cancellederror", "cancelederror"} + + def _repo_local_path(request: RunRequest) -> Path: data_dir = Path(set_working_data_dir(request.data_directory)) return data_dir / ".repo" @@ -1016,6 +1137,32 @@ def _influx_args(request: RunRequest) -> dict[str, Any]: return args +def _require_resolved_random_window( + request: RunRequest, + work_items: tuple[WorkItem, ...], +) -> None: + """Fail closed when a declared random window lost its resolved metadata.""" + if not request.random_window: + return + selections = tuple( + random_window_selection_from_metadata(item.metadata) + for item in work_items + ) + selection_ids = { + selection.selection_id + for selection in selections + if selection is not None + } + if ( + all(selection is not None for selection in selections) + and len(selection_ids) == 1 + ): + return + raise ValueError( + "random-window request is missing its resolved work-item selection" + ) + + def _influx_batch_writer(args: Mapping[str, Any]) -> Any: from histdatacom.influx import InfluxBatchWriter diff --git a/src/histdatacom/orchestration/client.py b/src/histdatacom/orchestration/client.py index 3f8ddb25..fbe13178 100644 --- a/src/histdatacom/orchestration/client.py +++ b/src/histdatacom/orchestration/client.py @@ -11,7 +11,7 @@ import json import logging from pathlib import Path -from typing import Any, Mapping, cast +from typing import TYPE_CHECKING, Any, Mapping, cast from histdatacom.cancellation import ( PartialArtifactDisposition, @@ -57,12 +57,18 @@ TOPOLOGY_SCHEMA_VERSION, ) +if TYPE_CHECKING: + from histdatacom.orchestration.reconstruction import ( + ReconstructionWorkflowRequestV1, + ) + TEMPORAL_EXTRA_HINT = ( "Temporal support requires temporalio. Base histdatacom installs include " "this dependency; reinstall histdatacom with dependencies enabled or " "install the temporal compatibility extra." ) DEFAULT_RUN_WORKFLOW_NAME = "HistDataRunWorkflow" +DEFAULT_RECONSTRUCTION_WORKFLOW_NAME = "ReconstructionRunWorkflow" RUN_REQUEST_METADATA_KEY = "run_request" CONTROL_ATTEMPTS_METADATA_KEY = "control_attempts" CONTROL_EXECUTION_METADATA_KEY = "control_execution" @@ -353,8 +359,7 @@ async def submit_run_request( status_store=status_store, ) LOGGER.info( - "Submitted HistData orchestration job request_id=%s workflow_id=%s " - "run_id=%s", + "Submitted HistData orchestration job request_id=%s workflow_id=%s run_id=%s", request.request_id, job_handle.workflow_id, job_handle.run_id, @@ -370,6 +375,84 @@ async def submit_run_request( return job_handle +async def submit_reconstruction_request( + request: ReconstructionWorkflowRequestV1, + *, + config: OrchestrationWorkerConfig | None = None, + supervisor: OrchestrationSupervisor | None = None, + client: Any | None = None, + client_class: Any | None = None, + status_store: ManifestStatusStore | None = None, + workflow: Any = DEFAULT_RECONSTRUCTION_WORKFLOW_NAME, + workflow_id: str = "", + execution_attempt_id: str = "", +) -> OrchestrationJobHandle: + """Submit an artifact-only synthetic reconstruction workflow. + + ``execution_attempt_id`` is deliberately outside the immutable scientific + request. A recovery submission can therefore use fresh parent/child + Temporal identities while reusing the exact request and durable window + checkpoints. Empty attempt IDs preserve the original workflow IDs. + """ + execution_attempt_id = _normalized_reconstruction_attempt_id( + execution_attempt_id + ) + resolved_config = resolve_orchestration_worker_config( + config=config, + supervisor=supervisor, + require_running=config is None, + ) + request = replace( + request, + task_queues=resolved_config.task_queues.to_dict(), + request_fingerprint="", + ) + temporal_client = client or await connect_temporal_client( + config=resolved_config, + supervisor=supervisor, + client_class=client_class, + ) + resolved_workflow_id = workflow_id or _reconstruction_workflow_id(request) + payload: dict[str, Any] = {"request": request.to_dict()} + if execution_attempt_id: + payload["execution_attempt_id"] = execution_attempt_id + handle = await _maybe_await( + temporal_client.start_workflow( + workflow, + payload, + id=resolved_workflow_id, + task_queue=resolved_config.task_queues.orchestration, + ) + ) + job_handle = OrchestrationJobHandle( + request_id=request.request_id, + workflow_id=str(getattr(handle, "id", resolved_workflow_id)), + run_id=str(getattr(handle, "run_id", "")), + task_queue=resolved_config.task_queues.orchestration, + namespace=resolved_config.namespace, + ) + store = status_store or ManifestStatusStore(request.manifest_store_root) + store.write_job_snapshot( + { + "schema_version": 1, + "job_id": resolved_workflow_id, + "request_id": request.request_id, + "workflow_id": resolved_workflow_id, + "run_id": job_handle.run_id, + "lifecycle": "submitted", + "status": WorkStatus.PLANNED.value, + "task_queue": job_handle.task_queue, + "metadata": { + "reconstruction_run_id": request.run.run_id, + "request_fingerprint": request.request_fingerprint, + "window_count": len(request.tasks), + "execution_attempt_id": execution_attempt_id, + }, + } + ) + return job_handle + + async def submit_run_request_and_observe( request: RunRequest, *, @@ -427,8 +510,7 @@ async def submit_run_request_and_observe( ) workflow_id = workflow_id_for_request(request) LOGGER.info( - "Submitting HistData orchestration job request_id=%s " - "workflow_id=%s", + "Submitting HistData orchestration job request_id=%s workflow_id=%s", request.request_id, workflow_id, extra=_request_log_context( @@ -504,8 +586,7 @@ async def submit_run_request_and_observe( _notify_progress_observer(progress_observer, submitted_snapshot) LOGGER.info( - "Waiting for HistData orchestration job request_id=%s " - "workflow_id=%s", + "Waiting for HistData orchestration job request_id=%s workflow_id=%s", request.request_id, handle.workflow_id, extra=_request_log_context( @@ -1351,6 +1432,28 @@ def workflow_id_for_request(request: RunRequest) -> str: return f"histdatacom-{request_id}" +def _reconstruction_workflow_id( + request: ReconstructionWorkflowRequestV1, +) -> str: + """Return a bounded deterministic workflow ID for reconstruction.""" + digest = hashlib.sha256( + f"{request.run.run_id}|{request.request_fingerprint}".encode("utf-8") + ).hexdigest()[:24] + return f"histdatacom-reconstruction-{request.request_id}-{digest}" + + +def _normalized_reconstruction_attempt_id(value: str) -> str: + """Return one bounded non-secret Temporal attempt discriminator.""" + attempt = str(value or "").strip() + if len(attempt) > 64: + raise ValueError("reconstruction execution_attempt_id exceeds 64 chars") + if any(not (char.isalnum() or char in {"-", "_", "."}) for char in attempt): + raise ValueError( + "reconstruction execution_attempt_id contains unsupported characters" + ) + return attempt + + def resolve_orchestration_worker_config( *, config: OrchestrationWorkerConfig | None = None, @@ -2322,8 +2425,7 @@ def _ensure_orchestration_available( except RuntimeError as err: if _is_missing_temporal_dependency_error(err): LOGGER.warning( - "Temporal orchestration startup dependency unavailable " - "error=%s", + "Temporal orchestration startup dependency unavailable error=%s", str(err), extra=_status_log_context( status, @@ -2364,7 +2466,7 @@ def _ensure_orchestration_available( ), ) raise OrchestrationUnavailableError( - "Temporal orchestration could not be started: " f"{started.message}" + f"Temporal orchestration could not be started: {started.message}" ) diff --git a/src/histdatacom/orchestration/performance.py b/src/histdatacom/orchestration/performance.py index 51ad77fb..131ae881 100644 --- a/src/histdatacom/orchestration/performance.py +++ b/src/histdatacom/orchestration/performance.py @@ -2,14 +2,13 @@ from __future__ import annotations -import sys import time from dataclasses import dataclass, field, replace -from importlib import import_module from enum import Enum from typing import Any, Callable, Mapping from histdatacom.concurrency import get_pool_cpu_count +from histdatacom.resource_usage import peak_rss_bytes from histdatacom.runtime_contracts import JSONValue, RunRequest, WorkItem DEFAULT_NETWORK_MULTIPLIER = 3 @@ -290,15 +289,7 @@ def measure_startup( def _peak_rss_bytes() -> int: - try: - resource = import_module("resource") - except ModuleNotFoundError: - return 0 - usage = resource.getrusage(resource.RUSAGE_SELF) - peak = int(getattr(usage, "ru_maxrss", 0) or 0) - if sys.platform.startswith("linux"): - return peak * 1024 - return peak + return peak_rss_bytes() def _normalize_lane(lane: object) -> str: diff --git a/src/histdatacom/orchestration/reconstruction.py b/src/histdatacom/orchestration/reconstruction.py new file mode 100644 index 00000000..93554e18 --- /dev/null +++ b/src/histdatacom/orchestration/reconstruction.py @@ -0,0 +1,2286 @@ +"""Durable, bounded orchestration for synthetic reconstruction windows. + +Temporal owns ordering, retries, and progress. Scientific rows stay in files +owned by stage handlers; workflow-visible values are contracts, counters, and +strong :class:`~histdatacom.runtime_contracts.ArtifactRef` objects only. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import inspect +import json +import os +import shutil +import tempfile +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from enum import Enum +from functools import lru_cache +from pathlib import Path +from typing import Any, Protocol, cast + +from histdatacom.manifest_store import ManifestStatusStore +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.contracts import canonical_contract_json +from histdatacom.synthetic.persistence import ( + verify_reconstruction_publication, +) +from histdatacom.synthetic.streaming import ( + ReconstructionCheckpointV1, + ReconstructionCommitPhase, + ReconstructionHeartbeatV1, + ReconstructionResourceEstimateV1, + ReconstructionResourceLimitError, + ReconstructionRunV1, + ReconstructionStoragePolicyV1, + ReconstructionWindowV1, +) + +RECONSTRUCTION_ORCHESTRATION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-orchestration.v1" +) +RECONSTRUCTION_STAGE_COMMAND_SCHEMA_VERSION = ( + "histdatacom.reconstruction-stage-command.v1" +) +RECONSTRUCTION_STAGE_OUTCOME_SCHEMA_VERSION = ( + "histdatacom.reconstruction-stage-outcome.v1" +) +RECONSTRUCTION_WINDOW_TASK_SCHEMA_VERSION = ( + "histdatacom.reconstruction-window-task.v1" +) +RECONSTRUCTION_WORKFLOW_REQUEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-workflow-request.v1" +) +RECONSTRUCTION_WINDOW_STATE_SCHEMA_VERSION = ( + "histdatacom.reconstruction-window-state.v1" +) +RECONSTRUCTION_REPORT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-run-report.v1" +) + +MAX_RECONSTRUCTION_WORKFLOW_BYTES = 1_048_576 +MAX_RECONSTRUCTION_WINDOW_STATE_BYTES = 1_048_576 +MAX_RECONSTRUCTION_REPORT_BYTES = 1_048_576 +MAX_RECONSTRUCTION_WINDOWS = 512 +MAX_STAGE_ARTIFACT_REFS = 32 +MAX_STAGE_RECEIPT_BYTES = 262_144 +MAX_STAGE_MESSAGE_LENGTH = 2_048 +DEFAULT_MAX_PARALLEL_RECONSTRUCTION_WINDOWS = 2 +RECONSTRUCTION_REPORT_DIRECTORY = "reconstruction-reports" +RECONSTRUCTION_RECEIPT_KIND = "reconstruction_stage_receipt" + +_FORBIDDEN_WORKFLOW_KEYS = frozenset( + {"dataframe", "event_batches", "events", "records", "rows", "table"} +) + + +class ReconstructionStage(str, Enum): + """Ordered data-plane boundaries controlled by Temporal.""" + + SOURCE_ENRICHMENT = "source_enrichment" + PROPOSAL = "proposal" + CARVING = "carving" + CROSS_SERIES_RECONCILIATION = "cross_series_reconciliation" + BROKER_TRANSFER = "broker_transfer" + VALIDATION = "validation" + ATOMIC_PARTITION_COMMIT = "atomic_partition_commit" + + @classmethod + def from_value( + cls, value: str | "ReconstructionStage" + ) -> "ReconstructionStage": + """Normalize one stage name.""" + if isinstance(value, cls): + return value + normalized = str(value).strip().lower().replace("-", "_") + try: + return cls(normalized) + except ValueError as err: + raise ValueError( + f"unsupported reconstruction stage: {value!r}" + ) from err + + +RECONSTRUCTION_STAGE_ORDER = tuple(ReconstructionStage) + + +class ReconstructionStageStatus(str, Enum): + """Bounded outcome status for one stage attempt.""" + + COMPLETED = "completed" + REFUSED = "refused" + + @classmethod + def from_value( + cls, value: str | "ReconstructionStageStatus" + ) -> "ReconstructionStageStatus": + """Normalize one stage status.""" + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError("unsupported reconstruction stage status") from err + + +class ReconstructionCheckpointConflict(RuntimeError): + """A stale worker tried to replace a newer checkpoint chain.""" + + +class ReconstructionArtifactError(ValueError): + """A stage receipt or referenced artifact failed integrity checks.""" + + +class ReconstructionReportMismatch(ValueError): + """Workflow state and committed storage evidence do not reconcile.""" + + +@dataclass(frozen=True, slots=True) +class ReconstructionStageCommandV1: + """Bounded command metadata for one artifact-producing stage.""" + + stage: ReconstructionStage + handler_name: str + receipt_path: str + input_manifest_refs: tuple[ArtifactRef, ...] = () + configuration_refs: tuple[ArtifactRef, ...] = () + command_id: str = "" + schema_version: str = RECONSTRUCTION_STAGE_COMMAND_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_STAGE_COMMAND_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction stage command schema") + object.__setattr__( + self, "stage", ReconstructionStage.from_value(self.stage) + ) + object.__setattr__( + self, "handler_name", _required_text(self.handler_name) + ) + object.__setattr__( + self, + "receipt_path", + str(Path(_required_text(self.receipt_path)).expanduser().resolve()), + ) + for name in ("input_manifest_refs", "configuration_refs"): + refs = _unique_strong_refs(getattr(self, name), name) + if len(refs) > MAX_STAGE_ARTIFACT_REFS: + raise ValueError(f"{name} exceeds artifact-reference limit") + object.__setattr__(self, name, refs) + expected = _stable_id("reconstruction-command", self.identity_payload()) + if self.command_id and self.command_id != expected: + raise ValueError("command_id does not match deterministic identity") + object.__setattr__(self, "command_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return retry-independent command identity.""" + return { + "schema_version": self.schema_version, + "stage": self.stage.value, + "handler_name": self.handler_name, + "receipt_path": self.receipt_path, + "input_manifest_refs": [ + _artifact_identity(ref) for ref in self.input_manifest_refs + ], + "configuration_refs": [ + _artifact_identity(ref) for ref in self.configuration_refs + ], + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return workflow-safe command metadata.""" + return { + **self.identity_payload(), + "command_id": self.command_id, + "input_manifest_refs": [ + ref.to_dict() for ref in self.input_manifest_refs + ], + "configuration_refs": [ + ref.to_dict() for ref in self.configuration_refs + ], + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionStageCommandV1": + """Restore and verify one stage command.""" + return cls( + stage=ReconstructionStage.from_value(str(data.get("stage", ""))), + handler_name=str(data.get("handler_name", "")), + receipt_path=str(data.get("receipt_path", "")), + input_manifest_refs=_artifact_refs(data.get("input_manifest_refs")), + configuration_refs=_artifact_refs(data.get("configuration_refs")), + command_id=str(data.get("command_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionStageOutcomeV1: + """Durable stage receipt containing references and bounded counters only.""" + + run_id: str + window_id: str + synchronization_unit_id: str + stage: ReconstructionStage + command_id: str + input_fingerprint: str + status: ReconstructionStageStatus + output_refs: tuple[ArtifactRef, ...] = () + observed_event_count: int = 0 + candidate_event_count: int = 0 + accepted_event_count: int = 0 + scratch_bytes: int = 0 + output_bytes: int = 0 + refusal_reasons: tuple[str, ...] = () + message: str = "" + reused: bool = False + outcome_id: str = "" + schema_version: str = RECONSTRUCTION_STAGE_OUTCOME_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_STAGE_OUTCOME_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction stage outcome schema") + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "command_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, "stage", ReconstructionStage.from_value(self.stage) + ) + object.__setattr__( + self, "status", ReconstructionStageStatus.from_value(self.status) + ) + object.__setattr__( + self, + "input_fingerprint", + _required_sha256(self.input_fingerprint, "input_fingerprint"), + ) + refs = _unique_strong_refs(self.output_refs, "output_refs") + if len(refs) > MAX_STAGE_ARTIFACT_REFS: + raise ValueError("output_refs exceeds artifact-reference limit") + object.__setattr__(self, "output_refs", refs) + for name in ( + "observed_event_count", + "candidate_event_count", + "accepted_event_count", + "scratch_bytes", + "output_bytes", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + reasons = tuple( + sorted({_bounded_text(item) for item in self.refusal_reasons}) + ) + object.__setattr__(self, "refusal_reasons", reasons) + object.__setattr__(self, "message", _bounded_text(self.message)) + if not isinstance(self.reused, bool): + raise ValueError("reused must be a boolean") + if self.status is ReconstructionStageStatus.REFUSED and not reasons: + raise ValueError("refused stage outcome requires refusal_reasons") + if self.status is ReconstructionStageStatus.COMPLETED and reasons: + raise ValueError("completed stage outcome rejects refusal_reasons") + expected = _stable_id("reconstruction-outcome", self.identity_payload()) + if self.outcome_id and self.outcome_id != expected: + raise ValueError("outcome_id does not match deterministic identity") + object.__setattr__(self, "outcome_id", expected) + _reject_inline_data(self.to_dict(), path="stage_outcome") + _ensure_payload_size( + self.to_dict(), MAX_STAGE_RECEIPT_BYTES, "stage outcome" + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return retry-independent receipt identity.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "stage": self.stage.value, + "command_id": self.command_id, + "input_fingerprint": self.input_fingerprint, + "status": self.status.value, + "output_refs": [ + _artifact_identity(ref) for ref in self.output_refs + ], + "observed_event_count": self.observed_event_count, + "candidate_event_count": self.candidate_event_count, + "accepted_event_count": self.accepted_event_count, + "scratch_bytes": self.scratch_bytes, + "output_bytes": self.output_bytes, + "refusal_reasons": list(self.refusal_reasons), + "message": self.message, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return a bounded Temporal-safe receipt.""" + return { + **self.identity_payload(), + "outcome_id": self.outcome_id, + "output_refs": [ref.to_dict() for ref in self.output_refs], + "reused": self.reused, + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionStageOutcomeV1": + """Restore and verify one stage receipt.""" + return cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + stage=ReconstructionStage.from_value(str(data.get("stage", ""))), + command_id=str(data.get("command_id", "")), + input_fingerprint=str(data.get("input_fingerprint", "")), + status=ReconstructionStageStatus.from_value( + str(data.get("status", "")) + ), + output_refs=_artifact_refs(data.get("output_refs")), + observed_event_count=cast(int, data.get("observed_event_count", 0)), + candidate_event_count=cast( + int, data.get("candidate_event_count", 0) + ), + accepted_event_count=cast(int, data.get("accepted_event_count", 0)), + scratch_bytes=cast(int, data.get("scratch_bytes", 0)), + output_bytes=cast(int, data.get("output_bytes", 0)), + refusal_reasons=_string_tuple(data.get("refusal_reasons")), + message=str(data.get("message", "")), + reused=cast(bool, data.get("reused", False)), + outcome_id=str(data.get("outcome_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionStageOutcomeV1": + """Restore one stage receipt from JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionWindowTaskV1: + """One synchronized window plus its bounded stage command plan.""" + + window: ReconstructionWindowV1 + resource_estimate: ReconstructionResourceEstimateV1 + commands: tuple[ReconstructionStageCommandV1, ...] + scratch_directory: str + task_id: str = "" + schema_version: str = RECONSTRUCTION_WINDOW_TASK_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_WINDOW_TASK_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction window task schema") + if not isinstance(self.window, ReconstructionWindowV1): + raise ValueError("window task requires ReconstructionWindowV1") + if not isinstance( + self.resource_estimate, ReconstructionResourceEstimateV1 + ): + raise ValueError("window task requires a resource estimate") + commands = tuple(self.commands) + if tuple(item.stage for item in commands) != RECONSTRUCTION_STAGE_ORDER: + raise ValueError( + "window task must contain the complete ordered stage plan" + ) + if len({item.command_id for item in commands}) != len(commands): + raise ValueError("window task command IDs must be unique") + object.__setattr__(self, "commands", commands) + scratch = ( + Path(_required_text(self.scratch_directory)).expanduser().resolve() + ) + for command in commands: + receipt = Path(command.receipt_path).expanduser().resolve() + if not receipt.is_relative_to(scratch): + raise ValueError( + "stage receipt path must remain inside window scratch" + ) + object.__setattr__(self, "scratch_directory", str(scratch)) + expected = _stable_id( + "reconstruction-window-task", self.identity_payload() + ) + if self.task_id and self.task_id != expected: + raise ValueError("task_id does not match deterministic identity") + object.__setattr__(self, "task_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic task identity.""" + return { + "schema_version": self.schema_version, + "window": self.window.to_dict(), + "resource_estimate": self.resource_estimate.to_dict(), + "commands": [item.to_dict() for item in self.commands], + "scratch_directory": self.scratch_directory, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return workflow-safe task metadata.""" + return {**self.identity_payload(), "task_id": self.task_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionWindowTaskV1": + """Restore and verify one window task.""" + return cls( + window=ReconstructionWindowV1.from_dict( + _mapping(data.get("window")) + ), + resource_estimate=ReconstructionResourceEstimateV1.from_dict( + _mapping(data.get("resource_estimate")) + ), + commands=tuple( + ReconstructionStageCommandV1.from_dict(_mapping(item)) + for item in _sequence(data.get("commands")) + ), + scratch_directory=str(data.get("scratch_directory", "")), + task_id=str(data.get("task_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionWorkflowRequestV1: + """A bounded period-scale plan suitable for Temporal workflow history.""" + + request_id: str + run: ReconstructionRunV1 + tasks: tuple[ReconstructionWindowTaskV1, ...] + manifest_store_root: str + report_root: str + task_queues: dict[str, str] = field(default_factory=dict) + max_parallel_windows: int = DEFAULT_MAX_PARALLEL_RECONSTRUCTION_WINDOWS + max_inflight_memory_bytes: int | None = None + request_fingerprint: str = "" + schema_version: str = RECONSTRUCTION_WORKFLOW_REQUEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_WORKFLOW_REQUEST_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction workflow request schema" + ) + object.__setattr__(self, "request_id", _required_text(self.request_id)) + if not isinstance(self.run, ReconstructionRunV1): + raise ValueError("workflow request requires ReconstructionRunV1") + tasks = tuple( + sorted(self.tasks, key=lambda item: item.window.core_start_ns) + ) + if not tasks: + raise ValueError( + "workflow request requires at least one window task" + ) + if len(tasks) > MAX_RECONSTRUCTION_WINDOWS: + raise ValueError("workflow request exceeds window limit") + if len({item.window.window_id for item in tasks}) != len(tasks): + raise ValueError("workflow request contains duplicate windows") + for task in tasks: + if task.window.run_id != self.run.run_id: + raise ValueError("window task run_id differs from request run") + if tuple(sorted(task.window.symbols)) != tuple( + sorted(self.run.symbols) + ): + raise ValueError( + "window task must contain the complete synchronized run symbol set" + ) + for previous, current in zip(tasks, tasks[1:]): + if current.window.core_start_ns < previous.window.core_end_ns: + raise ValueError( + "reconstruction window core intervals must not overlap" + ) + object.__setattr__(self, "tasks", tasks) + manifest_store_root = str( + Path(_required_text(self.manifest_store_root)) + .expanduser() + .resolve() + ) + report_root = str( + Path(_required_text(self.report_root)).expanduser().resolve() + ) + object.__setattr__(self, "manifest_store_root", manifest_store_root) + object.__setattr__(self, "report_root", report_root) + _validate_scratch_boundaries( + tasks, + durable_roots=(manifest_store_root, report_root), + ) + queues = { + _required_text(key): _required_text(value) + for key, value in sorted(self.task_queues.items()) + } + object.__setattr__(self, "task_queues", queues) + parallel = _positive_int( + self.max_parallel_windows, "max_parallel_windows" + ) + if parallel > self.run.storage_policy.max_inflight_batches: + raise ValueError( + "max_parallel_windows exceeds storage-policy inflight limit" + ) + object.__setattr__(self, "max_parallel_windows", parallel) + memory_limit = self.max_inflight_memory_bytes + if memory_limit is None: + memory_limit = self.run.storage_policy.max_memory_bytes + object.__setattr__( + self, + "max_inflight_memory_bytes", + _positive_int(memory_limit, "max_inflight_memory_bytes"), + ) + expected = _stable_id("reconstruction-request", self.identity_payload()) + if self.request_fingerprint and self.request_fingerprint != expected: + raise ValueError( + "request_fingerprint does not match deterministic identity" + ) + object.__setattr__(self, "request_fingerprint", expected) + payload = self.to_dict() + _reject_inline_data(payload) + _ensure_payload_size( + payload, + MAX_RECONSTRUCTION_WORKFLOW_BYTES, + "reconstruction workflow request", + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return stable semantic and execution policy inputs.""" + return { + "schema_version": self.schema_version, + "request_id": self.request_id, + "run": self.run.to_dict(), + "tasks": [item.to_dict() for item in self.tasks], + "manifest_store_root": self.manifest_store_root, + "report_root": self.report_root, + "task_queues": dict(self.task_queues), + "max_parallel_windows": self.max_parallel_windows, + "max_inflight_memory_bytes": self.max_inflight_memory_bytes, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return the bounded workflow payload.""" + return { + **self.identity_payload(), + "request_fingerprint": self.request_fingerprint, + "history_policy": "bounded_metadata_and_artifact_refs_only", + } + + def for_task( + self, task: ReconstructionWindowTaskV1 + ) -> "ReconstructionWorkflowRequestV1": + """Return a bounded child-workflow request for exactly one window.""" + if task.task_id not in {item.task_id for item in self.tasks}: + raise ValueError("child task is not part of the parent request") + return replace( + self, + tasks=(task,), + max_parallel_windows=1, + request_fingerprint="", + ) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionWorkflowRequestV1": + """Restore and verify a workflow request.""" + return cls( + request_id=str(data.get("request_id", "")), + run=ReconstructionRunV1.from_dict(_mapping(data.get("run"))), + tasks=tuple( + ReconstructionWindowTaskV1.from_dict(_mapping(item)) + for item in _sequence(data.get("tasks")) + ), + manifest_store_root=str(data.get("manifest_store_root", "")), + report_root=str(data.get("report_root", "")), + task_queues={ + str(key): str(value) + for key, value in _mapping(data.get("task_queues")).items() + }, + max_parallel_windows=cast(int, data.get("max_parallel_windows", 0)), + max_inflight_memory_bytes=cast( + int | None, data.get("max_inflight_memory_bytes") + ), + request_fingerprint=str(data.get("request_fingerprint", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionWindowStateV1: + """Checkpoint plus completed stage receipts for one synchronized window.""" + + request_id: str + task: ReconstructionWindowTaskV1 + checkpoint: ReconstructionCheckpointV1 + outcomes: tuple[ReconstructionStageOutcomeV1, ...] = () + state_id: str = "" + schema_version: str = RECONSTRUCTION_WINDOW_STATE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_WINDOW_STATE_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction window state schema") + object.__setattr__(self, "request_id", _required_text(self.request_id)) + if not isinstance(self.task, ReconstructionWindowTaskV1): + raise ValueError("window state requires ReconstructionWindowTaskV1") + if not isinstance(self.checkpoint, ReconstructionCheckpointV1): + raise ValueError("window state requires ReconstructionCheckpointV1") + window = self.task.window + if ( + self.checkpoint.run_id != window.run_id + or self.checkpoint.window_id != window.window_id + or self.checkpoint.synchronization_unit_id + != window.synchronization_unit_id + ): + raise ValueError("checkpoint scope differs from window task") + outcomes = tuple(self.outcomes) + expected_stages = RECONSTRUCTION_STAGE_ORDER[: len(outcomes)] + if tuple(item.stage for item in outcomes) != expected_stages: + raise ValueError( + "window outcomes must be a strict stage-order prefix" + ) + for index, (outcome, command) in enumerate( + zip(outcomes, self.task.commands) + ): + _validate_stage_outcome( + outcome, + window, + command, + outcomes[:index], + ) + object.__setattr__(self, "outcomes", outcomes) + self._validate_phase() + expected = _stable_id( + "reconstruction-window-state", self.identity_payload() + ) + if self.state_id and self.state_id != expected: + raise ValueError("state_id does not match deterministic identity") + object.__setattr__(self, "state_id", expected) + _reject_inline_data(self.to_dict(), path="window_state") + _ensure_payload_size( + self.to_dict(), + MAX_RECONSTRUCTION_WINDOW_STATE_BYTES, + "reconstruction window state", + ) + + @classmethod + def planned( + cls, request_id: str, task: ReconstructionWindowTaskV1 + ) -> "ReconstructionWindowStateV1": + """Return the initial durable state.""" + return cls( + request_id=request_id, + task=task, + checkpoint=ReconstructionCheckpointV1.planned(task.window), + ) + + @property + def terminal(self) -> bool: + """Return whether the window will not run more stages this attempt.""" + return self.checkpoint.phase in { + ReconstructionCommitPhase.COMMITTED, + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + } + + @property + def committed_manifest_ref(self) -> ArtifactRef | None: + """Return the discoverable committed manifest, if any.""" + return self.checkpoint.committed_manifest_ref + + def outcome_for( + self, stage: ReconstructionStage + ) -> ReconstructionStageOutcomeV1 | None: + """Return the stored receipt for a stage.""" + return next( + (item for item in self.outcomes if item.stage is stage), None + ) + + def running(self) -> "ReconstructionWindowStateV1": + """Start or resume work from the latest durable checkpoint.""" + if self.checkpoint.phase is ReconstructionCommitPhase.COMMITTED: + return self + if self.checkpoint.phase is ReconstructionCommitPhase.RUNNING: + return self + outcomes = self.outcomes + if self.checkpoint.phase is ReconstructionCommitPhase.CANCELLED: + # Cancellation cleanup removes the entire window scratch tree, + # including every uncommitted receipt and stage artifact. A + # resumed attempt must therefore rebuild the whole disposable + # prefix instead of retaining references to deleted files. + outcomes = () + elif ( + self.checkpoint.phase is ReconstructionCommitPhase.FAILED + and ReconstructionStage.VALIDATION + in tuple(item.stage for item in outcomes) + ): + validation_index = RECONSTRUCTION_STAGE_ORDER.index( + ReconstructionStage.VALIDATION + ) + outcomes = outcomes[:validation_index] + checkpoint = self.checkpoint.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=self.checkpoint.checkpoint_id, + ) + return replace( + self, + checkpoint=checkpoint, + outcomes=outcomes, + state_id="", + ) + + def complete( + self, outcome: ReconstructionStageOutcomeV1 + ) -> "ReconstructionWindowStateV1": + """Append one successful stage and advance its checkpoint phase.""" + if outcome.status is not ReconstructionStageStatus.COMPLETED: + raise ValueError("only completed outcomes can advance a window") + expected_stage = RECONSTRUCTION_STAGE_ORDER[len(self.outcomes)] + if outcome.stage is not expected_stage: + raise ValueError("stage completion is out of order") + command = self.task.commands[len(self.outcomes)] + _validate_stage_outcome( + outcome, self.task.window, command, self.outcomes + ) + checkpoint = self.checkpoint + if outcome.stage is ReconstructionStage.VALIDATION: + staged = _phase_artifact(outcome, "staged") + checkpoint = checkpoint.transition( + ReconstructionCommitPhase.STAGED, + expected_checkpoint_id=checkpoint.checkpoint_id, + staged_manifest_ref=staged, + ) + elif outcome.stage is ReconstructionStage.ATOMIC_PARTITION_COMMIT: + committed = _phase_artifact(outcome, "committed") + if ( + Path(committed.path) + .expanduser() + .resolve() + .is_relative_to( + Path(self.task.scratch_directory).expanduser().resolve() + ) + ): + raise ValueError( + "committed manifest must remain outside disposable window scratch" + ) + checkpoint = checkpoint.transition( + ReconstructionCommitPhase.COMMITTED, + expected_checkpoint_id=checkpoint.checkpoint_id, + committed_manifest_ref=committed, + output_watermark_ns=self.task.window.core_end_ns, + ) + else: + checkpoint = checkpoint.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=checkpoint.checkpoint_id, + ) + return replace( + self, + checkpoint=checkpoint, + outcomes=(*self.outcomes, replace(outcome, reused=False)), + state_id="", + ) + + def validated(self) -> "ReconstructionWindowStateV1": + """Persist the second phase of the validation/commit protocol.""" + if self.checkpoint.phase is ReconstructionCommitPhase.VALIDATED: + return self + if self.checkpoint.phase is not ReconstructionCommitPhase.STAGED: + raise ValueError("only a staged window can become validated") + checkpoint = self.checkpoint.transition( + ReconstructionCommitPhase.VALIDATED, + expected_checkpoint_id=self.checkpoint.checkpoint_id, + ) + return replace(self, checkpoint=checkpoint, state_id="") + + def interrupted( + self, + phase: ReconstructionCommitPhase, + reason: str, + ) -> "ReconstructionWindowStateV1": + """Record a resumable cancellation or explicit refusal.""" + if phase not in { + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + }: + raise ValueError("interrupted state must be cancelled or failed") + if self.checkpoint.phase is ReconstructionCommitPhase.COMMITTED: + return self + checkpoint = self.checkpoint.transition( + phase, + expected_checkpoint_id=self.checkpoint.checkpoint_id, + interruption_reason=_bounded_text(reason), + ) + return replace(self, checkpoint=checkpoint, state_id="") + + def _validate_phase(self) -> None: + stages = tuple(item.stage for item in self.outcomes) + phase = self.checkpoint.phase + committed = ReconstructionStage.ATOMIC_PARTITION_COMMIT in stages + validation = ReconstructionStage.VALIDATION in stages + if committed != (phase is ReconstructionCommitPhase.COMMITTED): + raise ValueError("committed outcome/checkpoint phase differs") + if ( + phase + in { + ReconstructionCommitPhase.STAGED, + ReconstructionCommitPhase.VALIDATED, + } + and not validation + ): + raise ValueError( + "staged/validated checkpoint lacks validation outcome" + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return durable state identity fields.""" + return { + "schema_version": self.schema_version, + "request_id": self.request_id, + "task": self.task.to_dict(), + "checkpoint": self.checkpoint.to_dict(), + "outcomes": [item.to_dict() for item in self.outcomes], + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded recovery metadata.""" + return {**self.identity_payload(), "state_id": self.state_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionWindowStateV1": + """Restore and verify durable window state.""" + return cls( + request_id=str(data.get("request_id", "")), + task=ReconstructionWindowTaskV1.from_dict( + _mapping(data.get("task")) + ), + checkpoint=ReconstructionCheckpointV1.from_dict( + _mapping(data.get("checkpoint")) + ), + outcomes=tuple( + ReconstructionStageOutcomeV1.from_dict(_mapping(item)) + for item in _sequence(data.get("outcomes")) + ), + state_id=str(data.get("state_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionRunReportV1: + """Compact reconciliation of workflow state and committed manifests.""" + + request_id: str + run_id: str + status: str + window_count: int + committed_window_count: int + cancelled_window_count: int + failed_window_count: int + observed_event_count: int + synthetic_event_count: int + committed_manifest_refs: tuple[ArtifactRef, ...] + window_states: tuple[dict[str, JSONValue], ...] + report_id: str = "" + schema_version: str = RECONSTRUCTION_REPORT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_REPORT_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction report schema") + object.__setattr__(self, "request_id", _required_text(self.request_id)) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + if self.status not in {"committed", "partial", "cancelled", "failed"}: + raise ValueError("unsupported reconstruction report status") + for name in ( + "window_count", + "committed_window_count", + "cancelled_window_count", + "failed_window_count", + "observed_event_count", + "synthetic_event_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + refs = _unique_strong_refs( + self.committed_manifest_refs, "committed_manifest_refs" + ) + object.__setattr__(self, "committed_manifest_refs", refs) + summaries = tuple(dict(item) for item in self.window_states) + object.__setattr__(self, "window_states", summaries) + if len(summaries) != self.window_count: + raise ValueError("window_states count differs from window_count") + if len(refs) != self.committed_window_count: + raise ValueError( + "committed manifest count differs from committed windows" + ) + terminal_count = ( + self.committed_window_count + + self.cancelled_window_count + + self.failed_window_count + ) + if terminal_count > self.window_count: + raise ValueError("terminal window counts exceed window_count") + expected = _stable_id("reconstruction-report", self.identity_payload()) + if self.report_id and self.report_id != expected: + raise ValueError("report_id does not match deterministic identity") + object.__setattr__(self, "report_id", expected) + _reject_inline_data(self.to_dict(), path="reconstruction_report") + _ensure_payload_size( + self.to_dict(), + MAX_RECONSTRUCTION_REPORT_BYTES, + "reconstruction report", + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic report evidence.""" + return { + "schema_version": self.schema_version, + "request_id": self.request_id, + "run_id": self.run_id, + "status": self.status, + "window_count": self.window_count, + "committed_window_count": self.committed_window_count, + "cancelled_window_count": self.cancelled_window_count, + "failed_window_count": self.failed_window_count, + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "committed_manifest_refs": [ + _artifact_identity(ref) for ref in self.committed_manifest_refs + ], + "window_states": list(self.window_states), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return report JSON with strong artifact references.""" + return { + **self.identity_payload(), + "report_id": self.report_id, + "committed_manifest_refs": [ + ref.to_dict() for ref in self.committed_manifest_refs + ], + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionRunReportV1": + """Restore and verify a reconstruction report.""" + return cls( + request_id=str(data.get("request_id", "")), + run_id=str(data.get("run_id", "")), + status=str(data.get("status", "")), + window_count=cast(int, data.get("window_count", 0)), + committed_window_count=cast( + int, data.get("committed_window_count", 0) + ), + cancelled_window_count=cast( + int, data.get("cancelled_window_count", 0) + ), + failed_window_count=cast(int, data.get("failed_window_count", 0)), + observed_event_count=cast(int, data.get("observed_event_count", 0)), + synthetic_event_count=cast( + int, data.get("synthetic_event_count", 0) + ), + committed_manifest_refs=_artifact_refs( + data.get("committed_manifest_refs") + ), + window_states=tuple( + dict(_mapping(item)) + for item in _sequence(data.get("window_states")) + ), + report_id=str(data.get("report_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +class ReconstructionCheckpointStore: + """Optimistic window-state repository backed by the manifest SQLite DB.""" + + def __init__(self, root: str | Path): + self.store = ManifestStatusStore(root) + + @staticmethod + def job_id_for(window: ReconstructionWindowV1) -> str: + """Return a compact stable manifest-store key.""" + digest = hashlib.sha256( + f"{window.run_id}|{window.window_id}".encode("utf-8") + ).hexdigest()[:24] + return f"reconstruction-window-{digest}" + + def initialize( + self, + request_id: str, + task: ReconstructionWindowTaskV1, + ) -> ReconstructionWindowStateV1: + """Create the initial state or return the already durable state.""" + state = ReconstructionWindowStateV1.planned(request_id, task) + if self.store.compare_and_swap_job_snapshot( + self._snapshot(state), expected_snapshot_id=None + ): + return state + existing = self.load(task.window) + if existing is None: + raise ReconstructionCheckpointConflict( + "checkpoint initialization lost without a durable winner" + ) + self._validate_task(existing, request_id, task) + return existing + + def load( + self, window: ReconstructionWindowV1 + ) -> ReconstructionWindowStateV1 | None: + """Load and verify the latest durable state after any process restart.""" + payload = self.store.get_job_snapshot(self.job_id_for(window)) + if payload is None: + return None + state = ReconstructionWindowStateV1.from_dict( + _mapping(payload.get("reconstruction_window_state")) + ) + if state.state_id != str(payload.get("snapshot_id", "")): + raise ReconstructionArtifactError( + "stored reconstruction snapshot identity differs from state" + ) + return state + + def save( + self, + state: ReconstructionWindowStateV1, + *, + expected_state_id: str, + ) -> ReconstructionWindowStateV1: + """Advance the chain, resolving identical duplicate completion.""" + if self.store.compare_and_swap_job_snapshot( + self._snapshot(state), expected_snapshot_id=expected_state_id + ): + return state + current = self.load(state.task.window) + if current is None: + raise ReconstructionCheckpointConflict( + "checkpoint disappeared during save" + ) + if _state_contains(current, state): + return current + raise ReconstructionCheckpointConflict( + "stale reconstruction checkpoint transition rejected" + ) + + @staticmethod + def _validate_task( + state: ReconstructionWindowStateV1, + request_id: str, + task: ReconstructionWindowTaskV1, + ) -> None: + if state.request_id != request_id or state.task.task_id != task.task_id: + raise ReconstructionCheckpointConflict( + "durable checkpoint belongs to a different request or task" + ) + + def _snapshot( + self, state: ReconstructionWindowStateV1 + ) -> dict[str, JSONValue]: + phase = state.checkpoint.phase + status = { + ReconstructionCommitPhase.COMMITTED: "COMPLETED", + ReconstructionCommitPhase.CANCELLED: "CANCELLED", + ReconstructionCommitPhase.FAILED: "FAILED", + }.get( + phase, + ( + "PLANNED" + if phase is ReconstructionCommitPhase.PLANNED + else "RUNNING" + ), + ) + job_id = self.job_id_for(state.task.window) + artifacts: list[JSONValue] = [ + ref.to_dict() + for outcome in state.outcomes + for ref in outcome.output_refs + ] + return { + "schema_version": 1, + "job_id": job_id, + "request_id": state.request_id, + "workflow_id": job_id, + "run_id": state.task.window.run_id, + "lifecycle": phase.value, + "status": status, + "snapshot_id": state.state_id, + "reconstruction_window_state": state.to_dict(), + "artifacts": artifacts, + } + + +@dataclass(frozen=True, slots=True) +class ReconstructionStageInvocationV1: + """Bounded activity input passed to a data-plane stage handler.""" + + run: ReconstructionRunV1 + task: ReconstructionWindowTaskV1 + command: ReconstructionStageCommandV1 + prior_outcomes: tuple[ReconstructionStageOutcomeV1, ...] + heartbeat_callback: Callable[[ReconstructionHeartbeatV1], Any] | None = ( + field( + default=None, + compare=False, + repr=False, + ) + ) + cancellation_check: Callable[[], bool] | None = field( + default=None, + compare=False, + repr=False, + ) + + @property + def input_fingerprint(self) -> str: + """Hash declared inputs and prior outputs for retry validation.""" + payload: dict[str, JSONValue] = { + "run_id": self.run.run_id, + "window_id": self.task.window.window_id, + "command_id": self.command.command_id, + "input_refs": [ + _artifact_identity(ref) + for ref in ( + *self.command.input_manifest_refs, + *self.command.configuration_refs, + *tuple( + ref + for outcome in self.prior_outcomes + for ref in outcome.output_refs + ), + ) + ], + "prior_outcome_ids": [ + item.outcome_id for item in self.prior_outcomes + ], + } + return hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + + @property + def cancellation_requested(self) -> bool: + """Return whether the activity should stop producing new work.""" + return bool(self.cancellation_check and self.cancellation_check()) + + def heartbeat( + self, + *, + sequence: int, + completed_units: int, + total_units: int, + observed_event_count: int = 0, + candidate_event_count: int = 0, + accepted_event_count: int = 0, + scratch_bytes: int = 0, + output_bytes: int = 0, + message: str = "", + ) -> None: + """Emit bounded intra-stage progress from a long-running handler.""" + if self.heartbeat_callback is None: + return + heartbeat = ReconstructionHeartbeatV1( + run_id=self.task.window.run_id, + window_id=self.task.window.window_id, + synchronization_unit_id=self.task.window.synchronization_unit_id, + phase=ReconstructionCommitPhase.RUNNING, + sequence=sequence, + completed_units=completed_units, + total_units=total_units, + observed_event_count=observed_event_count, + candidate_event_count=candidate_event_count, + accepted_event_count=accepted_event_count, + scratch_bytes=scratch_bytes, + output_bytes=output_bytes, + cancellation_requested=self.cancellation_requested, + message=message or self.command.stage.value, + ) + result = self.heartbeat_callback(heartbeat) + if inspect.isawaitable(result): + raise TypeError( + "stage-handler heartbeat callback must be synchronous" + ) + + def completed( + self, + *, + output_refs: Sequence[ArtifactRef], + observed_event_count: int = 0, + candidate_event_count: int = 0, + accepted_event_count: int = 0, + scratch_bytes: int = 0, + output_bytes: int = 0, + message: str = "", + ) -> ReconstructionStageOutcomeV1: + """Build a correctly scoped successful stage outcome.""" + window = self.task.window + return ReconstructionStageOutcomeV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + stage=self.command.stage, + command_id=self.command.command_id, + input_fingerprint=self.input_fingerprint, + status=ReconstructionStageStatus.COMPLETED, + output_refs=tuple(output_refs), + observed_event_count=observed_event_count, + candidate_event_count=candidate_event_count, + accepted_event_count=accepted_event_count, + scratch_bytes=scratch_bytes, + output_bytes=output_bytes, + message=message, + ) + + def refused( + self, + *reasons: str, + message: str = "", + ) -> ReconstructionStageOutcomeV1: + """Build a fail-closed scientific/resource refusal receipt.""" + window = self.task.window + return ReconstructionStageOutcomeV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + stage=self.command.stage, + command_id=self.command.command_id, + input_fingerprint=self.input_fingerprint, + status=ReconstructionStageStatus.REFUSED, + refusal_reasons=tuple(reasons), + message=message, + ) + + +ReconstructionStageHandler = Callable[ + [ReconstructionStageInvocationV1], + ReconstructionStageOutcomeV1 | Awaitable[ReconstructionStageOutcomeV1], +] +_STAGE_HANDLERS: dict[str, ReconstructionStageHandler] = {} + + +def register_reconstruction_stage_handler( + name: str, + handler: ReconstructionStageHandler, + *, + replace_existing: bool = False, +) -> None: + """Register an activity-side adapter outside workflow state.""" + normalized = _required_text(name) + if not callable(handler): + raise TypeError("reconstruction stage handler must be callable") + if normalized in _STAGE_HANDLERS and not replace_existing: + raise ValueError( + f"reconstruction stage handler already registered: {normalized}" + ) + _STAGE_HANDLERS[normalized] = handler + + +def unregister_reconstruction_stage_handler(name: str) -> None: + """Remove a process-local stage adapter, primarily for test isolation.""" + _STAGE_HANDLERS.pop(str(name).strip(), None) + + +def registered_reconstruction_stage_handlers() -> ( + Mapping[str, ReconstructionStageHandler] +): + """Return an isolated snapshot of installed activity-side adapters.""" + return dict(_STAGE_HANDLERS) + + +async def execute_reconstruction_stage( + invocation: ReconstructionStageInvocationV1, + *, + verify_outputs: bool = True, +) -> ReconstructionStageOutcomeV1: + """Execute or reuse one idempotent artifact-producing stage receipt.""" + for ref in ( + *invocation.command.input_manifest_refs, + *invocation.command.configuration_refs, + *tuple( + ref + for outcome in invocation.prior_outcomes + for ref in outcome.output_refs + ), + ): + verify_artifact_ref(ref) + receipt_path = Path(invocation.command.receipt_path).expanduser() + if receipt_path.exists(): + outcome = _read_stage_receipt(receipt_path) + _validate_stage_outcome( + outcome, + invocation.task.window, + invocation.command, + invocation.prior_outcomes, + ) + if verify_outputs: + for ref in outcome.output_refs: + verify_artifact_ref(ref) + return replace(outcome, reused=True) + + handler = _STAGE_HANDLERS.get(invocation.command.handler_name) + if handler is None: + raise ReconstructionArtifactError( + "no reconstruction stage handler is registered for " + f"{invocation.command.handler_name!r}, and no durable receipt exists" + ) + result = handler(invocation) + if inspect.isawaitable(result): + result = await result + outcome = result + _validate_stage_outcome( + outcome, + invocation.task.window, + invocation.command, + invocation.prior_outcomes, + ) + if verify_outputs: + for ref in outcome.output_refs: + verify_artifact_ref(ref) + _write_stage_receipt(receipt_path, outcome) + return outcome + + +class ReconstructionStageExecutor(Protocol): + """Activity boundary used by local tests and Temporal workflows.""" + + async def execute( + self, invocation: ReconstructionStageInvocationV1 + ) -> ReconstructionStageOutcomeV1: + """Run one stage and return bounded receipt metadata.""" + + +class RegisteredReconstructionStageExecutor: + """Execute stages through the process-local activity handler registry.""" + + async def execute( + self, invocation: ReconstructionStageInvocationV1 + ) -> ReconstructionStageOutcomeV1: + """Execute or reuse the stage receipt.""" + return await execute_reconstruction_stage(invocation) + + +HeartbeatCallback = Callable[ + [ReconstructionHeartbeatV1], None | Awaitable[None] +] +CancellationCheck = Callable[[], bool] + + +async def run_reconstruction_window( + request: ReconstructionWorkflowRequestV1, + task: ReconstructionWindowTaskV1, + *, + checkpoint_store: ReconstructionCheckpointStore, + stage_executor: ReconstructionStageExecutor, + heartbeat: HeartbeatCallback | None = None, + cancellation_requested: CancellationCheck | None = None, +) -> ReconstructionWindowStateV1: + """Run or resume one synchronized window from its last valid checkpoint.""" + state = checkpoint_store.initialize(request.request_id, task) + if state.checkpoint.phase is ReconstructionCommitPhase.COMMITTED: + return state + try: + request.run.storage_policy.preflight(task.resource_estimate) + except ReconstructionResourceLimitError as err: + if state.checkpoint.phase is not ReconstructionCommitPhase.FAILED: + failed = state.interrupted( + ReconstructionCommitPhase.FAILED, + "; ".join(err.violations), + ) + state = checkpoint_store.save( + failed, expected_state_id=state.state_id + ) + return state + + lane_limit = cast(int, request.max_inflight_memory_bytes) + if task.resource_estimate.estimated_memory_bytes > lane_limit: + if state.checkpoint.phase is not ReconstructionCommitPhase.FAILED: + failed = state.interrupted( + ReconstructionCommitPhase.FAILED, + "estimated_memory_bytes " + f"{task.resource_estimate.estimated_memory_bytes} exceeds " + f"lane limit {lane_limit}", + ) + state = checkpoint_store.save( + failed, expected_state_id=state.state_id + ) + return state + + if state.checkpoint.phase in { + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + }: + state = checkpoint_store.save( + state.running(), expected_state_id=state.state_id + ) + elif state.checkpoint.phase is ReconstructionCommitPhase.PLANNED: + state = checkpoint_store.save( + state.running(), expected_state_id=state.state_id + ) + + if state.checkpoint.phase is ReconstructionCommitPhase.STAGED: + state = checkpoint_store.save( + state.validated(), expected_state_id=state.state_id + ) + + while len(state.outcomes) < len(task.commands): + if cancellation_requested is not None and cancellation_requested(): + cancelled = state.interrupted( + ReconstructionCommitPhase.CANCELLED, + "cancellation requested before next stage", + ) + state = checkpoint_store.save( + cancelled, expected_state_id=state.state_id + ) + cleanup_reconstruction_window_scratch(task.scratch_directory) + return state + command = task.commands[len(state.outcomes)] + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=command, + prior_outcomes=state.outcomes, + heartbeat_callback=heartbeat, + cancellation_check=cancellation_requested, + ) + await _emit_heartbeat( + heartbeat, _heartbeat_for(state, command.stage, False) + ) + outcome = await stage_executor.execute(invocation) + resource_violations = _outcome_resource_violations( + request.run.storage_policy, + task.resource_estimate, + outcome, + ) + if resource_violations: + refused = state.interrupted( + ReconstructionCommitPhase.FAILED, + "; ".join(resource_violations), + ) + return checkpoint_store.save( + refused, expected_state_id=state.state_id + ) + if outcome.status is ReconstructionStageStatus.REFUSED: + refused = state.interrupted( + ReconstructionCommitPhase.FAILED, + _bounded_reason_summary(outcome.refusal_reasons), + ) + state = checkpoint_store.save( + refused, expected_state_id=state.state_id + ) + return state + next_state = state.complete(outcome) + state = checkpoint_store.save( + next_state, expected_state_id=state.state_id + ) + await _emit_heartbeat( + heartbeat, _heartbeat_for(state, command.stage, True) + ) + if ( + command.stage is ReconstructionStage.VALIDATION + and state.checkpoint.phase is ReconstructionCommitPhase.STAGED + ): + validated = state.validated() + state = checkpoint_store.save( + validated, expected_state_id=state.state_id + ) + return state + + +def plan_reconstruction_waves( + tasks: Sequence[ReconstructionWindowTaskV1], + *, + max_parallel_windows: int, + max_inflight_memory_bytes: int, +) -> tuple[tuple[ReconstructionWindowTaskV1, ...], ...]: + """Plan deterministic bounded waves that enforce producer backpressure.""" + maximum = _positive_int(max_parallel_windows, "max_parallel_windows") + byte_limit = _positive_int( + max_inflight_memory_bytes, "max_inflight_memory_bytes" + ) + waves: list[tuple[ReconstructionWindowTaskV1, ...]] = [] + current: list[ReconstructionWindowTaskV1] = [] + current_bytes = 0 + for task in sorted(tasks, key=lambda item: item.window.core_start_ns): + weight = max(1, task.resource_estimate.estimated_memory_bytes) + if weight > byte_limit: + if current: + waves.append(tuple(current)) + current = [] + current_bytes = 0 + waves.append((task,)) + continue + if current and ( + len(current) >= maximum or current_bytes + weight > byte_limit + ): + waves.append(tuple(current)) + current = [] + current_bytes = 0 + current.append(task) + current_bytes += weight + if current: + waves.append(tuple(current)) + return tuple(waves) + + +async def run_reconstruction_request( + request: ReconstructionWorkflowRequestV1, + *, + stage_executor: ReconstructionStageExecutor | None = None, + heartbeat: HeartbeatCallback | None = None, + cancellation_requested: CancellationCheck | None = None, +) -> tuple[ReconstructionWindowStateV1, ...]: + """Execute a request locally with the same wave backpressure as Temporal.""" + executor = stage_executor or RegisteredReconstructionStageExecutor() + store = ReconstructionCheckpointStore(request.manifest_store_root) + states: list[ReconstructionWindowStateV1] = [] + waves = plan_reconstruction_waves( + request.tasks, + max_parallel_windows=request.max_parallel_windows, + max_inflight_memory_bytes=cast(int, request.max_inflight_memory_bytes), + ) + for wave in waves: + wave_states = await asyncio.gather( + *( + run_reconstruction_window( + request, + task, + checkpoint_store=store, + stage_executor=executor, + heartbeat=heartbeat, + cancellation_requested=cancellation_requested, + ) + for task in wave + ) + ) + states.extend(wave_states) + if cancellation_requested is not None and cancellation_requested(): + break + return tuple( + sorted(states, key=lambda item: item.task.window.core_start_ns) + ) + + +def reconcile_reconstruction_report( + request: ReconstructionWorkflowRequestV1, + states: Sequence[ReconstructionWindowStateV1], + *, + progress: Callable[[Mapping[str, JSONValue]], None] | None = None, +) -> ReconstructionRunReportV1: + """Reconcile durable workflow checkpoints with final storage manifests.""" + ordered = tuple( + sorted(states, key=lambda item: item.task.window.core_start_ns) + ) + if len(ordered) != len(request.tasks): + raise ReconstructionReportMismatch( + "report state count differs from requested window count" + ) + expected = {item.window.window_id for item in request.tasks} + if {item.task.window.window_id for item in ordered} != expected: + raise ReconstructionReportMismatch( + "report window set differs from request" + ) + refs: list[ArtifactRef] = [] + window_summaries: list[dict[str, JSONValue]] = [] + observed = 0 + synthetic = 0 + for index, state in enumerate(ordered, start=1): + window = state.task.window + if progress is not None: + progress( + { + "phase": "report_reconciliation", + "window_id": window.window_id, + "completed_windows": index - 1, + "total_windows": len(ordered), + } + ) + summary: dict[str, JSONValue] = { + "window_id": window.window_id, + "synchronization_unit_id": window.synchronization_unit_id, + "phase": state.checkpoint.phase.value, + "checkpoint_id": state.checkpoint.checkpoint_id, + "completed_stages": [item.stage.value for item in state.outcomes], + "resource_usage": _aggregate_outcome_telemetry(state.outcomes), + } + manifest_ref = state.committed_manifest_ref + if manifest_ref is not None: + verify_artifact_ref(manifest_ref) + manifest = verify_reconstruction_publication(manifest_ref.path) + if ( + manifest.run_id != request.run.run_id + or manifest.window_id != window.window_id + or manifest.synchronization_unit_id + != window.synchronization_unit_id + or tuple(sorted(manifest.symbols)) + != tuple(sorted(window.symbols)) + ): + message = ( + "committed manifest scope differs for window " + f"{window.window_id}" + ) + raise ReconstructionReportMismatch(message) + refs.append(manifest_ref) + observed += manifest.observed_event_count + synthetic += manifest.synthetic_event_count + summary.update( + { + "manifest_id": manifest.manifest_id, + "publication_id": manifest.publication_id, + "event_count": manifest.event_count, + "observed_event_count": manifest.observed_event_count, + "synthetic_event_count": manifest.synthetic_event_count, + } + ) + elif state.checkpoint.phase is ReconstructionCommitPhase.COMMITTED: + raise ReconstructionReportMismatch( + f"committed window {window.window_id} lacks manifest reference" + ) + window_summaries.append(summary) + if progress is not None: + progress( + { + "phase": "report_reconciliation", + "window_id": window.window_id, + "completed_windows": index, + "total_windows": len(ordered), + } + ) + committed = sum( + item.checkpoint.phase is ReconstructionCommitPhase.COMMITTED + for item in ordered + ) + cancelled = sum( + item.checkpoint.phase is ReconstructionCommitPhase.CANCELLED + for item in ordered + ) + failed = sum( + item.checkpoint.phase is ReconstructionCommitPhase.FAILED + for item in ordered + ) + status = "committed" + if committed != len(ordered): + if failed: + status = "failed" if committed == 0 else "partial" + elif cancelled: + status = "cancelled" if committed == 0 else "partial" + else: + status = "partial" + return ReconstructionRunReportV1( + request_id=request.request_id, + run_id=request.run.run_id, + status=status, + window_count=len(ordered), + committed_window_count=committed, + cancelled_window_count=cancelled, + failed_window_count=failed, + observed_event_count=observed, + synthetic_event_count=synthetic, + committed_manifest_refs=tuple(refs), + window_states=tuple(window_summaries), + ) + + +def _aggregate_outcome_telemetry( + outcomes: Sequence[ReconstructionStageOutcomeV1], +) -> dict[str, JSONValue]: + """Reconcile stage telemetry without counting duplicate output refs.""" + runtimes: list[float] = [] + peak_rss: list[int] = [] + scratch: list[int] = [] + output: list[int] = [] + amplification: list[float] = [] + for outcome in outcomes: + metadata = [ref.metadata for ref in outcome.output_refs] + runtimes.append( + max( + ( + _metadata_number(item, "runtime_seconds") + for item in metadata + ), + default=0.0, + ) + ) + peak_rss.append( + max( + (_metadata_int(item, "peak_rss_bytes") for item in metadata), + default=0, + ) + ) + scratch.append( + max( + (_metadata_int(item, "scratch_bytes") for item in metadata), + default=0, + ) + ) + output.append( + max( + (_metadata_int(item, "output_bytes") for item in metadata), + default=0, + ) + ) + amplification.append( + max( + ( + _metadata_number(item, "candidate_amplification") + for item in metadata + ), + default=0.0, + ) + ) + return { + "runtime_seconds": round(sum(runtimes), 6), + "peak_rss_bytes": max(peak_rss, default=0), + "peak_scratch_bytes": max(scratch, default=0), + "stage_output_bytes_total": sum(output), + "peak_candidate_amplification": round( + max(amplification, default=0.0), 9 + ), + "basis": "sum-stage-runtime-max-stage-resources-v1", + } + + +def _metadata_number(metadata: Mapping[str, JSONValue], key: str) -> float: + value = metadata.get(key, 0.0) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0.0 + return float(value) + + +def _metadata_int(metadata: Mapping[str, JSONValue], key: str) -> int: + value = metadata.get(key, 0) + if isinstance(value, bool) or not isinstance(value, int): + return 0 + return max(0, value) + + +def write_reconstruction_report( + report: ReconstructionRunReportV1, + root: str | Path, +) -> ArtifactRef: + """Atomically write the compact reconciled run report.""" + directory = ( + Path(root).expanduser() + / ".histdatacom" + / RECONSTRUCTION_REPORT_DIRECTORY + ) + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{_safe_filename(report.request_id)}.json" + payload = report.to_json().encode("utf-8") + _atomic_write(path, payload) + return ArtifactRef( + kind="reconstruction_run_report", + path=str(path.resolve()), + size_bytes=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + metadata={"report_id": report.report_id, "run_id": report.run_id}, + ) + + +def verify_artifact_ref(ref: ArtifactRef) -> Path: + """Fail closed unless a local strong artifact reference matches bytes.""" + strong = _strong_ref(ref) + path = Path(strong.path).expanduser() + if not path.is_file(): + raise ReconstructionArtifactError(f"artifact is missing: {path}") + size = path.stat().st_size + if size != strong.size_bytes: + raise ReconstructionArtifactError( + f"artifact size differs for {path}: {size} != {strong.size_bytes}" + ) + digest = _file_sha256(path) + if digest != strong.sha256: + raise ReconstructionArtifactError(f"artifact sha256 differs for {path}") + return path + + +def artifact_ref_for_file( + path: str | Path, + *, + kind: str, + metadata: Mapping[str, JSONValue] | None = None, +) -> ArtifactRef: + """Build a strong reference for one existing local artifact.""" + target = Path(path).expanduser().resolve() + if not target.is_file(): + raise ReconstructionArtifactError(f"artifact is missing: {target}") + return ArtifactRef( + kind=_required_text(kind), + path=str(target), + size_bytes=target.stat().st_size, + sha256=_file_sha256(target), + metadata=dict(metadata or {}), + ) + + +def cleanup_reconstruction_window_scratch(path: str | Path) -> bool: + """Remove only the explicitly scoped uncommitted window scratch tree.""" + target = Path(path).expanduser() + if not target.exists(): + return False + if target.is_symlink() or not target.is_dir(): + raise ReconstructionArtifactError( + "reconstruction scratch path must be a real directory" + ) + if target.resolve() in {Path.home().resolve(), Path("/").resolve()}: + raise ReconstructionArtifactError( + "refusing unsafe scratch cleanup root" + ) + shutil.rmtree(target) + return True + + +def _heartbeat_for( + state: ReconstructionWindowStateV1, + stage: ReconstructionStage, + completed: bool, +) -> ReconstructionHeartbeatV1: + outcomes = state.outcomes + completed_units = len(outcomes) + current = outcomes[-1] if outcomes else None + stage_status = "completed" if completed else "starting" + return ReconstructionHeartbeatV1( + run_id=state.task.window.run_id, + window_id=state.task.window.window_id, + synchronization_unit_id=state.task.window.synchronization_unit_id, + phase=state.checkpoint.phase, + sequence=state.checkpoint.revision, + completed_units=completed_units, + total_units=len(RECONSTRUCTION_STAGE_ORDER), + observed_event_count=current.observed_event_count if current else 0, + candidate_event_count=current.candidate_event_count if current else 0, + accepted_event_count=current.accepted_event_count if current else 0, + scratch_bytes=current.scratch_bytes if current else 0, + output_bytes=current.output_bytes if current else 0, + checkpoint_id=state.checkpoint.checkpoint_id, + message=f"{stage.value}:{stage_status}", + ) + + +async def _emit_heartbeat( + callback: HeartbeatCallback | None, + heartbeat: ReconstructionHeartbeatV1, +) -> None: + if callback is None: + return + result = callback(heartbeat) + if inspect.isawaitable(result): + await result + + +def _outcome_resource_violations( + policy: ReconstructionStoragePolicyV1, + estimate: ReconstructionResourceEstimateV1, + outcome: ReconstructionStageOutcomeV1, +) -> tuple[str, ...]: + """Return fail-closed evidence when actual stage use exceeds admission.""" + violations: list[str] = [] + limits = ( + ( + "candidate_event_count", + outcome.candidate_event_count, + estimate.candidate_event_count, + ), + ("scratch_bytes", outcome.scratch_bytes, policy.max_scratch_bytes), + ("output_bytes", outcome.output_bytes, policy.max_output_bytes), + ) + for name, actual, limit in limits: + if actual > limit: + violations.append(f"{name} {actual} exceeds admitted limit {limit}") + peak_rss_bytes = max( + ( + value + for ref in outcome.output_refs + if type(value := ref.metadata.get("peak_rss_bytes")) is int + ), + default=0, + ) + if peak_rss_bytes > policy.max_memory_bytes: + violations.append( + f"peak_rss_bytes {peak_rss_bytes} exceeds admitted limit " + f"{policy.max_memory_bytes}" + ) + if outcome.accepted_event_count > outcome.candidate_event_count: + violations.append("accepted_event_count exceeds candidate_event_count") + return tuple(violations) + + +def _validate_stage_outcome( + outcome: ReconstructionStageOutcomeV1, + window: ReconstructionWindowV1, + command: ReconstructionStageCommandV1, + prior_outcomes: Sequence[ReconstructionStageOutcomeV1], +) -> None: + if ( + outcome.run_id != window.run_id + or outcome.window_id != window.window_id + or outcome.synchronization_unit_id != window.synchronization_unit_id + or outcome.stage is not command.stage + or outcome.command_id != command.command_id + ): + raise ReconstructionArtifactError( + "stage outcome scope differs from invocation" + ) + payload: dict[str, JSONValue] = { + "run_id": window.run_id, + "window_id": window.window_id, + "command_id": command.command_id, + "input_refs": [ + _artifact_identity(ref) + for ref in ( + *command.input_manifest_refs, + *command.configuration_refs, + *tuple( + ref for prior in prior_outcomes for ref in prior.output_refs + ), + ) + ], + "prior_outcome_ids": [item.outcome_id for item in prior_outcomes], + } + expected = hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + if outcome.input_fingerprint != expected: + raise ReconstructionArtifactError( + "stage outcome input fingerprint differs from invocation" + ) + + +def _phase_artifact( + outcome: ReconstructionStageOutcomeV1, phase: str +) -> ArtifactRef: + matching = tuple( + ref + for ref in outcome.output_refs + if str(ref.metadata.get("commit_phase", "")).strip().lower() == phase + ) + if len(matching) != 1: + raise ValueError( + f"{outcome.stage.value} requires exactly one {phase} manifest ref" + ) + return matching[0] + + +def _state_contains( + current: ReconstructionWindowStateV1, + proposed: ReconstructionWindowStateV1, +) -> bool: + if ( + current.request_id != proposed.request_id + or current.task.task_id != proposed.task.task_id + ): + return False + proposed_ids = tuple(item.outcome_id for item in proposed.outcomes) + current_ids = tuple(item.outcome_id for item in current.outcomes) + if current_ids[: len(proposed_ids)] != proposed_ids: + return False + if current.state_id == proposed.state_id: + return True + if ( + current.checkpoint.parent_checkpoint_id + == proposed.checkpoint.checkpoint_id + ): + return True + return ( + len(current_ids) > len(proposed_ids) + and current.checkpoint.revision > proposed.checkpoint.revision + ) + + +def _read_stage_receipt(path: Path) -> ReconstructionStageOutcomeV1: + if path.stat().st_size > MAX_STAGE_RECEIPT_BYTES: + raise ReconstructionArtifactError("stage receipt exceeds size limit") + try: + return ReconstructionStageOutcomeV1.from_json( + path.read_text(encoding="utf-8") + ) + except (OSError, UnicodeError, ValueError, json.JSONDecodeError) as err: + raise ReconstructionArtifactError( + f"invalid stage receipt: {path}" + ) from err + + +def _write_stage_receipt( + path: Path, outcome: ReconstructionStageOutcomeV1 +) -> None: + payload = outcome.to_json().encode("utf-8") + if path.exists(): + existing = _read_stage_receipt(path) + if existing.outcome_id != outcome.outcome_id: + raise ReconstructionArtifactError( + "stage receipt path already contains different evidence" + ) + return + path.parent.mkdir(parents=True, exist_ok=True) + _atomic_write(path, payload) + + +def _atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp( + prefix=f".{path.name}.", dir=path.parent + ) + temporary_path = Path(temporary) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary_path, path) + finally: + temporary_path.unlink(missing_ok=True) + + +def _reject_inline_data(value: Any, path: str = "request") -> None: + if isinstance(value, Mapping): + for key, item in value.items(): + normalized = str(key).strip().lower() + if normalized in _FORBIDDEN_WORKFLOW_KEYS: + raise ValueError( + f"workflow payload cannot contain {path}.{key}" + ) + _reject_inline_data(item, f"{path}.{key}") + elif isinstance(value, Sequence) and not isinstance( + value, (str, bytes, bytearray) + ): + for index, item in enumerate(value): + _reject_inline_data(item, f"{path}[{index}]") + + +def _unique_strong_refs( + values: Sequence[ArtifactRef], name: str +) -> tuple[ArtifactRef, ...]: + refs: dict[tuple[str, str, str, str], ArtifactRef] = {} + for value in values: + try: + ref = _strong_ref(value) + except ValueError as err: + raise ValueError(f"invalid {name}: {err}") from err + key = ( + ref.kind, + ref.path, + ref.sha256, + canonical_contract_json(ref.metadata), + ) + refs[key] = ref + return tuple(refs[key] for key in sorted(refs)) + + +def _strong_ref(value: ArtifactRef) -> ArtifactRef: + if not isinstance(value, ArtifactRef): + raise ValueError("artifact reference has the wrong type") + kind = _required_text(value.kind) + path = str(Path(_required_text(value.path)).expanduser().resolve()) + size = _nonnegative_int(value.size_bytes, "artifact size_bytes") + digest = _required_sha256(value.sha256, "artifact sha256") + return ArtifactRef( + kind=kind, + path=path, + size_bytes=size, + sha256=digest, + metadata=dict(value.metadata), + ) + + +def _artifact_identity(ref: ArtifactRef) -> dict[str, JSONValue]: + strong = _strong_ref(ref) + return { + "kind": strong.kind, + "path": strong.path, + "size_bytes": strong.size_bytes, + "sha256": strong.sha256, + "metadata": dict(strong.metadata), + } + + +def _artifact_refs(value: Any) -> tuple[ArtifactRef, ...]: + return tuple( + ArtifactRef.from_dict(_mapping(item)) for item in _sequence(value) + ) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _required_text(value: Any) -> str: + normalized = str(value).strip() if value is not None else "" + if not normalized: + raise ValueError("required text value is empty") + return normalized + + +def _bounded_text(value: Any) -> str: + normalized = str(value or "").strip() + if len(normalized.encode("utf-8")) > MAX_STAGE_MESSAGE_LENGTH: + raise ValueError("stage text exceeds bounded message limit") + return normalized + + +def _bounded_reason_summary(values: Sequence[str]) -> str: + reasons = tuple( + str(value).strip() for value in values if str(value).strip() + ) + full = "; ".join(reasons) + if len(full.encode("utf-8")) <= MAX_STAGE_MESSAGE_LENGTH: + return full + digest = hashlib.sha256(full.encode("utf-8")).hexdigest()[:16] + selected: list[str] = [] + for index, reason in enumerate(reasons): + suffix = f"; +{len(reasons) - index - 1} more [sha256:{digest}]" + candidate = "; ".join((*selected, reason)) + suffix + if len(candidate.encode("utf-8")) > MAX_STAGE_MESSAGE_LENGTH: + break + selected.append(reason) + suffix = f"; +{len(reasons) - len(selected)} more [sha256:{digest}]" + return "; ".join(selected) + suffix + + +def _required_sha256(value: Any, name: str) -> str: + normalized = str(value).strip().lower() + if len(normalized) != 64 or any( + char not in "0123456789abcdef" for char in normalized + ): + raise ValueError(f"{name} must be a lowercase sha256 digest") + return normalized + + +def _nonnegative_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{name} must be a nonnegative integer") + return value + + +def _positive_int(value: Any, name: str) -> int: + result = _nonnegative_int(value, name) + if result == 0: + raise ValueError(f"{name} must be positive") + return result + + +def _ensure_payload_size( + value: Mapping[str, JSONValue], maximum: int, name: str +) -> None: + size = len(canonical_contract_json(value).encode("utf-8")) + if size > maximum: + raise ValueError(f"{name} exceeds {maximum} bytes") + + +def _file_sha256(path: Path) -> str: + target = path.resolve() + stat = target.stat() + return _file_sha256_for_identity( + str(target), + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + ) + + +@lru_cache(maxsize=8192) +def _file_sha256_for_identity( + path: str, + device: int, + inode: int, + size: int, + modified_ns: int, + changed_ns: int, +) -> str: + del device, inode, size, modified_ns, changed_ns + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _safe_filename(value: str) -> str: + allowed = ( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_." + ) + normalized = "".join(char if char in allowed else "_" for char in value) + return normalized[:160] or "reconstruction-report" + + +def _validate_scratch_boundaries( + tasks: Sequence[ReconstructionWindowTaskV1], + *, + durable_roots: Sequence[str], +) -> None: + scratch_paths = tuple( + Path(task.scratch_directory).expanduser().resolve() for task in tasks + ) + for index, left in enumerate(scratch_paths): + for right in scratch_paths[index + 1 :]: + if _paths_overlap(left, right): + raise ValueError( + "reconstruction window scratch directories must be disjoint" + ) + durable_paths = tuple( + Path(root).expanduser().resolve() for root in durable_roots + ) + for scratch, task in zip(scratch_paths, tasks): + if any(_paths_overlap(scratch, root) for root in durable_paths): + raise ValueError( + "window scratch must not overlap manifest or report storage" + ) + for command in task.commands: + for ref in ( + *command.input_manifest_refs, + *command.configuration_refs, + ): + if ( + Path(ref.path) + .expanduser() + .resolve() + .is_relative_to(scratch) + ): + raise ValueError( + "durable stage inputs must remain outside window scratch" + ) + + +def _paths_overlap(left: Path, right: Path) -> bool: + return ( + left == right + or left.is_relative_to(right) + or right.is_relative_to(left) + ) + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError("expected a sequence") + return value + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + return _mapping(value) diff --git a/src/histdatacom/orchestration/worker.py b/src/histdatacom/orchestration/worker.py index e08fecec..ef67b8af 100644 --- a/src/histdatacom/orchestration/worker.py +++ b/src/histdatacom/orchestration/worker.py @@ -67,6 +67,12 @@ def build_temporal_worker( """Build a Temporal worker from centralized orchestration configuration.""" resolved_config = config or build_orchestration_worker_config() temporal_worker_class = worker_class or _load_temporal_worker_class() + if not activities: + from histdatacom.synthetic.reconstruction_handlers import ( + register_first_party_reconstruction_handlers, + ) + + register_first_party_reconstruction_handlers() workflow_classes = ( list(workflows) if workflows else list(default_workflows()) ) diff --git a/src/histdatacom/orchestration/workflows.py b/src/histdatacom/orchestration/workflows.py index 0eadb26f..bd3a36c8 100644 --- a/src/histdatacom/orchestration/workflows.py +++ b/src/histdatacom/orchestration/workflows.py @@ -8,8 +8,10 @@ from __future__ import annotations import asyncio +from contextlib import nullcontext from dataclasses import dataclass from datetime import timedelta +import hashlib from importlib import import_module import logging from typing import Any, Callable, Mapping, Protocol, TypeVar, cast @@ -93,6 +95,18 @@ def _load_workflow_api() -> Any: workflow = _load_workflow_api() +_passed_through_imports = ( + workflow.unsafe.imports_passed_through + if hasattr(workflow, "unsafe") + else nullcontext +) +with _passed_through_imports(): + from histdatacom.orchestration.reconstruction import ( + ReconstructionWorkflowRequestV1, + ReconstructionWindowStateV1, + plan_reconstruction_waves, + ) + _Decorated = TypeVar("_Decorated") _Callable = TypeVar("_Callable", bound=Callable[..., Any]) _WORKFLOW_LOGGER = logging.getLogger(__name__) @@ -516,6 +530,17 @@ async def execute_activity( lane=TaskQueueLane.INFLUX, operation_family="Influx import", ), + WorkflowSpec( + name="ReconstructionRunWorkflow", + lane=TaskQueueLane.ORCHESTRATION, + operation_family="synthetic reconstruction run", + children=("ReconstructionWindowWorkflow",), + ), + WorkflowSpec( + name="ReconstructionWindowWorkflow", + lane=TaskQueueLane.ORCHESTRATION, + operation_family="synchronized reconstruction window", + ), ) WORKFLOW_SPECS_BY_NAME = {spec.name: spec for spec in WORKFLOW_TOPOLOGY} OPERATION_ACTIVITIES = { @@ -584,6 +609,18 @@ async def execute_activity( heartbeat_timeout_seconds=30, retry_policy=IDEMPOTENT_WRITE_RETRY_POLICY, ), + "reconstruction_window": ActivityExecutionPolicy( + activity_name="reconstruction_window", + start_to_close_timeout_seconds=604800, + heartbeat_timeout_seconds=60, + retry_policy=IDEMPOTENT_WRITE_RETRY_POLICY, + ), + "reconstruction_report": ActivityExecutionPolicy( + activity_name="reconstruction_report", + start_to_close_timeout_seconds=129600, + heartbeat_timeout_seconds=3600, + retry_policy=IDEMPOTENT_WRITE_RETRY_POLICY, + ), } DEFAULT_MAX_WORK_ITEMS_PER_BATCH = 64 BATCHING_METADATA_KEY = "temporal_batching" @@ -1189,6 +1226,174 @@ async def run(self, payload: dict[str, Any]) -> dict: status = workflow_query(_ActivityWorkflowBase.status) +@workflow_defn +class ReconstructionWindowWorkflow: + """Durable child workflow for one synchronized synthetic-data window.""" + + def __init__(self) -> None: + self._status: dict[str, JSONValue] = { + "status": WorkStatus.PLANNED.value, + "current_stage": "planned", + "completed_stages": 0, + "total_stages": 7, + } + + @workflow_run + async def run(self, payload: dict[str, Any]) -> dict[str, Any]: + """Execute the retry-safe window activity on the CPU/file lane.""" + request = ReconstructionWorkflowRequestV1.from_dict( + _coerce_mapping(payload.get("request")) + ) + if len(request.tasks) != 1: + raise ValueError("reconstruction window workflow requires one task") + task = request.tasks[0] + self._status = { + "status": WorkStatus.PLANNED.value, + "current_stage": "reconstruction_window", + "completed_stages": 0, + "total_stages": 7, + "window_id": task.window.window_id, + } + policy = activity_execution_policy("reconstruction_window") + options: dict[str, Any] = { + "start_to_close_timeout": timedelta( + seconds=policy.start_to_close_timeout_seconds + ), + "heartbeat_timeout": timedelta( + seconds=policy.heartbeat_timeout_seconds + ), + "retry_policy": _temporal_retry_policy(policy.retry_policy), + } + queue = request.task_queues.get("cpu_file", "") + if queue: + options["task_queue"] = queue + raw = await workflow.execute_activity( + "reconstruction_window", + {"request": request.to_dict()}, + **options, + ) + state = ReconstructionWindowStateV1.from_dict(_coerce_mapping(raw)) + self._status = { + "status": _reconstruction_work_status(state).value, + "current_stage": state.checkpoint.phase.value, + "completed_stages": len(state.outcomes), + "total_stages": 7, + "window_id": task.window.window_id, + "checkpoint_id": state.checkpoint.checkpoint_id, + } + result_payload: dict[str, JSONValue] = state.to_dict() + return result_payload + + @workflow_query + def status(self) -> dict[str, JSONValue]: + """Return compact queryable window progress.""" + return dict(self._status) + + +@workflow_defn +class ReconstructionRunWorkflow: + """Parent planner with deterministic wave-level reconstruction backpressure.""" + + def __init__(self) -> None: + self._status: dict[str, JSONValue] = { + "status": WorkStatus.PLANNED.value, + "current_stage": "planned", + "completed_windows": 0, + "total_windows": 0, + } + + @workflow_run + async def run(self, payload: dict[str, Any]) -> dict[str, Any]: + """Run bounded child windows, then reconcile committed storage.""" + request = ReconstructionWorkflowRequestV1.from_dict( + _coerce_mapping(payload.get("request", payload)) + ) + execution_attempt_id = _reconstruction_execution_attempt_id(payload) + self._status = { + "status": WorkStatus.PLANNED.value, + "current_stage": "window_planning", + "completed_windows": 0, + "total_windows": len(request.tasks), + "request_id": request.request_id, + } + waves = plan_reconstruction_waves( + request.tasks, + max_parallel_windows=request.max_parallel_windows, + max_inflight_memory_bytes=cast( + int, request.max_inflight_memory_bytes + ), + ) + states: list[ReconstructionWindowStateV1] = [] + for wave_index, wave in enumerate(waves): + self._status["current_stage"] = f"window_wave_{wave_index}" + children = [] + for task in wave: + child_request = request.for_task(task) + child_id = _reconstruction_child_workflow_id( + request.request_id, + task.window.window_id, + execution_attempt_id=execution_attempt_id, + ) + child_options: dict[str, Any] = {"id": child_id} + queue = request.task_queues.get("orchestration", "") + if queue: + child_options["task_queue"] = queue + children.append( + workflow.execute_child_workflow( + "ReconstructionWindowWorkflow", + {"request": child_request.to_dict()}, + **child_options, + ) + ) + raw_states = await asyncio.gather(*children) + states.extend( + ReconstructionWindowStateV1.from_dict( + _coerce_mapping(raw_state) + ) + for raw_state in raw_states + ) + self._status["completed_windows"] = len(states) + self._status["current_stage"] = "report_reconciliation" + policy = activity_execution_policy("reconstruction_report") + report_options: dict[str, Any] = { + "start_to_close_timeout": timedelta( + seconds=policy.start_to_close_timeout_seconds + ), + "heartbeat_timeout": timedelta( + seconds=policy.heartbeat_timeout_seconds + ), + "retry_policy": _temporal_retry_policy(policy.retry_policy), + } + report_queue = request.task_queues.get("cpu_file", "") + if report_queue: + report_options["task_queue"] = report_queue + report = await workflow.execute_activity( + "reconstruction_report", + {"request": request.to_dict()}, + **report_options, + ) + report_payload = _coerce_mapping(report) + report_body = _coerce_mapping(report_payload.get("report")) + self._status = { + "status": ( + WorkStatus.COMPLETED.value + if report_body.get("status") == "committed" + else WorkStatus.FAILED.value + ), + "current_stage": "complete", + "completed_windows": len(states), + "total_windows": len(request.tasks), + "request_id": request.request_id, + "report_id": str(report_body.get("report_id", "")), + } + return dict(report_payload) + + @workflow_query + def status(self) -> dict[str, JSONValue]: + """Return compact queryable parent progress.""" + return dict(self._status) + + DEFAULT_WORKFLOWS = ( HistDataRunWorkflow, RepositoryRefreshWorkflow, @@ -1201,9 +1406,50 @@ async def run(self, payload: dict[str, Any]) -> dict: BuildCacheWorkflow, MergeCacheWorkflow, ImportWorkflow, + ReconstructionRunWorkflow, + ReconstructionWindowWorkflow, ) +def _reconstruction_child_workflow_id( + request_id: str, + window_id: str, + *, + execution_attempt_id: str = "", +) -> str: + """Return a bounded deterministic child workflow ID.""" + identity = window_id + if execution_attempt_id: + identity = f"{window_id}|{execution_attempt_id}" + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:24] + return f"{request_id}-reconstruction-window-{digest}" + + +def _reconstruction_execution_attempt_id(payload: dict[str, Any]) -> str: + """Validate the bounded recovery identity carried beside a request.""" + attempt = str(payload.get("execution_attempt_id", "") or "").strip() + if len(attempt) > 64: + raise ValueError("reconstruction execution_attempt_id exceeds 64 chars") + if any(not (char.isalnum() or char in {"-", "_", "."}) for char in attempt): + raise ValueError( + "reconstruction execution_attempt_id contains unsupported characters" + ) + return attempt + + +def _reconstruction_work_status( + state: ReconstructionWindowStateV1, +) -> WorkStatus: + """Map checkpoint phase to public orchestration status.""" + if state.checkpoint.phase.value == "committed": + return WorkStatus.COMPLETED + if state.checkpoint.phase.value == "cancelled": + return WorkStatus.CANCELLED + if state.checkpoint.phase.value == "failed": + return WorkStatus.FAILED + return WorkStatus.PLANNED + + def _execute_status(results: tuple[StageResult, ...]) -> WorkStatus: if any(result.status == WorkStatus.CANCELLED for result in results): return WorkStatus.CANCELLED @@ -1240,8 +1486,7 @@ async def _execute_child_plan( ) _workflow_log( logging.INFO, - "Workflow child plan started request_id=%s workflow_name=%s " - "child_count=%d", + "Workflow child plan started request_id=%s workflow_name=%s child_count=%d", request.request_id, workflow_name, len(invocations), @@ -1546,8 +1791,7 @@ async def _execute_parallel_symbol_invocations( break _workflow_log( logging.DEBUG, - "Workflow parallel symbol fanout finished request_id=%s " - "completed=%d", + "Workflow parallel symbol fanout finished request_id=%s completed=%d", request.request_id, len(results), request_id=request.request_id, diff --git a/src/histdatacom/quality_cli.py b/src/histdatacom/quality_cli.py index 898d9587..3cb0e623 100644 --- a/src/histdatacom/quality_cli.py +++ b/src/histdatacom/quality_cli.py @@ -6,8 +6,11 @@ import json import sys from collections.abc import Mapping, Sequence +from pathlib import Path from typing import cast +from polars.exceptions import PolarsError + from histdatacom.cli_config import ( CliConfigError, add_config_argument, @@ -24,6 +27,11 @@ format_fingerprint_contract_audit, format_fingerprint_schema_discovery, ) +from histdatacom.data_quality.fingerprint_next_work import ( + DEFAULT_FINGERPRINT_NEXT_WORK_ALTERNATE_LIMIT, + fingerprint_next_work_recommendation, + format_fingerprint_next_work, +) from histdatacom.data_quality.reporting import ( fingerprint_readiness_risk_summary, format_fingerprint_readiness_risk_lines, @@ -48,6 +56,40 @@ remediation_catalog_audit_has_warning_error_gaps, remediation_catalog_audit_to_json, ) +from histdatacom.data_quality.contracts import ( + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.discovery import quality_target_from_path +from histdatacom.data_quality.repair_plan import ( + DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + DEFAULT_QUALITY_REPAIR_PLAN_ITEM_LIMIT, + format_quality_repair_plan, + quality_repair_plan, + quality_repair_plan_to_json, +) +from histdatacom.data_quality.synthetic_constraints import ( + DEFAULT_SYNTHETIC_VALIDATION_MISMATCH_LIMIT, + DEFAULT_SYNTHETIC_VALIDATION_TARGET_LIMIT, + format_synthetic_validation, + validate_synthetic_constraint_reports, +) +from histdatacom.data_quality.synthetic_generation import ( + DEFAULT_SYNTHETIC_TICK_BLOCK_SIZE, + DEFAULT_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT, + DEFAULT_SYNTHETIC_TICK_MAX_ABS_LOG_RETURN, + DEFAULT_SYNTHETIC_TICK_MAX_GENERATED_ROWS, + DEFAULT_SYNTHETIC_TICK_MAX_REFERENCE_ROWS, + DEFAULT_SYNTHETIC_TICK_MINIMUM_REFERENCE_ROWS, + DEFAULT_SYNTHETIC_TICK_ROUNDING_DIGITS, + DEFAULT_SYNTHETIC_TICK_SEED, + SyntheticTickGenerationProfile, + format_synthetic_tick_generation, + generate_synthetic_ticks_from_reference, + reference_fingerprint_from_report, + validate_synthetic_tick_cache, +) +from histdatacom.data_quality.reporting import write_quality_report from histdatacom.fx_enums import ( Format, Pairs, @@ -58,6 +100,11 @@ from histdatacom.publication_safety import publish_safe_path from histdatacom.runtime_contracts import JSONValue from histdatacom.verbosity import configure_logging +from histdatacom.histdata_ascii import ( + TICK, + read_polars_cache, + write_polars_cache, +) FINGERPRINT_READINESS_RISK_COMMAND_SCHEMA_VERSION = ( "histdatacom.fingerprint-readiness-risk-command.v1" @@ -233,6 +280,43 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="emit the machine-readable audit payload", ) + repair_plan = subparsers.add_parser( + "repair-plan", + aliases=("remediation-repair-plan",), + help="derive a non-mutating repair plan from a saved quality report", + ) + repair_plan.add_argument( + "--report", + dest="report_path", + required=True, + metavar="PATH", + help="saved quality JSON report to translate into a repair plan", + ) + repair_plan.add_argument( + "--item-limit", + type=_non_negative_int, + default=DEFAULT_QUALITY_REPAIR_PLAN_ITEM_LIMIT, + metavar="N", + help=( + "maximum repair-plan items to include; defaults to " + f"{DEFAULT_QUALITY_REPAIR_PLAN_ITEM_LIMIT}" + ), + ) + repair_plan.add_argument( + "--evidence-limit", + type=_non_negative_int, + default=DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT, + metavar="N", + help=( + "maximum evidence values per item; defaults to " + f"{DEFAULT_QUALITY_REPAIR_PLAN_EVIDENCE_LIMIT}" + ), + ) + repair_plan.add_argument( + "--json", + action="store_true", + help="emit the machine-readable non-mutating repair plan", + ) fingerprint_schema = subparsers.add_parser( "fingerprint-schema", aliases=("fingerprint-contract", "fingerprint-discovery"), @@ -298,11 +382,170 @@ def build_parser() -> argparse.ArgumentParser: metavar="N", help="maximum reason codes to include; use -1 for all", ) + fingerprint_readiness.add_argument( + "--next-work", + action="store_true", + help=( + "recommend the next fingerprint product work from the saved " + "report evidence" + ), + ) + fingerprint_readiness.add_argument( + "--alternate-limit", + dest="alternate_limit", + type=_integer_limit, + default=DEFAULT_FINGERPRINT_NEXT_WORK_ALTERNATE_LIMIT, + metavar="N", + help=( + "maximum alternate next-work recommendations to include; " + "use -1 for all" + ), + ) fingerprint_readiness.add_argument( "--json", action="store_true", help="emit the machine-readable readiness risk ranking payload", ) + synthetic_generate = subparsers.add_parser( + "synthetic-generate", + aliases=("synthetic-tick-generate",), + help="generate deterministic synthetic tick columns from a reference set", + ) + synthetic_generate.add_argument( + "--reference-cache", + required=True, + metavar="PATH", + help="enriched or legacy ASCII tick .data reference cache", + ) + synthetic_generate.add_argument( + "--reference-report", + required=True, + metavar="PATH", + help="saved quality report containing the reference fingerprint", + ) + synthetic_generate.add_argument( + "--output-cache", + required=True, + metavar="PATH", + help="destination enriched .data cache with populated synth_* columns", + ) + synthetic_generate.add_argument( + "--candidate-report", + metavar="PATH", + help="optional saved ordinary fingerprint report for the candidate", + ) + synthetic_generate.add_argument( + "--seed", + type=int, + default=DEFAULT_SYNTHETIC_TICK_SEED, + help="deterministic bootstrap seed", + ) + synthetic_generate.add_argument( + "--block-size", + type=int, + default=DEFAULT_SYNTHETIC_TICK_BLOCK_SIZE, + metavar="N", + help="contiguous empirical transition block size", + ) + synthetic_generate.add_argument( + "--minimum-reference-rows", + type=int, + default=DEFAULT_SYNTHETIC_TICK_MINIMUM_REFERENCE_ROWS, + metavar="N", + help="minimum usable reference rows with an adjacent transition", + ) + synthetic_generate.add_argument( + "--max-reference-rows", + type=int, + default=DEFAULT_SYNTHETIC_TICK_MAX_REFERENCE_ROWS, + metavar="N", + help="maximum reference rows inspected", + ) + synthetic_generate.add_argument( + "--max-generated-rows", + type=int, + default=DEFAULT_SYNTHETIC_TICK_MAX_GENERATED_ROWS, + metavar="N", + help="maximum rows populated with synthetic values", + ) + synthetic_generate.add_argument( + "--max-abs-log-return", + type=float, + default=DEFAULT_SYNTHETIC_TICK_MAX_ABS_LOG_RETURN, + metavar="FLOAT", + help="absolute sampled log-return safety bound", + ) + synthetic_generate.add_argument( + "--rounding-digits", + type=int, + default=DEFAULT_SYNTHETIC_TICK_ROUNDING_DIGITS, + metavar="N", + help="synthetic quote rounding digits", + ) + synthetic_generate.add_argument( + "--diagnostic-sample-limit", + type=int, + default=DEFAULT_SYNTHETIC_TICK_DIAGNOSTIC_SAMPLE_LIMIT, + metavar="N", + help="bounded reference-transition evidence sample count", + ) + synthetic_generate.add_argument( + "--anchor-mode", + choices=("first_valid_mid", "median_valid_mid"), + default="first_valid_mid", + help="starting midpoint selection policy", + ) + synthetic_generate.add_argument( + "--overwrite-output", + action="store_true", + help="replace an existing output cache", + ) + synthetic_generate.add_argument( + "--overwrite-synthetic", + action="store_true", + help="replace populated synth_* values in the reference cache", + ) + synthetic_generate.add_argument( + "--json", + action="store_true", + help="emit machine-readable bounded generation diagnostics", + ) + synthetic_validate = subparsers.add_parser( + "synthetic-validate", + aliases=("synthetic-fingerprint-validate",), + help="compare candidate and reference synthetic fingerprint constraints", + ) + synthetic_validate.add_argument( + "--reference-report", + required=True, + metavar="PATH", + help="saved reference quality report containing fingerprint constraints", + ) + synthetic_validate.add_argument( + "--candidate-report", + required=True, + metavar="PATH", + help="saved candidate quality report containing fingerprint constraints", + ) + synthetic_validate.add_argument( + "--target-limit", + type=_integer_limit, + default=DEFAULT_SYNTHETIC_VALIDATION_TARGET_LIMIT, + metavar="N", + help="maximum target comparisons to include; use -1 for all", + ) + synthetic_validate.add_argument( + "--mismatch-limit", + type=_integer_limit, + default=DEFAULT_SYNTHETIC_VALIDATION_MISMATCH_LIMIT, + metavar="N", + help="maximum mismatch details per target; use -1 for all", + ) + synthetic_validate.add_argument( + "--json", + action="store_true", + help="emit the machine-readable advisory validation payload", + ) bounded_payload = subparsers.add_parser( "bounded-payload-contract", aliases=("bounded-payload-audit", "report-payload-contract"), @@ -326,6 +569,12 @@ def main(argv: Sequence[str] | None = None) -> int: print(f"config error: {exc}", file=sys.stderr) # noqa:T201 return 1 configure_logging(args.verbosity) + if args.quality_command in { + "synthetic-generate", + "synthetic-tick-generate", + }: + return _run_synthetic_generation(args) + if args.quality_command in { "evidence", "inspect-evidence", @@ -375,6 +624,29 @@ def main(argv: Sequence[str] | None = None) -> int: else 0 ) + if args.quality_command in { + "repair-plan", + "remediation-repair-plan", + }: + try: + report = load_quality_report(args.report_path) + repair_payload = quality_repair_plan( + report, + report_path=args.report_path, + item_limit=args.item_limit, + evidence_limit=args.evidence_limit, + ) + except (OSError, ValueError, TypeError) as exc: + print(f"quality report error: {exc}", file=sys.stderr) # noqa:T201 + return 1 + if args.json: + print( + quality_repair_plan_to_json(repair_payload), end="" + ) # noqa:T201 + else: + print(format_quality_repair_plan(repair_payload)) # noqa:T201 + return 0 + if args.quality_command in { "fingerprint-schema", "fingerprint-contract", @@ -419,6 +691,8 @@ def main(argv: Sequence[str] | None = None) -> int: target_limit=args.target_limit, section_limit=args.section_limit, reason_limit=args.reason_limit, + next_work=args.next_work, + alternate_limit=args.alternate_limit, ) except (OSError, ValueError, TypeError) as exc: print(f"quality report error: {exc}", file=sys.stderr) # noqa:T201 @@ -433,6 +707,28 @@ def main(argv: Sequence[str] | None = None) -> int: ) # noqa:T201 return 0 + if args.quality_command in { + "synthetic-validate", + "synthetic-fingerprint-validate", + }: + try: + reference = load_quality_report(args.reference_report) + candidate = load_quality_report(args.candidate_report) + validation = validate_synthetic_constraint_reports( + reference, + candidate, + target_limit=args.target_limit, + mismatch_limit=args.mismatch_limit, + ) + except (OSError, ValueError, TypeError) as exc: + print(f"quality report error: {exc}", file=sys.stderr) # noqa:T201 + return 1 + if args.json: + print(json.dumps(validation, indent=2, sort_keys=True)) # noqa:T201 + else: + print(format_synthetic_validation(validation)) # noqa:T201 + return 0 + if args.quality_command in { "bounded-payload-contract", "bounded-payload-audit", @@ -452,17 +748,149 @@ def main(argv: Sequence[str] | None = None) -> int: parser.error(f"unsupported quality command: {args.quality_command}") +def _run_synthetic_generation(args: argparse.Namespace) -> int: + reference_path = Path(args.reference_cache).expanduser() + output_path = Path(args.output_cache).expanduser() + try: + reference_report_path = Path(args.reference_report).expanduser() + candidate_report_path = ( + Path(args.candidate_report).expanduser() + if args.candidate_report + else None + ) + output_resolved = output_path.resolve() + protected_paths = { + reference_path.resolve(): "reference cache", + reference_report_path.resolve(): "reference report", + } + if candidate_report_path is not None: + protected_paths[candidate_report_path.resolve()] = ( + "candidate report" + ) + if output_resolved in protected_paths: + raise ValueError( + "output cache path conflicts with " + f"{protected_paths[output_resolved]}" + ) + if ( + candidate_report_path is not None + and candidate_report_path.resolve() + in {reference_path.resolve(), reference_report_path.resolve()} + ): + raise ValueError( + "candidate report path conflicts with a reference input" + ) + if output_path.exists() and not args.overwrite_output: + raise ValueError( + "output cache already exists: " + f"{publish_safe_path(str(output_path))}" + ) + if ( + candidate_report_path is not None + and candidate_report_path.exists() + and not args.overwrite_output + ): + raise ValueError( + "candidate report already exists: " + f"{publish_safe_path(str(candidate_report_path))}" + ) + reference_report = load_quality_report(reference_report_path) + parsed_target = quality_target_from_path(reference_path) + reference_fingerprint = reference_fingerprint_from_report( + reference_report, + target=parsed_target, + ) + axis_value = reference_fingerprint.get("target_axis") + axis = ( + cast(Mapping[str, JSONValue], axis_value) + if isinstance(axis_value, Mapping) + else {} + ) + reference_target = QualityTarget( + path=str(reference_path.resolve()), + kind=QualityTargetKind.CACHE, + data_format=str(axis.get("data_format") or "ascii"), + timeframe=str(axis.get("timeframe") or TICK), + symbol=str(axis.get("symbol") or ""), + period=str(axis.get("period") or ""), + ) + profile = SyntheticTickGenerationProfile( + seed=args.seed, + block_size=args.block_size, + minimum_reference_rows=args.minimum_reference_rows, + max_reference_rows=args.max_reference_rows, + max_generated_rows=args.max_generated_rows, + max_abs_log_return=args.max_abs_log_return, + rounding_digits=args.rounding_digits, + diagnostic_sample_limit=args.diagnostic_sample_limit, + anchor_mode=args.anchor_mode, + overwrite_existing=args.overwrite_synthetic, + ) + result = generate_synthetic_ticks_from_reference( + read_polars_cache(reference_path), + reference_fingerprint, + profile=profile, + target=reference_target, + ) + diagnostics = dict(result.diagnostics) + if diagnostics.get("status") == "unavailable": + raise ValueError( + "synthetic generation unavailable: " + f"{diagnostics.get('reason', 'unknown')}" + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + write_polars_cache(result.frame, output_path) + candidate_target = QualityTarget( + path=str(output_path.resolve()), + kind=QualityTargetKind.CACHE, + data_format=reference_target.data_format, + timeframe=reference_target.timeframe, + symbol=reference_target.symbol, + period=reference_target.period, + ) + candidate_report, validation = validate_synthetic_tick_cache( + output_path, + reference_fingerprint, + target=candidate_target, + ) + diagnostics["validation"] = validation + if candidate_report_path is not None: + write_quality_report(candidate_report, candidate_report_path) + except (OSError, ValueError, TypeError, PolarsError) as exc: + print( + f"synthetic generation error: {exc}", file=sys.stderr + ) # noqa:T201 + return 1 + if args.json: + print(json.dumps(diagnostics, indent=2, sort_keys=True)) # noqa:T201 + else: + print(format_synthetic_tick_generation(diagnostics)) # noqa:T201 + print( # noqa:T201 + f"output cache: {publish_safe_path(str(output_path))}" + ) + if candidate_report_path is not None: + print( # noqa:T201 + "candidate report: " + f"{publish_safe_path(str(candidate_report_path))}" + ) + return 0 + + def _fingerprint_readiness_risk_command_payload( report_paths: Sequence[str], *, target_limit: int | None, section_limit: int | None, reason_limit: int | None, + next_work: bool = False, + alternate_limit: int | None = None, ) -> dict[str, JSONValue]: reports: list[dict[str, JSONValue]] = [] + loaded_reports = [] risk_report_count = 0 for path in report_paths: report = load_quality_report(path) + loaded_reports.append((path, report)) summary = fingerprint_readiness_risk_summary( report, target_limit=target_limit, @@ -480,12 +908,19 @@ def _fingerprint_readiness_risk_command_payload( "summary": summary or {}, } ) - return { + payload: dict[str, JSONValue] = { "schema_version": FINGERPRINT_READINESS_RISK_COMMAND_SCHEMA_VERSION, "report_count": len(reports), "risk_report_count": risk_report_count, "reports": cast(JSONValue, reports), } + if next_work: + payload["next_work"] = fingerprint_next_work_recommendation( + loaded_reports, + alternate_limit=alternate_limit, + target_axis_limit=target_limit, + ) + return payload def _format_fingerprint_readiness_risk_command( @@ -510,6 +945,10 @@ def _format_fingerprint_readiness_risk_command( else: lines.append("Fingerprint readiness risk") lines.append("- no fingerprint readiness data") + next_work = payload.get("next_work") + if isinstance(next_work, dict): + lines.append("") + lines.append(format_fingerprint_next_work(next_work)) return "\n".join(lines) diff --git a/src/histdatacom/random_windows.py b/src/histdatacom/random_windows.py new file mode 100644 index 00000000..4b6ba096 --- /dev/null +++ b/src/histdatacom/random_windows.py @@ -0,0 +1,1047 @@ +"""Deterministic random and session-window selection for tick projections. + +HistData source archives and canonical caches are monthly evidence. This +module therefore resolves a compact selection contract during planning and +filters only consumer projections; it never rewrites source artifacts. +""" + +from __future__ import annotations + +import calendar +import hashlib +import json +import random +import re +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta, timezone +from functools import lru_cache, reduce +from operator import or_ +from typing import Any, Mapping, Sequence, cast +from zoneinfo import ZoneInfo + +RANDOM_WINDOW_EXPRESSION_SCHEMA_VERSION = ( + "histdatacom.random-window-expression.v1" +) +RANDOM_WINDOW_SELECTION_SCHEMA_VERSION = ( + "histdatacom.random-window-selection.v1" +) +RANDOM_WINDOW_SESSION_PROFILE_VERSION = ( + "histdatacom.random-window-session-profile.v1" +) +RANDOM_WINDOW_SELECTION_METADATA_KEY = "random_window_selection" + +RANDOM_WINDOW_MODE_RANDOM = "random" +RANDOM_WINDOW_MODE_ALL_SESSIONS = "all_sessions" + +MILLISECONDS_PER_MINUTE = 60_000 +MAX_RANDOM_WINDOW_DURATION_COUNT = 1_000_000 +MAX_RANDOM_WINDOW_SESSION_OCCURRENCES = 20_000 + +_DURATION_RE = re.compile(r"^([1-9][0-9]*)([yqMwdhm])$") +_SESSION_CODES = frozenset( + {"fra", "ldn", "ny", "chi", "la", "auk", "syd", "tyo", "hk"} +) +_SMALL_DURATION_UNITS = frozenset({"h", "m"}) +_LARGE_DURATION_UNITS = frozenset({"y", "q", "M", "w", "d"}) + + +class RandomWindowError(ValueError): + """Base class for bounded random-window contract failures.""" + + +class RandomWindowSyntaxError(RandomWindowError): + """A random-window expression is malformed or ambiguous.""" + + +class RandomWindowSupportError(RandomWindowError): + """Requested symbols or bounds do not contain a valid selection.""" + + +class RandomWindowEmptySelectionError(RandomWindowError): + """A resolved selection produced no projected tick rows.""" + + +@dataclass(frozen=True, slots=True) +class RandomWindowSessionProfileV1: + """Explicit local-clock sampling window for one legacy session code.""" + + code: str + label: str + timezone_name: str + start_minute_local: int = 8 * 60 + end_minute_local: int = 17 * 60 + schema_version: str = RANDOM_WINDOW_SESSION_PROFILE_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RANDOM_WINDOW_SESSION_PROFILE_VERSION: + raise ValueError("unsupported random-window session profile") + if self.code not in _SESSION_CODES: + raise ValueError("unsupported random-window session code") + if not self.label.strip(): + raise ValueError("random-window session label is required") + try: + ZoneInfo(self.timezone_name) + except Exception as err: + raise ValueError("invalid random-window IANA timezone") from err + for name, value in ( + ("start_minute_local", self.start_minute_local), + ("end_minute_local", self.end_minute_local), + ): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if not 0 <= value < 24 * 60: + raise ValueError(f"{name} must be within one local day") + if self.start_minute_local == self.end_minute_local: + raise ValueError("random-window session cannot have zero duration") + + def to_dict(self) -> dict[str, Any]: + """Return deterministic public session-profile metadata.""" + return { + "schema_version": self.schema_version, + "code": self.code, + "label": self.label, + "timezone": self.timezone_name, + "start_minute_local": self.start_minute_local, + "end_minute_local": self.end_minute_local, + "dst_policy": "iana_zone_rules", + "semantics": "sampling_window_not_exchange_hours", + } + + +RANDOM_WINDOW_SESSION_PROFILES = { + profile.code: profile + for profile in ( + RandomWindowSessionProfileV1( + code="fra", + label="Frankfurt/Paris", + timezone_name="Europe/Paris", + ), + RandomWindowSessionProfileV1( + code="ldn", + label="London", + timezone_name="Europe/London", + ), + RandomWindowSessionProfileV1( + code="ny", + label="New York", + timezone_name="America/New_York", + ), + RandomWindowSessionProfileV1( + code="chi", + label="Chicago", + timezone_name="America/Chicago", + ), + RandomWindowSessionProfileV1( + code="la", + label="San Francisco/Los Angeles", + timezone_name="America/Los_Angeles", + ), + RandomWindowSessionProfileV1( + code="auk", + label="Auckland/Wellington", + timezone_name="Pacific/Auckland", + ), + RandomWindowSessionProfileV1( + code="syd", + label="Sydney", + timezone_name="Australia/Sydney", + ), + RandomWindowSessionProfileV1( + code="tyo", + label="Tokyo", + timezone_name="Asia/Tokyo", + ), + RandomWindowSessionProfileV1( + code="hk", + label="Hong Kong/Singapore", + timezone_name="Asia/Hong_Kong", + ), + ) +} + + +@dataclass(frozen=True, slots=True) +class RandomWindowExpressionV1: + """Parsed deterministic duration or session expression.""" + + expression: str + duration_count: int | None = None + duration_unit: str = "" + prefix_minutes: int = 0 + start_session: str = "" + bridge_count: int | None = None + bridge_unit: str = "" + end_session: str = "" + suffix_minutes: int = 0 + schema_version: str = RANDOM_WINDOW_EXPRESSION_SCHEMA_VERSION + + @property + def has_session(self) -> bool: + """Return whether this expression is session anchored.""" + return bool(self.start_session) + + def to_dict(self) -> dict[str, Any]: + """Return a deterministic parser payload.""" + return { + "schema_version": self.schema_version, + "expression": self.expression, + "duration_count": self.duration_count, + "duration_unit": self.duration_unit, + "prefix_minutes": self.prefix_minutes, + "start_session": self.start_session, + "bridge_count": self.bridge_count, + "bridge_unit": self.bridge_unit, + "end_session": self.end_session, + "suffix_minutes": self.suffix_minutes, + } + + +@dataclass(frozen=True, slots=True) +class RandomWindowSelectionV1: + """Compact resolved selection carried through Temporal work items.""" + + expression: str + mode: str + support_start_utc_ms: int + support_end_utc_ms: int + seed: int | None = None + selected_start_utc_ms: int | None = None + selected_end_utc_ms: int | None = None + occurrence_count: int = 1 + selection_id: str = "" + schema_version: str = RANDOM_WINDOW_SELECTION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RANDOM_WINDOW_SELECTION_SCHEMA_VERSION: + raise ValueError("unsupported random-window selection schema") + specification = parse_random_window_expression(self.expression) + if self.mode not in { + RANDOM_WINDOW_MODE_RANDOM, + RANDOM_WINDOW_MODE_ALL_SESSIONS, + }: + raise ValueError("unsupported random-window selection mode") + _validate_ms_interval( + self.support_start_utc_ms, + self.support_end_utc_ms, + "random-window support", + ) + if self.seed is not None: + _validate_seed(self.seed) + if self.mode == RANDOM_WINDOW_MODE_RANDOM: + if self.seed is None: + raise ValueError("random-window selection requires a seed") + if ( + self.selected_start_utc_ms is None + or self.selected_end_utc_ms is None + ): + raise ValueError( + "random-window selection requires one interval" + ) + _validate_ms_interval( + self.selected_start_utc_ms, + self.selected_end_utc_ms, + "selected random window", + ) + if ( + self.selected_start_utc_ms < self.support_start_utc_ms + or self.selected_end_utc_ms > self.support_end_utc_ms + ): + raise ValueError("selected random window exceeds support") + if self.occurrence_count != 1: + raise ValueError("random mode must contain one occurrence") + if specification.has_session: + _validate_session_selection( + specification, + self.selected_start_utc_ms, + self.selected_end_utc_ms, + ) + else: + _validate_duration_selection( + specification, + self.selected_start_utc_ms, + self.selected_end_utc_ms, + ) + elif ( + self.selected_start_utc_ms is not None + or self.selected_end_utc_ms is not None + ): + raise ValueError( + "all-session mode cannot persist expanded intervals" + ) + elif not specification.has_session: + raise ValueError("all-session mode requires a session expression") + if ( + isinstance(self.occurrence_count, bool) + or not isinstance(self.occurrence_count, int) + or self.occurrence_count < 1 + or self.occurrence_count > MAX_RANDOM_WINDOW_SESSION_OCCURRENCES + ): + raise ValueError("random-window occurrence count is out of bounds") + expected = _stable_id( + "random-window-selection", self.identity_payload() + ) + if self.selection_id and self.selection_id != expected: + raise ValueError( + "random-window selection ID does not match content" + ) + object.__setattr__(self, "selection_id", expected) + + def identity_payload(self) -> dict[str, Any]: + """Return fields that determine this selection.""" + return { + "schema_version": self.schema_version, + "expression": self.expression, + "mode": self.mode, + "support_start_utc_ms": self.support_start_utc_ms, + "support_end_utc_ms": self.support_end_utc_ms, + "seed": self.seed, + "selected_start_utc_ms": self.selected_start_utc_ms, + "selected_end_utc_ms": self.selected_end_utc_ms, + "occurrence_count": self.occurrence_count, + "session_profile_version": RANDOM_WINDOW_SESSION_PROFILE_VERSION, + } + + def to_dict(self) -> dict[str, Any]: + """Return compact JSON-compatible selection metadata.""" + return {**self.identity_payload(), "selection_id": self.selection_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "RandomWindowSelectionV1": + """Restore and verify a serialized selection.""" + return cls( + expression=str(data.get("expression", "")), + mode=str(data.get("mode", "")), + support_start_utc_ms=cast(int, data.get("support_start_utc_ms")), + support_end_utc_ms=cast(int, data.get("support_end_utc_ms")), + seed=cast(int | None, data.get("seed")), + selected_start_utc_ms=cast( + int | None, data.get("selected_start_utc_ms") + ), + selected_end_utc_ms=cast( + int | None, data.get("selected_end_utc_ms") + ), + occurrence_count=cast(int, data.get("occurrence_count", 0)), + selection_id=str(data.get("selection_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +def parse_random_window_expression( + expression: str, +) -> RandomWindowExpressionV1: + """Parse one documented duration or session expression.""" + normalized = str(expression).strip() + if not normalized or normalized != expression: + raise RandomWindowSyntaxError( + "random-window expression must be non-empty without outer whitespace" + ) + tokens = normalized.split("-") + if any(not token for token in tokens): + raise RandomWindowSyntaxError( + "random-window expression contains an empty token" + ) + session_positions = [ + index for index, token in enumerate(tokens) if token in _SESSION_CODES + ] + unknown = [ + token + for token in tokens + if token not in _SESSION_CODES and _duration_token(token) is None + ] + if unknown: + raise RandomWindowSyntaxError( + f"unsupported random-window token: {unknown[0]!r}" + ) + if not session_positions: + if len(tokens) != 1: + raise RandomWindowSyntaxError( + "duration-only random windows require exactly one token" + ) + count, unit = cast(tuple[int, str], _duration_token(tokens[0])) + return RandomWindowExpressionV1( + expression=normalized, + duration_count=count, + duration_unit=unit, + ) + if len(session_positions) > 2: + raise RandomWindowSyntaxError( + "session expressions support at most two session anchors" + ) + + first = session_positions[0] + last = session_positions[-1] + prefix = tokens[:first] + between = tokens[first + 1 : last] if len(session_positions) == 2 else [] + suffix = tokens[last + 1 :] + if len(prefix) > 1 or len(between) > 1 or len(suffix) > 1: + raise RandomWindowSyntaxError( + "random-window padding and bridge sections accept one token each" + ) + prefix_minutes = _padding_minutes(prefix, "prefix") + bridge_count: int | None = None + bridge_unit = "" + suffix_minutes = 0 + end_session = "" + + if len(session_positions) == 2: + end_session = tokens[last] + if between: + bridge_count, bridge_unit = _large_duration( + between[0], "session bridge" + ) + suffix_minutes = _padding_minutes(suffix, "suffix") + elif suffix: + count, unit = cast(tuple[int, str], _duration_token(suffix[0])) + if unit in _LARGE_DURATION_UNITS: + bridge_count, bridge_unit = count, unit + elif unit in _SMALL_DURATION_UNITS: + suffix_minutes = _duration_minutes(count, unit) + else: # pragma: no cover - parser unit set is closed + raise RandomWindowSyntaxError("unsupported session suffix") + + return RandomWindowExpressionV1( + expression=normalized, + prefix_minutes=prefix_minutes, + start_session=tokens[first], + bridge_count=bridge_count, + bridge_unit=bridge_unit, + end_session=end_session, + suffix_minutes=suffix_minutes, + ) + + +def random_window_requires_seed( + expression: str, + *, + start_yearmonth: str | None, + end_yearmonth: str | None, +) -> bool: + """Return whether CLI inputs resolve by seeded random selection.""" + spec = parse_random_window_expression(expression) + return not (spec.has_session and start_yearmonth and end_yearmonth) + + +def resolve_random_window_selection( + expression: str, + *, + seed: int | None, + pairs: Sequence[str], + repository_ranges: Mapping[str, Any], + start_yearmonth: str | None = None, + end_yearmonth: str | None = None, + current_yearmonth: str | None = None, +) -> RandomWindowSelectionV1: + """Resolve a selection against common repository support.""" + spec = parse_random_window_expression(expression) + support_start, support_end = _common_support_interval( + pairs, + repository_ranges=repository_ranges, + start_yearmonth=start_yearmonth, + end_yearmonth=end_yearmonth, + current_yearmonth=current_yearmonth, + ) + all_sessions = bool(spec.has_session and start_yearmonth and end_yearmonth) + if all_sessions: + occurrences = _session_candidates(spec, support_start, support_end) + if not occurrences: + raise RandomWindowSupportError( + "session expression has no occurrence inside common support" + ) + return RandomWindowSelectionV1( + expression=spec.expression, + mode=RANDOM_WINDOW_MODE_ALL_SESSIONS, + support_start_utc_ms=_to_ms(support_start), + support_end_utc_ms=_to_ms(support_end), + seed=seed, + occurrence_count=len(occurrences), + ) + + if seed is None: + raise RandomWindowSupportError( + "random-window selection requires --random-seed" + ) + _validate_seed(seed) + rng = random.Random(seed) + if spec.has_session: + candidates = _session_candidates(spec, support_start, support_end) + if not candidates: + raise RandomWindowSupportError( + "session expression has no occurrence inside common support" + ) + selected_start, selected_end = candidates[ + rng.randrange(len(candidates)) + ] + else: + selected_start, selected_end = _duration_selection( + spec, + support_start, + support_end, + rng, + ) + return RandomWindowSelectionV1( + expression=spec.expression, + mode=RANDOM_WINDOW_MODE_RANDOM, + support_start_utc_ms=_to_ms(support_start), + support_end_utc_ms=_to_ms(support_end), + seed=seed, + selected_start_utc_ms=_to_ms(selected_start), + selected_end_utc_ms=_to_ms(selected_end), + ) + + +def random_window_planning_yearmonths( + selection: RandomWindowSelectionV1, +) -> tuple[str, str]: + """Return inclusive monthly source bounds for a resolved selection.""" + if selection.mode == RANDOM_WINDOW_MODE_RANDOM: + assert selection.selected_start_utc_ms is not None + assert selection.selected_end_utc_ms is not None + start_ms = selection.selected_start_utc_ms + end_ms = selection.selected_end_utc_ms + else: + start_ms = selection.support_start_utc_ms + end_ms = selection.support_end_utc_ms + start = _from_ms(start_ms) + inclusive_end = _from_ms(end_ms - 1) + return start.strftime("%Y%m"), inclusive_end.strftime("%Y%m") + + +def random_window_selection_from_metadata( + metadata: Mapping[str, Any] | None, +) -> RandomWindowSelectionV1 | None: + """Return a verified selection from work-item metadata.""" + if not metadata: + return None + if RANDOM_WINDOW_SELECTION_METADATA_KEY not in metadata: + return None + payload = metadata[RANDOM_WINDOW_SELECTION_METADATA_KEY] + if not isinstance(payload, Mapping): + raise RandomWindowError( + "random-window work-item metadata must be a mapping" + ) + return RandomWindowSelectionV1.from_dict(payload) + + +def random_window_intervals_for_range( + selection: RandomWindowSelectionV1, + *, + range_start_utc_ms: int, + range_end_utc_ms: int, +) -> tuple[tuple[int, int], ...]: + """Return selected half-open intervals overlapping one bounded range.""" + _validate_ms_interval( + range_start_utc_ms, + range_end_utc_ms, + "random-window projection range", + ) + candidates: tuple[tuple[int, int], ...] + if selection.mode == RANDOM_WINDOW_MODE_RANDOM: + assert selection.selected_start_utc_ms is not None + assert selection.selected_end_utc_ms is not None + candidates = ( + ( + selection.selected_start_utc_ms, + selection.selected_end_utc_ms, + ), + ) + else: + candidates = _cached_all_session_intervals( + selection.expression, + selection.support_start_utc_ms, + selection.support_end_utc_ms, + ) + return tuple( + (max(start, range_start_utc_ms), min(end, range_end_utc_ms)) + for start, end in candidates + if start < range_end_utc_ms and end > range_start_utc_ms + ) + + +def filter_polars_frame_to_random_window( + frame: Any, + selection: RandomWindowSelectionV1 | Mapping[str, Any] | None, + *, + timestamp_column: str = "datetime", +) -> Any: + """Filter one eager Polars frame to the exact selected interval union.""" + if selection is None: + return frame + resolved = ( + selection + if isinstance(selection, RandomWindowSelectionV1) + else RandomWindowSelectionV1.from_dict(selection) + ) + if timestamp_column not in getattr(frame, "columns", ()): + raise RandomWindowError( + f"random-window filtering requires {timestamp_column!r}" + ) + if int(getattr(frame, "height", 0)) == 0: + return frame + import polars as pl + + bounds = frame.select( + pl.col(timestamp_column).min().alias("start"), + pl.col(timestamp_column).max().alias("end"), + ).row(0) + range_start = int(bounds[0]) + range_end = int(bounds[1]) + 1 + intervals = random_window_intervals_for_range( + resolved, + range_start_utc_ms=range_start, + range_end_utc_ms=range_end, + ) + if not intervals: + return frame.head(0) + predicates = [ + (pl.col(timestamp_column) >= start) & (pl.col(timestamp_column) < end) + for start, end in intervals + ] + return frame.filter(reduce(or_, predicates)) + + +@lru_cache(maxsize=64) +def _cached_all_session_intervals( + expression: str, + support_start_utc_ms: int, + support_end_utc_ms: int, +) -> tuple[tuple[int, int], ...]: + spec = parse_random_window_expression(expression) + candidates = _session_candidates( + spec, + _from_ms(support_start_utc_ms), + _from_ms(support_end_utc_ms), + ) + return tuple((_to_ms(start), _to_ms(end)) for start, end in candidates) + + +def _common_support_interval( + pairs: Sequence[str], + *, + repository_ranges: Mapping[str, Any], + start_yearmonth: str | None, + end_yearmonth: str | None, + current_yearmonth: str | None, +) -> tuple[datetime, datetime]: + normalized_pairs = tuple(sorted({str(pair).lower() for pair in pairs})) + if not normalized_pairs: + raise RandomWindowSupportError( + "random-window selection requires at least one symbol" + ) + user_start = _yearmonth_start(start_yearmonth) if start_yearmonth else None + user_end = _yearmonth_end(end_yearmonth) if end_yearmonth else None + current_end = ( + _yearmonth_end(current_yearmonth) if current_yearmonth else None + ) + starts: list[datetime] = [] + ends: list[datetime] = [] + inventory_available = bool(repository_ranges) + for pair in normalized_pairs: + pair_range = repository_ranges.get(pair) + repo_start: datetime | None = None + repo_end: datetime | None = None + if isinstance(pair_range, Mapping): + if pair_range.get("start"): + repo_start = _yearmonth_start(str(pair_range["start"])) + if pair_range.get("end"): + repo_end = _yearmonth_end(str(pair_range["end"])) + if repo_start is None or repo_end is None: + if inventory_available: + raise RandomWindowSupportError( + f"repository support is unavailable for {pair}" + ) + if user_start is None or user_end is None: + raise RandomWindowSupportError( + f"repository support is unavailable for {pair}" + ) + repo_start, repo_end = user_start, user_end + starts.append(max(item for item in (repo_start, user_start) if item)) + pair_ends = [item for item in (repo_end, user_end, current_end) if item] + ends.append(min(pair_ends)) + support_start = max(starts) + support_end = min(ends) + if support_start >= support_end: + raise RandomWindowSupportError( + "requested symbols and bounds have no common repository support" + ) + return support_start, support_end + + +def _duration_selection( + spec: RandomWindowExpressionV1, + support_start: datetime, + support_end: datetime, + rng: random.Random, +) -> tuple[datetime, datetime]: + assert spec.duration_count is not None + count = spec.duration_count + unit = spec.duration_unit + if unit in {"y", "q", "M"}: + month_step = {"y": 12, "q": 3, "M": 1}[unit] + duration_months = count * month_step + starts: list[datetime] = [] + cursor = datetime( + support_start.year, + support_start.month, + 1, + tzinfo=timezone.utc, + ) + if unit == "y": + cursor = datetime(cursor.year, 1, 1, tzinfo=timezone.utc) + if cursor < support_start: + cursor = datetime(cursor.year + 1, 1, 1, tzinfo=timezone.utc) + elif unit == "q": + quarter_month = ((cursor.month - 1) // 3) * 3 + 1 + cursor = datetime( + cursor.year, quarter_month, 1, tzinfo=timezone.utc + ) + if cursor < support_start: + cursor = _add_months(cursor, 3) + elif cursor < support_start: + cursor = _add_months(cursor, 1) + while True: + end = _add_months(cursor, duration_months) + if end > support_end: + break + starts.append(cursor) + cursor = _add_months(cursor, month_step) + if not starts: + raise RandomWindowSupportError( + "calendar-aligned duration does not fit inside common support" + ) + start = starts[rng.randrange(len(starts))] + return start, _add_months(start, duration_months) + + duration_minutes = _duration_minutes(count, unit) + duration = timedelta(minutes=duration_minutes) + latest_start = support_end - duration + if latest_start < support_start: + raise RandomWindowSupportError( + "random-window duration exceeds common repository support" + ) + slots = int((latest_start - support_start).total_seconds() // 60) + 1 + start_index = rng.randrange(slots) + search_limit = min(slots, 7 * 24 * 60) + for offset in range(search_limit): + start = support_start + timedelta( + minutes=(start_index + offset) % slots + ) + if start.weekday() < 5: + return start, start + duration + raise RandomWindowSupportError( + "duration selection has no weekday-aligned candidate" + ) + + +def _session_candidates( + spec: RandomWindowExpressionV1, + support_start: datetime, + support_end: datetime, +) -> tuple[tuple[datetime, datetime], ...]: + if not spec.has_session: + raise RandomWindowSyntaxError( + "session candidate generation requires a session expression" + ) + profile = RANDOM_WINDOW_SESSION_PROFILES[spec.start_session] + zone = ZoneInfo(profile.timezone_name) + cursor = support_start.astimezone(zone).date() - timedelta(days=2) + last = support_end.astimezone(zone).date() + timedelta(days=2) + candidates: list[tuple[datetime, datetime]] = [] + while cursor <= last: + if cursor.weekday() < 5: + interval = _session_interval_for_date(spec, cursor) + if interval[0] >= support_start and interval[1] <= support_end: + candidates.append(interval) + if len(candidates) > MAX_RANDOM_WINDOW_SESSION_OCCURRENCES: + raise RandomWindowSupportError( + "session selection exceeds occurrence resource limit" + ) + cursor += timedelta(days=1) + return tuple(candidates) + + +def _session_interval_for_date( + spec: RandomWindowExpressionV1, + local_date: date, +) -> tuple[datetime, datetime]: + start_profile = RANDOM_WINDOW_SESSION_PROFILES[spec.start_session] + start_open, start_close = _session_bounds(start_profile, local_date) + start = start_open - timedelta(minutes=spec.prefix_minutes) + if spec.end_session: + base = ( + _add_duration(start_open, spec.bridge_count, spec.bridge_unit) + if spec.bridge_count is not None + else start_open + ) + end_profile = RANDOM_WINDOW_SESSION_PROFILES[spec.end_session] + end_zone = ZoneInfo(end_profile.timezone_name) + target_date = base.astimezone(end_zone).date() + end_open, end_close = _session_bounds(end_profile, target_date) + if spec.bridge_count is None: + if spec.end_session == spec.start_session or end_open <= start_open: + target_date += timedelta(days=1) + end_open, end_close = _session_bounds(end_profile, target_date) + while end_close <= base: + target_date += timedelta(days=1) + end_open, end_close = _session_bounds(end_profile, target_date) + end = end_close + elif spec.bridge_count is not None: + end = _add_duration(start_open, spec.bridge_count, spec.bridge_unit) + else: + end = start_close + end += timedelta(minutes=spec.suffix_minutes) + start_utc = start.astimezone(timezone.utc) + end_utc = end.astimezone(timezone.utc) + if start_utc >= end_utc: + raise RandomWindowSyntaxError( + "session expression resolves to an empty or reversed interval" + ) + return start_utc, end_utc + + +def _session_bounds( + profile: RandomWindowSessionProfileV1, + local_date: date, +) -> tuple[datetime, datetime]: + zone = ZoneInfo(profile.timezone_name) + start = datetime.combine( + local_date, + time( + hour=profile.start_minute_local // 60, + minute=profile.start_minute_local % 60, + ), + tzinfo=zone, + ) + end_date = local_date + if profile.end_minute_local < profile.start_minute_local: + end_date += timedelta(days=1) + end = datetime.combine( + end_date, + time( + hour=profile.end_minute_local // 60, + minute=profile.end_minute_local % 60, + ), + tzinfo=zone, + ) + return start, end + + +def _add_duration( + value: datetime, + count: int | None, + unit: str, +) -> datetime: + if count is None: + return value + if unit == "y": + return _add_months(value, count * 12) + if unit == "q": + return _add_months(value, count * 3) + if unit == "M": + return _add_months(value, count) + return value + timedelta(minutes=_duration_minutes(count, unit)) + + +def _add_months(value: datetime, months: int) -> datetime: + month_index = value.year * 12 + value.month - 1 + months + year, zero_month = divmod(month_index, 12) + month = zero_month + 1 + day = min(value.day, calendar.monthrange(year, month)[1]) + return value.replace(year=year, month=month, day=day) + + +def _duration_token(token: str) -> tuple[int, str] | None: + match = _DURATION_RE.fullmatch(token) + if match is None: + return None + count = int(match.group(1)) + if count > MAX_RANDOM_WINDOW_DURATION_COUNT: + raise RandomWindowSyntaxError( + "random-window duration count exceeds resource limit" + ) + return count, match.group(2) + + +def _large_duration(token: str, name: str) -> tuple[int, str]: + count, unit = cast(tuple[int, str], _duration_token(token)) + if unit not in _LARGE_DURATION_UNITS: + raise RandomWindowSyntaxError(f"{name} requires d, w, M, q, or y") + return count, unit + + +def _padding_minutes(tokens: Sequence[str], name: str) -> int: + if not tokens: + return 0 + count, unit = cast(tuple[int, str], _duration_token(tokens[0])) + if unit not in _SMALL_DURATION_UNITS: + raise RandomWindowSyntaxError(f"{name} padding requires h or m") + return _duration_minutes(count, unit) + + +def _duration_minutes(count: int, unit: str) -> int: + factors = { + "w": 7 * 24 * 60, + "d": 24 * 60, + "h": 60, + "m": 1, + } + try: + return count * factors[unit] + except KeyError as err: + raise RandomWindowSyntaxError( + f"{unit!r} is not a fixed-duration unit" + ) from err + + +def _yearmonth_start(value: str | None) -> datetime: + normalized = _normalize_yearmonth(value, end=False) + return datetime( + int(normalized[:4]), + int(normalized[4:]), + 1, + tzinfo=timezone.utc, + ) + + +def _yearmonth_end(value: str | None) -> datetime: + normalized = _normalize_yearmonth(value, end=True) + start = datetime( + int(normalized[:4]), + int(normalized[4:]), + 1, + tzinfo=timezone.utc, + ) + return _add_months(start, 1) + + +def _normalize_yearmonth(value: str | None, *, end: bool) -> str: + normalized = str(value or "").replace("-", "") + if len(normalized) == 4 and normalized.isdigit(): + return f"{normalized}{'12' if end else '01'}" + if len(normalized) != 6 or not normalized.isdigit(): + raise RandomWindowSupportError( + "random-window bounds require YYYY or YYYYMM values" + ) + month = int(normalized[4:]) + if not 1 <= month <= 12: + raise RandomWindowSupportError( + "random-window bound month must be between 01 and 12" + ) + return normalized + + +def _validate_seed(seed: int) -> None: + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ValueError("random-window seed must be a non-negative integer") + if seed > 2**63 - 1: + raise ValueError("random-window seed exceeds signed int64 range") + + +def _validate_ms_interval(start: int, end: int, name: str) -> None: + for field_name, value in (("start", start), ("end", end)): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} {field_name} must be an integer") + if value < 0 or value > 2**63 - 1: + raise ValueError(f"{name} {field_name} exceeds int64 range") + if start >= end: + raise ValueError(f"{name} must be a non-empty half-open interval") + + +def _validate_duration_selection( + specification: RandomWindowExpressionV1, + start_ms: int, + end_ms: int, +) -> None: + """Verify a serialized duration selection still matches its expression.""" + assert specification.duration_count is not None + count = specification.duration_count + unit = specification.duration_unit + start = _from_ms(start_ms) + end = _from_ms(end_ms) + if unit in {"y", "q", "M"}: + month_step = {"y": 12, "q": 3, "M": 1}[unit] + if start != datetime( + start.year, + start.month, + 1, + tzinfo=timezone.utc, + ): + raise ValueError("calendar random window is not month aligned") + if unit == "y" and start.month != 1: + raise ValueError("calendar-year random window is not year aligned") + if unit == "q" and start.month not in {1, 4, 7, 10}: + raise ValueError("calendar-quarter random window is not aligned") + expected_end = _add_months(start, count * month_step) + else: + if start_ms % MILLISECONDS_PER_MINUTE: + raise ValueError("fixed random window is not UTC-minute aligned") + expected_end = start + timedelta(minutes=_duration_minutes(count, unit)) + if end != expected_end: + raise ValueError("selected random window does not match its expression") + + +def _validate_session_selection( + specification: RandomWindowExpressionV1, + start_ms: int, + end_ms: int, +) -> None: + """Verify a compact session selection against its local-clock profile.""" + profile = RANDOM_WINDOW_SESSION_PROFILES[specification.start_session] + local_date = ( + (_from_ms(start_ms) + timedelta(minutes=specification.prefix_minutes)) + .astimezone(ZoneInfo(profile.timezone_name)) + .date() + ) + if local_date.weekday() >= 5: + raise ValueError("selected session random window starts on a weekend") + expected_start, expected_end = _session_interval_for_date( + specification, + local_date, + ) + if (start_ms, end_ms) != (_to_ms(expected_start), _to_ms(expected_end)): + raise ValueError( + "selected session window does not match its expression" + ) + + +def _to_ms(value: datetime) -> int: + return int(value.timestamp() * 1000) + + +def _from_ms(value: int) -> datetime: + return datetime.fromtimestamp(value / 1000, tz=timezone.utc) + + +def _stable_id(prefix: str, payload: Mapping[str, Any]) -> str: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +__all__ = [ + "MAX_RANDOM_WINDOW_SESSION_OCCURRENCES", + "RANDOM_WINDOW_EXPRESSION_SCHEMA_VERSION", + "RANDOM_WINDOW_MODE_ALL_SESSIONS", + "RANDOM_WINDOW_MODE_RANDOM", + "RANDOM_WINDOW_SELECTION_METADATA_KEY", + "RANDOM_WINDOW_SELECTION_SCHEMA_VERSION", + "RANDOM_WINDOW_SESSION_PROFILES", + "RANDOM_WINDOW_SESSION_PROFILE_VERSION", + "RandomWindowEmptySelectionError", + "RandomWindowError", + "RandomWindowExpressionV1", + "RandomWindowSelectionV1", + "RandomWindowSessionProfileV1", + "RandomWindowSupportError", + "RandomWindowSyntaxError", + "filter_polars_frame_to_random_window", + "parse_random_window_expression", + "random_window_intervals_for_range", + "random_window_planning_yearmonths", + "random_window_requires_seed", + "random_window_selection_from_metadata", + "resolve_random_window_selection", +] diff --git a/src/histdatacom/reconstruction.py b/src/histdatacom/reconstruction.py new file mode 100644 index 00000000..5d7ef670 --- /dev/null +++ b/src/histdatacom/reconstruction.py @@ -0,0 +1,2134 @@ +"""Typed public facade for first-party reconstruction operations. + +The facade keeps operator intent, scientific plan identity, orchestration +control, and product inspection at one supported import boundary. Tick rows +remain in Arrow/Parquet artifacts; public requests and receipts carry only +bounded metadata and strong references. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, replace +from datetime import datetime, timezone +from enum import IntEnum +import hashlib +import json +from pathlib import Path +from typing import Any, cast + +from histdatacom.manifest_store import ManifestStatusStore +from histdatacom.orchestration.client import ( + OrchestrationJobHandle, + cancel_job, + get_job_result, + inspect_job_status, + submit_reconstruction_request, +) +from histdatacom.orchestration.queues import OrchestrationWorkerConfig +from histdatacom.orchestration.reconstruction import ( + ReconstructionRunReportV1, + ReconstructionWorkflowRequestV1, + artifact_ref_for_file, + reconcile_reconstruction_report, + run_reconstruction_request, + verify_artifact_ref, + write_reconstruction_report, +) +from histdatacom.orchestration.supervisor import OrchestrationSupervisor +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.certification import ( + ReconstructionCertificationDossierV2, +) +from histdatacom.synthetic.certification_campaign import ( + ModernReferenceCertificationCampaignResultV1, + ModernReferenceCertificationCampaignSpecV1, + read_modern_reference_certification_campaign_spec, + run_modern_reference_certification_campaign, +) +from histdatacom.synthetic.contracts import canonical_contract_json +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.persistence import ( + discover_reconstruction_manifests, + iter_reconstruction_event_batches, + load_reconstruction_manifest, + read_reconstruction_streams, + verify_reconstruction_publication, +) +from histdatacom.synthetic.reconstruction_handlers import ( + register_first_party_reconstruction_handlers, +) +from histdatacom.synthetic.reconstruction_plan import ( + DEFAULT_RECONSTRUCTION_WINDOW_SIZE_NS, + SCIENTIFIC_NONCLAIM, + ReconstructionDeliveryMode, + ReconstructionPlanResourceSummaryV1, + SyntheticInfillPlanV1, + build_synthetic_infill_plan, + read_reconstruction_plan_execution_manifest, + read_reconstruction_source_inventory, + read_synthetic_infill_plan, + validate_synthetic_infill_plan_for_execution, + write_synthetic_infill_plan, +) + +RECONSTRUCTION_PLAN_SPEC_SCHEMA_VERSION = ( + "histdatacom.reconstruction-plan-spec.v1" +) +RECONSTRUCTION_PLAN_SET_SCHEMA_VERSION = ( + "histdatacom.reconstruction-plan-set.v1" +) +RECONSTRUCTION_PLAN_SET_PREFLIGHT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-plan-set-preflight.v1" +) +RECONSTRUCTION_PLAN_SHARD_SCHEMA_VERSION = ( + "histdatacom.reconstruction-plan-shard.v1" +) +RECONSTRUCTION_EXECUTION_REQUEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-execution-request.v1" +) +RECONSTRUCTION_PREFLIGHT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-preflight.v1" +) +RECONSTRUCTION_RECEIPT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-operation-receipt.v1" +) +RECONSTRUCTION_OUTPUT_LIST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-output-list.v1" +) +RECONSTRUCTION_PREVIEW_SCHEMA_VERSION = "histdatacom.reconstruction-preview.v1" +RECONSTRUCTION_REPLAY_SCHEMA_VERSION = "histdatacom.reconstruction-replay.v1" + +RECONSTRUCTION_SYMBOLS = ("eurgbp", "eurusd", "gbpusd") +RECONSTRUCTION_SOURCE_FORMAT = "ascii" +RECONSTRUCTION_TIMEFRAME = "T" +DEFAULT_PREVIEW_LIMIT = 20 +MAX_PREVIEW_LIMIT = 100 +DEFAULT_PLAN_SET_PERIODS_PER_SHARD = 12 +MAX_PLAN_SET_PERIODS_PER_SHARD = 24 +MAX_RECONSTRUCTION_PLAN_SHARDS = 4096 + + +class ReconstructionExitCode(IntEnum): + """Stable CLI outcome categories for public reconstruction commands.""" + + SUCCESS = 0 + INVALID_PLAN = 2 + REFUSED = 3 + RUNTIME_FAILURE = 4 + VALIDATION_FAILURE = 5 + + +class ReconstructionPublicError(RuntimeError): + """Base error carrying a stable machine-readable public reason code.""" + + reason_code = "reconstruction_error" + exit_code = ReconstructionExitCode.RUNTIME_FAILURE + + +class ReconstructionUnsupportedError(ReconstructionPublicError): + """The requested public source, timeframe, symbol set, or mode is invalid.""" + + reason_code = "unsupported_reconstruction_request" + exit_code = ReconstructionExitCode.INVALID_PLAN + + +class ReconstructionPlanError(ReconstructionPublicError): + """The bound plan is missing, changed, malformed, or not executable.""" + + reason_code = "invalid_reconstruction_plan" + exit_code = ReconstructionExitCode.INVALID_PLAN + + +class ReconstructionRefusedError(ReconstructionPublicError): + """Declared scientific or resource policy refuses execution.""" + + reason_code = "reconstruction_refused" + exit_code = ReconstructionExitCode.REFUSED + + +class ReconstructionValidationError(ReconstructionPublicError): + """Executed output did not reach a fully committed validated state.""" + + reason_code = "reconstruction_validation_failed" + exit_code = ReconstructionExitCode.VALIDATION_FAILURE + + +@dataclass(frozen=True, slots=True) +class ReconstructionPlanSpecV1: + """Serializable public inputs for constructing one first-party plan.""" + + source_root: str + feed_epoch_definition_path: str + observation_operator_path: str + market_context_corpus_path: str + cftc_positioning_corpus_path: str + benchmark_manifest_path: str + motif_manifest_path: str + motif_index_path: str + motif_qualification_path: str + motif_leakage_audit_path: str + artifact_root: str + output_root: str + checkpoint_root: str + scratch_root: str + information_mode: InformationMode + start_period: str | None = None + end_period: str | None = None + requested_start_ns: int | None = None + requested_end_ns: int | None = None + window_size_ns: int = DEFAULT_RECONSTRUCTION_WINDOW_SIZE_NS + delivery_mode: ReconstructionDeliveryMode = ( + ReconstructionDeliveryMode.MODERN_REFERENCE + ) + broker_delivery_artifact: ArtifactRef | None = None + source_format: str = RECONSTRUCTION_SOURCE_FORMAT + timeframe: str = RECONSTRUCTION_TIMEFRAME + symbols: tuple[str, ...] = RECONSTRUCTION_SYMBOLS + schema_version: str = RECONSTRUCTION_PLAN_SPEC_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_PLAN_SPEC_SCHEMA_VERSION: + raise ReconstructionUnsupportedError( + "unsupported reconstruction plan-spec schema" + ) + for name in ( + "source_root", + "feed_epoch_definition_path", + "observation_operator_path", + "market_context_corpus_path", + "cftc_positioning_corpus_path", + "benchmark_manifest_path", + "motif_manifest_path", + "motif_index_path", + "motif_qualification_path", + "motif_leakage_audit_path", + "artifact_root", + "output_root", + "checkpoint_root", + "scratch_root", + ): + value = str(Path(_required_text(getattr(self, name))).expanduser()) + object.__setattr__(self, name, value) + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + object.__setattr__( + self, + "delivery_mode", + ReconstructionDeliveryMode.from_value(self.delivery_mode), + ) + _validate_public_input_contract( + source_format=self.source_format, + timeframe=self.timeframe, + symbols=self.symbols, + ) + object.__setattr__(self, "source_format", RECONSTRUCTION_SOURCE_FORMAT) + object.__setattr__(self, "timeframe", RECONSTRUCTION_TIMEFRAME) + object.__setattr__(self, "symbols", RECONSTRUCTION_SYMBOLS) + requested_start = self.requested_start_ns + requested_end = self.requested_end_ns + exact_bounds = (requested_start, requested_end) + if (requested_start is None) != (requested_end is None): + raise ReconstructionUnsupportedError( + "requested_start_ns and requested_end_ns must be supplied together" + ) + if requested_start is not None and requested_end is not None: + if any( + isinstance(value, bool) or not isinstance(value, int) + for value in exact_bounds + ): + raise ReconstructionUnsupportedError( + "requested nanosecond bounds must be integers" + ) + if requested_end <= requested_start: + raise ReconstructionUnsupportedError( + "requested nanosecond interval must be nonempty" + ) + if ( + isinstance(self.window_size_ns, bool) + or not isinstance(self.window_size_ns, int) + or self.window_size_ns <= 0 + ): + raise ReconstructionUnsupportedError( + "window_size_ns must be a positive integer" + ) + if ( + self.delivery_mode is ReconstructionDeliveryMode.BROKER_CONDITIONED + and self.broker_delivery_artifact is None + ): + raise ReconstructionUnsupportedError( + "broker-conditioned delivery requires broker_delivery_artifact" + ) + if ( + self.delivery_mode is ReconstructionDeliveryMode.MODERN_REFERENCE + and self.broker_delivery_artifact is not None + ): + raise ReconstructionUnsupportedError( + "modern-reference delivery rejects broker_delivery_artifact" + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return machine-readable planning metadata without row payloads.""" + return { + "schema_version": self.schema_version, + "source_root": self.source_root, + "feed_epoch_definition_path": self.feed_epoch_definition_path, + "observation_operator_path": self.observation_operator_path, + "market_context_corpus_path": self.market_context_corpus_path, + "cftc_positioning_corpus_path": self.cftc_positioning_corpus_path, + "benchmark_manifest_path": self.benchmark_manifest_path, + "motif_manifest_path": self.motif_manifest_path, + "motif_index_path": self.motif_index_path, + "motif_qualification_path": self.motif_qualification_path, + "motif_leakage_audit_path": self.motif_leakage_audit_path, + "artifact_root": self.artifact_root, + "output_root": self.output_root, + "checkpoint_root": self.checkpoint_root, + "scratch_root": self.scratch_root, + "information_mode": self.information_mode.value, + "start_period": self.start_period, + "end_period": self.end_period, + "requested_start_ns": self.requested_start_ns, + "requested_end_ns": self.requested_end_ns, + "window_size_ns": self.window_size_ns, + "delivery_mode": self.delivery_mode.value, + "broker_delivery_artifact": ( + self.broker_delivery_artifact.to_dict() + if self.broker_delivery_artifact is not None + else None + ), + "source_format": self.source_format, + "timeframe": self.timeframe, + "symbols": list(self.symbols), + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionPlanSpecV1": + """Restore a strict public plan specification.""" + broker_payload = data.get("broker_delivery_artifact") + broker_ref = ( + ArtifactRef.from_dict(_mapping(broker_payload)) + if broker_payload is not None + else None + ) + return cls( + source_root=str(data.get("source_root", "")), + feed_epoch_definition_path=str( + data.get("feed_epoch_definition_path", "") + ), + observation_operator_path=str( + data.get("observation_operator_path", "") + ), + market_context_corpus_path=str( + data.get("market_context_corpus_path", "") + ), + cftc_positioning_corpus_path=str( + data.get("cftc_positioning_corpus_path", "") + ), + benchmark_manifest_path=str( + data.get("benchmark_manifest_path", "") + ), + motif_manifest_path=str(data.get("motif_manifest_path", "")), + motif_index_path=str(data.get("motif_index_path", "")), + motif_qualification_path=str( + data.get("motif_qualification_path", "") + ), + motif_leakage_audit_path=str( + data.get("motif_leakage_audit_path", "") + ), + artifact_root=str(data.get("artifact_root", "")), + output_root=str(data.get("output_root", "")), + checkpoint_root=str(data.get("checkpoint_root", "")), + scratch_root=str(data.get("scratch_root", "")), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + start_period=_optional_text(data.get("start_period")), + end_period=_optional_text(data.get("end_period")), + requested_start_ns=( + cast(int, data["requested_start_ns"]) + if data.get("requested_start_ns") is not None + else None + ), + requested_end_ns=( + cast(int, data["requested_end_ns"]) + if data.get("requested_end_ns") is not None + else None + ), + window_size_ns=int( + data.get( + "window_size_ns", + DEFAULT_RECONSTRUCTION_WINDOW_SIZE_NS, + ) + ), + delivery_mode=ReconstructionDeliveryMode.from_value( + str(data.get("delivery_mode", "modern_reference")) + ), + broker_delivery_artifact=broker_ref, + source_format=str(data.get("source_format", "ascii")), + timeframe=str(data.get("timeframe", "T")), + symbols=tuple( + str(value) + for value in _sequence( + data.get("symbols", RECONSTRUCTION_SYMBOLS) + ) + ), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionPlanShardV1: + """One bounded executable plan in a contiguous full-range plan set.""" + + start_period: str + end_period: str + requested_start_ns: int + requested_end_ns: int + plan_id: str + plan_ref: ArtifactRef + preflight_status: str + executable: bool + refusal_count: int + resource_summary: Mapping[str, JSONValue] + shard_id: str = "" + schema_version: str = RECONSTRUCTION_PLAN_SHARD_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_PLAN_SHARD_SCHEMA_VERSION: + raise ReconstructionPlanError( + "unsupported reconstruction plan shard" + ) + start = _period(self.start_period) + end = _period(self.end_period) + if start > end: + raise ReconstructionPlanError("plan shard period range is reversed") + object.__setattr__(self, "start_period", start) + object.__setattr__(self, "end_period", end) + if ( + isinstance(self.requested_start_ns, bool) + or not isinstance(self.requested_start_ns, int) + or isinstance(self.requested_end_ns, bool) + or not isinstance(self.requested_end_ns, int) + or self.requested_end_ns <= self.requested_start_ns + ): + raise ReconstructionPlanError( + "plan shard nanosecond range is invalid" + ) + object.__setattr__(self, "plan_id", _required_text(self.plan_id)) + if self.plan_ref.kind != "synthetic_infill_plan_v1": + raise ReconstructionPlanError("plan shard artifact kind differs") + status = _required_text(self.preflight_status) + if status not in {"ready", "ready_with_refusals", "refused"}: + raise ReconstructionPlanError("plan shard preflight status differs") + object.__setattr__(self, "preflight_status", status) + if not isinstance(self.executable, bool): + raise ReconstructionPlanError( + "plan shard executable must be boolean" + ) + if ( + isinstance(self.refusal_count, bool) + or not isinstance(self.refusal_count, int) + or self.refusal_count < 0 + ): + raise ReconstructionPlanError("plan shard refusal count is invalid") + resources = { + str(key): value + for key, value in sorted(self.resource_summary.items()) + } + object.__setattr__(self, "resource_summary", resources) + expected = _stable_id( + "reconstruction-plan-shard", self.identity_payload() + ) + if self.shard_id and self.shard_id != expected: + raise ReconstructionPlanError( + "reconstruction plan shard identity differs" + ) + object.__setattr__(self, "shard_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return stable shard content without the derived identity.""" + return { + "schema_version": self.schema_version, + "start_period": self.start_period, + "end_period": self.end_period, + "requested_start_ns": self.requested_start_ns, + "requested_end_ns": self.requested_end_ns, + "plan_id": self.plan_id, + "plan_ref": self.plan_ref.to_dict(), + "preflight_status": self.preflight_status, + "executable": self.executable, + "refusal_count": self.refusal_count, + "resource_summary": dict(self.resource_summary), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded machine-readable shard metadata.""" + return {**self.identity_payload(), "shard_id": self.shard_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionPlanShardV1": + """Restore and identity-check one plan shard.""" + return cls( + start_period=str(data.get("start_period", "")), + end_period=str(data.get("end_period", "")), + requested_start_ns=_strict_int( + data.get("requested_start_ns"), "requested_start_ns" + ), + requested_end_ns=_strict_int( + data.get("requested_end_ns"), "requested_end_ns" + ), + plan_id=str(data.get("plan_id", "")), + plan_ref=ArtifactRef.from_dict(_mapping(data.get("plan_ref"))), + preflight_status=str(data.get("preflight_status", "")), + executable=_strict_bool(data.get("executable"), "executable"), + refusal_count=_strict_int( + data.get("refusal_count"), "refusal_count" + ), + resource_summary=_mapping(data.get("resource_summary")), + shard_id=str(data.get("shard_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionPlanSetV1: + """Content-addressed full-range plan composed of bounded plan shards.""" + + source_spec: ReconstructionPlanSpecV1 + shards: tuple[ReconstructionPlanShardV1, ...] + requested_start_ns: int + requested_end_ns: int + resource_summary: Mapping[str, JSONValue] + status: str + plan_set_id: str = "" + schema_version: str = RECONSTRUCTION_PLAN_SET_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_PLAN_SET_SCHEMA_VERSION: + raise ReconstructionPlanError("unsupported reconstruction plan set") + shards = tuple( + sorted(self.shards, key=lambda item: item.requested_start_ns) + ) + if not shards or len(shards) > MAX_RECONSTRUCTION_PLAN_SHARDS: + raise ReconstructionPlanError( + "plan set shard count is outside limits" + ) + if len({item.shard_id for item in shards}) != len(shards): + raise ReconstructionPlanError("plan set contains duplicate shards") + for previous, current in zip(shards, shards[1:], strict=False): + if previous.requested_end_ns != current.requested_start_ns: + raise ReconstructionPlanError( + "plan set shards are not contiguous" + ) + if ( + self.requested_start_ns != shards[0].requested_start_ns + or self.requested_end_ns != shards[-1].requested_end_ns + ): + raise ReconstructionPlanError( + "plan set bounds differ from its shards" + ) + if ( + self.source_spec.start_period != shards[0].start_period + or self.source_spec.end_period != shards[-1].end_period + ): + raise ReconstructionPlanError( + "plan set periods differ from source spec" + ) + object.__setattr__(self, "shards", shards) + resources = { + str(key): value + for key, value in sorted(self.resource_summary.items()) + } + object.__setattr__(self, "resource_summary", resources) + expected_status = ( + "refused" + if any(not item.executable for item in shards) + else ( + "ready_with_refusals" + if any(item.refusal_count for item in shards) + else "ready" + ) + ) + if self.status != expected_status: + raise ReconstructionPlanError("plan set status differs from shards") + expected = _stable_id( + "reconstruction-plan-set", self.identity_payload() + ) + if self.plan_set_id and self.plan_set_id != expected: + raise ReconstructionPlanError( + "reconstruction plan-set identity differs" + ) + object.__setattr__(self, "plan_set_id", expected) + + @property + def executable(self) -> bool: + """Return whether every bounded shard can execute its supported windows.""" + return all(item.executable for item in self.shards) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return stable plan-set content without the derived identity.""" + return { + "schema_version": self.schema_version, + "source_spec": self.source_spec.to_dict(), + "shards": [item.to_dict() for item in self.shards], + "requested_start_ns": self.requested_start_ns, + "requested_end_ns": self.requested_end_ns, + "resource_summary": dict(self.resource_summary), + "status": self.status, + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded machine-readable full-range planning evidence.""" + return {**self.identity_payload(), "plan_set_id": self.plan_set_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionPlanSetV1": + """Restore and identity-check one full-range plan set.""" + if data.get("scientific_nonclaim") != SCIENTIFIC_NONCLAIM: + raise ReconstructionPlanError( + "plan set scientific nonclaim differs" + ) + return cls( + source_spec=ReconstructionPlanSpecV1.from_dict( + _mapping(data.get("source_spec")) + ), + shards=tuple( + ReconstructionPlanShardV1.from_dict(_mapping(item)) + for item in _sequence(data.get("shards")) + ), + requested_start_ns=_strict_int( + data.get("requested_start_ns"), "requested_start_ns" + ), + requested_end_ns=_strict_int( + data.get("requested_end_ns"), "requested_end_ns" + ), + resource_summary=_mapping(data.get("resource_summary")), + status=str(data.get("status", "")), + plan_set_id=str(data.get("plan_set_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionPlanSetPreflightV1: + """Fresh public verification of every shard in a plan set.""" + + plan_set_id: str + status: str + executable: bool + shard_count: int + verified_shard_count: int + refusal_count: int + resource_summary: Mapping[str, JSONValue] + shard_preflights: tuple[Mapping[str, JSONValue], ...] + schema_version: str = RECONSTRUCTION_PLAN_SET_PREFLIGHT_SCHEMA_VERSION + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded public full-range preflight evidence.""" + return { + "schema_version": self.schema_version, + "plan_set_id": self.plan_set_id, + "status": self.status, + "executable": self.executable, + "shard_count": self.shard_count, + "verified_shard_count": self.verified_shard_count, + "refusal_count": self.refusal_count, + "resource_summary": dict(self.resource_summary), + "shard_preflights": [dict(item) for item in self.shard_preflights], + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + } + + +@dataclass(frozen=True, slots=True) +class ReconstructionExecutionRequestV1: + """Operator intent bound to one immutable reconstruction plan artifact.""" + + plan_path: str + plan_id: str + information_mode: InformationMode + scientific_nonclaim_acknowledged: bool + source_format: str = RECONSTRUCTION_SOURCE_FORMAT + timeframe: str = RECONSTRUCTION_TIMEFRAME + symbols: tuple[str, ...] = RECONSTRUCTION_SYMBOLS + allow_refusals: bool = False + request_id: str = "" + schema_version: str = RECONSTRUCTION_EXECUTION_REQUEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_EXECUTION_REQUEST_SCHEMA_VERSION + ): + raise ReconstructionUnsupportedError( + "unsupported reconstruction execution-request schema" + ) + object.__setattr__( + self, + "plan_path", + str(Path(_required_text(self.plan_path)).expanduser().resolve()), + ) + object.__setattr__(self, "plan_id", _required_text(self.plan_id)) + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + if not self.scientific_nonclaim_acknowledged: + raise ReconstructionRefusedError( + "scientific nonclaim acknowledgement is required" + ) + _validate_public_input_contract( + source_format=self.source_format, + timeframe=self.timeframe, + symbols=self.symbols, + ) + object.__setattr__(self, "source_format", RECONSTRUCTION_SOURCE_FORMAT) + object.__setattr__(self, "timeframe", RECONSTRUCTION_TIMEFRAME) + object.__setattr__(self, "symbols", RECONSTRUCTION_SYMBOLS) + expected = _stable_id( + "reconstruction-execution-request", self.identity_payload() + ) + if self.request_id and self.request_id != expected: + raise ReconstructionPlanError( + "reconstruction execution request identity differs" + ) + object.__setattr__(self, "request_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the exact operator and plan inputs bound by request_id.""" + return { + "schema_version": self.schema_version, + "plan_path": self.plan_path, + "plan_id": self.plan_id, + "information_mode": self.information_mode.value, + "source_format": self.source_format, + "timeframe": self.timeframe, + "symbols": list(self.symbols), + "allow_refusals": self.allow_refusals, + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + "scientific_nonclaim_acknowledged": True, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded machine-readable operator metadata.""" + return {**self.identity_payload(), "request_id": self.request_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionExecutionRequestV1": + """Restore and identity-check an operator execution request.""" + nonclaim = str(data.get("scientific_nonclaim", "")) + if nonclaim != SCIENTIFIC_NONCLAIM: + raise ReconstructionRefusedError( + "execution request scientific nonclaim text differs" + ) + return cls( + plan_path=str(data.get("plan_path", "")), + plan_id=str(data.get("plan_id", "")), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + scientific_nonclaim_acknowledged=_strict_bool( + data.get("scientific_nonclaim_acknowledged"), + "scientific_nonclaim_acknowledged", + ), + source_format=str(data.get("source_format", "")), + timeframe=str(data.get("timeframe", "")), + symbols=tuple( + str(value) for value in _sequence(data.get("symbols")) + ), + allow_refusals=_strict_bool( + data.get("allow_refusals", False), "allow_refusals" + ), + request_id=str(data.get("request_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionPreflightV1: + """Bounded readiness, refusal, resource, and evidence decision.""" + + request_id: str + plan_id: str + status: str + executable: bool + plan_status: str + dry_run: Mapping[str, JSONValue] + evidence_refs: Mapping[str, ArtifactRef] + refusal_reasons: tuple[Mapping[str, JSONValue], ...] = () + schema_version: str = RECONSTRUCTION_PREFLIGHT_SCHEMA_VERSION + + def to_dict(self) -> dict[str, JSONValue]: + """Return a public preflight report.""" + return { + "schema_version": self.schema_version, + "request_id": self.request_id, + "plan_id": self.plan_id, + "status": self.status, + "executable": self.executable, + "plan_status": self.plan_status, + "dry_run": dict(self.dry_run), + "evidence_refs": { + name: ref.to_dict() for name, ref in self.evidence_refs.items() + }, + "refusal_reasons": [dict(value) for value in self.refusal_reasons], + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + } + + +@dataclass(frozen=True, slots=True) +class ReconstructionOperationReceiptV1: + """Serializable submission, execution, status, cancel, or resume receipt.""" + + operation: str + request: ReconstructionExecutionRequestV1 + status: str + handles: tuple[OrchestrationJobHandle, ...] = () + status_store_roots: tuple[str, ...] = () + execution_attempt_id: str = "" + job_snapshots: tuple[Mapping[str, JSONValue], ...] = () + reports: tuple[ReconstructionRunReportV1, ...] = () + report_refs: tuple[ArtifactRef, ...] = () + receipt_id: str = "" + schema_version: str = RECONSTRUCTION_RECEIPT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_RECEIPT_SCHEMA_VERSION: + raise ReconstructionPlanError( + "unsupported reconstruction operation-receipt schema" + ) + if len(self.handles) != len(self.status_store_roots): + raise ReconstructionPlanError( + "receipt handles and status-store roots differ" + ) + if self.report_refs and len(self.reports) != len(self.report_refs): + raise ReconstructionPlanError( + "receipt reports and report references differ" + ) + expected = _stable_id( + "reconstruction-operation-receipt", self.identity_payload() + ) + if self.receipt_id and self.receipt_id != expected: + raise ReconstructionPlanError( + "reconstruction receipt identity differs" + ) + object.__setattr__(self, "receipt_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return stable receipt content.""" + return { + "schema_version": self.schema_version, + "operation": self.operation, + "request": self.request.to_dict(), + "status": self.status, + "handles": [ + cast(dict[str, JSONValue], handle.to_dict()) + for handle in self.handles + ], + "status_store_roots": list(self.status_store_roots), + "execution_attempt_id": self.execution_attempt_id, + "job_snapshots": [dict(item) for item in self.job_snapshots], + "reports": [report.to_dict() for report in self.reports], + "report_refs": [ref.to_dict() for ref in self.report_refs], + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return machine-readable receipt content.""" + return {**self.identity_payload(), "receipt_id": self.receipt_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionOperationReceiptV1": + """Restore an identity-checked public operation receipt.""" + return cls( + operation=str(data.get("operation", "")), + request=ReconstructionExecutionRequestV1.from_dict( + _mapping(data.get("request")) + ), + status=str(data.get("status", "")), + handles=tuple( + OrchestrationJobHandle( + request_id=str(item.get("request_id", "")), + workflow_id=str(item.get("workflow_id", "")), + run_id=str(item.get("run_id", "")), + task_queue=str(item.get("task_queue", "")), + namespace=str(item.get("namespace", "")), + ) + for item in ( + _mapping(value) for value in _sequence(data.get("handles")) + ) + ), + status_store_roots=tuple( + str(value) + for value in _sequence(data.get("status_store_roots")) + ), + execution_attempt_id=str(data.get("execution_attempt_id", "")), + job_snapshots=tuple( + dict(_mapping(value)) + for value in _sequence(data.get("job_snapshots")) + ), + reports=tuple( + ReconstructionRunReportV1.from_dict(_mapping(value)) + for value in _sequence(data.get("reports")) + ), + report_refs=tuple( + ArtifactRef.from_dict(_mapping(value)) + for value in _sequence(data.get("report_refs")) + ), + receipt_id=str(data.get("receipt_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +class ReconstructionClient: + """Supported synchronous and asynchronous reconstruction control facade.""" + + def __init__( + self, + *, + config: OrchestrationWorkerConfig | None = None, + supervisor: OrchestrationSupervisor | None = None, + temporal_client: Any | None = None, + ) -> None: + self.config = config + self.supervisor = supervisor + self.temporal_client = temporal_client + + def construct_plan(self, spec: ReconstructionPlanSpecV1) -> ArtifactRef: + """Build, execution-validate, and persist one content-addressed plan.""" + plan = self._construct_plan_model(spec) + return write_synthetic_infill_plan(plan, spec.artifact_root) + + def _construct_plan_model( + self, spec: ReconstructionPlanSpecV1 + ) -> SyntheticInfillPlanV1: + """Build one validated plan without a redundant persistence readback.""" + try: + plan = build_synthetic_infill_plan( + spec.source_root, + feed_epoch_definition_path=spec.feed_epoch_definition_path, + observation_operator_path=spec.observation_operator_path, + market_context_corpus_path=spec.market_context_corpus_path, + cftc_positioning_corpus_path=spec.cftc_positioning_corpus_path, + benchmark_manifest_path=spec.benchmark_manifest_path, + motif_manifest_path=spec.motif_manifest_path, + motif_index_path=spec.motif_index_path, + motif_qualification_path=spec.motif_qualification_path, + motif_leakage_audit_path=spec.motif_leakage_audit_path, + artifact_root=spec.artifact_root, + output_root=spec.output_root, + checkpoint_root=spec.checkpoint_root, + scratch_root=spec.scratch_root, + symbols=spec.symbols, + start_period=spec.start_period, + end_period=spec.end_period, + requested_start_ns=spec.requested_start_ns, + requested_end_ns=spec.requested_end_ns, + window_size_ns=spec.window_size_ns, + information_mode=spec.information_mode, + delivery_mode=spec.delivery_mode, + broker_delivery_artifact=spec.broker_delivery_artifact, + ) + validate_synthetic_infill_plan_for_execution(plan) + return plan + except ReconstructionPublicError: + raise + except (OSError, TypeError, ValueError) as err: + raise ReconstructionPlanError(str(err)) from err + + def construct_plan_set( + self, + spec: ReconstructionPlanSpecV1, + *, + periods_per_shard: int = DEFAULT_PLAN_SET_PERIODS_PER_SHARD, + ) -> ArtifactRef: + """Build one full-range plan as bounded contiguous executable shards.""" + if ( + spec.requested_start_ns is not None + or spec.requested_end_ns is not None + ): + raise ReconstructionUnsupportedError( + "plan sets currently require explicit start_period and end_period" + ) + if spec.start_period is None or spec.end_period is None: + raise ReconstructionUnsupportedError( + "plan sets require explicit start_period and end_period" + ) + if ( + isinstance(periods_per_shard, bool) + or not isinstance(periods_per_shard, int) + or not 1 <= periods_per_shard <= MAX_PLAN_SET_PERIODS_PER_SHARD + ): + raise ReconstructionUnsupportedError( + "periods_per_shard is outside public limits" + ) + ranges = _period_shards( + spec.start_period, + spec.end_period, + periods_per_shard=periods_per_shard, + ) + if len(ranges) > MAX_RECONSTRUCTION_PLAN_SHARDS: + raise ReconstructionUnsupportedError( + "requested range exceeds the public plan-set shard limit" + ) + shards: list[ReconstructionPlanShardV1] = [] + resource_summaries: list[ReconstructionPlanResourceSummaryV1] = [] + source_partitions: dict[str, tuple[str, str, int, int]] = {} + root = Path(spec.artifact_root).expanduser().resolve() + + def construct_interval( + requested_start_ns: int, requested_end_ns: int + ) -> None: + start_period = _period_for_ns(requested_start_ns) + end_period = _period_for_ns(requested_end_ns - 1) + shard_root = ( + root + / "shards" + / ( + f"{start_period}-{end_period}-" + f"{requested_start_ns}-{requested_end_ns}" + ) + ) + shard_spec = replace( + spec, + start_period=start_period, + end_period=end_period, + requested_start_ns=requested_start_ns, + requested_end_ns=requested_end_ns, + artifact_root=str(shard_root / "artifacts"), + output_root=str(shard_root / "output"), + checkpoint_root=str(shard_root / "checkpoints"), + scratch_root=str(shard_root / "scratch"), + ) + try: + plan = self._construct_plan_model(shard_spec) + except ReconstructionPlanError as error: + window_count = ( + requested_end_ns + - requested_start_ns + + spec.window_size_ns + - 1 + ) // spec.window_size_ns + if not _splittable_plan_error(error) or window_count <= 1: + raise + left_window_count = max(1, window_count // 2) + split_ns = min( + requested_end_ns, + requested_start_ns + + left_window_count * spec.window_size_ns, + ) + if ( + split_ns <= requested_start_ns + or split_ns >= requested_end_ns + ): + raise + construct_interval(requested_start_ns, split_ns) + construct_interval(split_ns, requested_end_ns) + return + plan_ref = write_synthetic_infill_plan( + plan, shard_spec.artifact_root + ) + preflight_status = ( + "ready_with_refusals" if plan.refusals else "ready" + ) + shards.append( + ReconstructionPlanShardV1( + start_period=start_period, + end_period=end_period, + requested_start_ns=plan.requested_start_ns, + requested_end_ns=plan.requested_end_ns, + plan_id=plan.plan_id, + plan_ref=plan_ref, + preflight_status=preflight_status, + executable=True, + refusal_count=len(plan.refusals), + resource_summary=plan.resources.to_dict(), + ) + ) + _accumulate_plan_set_resources( + plan, + resource_summaries=resource_summaries, + source_partitions=source_partitions, + ) + if len(shards) > MAX_RECONSTRUCTION_PLAN_SHARDS: + raise ReconstructionUnsupportedError( + "resource-safe plan set exceeds the public shard limit" + ) + + for start_period, end_period in ranges: + construct_interval( + _period_start_ns(start_period), + _period_start_ns(_next_period(end_period)), + ) + resources = _aggregate_plan_set_resources( + resource_summaries, source_partitions + ) + status = ( + "refused" + if any(not item.executable for item in shards) + else ( + "ready_with_refusals" + if any(item.refusal_count for item in shards) + else "ready" + ) + ) + plan_set = ReconstructionPlanSetV1( + source_spec=spec, + shards=tuple(shards), + requested_start_ns=shards[0].requested_start_ns, + requested_end_ns=shards[-1].requested_end_ns, + resource_summary=resources, + status=status, + ) + return write_reconstruction_plan_set(plan_set, root) + + def preflight_plan_set( + self, plan_set_path: str | Path + ) -> ReconstructionPlanSetPreflightV1: + """Re-verify every artifact, identity, resource bound, and refusal.""" + plan_set = read_reconstruction_plan_set(plan_set_path) + shard_preflights: list[Mapping[str, JSONValue]] = [] + resource_summaries: list[ReconstructionPlanResourceSummaryV1] = [] + source_partitions: dict[str, tuple[str, str, int, int]] = {} + refusal_count = 0 + all_executable = True + verified_refs: set[tuple[str, str, int | None, str]] = set() + + def verify_once(ref: ArtifactRef) -> None: + key = (ref.kind, ref.path, ref.size_bytes, ref.sha256) + if key not in verified_refs: + verify_artifact_ref(ref) + verified_refs.add(key) + + for shard in plan_set.shards: + try: + verify_once(shard.plan_ref) + except (OSError, TypeError, ValueError) as error: + raise ReconstructionPlanError( + "plan-set shard artifact differs" + ) from error + plan = read_synthetic_infill_plan(shard.plan_ref.path) + if ( + plan.plan_id != shard.plan_id + or plan.requested_start_ns != shard.requested_start_ns + or plan.requested_end_ns != shard.requested_end_ns + or plan.resources.to_dict() != dict(shard.resource_summary) + ): + raise ReconstructionPlanError("plan-set shard content differs") + for ref in plan.artifact_graph.values(): + verify_once(ref) + for workflow_request in plan.workflow_requests: + for task in workflow_request.tasks: + for command in task.commands: + for ref in command.input_manifest_refs: + verify_once(ref) + validate_synthetic_infill_plan_for_execution( + plan, verify_artifacts=False + ) + request = self.create_request( + shard.plan_ref.path, + information_mode=plan_set.source_spec.information_mode, + acknowledge_scientific_nonclaim=True, + allow_refusals=True, + ) + preflight = self.preflight(request, verify_artifacts=False) + current_refusals = len(preflight.refusal_reasons) + refusal_count += current_refusals + all_executable = all_executable and preflight.executable + shard_preflights.append( + { + "shard_id": shard.shard_id, + "plan_id": shard.plan_id, + "start_period": shard.start_period, + "end_period": shard.end_period, + "status": preflight.status, + "executable": preflight.executable, + "refusal_count": current_refusals, + } + ) + _accumulate_plan_set_resources( + plan, + resource_summaries=resource_summaries, + source_partitions=source_partitions, + ) + resources = _aggregate_plan_set_resources( + resource_summaries, source_partitions + ) + if resources != dict(plan_set.resource_summary): + raise ReconstructionPlanError("plan-set aggregate resources differ") + status = ( + "refused" + if not all_executable + else ("ready_with_refusals" if refusal_count else "ready") + ) + if status != plan_set.status: + raise ReconstructionPlanError("plan-set preflight status differs") + return ReconstructionPlanSetPreflightV1( + plan_set_id=plan_set.plan_set_id, + status=status, + executable=all_executable, + shard_count=len(plan_set.shards), + verified_shard_count=len(shard_preflights), + refusal_count=refusal_count, + resource_summary=resources, + shard_preflights=tuple(shard_preflights), + ) + + def create_request( + self, + plan_path: str | Path, + *, + information_mode: InformationMode | str, + acknowledge_scientific_nonclaim: bool, + allow_refusals: bool = False, + ) -> ReconstructionExecutionRequestV1: + """Bind explicit operator intent to a verified plan identity.""" + plan = _read_plan(plan_path) + return ReconstructionExecutionRequestV1( + plan_path=str(Path(plan_path).expanduser().resolve()), + plan_id=plan.plan_id, + information_mode=InformationMode.from_value(information_mode), + scientific_nonclaim_acknowledged=acknowledge_scientific_nonclaim, + allow_refusals=allow_refusals, + ) + + def preflight( + self, + request: ReconstructionExecutionRequestV1, + *, + verify_artifacts: bool = True, + ) -> ReconstructionPreflightV1: + """Validate plan identity, artifacts, support, refusals, and resources.""" + plan = _bound_plan(request) + try: + validate_synthetic_infill_plan_for_execution( + plan, verify_artifacts=verify_artifacts + ) + except (OSError, TypeError, ValueError) as err: + raise ReconstructionPlanError(str(err)) from err + refusals = tuple(item.to_dict() for item in plan.refusals) + executable = not refusals or request.allow_refusals + status = "ready" + if refusals: + status = "ready_with_refusals" if executable else "refused" + evidence = { + name: ref + for name, ref in plan.artifact_graph.items() + if any( + token in name + for token in ( + "audit", + "benchmark", + "certification", + "information", + "qualification", + "validation", + ) + ) + } + return ReconstructionPreflightV1( + request_id=request.request_id, + plan_id=plan.plan_id, + status=status, + executable=executable, + plan_status=plan.status, + dry_run=plan.dry_run_payload(), + evidence_refs=evidence, + refusal_reasons=refusals, + ) + + def submit( + self, + request: ReconstructionExecutionRequestV1, + *, + wait: bool = False, + execution_attempt_id: str = "", + ) -> ReconstructionOperationReceiptV1: + """Synchronously submit all plan batches and optionally wait.""" + return asyncio.run( + self.submit_async( + request, + wait=wait, + execution_attempt_id=execution_attempt_id, + ) + ) + + async def submit_async( + self, + request: ReconstructionExecutionRequestV1, + *, + wait: bool = False, + execution_attempt_id: str = "", + ) -> ReconstructionOperationReceiptV1: + """Submit all plan batches and optionally attach terminal snapshots.""" + plan = self._executable_plan(request) + handles: list[OrchestrationJobHandle] = [] + roots: list[str] = [] + snapshots: list[Mapping[str, JSONValue]] = [] + for workflow_request in plan.workflow_requests: + store = ManifestStatusStore(workflow_request.manifest_store_root) + workflow_id = _attempt_workflow_id( + workflow_request, execution_attempt_id + ) + handle = await submit_reconstruction_request( + workflow_request, + config=self.config, + supervisor=self.supervisor, + client=self.temporal_client, + status_store=store, + workflow_id=workflow_id, + execution_attempt_id=execution_attempt_id, + ) + handles.append(handle) + roots.append(workflow_request.manifest_store_root) + if wait: + snapshot = await get_job_result( + handle.workflow_id, + run_id=handle.run_id, + config=self.config, + supervisor=self.supervisor, + client=self.temporal_client, + status_store=store, + ) + snapshots.append( + cast(Mapping[str, JSONValue], snapshot.to_dict()) + ) + status = "submitted" + if wait: + status = _snapshot_collection_status(snapshots) + return ReconstructionOperationReceiptV1( + operation="submit_and_wait" if wait else "submit_only", + request=request, + status=status, + handles=tuple(handles), + status_store_roots=tuple(roots), + execution_attempt_id=execution_attempt_id, + job_snapshots=tuple(snapshots), + ) + + def execute_local( + self, + request: ReconstructionExecutionRequestV1, + *, + window_id: str = "", + cancellation_requested: Callable[[], bool] | None = None, + ) -> ReconstructionOperationReceiptV1: + """Execute the real registered pipeline in-process for bounded recovery. + + Production submission remains Temporal-backed. This explicit method is + for one-process smoke, deterministic parity, and checkpoint recovery; + it never silently replaces a failed Temporal submission. + """ + plan = self._executable_plan(request) + register_first_party_reconstruction_handlers() + reports: list[ReconstructionRunReportV1] = [] + report_refs: list[ArtifactRef] = [] + matched_window = not window_id + for workflow_request in plan.workflow_requests: + selected = _selected_workflow_request(workflow_request, window_id) + if selected is None: + continue + matched_window = True + states = asyncio.run( + run_reconstruction_request( + selected, + cancellation_requested=cancellation_requested, + ) + ) + report = reconcile_reconstruction_report(selected, states) + reports.append(report) + report_refs.append( + write_reconstruction_report(report, selected.report_root) + ) + if not matched_window: + raise ReconstructionPlanError( + f"window_id is absent from plan: {window_id}" + ) + status = _report_collection_status(reports) + return ReconstructionOperationReceiptV1( + operation="execute_local", + request=request, + status=status, + reports=tuple(reports), + report_refs=tuple(report_refs), + ) + + def inspect( + self, + receipt: ReconstructionOperationReceiptV1, + *, + offline: bool = False, + ) -> ReconstructionOperationReceiptV1: + """Inspect every submitted handle using its exact persisted store.""" + return asyncio.run(self.inspect_async(receipt, offline=offline)) + + async def inspect_async( + self, + receipt: ReconstructionOperationReceiptV1, + *, + offline: bool = False, + ) -> ReconstructionOperationReceiptV1: + """Asynchronously inspect every submitted reconstruction handle.""" + snapshots: list[Mapping[str, JSONValue]] = [] + for handle, root in zip( + receipt.handles, receipt.status_store_roots, strict=True + ): + snapshot = await inspect_job_status( + handle.workflow_id, + run_id=handle.run_id, + config=self.config, + supervisor=self.supervisor, + client=self.temporal_client, + status_store=ManifestStatusStore(root), + offline=offline, + ) + snapshots.append(cast(Mapping[str, JSONValue], snapshot.to_dict())) + return ReconstructionOperationReceiptV1( + operation="status", + request=receipt.request, + status=_snapshot_collection_status(snapshots), + handles=receipt.handles, + status_store_roots=receipt.status_store_roots, + execution_attempt_id=receipt.execution_attempt_id, + job_snapshots=tuple(snapshots), + ) + + def cancel( + self, + receipt: ReconstructionOperationReceiptV1, + *, + reason: str = "", + ) -> ReconstructionOperationReceiptV1: + """Request live Temporal cancellation for every receipt handle.""" + return asyncio.run(self.cancel_async(receipt, reason=reason)) + + async def cancel_async( + self, + receipt: ReconstructionOperationReceiptV1, + *, + reason: str = "", + ) -> ReconstructionOperationReceiptV1: + """Asynchronously request cancellation using aligned status stores.""" + snapshots: list[Mapping[str, JSONValue]] = [] + for handle, root in zip( + receipt.handles, receipt.status_store_roots, strict=True + ): + snapshot = await cancel_job( + handle.workflow_id, + run_id=handle.run_id, + reason=reason, + config=self.config, + supervisor=self.supervisor, + client=self.temporal_client, + status_store=ManifestStatusStore(root), + ) + snapshots.append(cast(Mapping[str, JSONValue], snapshot.to_dict())) + return ReconstructionOperationReceiptV1( + operation="cancel", + request=receipt.request, + status="cancellation_requested", + handles=receipt.handles, + status_store_roots=receipt.status_store_roots, + execution_attempt_id=receipt.execution_attempt_id, + job_snapshots=tuple(snapshots), + ) + + def resume( + self, + receipt: ReconstructionOperationReceiptV1, + *, + wait: bool = False, + local: bool = False, + ) -> ReconstructionOperationReceiptV1: + """Resume from durable checkpoints with fresh workflow identities.""" + if local: + return replace( + self.execute_local(receipt.request), + operation="resume_local", + receipt_id="", + ) + attempt = _next_resume_attempt(receipt.execution_attempt_id) + resumed = self.submit( + receipt.request, + wait=wait, + execution_attempt_id=attempt, + ) + return replace(resumed, operation="resume", receipt_id="") + + def outputs( + self, request: ReconstructionExecutionRequestV1 + ) -> dict[str, JSONValue]: + """List compact verified committed product manifests for the plan.""" + plan = _bound_plan(request) + execution = read_reconstruction_plan_execution_manifest( + plan.artifact_graph["execution_manifest"].path + ) + outputs: list[JSONValue] = [] + ignored = 0 + planned_scopes = { + (task.window.window_id, task.window.ensemble_member_id) + for workflow_request in plan.workflow_requests + for task in workflow_request.tasks + } + for path in discover_reconstruction_manifests( + execution.output_root, run_id=plan.run.run_id + ): + manifest = verify_reconstruction_publication(path) + scope = (manifest.window_id, manifest.ensemble_member_id) + if scope not in planned_scopes: + ignored += 1 + continue + outputs.append(_manifest_summary(path, manifest)) + return { + "schema_version": RECONSTRUCTION_OUTPUT_LIST_SCHEMA_VERSION, + "request_id": request.request_id, + "plan_id": plan.plan_id, + "run_id": plan.run.run_id, + "output_root": execution.output_root, + "output_count": len(outputs), + "ignored_out_of_plan_count": ignored, + "outputs": outputs, + } + + def preview( + self, + manifest_path: str | Path, + *, + limit: int = DEFAULT_PREVIEW_LIMIT, + ) -> dict[str, JSONValue]: + """Return bounded rows with origin, lineage, method, and decisions.""" + selected_limit = _preview_limit(limit) + path = Path(manifest_path).expanduser().resolve() + manifest = verify_reconstruction_publication(path) + rows: list[JSONValue] = [] + for batch in iter_reconstruction_event_batches( + path, batch_size=selected_limit + ): + for row in batch.to_pylist(): + rows.append(_preview_row(row)) + if len(rows) >= selected_limit: + break + if len(rows) >= selected_limit: + break + return { + "schema_version": RECONSTRUCTION_PREVIEW_SCHEMA_VERSION, + "manifest_path": str(path), + "manifest_id": manifest.manifest_id, + "publication_id": manifest.publication_id, + "run_id": manifest.run_id, + "logical_content_sha256": manifest.replay.logical_content_sha256, + "validation": manifest.quality.to_dict(), + "constraints": manifest.constraints.to_dict(), + "preview_limit": selected_limit, + "preview_count": len(rows), + "rows": rows, + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + } + + def replay(self, manifest_path: str | Path) -> dict[str, JSONValue]: + """Integrity-replay a committed output and return compact evidence.""" + path = Path(manifest_path).expanduser().resolve() + manifest = load_reconstruction_manifest(path) + streams = read_reconstruction_streams(path) + event_count = sum(len(stream.events) for stream in streams) + return { + "schema_version": RECONSTRUCTION_REPLAY_SCHEMA_VERSION, + "manifest_path": str(path), + "manifest_id": manifest.manifest_id, + "publication_id": manifest.publication_id, + "run_id": manifest.run_id, + "symbols": [stream.symbol for stream in streams], + "stream_count": len(streams), + "event_count": event_count, + "logical_content_sha256": manifest.replay.logical_content_sha256, + "replay_verified": event_count == manifest.event_count, + } + + def certify( + self, + spec_path: str | Path, + *, + output_directory: str | Path, + ) -> tuple[ + ReconstructionCertificationDossierV2, + ModernReferenceCertificationCampaignResultV1, + ]: + """Run the public hash-verified modern-reference evidence campaign.""" + return run_modern_reference_certification_campaign( + spec_path, output_directory=output_directory + ) + + def _executable_plan( + self, request: ReconstructionExecutionRequestV1 + ) -> SyntheticInfillPlanV1: + preflight = self.preflight(request) + if not preflight.executable: + reasons = "; ".join( + str(item.get("reason", "refused")) + for item in preflight.refusal_reasons + ) + raise ReconstructionRefusedError(reasons or "plan was refused") + return _bound_plan(request) + + +def read_plan_spec(path: str | Path) -> ReconstructionPlanSpecV1: + """Read a public plan-spec JSON artifact.""" + return ReconstructionPlanSpecV1.from_dict(_read_json_mapping(path)) + + +def write_reconstruction_plan_set( + plan_set: ReconstructionPlanSetV1, directory: str | Path +) -> ArtifactRef: + """Atomically persist one content-addressed bounded plan set.""" + root = Path(directory).expanduser().resolve() + path = ( + root + / f"reconstruction-plan-set-{plan_set.plan_set_id.rsplit(':', 1)[-1]}.json" + ) + written = _write_json(path, plan_set.to_dict()) + return artifact_ref_for_file( + written, + kind="reconstruction_plan_set_v1", + metadata={ + "plan_set_id": plan_set.plan_set_id, + "shard_count": len(plan_set.shards), + "status": plan_set.status, + }, + ) + + +def read_reconstruction_plan_set(path: str | Path) -> ReconstructionPlanSetV1: + """Read and identity-check one bounded plan-set artifact.""" + return ReconstructionPlanSetV1.from_dict(_read_json_mapping(path)) + + +def write_execution_request( + request: ReconstructionExecutionRequestV1, path: str | Path +) -> Path: + """Atomically write operator request metadata.""" + return _write_json(path, request.to_dict()) + + +def read_execution_request( + path: str | Path, +) -> ReconstructionExecutionRequestV1: + """Read and verify operator request metadata.""" + return ReconstructionExecutionRequestV1.from_dict(_read_json_mapping(path)) + + +def write_operation_receipt( + receipt: ReconstructionOperationReceiptV1, path: str | Path +) -> Path: + """Atomically write a reconstruction operation receipt.""" + return _write_json(path, receipt.to_dict()) + + +def read_operation_receipt( + path: str | Path, +) -> ReconstructionOperationReceiptV1: + """Read and identity-check a reconstruction operation receipt.""" + return ReconstructionOperationReceiptV1.from_dict(_read_json_mapping(path)) + + +def reconstruction_exit_code( + result: ReconstructionPreflightV1 | ReconstructionOperationReceiptV1, +) -> ReconstructionExitCode: + """Map a public report or receipt to its stable CLI exit category.""" + if isinstance(result, ReconstructionPreflightV1): + return ( + ReconstructionExitCode.SUCCESS + if result.executable + else ReconstructionExitCode.REFUSED + ) + if result.status in { + "cancelled", + "cancellation_requested", + "committed", + "completed", + "running", + "submitted", + }: + return ReconstructionExitCode.SUCCESS + if result.status == "refused": + return ReconstructionExitCode.REFUSED + if result.status in {"failed", "partial"}: + return ReconstructionExitCode.VALIDATION_FAILURE + return ReconstructionExitCode.RUNTIME_FAILURE + + +def _read_plan(path: str | Path) -> SyntheticInfillPlanV1: + try: + return read_synthetic_infill_plan(path) + except (OSError, TypeError, ValueError) as err: + raise ReconstructionPlanError(str(err)) from err + + +def _bound_plan( + request: ReconstructionExecutionRequestV1, +) -> SyntheticInfillPlanV1: + plan = _read_plan(request.plan_path) + if plan.plan_id != request.plan_id: + raise ReconstructionPlanError("execution request plan_id differs") + if plan.information_mode is not request.information_mode: + raise ReconstructionRefusedError( + "operator information mode differs from the immutable plan" + ) + if tuple(plan.run.symbols) != RECONSTRUCTION_SYMBOLS: + raise ReconstructionUnsupportedError( + "plan does not contain the supported complete EURUSD triangle" + ) + return plan + + +def _selected_workflow_request( + request: ReconstructionWorkflowRequestV1, window_id: str +) -> ReconstructionWorkflowRequestV1 | None: + if not window_id: + return request + tasks = tuple( + task for task in request.tasks if task.window.window_id == window_id + ) + if not tasks: + return None + return replace( + request, + tasks=tasks, + max_parallel_windows=1, + request_fingerprint="", + ) + + +def _attempt_workflow_id( + request: ReconstructionWorkflowRequestV1, execution_attempt_id: str +) -> str: + if not execution_attempt_id: + return "" + digest = hashlib.sha256( + ( + f"{request.run.run_id}|{request.request_fingerprint}|{execution_attempt_id}" + ).encode("utf-8") + ).hexdigest()[:24] + return f"histdatacom-reconstruction-{request.request_id}-{digest}" + + +def _next_resume_attempt(previous: str) -> str: + prefix = "resume-" + if previous.startswith(prefix) and previous[len(prefix) :].isdigit(): + ordinal = int(previous[len(prefix) :]) + 1 + else: + ordinal = 1 + return f"{prefix}{ordinal:03d}" + + +def _report_collection_status( + reports: Sequence[ReconstructionRunReportV1], +) -> str: + statuses = {report.status for report in reports} + if statuses == {"committed"} and reports: + return "committed" + if "failed" in statuses: + return "failed" + if "partial" in statuses: + return "partial" + if statuses == {"cancelled"}: + return "cancelled" + return "failed" + + +def _snapshot_collection_status( + snapshots: Sequence[Mapping[str, JSONValue]], +) -> str: + if not snapshots: + return "unknown" + values = { + str(snapshot.get("status", "")).strip().lower() + for snapshot in snapshots + } + if values.issubset({"completed", "succeeded"}): + return "completed" + if "failed" in values: + return "failed" + if values.issubset({"cancelled", "canceled"}): + return "cancelled" + return "running" + + +def _manifest_summary(path: Path, manifest: Any) -> dict[str, JSONValue]: + return { + "manifest_path": str(path), + "manifest_id": manifest.manifest_id, + "publication_id": manifest.publication_id, + "run_id": manifest.run_id, + "window_id": manifest.window_id, + "ensemble_member_id": manifest.ensemble_member_id, + "symbols": list(manifest.symbols), + "event_count": manifest.event_count, + "observed_event_count": manifest.observed_event_count, + "synthetic_event_count": manifest.synthetic_event_count, + "logical_content_sha256": manifest.replay.logical_content_sha256, + "validation_manifest_id": manifest.quality.quality_manifest_id, + "constraint_manifest_id": (manifest.constraints.constraint_manifest_id), + } + + +def _preview_row(row: Mapping[str, Any]) -> dict[str, JSONValue]: + origin = str(row.get("origin", "")) + observed = origin == "observed" + return { + "event_id": str(row.get("event_id", "")), + "origin": origin, + "symbol": str(row.get("symbol", "")), + "event_time_ns": cast(int, row.get("event_time_ns", 0)), + "event_sequence": cast(int, row.get("event_sequence", 0)), + "bid": cast(float, row.get("bid", 0.0)), + "ask": cast(float, row.get("ask", 0.0)), + "lineage": { + "source_version_id": row.get("source_version_id"), + "source_series_id": row.get("source_series_id"), + "source_period": row.get("source_period"), + "source_row_id": row.get("source_row_id"), + "anchor_interval_id": row.get("anchor_interval_id"), + "left_anchor_event_id": row.get("left_anchor_event_id"), + "right_anchor_event_id": row.get("right_anchor_event_id"), + "immutable_observed_anchor": observed, + }, + "generation": { + "method": ( + "immutable_observed_anchor" + if observed + else row.get("generator_id") + ), + "generator_id": row.get("generator_id"), + "generator_version": row.get("generator_version"), + "generator_config_id": row.get("generator_config_id"), + "reference_id": row.get("reference_id"), + "motif_id": row.get("motif_id"), + "feed_epoch_id": row.get("feed_epoch_id"), + "broker_profile_id": row.get("broker_profile_id"), + "confidence": row.get("confidence"), + }, + "constraint_decision": { + "decision": "immutable_anchor" if observed else "accepted", + "constraint_set_id": row.get("constraint_set_id"), + }, + } + + +def _validate_public_input_contract( + *, source_format: str, timeframe: str, symbols: Sequence[str] +) -> None: + if str(source_format).strip().lower() != RECONSTRUCTION_SOURCE_FORMAT: + raise ReconstructionUnsupportedError( + "unsupported source format; reconstruction requires ASCII" + ) + if str(timeframe).strip().upper() != RECONSTRUCTION_TIMEFRAME: + raise ReconstructionUnsupportedError( + "unsupported timeframe; reconstruction requires tick timeframe T" + ) + selected = tuple(sorted(str(value).strip().lower() for value in symbols)) + if selected != RECONSTRUCTION_SYMBOLS: + raise ReconstructionUnsupportedError( + "unsupported symbols; reconstruction requires EURGBP/EURUSD/GBPUSD" + ) + + +def _preview_limit(value: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ReconstructionUnsupportedError("preview limit must be an integer") + if value < 1 or value > MAX_PREVIEW_LIMIT: + raise ReconstructionUnsupportedError( + f"preview limit must be between 1 and {MAX_PREVIEW_LIMIT}" + ) + return value + + +def _period(value: Any) -> str: + selected = str(value or "").strip() + if ( + len(selected) != 6 + or not selected.isdigit() + or not 1 <= int(selected[4:]) <= 12 + ): + raise ReconstructionUnsupportedError( + "reconstruction period must use YYYYMM" + ) + return selected + + +def _next_period(value: str) -> str: + selected = _period(value) + year = int(selected[:4]) + month = int(selected[4:]) + if month == 12: + return f"{year + 1:04d}01" + return f"{year:04d}{month + 1:02d}" + + +def _period_start_ns(value: str) -> int: + selected = _period(value) + timestamp = datetime( + int(selected[:4]), int(selected[4:]), 1, tzinfo=timezone.utc + ) + return int(timestamp.timestamp()) * 1_000_000_000 + + +def _period_for_ns(value: int) -> str: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ReconstructionUnsupportedError("event time must be nanoseconds") + timestamp = datetime.fromtimestamp(value // 1_000_000_000, tz=timezone.utc) + return f"{timestamp.year:04d}{timestamp.month:02d}" + + +def _splittable_plan_error(error: ReconstructionPlanError) -> bool: + message = str(error).lower() + return any( + token in message + for token in ( + "reconstruction persistence preflight failed", + "reconstruction resource preflight failed", + "synthetic infill plan exceeds bounded artifact size", + ) + ) + + +def _period_shards( + start_period: str, + end_period: str, + *, + periods_per_shard: int, +) -> tuple[tuple[str, str], ...]: + start = _period(start_period) + end = _period(end_period) + if start > end: + raise ReconstructionUnsupportedError( + "plan-set start_period follows end_period" + ) + periods: list[str] = [] + current = start + while current <= end: + periods.append(current) + current = _next_period(current) + return tuple( + (selected[0], selected[-1]) + for offset in range(0, len(periods), periods_per_shard) + for selected in (periods[offset : offset + periods_per_shard],) + ) + + +def _accumulate_plan_set_resources( + plan: SyntheticInfillPlanV1, + *, + resource_summaries: list[ReconstructionPlanResourceSummaryV1], + source_partitions: dict[str, tuple[str, str, int, int]], +) -> None: + """Retain only compact shard resources and unique source identities.""" + inventory = read_reconstruction_source_inventory( + plan.artifact_graph["source_inventory"].path + ) + for partition in inventory.partitions: + identity = ( + partition.period, + partition.symbol, + partition.row_count, + cast(int, partition.artifact.size_bytes), + ) + existing = source_partitions.setdefault( + partition.partition_id, identity + ) + if existing != identity: + raise ReconstructionPlanError( + "plan-set source partition identity is inconsistent" + ) + resource_summaries.append(plan.resources) + + +def _aggregate_plan_set_resources( + resources: Sequence[ReconstructionPlanResourceSummaryV1], + source_partitions: Mapping[str, tuple[str, str, int, int]], +) -> dict[str, JSONValue]: + if not resources: + raise ReconstructionPlanError("cannot aggregate an empty plan set") + input_events = sum(item.estimated_input_event_count for item in resources) + candidate_events = sum( + item.estimated_candidate_event_count for item in resources + ) + payload: dict[str, JSONValue] = { + "schema_version": "histdatacom.reconstruction-plan-set-resources.v1", + "plan_shard_count": len(resources), + "source_partition_count": len(source_partitions), + "source_event_count": sum( + item[2] for item in source_partitions.values() + ), + "source_size_bytes": sum( + item[3] for item in source_partitions.values() + ), + "planned_window_count": sum( + item.planned_window_count for item in resources + ), + "executable_window_count": sum( + item.executable_window_count for item in resources + ), + "refused_window_count": sum( + item.refused_window_count for item in resources + ), + "ensemble_member_count": max( + item.ensemble_member_count for item in resources + ), + "retained_member_count": max( + item.retained_member_count for item in resources + ), + "workflow_request_count": sum( + item.workflow_request_count for item in resources + ), + "estimated_input_event_count": input_events, + "estimated_candidate_event_count": candidate_events, + "estimated_candidate_bytes": sum( + item.estimated_candidate_bytes for item in resources + ), + "estimated_peak_memory_bytes": max( + item.estimated_peak_memory_bytes for item in resources + ), + "estimated_peak_scratch_bytes": max( + item.estimated_peak_scratch_bytes for item in resources + ), + "estimated_output_bytes": sum( + item.estimated_output_bytes for item in resources + ), + "estimated_partition_count": sum( + item.estimated_partition_count for item in resources + ), + "candidate_amplification": ( + candidate_events / input_events if input_events else 0.0 + ), + "output_basis": "sharded-sum-of-retained-member-compressed-upper-bound-v1", + "scratch_basis": "maximum-shard-peak-concurrent-window-scratch-v1", + } + payload["summary_id"] = _stable_id( + "reconstruction-plan-set-resources", payload + ) + return payload + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _read_json_mapping(path: str | Path) -> dict[str, Any]: + target = Path(path).expanduser().resolve() + try: + payload = json.loads(target.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as err: + raise ReconstructionPlanError(f"cannot read {target}: {err}") from err + if not isinstance(payload, Mapping): + raise ReconstructionPlanError(f"JSON root must be an object: {target}") + return dict(payload) + + +def _write_json(path: str | Path, payload: Mapping[str, JSONValue]) -> Path: + target = Path(path).expanduser().resolve() + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(f".{target.name}.partial") + encoded = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode( + "utf-8" + ) + temporary.write_bytes(encoded) + temporary.replace(target) + return target + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ReconstructionPlanError("expected a JSON object") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ReconstructionPlanError("expected a JSON array") + return value + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ReconstructionPlanError(f"{name} must be a JSON boolean") + return value + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ReconstructionPlanError(f"{name} must be a JSON integer") + return value + + +def _required_text(value: Any) -> str: + text = str(value or "").strip() + if not text: + raise ReconstructionPlanError("required reconstruction text is empty") + return text + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +__all__ = [ + "DEFAULT_PLAN_SET_PERIODS_PER_SHARD", + "DEFAULT_PREVIEW_LIMIT", + "MAX_PREVIEW_LIMIT", + "MAX_RECONSTRUCTION_PLAN_SHARDS", + "InformationMode", + "RECONSTRUCTION_EXECUTION_REQUEST_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_SET_PREFLIGHT_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_SET_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_SHARD_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_SPEC_SCHEMA_VERSION", + "RECONSTRUCTION_PREVIEW_SCHEMA_VERSION", + "RECONSTRUCTION_RECEIPT_SCHEMA_VERSION", + "RECONSTRUCTION_REPLAY_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_FORMAT", + "RECONSTRUCTION_SYMBOLS", + "RECONSTRUCTION_TIMEFRAME", + "ReconstructionClient", + "ReconstructionExecutionRequestV1", + "ReconstructionExitCode", + "ReconstructionOperationReceiptV1", + "ReconstructionPlanError", + "ReconstructionPlanSetPreflightV1", + "ReconstructionPlanSetV1", + "ReconstructionPlanShardV1", + "ReconstructionPlanSpecV1", + "ReconstructionPreflightV1", + "ReconstructionPublicError", + "ReconstructionRefusedError", + "ReconstructionUnsupportedError", + "ReconstructionValidationError", + "ModernReferenceCertificationCampaignResultV1", + "ModernReferenceCertificationCampaignSpecV1", + "ReconstructionCertificationDossierV2", + "read_execution_request", + "read_modern_reference_certification_campaign_spec", + "read_operation_receipt", + "read_plan_spec", + "read_reconstruction_plan_set", + "reconstruction_exit_code", + "run_modern_reference_certification_campaign", + "write_execution_request", + "write_operation_receipt", + "write_reconstruction_plan_set", +] diff --git a/src/histdatacom/reconstruction_cli.py b/src/histdatacom/reconstruction_cli.py new file mode 100644 index 00000000..71b13266 --- /dev/null +++ b/src/histdatacom/reconstruction_cli.py @@ -0,0 +1,389 @@ +"""Installed command-line surface for typed reconstruction operations.""" + +from __future__ import annotations + +import argparse +from collections.abc import Mapping, Sequence +import json +from pathlib import Path +import sys +from typing import Any + +from histdatacom.cli_config import ( + CliConfigError, + add_config_argument, + configured_reconstruction_argv, +) +from histdatacom.orchestration.supervisor import OrchestrationSupervisor +from histdatacom.reconstruction import ( + MAX_PREVIEW_LIMIT, + ReconstructionClient, + ReconstructionExitCode, + ReconstructionPublicError, + read_execution_request, + read_operation_receipt, + read_plan_spec, + reconstruction_exit_code, + write_execution_request, + write_operation_receipt, +) +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.certification import CertificationState + + +def build_parser() -> argparse.ArgumentParser: + """Build the reconstruction command family parser.""" + parser = argparse.ArgumentParser( + prog="histdatacom reconstruction", + description=( + "Plan, preflight, execute, control, and inspect first-party " + "ASCII tick reconstruction." + ), + epilog=( + "Every execution request must explicitly select an information " + "mode and acknowledge that output is not recovered historical " + "truth. M1/bar inputs and partial symbol groups are unsupported." + ), + ) + add_config_argument(parser) + parser.add_argument( + "--json", + action="store_true", + help="emit machine-readable JSON instead of a compact summary", + ) + parser.add_argument( + "--start-runtime", + action="store_true", + help="start the local Temporal runtime before submit, cancel, or resume", + ) + subparsers = parser.add_subparsers( + dest="reconstruction_command", required=True + ) + + plan = subparsers.add_parser( + "plan", help="construct and validate a plan from a JSON plan spec" + ) + plan.add_argument("--spec", required=True, metavar="PATH") + + plan_set = subparsers.add_parser( + "plan-set", + help="construct a full range as bounded contiguous plan shards", + ) + plan_set.add_argument("--spec", required=True, metavar="PATH") + plan_set.add_argument( + "--periods-per-shard", type=int, default=12, metavar="COUNT" + ) + + preflight_set = subparsers.add_parser( + "preflight-set", help="verify every shard in a full-range plan set" + ) + preflight_set.add_argument("--plan-set", required=True, metavar="PATH") + + request = subparsers.add_parser( + "request", help="bind explicit operator intent to an immutable plan" + ) + request.add_argument("--plan", required=True, metavar="PATH") + request.add_argument( + "--information-mode", + required=True, + choices=tuple(mode.value for mode in InformationMode), + ) + request.add_argument( + "--acknowledge-scientific-nonclaim", + action="store_true", + help="acknowledge that output is plausible, not recovered truth", + ) + request.add_argument( + "--allow-refusals", + action="store_true", + help="execute supported windows while retaining explicit refusals", + ) + request.add_argument("--output", required=True, metavar="PATH") + + preflight = subparsers.add_parser( + "preflight", help="validate readiness and print dry-run resources" + ) + preflight.add_argument("--request", required=True, metavar="PATH") + + run = subparsers.add_parser( + "run", + help="execute and wait, submit only, or run a bounded local smoke", + ) + run.add_argument("--request", required=True, metavar="PATH") + mode = run.add_mutually_exclusive_group() + mode.add_argument( + "--submit-only", + action="store_true", + help="return after Temporal submit", + ) + mode.add_argument( + "--local", + action="store_true", + help="run the registered pipeline in-process for smoke/recovery", + ) + run.add_argument( + "--window-id", + default="", + help="limit explicit local execution to one plan window", + ) + run.add_argument("--receipt", required=True, metavar="PATH") + + status = subparsers.add_parser( + "status", help="inspect every job in an operation receipt" + ) + status.add_argument("--receipt", required=True, metavar="PATH") + status.add_argument( + "--offline", action="store_true", help="read durable status only" + ) + status.add_argument("--output", default="", metavar="PATH") + + cancel = subparsers.add_parser( + "cancel", help="request cancellation for every submitted job" + ) + cancel.add_argument("--receipt", required=True, metavar="PATH") + cancel.add_argument("--reason", default="") + cancel.add_argument("--output", required=True, metavar="PATH") + + resume = subparsers.add_parser( + "resume", help="resume durable checkpoints using fresh workflow IDs" + ) + resume.add_argument("--receipt", required=True, metavar="PATH") + resume_mode = resume.add_mutually_exclusive_group() + resume_mode.add_argument("--submit-only", action="store_true") + resume_mode.add_argument("--local", action="store_true") + resume.add_argument("--output", required=True, metavar="PATH") + + outputs = subparsers.add_parser( + "outputs", help="list verified committed products for a request" + ) + outputs.add_argument("--request", required=True, metavar="PATH") + + preview = subparsers.add_parser( + "preview", help="show bounded event origin, lineage, and decisions" + ) + preview.add_argument("--manifest", required=True, metavar="PATH") + preview.add_argument( + "--limit", type=int, default=20, choices=range(1, MAX_PREVIEW_LIMIT + 1) + ) + + replay = subparsers.add_parser( + "replay", help="integrity-replay a committed reconstruction product" + ) + replay.add_argument("--manifest", required=True, metavar="PATH") + + certify = subparsers.add_parser( + "certify", + help="evaluate a hash-verified modern-reference evidence campaign", + ) + certify.add_argument("--spec", required=True, metavar="PATH") + certify.add_argument("--output-directory", required=True, metavar="PATH") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the installed reconstruction command family.""" + parser = build_parser() + raw_argv = list(argv) if argv is not None else sys.argv[1:] + try: + args = parser.parse_args(configured_reconstruction_argv(raw_argv)) + client = _client(args) + result, code = _run_command(client, args) + _write_result(result, as_json=bool(args.json)) + return int(code) + except CliConfigError as err: + return _write_error( + err, + reason_code="reconstruction_config_error", + exit_code=ReconstructionExitCode.INVALID_PLAN, + as_json="--json" in raw_argv, + ) + except ReconstructionPublicError as err: + return _write_error( + err, + reason_code=err.reason_code, + exit_code=err.exit_code, + as_json="--json" in raw_argv, + ) + except (OSError, TypeError, ValueError) as err: + return _write_error( + err, + reason_code="invalid_reconstruction_plan", + exit_code=ReconstructionExitCode.INVALID_PLAN, + as_json="--json" in raw_argv, + ) + except Exception as err: # pragma: no cover - defensive CLI boundary + return _write_error( + err, + reason_code="reconstruction_runtime_failure", + exit_code=ReconstructionExitCode.RUNTIME_FAILURE, + as_json="--json" in raw_argv, + ) + + +def _client(args: argparse.Namespace) -> ReconstructionClient: + supervisor = None + if args.start_runtime: + supervisor = OrchestrationSupervisor() + supervisor.start() + return ReconstructionClient(supervisor=supervisor) + + +def _run_command( + client: ReconstructionClient, args: argparse.Namespace +) -> tuple[Mapping[str, Any], ReconstructionExitCode]: + command = args.reconstruction_command + if command == "plan": + ref = client.construct_plan(read_plan_spec(args.spec)) + return ref.to_dict(), ReconstructionExitCode.SUCCESS + if command == "plan-set": + ref = client.construct_plan_set( + read_plan_spec(args.spec), + periods_per_shard=args.periods_per_shard, + ) + return ref.to_dict(), ReconstructionExitCode.SUCCESS + if command == "preflight-set": + set_preflight = client.preflight_plan_set(args.plan_set) + return ( + set_preflight.to_dict(), + ( + ReconstructionExitCode.SUCCESS + if set_preflight.executable + else ReconstructionExitCode.REFUSED + ), + ) + if command == "request": + request = client.create_request( + args.plan, + information_mode=args.information_mode, + acknowledge_scientific_nonclaim=( + args.acknowledge_scientific_nonclaim + ), + allow_refusals=args.allow_refusals, + ) + path = write_execution_request(request, args.output) + return { + "request": request.to_dict(), + "request_path": str(path), + }, ReconstructionExitCode.SUCCESS + if command == "preflight": + preflight = client.preflight(read_execution_request(args.request)) + return preflight.to_dict(), reconstruction_exit_code(preflight) + if command == "run": + request = read_execution_request(args.request) + receipt = ( + client.execute_local(request, window_id=args.window_id) + if args.local + else client.submit(request, wait=not args.submit_only) + ) + path = write_operation_receipt(receipt, args.receipt) + return _receipt_result(receipt, path), reconstruction_exit_code(receipt) + if command == "status": + receipt = client.inspect( + read_operation_receipt(args.receipt), offline=args.offline + ) + status_path = ( + write_operation_receipt(receipt, args.output) + if args.output + else None + ) + return _receipt_result(receipt, status_path), reconstruction_exit_code( + receipt + ) + if command == "cancel": + receipt = client.cancel( + read_operation_receipt(args.receipt), reason=args.reason + ) + path = write_operation_receipt(receipt, args.output) + return _receipt_result(receipt, path), reconstruction_exit_code(receipt) + if command == "resume": + receipt = client.resume( + read_operation_receipt(args.receipt), + wait=not args.submit_only, + local=args.local, + ) + path = write_operation_receipt(receipt, args.output) + return _receipt_result(receipt, path), reconstruction_exit_code(receipt) + if command == "outputs": + return ( + client.outputs(read_execution_request(args.request)), + ReconstructionExitCode.SUCCESS, + ) + if command == "preview": + return ( + client.preview(args.manifest, limit=args.limit), + ReconstructionExitCode.SUCCESS, + ) + if command == "replay": + return client.replay(args.manifest), ReconstructionExitCode.SUCCESS + if command == "certify": + dossier, result = client.certify( + args.spec, output_directory=args.output_directory + ) + payload = result.to_dict() + payload["summary"] = dossier.summary + return payload, _certification_exit_code(dossier.state) + raise ValueError(f"unsupported reconstruction command: {command}") + + +def _receipt_result(receipt: Any, path: Path | None) -> dict[str, Any]: + payload = dict(receipt.to_dict()) + if path is not None: + payload["receipt_path"] = str(path) + return payload + + +def _certification_exit_code( + state: CertificationState, +) -> ReconstructionExitCode: + if state in { + CertificationState.CERTIFIED, + CertificationState.READY_FOR_PROMOTION, + }: + return ReconstructionExitCode.SUCCESS + if state is CertificationState.INCOMPLETE: + return ReconstructionExitCode.REFUSED + return ReconstructionExitCode.VALIDATION_FAILURE + + +def _write_result(result: Mapping[str, Any], *, as_json: bool) -> None: + if as_json: + print(json.dumps(result, indent=2, sort_keys=True)) # noqa:T201 + return + status = result.get("status") or result.get("kind") or "complete" + identity = ( + result.get("receipt_id") + or result.get("request_id") + or result.get("plan_id") + or result.get("sha256") + or "" + ) + suffix = f" ({identity})" if identity else "" + print(f"Reconstruction {status}{suffix}") # noqa:T201 + for key in ("path", "request_path", "receipt_path", "manifest_path"): + if result.get(key): + print(f"{key}: {result[key]}") # noqa:T201 + + +def _write_error( + error: Exception, + *, + reason_code: str, + exit_code: ReconstructionExitCode, + as_json: bool, +) -> int: + payload = { + "status": "error", + "reason_code": reason_code, + "message": str(error), + "exit_code": int(exit_code), + } + if as_json: + print(json.dumps(payload, sort_keys=True), file=sys.stderr) # noqa:T201 + else: + print( # noqa:T201 + f"reconstruction error [{reason_code}]: {error}", file=sys.stderr + ) + return int(exit_code) + + +__all__ = ["build_parser", "main"] diff --git a/src/histdatacom/resource_usage.py b/src/histdatacom/resource_usage.py new file mode 100644 index 00000000..428fc983 --- /dev/null +++ b/src/histdatacom/resource_usage.py @@ -0,0 +1,125 @@ +"""Portable process resource-usage probes. + +Python's :mod:`resource` module is unavailable on Windows. Release and +scientific resource audits still need a process-level peak resident-memory +measurement there, so this module normalizes Unix ``ru_maxrss`` and uses the +Windows process API without adding a runtime dependency. +""" + +from __future__ import annotations + +import ctypes +from dataclasses import dataclass +from importlib import import_module +import sys +from typing import Any + + +@dataclass(frozen=True) +class PeakRssMeasurement: + """A peak resident-memory observation and its platform source.""" + + bytes: int + source: str + available: bool + + +class _WindowsProcessMemoryCounters(ctypes.Structure): + """Subset-compatible ``PROCESS_MEMORY_COUNTERS`` layout.""" + + _fields_ = [ + ("cb", ctypes.c_ulong), + ("PageFaultCount", ctypes.c_ulong), + ("PeakWorkingSetSize", ctypes.c_size_t), + ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), + ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), + ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), + ("PeakPagefileUsage", ctypes.c_size_t), + ] + + +def peak_rss_measurement() -> PeakRssMeasurement: + """Return peak process RSS in bytes with explicit source availability.""" + try: + resource_module = import_module("resource") + except (ImportError, ModuleNotFoundError): + resource_module = None + if resource_module is not None: + measurement = _resource_peak_rss_measurement( + resource_module, + platform_name=sys.platform, + ) + if measurement.available: + return measurement + if sys.platform.startswith("win"): + peak = _windows_peak_working_set_bytes() + if peak > 0: + return PeakRssMeasurement( + bytes=peak, + source="windows.PeakWorkingSetSize", + available=True, + ) + return PeakRssMeasurement(bytes=0, source="unavailable", available=False) + + +def peak_rss_bytes() -> int: + """Return peak process RSS bytes or the explicit unavailable sentinel ``0``.""" + return peak_rss_measurement().bytes + + +def _resource_peak_rss_measurement( + resource_module: Any, + *, + platform_name: str, +) -> PeakRssMeasurement: + try: + usage = resource_module.getrusage(resource_module.RUSAGE_SELF) + peak = int(getattr(usage, "ru_maxrss", 0) or 0) + except (AttributeError, OSError, TypeError, ValueError): + return PeakRssMeasurement( + bytes=0, source="unavailable", available=False + ) + if peak <= 0: + return PeakRssMeasurement( + bytes=0, source="unavailable", available=False + ) + if platform_name.startswith("linux"): + peak *= 1024 + return PeakRssMeasurement( + bytes=peak, + source="resource.ru_maxrss", + available=True, + ) + + +def _windows_peak_working_set_bytes() -> int: + """Return Windows peak working-set bytes, or ``0`` when the API is absent.""" + try: + windll_factory: Any = getattr(ctypes, "WinDLL") + kernel32 = windll_factory("kernel32", use_last_error=True) + psapi = windll_factory("psapi", use_last_error=True) + get_current_process = kernel32.GetCurrentProcess + get_current_process.argtypes = [] + get_current_process.restype = ctypes.c_void_p + get_process_memory_info = psapi.GetProcessMemoryInfo + get_process_memory_info.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(_WindowsProcessMemoryCounters), + ctypes.c_ulong, + ] + get_process_memory_info.restype = ctypes.c_int + counters = _WindowsProcessMemoryCounters() + counters.cb = ctypes.sizeof(counters) + succeeded = get_process_memory_info( + get_current_process(), + ctypes.byref(counters), + counters.cb, + ) + except (AttributeError, OSError, TypeError, ValueError): + return 0 + if not succeeded: + return 0 + return max(0, int(counters.PeakWorkingSetSize)) diff --git a/src/histdatacom/runtime_contracts.py b/src/histdatacom/runtime_contracts.py index 294e26cd..86413db5 100644 --- a/src/histdatacom/runtime_contracts.py +++ b/src/histdatacom/runtime_contracts.py @@ -406,6 +406,8 @@ class RunRequest: timeframes: tuple[str, ...] = () start_yearmonth: str = "" end_yearmonth: str = "" + random_window: str = "" + random_seed: int | None = None data_directory: str = "data" api_return_type: str = "" cpu_utilization: str = "medium" @@ -453,6 +455,11 @@ def from_options(cls, options: Any, request_id: str = "") -> "RunRequest": schedule_key = str(getattr(options, "schedule_key", "") or "").strip() if schedule_key: metadata["schedule_key"] = schedule_key + output_timezone = str( + getattr(options, "output_timezone", "") or "" + ).strip() + if output_timezone: + metadata["output_timezone"] = output_timezone verbosity = max(0, int(getattr(options, "verbosity", 0) or 0)) return cls( request_id=request_id or new_request_id(), @@ -464,6 +471,12 @@ def from_options(cls, options: Any, request_id: str = "") -> "RunRequest": timeframes=tuple(sorted(getattr(options, "timeframes", ()) or ())), start_yearmonth=str(getattr(options, "start_yearmonth", "") or ""), end_yearmonth=str(getattr(options, "end_yearmonth", "") or ""), + random_window=str(getattr(options, "random_window", "") or ""), + random_seed=( + int(options.random_seed) + if getattr(options, "random_seed", None) is not None + else None + ), data_directory=str( getattr(options, "data_directory", "data") or "data" ), @@ -539,6 +552,8 @@ def to_dict(self) -> dict[str, JSONValue]: "timeframes": list(self.timeframes), "start_yearmonth": self.start_yearmonth, "end_yearmonth": self.end_yearmonth, + "random_window": self.random_window, + "random_seed": self.random_seed, "data_directory": self.data_directory, "api_return_type": self.api_return_type, "cpu_utilization": self.cpu_utilization, @@ -577,6 +592,12 @@ def from_dict(cls, data: Mapping[str, Any]) -> "RunRequest": timeframes=tuple(str(item) for item in data.get("timeframes", [])), start_yearmonth=str(data.get("start_yearmonth", "") or ""), end_yearmonth=str(data.get("end_yearmonth", "") or ""), + random_window=str(data.get("random_window", "") or ""), + random_seed=( + int(data["random_seed"]) + if data.get("random_seed") is not None + else None + ), data_directory=str(data.get("data_directory", "data") or "data"), api_return_type=str(data.get("api_return_type", "") or ""), cpu_utilization=str( diff --git a/src/histdatacom/scraper/scraper.py b/src/histdatacom/scraper/scraper.py index 161ede72..7e9cea91 100644 --- a/src/histdatacom/scraper/scraper.py +++ b/src/histdatacom/scraper/scraper.py @@ -25,9 +25,11 @@ fetch_histdata_page_data, plan_dataset_work_items, parse_histdata_form_metadata, + post_histdata_archive, validate_url_work_item, ) from histdatacom.helper_args import helper_runtime_args +from histdatacom.exceptions import ArchiveDownloadError from histdatacom.legacy_boundary import warn_legacy_side_effect from histdatacom.observability import ProgressState from histdatacom.records import Record @@ -106,6 +108,19 @@ def _write_file(cls, record: Record, zip_content: bytes) -> None: record (Record): a record to write zip_content (bytes): binary zip data """ + if ( + record.data_format.lower() != "ascii" + or record.data_timeframe != "T" + ): + raise ArchiveDownloadError( + "UNSUPPORTED_RAW_INPUT", + "Archive writes support HistData ASCII tick inputs only.", + retryable=False, + detail={ + "data_format": record.data_format, + "timeframe": record.data_timeframe, + }, + ) atomic_write_zip_archive( Path(record.data_dir), record.zip_filename, @@ -124,20 +139,13 @@ def _request_file(cls, record: Record, timeout: int) -> requests.Response: Returns: requests.Response: zip file response """ - post_headers = config.default_post_headers() - post_headers["Referer"] = record.url - return requests.post( - "http://www.histdata.com/get.php", - data={ - "tk": record.data_tk, - "date": record.data_date, - "datemonth": record.data_datemonth, - "platform": record.data_format, - "timeframe": record.data_timeframe, - "fxpair": record.data_fxpair, - }, - headers=post_headers, - timeout=timeout, + return cast( + requests.Response, + post_histdata_archive( + record, + timeout=timeout, + post_headers=config.default_post_headers(), + ), ) def plan_initial_records( diff --git a/src/histdatacom/synthetic/__init__.py b/src/histdatacom/synthetic/__init__.py new file mode 100644 index 00000000..9e4904c5 --- /dev/null +++ b/src/histdatacom/synthetic/__init__.py @@ -0,0 +1,1432 @@ +"""Versioned contracts for synthetic market-event products.""" + +from typing import Any + + +from histdatacom.synthetic.activity import ( + ACTIVITY_EVENT_COLUMNS, + ACTIVITY_REVERSE_DEGRADATION_METRICS, + DEFAULT_ACTIVITY_BATCH_SIZE, + DEFAULT_ACTIVITY_MAX_PAYLOAD_BYTES, + DEFAULT_ACTIVITY_MAX_PROVENANCE_VALUES, + DEFAULT_ACTIVITY_MAX_SLICES, + DEFAULT_ACTIVITY_MAX_SYMBOLS, + DEFAULT_ACTIVITY_ROUNDING_DIGITS, + RECONSTRUCTION_ACTIVITY_BENCHMARK_EVIDENCE_SCHEMA_VERSION, + RECONSTRUCTION_ACTIVITY_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_ACTIVITY_METRIC_SCHEMA_VERSION, + RECONSTRUCTION_ACTIVITY_POLICY_SCHEMA_VERSION, + RECONSTRUCTION_ACTIVITY_SLICE_SCHEMA_VERSION, + ActivityAggregationSemantics, + ActivityMetricSemantics, + ActivitySliceScope, + ActivityVolumeState, + ReconstructionActivityBenchmarkEvidenceV1, + ReconstructionActivityManifestV1, + ReconstructionActivityMetricV1, + ReconstructionActivityPolicyV1, + ReconstructionActivitySliceV1, + activity_bar_projection_semantics, + activity_metric_definitions, + read_reconstruction_activity_manifest, + reconstruction_activity_benchmark_evidence, + summarize_committed_reconstruction_activity, + summarize_reconstruction_activity, + summarize_reconstruction_activity_streams, + write_reconstruction_activity_manifest, +) +from histdatacom.synthetic.bars import ( + DEFAULT_DERIVED_BAR_BATCH_SIZE, + DEFAULT_DERIVED_BAR_MAX_BARS, + DEFAULT_DERIVED_BAR_MAX_PROVENANCE_VALUES, + DEFAULT_DERIVED_BAR_MAX_SYMBOLS, + DEFAULT_DERIVED_BAR_ROUNDING_DIGITS, + DEFAULT_DERIVED_BAR_ROW_GROUP_SIZE, + DEFAULT_DERIVED_BAR_WRITE_BUFFER_ROWS, + DERIVED_BAR_ARROW_COLUMNS, + DERIVED_BAR_EVENT_COLUMNS, + DERIVED_BAR_INTERVAL_SCHEMA_VERSION, + DERIVED_BAR_MANIFEST_ARTIFACT_KIND, + DERIVED_BAR_PARTITION_SCHEMA_VERSION, + DERIVED_BAR_POLICY_SCHEMA_VERSION, + DERIVED_BAR_PRODUCT_DIRECTORY, + DERIVED_BAR_PRODUCT_SCHEMA_VERSION, + DERIVED_BAR_SCHEMA_VERSION, + STANDARD_DERIVED_BAR_INTERVALS, + DerivedBarIntervalV1, + DerivedBarPartitionV1, + DerivedBarPersistenceError, + DerivedBarPolicyV1, + DerivedBarProductManifestV1, + DerivedBarV1, + PublishedDerivedBarsV1, + StagedDerivedBarPublicationV1, + commit_derived_bar_publication, + derive_reconstruction_bars, + derived_bar_arrow_schema, + discover_derived_bar_manifests, + iter_committed_reconstruction_bars, + iter_derived_bar_batches, + load_derived_bar_manifest, + publish_derived_bars, + scan_derived_bars_polars, + stage_derived_bar_publication, + verify_derived_bar_publication, +) +from histdatacom.synthetic.benchmark import ( + BENCHMARK_CANDIDATE_SCHEMA_VERSION, + BENCHMARK_CANDIDATE_SCORE_SCHEMA_VERSION, + BENCHMARK_CANDIDATE_WINDOW_SCHEMA_VERSION, + BENCHMARK_EVENT_SCHEMA_VERSION, + BENCHMARK_EXECUTION_EVIDENCE_SCHEMA_VERSION, + BENCHMARK_MANIFEST_SCHEMA_VERSION, + BENCHMARK_PROFILE_SCHEMA_VERSION, + BENCHMARK_SCENARIO_SCHEMA_VERSION, + BENCHMARK_SCORECARD_SCHEMA_VERSION, + BENCHMARK_SLICE_SCORE_SCHEMA_VERSION, + BENCHMARK_SPLIT_SCHEMA_VERSION, + DEFAULT_BENCHMARK_MAX_CANDIDATES, + DEFAULT_BENCHMARK_MAX_EVENTS_PER_WINDOW, + DEFAULT_BENCHMARK_MAX_HOOK_METRICS, + DEFAULT_BENCHMARK_MAX_PAYLOAD_BYTES, + DEFAULT_BENCHMARK_MAX_REASON_CODES, + DEFAULT_BENCHMARK_MAX_SCENARIOS, + DEFAULT_BENCHMARK_MAX_SLICES, + DEFAULT_BENCHMARK_ROUNDING_DIGITS, + MAX_BENCHMARK_CANDIDATES, + MAX_BENCHMARK_ENSEMBLE_MEMBERS, + MAX_BENCHMARK_EVENTS_PER_WINDOW, + MAX_BENCHMARK_HOOK_METRICS, + MAX_BENCHMARK_PAYLOAD_BYTES, + MAX_BENCHMARK_REASON_CODES, + MAX_BENCHMARK_SCENARIOS, + MAX_BENCHMARK_SLICES, + BenchmarkCandidateKind, + BenchmarkCandidateScoreV1, + BenchmarkCandidateV1, + BenchmarkCandidateWindowV1, + BenchmarkControlKind, + BenchmarkEventV1, + BenchmarkExecutionEvidenceV1, + BenchmarkGeneratorV1, + BenchmarkProfileV1, + BenchmarkScenarioV1, + BenchmarkSliceScoreV1, + BenchmarkSplitKind, + BenchmarkSplitV1, + ReverseDegradationBenchmarkManifestV1, + ReverseDegradationBenchmarkV1, + ReverseDegradationScorecardV1, + benchmark_events_from_empirical_overlay, + build_benchmark_control_events, + build_benchmark_control_windows, + degrade_benchmark_window, + generate_benchmark_candidate_window, + validate_benchmark_information_boundary, +) +from histdatacom.synthetic.benchmark_corpus import ( + BENCHMARK_CANDIDATE_REPORT_SCHEMA_VERSION, + BENCHMARK_CORPUS_PROFILE_SCHEMA_VERSION, + BENCHMARK_SOURCE_PARTITION_SCHEMA_VERSION, + BENCHMARK_WINDOW_PARTITION_SCHEMA_VERSION, + PREDECLARED_GATE_COMMIT, + REVERSE_DEGRADATION_CAMPAIGN_SCHEMA_VERSION, + REVERSE_DEGRADATION_CORPUS_SCHEMA_VERSION, + BenchmarkCandidateReportV1, + BenchmarkSourcePartitionV1, + BenchmarkWindowPartitionV1, + ReverseDegradationBenchmarkCampaignV1, + ReverseDegradationBenchmarkCorpusV1, + ReverseDegradationCorpusProfileV1, + audit_holdout_neighbor_leakage, + build_reverse_degradation_benchmark_corpus, + read_reverse_degradation_benchmark_campaign, + read_reverse_degradation_benchmark_corpus, + replay_reverse_degradation_benchmark_corpus, + run_reverse_degradation_benchmark_campaign, + write_reverse_degradation_benchmark_artifacts, +) +from histdatacom.synthetic.benchmark_gates import ( + BENCHMARK_GATE_CHECK_SCHEMA_VERSION, + BENCHMARK_GATE_OBSERVATION_SCHEMA_VERSION, + BENCHMARK_GATE_REQUIREMENT_SCHEMA_VERSION, + BENCHMARK_PROMOTION_DECISION_SCHEMA_VERSION, + BENCHMARK_PROMOTION_GATE_POLICY_SCHEMA_VERSION, + DEFAULT_BENCHMARK_GATE_ASSET, + BenchmarkGateCheckV1, + BenchmarkGateComparator, + BenchmarkGateObservationV1, + BenchmarkGateRequirementV1, + BenchmarkGateScope, + BenchmarkGateSeverity, + BenchmarkGateStatus, + BenchmarkPromotionDecisionV1, + BenchmarkPromotionGatePolicyV1, + evaluate_benchmark_promotion_gates, + load_default_benchmark_promotion_gate_policy, +) +from histdatacom.synthetic.broker_transfer import ( + BROKER_BENCHMARK_COMPARISON_SCHEMA_VERSION, + BROKER_CONDITIONED_PROPOSAL_SCHEMA_VERSION, + BROKER_PROFILE_SELECTION_SCHEMA_VERSION, + BROKER_RENDERED_GROUP_SCHEMA_VERSION, + BROKER_RENDER_LINEAGE_SCHEMA_VERSION, + BROKER_TRANSFER_CONFIG_SCHEMA_VERSION, + BROKER_TRANSFER_ENGINE_ID, + BROKER_TRANSFER_ENGINE_VERSION, + BROKER_TRANSFER_MANIFEST_SCHEMA_VERSION, + BrokerBenchmarkComparisonV1, + BrokerConditionedProposalV1, + BrokerProfileSelectionV1, + BrokerRenderedGroupV1, + BrokerRenderLineageV1, + BrokerTransferConfigV1, + BrokerTransferManifestV1, + BrokerTransferStatus, + compare_broker_benchmark_results, + condition_broker_proposal, + render_broker_delivery, + select_broker_profile, +) +from histdatacom.synthetic.certification import ( + CERTIFICATION_ARTIFACT_SCHEMA_VERSION, + CERTIFICATION_CHECK_RESULT_SCHEMA_VERSION, + CERTIFICATION_GATE_RESULT_SCHEMA_VERSION, + CERTIFICATION_OBSERVATION_SCHEMA_VERSION, + CERTIFICATION_REQUIREMENT_SCHEMA_VERSION, + DEFAULT_CERTIFICATION_MAX_ARTIFACTS, + DEFAULT_CERTIFICATION_MAX_METADATA_ITEMS, + DEFAULT_CERTIFICATION_MAX_OBSERVATIONS, + DEFAULT_CERTIFICATION_MAX_PAYLOAD_BYTES, + DEFAULT_CERTIFICATION_MAX_REQUIREMENTS, + DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH, + EURUSD_TRIANGLE_COMMON_START_PERIOD, + MODERN_REFERENCE_DELIVERY_CLAIM, + MODERN_REFERENCE_DELIVERY_MODE, + PROMOTION_ONLY_CHECK_IDS, + RECONSTRUCTION_CERTIFICATION_DOSSIER_SCHEMA_VERSION, + RECONSTRUCTION_CERTIFICATION_DOSSIER_V2_SCHEMA_VERSION, + RECONSTRUCTION_CERTIFICATION_POLICY_SCHEMA_VERSION, + RECONSTRUCTION_CERTIFICATION_POLICY_V2_SCHEMA_VERSION, + CertificationArtifactV1, + CertificationCheckResultV1, + CertificationCheckStatus, + CertificationComparator, + CertificationGate, + CertificationGateResultV1, + CertificationObservationV1, + CertificationRequirementV1, + CertificationState, + ReconstructionCertificationDossierV1, + ReconstructionCertificationDossierV2, + ReconstructionCertificationPolicyV1, + ReconstructionCertificationPolicyV2, + eurusd_triangle_certification_policy, + evaluate_modern_reference_reconstruction_certification, + evaluate_reconstruction_certification, + load_modern_reference_reconstruction_certification_dossier, + load_reconstruction_certification_dossier, + modern_reference_triangle_certification_policy, + write_modern_reference_reconstruction_certification_dossier, + write_reconstruction_certification_dossier, +) +from histdatacom.synthetic.certification_campaign import ( + CAMPAIGN_MANIFEST_EVIDENCE_KEY, + CERTIFICATION_CAMPAIGN_ARTIFACT_SCHEMA_VERSION, + CERTIFICATION_CAMPAIGN_OBSERVATION_SCHEMA_VERSION, + CERTIFICATION_CAMPAIGN_RESULT_SCHEMA_VERSION, + CERTIFICATION_CAMPAIGN_SPEC_SCHEMA_VERSION, + CERTIFICATION_METHODOLOGY_REPORT_SCHEMA_VERSION, + METHODOLOGY_REPORT_EVIDENCE_KEY, + CertificationCampaignArtifactV1, + CertificationCampaignObservationV1, + ModernReferenceCertificationCampaignResultV1, + ModernReferenceCertificationCampaignSpecV1, + read_modern_reference_certification_campaign_spec, + run_modern_reference_certification_campaign, +) +from histdatacom.synthetic.contracts import ( + SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION, + SYNTHETIC_EVENT_SCHEMA_VERSION, + SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION, + SyntheticEnsembleManifestV1, + SyntheticEnsembleMemberV1, + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + canonical_contract_json, + derive_anchor_interval_id, + read_synthetic_event_stream_parquet, + synthetic_event_arrow_schema, + synthetic_event_stream_from_arrow, + synthetic_event_stream_from_parquet_bytes, + synthetic_event_stream_to_arrow, + synthetic_event_stream_to_parquet_bytes, + write_synthetic_event_stream_parquet, +) +from histdatacom.synthetic.delivery import ( + MODERN_REFERENCE_DELIVERY_ENGINE_ID, + MODERN_REFERENCE_DELIVERY_ENGINE_VERSION, + RECONSTRUCTION_DELIVERED_GROUP_SCHEMA_VERSION, + RECONSTRUCTION_DELIVERY_MANIFEST_SCHEMA_VERSION, + ReconstructionDeliveredGroupV1, + ReconstructionDeliveryManifestV1, + ReconstructionDeliveryStatus, + project_modern_reference_delivery, + reconstruction_streams_content_sha256, +) +from histdatacom.synthetic.persistence import ( + DEFAULT_ESTIMATED_BYTES_PER_EVENT, + DEFAULT_ESTIMATED_COMPRESSION_RATIO, + DEFAULT_RECONSTRUCTION_ROW_GROUP_SIZE, + RECONSTRUCTION_BYTE_HASH_ALGORITHM, + RECONSTRUCTION_COMPRESSION, + RECONSTRUCTION_CONSTRAINT_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_DELIVERY_QUALITY_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_ENSEMBLE_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_LOGICAL_HASH_ALGORITHM, + RECONSTRUCTION_MANIFEST_ARTIFACT_KIND, + RECONSTRUCTION_MANIFEST_FILENAME, + RECONSTRUCTION_PARTITION_SCHEMA_VERSION, + RECONSTRUCTION_PRODUCT_DIRECTORY, + RECONSTRUCTION_PRODUCT_SCHEMA_VERSION, + RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION, + RECONSTRUCTION_QUALITY_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_REPLAY_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_RETENTION_PLAN_SCHEMA_VERSION, + RECONSTRUCTION_SOURCE_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_WRITER_ID, + PublishedReconstructionV1, + PublishedReconstructionV2, + ReconstructionConstraintManifestV1, + ReconstructionDeliveryQualityManifestV1, + ReconstructionEnsembleManifestV1, + ReconstructionPersistenceError, + ReconstructionProductManifestV1, + ReconstructionProductManifestV2, + ReconstructionProductPartitionV1, + ReconstructionQualityManifestV1, + ReconstructionReplayManifestV1, + ReconstructionRetentionPlanV1, + ReconstructionSourceManifestV1, + ReconstructionStoragePreflightError, + StagedReconstructionPublicationV1, + StagedReconstructionPublicationV2, + cleanup_reconstruction_scratch, + commit_reconstruction_publication, + commit_delivery_reconstruction_publication, + discover_reconstruction_manifests, + estimate_reconstruction_retention, + iter_reconstruction_event_batches, + load_reconstruction_manifest, + publish_reconstruction_group, + read_reconstruction_streams, + reconstruction_logical_content_sha256, + reconstruction_parquet_paths, + scan_reconstruction_events_polars, + stage_reconstruction_publication, + stage_delivery_reconstruction_publication, + verify_reconstruction_publication, +) +from histdatacom.synthetic.ensembles import ( + DEFAULT_ENSEMBLE_HORIZONS_NS, + DEFAULT_ENSEMBLE_MAX_PAYLOAD_BYTES, + DEFAULT_ENSEMBLE_MAX_SAMPLES, + DEFAULT_ENSEMBLE_MAX_SLICES, + ENSEMBLE_ARTIFACT_DIGEST_SCHEMA_VERSION, + ENSEMBLE_CALIBRATION_FIT_SPLIT, + ENSEMBLE_CALIBRATION_METRIC_GROUPS, + ENSEMBLE_CALIBRATION_METRIC_NAMES, + ENSEMBLE_CALIBRATION_REPORT_SCHEMA_VERSION, + ENSEMBLE_CALIBRATION_SAMPLE_SCHEMA_VERSION, + ENSEMBLE_CALIBRATION_EVALUATION_SPLIT, + ENSEMBLE_CONFIDENCE_QUANTITY, + ENSEMBLE_CONFIG_SCHEMA_VERSION, + ENSEMBLE_DIVERSITY_SUMMARY_SCHEMA_VERSION, + ENSEMBLE_ENGINE_ID, + ENSEMBLE_ENGINE_VERSION, + ENSEMBLE_MEMBER_CALIBRATION_SCHEMA_VERSION, + ENSEMBLE_MEMBER_PLAN_SCHEMA_VERSION, + ENSEMBLE_MEMBER_SELECTION_SCHEMA_VERSION, + ENSEMBLE_METRIC_CALIBRATION_SCHEMA_VERSION, + ENSEMBLE_OUTCOME_SUMMARY_SCHEMA_VERSION, + ENSEMBLE_PLAN_SCHEMA_VERSION, + ENSEMBLE_PRIMARY_SELECTION_BASIS, + ENSEMBLE_REGENERATION_REQUEST_SCHEMA_VERSION, + ENSEMBLE_STORAGE_ESTIMATE_SCHEMA_VERSION, + ENSEMBLE_STRATUM_SCHEMA_VERSION, + EnsembleArtifactDigestV1, + EnsembleArtifactKind, + EnsembleCalibrationConfigV1, + EnsembleCalibrationReportV1, + EnsembleCalibrationSampleV1, + EnsembleCalibrationStratumV1, + EnsembleDiversityStatus, + EnsembleDiversitySummaryV1, + EnsembleMemberCalibrationV1, + EnsembleMemberPlanV1, + EnsembleMemberSelectionV1, + EnsembleMemberStatus, + EnsembleMetricCalibrationV1, + EnsembleMetricStatus, + EnsembleOutcomeSummaryV1, + EnsembleRegenerationRequestV1, + EnsembleReportStatus, + EnsembleStorageEstimateV1, + ReconstructionEnsemblePlanV1, + benchmark_ensemble_calibration_sample, + benchmark_logical_content_sha256, + build_ensemble_regeneration_request, + calibrate_reconstruction_ensemble, + ensemble_logical_content_sha256, + estimate_reconstruction_ensemble_resources, + plan_reconstruction_ensemble, + verify_ensemble_regeneration, +) +from histdatacom.synthetic.generation import ( + CANDIDATE_ONLY_CONSTRAINT_SET_ID, + EMPIRICAL_MOTIF_GENERATOR_ID, + EMPIRICAL_MOTIF_GENERATOR_VERSION, + MAX_MOTIF_GENERATED_EVENTS_PER_INTERVAL, + MAX_MOTIF_TRANSFORMATIONS_PER_INTERVAL, + MOTIF_CANDIDATE_BATCH_SCHEMA_VERSION, + MOTIF_EVENT_LINEAGE_SCHEMA_VERSION, + MOTIF_GENERATOR_CONFIG_SCHEMA_VERSION, + MOTIF_TRANSFORMATION_CONFIDENCE_QUANTITY, + MOTIF_TRANSFORMATION_SCHEMA_VERSION, + EmpiricalMotifBenchmarkGeneratorV1, + EmpiricalMotifCandidateBatchV1, + EmpiricalMotifEventLineageV1, + EmpiricalMotifGeneratorConfigV1, + EmpiricalMotifTransformationV1, + MotifGenerationDecision, + MotifGenerationStatus, + generate_empirical_motif_candidates, +) +from histdatacom.synthetic.carving import ( + CARVED_CANDIDATE_BATCH_SCHEMA_VERSION, + CARVING_CONDITION_POLICY_SCHEMA_VERSION, + CARVING_CONSTRAINT_SET_SCHEMA_VERSION, + CARVING_EVENT_LINEAGE_SCHEMA_VERSION, + CARVING_FINGERPRINT_EVIDENCE_SCHEMA_VERSION, + CARVING_QUARANTINE_SCHEMA_VERSION, + CARVING_REJECTION_EXAMPLE_SCHEMA_VERSION, + CARVING_VALIDATION_EVIDENCE_SCHEMA_VERSION, + HISTORICAL_CARVING_ENGINE_ID, + HISTORICAL_CARVING_ENGINE_VERSION, + HISTORICAL_CARVING_RULE_PRECEDENCE, + CarvingBatchStatus, + CarvingEventAction, + CarvingFingerprintEvidenceV1, + CarvingReason, + HistoricalCarvedCandidateBatchV1, + HistoricalCarvingConditionPolicyV1, + HistoricalCarvingConstraintSetV1, + HistoricalCarvingEventLineageV1, + HistoricalCarvingQuarantineV1, + HistoricalCarvingRejectionExampleV1, + HistoricalCarvingValidationEvidenceV1, + carve_empirical_motif_candidates, +) +from histdatacom.synthetic.cross_currency import ( + CROSS_CURRENCY_CONDITION_SCHEMA_VERSION, + CROSS_CURRENCY_CONFIG_SCHEMA_VERSION, + CROSS_CURRENCY_ENGINE_ID, + CROSS_CURRENCY_ENGINE_VERSION, + CROSS_CURRENCY_EXCLUDED_SPAN_SCHEMA_VERSION, + CROSS_CURRENCY_GROUP_SCHEMA_VERSION, + CROSS_CURRENCY_PROJECTION_LINEAGE_SCHEMA_VERSION, + CROSS_CURRENCY_RELATIONSHIP_SCHEMA_VERSION, + CROSS_CURRENCY_RELATIONSHIP_SUPPORT_SCHEMA_VERSION, + CROSS_CURRENCY_RESIDUAL_SLICE_SCHEMA_VERSION, + CROSS_CURRENCY_SYMBOL_COVERAGE_SCHEMA_VERSION, + CROSS_CURRENCY_VALIDATION_SCHEMA_VERSION, + CROSS_CURRENCY_WINDOW_PLAN_SCHEMA_VERSION, + EURUSD_TRIANGLE_SYMBOLS, + CrossCurrencyConditionV1, + CrossCurrencyCoverageStatus, + CrossCurrencyExcludedReason, + CrossCurrencyExcludedSpanV1, + CrossCurrencyGroupStatus, + CrossCurrencyProjectionLineageV1, + CrossCurrencyReconciledGroupV1, + CrossCurrencyReconciliationConfigV1, + CrossCurrencyRelationshipKind, + CrossCurrencyRelationshipSupportV1, + CrossCurrencyRelationshipV1, + CrossCurrencyResidualSliceV1, + CrossCurrencySymbolCoverageV1, + CrossCurrencyValidationReportV1, + CrossCurrencyValidationStage, + CrossCurrencyValidationStatus, + CrossCurrencyWindowPlanStatus, + CrossCurrencyWindowPlanV1, + cross_currency_quality_report, + cross_currency_series_inputs, + eurusd_triangle_reconciliation_config, + plan_cross_currency_windows, + reconcile_cross_currency_window, + validate_cross_currency_atomic_manifest, + validate_cross_currency_output, +) +from histdatacom.synthetic.information import ( + DEFAULT_INFORMATION_AUDIT_FINDINGS, + MAX_INFORMATION_AUDIT_FINDINGS, + MAX_INFORMATION_INPUTS, + RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION, + InformationAuditFindingV1, + InformationAuditReportV1, + InformationAuditRule, + InformationInputKind, + InformationLeakageError, + InformationMode, + InformationScope, + InformationSplitKind, + InformationStage, + ReconstructionInformationInputV1, + ReconstructionInformationManifestV1, + ReconstructionInformationPolicyV1, + ReconstructionInformationSplitV1, + audit_reconstruction_information, + reconstruction_information_window_plan_id, + require_reconstruction_information_audit, +) +from histdatacom.synthetic.motifs import ( + MAX_REFERENCE_MOTIF_ARTIFACT_BYTES, + MAX_REFERENCE_MOTIF_EVENTS, + MAX_REFERENCE_MOTIF_FRAGMENTS, + MAX_REFERENCE_MOTIF_MATCHES, + MAX_REFERENCE_MOTIF_SOURCE_WINDOWS, + REFERENCE_MOTIF_ARTIFACT_KIND, + REFERENCE_MOTIF_BACKOFF_ATTEMPT_SCHEMA_VERSION, + REFERENCE_MOTIF_BACKOFF_LEVELS, + REFERENCE_MOTIF_CONDITION_SCHEMA_VERSION, + REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION, + REFERENCE_MOTIF_FEATURE_SCHEMA_VERSION, + REFERENCE_MOTIF_INDEX_CONFIG_SCHEMA_VERSION, + REFERENCE_MOTIF_INDEX_SCHEMA_VERSION, + REFERENCE_MOTIF_MATCH_SCHEMA_VERSION, + REFERENCE_MOTIF_METRIC_NAMES, + REFERENCE_MOTIF_QUERY_RESULT_SCHEMA_VERSION, + REFERENCE_MOTIF_QUERY_SCHEMA_VERSION, + REFERENCE_MOTIF_SOURCE_EVENT_SCHEMA_VERSION, + REFERENCE_MOTIF_SOURCE_WINDOW_SCHEMA_VERSION, + REFERENCE_MOTIF_SPLIT_SCHEMA_VERSION, + REFERENCE_MOTIF_TRANSFORM_SCHEMA_VERSION, + ReferenceMotifBackoffAttemptV1, + ReferenceMotifConditionV1, + ReferenceMotifFragmentV1, + ReferenceMotifIndexConfigV1, + ReferenceMotifIndexV1, + ReferenceMotifLeakageError, + ReferenceMotifMatchV1, + ReferenceMotifQueryResultV1, + ReferenceMotifQueryStatus, + ReferenceMotifQueryV1, + ReferenceMotifResourceLimitError, + ReferenceMotifSourceEventV1, + ReferenceMotifSourceWindowV1, + ReferenceMotifSplitKind, + ReferenceMotifSplitV1, + ReferenceMotifTransformPolicyV1, + ReferenceMotifTransition, + build_reference_motif_index, + extract_reference_motif_fragment, + query_reference_motifs, + read_reference_motif_index, + reference_motif_condition_from_quotes, + reference_motif_information_inputs, + reference_motif_source_window_from_training_frame, + write_reference_motif_index, +) +from histdatacom.synthetic.motif_library import ( + DEFAULT_MODERN_MOTIF_SPLIT_PERIODS, + MODERN_REFERENCE_MOTIF_COVERAGE_SCHEMA_VERSION, + MODERN_REFERENCE_MOTIF_LEAKAGE_SCHEMA_VERSION, + MODERN_REFERENCE_MOTIF_MANIFEST_SCHEMA_VERSION, + MODERN_REFERENCE_MOTIF_PROFILE_SCHEMA_VERSION, + MODERN_REFERENCE_MOTIF_QUALIFICATION_SCHEMA_VERSION, + MODERN_REFERENCE_MOTIF_RESOURCE_SCHEMA_VERSION, + ModernReferenceMotifBuildV1, + ModernReferenceMotifProfileV1, + build_modern_reference_motif_library, + read_modern_reference_motif_artifact, + read_modern_reference_motif_index, + write_modern_reference_motif_artifacts, +) +from histdatacom.synthetic.observation import ( + MAX_OBSERVATION_ARTIFACT_BYTES, + MAX_OBSERVATION_DIAGNOSTIC_SAMPLES, + MAX_OBSERVATION_DURATION_NS, + MAX_OBSERVATION_EVIDENCE, + MAX_OBSERVATION_INPUT_EVENTS, + MAX_OBSERVATION_OUTPUTS_PER_INPUT, + MAX_OBSERVATION_STRATA, + MAX_OBSERVATION_WINDOW_ALIGNMENT_NS, + OBSERVATION_APPLICATION_RESULT_SCHEMA_VERSION, + OBSERVATION_BACKOFF_LEVELS, + OBSERVATION_CARRY_FIELDS, + OBSERVATION_CARRY_STATE_SCHEMA_VERSION, + OBSERVATION_CONTEXT_SCHEMA_VERSION, + OBSERVATION_FIT_CONFIG_SCHEMA_VERSION, + OBSERVATION_FIT_DIAGNOSTICS_SCHEMA_VERSION, + OBSERVATION_FIT_EVIDENCE_SCHEMA_VERSION, + OBSERVATION_INPUT_EVENT_SCHEMA_VERSION, + OBSERVATION_OPERATOR_SCHEMA_VERSION, + OBSERVATION_OUTPUT_EVENT_SCHEMA_VERSION, + OBSERVATION_PARAMETER_NAMES, + OBSERVATION_PARAMETER_SCHEMA_VERSION, + OBSERVATION_STRATUM_SCHEMA_VERSION, + ObservationApplicationResultV1, + ObservationCarryStateV1, + ObservationContextV1, + ObservationFitDiagnosticsV1, + ObservationFitEvidenceV1, + ObservationInputEventV1, + ObservationOperatorFitConfigV1, + ObservationOperatorV1, + ObservationOutputEventV1, + ObservationParameterEstimateV1, + ObservationStratumV1, + fit_observation_operator, + read_observation_operator_artifact, + write_observation_operator, +) +from histdatacom.synthetic.observation_calibration import ( + OBSERVATION_CALIBRATION_CAMPAIGN_V2_SCHEMA_VERSION, + OBSERVATION_CALIBRATION_ENGINE_ID, + OBSERVATION_CALIBRATION_ENGINE_VERSION, + OBSERVATION_CALIBRATION_MECHANISMS, + OBSERVATION_CALIBRATION_PROFILE_V2_SCHEMA_VERSION, + OBSERVATION_CALIBRATION_REQUIRED_PARAMETERS, + OBSERVATION_CALIBRATION_SESSIONS, + OBSERVATION_CALIBRATION_SPLITS, + OBSERVATION_CALIBRATION_TARGET_V2_SCHEMA_VERSION, + OBSERVATION_CALIBRATION_WINDOW_V2_SCHEMA_VERSION, + OBSERVATION_UPDATE_STATES, + ObservationCalibrationCampaignV2, + ObservationCalibrationProfileV2, + ObservationCalibrationTargetV2, + ObservationCalibrationWindowV2, + calibrate_historical_observation_operators, + estimate_paired_observation_evidence, + read_feed_epoch_evidence_v2, + read_observation_calibration_campaign, + write_observation_calibration_campaign, +) +from histdatacom.synthetic.streaming import ( + CARRY_STATE_SCHEMA_VERSION, + EVENT_BATCH_SCHEMA_VERSION, + PARTITION_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_CHECKPOINT_SCHEMA_VERSION, + RECONSTRUCTION_HEARTBEAT_SCHEMA_VERSION, + RECONSTRUCTION_RESOURCE_ESTIMATE_SCHEMA_VERSION, + RECONSTRUCTION_RUN_SCHEMA_VERSION, + RECONSTRUCTION_STORAGE_POLICY_SCHEMA_VERSION, + RECONSTRUCTION_WINDOW_SCHEMA_VERSION, + REJECTION_SUMMARY_SCHEMA_VERSION, + CarryStateV1, + EventBatchV1, + PartitionManifestV1, + ReconstructionCheckpointV1, + ReconstructionCommitPhase, + ReconstructionHeartbeatV1, + ReconstructionResourceEstimateV1, + ReconstructionResourceLimitError, + ReconstructionRunV1, + ReconstructionStoragePolicyV1, + ReconstructionWindowV1, + RejectionSummaryV1, + artifact_ref_for_json_contract, + plan_reconstruction_windows, + validate_reconstruction_window_plan, +) +from histdatacom.synthetic.strategy_sensitivity import ( + DEFAULT_STRATEGY_HORIZONS_NS, + REFERENCE_MOMENTUM_STRATEGY_ID, + REFERENCE_MOMENTUM_STRATEGY_VERSION, + STRATEGY_EVALUATION_CASE_SCHEMA_VERSION, + STRATEGY_EVALUATION_PLAN_SCHEMA_VERSION, + STRATEGY_EVALUATION_POLICY_SCHEMA_VERSION, + STRATEGY_EXECUTION_SPECIFICATION_SCHEMA_VERSION, + STRATEGY_INVALID_FOR_BACKTEST_LABEL, + STRATEGY_QUOTE_SCHEMA_VERSION, + STRATEGY_RESTORATION_RESULT_SCHEMA_VERSION, + STRATEGY_SENSITIVITY_REPORT_SCHEMA_VERSION, + STRATEGY_SIGNAL_SCHEMA_VERSION, + STRATEGY_SLICE_RESULT_SCHEMA_VERSION, + STRATEGY_SPECIFICATION_SCHEMA_VERSION, + STRATEGY_UNCERTAINTY_SUMMARY_SCHEMA_VERSION, + STRATEGY_WINDOW_RESULT_SCHEMA_VERSION, + ReferenceMomentumStrategyV1, + StrategyEvaluationCaseV1, + StrategyEvaluationFailure, + StrategyEvaluationPlanV1, + StrategyEvaluationPolicyV1, + StrategyExecutionSpecificationV1, + StrategyQuoteV1, + StrategyResourceLimitError, + StrategyRestorationResultV1, + StrategySensitivityReportV1, + StrategySide, + StrategySignalEngineV1, + StrategySignalStateV1, + StrategySignalV1, + StrategySliceResultV1, + StrategySourceKind, + StrategySpecificationV1, + StrategyUncertaintySummaryV1, + StrategyWindowResultV1, + StrategyWindowStatus, + evaluate_strategy_sensitivity, + strategy_sensitivity_benchmark_hooks, +) + +_RECONSTRUCTION_PLAN_EXPORTS = frozenset( + { + "ASCII_TICK_SOURCE_KIND", + "DEFAULT_RECONSTRUCTION_BASE_SEED", + "DEFAULT_RECONSTRUCTION_MAX_PARALLEL_WINDOWS", + "DEFAULT_RECONSTRUCTION_REQUEST_WINDOW_LIMIT", + "DEFAULT_RECONSTRUCTION_WINDOW_SIZE_NS", + "FIRST_PARTY_RECONSTRUCTION_HANDLERS", + "IMMUTABLE_ANCHOR_POLICY", + "PLAN_CONFIGURATION_ARTIFACT_KIND", + "PLAN_EXECUTION_MANIFEST_ARTIFACT_KIND", + "RECONSTRUCTION_PLAN_CONFIGURATION_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_EXECUTION_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_REFUSAL_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_RESOURCE_SUMMARY_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_INVENTORY_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_PARTITION_SCHEMA_VERSION", + "SCIENTIFIC_NONCLAIM", + "SOURCE_INVENTORY_ARTIFACT_KIND", + "SYNTHETIC_INFILL_PLAN_ARTIFACT_KIND", + "SYNTHETIC_INFILL_PLAN_SCHEMA_VERSION", + "TICK_ONLY_INPUT_POLICY", + "ReconstructionDeliveryMode", + "ReconstructionPlanCompatibilityError", + "ReconstructionPlanConfigurationV1", + "ReconstructionPlanExecutionManifestV1", + "ReconstructionPlanRefusalCode", + "ReconstructionPlanRefusalV1", + "ReconstructionPlanResourceSummaryV1", + "ReconstructionStagePlanV1", + "ReconstructionSourceInventoryV1", + "ReconstructionSourcePartitionV1", + "SyntheticInfillPlanV1", + "build_synthetic_infill_plan", + "load_reconstruction_stage_plan", + "read_reconstruction_plan_configuration", + "read_reconstruction_plan_execution_manifest", + "read_reconstruction_source_inventory", + "read_synthetic_infill_plan", + "validate_synthetic_infill_plan_for_execution", + "write_synthetic_infill_plan", + } +) + + +def __getattr__(name: str) -> Any: + """Load orchestration-bound planning exports without an import cycle.""" + if name not in _RECONSTRUCTION_PLAN_EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + reconstruction_plan = import_module( + "histdatacom.synthetic.reconstruction_plan" + ) + value = getattr(reconstruction_plan, name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals(), *_RECONSTRUCTION_PLAN_EXPORTS}) + + +__all__ = [ + "ASCII_TICK_SOURCE_KIND", + "DEFAULT_RECONSTRUCTION_BASE_SEED", + "DEFAULT_RECONSTRUCTION_MAX_PARALLEL_WINDOWS", + "DEFAULT_RECONSTRUCTION_REQUEST_WINDOW_LIMIT", + "DEFAULT_RECONSTRUCTION_WINDOW_SIZE_NS", + "FIRST_PARTY_RECONSTRUCTION_HANDLERS", + "IMMUTABLE_ANCHOR_POLICY", + "PLAN_CONFIGURATION_ARTIFACT_KIND", + "PLAN_EXECUTION_MANIFEST_ARTIFACT_KIND", + "RECONSTRUCTION_PLAN_CONFIGURATION_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_EXECUTION_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_REFUSAL_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_RESOURCE_SUMMARY_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_INVENTORY_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_PARTITION_SCHEMA_VERSION", + "SCIENTIFIC_NONCLAIM", + "SOURCE_INVENTORY_ARTIFACT_KIND", + "SYNTHETIC_INFILL_PLAN_ARTIFACT_KIND", + "SYNTHETIC_INFILL_PLAN_SCHEMA_VERSION", + "TICK_ONLY_INPUT_POLICY", + "ReconstructionDeliveryMode", + "ReconstructionPlanCompatibilityError", + "ReconstructionPlanConfigurationV1", + "ReconstructionPlanExecutionManifestV1", + "ReconstructionPlanRefusalCode", + "ReconstructionPlanRefusalV1", + "ReconstructionPlanResourceSummaryV1", + "ReconstructionStagePlanV1", + "ReconstructionSourceInventoryV1", + "ReconstructionSourcePartitionV1", + "SyntheticInfillPlanV1", + "build_synthetic_infill_plan", + "load_reconstruction_stage_plan", + "read_reconstruction_plan_configuration", + "read_reconstruction_plan_execution_manifest", + "read_reconstruction_source_inventory", + "read_synthetic_infill_plan", + "validate_synthetic_infill_plan_for_execution", + "write_synthetic_infill_plan", + "DEFAULT_MODERN_MOTIF_SPLIT_PERIODS", + "MODERN_REFERENCE_MOTIF_COVERAGE_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_LEAKAGE_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_MANIFEST_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_PROFILE_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_QUALIFICATION_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_RESOURCE_SCHEMA_VERSION", + "REFERENCE_MOTIF_FEATURE_SCHEMA_VERSION", + "ACTIVITY_EVENT_COLUMNS", + "ACTIVITY_REVERSE_DEGRADATION_METRICS", + "DEFAULT_ACTIVITY_BATCH_SIZE", + "DEFAULT_ACTIVITY_MAX_PAYLOAD_BYTES", + "DEFAULT_ACTIVITY_MAX_PROVENANCE_VALUES", + "DEFAULT_ACTIVITY_MAX_SLICES", + "DEFAULT_ACTIVITY_MAX_SYMBOLS", + "DEFAULT_ACTIVITY_ROUNDING_DIGITS", + "RECONSTRUCTION_ACTIVITY_BENCHMARK_EVIDENCE_SCHEMA_VERSION", + "RECONSTRUCTION_ACTIVITY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_ACTIVITY_METRIC_SCHEMA_VERSION", + "RECONSTRUCTION_ACTIVITY_POLICY_SCHEMA_VERSION", + "RECONSTRUCTION_ACTIVITY_SLICE_SCHEMA_VERSION", + "ActivityAggregationSemantics", + "ActivityMetricSemantics", + "ActivitySliceScope", + "ActivityVolumeState", + "DEFAULT_DERIVED_BAR_BATCH_SIZE", + "DEFAULT_DERIVED_BAR_MAX_BARS", + "DEFAULT_DERIVED_BAR_MAX_PROVENANCE_VALUES", + "DEFAULT_DERIVED_BAR_MAX_SYMBOLS", + "DEFAULT_DERIVED_BAR_ROUNDING_DIGITS", + "DEFAULT_DERIVED_BAR_ROW_GROUP_SIZE", + "DEFAULT_DERIVED_BAR_WRITE_BUFFER_ROWS", + "DERIVED_BAR_ARROW_COLUMNS", + "DERIVED_BAR_EVENT_COLUMNS", + "DERIVED_BAR_INTERVAL_SCHEMA_VERSION", + "DERIVED_BAR_MANIFEST_ARTIFACT_KIND", + "DERIVED_BAR_PARTITION_SCHEMA_VERSION", + "DERIVED_BAR_POLICY_SCHEMA_VERSION", + "DERIVED_BAR_PRODUCT_DIRECTORY", + "DERIVED_BAR_PRODUCT_SCHEMA_VERSION", + "DERIVED_BAR_SCHEMA_VERSION", + "STANDARD_DERIVED_BAR_INTERVALS", + "DerivedBarIntervalV1", + "DerivedBarPartitionV1", + "DerivedBarPersistenceError", + "DerivedBarPolicyV1", + "DerivedBarProductManifestV1", + "DerivedBarV1", + "PublishedDerivedBarsV1", + "ReconstructionActivityBenchmarkEvidenceV1", + "ReconstructionActivityManifestV1", + "ReconstructionActivityMetricV1", + "ReconstructionActivityPolicyV1", + "ReconstructionActivitySliceV1", + "StagedDerivedBarPublicationV1", + "activity_bar_projection_semantics", + "activity_metric_definitions", + "commit_derived_bar_publication", + "derive_reconstruction_bars", + "derived_bar_arrow_schema", + "discover_derived_bar_manifests", + "reconstruction_activity_benchmark_evidence", + "read_reconstruction_activity_manifest", + "iter_committed_reconstruction_bars", + "iter_derived_bar_batches", + "load_derived_bar_manifest", + "publish_derived_bars", + "scan_derived_bars_polars", + "stage_derived_bar_publication", + "summarize_committed_reconstruction_activity", + "summarize_reconstruction_activity", + "summarize_reconstruction_activity_streams", + "write_reconstruction_activity_manifest", + "verify_derived_bar_publication", + "BENCHMARK_CANDIDATE_SCHEMA_VERSION", + "CROSS_CURRENCY_CONDITION_SCHEMA_VERSION", + "CROSS_CURRENCY_CONFIG_SCHEMA_VERSION", + "CROSS_CURRENCY_ENGINE_ID", + "CROSS_CURRENCY_ENGINE_VERSION", + "CROSS_CURRENCY_EXCLUDED_SPAN_SCHEMA_VERSION", + "CROSS_CURRENCY_GROUP_SCHEMA_VERSION", + "CROSS_CURRENCY_PROJECTION_LINEAGE_SCHEMA_VERSION", + "CROSS_CURRENCY_RELATIONSHIP_SCHEMA_VERSION", + "CROSS_CURRENCY_RELATIONSHIP_SUPPORT_SCHEMA_VERSION", + "CROSS_CURRENCY_RESIDUAL_SLICE_SCHEMA_VERSION", + "CROSS_CURRENCY_SYMBOL_COVERAGE_SCHEMA_VERSION", + "CROSS_CURRENCY_VALIDATION_SCHEMA_VERSION", + "CROSS_CURRENCY_WINDOW_PLAN_SCHEMA_VERSION", + "CrossCurrencyConditionV1", + "CrossCurrencyCoverageStatus", + "CrossCurrencyExcludedReason", + "CrossCurrencyExcludedSpanV1", + "CrossCurrencyGroupStatus", + "CrossCurrencyProjectionLineageV1", + "CrossCurrencyReconciledGroupV1", + "CrossCurrencyReconciliationConfigV1", + "CrossCurrencyRelationshipKind", + "CrossCurrencyRelationshipSupportV1", + "CrossCurrencyRelationshipV1", + "CrossCurrencyResidualSliceV1", + "CrossCurrencySymbolCoverageV1", + "CrossCurrencyValidationReportV1", + "CrossCurrencyValidationStage", + "CrossCurrencyValidationStatus", + "CrossCurrencyWindowPlanStatus", + "CrossCurrencyWindowPlanV1", + "EURUSD_TRIANGLE_SYMBOLS", + "BENCHMARK_CANDIDATE_SCORE_SCHEMA_VERSION", + "BENCHMARK_CANDIDATE_WINDOW_SCHEMA_VERSION", + "BENCHMARK_EVENT_SCHEMA_VERSION", + "BENCHMARK_EXECUTION_EVIDENCE_SCHEMA_VERSION", + "BENCHMARK_MANIFEST_SCHEMA_VERSION", + "BENCHMARK_PROFILE_SCHEMA_VERSION", + "BENCHMARK_SCENARIO_SCHEMA_VERSION", + "BENCHMARK_SCORECARD_SCHEMA_VERSION", + "BENCHMARK_SLICE_SCORE_SCHEMA_VERSION", + "BENCHMARK_SPLIT_SCHEMA_VERSION", + "DEFAULT_BENCHMARK_MAX_CANDIDATES", + "DEFAULT_BENCHMARK_MAX_EVENTS_PER_WINDOW", + "DEFAULT_BENCHMARK_MAX_HOOK_METRICS", + "DEFAULT_BENCHMARK_MAX_PAYLOAD_BYTES", + "DEFAULT_BENCHMARK_MAX_REASON_CODES", + "DEFAULT_BENCHMARK_MAX_SCENARIOS", + "DEFAULT_BENCHMARK_MAX_SLICES", + "DEFAULT_BENCHMARK_ROUNDING_DIGITS", + "DEFAULT_INFORMATION_AUDIT_FINDINGS", + "CANDIDATE_ONLY_CONSTRAINT_SET_ID", + "CARVED_CANDIDATE_BATCH_SCHEMA_VERSION", + "CARVING_CONDITION_POLICY_SCHEMA_VERSION", + "CARVING_CONSTRAINT_SET_SCHEMA_VERSION", + "CARVING_EVENT_LINEAGE_SCHEMA_VERSION", + "CARVING_FINGERPRINT_EVIDENCE_SCHEMA_VERSION", + "CARVING_QUARANTINE_SCHEMA_VERSION", + "CARVING_REJECTION_EXAMPLE_SCHEMA_VERSION", + "CARVING_VALIDATION_EVIDENCE_SCHEMA_VERSION", + "EMPIRICAL_MOTIF_GENERATOR_ID", + "EMPIRICAL_MOTIF_GENERATOR_VERSION", + "HISTORICAL_CARVING_ENGINE_ID", + "HISTORICAL_CARVING_ENGINE_VERSION", + "HISTORICAL_CARVING_RULE_PRECEDENCE", + "MAX_REFERENCE_MOTIF_ARTIFACT_BYTES", + "MAX_REFERENCE_MOTIF_EVENTS", + "MAX_REFERENCE_MOTIF_FRAGMENTS", + "MAX_REFERENCE_MOTIF_MATCHES", + "MAX_REFERENCE_MOTIF_SOURCE_WINDOWS", + "MAX_INFORMATION_AUDIT_FINDINGS", + "MAX_INFORMATION_INPUTS", + "MAX_MOTIF_GENERATED_EVENTS_PER_INTERVAL", + "MAX_MOTIF_TRANSFORMATIONS_PER_INTERVAL", + "MAX_BENCHMARK_CANDIDATES", + "MAX_BENCHMARK_ENSEMBLE_MEMBERS", + "MAX_BENCHMARK_EVENTS_PER_WINDOW", + "MAX_BENCHMARK_HOOK_METRICS", + "MAX_BENCHMARK_PAYLOAD_BYTES", + "MAX_BENCHMARK_REASON_CODES", + "MAX_BENCHMARK_SCENARIOS", + "MAX_BENCHMARK_SLICES", + "MAX_OBSERVATION_ARTIFACT_BYTES", + "MAX_OBSERVATION_DIAGNOSTIC_SAMPLES", + "MAX_OBSERVATION_DURATION_NS", + "MAX_OBSERVATION_EVIDENCE", + "MAX_OBSERVATION_INPUT_EVENTS", + "MAX_OBSERVATION_OUTPUTS_PER_INPUT", + "MAX_OBSERVATION_STRATA", + "MAX_OBSERVATION_WINDOW_ALIGNMENT_NS", + "OBSERVATION_APPLICATION_RESULT_SCHEMA_VERSION", + "OBSERVATION_BACKOFF_LEVELS", + "OBSERVATION_CARRY_FIELDS", + "OBSERVATION_CARRY_STATE_SCHEMA_VERSION", + "OBSERVATION_CONTEXT_SCHEMA_VERSION", + "OBSERVATION_FIT_CONFIG_SCHEMA_VERSION", + "OBSERVATION_FIT_DIAGNOSTICS_SCHEMA_VERSION", + "OBSERVATION_FIT_EVIDENCE_SCHEMA_VERSION", + "OBSERVATION_INPUT_EVENT_SCHEMA_VERSION", + "OBSERVATION_OPERATOR_SCHEMA_VERSION", + "OBSERVATION_OUTPUT_EVENT_SCHEMA_VERSION", + "OBSERVATION_PARAMETER_NAMES", + "OBSERVATION_PARAMETER_SCHEMA_VERSION", + "OBSERVATION_STRATUM_SCHEMA_VERSION", + "OBSERVATION_CALIBRATION_CAMPAIGN_V2_SCHEMA_VERSION", + "OBSERVATION_CALIBRATION_ENGINE_ID", + "OBSERVATION_CALIBRATION_ENGINE_VERSION", + "OBSERVATION_CALIBRATION_MECHANISMS", + "OBSERVATION_CALIBRATION_PROFILE_V2_SCHEMA_VERSION", + "OBSERVATION_CALIBRATION_REQUIRED_PARAMETERS", + "OBSERVATION_CALIBRATION_SESSIONS", + "OBSERVATION_CALIBRATION_SPLITS", + "OBSERVATION_CALIBRATION_TARGET_V2_SCHEMA_VERSION", + "OBSERVATION_CALIBRATION_WINDOW_V2_SCHEMA_VERSION", + "OBSERVATION_UPDATE_STATES", + "MOTIF_CANDIDATE_BATCH_SCHEMA_VERSION", + "MOTIF_EVENT_LINEAGE_SCHEMA_VERSION", + "MOTIF_GENERATOR_CONFIG_SCHEMA_VERSION", + "MOTIF_TRANSFORMATION_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION", + "REFERENCE_MOTIF_ARTIFACT_KIND", + "REFERENCE_MOTIF_BACKOFF_ATTEMPT_SCHEMA_VERSION", + "REFERENCE_MOTIF_BACKOFF_LEVELS", + "REFERENCE_MOTIF_CONDITION_SCHEMA_VERSION", + "REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION", + "REFERENCE_MOTIF_INDEX_CONFIG_SCHEMA_VERSION", + "REFERENCE_MOTIF_INDEX_SCHEMA_VERSION", + "REFERENCE_MOTIF_MATCH_SCHEMA_VERSION", + "REFERENCE_MOTIF_METRIC_NAMES", + "REFERENCE_MOTIF_QUERY_RESULT_SCHEMA_VERSION", + "REFERENCE_MOTIF_QUERY_SCHEMA_VERSION", + "REFERENCE_MOTIF_SOURCE_EVENT_SCHEMA_VERSION", + "REFERENCE_MOTIF_SOURCE_WINDOW_SCHEMA_VERSION", + "REFERENCE_MOTIF_SPLIT_SCHEMA_VERSION", + "REFERENCE_MOTIF_TRANSFORM_SCHEMA_VERSION", + "MODERN_REFERENCE_DELIVERY_ENGINE_ID", + "MODERN_REFERENCE_DELIVERY_ENGINE_VERSION", + "SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION", + "SYNTHETIC_EVENT_SCHEMA_VERSION", + "SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION", + "DEFAULT_ESTIMATED_BYTES_PER_EVENT", + "DEFAULT_ESTIMATED_COMPRESSION_RATIO", + "DEFAULT_RECONSTRUCTION_ROW_GROUP_SIZE", + "RECONSTRUCTION_BYTE_HASH_ALGORITHM", + "RECONSTRUCTION_COMPRESSION", + "RECONSTRUCTION_CONSTRAINT_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_DELIVERED_GROUP_SCHEMA_VERSION", + "RECONSTRUCTION_DELIVERY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_DELIVERY_QUALITY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_ENSEMBLE_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_LOGICAL_HASH_ALGORITHM", + "RECONSTRUCTION_MANIFEST_ARTIFACT_KIND", + "RECONSTRUCTION_MANIFEST_FILENAME", + "RECONSTRUCTION_PARTITION_SCHEMA_VERSION", + "RECONSTRUCTION_PRODUCT_DIRECTORY", + "RECONSTRUCTION_PRODUCT_SCHEMA_VERSION", + "RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION", + "RECONSTRUCTION_QUALITY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_REPLAY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_RETENTION_PLAN_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_WRITER_ID", + "CARRY_STATE_SCHEMA_VERSION", + "EVENT_BATCH_SCHEMA_VERSION", + "PARTITION_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_CHECKPOINT_SCHEMA_VERSION", + "RECONSTRUCTION_HEARTBEAT_SCHEMA_VERSION", + "RECONSTRUCTION_RESOURCE_ESTIMATE_SCHEMA_VERSION", + "RECONSTRUCTION_RUN_SCHEMA_VERSION", + "RECONSTRUCTION_STORAGE_POLICY_SCHEMA_VERSION", + "RECONSTRUCTION_WINDOW_SCHEMA_VERSION", + "REJECTION_SUMMARY_SCHEMA_VERSION", + "CarryStateV1", + "CarvingBatchStatus", + "CarvingEventAction", + "CarvingFingerprintEvidenceV1", + "CarvingReason", + "BenchmarkCandidateKind", + "BenchmarkCandidateScoreV1", + "BenchmarkCandidateV1", + "BenchmarkCandidateWindowV1", + "BenchmarkControlKind", + "BenchmarkEventV1", + "BenchmarkExecutionEvidenceV1", + "BenchmarkGeneratorV1", + "BenchmarkProfileV1", + "BenchmarkScenarioV1", + "BenchmarkSliceScoreV1", + "BenchmarkSplitKind", + "BenchmarkSplitV1", + "EventBatchV1", + "EmpiricalMotifBenchmarkGeneratorV1", + "EmpiricalMotifCandidateBatchV1", + "EmpiricalMotifEventLineageV1", + "EmpiricalMotifGeneratorConfigV1", + "EmpiricalMotifTransformationV1", + "InformationAuditFindingV1", + "InformationAuditReportV1", + "InformationAuditRule", + "InformationInputKind", + "InformationLeakageError", + "InformationMode", + "InformationScope", + "InformationSplitKind", + "InformationStage", + "HistoricalCarvedCandidateBatchV1", + "HistoricalCarvingConditionPolicyV1", + "HistoricalCarvingConstraintSetV1", + "HistoricalCarvingEventLineageV1", + "HistoricalCarvingQuarantineV1", + "HistoricalCarvingRejectionExampleV1", + "HistoricalCarvingValidationEvidenceV1", + "ObservationApplicationResultV1", + "ObservationCalibrationCampaignV2", + "ObservationCalibrationProfileV2", + "ObservationCalibrationTargetV2", + "ObservationCalibrationWindowV2", + "ObservationCarryStateV1", + "ObservationContextV1", + "ObservationFitDiagnosticsV1", + "ObservationFitEvidenceV1", + "ObservationInputEventV1", + "ObservationOperatorFitConfigV1", + "ObservationOperatorV1", + "ObservationOutputEventV1", + "ObservationParameterEstimateV1", + "ObservationStratumV1", + "MotifGenerationDecision", + "MotifGenerationStatus", + "PartitionManifestV1", + "ReconstructionCheckpointV1", + "ReconstructionCommitPhase", + "ReconstructionHeartbeatV1", + "ReconstructionInformationInputV1", + "ReconstructionInformationManifestV1", + "ReconstructionInformationPolicyV1", + "ReconstructionInformationSplitV1", + "ReconstructionResourceEstimateV1", + "ReconstructionResourceLimitError", + "ReconstructionRunV1", + "ReconstructionStoragePolicyV1", + "ReconstructionWindowV1", + "PublishedReconstructionV1", + "PublishedReconstructionV2", + "ReconstructionConstraintManifestV1", + "ReconstructionDeliveredGroupV1", + "ReconstructionDeliveryManifestV1", + "ReconstructionDeliveryQualityManifestV1", + "ReconstructionDeliveryStatus", + "ReconstructionEnsembleManifestV1", + "ReconstructionPersistenceError", + "ReconstructionProductManifestV1", + "ReconstructionProductManifestV2", + "ReconstructionProductPartitionV1", + "ReconstructionQualityManifestV1", + "ReconstructionReplayManifestV1", + "ReconstructionRetentionPlanV1", + "ReconstructionSourceManifestV1", + "ReconstructionStoragePreflightError", + "StagedReconstructionPublicationV1", + "StagedReconstructionPublicationV2", + "ReferenceMotifBackoffAttemptV1", + "ReferenceMotifConditionV1", + "ReferenceMotifFragmentV1", + "ReferenceMotifIndexConfigV1", + "ReferenceMotifIndexV1", + "ReferenceMotifLeakageError", + "ReferenceMotifMatchV1", + "ReferenceMotifQueryResultV1", + "ReferenceMotifQueryStatus", + "ReferenceMotifQueryV1", + "ReferenceMotifResourceLimitError", + "ReferenceMotifSourceEventV1", + "ReferenceMotifSourceWindowV1", + "ReferenceMotifSplitKind", + "ReferenceMotifSplitV1", + "ReferenceMotifTransformPolicyV1", + "ReferenceMotifTransition", + "ModernReferenceMotifBuildV1", + "ModernReferenceMotifProfileV1", + "RejectionSummaryV1", + "ReverseDegradationBenchmarkManifestV1", + "ReverseDegradationBenchmarkV1", + "ReverseDegradationScorecardV1", + "SyntheticEnsembleManifestV1", + "SyntheticEnsembleMemberV1", + "SyntheticEventOrigin", + "SyntheticEventStreamV1", + "SyntheticEventV1", + "artifact_ref_for_json_contract", + "audit_reconstruction_information", + "benchmark_events_from_empirical_overlay", + "cross_currency_quality_report", + "cross_currency_series_inputs", + "build_benchmark_control_events", + "build_benchmark_control_windows", + "build_reference_motif_index", + "build_modern_reference_motif_library", + "canonical_contract_json", + "cleanup_reconstruction_scratch", + "commit_delivery_reconstruction_publication", + "commit_reconstruction_publication", + "carve_empirical_motif_candidates", + "derive_anchor_interval_id", + "degrade_benchmark_window", + "discover_reconstruction_manifests", + "estimate_reconstruction_retention", + "eurusd_triangle_reconciliation_config", + "fit_observation_operator", + "calibrate_historical_observation_operators", + "estimate_paired_observation_evidence", + "extract_reference_motif_fragment", + "generate_benchmark_candidate_window", + "generate_empirical_motif_candidates", + "plan_reconstruction_windows", + "plan_cross_currency_windows", + "query_reference_motifs", + "read_modern_reference_motif_artifact", + "read_modern_reference_motif_index", + "reconcile_cross_currency_window", + "read_reference_motif_index", + "read_synthetic_event_stream_parquet", + "iter_reconstruction_event_batches", + "load_reconstruction_manifest", + "publish_reconstruction_group", + "read_reconstruction_streams", + "reconstruction_logical_content_sha256", + "reconstruction_parquet_paths", + "reconstruction_streams_content_sha256", + "read_observation_operator_artifact", + "read_feed_epoch_evidence_v2", + "read_observation_calibration_campaign", + "reconstruction_information_window_plan_id", + "reference_motif_information_inputs", + "reference_motif_condition_from_quotes", + "reference_motif_source_window_from_training_frame", + "require_reconstruction_information_audit", + "synthetic_event_arrow_schema", + "synthetic_event_stream_from_arrow", + "synthetic_event_stream_from_parquet_bytes", + "synthetic_event_stream_to_arrow", + "synthetic_event_stream_to_parquet_bytes", + "scan_reconstruction_events_polars", + "stage_reconstruction_publication", + "stage_delivery_reconstruction_publication", + "validate_reconstruction_window_plan", + "validate_benchmark_information_boundary", + "validate_cross_currency_atomic_manifest", + "validate_cross_currency_output", + "verify_reconstruction_publication", + "project_modern_reference_delivery", + "write_observation_operator", + "write_observation_calibration_campaign", + "write_reference_motif_index", + "write_modern_reference_motif_artifacts", + "write_synthetic_event_stream_parquet", + "DEFAULT_ENSEMBLE_HORIZONS_NS", + "DEFAULT_ENSEMBLE_MAX_PAYLOAD_BYTES", + "DEFAULT_ENSEMBLE_MAX_SAMPLES", + "DEFAULT_ENSEMBLE_MAX_SLICES", + "ENSEMBLE_ARTIFACT_DIGEST_SCHEMA_VERSION", + "ENSEMBLE_CALIBRATION_EVALUATION_SPLIT", + "ENSEMBLE_CALIBRATION_FIT_SPLIT", + "ENSEMBLE_CALIBRATION_METRIC_GROUPS", + "ENSEMBLE_CALIBRATION_METRIC_NAMES", + "ENSEMBLE_CALIBRATION_REPORT_SCHEMA_VERSION", + "ENSEMBLE_CALIBRATION_SAMPLE_SCHEMA_VERSION", + "ENSEMBLE_CONFIDENCE_QUANTITY", + "ENSEMBLE_CONFIG_SCHEMA_VERSION", + "ENSEMBLE_DIVERSITY_SUMMARY_SCHEMA_VERSION", + "ENSEMBLE_ENGINE_ID", + "ENSEMBLE_ENGINE_VERSION", + "ENSEMBLE_MEMBER_CALIBRATION_SCHEMA_VERSION", + "ENSEMBLE_MEMBER_PLAN_SCHEMA_VERSION", + "ENSEMBLE_MEMBER_SELECTION_SCHEMA_VERSION", + "ENSEMBLE_METRIC_CALIBRATION_SCHEMA_VERSION", + "ENSEMBLE_OUTCOME_SUMMARY_SCHEMA_VERSION", + "ENSEMBLE_PLAN_SCHEMA_VERSION", + "ENSEMBLE_PRIMARY_SELECTION_BASIS", + "ENSEMBLE_REGENERATION_REQUEST_SCHEMA_VERSION", + "ENSEMBLE_STORAGE_ESTIMATE_SCHEMA_VERSION", + "ENSEMBLE_STRATUM_SCHEMA_VERSION", + "MOTIF_TRANSFORMATION_CONFIDENCE_QUANTITY", + "EnsembleArtifactDigestV1", + "EnsembleArtifactKind", + "EnsembleCalibrationConfigV1", + "EnsembleCalibrationReportV1", + "EnsembleCalibrationSampleV1", + "EnsembleCalibrationStratumV1", + "EnsembleDiversityStatus", + "EnsembleDiversitySummaryV1", + "EnsembleMemberCalibrationV1", + "EnsembleMemberPlanV1", + "EnsembleMemberSelectionV1", + "EnsembleMemberStatus", + "EnsembleMetricCalibrationV1", + "EnsembleMetricStatus", + "EnsembleOutcomeSummaryV1", + "EnsembleRegenerationRequestV1", + "EnsembleReportStatus", + "EnsembleStorageEstimateV1", + "ReconstructionEnsemblePlanV1", + "benchmark_ensemble_calibration_sample", + "benchmark_logical_content_sha256", + "build_ensemble_regeneration_request", + "calibrate_reconstruction_ensemble", + "ensemble_logical_content_sha256", + "estimate_reconstruction_ensemble_resources", + "plan_reconstruction_ensemble", + "verify_ensemble_regeneration", + "BROKER_BENCHMARK_COMPARISON_SCHEMA_VERSION", + "BROKER_CONDITIONED_PROPOSAL_SCHEMA_VERSION", + "BROKER_PROFILE_SELECTION_SCHEMA_VERSION", + "BROKER_RENDERED_GROUP_SCHEMA_VERSION", + "BROKER_RENDER_LINEAGE_SCHEMA_VERSION", + "BROKER_TRANSFER_CONFIG_SCHEMA_VERSION", + "BROKER_TRANSFER_ENGINE_ID", + "BROKER_TRANSFER_ENGINE_VERSION", + "BROKER_TRANSFER_MANIFEST_SCHEMA_VERSION", + "BrokerBenchmarkComparisonV1", + "BrokerConditionedProposalV1", + "BrokerProfileSelectionV1", + "BrokerRenderedGroupV1", + "BrokerRenderLineageV1", + "BrokerTransferConfigV1", + "BrokerTransferManifestV1", + "BrokerTransferStatus", + "compare_broker_benchmark_results", + "condition_broker_proposal", + "render_broker_delivery", + "select_broker_profile", + "CERTIFICATION_ARTIFACT_SCHEMA_VERSION", + "CERTIFICATION_CHECK_RESULT_SCHEMA_VERSION", + "CERTIFICATION_GATE_RESULT_SCHEMA_VERSION", + "CERTIFICATION_OBSERVATION_SCHEMA_VERSION", + "CERTIFICATION_REQUIREMENT_SCHEMA_VERSION", + "DEFAULT_CERTIFICATION_MAX_ARTIFACTS", + "DEFAULT_CERTIFICATION_MAX_METADATA_ITEMS", + "DEFAULT_CERTIFICATION_MAX_OBSERVATIONS", + "DEFAULT_CERTIFICATION_MAX_PAYLOAD_BYTES", + "DEFAULT_CERTIFICATION_MAX_REQUIREMENTS", + "DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH", + "EURUSD_TRIANGLE_COMMON_START_PERIOD", + "MODERN_REFERENCE_DELIVERY_CLAIM", + "MODERN_REFERENCE_DELIVERY_MODE", + "PROMOTION_ONLY_CHECK_IDS", + "RECONSTRUCTION_CERTIFICATION_DOSSIER_SCHEMA_VERSION", + "RECONSTRUCTION_CERTIFICATION_DOSSIER_V2_SCHEMA_VERSION", + "RECONSTRUCTION_CERTIFICATION_POLICY_SCHEMA_VERSION", + "RECONSTRUCTION_CERTIFICATION_POLICY_V2_SCHEMA_VERSION", + "CertificationArtifactV1", + "CertificationCheckResultV1", + "CertificationCheckStatus", + "CertificationComparator", + "CertificationGate", + "CertificationGateResultV1", + "CertificationObservationV1", + "CertificationRequirementV1", + "CertificationState", + "ReconstructionCertificationDossierV1", + "ReconstructionCertificationDossierV2", + "ReconstructionCertificationPolicyV1", + "ReconstructionCertificationPolicyV2", + "eurusd_triangle_certification_policy", + "evaluate_modern_reference_reconstruction_certification", + "evaluate_reconstruction_certification", + "load_modern_reference_reconstruction_certification_dossier", + "load_reconstruction_certification_dossier", + "modern_reference_triangle_certification_policy", + "write_modern_reference_reconstruction_certification_dossier", + "write_reconstruction_certification_dossier", + "CAMPAIGN_MANIFEST_EVIDENCE_KEY", + "CERTIFICATION_CAMPAIGN_ARTIFACT_SCHEMA_VERSION", + "CERTIFICATION_CAMPAIGN_OBSERVATION_SCHEMA_VERSION", + "CERTIFICATION_CAMPAIGN_RESULT_SCHEMA_VERSION", + "CERTIFICATION_CAMPAIGN_SPEC_SCHEMA_VERSION", + "CERTIFICATION_METHODOLOGY_REPORT_SCHEMA_VERSION", + "METHODOLOGY_REPORT_EVIDENCE_KEY", + "CertificationCampaignArtifactV1", + "CertificationCampaignObservationV1", + "ModernReferenceCertificationCampaignResultV1", + "ModernReferenceCertificationCampaignSpecV1", + "read_modern_reference_certification_campaign_spec", + "run_modern_reference_certification_campaign", + "DEFAULT_STRATEGY_HORIZONS_NS", + "REFERENCE_MOMENTUM_STRATEGY_ID", + "REFERENCE_MOMENTUM_STRATEGY_VERSION", + "STRATEGY_EVALUATION_CASE_SCHEMA_VERSION", + "STRATEGY_EVALUATION_PLAN_SCHEMA_VERSION", + "STRATEGY_EVALUATION_POLICY_SCHEMA_VERSION", + "STRATEGY_EXECUTION_SPECIFICATION_SCHEMA_VERSION", + "STRATEGY_INVALID_FOR_BACKTEST_LABEL", + "STRATEGY_QUOTE_SCHEMA_VERSION", + "STRATEGY_RESTORATION_RESULT_SCHEMA_VERSION", + "STRATEGY_SENSITIVITY_REPORT_SCHEMA_VERSION", + "STRATEGY_SIGNAL_SCHEMA_VERSION", + "STRATEGY_SLICE_RESULT_SCHEMA_VERSION", + "STRATEGY_SPECIFICATION_SCHEMA_VERSION", + "STRATEGY_UNCERTAINTY_SUMMARY_SCHEMA_VERSION", + "STRATEGY_WINDOW_RESULT_SCHEMA_VERSION", + "ReferenceMomentumStrategyV1", + "StrategyEvaluationCaseV1", + "StrategyEvaluationFailure", + "StrategyEvaluationPlanV1", + "StrategyEvaluationPolicyV1", + "StrategyExecutionSpecificationV1", + "StrategyQuoteV1", + "StrategyResourceLimitError", + "StrategyRestorationResultV1", + "StrategySensitivityReportV1", + "StrategySide", + "StrategySignalEngineV1", + "StrategySignalStateV1", + "StrategySignalV1", + "StrategySliceResultV1", + "StrategySourceKind", + "StrategySpecificationV1", + "StrategyUncertaintySummaryV1", + "StrategyWindowResultV1", + "StrategyWindowStatus", + "evaluate_strategy_sensitivity", + "strategy_sensitivity_benchmark_hooks", + "BENCHMARK_CANDIDATE_REPORT_SCHEMA_VERSION", + "BENCHMARK_CORPUS_PROFILE_SCHEMA_VERSION", + "BENCHMARK_SOURCE_PARTITION_SCHEMA_VERSION", + "BENCHMARK_WINDOW_PARTITION_SCHEMA_VERSION", + "PREDECLARED_GATE_COMMIT", + "REVERSE_DEGRADATION_CAMPAIGN_SCHEMA_VERSION", + "REVERSE_DEGRADATION_CORPUS_SCHEMA_VERSION", + "BenchmarkCandidateReportV1", + "BenchmarkSourcePartitionV1", + "BenchmarkWindowPartitionV1", + "ReverseDegradationBenchmarkCampaignV1", + "ReverseDegradationBenchmarkCorpusV1", + "ReverseDegradationCorpusProfileV1", + "audit_holdout_neighbor_leakage", + "build_reverse_degradation_benchmark_corpus", + "read_reverse_degradation_benchmark_campaign", + "read_reverse_degradation_benchmark_corpus", + "replay_reverse_degradation_benchmark_corpus", + "run_reverse_degradation_benchmark_campaign", + "write_reverse_degradation_benchmark_artifacts", + "BENCHMARK_GATE_CHECK_SCHEMA_VERSION", + "BENCHMARK_GATE_OBSERVATION_SCHEMA_VERSION", + "BENCHMARK_GATE_REQUIREMENT_SCHEMA_VERSION", + "BENCHMARK_PROMOTION_DECISION_SCHEMA_VERSION", + "BENCHMARK_PROMOTION_GATE_POLICY_SCHEMA_VERSION", + "DEFAULT_BENCHMARK_GATE_ASSET", + "BenchmarkGateCheckV1", + "BenchmarkGateComparator", + "BenchmarkGateObservationV1", + "BenchmarkGateRequirementV1", + "BenchmarkGateScope", + "BenchmarkGateSeverity", + "BenchmarkGateStatus", + "BenchmarkPromotionDecisionV1", + "BenchmarkPromotionGatePolicyV1", + "evaluate_benchmark_promotion_gates", + "load_default_benchmark_promotion_gate_policy", +] diff --git a/src/histdatacom/synthetic/activity.py b/src/histdatacom/synthetic/activity.py new file mode 100644 index 00000000..533bbe8e --- /dev/null +++ b/src/histdatacom/synthetic/activity.py @@ -0,0 +1,2112 @@ +"""Honest activity and liquidity-proxy evidence for reconstructed events. + +The final event schema records quote deliveries, not centralized FX trades. +This module therefore derives bounded activity metadata from immutable +``SyntheticEventV1`` rows without adding a fabricated volume column. The +same contracts work over in-memory streams and projected batches from the +committed Parquet product. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any, TypeVar, cast + +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.benchmark import ReverseDegradationScorecardV1 +from histdatacom.synthetic.contracts import ( + SYNTHETIC_EVENT_SCHEMA_VERSION, + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + canonical_contract_json, +) +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.persistence import ( + iter_reconstruction_event_batches, + verify_reconstruction_publication, +) + +RECONSTRUCTION_ACTIVITY_POLICY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-activity-policy.v1" +) +RECONSTRUCTION_ACTIVITY_METRIC_SCHEMA_VERSION = ( + "histdatacom.reconstruction-activity-metric.v1" +) +RECONSTRUCTION_ACTIVITY_SLICE_SCHEMA_VERSION = ( + "histdatacom.reconstruction-activity-slice.v1" +) +RECONSTRUCTION_ACTIVITY_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-activity-manifest.v1" +) +RECONSTRUCTION_ACTIVITY_BENCHMARK_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.reconstruction-activity-benchmark-evidence.v1" +) + +DEFAULT_ACTIVITY_MAX_SYMBOLS = 64 +DEFAULT_ACTIVITY_MAX_SLICES = 192 +DEFAULT_ACTIVITY_MAX_PROVENANCE_VALUES = 256 +DEFAULT_ACTIVITY_MAX_PAYLOAD_BYTES = 1_048_576 +DEFAULT_ACTIVITY_ROUNDING_DIGITS = 12 +DEFAULT_ACTIVITY_BATCH_SIZE = 8_192 +MAX_ACTIVITY_SYMBOLS = 256 +MAX_ACTIVITY_SLICES = 768 +MAX_ACTIVITY_PROVENANCE_VALUES = 4_096 +MAX_ACTIVITY_PAYLOAD_BYTES = 4_194_304 +MAX_ACTIVITY_TEXT = 1_024 +NANOSECONDS_PER_SECOND = 1_000_000_000 + +_EnumT = TypeVar("_EnumT", bound=Enum) + +ACTIVITY_EVENT_COLUMNS = ( + "event_id", + "origin", + "symbol", + "event_time_ns", + "event_sequence", + "bid", + "ask", + "run_id", + "ensemble_member_id", + "source_version_id", + "generator_id", + "generator_version", + "generator_config_id", + "reference_id", + "motif_id", + "feed_epoch_id", + "broker_profile_id", + "constraint_set_id", + "confidence", +) + +ACTIVITY_REVERSE_DEGRADATION_METRICS = ( + "event_count_relative_error", + "intensity_relative_error", + "interarrival_hist_l1", + "burst_rate_absolute_error", + "quiet_rate_absolute_error", + "spread_mean_relative_error", + "spread_hist_l1", +) + + +class ActivitySliceScope(str, Enum): + """Which final-event population one activity slice describes.""" + + OBSERVED = "observed" + SYNTHETIC = "synthetic" + MERGED = "merged" + + @classmethod + def from_value( + cls, value: str | "ActivitySliceScope" + ) -> "ActivitySliceScope": + """Return a strict normalized activity scope.""" + return _enum_value(cls, value, "activity slice scope") + + +class ActivityVolumeState(str, Enum): + """Honest handling state for fields sometimes mislabeled as volume.""" + + UNAVAILABLE = "unavailable" + OMITTED = "omitted" + OBSERVED_SOURCE_SIZE = "observed_source_size" + BROKER_SUPPLIED_SIZE = "broker_supplied_size" + SYNTHETIC_ACTIVITY_PROXY = "synthetic_activity_proxy" + + @classmethod + def from_value( + cls, value: str | "ActivityVolumeState" + ) -> "ActivityVolumeState": + """Return strict volume handling without implying traded volume.""" + return _enum_value(cls, value, "activity volume state") + + +class ActivityMetricSemantics(str, Enum): + """Scientific meaning of one derived activity metric.""" + + QUOTE_ACTIVITY = "quote_activity" + CADENCE = "cadence" + SPREAD_LIQUIDITY_PROXY = "spread_liquidity_proxy" + EVENT_CONFIDENCE = "event_confidence" + + @classmethod + def from_value( + cls, value: str | "ActivityMetricSemantics" + ) -> "ActivityMetricSemantics": + """Return strict activity metric semantics.""" + return _enum_value(cls, value, "activity metric semantics") + + +class ActivityAggregationSemantics(str, Enum): + """How a metric must be projected into a larger interval.""" + + COUNT = "count" + INTERVAL_DURATION = "interval_duration" + RECOMPUTE_RATE = "recompute_rate" + SUPPORT_WEIGHTED_MEAN = "support_weighted_mean" + MINIMUM = "minimum" + MAXIMUM = "maximum" + + @classmethod + def from_value( + cls, value: str | "ActivityAggregationSemantics" + ) -> "ActivityAggregationSemantics": + """Return strict aggregation semantics.""" + return _enum_value(cls, value, "activity aggregation semantics") + + +_METRIC_DEFINITIONS: dict[ + str, + tuple[str, ActivityAggregationSemantics, ActivityMetricSemantics], +] = { + "event_count": ( + "event", + ActivityAggregationSemantics.COUNT, + ActivityMetricSemantics.QUOTE_ACTIVITY, + ), + "quote_update_count": ( + "quote_update", + ActivityAggregationSemantics.COUNT, + ActivityMetricSemantics.QUOTE_ACTIVITY, + ), + "exposure_duration_ns": ( + "nanosecond", + ActivityAggregationSemantics.INTERVAL_DURATION, + ActivityMetricSemantics.QUOTE_ACTIVITY, + ), + "tick_intensity_per_second": ( + "event_per_second", + ActivityAggregationSemantics.RECOMPUTE_RATE, + ActivityMetricSemantics.CADENCE, + ), + "mean_interarrival_ns": ( + "nanosecond", + ActivityAggregationSemantics.SUPPORT_WEIGHTED_MEAN, + ActivityMetricSemantics.CADENCE, + ), + "min_interarrival_ns": ( + "nanosecond", + ActivityAggregationSemantics.MINIMUM, + ActivityMetricSemantics.CADENCE, + ), + "max_interarrival_ns": ( + "nanosecond", + ActivityAggregationSemantics.MAXIMUM, + ActivityMetricSemantics.CADENCE, + ), + "price_change_count": ( + "transition", + ActivityAggregationSemantics.COUNT, + ActivityMetricSemantics.QUOTE_ACTIVITY, + ), + "stale_quote_count": ( + "transition", + ActivityAggregationSemantics.COUNT, + ActivityMetricSemantics.QUOTE_ACTIVITY, + ), + "stale_quote_rate": ( + "ratio", + ActivityAggregationSemantics.RECOMPUTE_RATE, + ActivityMetricSemantics.QUOTE_ACTIVITY, + ), + "mean_spread": ( + "price", + ActivityAggregationSemantics.SUPPORT_WEIGHTED_MEAN, + ActivityMetricSemantics.SPREAD_LIQUIDITY_PROXY, + ), + "min_spread": ( + "price", + ActivityAggregationSemantics.MINIMUM, + ActivityMetricSemantics.SPREAD_LIQUIDITY_PROXY, + ), + "max_spread": ( + "price", + ActivityAggregationSemantics.MAXIMUM, + ActivityMetricSemantics.SPREAD_LIQUIDITY_PROXY, + ), + "mean_event_confidence": ( + "probability", + ActivityAggregationSemantics.SUPPORT_WEIGHTED_MEAN, + ActivityMetricSemantics.EVENT_CONFIDENCE, + ), +} + + +def activity_metric_definitions() -> dict[str, JSONValue]: + """Return the immutable metric dictionary in JSON-compatible form.""" + return { + name: { + "unit": definition[0], + "aggregation": definition[1].value, + "semantics": definition[2].value, + } + for name, definition in sorted(_METRIC_DEFINITIONS.items()) + } + + +def activity_bar_projection_semantics() -> dict[str, JSONValue]: + """Return the exact #18 projection rules for derived candlestick bars.""" + return { + "tick_count": { + "source_metric": "event_count", + "operation": "sum", + "unit": "event", + }, + "quote_update_count": { + "source_metric": "quote_update_count", + "operation": "sum", + "unit": "quote_update", + }, + "activity_duration_ns": { + "source_metric": "exposure_duration_ns", + "operation": "recompute_from_bar_event_bounds", + "unit": "nanosecond", + }, + "tick_intensity_per_second": { + "source_metric": "event_count", + "operation": "recompute_count_divided_by_activity_duration", + "unit": "event_per_second", + }, + "price_change_count": { + "source_metric": "price_change_count", + "operation": "sum_with_boundary_carry", + "unit": "transition", + }, + "stale_quote_count": { + "source_metric": "stale_quote_count", + "operation": "sum_with_boundary_carry", + "unit": "transition", + }, + "stale_quote_rate": { + "source_metric": "stale_quote_count", + "operation": "recompute_over_quote_transitions", + "unit": "ratio", + }, + "mean_spread": { + "source_metric": "mean_spread", + "operation": "event_support_weighted_mean", + "weight_metric": "event_count", + "unit": "price", + }, + "volume": { + "source_metric": None, + "operation": "unavailable_unless_separately_sourced", + "unit": None, + }, + } + + +@dataclass(frozen=True, slots=True) +class ReconstructionActivityPolicyV1: + """Bounded deterministic aggregation and honest volume policy.""" + + scopes: tuple[ActivitySliceScope, ...] = ( + ActivitySliceScope.OBSERVED, + ActivitySliceScope.SYNTHETIC, + ActivitySliceScope.MERGED, + ) + volume_state: ActivityVolumeState = ActivityVolumeState.UNAVAILABLE + rounding_digits: int = DEFAULT_ACTIVITY_ROUNDING_DIGITS + max_symbols: int = DEFAULT_ACTIVITY_MAX_SYMBOLS + max_slices: int = DEFAULT_ACTIVITY_MAX_SLICES + max_provenance_values: int = DEFAULT_ACTIVITY_MAX_PROVENANCE_VALUES + max_payload_bytes: int = DEFAULT_ACTIVITY_MAX_PAYLOAD_BYTES + policy_id: str = "" + schema_version: str = RECONSTRUCTION_ACTIVITY_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_ACTIVITY_POLICY_SCHEMA_VERSION, + "reconstruction activity policy", + ) + scopes = tuple( + sorted( + {ActivitySliceScope.from_value(item) for item in self.scopes}, + key=lambda item: item.value, + ) + ) + if not scopes: + raise ValueError("activity policy requires at least one scope") + object.__setattr__(self, "scopes", scopes) + object.__setattr__( + self, + "volume_state", + ActivityVolumeState.from_value(self.volume_state), + ) + rounding = _bounded_int(self.rounding_digits, "rounding_digits", 0, 15) + object.__setattr__(self, "rounding_digits", rounding) + for name, maximum in ( + ("max_symbols", MAX_ACTIVITY_SYMBOLS), + ("max_slices", MAX_ACTIVITY_SLICES), + ("max_provenance_values", MAX_ACTIVITY_PROVENANCE_VALUES), + ("max_payload_bytes", MAX_ACTIVITY_PAYLOAD_BYTES), + ): + value = _bounded_int(getattr(self, name), name, 1, maximum) + object.__setattr__(self, name, value) + if self.max_slices < len(scopes): + raise ValueError("max_slices is smaller than configured scopes") + expected = _stable_id("activity-policy", self.identity_payload()) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("activity policy_id differs from its content") + object.__setattr__(self, "policy_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic policy identity.""" + return { + "schema_version": self.schema_version, + "scopes": [item.value for item in self.scopes], + "volume_state": self.volume_state.value, + "rounding_digits": self.rounding_digits, + "max_symbols": self.max_symbols, + "max_slices": self.max_slices, + "max_provenance_values": self.max_provenance_values, + "max_payload_bytes": self.max_payload_bytes, + "output_mode": "derived_metadata", + "event_schema_augmented": False, + "centralized_traded_volume_claim": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic policy JSON.""" + return {**self.identity_payload(), "policy_id": self.policy_id} + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionActivityPolicyV1": + """Restore and verify an activity policy.""" + _require_schema(data, RECONSTRUCTION_ACTIVITY_POLICY_SCHEMA_VERSION) + _require_derived(data, "output_mode", "derived_metadata") + _require_derived(data, "event_schema_augmented", False) + _require_derived(data, "centralized_traded_volume_claim", False) + return cls( + scopes=tuple( + ActivitySliceScope.from_value(str(item)) + for item in _sequence(data.get("scopes"), "scopes") + ), + volume_state=ActivityVolumeState.from_value( + str(data.get("volume_state", "")) + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + max_symbols=_strict_int(data.get("max_symbols"), "max_symbols"), + max_slices=_strict_int(data.get("max_slices"), "max_slices"), + max_provenance_values=_strict_int( + data.get("max_provenance_values"), "max_provenance_values" + ), + max_payload_bytes=_strict_int( + data.get("max_payload_bytes"), "max_payload_bytes" + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionActivityPolicyV1": + """Restore a policy from JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionActivityMetricV1: + """One unit-bearing activity or liquidity-proxy estimate.""" + + name: str + value: int | float | None + unit: str + aggregation: ActivityAggregationSemantics + semantics: ActivityMetricSemantics + support_count: int + confidence: float | None = None + limitations: tuple[str, ...] = () + schema_version: str = RECONSTRUCTION_ACTIVITY_METRIC_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_ACTIVITY_METRIC_SCHEMA_VERSION, + "reconstruction activity metric", + ) + name = _required_text(self.name) + if name not in _METRIC_DEFINITIONS: + raise ValueError("unsupported reconstruction activity metric") + object.__setattr__(self, "name", name) + expected_unit, expected_aggregation, expected_semantics = ( + _METRIC_DEFINITIONS[name] + ) + unit = _required_text(self.unit) + aggregation = ActivityAggregationSemantics.from_value(self.aggregation) + semantics = ActivityMetricSemantics.from_value(self.semantics) + if ( + unit != expected_unit + or aggregation is not expected_aggregation + or semantics is not expected_semantics + ): + raise ValueError("activity metric definition differs") + object.__setattr__(self, "unit", unit) + object.__setattr__(self, "aggregation", aggregation) + object.__setattr__(self, "semantics", semantics) + if self.value is not None: + if isinstance(self.value, bool) or not isinstance( + self.value, (int, float) + ): + raise ValueError("activity metric value must be numeric") + if not math.isfinite(float(self.value)): + raise ValueError("activity metric value must be finite") + if float(self.value) < 0.0: + raise ValueError("activity metric value must be non-negative") + object.__setattr__( + self, + "support_count", + _bounded_int(self.support_count, "support_count", 0, 2**63 - 1), + ) + if self.value is None and self.support_count: + raise ValueError("unavailable metric cannot claim support") + if self.confidence is not None: + confidence = _finite_float(self.confidence, "confidence") + if not 0.0 <= confidence <= 1.0: + raise ValueError("activity metric confidence is out of range") + object.__setattr__(self, "confidence", confidence) + object.__setattr__( + self, "limitations", _normalized_text_tuple(self.limitations) + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return stable metric metadata.""" + return { + "schema_version": self.schema_version, + "name": self.name, + "value": self.value, + "unit": self.unit, + "aggregation": self.aggregation.value, + "semantics": self.semantics.value, + "support_count": self.support_count, + "confidence": self.confidence, + "limitations": list(self.limitations), + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionActivityMetricV1": + """Restore a strict activity metric.""" + _require_schema(data, RECONSTRUCTION_ACTIVITY_METRIC_SCHEMA_VERSION) + raw_value = data.get("value") + if raw_value is not None and ( + isinstance(raw_value, bool) + or not isinstance(raw_value, (int, float)) + ): + raise ValueError("activity metric value must be numeric") + return cls( + name=str(data.get("name", "")), + value=raw_value, + unit=str(data.get("unit", "")), + aggregation=ActivityAggregationSemantics.from_value( + str(data.get("aggregation", "")) + ), + semantics=ActivityMetricSemantics.from_value( + str(data.get("semantics", "")) + ), + support_count=_strict_int( + data.get("support_count"), "support_count" + ), + confidence=_optional_float(data.get("confidence"), "confidence"), + limitations=_string_tuple(data.get("limitations"), "limitations"), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionActivitySliceV1: + """One symbol/origin activity summary with bounded provenance.""" + + symbol: str + scope: ActivitySliceScope + start_event_time_ns: int + end_event_time_ns: int + metrics: tuple[ReconstructionActivityMetricV1, ...] + source_version_ids: tuple[str, ...] + generator_ids: tuple[str, ...] + generator_versions: tuple[str, ...] + generator_config_ids: tuple[str, ...] + reference_ids: tuple[str, ...] + motif_ids: tuple[str, ...] + feed_epoch_ids: tuple[str, ...] + broker_profile_ids: tuple[str, ...] + constraint_set_ids: tuple[str, ...] + stream_ids: tuple[str, ...] + event_content_sha256: str + confidence_support_count: int + limitations: tuple[str, ...] = () + slice_id: str = "" + schema_version: str = RECONSTRUCTION_ACTIVITY_SLICE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_ACTIVITY_SLICE_SCHEMA_VERSION, + "reconstruction activity slice", + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, "scope", ActivitySliceScope.from_value(self.scope) + ) + start = _strict_int(self.start_event_time_ns, "start_event_time_ns") + end = _strict_int(self.end_event_time_ns, "end_event_time_ns") + if end < start: + raise ValueError("activity slice time bounds are reversed") + object.__setattr__(self, "start_event_time_ns", start) + object.__setattr__(self, "end_event_time_ns", end) + metrics = tuple(sorted(self.metrics, key=lambda item: item.name)) + if any( + not isinstance(item, ReconstructionActivityMetricV1) + for item in metrics + ): + raise TypeError("activity slice metrics require v1 contracts") + if {item.name for item in metrics} != set(_METRIC_DEFINITIONS): + raise ValueError("activity slice metric coverage differs") + object.__setattr__(self, "metrics", metrics) + for name in ( + "source_version_ids", + "generator_ids", + "generator_versions", + "generator_config_ids", + "reference_ids", + "motif_ids", + "feed_epoch_ids", + "broker_profile_ids", + "constraint_set_ids", + "stream_ids", + ): + object.__setattr__( + self, name, _normalized_text_tuple(getattr(self, name)) + ) + if not self.source_version_ids: + raise ValueError("activity slice requires source version identity") + object.__setattr__( + self, + "event_content_sha256", + _required_sha256(self.event_content_sha256, "event_content_sha256"), + ) + object.__setattr__( + self, + "confidence_support_count", + _bounded_int( + self.confidence_support_count, + "confidence_support_count", + 0, + self.event_count, + ), + ) + object.__setattr__( + self, "limitations", _normalized_text_tuple(self.limitations) + ) + expected = _stable_id("activity-slice", self.identity_payload()) + supplied = _optional_text(self.slice_id) + if supplied is not None and supplied != expected: + raise ValueError("activity slice_id differs from its content") + object.__setattr__(self, "slice_id", expected) + + @property + def metric_by_name(self) -> dict[str, ReconstructionActivityMetricV1]: + """Return metrics keyed by stable name.""" + return {item.name: item for item in self.metrics} + + @property + def event_count(self) -> int: + """Return the number of delivered quote events.""" + value = self.metric_by_name["event_count"].value + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError("activity event_count is not integral") + return value + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic slice evidence.""" + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "scope": self.scope.value, + "start_event_time_ns": self.start_event_time_ns, + "end_event_time_ns": self.end_event_time_ns, + "metrics": [item.to_dict() for item in self.metrics], + "source_version_ids": list(self.source_version_ids), + "generator_ids": list(self.generator_ids), + "generator_versions": list(self.generator_versions), + "generator_config_ids": list(self.generator_config_ids), + "reference_ids": list(self.reference_ids), + "motif_ids": list(self.motif_ids), + "feed_epoch_ids": list(self.feed_epoch_ids), + "broker_profile_ids": list(self.broker_profile_ids), + "constraint_set_ids": list(self.constraint_set_ids), + "stream_ids": list(self.stream_ids), + "event_content_sha256": self.event_content_sha256, + "confidence_support_count": self.confidence_support_count, + "limitations": list(self.limitations), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return stable slice metadata.""" + return {**self.identity_payload(), "slice_id": self.slice_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionActivitySliceV1": + """Restore and verify an activity slice.""" + _require_schema(data, RECONSTRUCTION_ACTIVITY_SLICE_SCHEMA_VERSION) + return cls( + symbol=str(data.get("symbol", "")), + scope=ActivitySliceScope.from_value(str(data.get("scope", ""))), + start_event_time_ns=_strict_int( + data.get("start_event_time_ns"), "start_event_time_ns" + ), + end_event_time_ns=_strict_int( + data.get("end_event_time_ns"), "end_event_time_ns" + ), + metrics=tuple( + ReconstructionActivityMetricV1.from_dict(item) + for item in _mapping_sequence(data.get("metrics"), "metrics") + ), + source_version_ids=_string_tuple( + data.get("source_version_ids"), "source_version_ids" + ), + generator_ids=_string_tuple( + data.get("generator_ids"), "generator_ids" + ), + generator_versions=_string_tuple( + data.get("generator_versions"), "generator_versions" + ), + generator_config_ids=_string_tuple( + data.get("generator_config_ids"), "generator_config_ids" + ), + reference_ids=_string_tuple( + data.get("reference_ids"), "reference_ids" + ), + motif_ids=_string_tuple(data.get("motif_ids"), "motif_ids"), + feed_epoch_ids=_string_tuple( + data.get("feed_epoch_ids"), "feed_epoch_ids" + ), + broker_profile_ids=_string_tuple( + data.get("broker_profile_ids"), "broker_profile_ids" + ), + constraint_set_ids=_string_tuple( + data.get("constraint_set_ids"), "constraint_set_ids" + ), + stream_ids=_string_tuple(data.get("stream_ids"), "stream_ids"), + event_content_sha256=str(data.get("event_content_sha256", "")), + confidence_support_count=_strict_int( + data.get("confidence_support_count"), + "confidence_support_count", + ), + limitations=_string_tuple(data.get("limitations"), "limitations"), + slice_id=str(data.get("slice_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionActivityBenchmarkEvidenceV1: + """Activity-specific evidence extracted from the shared benchmark.""" + + scorecard_id: str + candidate_score_ids: tuple[str, ...] + split_kinds: tuple[str, ...] + metric_support_counts: Mapping[str, int] + metric_mean_errors: Mapping[str, float] + mean_restoration_gain_vs_degraded: float + promotion_eligible_candidate_count: int + calibration_supported_candidate_count: int + execution_failure_count: int + evidence_id: str = "" + schema_version: str = ( + RECONSTRUCTION_ACTIVITY_BENCHMARK_EVIDENCE_SCHEMA_VERSION + ) + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_ACTIVITY_BENCHMARK_EVIDENCE_SCHEMA_VERSION, + "reconstruction activity benchmark evidence", + ) + object.__setattr__( + self, "scorecard_id", _required_text(self.scorecard_id) + ) + candidate_ids = _normalized_text_tuple(self.candidate_score_ids) + if not candidate_ids: + raise ValueError("activity benchmark evidence requires candidates") + object.__setattr__(self, "candidate_score_ids", candidate_ids) + split_kinds = _normalized_text_tuple(self.split_kinds) + if not split_kinds: + raise ValueError("activity benchmark evidence requires split kinds") + object.__setattr__(self, "split_kinds", split_kinds) + supports = { + _required_text(name): _bounded_int( + value, f"metric_support_counts.{name}", 1, 2**63 - 1 + ) + for name, value in self.metric_support_counts.items() + } + errors = { + _required_text(name): _nonnegative_float( + value, f"metric_mean_errors.{name}" + ) + for name, value in self.metric_mean_errors.items() + } + required = set(ACTIVITY_REVERSE_DEGRADATION_METRICS) + if set(supports) != required or set(errors) != required: + raise ValueError("activity benchmark metric coverage differs") + object.__setattr__( + self, "metric_support_counts", dict(sorted(supports.items())) + ) + object.__setattr__( + self, "metric_mean_errors", dict(sorted(errors.items())) + ) + object.__setattr__( + self, + "mean_restoration_gain_vs_degraded", + _finite_float( + self.mean_restoration_gain_vs_degraded, + "mean_restoration_gain_vs_degraded", + ), + ) + for name in ( + "promotion_eligible_candidate_count", + "calibration_supported_candidate_count", + ): + object.__setattr__( + self, + name, + _bounded_int(getattr(self, name), name, 0, len(candidate_ids)), + ) + object.__setattr__( + self, + "execution_failure_count", + _bounded_int( + self.execution_failure_count, + "execution_failure_count", + 0, + 2**63 - 1, + ), + ) + expected = _stable_id("activity-benchmark", self.identity_payload()) + supplied = _optional_text(self.evidence_id) + if supplied is not None and supplied != expected: + raise ValueError("activity benchmark evidence_id differs") + object.__setattr__(self, "evidence_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return bounded benchmark evidence without selecting a winner.""" + return { + "schema_version": self.schema_version, + "scorecard_id": self.scorecard_id, + "candidate_score_ids": list(self.candidate_score_ids), + "split_kinds": list(self.split_kinds), + "metric_support_counts": dict(self.metric_support_counts), + "metric_mean_errors": dict(self.metric_mean_errors), + "mean_restoration_gain_vs_degraded": ( + self.mean_restoration_gain_vs_degraded + ), + "promotion_eligible_candidate_count": ( + self.promotion_eligible_candidate_count + ), + "calibration_supported_candidate_count": ( + self.calibration_supported_candidate_count + ), + "execution_failure_count": self.execution_failure_count, + "automatic_winner": False, + "winner_candidate_id": None, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return stable benchmark evidence.""" + return {**self.identity_payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionActivityBenchmarkEvidenceV1": + """Restore activity benchmark evidence.""" + _require_schema( + data, RECONSTRUCTION_ACTIVITY_BENCHMARK_EVIDENCE_SCHEMA_VERSION + ) + _require_derived(data, "automatic_winner", False) + _require_derived(data, "winner_candidate_id", None) + return cls( + scorecard_id=str(data.get("scorecard_id", "")), + candidate_score_ids=_string_tuple( + data.get("candidate_score_ids"), "candidate_score_ids" + ), + split_kinds=_string_tuple(data.get("split_kinds"), "split_kinds"), + metric_support_counts={ + str(name): _strict_int(value, str(name)) + for name, value in _mapping( + data.get("metric_support_counts"), + "metric_support_counts", + ).items() + }, + metric_mean_errors={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping( + data.get("metric_mean_errors"), "metric_mean_errors" + ).items() + }, + mean_restoration_gain_vs_degraded=_finite_float( + data.get("mean_restoration_gain_vs_degraded"), + "mean_restoration_gain_vs_degraded", + ), + promotion_eligible_candidate_count=_strict_int( + data.get("promotion_eligible_candidate_count"), + "promotion_eligible_candidate_count", + ), + calibration_supported_candidate_count=_strict_int( + data.get("calibration_supported_candidate_count"), + "calibration_supported_candidate_count", + ), + execution_failure_count=_strict_int( + data.get("execution_failure_count"), "execution_failure_count" + ), + evidence_id=str(data.get("evidence_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionActivityManifestV1: + """Bounded derived activity metadata for one final event population.""" + + run_id: str + ensemble_member_id: str + information_mode: InformationMode + information_manifest_id: str + as_of_ns: int | None + policy: ReconstructionActivityPolicyV1 + slices: tuple[ReconstructionActivitySliceV1, ...] + input_content_sha256: str + window_id: str | None = None + synchronization_unit_id: str | None = None + product_manifest_id: str | None = None + calibration_report_id: str | None = None + benchmark_evidence: ReconstructionActivityBenchmarkEvidenceV1 | None = None + manifest_id: str = "" + schema_version: str = RECONSTRUCTION_ACTIVITY_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_ACTIVITY_MANIFEST_SCHEMA_VERSION, + "reconstruction activity manifest", + ) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, "ensemble_member_id", _required_text(self.ensemble_member_id) + ) + mode = InformationMode.from_value(self.information_mode) + object.__setattr__(self, "information_mode", mode) + object.__setattr__( + self, + "information_manifest_id", + _required_text(self.information_manifest_id), + ) + as_of = _optional_int(self.as_of_ns, "as_of_ns") + if mode is InformationMode.EX_ANTE_SIMULATION and as_of is None: + raise ValueError("ex-ante activity evidence requires as_of_ns") + if mode is InformationMode.EX_POST_RECONSTRUCTION and as_of is not None: + raise ValueError("ex-post activity evidence rejects as_of_ns") + object.__setattr__(self, "as_of_ns", as_of) + if not isinstance(self.policy, ReconstructionActivityPolicyV1): + raise TypeError("activity manifest requires a v1 policy") + slices = tuple( + sorted( + self.slices, key=lambda item: (item.symbol, item.scope.value) + ) + ) + if not slices or len(slices) > self.policy.max_slices: + raise ValueError( + "activity manifest slice count is empty or unbounded" + ) + if any( + not isinstance(item, ReconstructionActivitySliceV1) + for item in slices + ): + raise TypeError("activity manifest slices require v1 contracts") + if len({(item.symbol, item.scope) for item in slices}) != len(slices): + raise ValueError("activity manifest contains duplicate slices") + if any(item.scope not in self.policy.scopes for item in slices): + raise ValueError("activity manifest scope is outside policy") + symbols = {item.symbol for item in slices} + if len(symbols) > self.policy.max_symbols: + raise ValueError("activity manifest symbol count is unbounded") + _validate_slice_reconciliation(slices) + object.__setattr__(self, "slices", slices) + object.__setattr__( + self, + "input_content_sha256", + _required_sha256(self.input_content_sha256, "input_content_sha256"), + ) + for name in ( + "window_id", + "synchronization_unit_id", + "product_manifest_id", + "calibration_report_id", + ): + object.__setattr__(self, name, _optional_text(getattr(self, name))) + if self.benchmark_evidence is not None and not isinstance( + self.benchmark_evidence, + ReconstructionActivityBenchmarkEvidenceV1, + ): + raise TypeError("activity benchmark evidence requires v1 contract") + expected = _stable_id("activity-manifest", self.identity_payload()) + supplied = _optional_text(self.manifest_id) + if supplied is not None and supplied != expected: + raise ValueError("activity manifest_id differs from its content") + object.__setattr__(self, "manifest_id", expected) + if len(self.to_json().encode("utf-8")) > self.policy.max_payload_bytes: + raise ValueError("activity manifest exceeds policy payload limit") + + @property + def symbols(self) -> tuple[str, ...]: + """Return represented symbols.""" + return tuple(sorted({item.symbol for item in self.slices})) + + @property + def event_count(self) -> int: + """Return total events without double-counting origin slices.""" + merged = [ + item + for item in self.slices + if item.scope is ActivitySliceScope.MERGED + ] + if merged: + return sum(item.event_count for item in merged) + return sum( + item.event_count + for item in self.slices + if item.scope + in (ActivitySliceScope.OBSERVED, ActivitySliceScope.SYNTHETIC) + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return complete scientific and provenance identity.""" + return { + "schema_version": self.schema_version, + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "information_mode": self.information_mode.value, + "information_manifest_id": self.information_manifest_id, + "as_of_ns": self.as_of_ns, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "product_manifest_id": self.product_manifest_id, + "calibration_report_id": self.calibration_report_id, + "policy": self.policy.to_dict(), + "slices": [item.to_dict() for item in self.slices], + "input_content_sha256": self.input_content_sha256, + "benchmark_evidence": ( + self.benchmark_evidence.to_dict() + if self.benchmark_evidence is not None + else None + ), + "metric_definitions": activity_metric_definitions(), + "bar_projection_semantics": activity_bar_projection_semantics(), + "output_mode": "derived_metadata", + "event_schema_augmented": False, + "centralized_traded_volume_claim": False, + "automatic_winner": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic activity-manifest JSON.""" + return { + **self.identity_payload(), + "symbols": list(self.symbols), + "event_count": self.event_count, + "manifest_id": self.manifest_id, + } + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionActivityManifestV1": + """Restore and reconcile an activity manifest.""" + _require_schema(data, RECONSTRUCTION_ACTIVITY_MANIFEST_SCHEMA_VERSION) + _require_derived( + data, "event_schema_version", SYNTHETIC_EVENT_SCHEMA_VERSION + ) + _require_derived( + data, "metric_definitions", activity_metric_definitions() + ) + _require_derived( + data, + "bar_projection_semantics", + activity_bar_projection_semantics(), + ) + _require_derived(data, "output_mode", "derived_metadata") + _require_derived(data, "event_schema_augmented", False) + _require_derived(data, "centralized_traded_volume_claim", False) + _require_derived(data, "automatic_winner", False) + benchmark = data.get("benchmark_evidence") + manifest = cls( + run_id=str(data.get("run_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + information_manifest_id=str( + data.get("information_manifest_id", "") + ), + as_of_ns=_optional_int(data.get("as_of_ns"), "as_of_ns"), + policy=ReconstructionActivityPolicyV1.from_dict( + _mapping(data.get("policy"), "policy") + ), + slices=tuple( + ReconstructionActivitySliceV1.from_dict(item) + for item in _mapping_sequence(data.get("slices"), "slices") + ), + input_content_sha256=str(data.get("input_content_sha256", "")), + window_id=_mapping_optional_text(data, "window_id"), + synchronization_unit_id=_mapping_optional_text( + data, "synchronization_unit_id" + ), + product_manifest_id=_mapping_optional_text( + data, "product_manifest_id" + ), + calibration_report_id=_mapping_optional_text( + data, "calibration_report_id" + ), + benchmark_evidence=( + ReconstructionActivityBenchmarkEvidenceV1.from_dict( + _mapping(benchmark, "benchmark_evidence") + ) + if benchmark is not None + else None + ), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived(data, "symbols", list(manifest.symbols)) + _require_derived(data, "event_count", manifest.event_count) + return manifest + + @classmethod + def from_json(cls, text: str) -> "ReconstructionActivityManifestV1": + """Restore an activity manifest from JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class _ActivityEventView: + """Minimal projected event fields required by the online accumulator.""" + + event_id: str + origin: SyntheticEventOrigin + symbol: str + event_time_ns: int + event_sequence: int + bid: float + ask: float + run_id: str + ensemble_member_id: str + source_version_id: str + generator_id: str | None + generator_version: str | None + generator_config_id: str | None + reference_id: str | None + motif_id: str | None + feed_epoch_id: str | None + broker_profile_id: str | None + constraint_set_id: str | None + confidence: float | None + stream_ids: tuple[str, ...] = () + + +@dataclass(slots=True) +class _ActivitySliceState: + """Constant-size numeric and bounded-provenance state for one slice.""" + + symbol: str + scope: ActivitySliceScope + policy: ReconstructionActivityPolicyV1 + event_count: int = 0 + first_time_ns: int | None = None + last_time_ns: int | None = None + last_bid: float | None = None + last_ask: float | None = None + interarrival_count: int = 0 + interarrival_total_ns: int = 0 + min_interarrival_ns: int | None = None + max_interarrival_ns: int | None = None + price_change_count: int = 0 + stale_quote_count: int = 0 + spread_total: float = 0.0 + min_spread: float | None = None + max_spread: float | None = None + confidence_total: float = 0.0 + confidence_count: int = 0 + source_version_ids: set[str] = field(default_factory=set) + generator_ids: set[str] = field(default_factory=set) + generator_versions: set[str] = field(default_factory=set) + generator_config_ids: set[str] = field(default_factory=set) + reference_ids: set[str] = field(default_factory=set) + motif_ids: set[str] = field(default_factory=set) + feed_epoch_ids: set[str] = field(default_factory=set) + broker_profile_ids: set[str] = field(default_factory=set) + constraint_set_ids: set[str] = field(default_factory=set) + stream_ids: set[str] = field(default_factory=set) + provenance_occurrence_counts: dict[str, int] = field(default_factory=dict) + provenance_digests: dict[str, Any] = field(default_factory=dict) + truncated_provenance: set[str] = field(default_factory=set) + digest: Any = field( + default_factory=lambda: hashlib.sha256(b"activity-v1\n") + ) + + def add(self, event: _ActivityEventView) -> None: + """Consume one ordered event using constant numeric state.""" + if self.last_time_ns is not None: + gap = event.event_time_ns - self.last_time_ns + if gap < 0: + raise ValueError("activity events are not time ordered") + self.interarrival_count += 1 + self.interarrival_total_ns += gap + self.min_interarrival_ns = ( + gap + if self.min_interarrival_ns is None + else min(self.min_interarrival_ns, gap) + ) + self.max_interarrival_ns = ( + gap + if self.max_interarrival_ns is None + else max(self.max_interarrival_ns, gap) + ) + if event.bid == self.last_bid and event.ask == self.last_ask: + self.stale_quote_count += 1 + else: + self.price_change_count += 1 + else: + self.first_time_ns = event.event_time_ns + self.last_time_ns = event.event_time_ns + self.last_bid = event.bid + self.last_ask = event.ask + self.event_count += 1 + spread = event.ask - event.bid + self.spread_total += spread + self.min_spread = ( + spread if self.min_spread is None else min(self.min_spread, spread) + ) + self.max_spread = ( + spread if self.max_spread is None else max(self.max_spread, spread) + ) + if event.confidence is not None: + self.confidence_total += event.confidence + self.confidence_count += 1 + self._add_provenance("source_version_ids", event.source_version_id) + for name in ( + "generator_id", + "generator_version", + "generator_config_id", + "reference_id", + "motif_id", + "feed_epoch_id", + "broker_profile_id", + "constraint_set_id", + ): + value = getattr(event, name) + if value is not None: + self._add_provenance(name + "s", value) + for stream_id in event.stream_ids: + self._add_provenance("stream_ids", stream_id) + row = { + "event_id": event.event_id, + "origin": event.origin.value, + "symbol": event.symbol, + "event_time_ns": event.event_time_ns, + "event_sequence": event.event_sequence, + "bid": event.bid, + "ask": event.ask, + "run_id": event.run_id, + "ensemble_member_id": event.ensemble_member_id, + "source_version_id": event.source_version_id, + "generator_id": event.generator_id, + "generator_version": event.generator_version, + "generator_config_id": event.generator_config_id, + "reference_id": event.reference_id, + "motif_id": event.motif_id, + "feed_epoch_id": event.feed_epoch_id, + "broker_profile_id": event.broker_profile_id, + "constraint_set_id": event.constraint_set_id, + "confidence": event.confidence, + } + self.digest.update(canonical_contract_json(row).encode("utf-8")) + self.digest.update(b"\n") + + def _add_provenance(self, name: str, value: str) -> None: + selected = _required_text(value) + self.provenance_occurrence_counts[name] = ( + self.provenance_occurrence_counts.get(name, 0) + 1 + ) + provenance_digest = self.provenance_digests.get(name) + if provenance_digest is None: + provenance_digest = hashlib.sha256( + f"activity-provenance-v1:{name}\n".encode("utf-8") + ) + self.provenance_digests[name] = provenance_digest + provenance_digest.update( + canonical_contract_json(selected).encode("utf-8") + ) + provenance_digest.update(b"\n") + values = cast(set[str], getattr(self, name)) + if selected in values: + return + if len(values) >= self.policy.max_provenance_values: + self.truncated_provenance.add(name) + return + values.add(selected) + + def finalize(self) -> ReconstructionActivitySliceV1: + """Freeze the online state into a strict bounded slice.""" + if ( + not self.event_count + or self.first_time_ns is None + or self.last_time_ns is None + or self.min_spread is None + or self.max_spread is None + ): + raise ValueError("cannot finalize an empty activity slice") + duration = self.last_time_ns - self.first_time_ns + mean_confidence = ( + self.confidence_total / self.confidence_count + if self.confidence_count + else None + ) + limitations = ["centralized_traded_volume_unavailable"] + if duration == 0: + limitations.append("zero_exposure_duration") + if self.confidence_count < self.event_count: + limitations.append("event_confidence_partial_or_unavailable") + if not self.stream_ids: + limitations.append("stream_identity_unavailable") + limitations.extend( + ( + f"{name}_truncated:occurrence_count=" + f"{self.provenance_occurrence_counts[name]}:sha256=" + f"{self.provenance_digests[name].hexdigest()}" + ) + for name in sorted(self.truncated_provenance) + ) + values: dict[str, tuple[int | float | None, int, tuple[str, ...]]] = { + "event_count": (self.event_count, self.event_count, ()), + "quote_update_count": (self.event_count, self.event_count, ()), + "exposure_duration_ns": (duration, self.event_count, ()), + "tick_intensity_per_second": ( + ( + self.event_count * NANOSECONDS_PER_SECOND / duration + if duration + else None + ), + self.event_count if duration else 0, + () if duration else ("zero_exposure_duration",), + ), + "mean_interarrival_ns": ( + ( + self.interarrival_total_ns / self.interarrival_count + if self.interarrival_count + else None + ), + self.interarrival_count, + () if self.interarrival_count else ("no_quote_transition",), + ), + "min_interarrival_ns": ( + self.min_interarrival_ns, + self.interarrival_count, + () if self.interarrival_count else ("no_quote_transition",), + ), + "max_interarrival_ns": ( + self.max_interarrival_ns, + self.interarrival_count, + () if self.interarrival_count else ("no_quote_transition",), + ), + "price_change_count": ( + self.price_change_count, + self.interarrival_count, + (), + ), + "stale_quote_count": ( + self.stale_quote_count, + self.interarrival_count, + (), + ), + "stale_quote_rate": ( + ( + self.stale_quote_count / self.interarrival_count + if self.interarrival_count + else None + ), + self.interarrival_count, + () if self.interarrival_count else ("no_quote_transition",), + ), + "mean_spread": ( + self.spread_total / self.event_count, + self.event_count, + ("spread_is_a_liquidity_proxy_not_traded_volume",), + ), + "min_spread": ( + self.min_spread, + self.event_count, + ("spread_is_a_liquidity_proxy_not_traded_volume",), + ), + "max_spread": ( + self.max_spread, + self.event_count, + ("spread_is_a_liquidity_proxy_not_traded_volume",), + ), + "mean_event_confidence": ( + mean_confidence, + self.confidence_count, + ( + () + if self.confidence_count == self.event_count + else ("event_confidence_partial_or_unavailable",) + ), + ), + } + metrics = tuple( + _activity_metric( + name, + _rounded_optional(value[0], self.policy.rounding_digits), + value[1], + value[2], + ) + for name, value in values.items() + ) + return ReconstructionActivitySliceV1( + symbol=self.symbol, + scope=self.scope, + start_event_time_ns=self.first_time_ns, + end_event_time_ns=self.last_time_ns, + metrics=metrics, + source_version_ids=tuple(self.source_version_ids), + generator_ids=tuple(self.generator_ids), + generator_versions=tuple(self.generator_versions), + generator_config_ids=tuple(self.generator_config_ids), + reference_ids=tuple(self.reference_ids), + motif_ids=tuple(self.motif_ids), + feed_epoch_ids=tuple(self.feed_epoch_ids), + broker_profile_ids=tuple(self.broker_profile_ids), + constraint_set_ids=tuple(self.constraint_set_ids), + stream_ids=tuple(self.stream_ids), + event_content_sha256=self.digest.hexdigest(), + confidence_support_count=self.confidence_count, + limitations=tuple(limitations), + ) + + +class _ActivityAccumulator: + """Route ordered events into bounded symbol/origin slice states.""" + + def __init__( + self, + *, + run_id: str, + ensemble_member_id: str, + policy: ReconstructionActivityPolicyV1, + ) -> None: + self.run_id = _required_text(run_id) + self.ensemble_member_id = _required_text(ensemble_member_id) + self.policy = policy + self.states: dict[ + tuple[str, ActivitySliceScope], _ActivitySliceState + ] = {} + self.last_positions: dict[str, tuple[int, int]] = {} + self.symbols: set[str] = set() + + def add(self, event: _ActivityEventView) -> None: + if event.run_id != self.run_id: + raise ValueError("activity event run_id differs") + if event.ensemble_member_id != self.ensemble_member_id: + raise ValueError("activity event ensemble member differs") + symbol = _normalized_symbol(event.symbol) + position = (event.event_time_ns, event.event_sequence) + previous = self.last_positions.get(symbol) + if previous is not None and position <= previous: + raise ValueError( + "activity events are not strictly ordered per symbol" + ) + self.last_positions[symbol] = position + self.symbols.add(symbol) + if len(self.symbols) > self.policy.max_symbols: + raise ValueError("activity symbols exceed policy limit") + origin_scope = ( + ActivitySliceScope.OBSERVED + if event.origin is SyntheticEventOrigin.OBSERVED + else ActivitySliceScope.SYNTHETIC + ) + scopes = (origin_scope, ActivitySliceScope.MERGED) + for scope in scopes: + if scope not in self.policy.scopes: + continue + key = (symbol, scope) + state = self.states.get(key) + if state is None: + if len(self.states) >= self.policy.max_slices: + raise ValueError("activity slices exceed policy limit") + state = _ActivitySliceState(symbol, scope, self.policy) + self.states[key] = state + state.add(event) + + def finalize(self) -> tuple[ReconstructionActivitySliceV1, ...]: + if not self.states: + raise ValueError("activity aggregation requires events") + return tuple(state.finalize() for state in self.states.values()) + + +def summarize_reconstruction_activity( + events: Iterable[SyntheticEventV1], + *, + run_id: str, + ensemble_member_id: str, + information_mode: InformationMode, + information_manifest_id: str, + as_of_ns: int | None = None, + policy: ReconstructionActivityPolicyV1 | None = None, + stream_ids_by_symbol: Mapping[str, Iterable[str]] | None = None, + window_id: str | None = None, + synchronization_unit_id: str | None = None, + product_manifest_id: str | None = None, + calibration_report_id: str | None = None, + benchmark_evidence: ReconstructionActivityBenchmarkEvidenceV1 | None = None, +) -> ReconstructionActivityManifestV1: + """Aggregate ordered narrow events without retaining their rows.""" + selected_policy = policy or ReconstructionActivityPolicyV1() + stream_map = { + _normalized_symbol(symbol): _normalized_text_tuple(values) + for symbol, values in (stream_ids_by_symbol or {}).items() + } + views = ( + _activity_event_view( + event, + stream_ids=stream_map.get(event.symbol, ()), + ) + for event in events + ) + return _summarize_activity_views( + views, + run_id=run_id, + ensemble_member_id=ensemble_member_id, + information_mode=information_mode, + information_manifest_id=information_manifest_id, + as_of_ns=as_of_ns, + policy=selected_policy, + window_id=window_id, + synchronization_unit_id=synchronization_unit_id, + product_manifest_id=product_manifest_id, + calibration_report_id=calibration_report_id, + benchmark_evidence=benchmark_evidence, + ) + + +def summarize_reconstruction_activity_streams( + streams: Iterable[SyntheticEventStreamV1], + *, + information_mode: InformationMode, + information_manifest_id: str, + as_of_ns: int | None = None, + policy: ReconstructionActivityPolicyV1 | None = None, + window_id: str | None = None, + synchronization_unit_id: str | None = None, + product_manifest_id: str | None = None, + calibration_report_id: str | None = None, + benchmark_evidence: ReconstructionActivityBenchmarkEvidenceV1 | None = None, +) -> ReconstructionActivityManifestV1: + """Aggregate one compatible stream group in bounded numeric state.""" + stream_tuple = tuple(streams) + if not stream_tuple: + raise ValueError("activity aggregation requires streams") + if any( + not isinstance(item, SyntheticEventStreamV1) for item in stream_tuple + ): + raise TypeError("activity aggregation requires v1 streams") + run_ids = {item.run_id for item in stream_tuple} + member_ids = {item.ensemble_member_id for item in stream_tuple} + if len(run_ids) != 1 or len(member_ids) != 1: + raise ValueError("activity streams differ in run or ensemble member") + stream_map = {item.symbol: (item.stream_id,) for item in stream_tuple} + return summarize_reconstruction_activity( + (event for stream in stream_tuple for event in stream.events), + run_id=next(iter(run_ids)), + ensemble_member_id=next(iter(member_ids)), + information_mode=information_mode, + information_manifest_id=information_manifest_id, + as_of_ns=as_of_ns, + policy=policy, + stream_ids_by_symbol=stream_map, + window_id=window_id, + synchronization_unit_id=synchronization_unit_id, + product_manifest_id=product_manifest_id, + calibration_report_id=calibration_report_id, + benchmark_evidence=benchmark_evidence, + ) + + +def summarize_committed_reconstruction_activity( + manifest_path: str | Path, + *, + information_mode: InformationMode, + information_manifest_id: str, + as_of_ns: int | None = None, + policy: ReconstructionActivityPolicyV1 | None = None, + calibration_report_id: str | None = None, + benchmark_evidence: ReconstructionActivityBenchmarkEvidenceV1 | None = None, + batch_size: int = DEFAULT_ACTIVITY_BATCH_SIZE, +) -> ReconstructionActivityManifestV1: + """Stream projected final Parquet columns into compact activity evidence.""" + product = verify_reconstruction_publication(manifest_path) + size = _bounded_int(batch_size, "batch_size", 1, 1_000_000) + stream_map: dict[str, tuple[str, ...]] = {} + for symbol in product.symbols: + stream_map[symbol] = _normalized_text_tuple( + partition.stream_id + for partition in product.partitions + if partition.symbol == symbol + ) + + def views() -> Iterable[_ActivityEventView]: + for batch in iter_reconstruction_event_batches( + manifest_path, + columns=ACTIVITY_EVENT_COLUMNS, + batch_size=size, + ): + for row in batch.to_pylist(): + mapping = _mapping(row, "activity event row") + symbol = _normalized_symbol(str(mapping.get("symbol", ""))) + yield _activity_event_view_from_mapping( + mapping, + stream_ids=stream_map.get(symbol, ()), + ) + + return _summarize_activity_views( + views(), + run_id=product.run_id, + ensemble_member_id=product.ensemble_member_id, + information_mode=information_mode, + information_manifest_id=information_manifest_id, + as_of_ns=as_of_ns, + policy=policy or ReconstructionActivityPolicyV1(), + window_id=product.window_id, + synchronization_unit_id=product.synchronization_unit_id, + product_manifest_id=product.manifest_id, + calibration_report_id=calibration_report_id, + benchmark_evidence=benchmark_evidence, + ) + + +def write_reconstruction_activity_manifest( + manifest: ReconstructionActivityManifestV1, + directory: str | Path, +) -> ArtifactRef: + """Atomically persist and read back one compact activity manifest.""" + if not isinstance(manifest, ReconstructionActivityManifestV1): + raise TypeError("activity persistence requires a v1 manifest") + encoded = manifest.to_json().encode("utf-8") + b"\n" + digest = hashlib.sha256(encoded).hexdigest() + root = Path(directory).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + path = root / f"reconstruction-activity-manifest-{digest}.json" + if path.exists(): + if path.read_bytes() != encoded: + raise ValueError("content-addressed activity manifest collision") + else: + temporary = root / f".{path.name}.{os.getpid()}.tmp" + try: + with temporary.open("wb") as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + restored = read_reconstruction_activity_manifest(path) + if restored != manifest: + raise ValueError("persisted activity manifest differs on readback") + return ArtifactRef( + kind="activity-manifest", + path=str(path), + size_bytes=len(encoded), + sha256=digest, + metadata={ + "manifest_id": manifest.manifest_id, + "product_manifest_id": manifest.product_manifest_id, + "event_count": manifest.event_count, + }, + ) + + +def read_reconstruction_activity_manifest( + path: str | Path, +) -> ReconstructionActivityManifestV1: + """Hash, filename, size, and identity-check one activity manifest.""" + source = Path(path).expanduser().resolve() + encoded = source.read_bytes() + if len(encoded) > MAX_ACTIVITY_PAYLOAD_BYTES + 1: + raise ValueError("activity manifest exceeds absolute payload limit") + prefix = "reconstruction-activity-manifest-" + if not source.name.startswith(prefix) or not source.name.endswith(".json"): + raise ValueError("activity manifest filename is not content addressed") + expected = source.name[len(prefix) : -len(".json")] + if hashlib.sha256(encoded).hexdigest() != expected: + raise ValueError("activity manifest content hash differs from filename") + return ReconstructionActivityManifestV1.from_json(encoded.decode("utf-8")) + + +def reconstruction_activity_benchmark_evidence( + scorecard: ReverseDegradationScorecardV1, + *, + rounding_digits: int = DEFAULT_ACTIVITY_ROUNDING_DIGITS, +) -> ReconstructionActivityBenchmarkEvidenceV1: + """Extract the canonical activity metrics from shared benchmark evidence.""" + if not isinstance(scorecard, ReverseDegradationScorecardV1): + raise TypeError("activity evidence requires a v1 benchmark scorecard") + rounding = _bounded_int(rounding_digits, "rounding_digits", 0, 15) + totals = dict.fromkeys(ACTIVITY_REVERSE_DEGRADATION_METRICS, 0.0) + supports = dict.fromkeys(ACTIVITY_REVERSE_DEGRADATION_METRICS, 0) + restoration: list[float] = [] + calibration_supported = 0 + execution_failures = 0 + for candidate in scorecard.candidate_scores: + for slice_score in candidate.slice_scores: + for name in ACTIVITY_REVERSE_DEGRADATION_METRICS: + if name not in slice_score.metrics: + raise ValueError( + f"benchmark slice lacks activity metric {name}" + ) + value = _nonnegative_float( + slice_score.metrics[name], f"benchmark.{name}" + ) + totals[name] += value + supports[name] += 1 + restoration.append( + _finite_float( + slice_score.metrics.get( + "restoration_gain_vs_degraded", 0.0 + ), + "restoration_gain_vs_degraded", + ) + ) + support_intervals = candidate.uncertainty_metrics.get( + "support_interval_count", 0 + ) + if isinstance(support_intervals, int) and support_intervals > 0: + calibration_supported += 1 + failures = candidate.execution_summary.get("failure_count", 0) + if not isinstance(failures, int) or isinstance(failures, bool): + raise ValueError("benchmark failure_count must be integral") + execution_failures += failures + mean_errors = { + name: round(totals[name] / supports[name], rounding) + for name in ACTIVITY_REVERSE_DEGRADATION_METRICS + } + return ReconstructionActivityBenchmarkEvidenceV1( + scorecard_id=scorecard.scorecard_id, + candidate_score_ids=tuple( + item.candidate_score_id for item in scorecard.candidate_scores + ), + split_kinds=tuple( + item.split_kind.value for item in scorecard.candidate_scores + ), + metric_support_counts=supports, + metric_mean_errors=mean_errors, + mean_restoration_gain_vs_degraded=round( + sum(restoration) / len(restoration), rounding + ), + promotion_eligible_candidate_count=sum( + item.promotion_eligible for item in scorecard.candidate_scores + ), + calibration_supported_candidate_count=calibration_supported, + execution_failure_count=execution_failures, + ) + + +def _summarize_activity_views( + views: Iterable[_ActivityEventView], + *, + run_id: str, + ensemble_member_id: str, + information_mode: InformationMode, + information_manifest_id: str, + as_of_ns: int | None, + policy: ReconstructionActivityPolicyV1, + window_id: str | None, + synchronization_unit_id: str | None, + product_manifest_id: str | None, + calibration_report_id: str | None, + benchmark_evidence: ReconstructionActivityBenchmarkEvidenceV1 | None, +) -> ReconstructionActivityManifestV1: + if policy.volume_state in ( + ActivityVolumeState.OBSERVED_SOURCE_SIZE, + ActivityVolumeState.BROKER_SUPPLIED_SIZE, + ): + raise ValueError( + "final event schema has no source-supported size fields" + ) + accumulator = _ActivityAccumulator( + run_id=run_id, + ensemble_member_id=ensemble_member_id, + policy=policy, + ) + for event in views: + accumulator.add(event) + slices = accumulator.finalize() + input_hash = _content_sha256( + { + "slice_content": [ + [item.symbol, item.scope.value, item.event_content_sha256] + for item in sorted( + slices, key=lambda value: (value.symbol, value.scope.value) + ) + ] + } + ) + return ReconstructionActivityManifestV1( + run_id=run_id, + ensemble_member_id=ensemble_member_id, + information_mode=information_mode, + information_manifest_id=information_manifest_id, + as_of_ns=as_of_ns, + policy=policy, + slices=slices, + input_content_sha256=input_hash, + window_id=window_id, + synchronization_unit_id=synchronization_unit_id, + product_manifest_id=product_manifest_id, + calibration_report_id=calibration_report_id, + benchmark_evidence=benchmark_evidence, + ) + + +def _activity_event_view( + event: SyntheticEventV1, + *, + stream_ids: tuple[str, ...], +) -> _ActivityEventView: + if not isinstance(event, SyntheticEventV1): + raise TypeError("activity aggregation requires v1 events") + return _ActivityEventView( + event_id=event.event_id, + origin=event.origin, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + run_id=event.run_id, + ensemble_member_id=event.ensemble_member_id, + source_version_id=event.source_version_id, + generator_id=event.generator_id, + generator_version=event.generator_version, + generator_config_id=event.generator_config_id, + reference_id=event.reference_id, + motif_id=event.motif_id, + feed_epoch_id=event.feed_epoch_id, + broker_profile_id=event.broker_profile_id, + constraint_set_id=event.constraint_set_id, + confidence=event.confidence, + stream_ids=stream_ids, + ) + + +def _activity_event_view_from_mapping( + data: Mapping[str, Any], + *, + stream_ids: tuple[str, ...], +) -> _ActivityEventView: + bid = _positive_float(data.get("bid"), "bid") + ask = _positive_float(data.get("ask"), "ask") + if ask < bid: + raise ValueError("activity ask must not be below bid") + return _ActivityEventView( + event_id=_required_text(str(data.get("event_id", ""))), + origin=SyntheticEventOrigin.from_value(str(data.get("origin", ""))), + symbol=_normalized_symbol(str(data.get("symbol", ""))), + event_time_ns=_strict_int(data.get("event_time_ns"), "event_time_ns"), + event_sequence=_strict_int( + data.get("event_sequence"), "event_sequence" + ), + bid=bid, + ask=ask, + run_id=_required_text(str(data.get("run_id", ""))), + ensemble_member_id=_required_text( + str(data.get("ensemble_member_id", "")) + ), + source_version_id=_required_text( + str(data.get("source_version_id", "")) + ), + generator_id=_mapping_optional_text(data, "generator_id"), + generator_version=_mapping_optional_text(data, "generator_version"), + generator_config_id=_mapping_optional_text(data, "generator_config_id"), + reference_id=_mapping_optional_text(data, "reference_id"), + motif_id=_mapping_optional_text(data, "motif_id"), + feed_epoch_id=_mapping_optional_text(data, "feed_epoch_id"), + broker_profile_id=_mapping_optional_text(data, "broker_profile_id"), + constraint_set_id=_mapping_optional_text(data, "constraint_set_id"), + confidence=_optional_float(data.get("confidence"), "confidence"), + stream_ids=stream_ids, + ) + + +def _activity_metric( + name: str, + value: int | float | None, + support_count: int, + limitations: tuple[str, ...], +) -> ReconstructionActivityMetricV1: + unit, aggregation, semantics = _METRIC_DEFINITIONS[name] + return ReconstructionActivityMetricV1( + name=name, + value=value, + unit=unit, + aggregation=aggregation, + semantics=semantics, + support_count=support_count, + limitations=limitations, + ) + + +def _validate_slice_reconciliation( + slices: Sequence[ReconstructionActivitySliceV1], +) -> None: + by_symbol: dict[str, dict[ActivitySliceScope, int]] = {} + for item in slices: + by_symbol.setdefault(item.symbol, {})[item.scope] = item.event_count + for scopes in by_symbol.values(): + if ActivitySliceScope.MERGED not in scopes: + continue + origin_total = sum( + scopes.get(scope, 0) + for scope in ( + ActivitySliceScope.OBSERVED, + ActivitySliceScope.SYNTHETIC, + ) + ) + if ( + ActivitySliceScope.OBSERVED in scopes + and ActivitySliceScope.SYNTHETIC in scopes + and scopes[ActivitySliceScope.MERGED] != origin_total + ): + raise ValueError("activity merged/origin counts do not reconcile") + + +def _rounded_optional( + value: int | float | None, digits: int +) -> int | float | None: + if value is None or isinstance(value, int): + return value + return round(value, digits) + + +def _stable_id(kind: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + return f"{kind}:sha256:{digest}" + + +def _content_sha256(payload: Mapping[str, JSONValue]) -> str: + return hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + + +def _enum_value(cls: type[_EnumT], value: str | _EnumT, name: str) -> _EnumT: + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError(f"unsupported {name}") from err + + +def _required_text(value: Any) -> str: + if not isinstance(value, str): + raise ValueError("expected text") + normalized = value.strip() + if not normalized or len(normalized) > MAX_ACTIVITY_TEXT: + raise ValueError("text is empty or too long") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, str) and not value.strip(): + return None + return _required_text(value) + + +def _normalized_text_tuple(values: Iterable[str]) -> tuple[str, ...]: + return tuple(sorted({_required_text(value) for value in values})) + + +def _normalized_symbol(value: str) -> str: + symbol = _required_text(value).lower() + if not symbol.isalnum() or len(symbol) > 32: + raise ValueError("unsupported activity symbol") + return symbol + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _optional_int(value: Any, name: str) -> int | None: + if value is None: + return None + return _strict_int(value, name) + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + selected = _strict_int(value, name) + if selected < minimum or selected > maximum: + raise ValueError(f"{name} is outside the supported range") + return selected + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + selected = float(value) + if not math.isfinite(selected): + raise ValueError(f"{name} must be finite") + return selected + + +def _positive_float(value: Any, name: str) -> float: + selected = _finite_float(value, name) + if selected <= 0.0: + raise ValueError(f"{name} must be positive") + return selected + + +def _nonnegative_float(value: Any, name: str) -> float: + selected = _finite_float(value, name) + if selected < 0.0: + raise ValueError(f"{name} must be non-negative") + return selected + + +def _optional_float(value: Any, name: str) -> float | None: + if value is None: + return None + return _finite_float(value, name) + + +def _required_sha256(value: Any, name: str) -> str: + text = _required_text(value) + if len(text) != 64 or any(char not in "0123456789abcdef" for char in text): + raise ValueError(f"{name} must be lowercase SHA-256") + return text + + +def _require_version(actual: str, expected: str, name: str) -> None: + if actual != expected: + raise ValueError(f"unsupported {name} schema") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if data.get("schema_version") != expected: + raise ValueError(f"unsupported schema version; expected {expected}") + + +def _require_derived(data: Mapping[str, Any], name: str, expected: Any) -> None: + if data.get(name) != expected: + raise ValueError(f"derived activity field {name} differs") + + +def _mapping(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be a mapping") + return cast(Mapping[str, Any], value) + + +def _mapping_sequence(value: Any, name: str) -> tuple[Mapping[str, Any], ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError(f"{name} must be a sequence") + return tuple(_mapping(item, name) for item in value) + + +def _sequence(value: Any, name: str) -> Sequence[Any]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError(f"{name} must be a sequence") + return value + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + selected = _sequence(value, name) + if any(not isinstance(item, str) for item in selected): + raise ValueError(f"{name} must contain strings") + return tuple(cast(Sequence[str], selected)) + + +def _mapping_optional_text(data: Mapping[str, Any], name: str) -> str | None: + return _optional_text(data.get(name)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + try: + value = json.loads(text) + except (TypeError, json.JSONDecodeError) as err: + raise ValueError("invalid activity contract JSON") from err + return _mapping(value, "activity JSON") + + +__all__ = [ + "ACTIVITY_EVENT_COLUMNS", + "ACTIVITY_REVERSE_DEGRADATION_METRICS", + "DEFAULT_ACTIVITY_BATCH_SIZE", + "DEFAULT_ACTIVITY_MAX_PAYLOAD_BYTES", + "DEFAULT_ACTIVITY_MAX_PROVENANCE_VALUES", + "DEFAULT_ACTIVITY_MAX_SLICES", + "DEFAULT_ACTIVITY_MAX_SYMBOLS", + "DEFAULT_ACTIVITY_ROUNDING_DIGITS", + "RECONSTRUCTION_ACTIVITY_BENCHMARK_EVIDENCE_SCHEMA_VERSION", + "RECONSTRUCTION_ACTIVITY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_ACTIVITY_METRIC_SCHEMA_VERSION", + "RECONSTRUCTION_ACTIVITY_POLICY_SCHEMA_VERSION", + "RECONSTRUCTION_ACTIVITY_SLICE_SCHEMA_VERSION", + "ActivityAggregationSemantics", + "ActivityMetricSemantics", + "ActivitySliceScope", + "ActivityVolumeState", + "ReconstructionActivityBenchmarkEvidenceV1", + "ReconstructionActivityManifestV1", + "ReconstructionActivityMetricV1", + "ReconstructionActivityPolicyV1", + "ReconstructionActivitySliceV1", + "activity_bar_projection_semantics", + "activity_metric_definitions", + "reconstruction_activity_benchmark_evidence", + "read_reconstruction_activity_manifest", + "summarize_committed_reconstruction_activity", + "summarize_reconstruction_activity", + "summarize_reconstruction_activity_streams", + "write_reconstruction_activity_manifest", +] diff --git a/src/histdatacom/synthetic/assets/reverse_degradation_promotion_gates_v1.json b/src/histdatacom/synthetic/assets/reverse_degradation_promotion_gates_v1.json new file mode 100644 index 00000000..63829aa2 --- /dev/null +++ b/src/histdatacom/synthetic/assets/reverse_degradation_promotion_gates_v1.json @@ -0,0 +1,240 @@ +{ + "schema_version": "histdatacom.benchmark-promotion-gate-policy.v1", + "policy_name": "eurusd-triangle-reverse-degradation", + "policy_version": "v1", + "issue_number": 463, + "frozen_before_candidate_results": true, + "requirements": [ + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-artifact-budget", + "scope": "campaign", + "severity": "hard", + "metric_name": "artifact_bytes", + "comparator": "less-or-equal", + "threshold": 67108864, + "description": "Compact persisted benchmark evidence must remain at or below 64 MiB." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-bounded-hook-count", + "scope": "campaign", + "severity": "hard", + "metric_name": "max_hook_metric_count", + "comparator": "less-or-equal", + "threshold": 64, + "description": "Every candidate window must retain no more than 64 bounded hook metrics." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-dense-identity-control", + "scope": "campaign", + "severity": "hard", + "metric_name": "dense_identity_failure_count", + "comparator": "zero", + "threshold": 0, + "description": "Untouched dense reference controls must pass every applicable integrity gate." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-ensemble-support", + "scope": "campaign", + "severity": "advisory", + "metric_name": "minimum_ensemble_member_count", + "comparator": "greater-or-equal", + "threshold": 2, + "description": "Candidate uncertainty should include at least two deterministic ensemble members." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-information-audit", + "scope": "campaign", + "severity": "hard", + "metric_name": "information_audit_violation_count", + "comparator": "zero", + "threshold": 0, + "description": "Blocked splits and every information input must pass the existing leakage audit." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-negative-control", + "scope": "campaign", + "severity": "hard", + "metric_name": "negative_control_unexpected_pass_count", + "comparator": "zero", + "threshold": 0, + "description": "Every declared negative control must fail at least one hard promotion gate." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-neighbor-leakage", + "scope": "campaign", + "severity": "hard", + "metric_name": "holdout_neighbor_leakage_count", + "comparator": "zero", + "threshold": 0, + "description": "No calibration source or near-neighbor window may overlap validation or final holdout." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-peak-memory-budget", + "scope": "campaign", + "severity": "hard", + "metric_name": "peak_memory_bytes", + "comparator": "less-or-equal", + "threshold": 2147483648, + "description": "Peak benchmark memory must remain at or below the predeclared 2 GiB budget." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-required-strata", + "scope": "campaign", + "severity": "hard", + "metric_name": "required_stratum_missing_count", + "comparator": "zero", + "threshold": 0, + "description": "Every predeclared symbol, split, session, epoch, context, and severity stratum must have support." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-runtime-budget", + "scope": "campaign", + "severity": "hard", + "metric_name": "runtime_seconds", + "comparator": "less-or-equal", + "threshold": 900.0, + "description": "The bounded real benchmark campaign must complete within 15 minutes." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-source-hashes", + "scope": "campaign", + "severity": "hard", + "metric_name": "source_hash_mismatch_count", + "comparator": "zero", + "threshold": 0, + "description": "Every selected source partition must match its recorded content hash." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "campaign-window-support", + "scope": "campaign", + "severity": "advisory", + "metric_name": "measured_window_count", + "comparator": "greater-or-equal", + "threshold": 18, + "description": "The campaign should measure at least eighteen real synchronized windows." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-anchor-integrity", + "scope": "candidate", + "severity": "hard", + "metric_name": "immutable_anchor_violation_count", + "comparator": "zero", + "threshold": 0, + "description": "A candidate cannot alter or omit a protected observed anchor." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-event-count", + "scope": "candidate", + "severity": "hard", + "metric_name": "max_event_count_relative_error", + "comparator": "less-or-equal", + "threshold": 0.5, + "description": "Worst supported holdout event-count relative error must not exceed 0.50." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-execution-failures", + "scope": "candidate", + "severity": "hard", + "metric_name": "candidate_failure_count", + "comparator": "zero", + "threshold": 0, + "description": "A promotable candidate cannot have a failed or non-converged measured window." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-interarrival-shape", + "scope": "candidate", + "severity": "hard", + "metric_name": "max_interarrival_hist_l1", + "comparator": "less-or-equal", + "threshold": 0.45, + "description": "Worst supported holdout inter-arrival histogram L1 distance must not exceed 0.45." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-path-variation", + "scope": "candidate", + "severity": "hard", + "metric_name": "max_path_realized_variation_relative_error", + "comparator": "less-or-equal", + "threshold": 0.75, + "description": "Worst supported holdout realized-variation relative error must not exceed 0.75." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-refusal-reporting", + "scope": "candidate", + "severity": "advisory", + "metric_name": "refusal_rate_reported", + "comparator": "true", + "threshold": true, + "description": "Every candidate report should expose measured refusal and unsupported-context rates." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-spread-tail", + "scope": "candidate", + "severity": "hard", + "metric_name": "max_spread_tail_relative_error", + "comparator": "less-or-equal", + "threshold": 0.75, + "description": "Worst supported holdout spread-tail relative error must not exceed 0.75." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-triangle-residual", + "scope": "candidate", + "severity": "advisory", + "metric_name": "triangle_residual_p99_pips", + "comparator": "less-or-equal", + "threshold": 5.0, + "description": "The synchronized triangle p99 residual should remain at or below five pips." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-uncertainty-support", + "scope": "candidate", + "severity": "advisory", + "metric_name": "uncertainty_interval_count", + "comparator": "greater-or-equal", + "threshold": 6, + "description": "Candidate results should report uncertainty over at least six supported holdout slices." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-unsupported-emission", + "scope": "candidate", + "severity": "hard", + "metric_name": "unsupported_context_emission_count", + "comparator": "zero", + "threshold": 0, + "description": "Unsupported contexts must refuse instead of emitting plausible-looking events." + }, + { + "schema_version": "histdatacom.benchmark-gate-requirement.v1", + "requirement_id": "candidate-update-transition", + "scope": "candidate", + "severity": "hard", + "metric_name": "max_update_transition_l1", + "comparator": "less-or-equal", + "threshold": 0.55, + "description": "Worst supported holdout update-transition matrix L1 distance must not exceed 0.55." + } + ], + "policy_id": "" +} diff --git a/src/histdatacom/synthetic/bars.py b/src/histdatacom/synthetic/bars.py new file mode 100644 index 00000000..22aa7d03 --- /dev/null +++ b/src/histdatacom/synthetic/bars.py @@ -0,0 +1,3055 @@ +"""Deterministic derived candlestick products from committed reconstruction. + +Bars are an optional export projection of the final narrow event product. The +module never reads raw HistData M1 rows or the enriched analytical frame, never +widens ``SyntheticEventV1``, and never invents volume. Publication mirrors the +reconstruction transaction boundary: projected event batches are aggregated in +bounded state below hidden scratch, verified, and promoted with one rename. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import shutil +import sys +import tempfile +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, cast +from urllib.parse import quote + +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.activity import ( + ActivitySliceScope, + ActivityVolumeState, +) +from histdatacom.synthetic.contracts import ( + SYNTHETIC_EVENT_SCHEMA_VERSION, + SyntheticEventOrigin, + SyntheticEventV1, + canonical_contract_json, +) +from histdatacom.synthetic.persistence import ( + RECONSTRUCTION_PRODUCT_SCHEMA_VERSION, + iter_reconstruction_event_batches, + verify_reconstruction_publication, +) + +DERIVED_BAR_INTERVAL_SCHEMA_VERSION = "histdatacom.derived-bar-interval.v1" +DERIVED_BAR_POLICY_SCHEMA_VERSION = "histdatacom.derived-bar-policy.v1" +DERIVED_BAR_SCHEMA_VERSION = "histdatacom.derived-bar.v1" +DERIVED_BAR_PARTITION_SCHEMA_VERSION = ( + "histdatacom.derived-bar-product-partition.v1" +) +DERIVED_BAR_PRODUCT_SCHEMA_VERSION = "histdatacom.derived-bar-product.v1" + +DERIVED_BAR_PRODUCT_DIRECTORY = "derived-bar-products" +DERIVED_BAR_MANIFEST_FILENAME = "manifest.json" +DERIVED_BAR_MANIFEST_ARTIFACT_KIND = "derived-bar-product-manifest" +DERIVED_BAR_COMPRESSION = "zstd" +DERIVED_BAR_WRITER_ID = "histdatacom.pyarrow-parquet-zstd.v1" +DERIVED_BAR_LOGICAL_HASH_ALGORITHM = "sha256-canonical-bar-json-lines-v1" +DERIVED_BAR_BYTE_HASH_ALGORITHM = "sha256-path-byte-digests-v1" +DEFAULT_DERIVED_BAR_BATCH_SIZE = 65_536 +DEFAULT_DERIVED_BAR_ROW_GROUP_SIZE = 16_384 +DEFAULT_DERIVED_BAR_WRITE_BUFFER_ROWS = 1_024 +DEFAULT_DERIVED_BAR_MAX_BARS = 100_000_000 +DEFAULT_DERIVED_BAR_MAX_PROVENANCE_VALUES = 256 +DEFAULT_DERIVED_BAR_MAX_SYMBOLS = 64 +DEFAULT_DERIVED_BAR_ROUNDING_DIGITS = 12 +MAX_DERIVED_BAR_MANIFEST_BYTES = 8 * 1024**2 +MAX_DERIVED_BAR_PARTITIONS = 100_000 +MAX_DERIVED_BAR_TEXT = 1_024 + +NANOSECONDS_PER_SECOND = 1_000_000_000 +STANDARD_DERIVED_BAR_INTERVALS: dict[str, int] = { + "1m": 60 * NANOSECONDS_PER_SECOND, + "5m": 5 * 60 * NANOSECONDS_PER_SECOND, + "15m": 15 * 60 * NANOSECONDS_PER_SECOND, + "30m": 30 * 60 * NANOSECONDS_PER_SECOND, + "1h": 60 * 60 * NANOSECONDS_PER_SECOND, + "4h": 4 * 60 * 60 * NANOSECONDS_PER_SECOND, + "1d": 24 * 60 * 60 * NANOSECONDS_PER_SECOND, +} + +DERIVED_BAR_EVENT_COLUMNS = ( + "event_id", + "origin", + "symbol", + "event_time_ns", + "event_sequence", + "bid", + "ask", + "run_id", + "ensemble_member_id", + "source_version_id", + "generator_id", + "generator_version", + "generator_config_id", + "reference_id", + "motif_id", + "feed_epoch_id", + "broker_profile_id", + "constraint_set_id", + "confidence", +) + +DERIVED_BAR_ARROW_COLUMNS = ( + "schema_version", + "event_schema_version", + "bar_id", + "source_product_manifest_id", + "policy_id", + "rounding_digits", + "run_id", + "ensemble_member_id", + "symbol", + "scope", + "interval_code", + "interval_ns", + "bar_start_ns", + "bar_end_ns", + "first_event_id", + "last_event_id", + "first_event_time_ns", + "last_event_time_ns", + "event_count", + "observed_event_count", + "synthetic_event_count", + "quote_update_count", + "transition_count", + "bid_open", + "bid_high", + "bid_low", + "bid_close", + "ask_open", + "ask_high", + "ask_low", + "ask_close", + "mid_open", + "mid_high", + "mid_low", + "mid_close", + "spread_open", + "spread_high", + "spread_low", + "spread_close", + "mean_spread", + "activity_duration_ns", + "tick_intensity_per_second", + "price_change_count", + "stale_quote_count", + "stale_quote_rate", + "mean_event_confidence", + "confidence_support_count", + "volume_state", + "volume", + "is_partial_start", + "is_partial_end", + "source_version_ids", + "generator_ids", + "generator_versions", + "generator_config_ids", + "reference_ids", + "motif_ids", + "feed_epoch_ids", + "broker_profile_ids", + "constraint_set_ids", + "event_content_sha256", + "event_schema_augmented", + "raw_m1_input", + "centralized_traded_volume_claim", +) + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_BAR_MONTH_RE = re.compile(r"^\d{4}-\d{2}$") + + +class DerivedBarPersistenceError(ValueError): + """A derived-bar product is malformed or unsafe to publish.""" + + +@dataclass(frozen=True, slots=True) +class DerivedBarIntervalV1: + """One supported, UTC epoch-aligned half-open bar interval.""" + + code: str + duration_ns: int = 0 + alignment_epoch_ns: int = 0 + schema_version: str = DERIVED_BAR_INTERVAL_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + DERIVED_BAR_INTERVAL_SCHEMA_VERSION, + "derived bar interval", + ) + code = _required_text(self.code).lower() + if code not in STANDARD_DERIVED_BAR_INTERVALS: + raise ValueError("unsupported derived bar interval") + expected = STANDARD_DERIVED_BAR_INTERVALS[code] + supplied = _strict_int(self.duration_ns, "duration_ns") + if supplied not in (0, expected): + raise ValueError("derived bar interval duration differs") + alignment = _strict_int(self.alignment_epoch_ns, "alignment_epoch_ns") + if alignment != 0: + raise ValueError("v1 derived bars require UTC Unix-epoch alignment") + object.__setattr__(self, "code", code) + object.__setattr__(self, "duration_ns", expected) + + def to_dict(self) -> dict[str, JSONValue]: + """Return the exact interval contract.""" + return { + "schema_version": self.schema_version, + "code": self.code, + "duration_ns": self.duration_ns, + "alignment_epoch_ns": self.alignment_epoch_ns, + "timezone": "UTC", + "bin_semantics": "[start,end)", + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "DerivedBarIntervalV1": + """Restore and verify one interval contract.""" + _require_schema(data, DERIVED_BAR_INTERVAL_SCHEMA_VERSION) + _require_derived(data, "timezone", "UTC") + _require_derived(data, "bin_semantics", "[start,end)") + return cls( + code=str(data.get("code", "")), + duration_ns=_strict_int(data.get("duration_ns"), "duration_ns"), + alignment_epoch_ns=_strict_int( + data.get("alignment_epoch_ns"), "alignment_epoch_ns" + ), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class DerivedBarPolicyV1: + """Versioned scope, interval, empty-bin, and resource semantics.""" + + intervals: tuple[str, ...] = tuple(STANDARD_DERIVED_BAR_INTERVALS) + scopes: tuple[ActivitySliceScope, ...] = (ActivitySliceScope.MERGED,) + max_bars: int = DEFAULT_DERIVED_BAR_MAX_BARS + max_symbols: int = DEFAULT_DERIVED_BAR_MAX_SYMBOLS + max_provenance_values: int = DEFAULT_DERIVED_BAR_MAX_PROVENANCE_VALUES + rounding_digits: int = DEFAULT_DERIVED_BAR_ROUNDING_DIGITS + policy_id: str = "" + schema_version: str = DERIVED_BAR_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + DERIVED_BAR_POLICY_SCHEMA_VERSION, + "derived bar policy", + ) + requested = {_required_text(value).lower() for value in self.intervals} + if not requested: + raise ValueError("derived bar policy requires intervals") + unknown = requested.difference(STANDARD_DERIVED_BAR_INTERVALS) + if unknown: + raise ValueError( + f"unsupported derived bar intervals: {sorted(unknown)}" + ) + intervals = tuple( + code for code in STANDARD_DERIVED_BAR_INTERVALS if code in requested + ) + object.__setattr__(self, "intervals", intervals) + scopes = tuple( + scope + for scope in ActivitySliceScope + if scope in {ActivitySliceScope(value) for value in self.scopes} + ) + if not scopes: + raise ValueError("derived bar policy requires scopes") + object.__setattr__(self, "scopes", scopes) + object.__setattr__( + self, + "max_bars", + _bounded_int(self.max_bars, "max_bars", 1, 1_000_000_000), + ) + object.__setattr__( + self, + "max_symbols", + _bounded_int(self.max_symbols, "max_symbols", 1, 10_000), + ) + object.__setattr__( + self, + "max_provenance_values", + _bounded_int( + self.max_provenance_values, + "max_provenance_values", + 1, + 10_000, + ), + ) + object.__setattr__( + self, + "rounding_digits", + _bounded_int(self.rounding_digits, "rounding_digits", 0, 15), + ) + expected = _stable_id("derived-bar-policy", self.identity_payload()) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("derived bar policy_id differs") + object.__setattr__(self, "policy_id", expected) + + @property + def interval_contracts(self) -> tuple[DerivedBarIntervalV1, ...]: + """Return canonical interval contracts in increasing duration order.""" + return tuple(DerivedBarIntervalV1(code) for code in self.intervals) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return policy fields participating in stable identity.""" + return { + "schema_version": self.schema_version, + "intervals": [item.to_dict() for item in self.interval_contracts], + "scopes": [item.value for item in self.scopes], + "max_bars": self.max_bars, + "max_symbols": self.max_symbols, + "max_provenance_values": self.max_provenance_values, + "rounding_digits": self.rounding_digits, + "empty_bin_policy": "omit_without_fill", + "market_closure_policy": "omit_empty_without_liquidity", + "partial_bin_policy": "emit_and_flag_query_boundary_overlap", + "duplicate_timestamp_policy": ( + "order_by_event_time_sequence_and_event_id" + ), + "transition_boundary_policy": "carry_previous_quote_into_next_bar", + "volume_state": ActivityVolumeState.UNAVAILABLE.value, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return the strict policy contract.""" + return { + **self.identity_payload(), + "policy_id": self.policy_id, + "raw_m1_input": False, + "event_schema_augmented": False, + "centralized_traded_volume_claim": False, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "DerivedBarPolicyV1": + """Restore and verify a derived-bar policy.""" + _require_schema(data, DERIVED_BAR_POLICY_SCHEMA_VERSION) + for name, expected in ( + ("empty_bin_policy", "omit_without_fill"), + ("market_closure_policy", "omit_empty_without_liquidity"), + ( + "partial_bin_policy", + "emit_and_flag_query_boundary_overlap", + ), + ( + "duplicate_timestamp_policy", + "order_by_event_time_sequence_and_event_id", + ), + ( + "transition_boundary_policy", + "carry_previous_quote_into_next_bar", + ), + ("volume_state", ActivityVolumeState.UNAVAILABLE.value), + ("raw_m1_input", False), + ("event_schema_augmented", False), + ("centralized_traded_volume_claim", False), + ): + _require_derived(data, name, expected) + interval_rows = _mapping_sequence(data.get("intervals"), "intervals") + intervals = tuple( + DerivedBarIntervalV1.from_dict(item).code for item in interval_rows + ) + return cls( + intervals=intervals, + scopes=tuple( + ActivitySliceScope(str(item)) + for item in _sequence(data.get("scopes"), "scopes") + ), + max_bars=_strict_int(data.get("max_bars"), "max_bars"), + max_symbols=_strict_int(data.get("max_symbols"), "max_symbols"), + max_provenance_values=_strict_int( + data.get("max_provenance_values"), "max_provenance_values" + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class DerivedBarV1: + """One deterministic OHLC/activity projection of ordered event rows.""" + + source_product_manifest_id: str + policy_id: str + rounding_digits: int + run_id: str + ensemble_member_id: str + symbol: str + scope: ActivitySliceScope + interval_code: str + interval_ns: int + bar_start_ns: int + bar_end_ns: int + first_event_id: str + last_event_id: str + first_event_time_ns: int + last_event_time_ns: int + event_count: int + observed_event_count: int + synthetic_event_count: int + quote_update_count: int + transition_count: int + bid_open: float + bid_high: float + bid_low: float + bid_close: float + ask_open: float + ask_high: float + ask_low: float + ask_close: float + mid_open: float + mid_high: float + mid_low: float + mid_close: float + spread_open: float + spread_high: float + spread_low: float + spread_close: float + mean_spread: float + activity_duration_ns: int + tick_intensity_per_second: float | None + price_change_count: int + stale_quote_count: int + stale_quote_rate: float | None + mean_event_confidence: float | None + confidence_support_count: int + is_partial_start: bool + is_partial_end: bool + source_version_ids: tuple[str, ...] + generator_ids: tuple[str, ...] = () + generator_versions: tuple[str, ...] = () + generator_config_ids: tuple[str, ...] = () + reference_ids: tuple[str, ...] = () + motif_ids: tuple[str, ...] = () + feed_epoch_ids: tuple[str, ...] = () + broker_profile_ids: tuple[str, ...] = () + constraint_set_ids: tuple[str, ...] = () + event_content_sha256: str = "" + bar_id: str = "" + schema_version: str = DERIVED_BAR_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, DERIVED_BAR_SCHEMA_VERSION, "derived bar" + ) + for name in ( + "source_product_manifest_id", + "policy_id", + "run_id", + "ensemble_member_id", + "first_event_id", + "last_event_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "rounding_digits", + _bounded_int(self.rounding_digits, "rounding_digits", 0, 15), + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__(self, "scope", ActivitySliceScope(self.scope)) + interval = DerivedBarIntervalV1( + self.interval_code, duration_ns=self.interval_ns + ) + object.__setattr__(self, "interval_code", interval.code) + object.__setattr__(self, "interval_ns", interval.duration_ns) + start = _strict_int(self.bar_start_ns, "bar_start_ns") + end = _strict_int(self.bar_end_ns, "bar_end_ns") + if start % interval.duration_ns or end != start + interval.duration_ns: + raise ValueError( + "derived bar bounds differ from interval alignment" + ) + object.__setattr__(self, "bar_start_ns", start) + object.__setattr__(self, "bar_end_ns", end) + first = _strict_int(self.first_event_time_ns, "first_event_time_ns") + last = _strict_int(self.last_event_time_ns, "last_event_time_ns") + if not start <= first <= last < end: + raise ValueError("derived bar event bounds fall outside the bin") + object.__setattr__(self, "first_event_time_ns", first) + object.__setattr__(self, "last_event_time_ns", last) + for name in ( + "event_count", + "observed_event_count", + "synthetic_event_count", + "quote_update_count", + "transition_count", + "activity_duration_ns", + "price_change_count", + "stale_quote_count", + "confidence_support_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + if self.event_count < 1: + raise ValueError("derived bars cannot be empty") + if ( + self.event_count + != self.observed_event_count + self.synthetic_event_count + ): + raise ValueError("derived bar origin counts do not reconcile") + if self.quote_update_count != self.event_count: + raise ValueError("derived bar quote-update count differs") + if ( + self.transition_count + != self.price_change_count + self.stale_quote_count + ): + raise ValueError("derived bar transition counts do not reconcile") + if self.transition_count > self.event_count: + raise ValueError("derived bar transition support is impossible") + if self.transition_count not in { + self.event_count - 1, + self.event_count, + }: + raise ValueError("derived bar transition support is incomplete") + if self.activity_duration_ns != last - first: + raise ValueError("derived bar activity duration differs") + if (self.event_count == 1) != ( + self.first_event_id == self.last_event_id + ): + raise ValueError("derived bar endpoint identities do not reconcile") + if self.confidence_support_count > self.event_count: + raise ValueError("derived bar confidence support exceeds rows") + if ( + self.scope is ActivitySliceScope.OBSERVED + and self.synthetic_event_count + ): + raise ValueError("observed-only bar contains synthetic support") + if ( + self.scope is ActivitySliceScope.SYNTHETIC + and self.observed_event_count + ): + raise ValueError("synthetic-only bar contains observed support") + for prefix in ("bid", "ask", "mid", "spread"): + values = tuple( + _finite_float( + getattr(self, f"{prefix}_{suffix}"), f"{prefix}_{suffix}" + ) + for suffix in ("open", "high", "low", "close") + ) + if values[1] < max(values[0], values[3]) or values[2] > min( + values[0], values[3] + ): + raise ValueError(f"derived bar {prefix} OHLC is inconsistent") + if values[1] < values[2]: + raise ValueError(f"derived bar {prefix} range is reversed") + for suffix, value in zip(("open", "high", "low", "close"), values): + object.__setattr__(self, f"{prefix}_{suffix}", value) + if min(self.bid_low, self.ask_low, self.mid_low) <= 0: + raise ValueError("derived bar prices must be positive") + for suffix in ("open", "high", "low", "close"): + bid = getattr(self, f"bid_{suffix}") + ask = getattr(self, f"ask_{suffix}") + if ask < bid: + raise ValueError( + f"derived bar ask_{suffix} is below bid_{suffix}" + ) + if not ( + self.bid_low <= self.mid_low <= self.ask_low + and self.bid_high <= self.mid_high <= self.ask_high + ): + raise ValueError("derived bar midpoint extrema are inconsistent") + for suffix in ("open", "close"): + bid = getattr(self, f"bid_{suffix}") + ask = getattr(self, f"ask_{suffix}") + midpoint = getattr(self, f"mid_{suffix}") + spread = getattr(self, f"spread_{suffix}") + if not _matches_rounded_value( + midpoint, (bid + ask) / 2, self.rounding_digits + ): + raise ValueError( + f"derived bar mid_{suffix} differs from bid/ask" + ) + if not _matches_rounded_value( + spread, ask - bid, self.rounding_digits + ): + raise ValueError( + f"derived bar spread_{suffix} differs from bid/ask" + ) + if ( + min( + self.spread_open, + self.spread_high, + self.spread_low, + self.spread_close, + ) + < 0 + ): + raise ValueError("derived bar spread cannot be negative") + mean_spread = _nonnegative_float(self.mean_spread, "mean_spread") + if not self.spread_low <= mean_spread <= self.spread_high: + raise ValueError("derived bar mean spread is outside its range") + object.__setattr__(self, "mean_spread", mean_spread) + intensity = _optional_nonnegative_float( + self.tick_intensity_per_second, "tick_intensity_per_second" + ) + if self.activity_duration_ns == 0 and intensity is not None: + raise ValueError("zero-duration bar cannot claim tick intensity") + if self.activity_duration_ns > 0 and intensity is None: + raise ValueError("positive-duration bar requires tick intensity") + if intensity is not None: + expected_intensity = ( + self.event_count + * NANOSECONDS_PER_SECOND + / self.activity_duration_ns + ) + if not _matches_rounded_value( + intensity, expected_intensity, self.rounding_digits + ): + raise ValueError("derived bar tick intensity differs") + object.__setattr__(self, "tick_intensity_per_second", intensity) + stale_rate = _optional_ratio(self.stale_quote_rate, "stale_quote_rate") + if self.transition_count == 0 and stale_rate is not None: + raise ValueError("transition-free bar cannot claim stale rate") + if self.transition_count > 0 and stale_rate is None: + raise ValueError("derived bar transitions require stale rate") + if stale_rate is not None and not _matches_rounded_value( + stale_rate, + self.stale_quote_count / self.transition_count, + self.rounding_digits, + ): + raise ValueError("derived bar stale rate differs") + object.__setattr__(self, "stale_quote_rate", stale_rate) + mean_confidence = _optional_ratio( + self.mean_event_confidence, "mean_event_confidence" + ) + if (self.confidence_support_count == 0) != (mean_confidence is None): + raise ValueError("derived bar confidence support differs") + object.__setattr__(self, "mean_event_confidence", mean_confidence) + object.__setattr__( + self, "is_partial_start", _strict_bool(self.is_partial_start) + ) + object.__setattr__( + self, "is_partial_end", _strict_bool(self.is_partial_end) + ) + for name in ( + "source_version_ids", + "generator_ids", + "generator_versions", + "generator_config_ids", + "reference_ids", + "motif_ids", + "feed_epoch_ids", + "broker_profile_ids", + "constraint_set_ids", + ): + object.__setattr__( + self, name, _normalized_text_tuple(getattr(self, name)) + ) + if not self.source_version_ids: + raise ValueError("derived bar requires source-version lineage") + object.__setattr__( + self, + "event_content_sha256", + _required_sha256(self.event_content_sha256, "event_content_sha256"), + ) + expected = _stable_id("derived-bar", self.identity_payload()) + supplied = _optional_text(self.bar_id) + if supplied is not None and supplied != expected: + raise ValueError("derived bar_id differs") + object.__setattr__(self, "bar_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return all logical bar fields except the derived bar identifier.""" + return { + key: value + for key, value in self.to_dict(include_bar_id=False).items() + if key != "bar_id" + } + + def to_dict(self, *, include_bar_id: bool = True) -> dict[str, JSONValue]: + """Return the exact narrow Arrow/JSON row.""" + payload: dict[str, JSONValue] = { + "schema_version": self.schema_version, + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "source_product_manifest_id": self.source_product_manifest_id, + "policy_id": self.policy_id, + "rounding_digits": self.rounding_digits, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "symbol": self.symbol, + "scope": self.scope.value, + "interval_code": self.interval_code, + "interval_ns": self.interval_ns, + "bar_start_ns": self.bar_start_ns, + "bar_end_ns": self.bar_end_ns, + "first_event_id": self.first_event_id, + "last_event_id": self.last_event_id, + "first_event_time_ns": self.first_event_time_ns, + "last_event_time_ns": self.last_event_time_ns, + "event_count": self.event_count, + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "quote_update_count": self.quote_update_count, + "transition_count": self.transition_count, + "bid_open": self.bid_open, + "bid_high": self.bid_high, + "bid_low": self.bid_low, + "bid_close": self.bid_close, + "ask_open": self.ask_open, + "ask_high": self.ask_high, + "ask_low": self.ask_low, + "ask_close": self.ask_close, + "mid_open": self.mid_open, + "mid_high": self.mid_high, + "mid_low": self.mid_low, + "mid_close": self.mid_close, + "spread_open": self.spread_open, + "spread_high": self.spread_high, + "spread_low": self.spread_low, + "spread_close": self.spread_close, + "mean_spread": self.mean_spread, + "activity_duration_ns": self.activity_duration_ns, + "tick_intensity_per_second": self.tick_intensity_per_second, + "price_change_count": self.price_change_count, + "stale_quote_count": self.stale_quote_count, + "stale_quote_rate": self.stale_quote_rate, + "mean_event_confidence": self.mean_event_confidence, + "confidence_support_count": self.confidence_support_count, + "volume_state": ActivityVolumeState.UNAVAILABLE.value, + "volume": None, + "is_partial_start": self.is_partial_start, + "is_partial_end": self.is_partial_end, + "source_version_ids": list(self.source_version_ids), + "generator_ids": list(self.generator_ids), + "generator_versions": list(self.generator_versions), + "generator_config_ids": list(self.generator_config_ids), + "reference_ids": list(self.reference_ids), + "motif_ids": list(self.motif_ids), + "feed_epoch_ids": list(self.feed_epoch_ids), + "broker_profile_ids": list(self.broker_profile_ids), + "constraint_set_ids": list(self.constraint_set_ids), + "event_content_sha256": self.event_content_sha256, + "event_schema_augmented": False, + "raw_m1_input": False, + "centralized_traded_volume_claim": False, + } + if include_bar_id: + payload["bar_id"] = self.bar_id + return {name: payload[name] for name in DERIVED_BAR_ARROW_COLUMNS} + return payload + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "DerivedBarV1": + """Restore and verify one derived bar row.""" + _require_schema(data, DERIVED_BAR_SCHEMA_VERSION) + for name, expected in ( + ("event_schema_version", SYNTHETIC_EVENT_SCHEMA_VERSION), + ("volume_state", ActivityVolumeState.UNAVAILABLE.value), + ("volume", None), + ("event_schema_augmented", False), + ("raw_m1_input", False), + ("centralized_traded_volume_claim", False), + ): + _require_derived(data, name, expected) + kwargs: dict[str, Any] = { + name: data.get(name) + for name in DERIVED_BAR_ARROW_COLUMNS + if name + not in { + "event_schema_version", + "volume_state", + "volume", + "event_schema_augmented", + "raw_m1_input", + "centralized_traded_volume_claim", + } + } + for name in ( + "source_version_ids", + "generator_ids", + "generator_versions", + "generator_config_ids", + "reference_ids", + "motif_ids", + "feed_epoch_ids", + "broker_profile_ids", + "constraint_set_ids", + ): + kwargs[name] = _string_tuple(data.get(name), name) + kwargs["scope"] = ActivitySliceScope(str(data.get("scope", ""))) + return cls(**kwargs) + + def to_json(self) -> str: + """Serialize one bar deterministically.""" + serialized: str = canonical_contract_json(self.to_dict()) + return serialized + + @classmethod + def from_json(cls, text: str) -> "DerivedBarV1": + """Restore one bar from JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class DerivedBarPartitionV1: + """Physical and logical evidence for one monthly bar partition.""" + + relative_path: str + symbol: str + scope: ActivitySliceScope + interval_code: str + bar_month: str + row_count: int + min_bar_start_ns: int + max_bar_start_ns: int + logical_content_sha256: str + byte_sha256: str + size_bytes: int + row_group_count: int + partition_id: str = "" + schema_version: str = DERIVED_BAR_PARTITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + DERIVED_BAR_PARTITION_SCHEMA_VERSION, + "derived bar partition", + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__(self, "scope", ActivitySliceScope(self.scope)) + interval = DerivedBarIntervalV1(self.interval_code) + object.__setattr__(self, "interval_code", interval.code) + month = _required_bar_month(self.bar_month) + object.__setattr__(self, "bar_month", month) + relative = _safe_relative_path(self.relative_path) + expected_path = _bar_partition_relative_path( + self.symbol, self.scope, interval.code, month + ) + if relative != expected_path: + raise ValueError("derived bar partition path differs from layout") + object.__setattr__(self, "relative_path", relative) + for name in ("row_count", "size_bytes", "row_group_count"): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + if ( + self.row_count < 1 + or self.size_bytes < 1 + or self.row_group_count < 1 + or self.row_group_count > self.row_count + ): + raise ValueError("derived bar partitions cannot be empty") + minimum = _strict_int(self.min_bar_start_ns, "min_bar_start_ns") + maximum = _strict_int(self.max_bar_start_ns, "max_bar_start_ns") + if maximum < minimum: + raise ValueError("derived bar partition bounds are reversed") + if _bar_month(minimum) != month or _bar_month(maximum) != month: + raise ValueError("derived bar partition crosses its month") + object.__setattr__(self, "min_bar_start_ns", minimum) + object.__setattr__(self, "max_bar_start_ns", maximum) + for name in ("logical_content_sha256", "byte_sha256"): + object.__setattr__( + self, name, _required_sha256(getattr(self, name), name) + ) + expected = _stable_id("derived-bar-partition", self.payload()) + supplied = _optional_text(self.partition_id) + if supplied is not None and supplied != expected: + raise ValueError("derived bar partition_id differs") + object.__setattr__(self, "partition_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return deterministic physical and logical partition evidence.""" + return { + "schema_version": self.schema_version, + "bar_schema_version": DERIVED_BAR_SCHEMA_VERSION, + "relative_path": self.relative_path, + "symbol": self.symbol, + "scope": self.scope.value, + "interval_code": self.interval_code, + "bar_month": self.bar_month, + "row_count": self.row_count, + "min_bar_start_ns": self.min_bar_start_ns, + "max_bar_start_ns": self.max_bar_start_ns, + "logical_content_sha256": self.logical_content_sha256, + "byte_sha256": self.byte_sha256, + "size_bytes": self.size_bytes, + "row_group_count": self.row_group_count, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact partition evidence.""" + return {**self.payload(), "partition_id": self.partition_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "DerivedBarPartitionV1": + """Restore and verify partition evidence.""" + _require_schema(data, DERIVED_BAR_PARTITION_SCHEMA_VERSION) + _require_derived(data, "bar_schema_version", DERIVED_BAR_SCHEMA_VERSION) + return cls( + relative_path=str(data.get("relative_path", "")), + symbol=str(data.get("symbol", "")), + scope=ActivitySliceScope(str(data.get("scope", ""))), + interval_code=str(data.get("interval_code", "")), + bar_month=str(data.get("bar_month", "")), + row_count=_strict_int(data.get("row_count"), "row_count"), + min_bar_start_ns=_strict_int( + data.get("min_bar_start_ns"), "min_bar_start_ns" + ), + max_bar_start_ns=_strict_int( + data.get("max_bar_start_ns"), "max_bar_start_ns" + ), + logical_content_sha256=str(data.get("logical_content_sha256", "")), + byte_sha256=str(data.get("byte_sha256", "")), + size_bytes=_strict_int(data.get("size_bytes"), "size_bytes"), + row_group_count=_strict_int( + data.get("row_group_count"), "row_group_count" + ), + partition_id=str(data.get("partition_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class DerivedBarProductManifestV1: + """Compact identity and replay evidence for one atomic bar product.""" + + source_product_manifest_id: str + source_product_publication_id: str + source_product_logical_sha256: str + run_id: str + ensemble_member_id: str + query_start_ns: int | None + query_end_ns: int | None + policy: DerivedBarPolicyV1 + symbols: tuple[str, ...] + symbol_bar_counts: Mapping[str, int] + partitions: tuple[DerivedBarPartitionV1, ...] + logical_content_sha256: str + writer_id: str + writer_library_version: str + python_runtime: str + compression: str + row_group_size: int + publication_id: str = "" + manifest_id: str = "" + schema_version: str = DERIVED_BAR_PRODUCT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + DERIVED_BAR_PRODUCT_SCHEMA_VERSION, + "derived bar product", + ) + for name in ( + "source_product_manifest_id", + "source_product_publication_id", + "run_id", + "ensemble_member_id", + "writer_id", + "writer_library_version", + "python_runtime", + "compression", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if not isinstance(self.policy, DerivedBarPolicyV1): + raise TypeError("derived bar manifest requires v1 policy") + query_start, query_end = _query_bounds( + self.query_start_ns, self.query_end_ns + ) + object.__setattr__(self, "query_start_ns", query_start) + object.__setattr__(self, "query_end_ns", query_end) + object.__setattr__( + self, + "source_product_logical_sha256", + _required_sha256( + self.source_product_logical_sha256, + "source_product_logical_sha256", + ), + ) + symbols = tuple( + sorted({_normalized_symbol(item) for item in self.symbols}) + ) + if not symbols: + raise ValueError("derived bar product requires symbols") + object.__setattr__(self, "symbols", symbols) + counts = { + _normalized_symbol(symbol): _nonnegative_int( + count, f"symbol_bar_counts.{symbol}" + ) + for symbol, count in self.symbol_bar_counts.items() + } + if set(counts) != set(symbols) or any( + not value for value in counts.values() + ): + raise ValueError("derived bar symbol counts do not reconcile") + object.__setattr__( + self, "symbol_bar_counts", dict(sorted(counts.items())) + ) + partitions = tuple( + sorted( + self.partitions, + key=lambda item: ( + item.symbol, + item.scope.value, + STANDARD_DERIVED_BAR_INTERVALS[item.interval_code], + item.bar_month, + ), + ) + ) + if not partitions or len(partitions) > MAX_DERIVED_BAR_PARTITIONS: + raise ValueError( + "derived bar partition count is empty or unbounded" + ) + if len({item.relative_path for item in partitions}) != len(partitions): + raise ValueError( + "derived bar product contains duplicate partitions" + ) + partition_counts = dict.fromkeys(symbols, 0) + for partition in partitions: + partition_counts[partition.symbol] += partition.row_count + if partition_counts != counts: + raise ValueError("derived bar partition rows do not reconcile") + if {item.symbol for item in partitions} != set(symbols): + raise ValueError("derived bar partitions do not cover symbols") + if any(item.scope not in self.policy.scopes for item in partitions): + raise ValueError("derived bar partition scope differs from policy") + if any( + item.interval_code not in self.policy.intervals + for item in partitions + ): + raise ValueError( + "derived bar partition interval differs from policy" + ) + object.__setattr__(self, "partitions", partitions) + logical = _required_sha256( + self.logical_content_sha256, "logical_content_sha256" + ) + if logical != _bar_product_logical_sha256(partitions): + raise ValueError("derived bar logical content hash differs") + object.__setattr__(self, "logical_content_sha256", logical) + if self.writer_id != DERIVED_BAR_WRITER_ID: + raise ValueError("unsupported derived bar writer") + if self.compression != DERIVED_BAR_COMPRESSION: + raise ValueError("unsupported derived bar compression") + object.__setattr__( + self, + "row_group_size", + _bounded_int(self.row_group_size, "row_group_size", 1, 1_000_000), + ) + expected_publication = _stable_id( + "derived-bar-publication", self.publication_payload() + ) + supplied_publication = _optional_text(self.publication_id) + if ( + supplied_publication is not None + and supplied_publication != expected_publication + ): + raise ValueError("derived bar publication_id differs") + object.__setattr__(self, "publication_id", expected_publication) + expected_manifest = _stable_id("derived-bar-manifest", self.payload()) + supplied_manifest = _optional_text(self.manifest_id) + if ( + supplied_manifest is not None + and supplied_manifest != expected_manifest + ): + raise ValueError("derived bar manifest_id differs") + object.__setattr__(self, "manifest_id", expected_manifest) + encoded = self.to_json().encode("utf-8") + if len(encoded) > MAX_DERIVED_BAR_MANIFEST_BYTES: + raise ValueError("derived bar manifest exceeds size limit") + + @property + def bar_count(self) -> int: + """Return total durable bar rows.""" + return sum(self.symbol_bar_counts.values()) + + def publication_payload(self) -> dict[str, JSONValue]: + """Return logical fields defining the commit directory identity.""" + return { + "schema_version": self.schema_version, + "source_product_manifest_id": self.source_product_manifest_id, + "policy_id": self.policy.policy_id, + "query_start_ns": self.query_start_ns, + "query_end_ns": self.query_end_ns, + "logical_content_sha256": self.logical_content_sha256, + } + + def payload(self) -> dict[str, JSONValue]: + """Return full compact manifest evidence except manifest identity.""" + return { + "schema_version": self.schema_version, + "source_product_schema_version": ( + RECONSTRUCTION_PRODUCT_SCHEMA_VERSION + ), + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "bar_schema_version": DERIVED_BAR_SCHEMA_VERSION, + "source_product_manifest_id": self.source_product_manifest_id, + "source_product_publication_id": self.source_product_publication_id, + "source_product_logical_sha256": self.source_product_logical_sha256, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "query_start_ns": self.query_start_ns, + "query_end_ns": self.query_end_ns, + "policy": self.policy.to_dict(), + "symbols": list(self.symbols), + "symbol_bar_counts": dict(self.symbol_bar_counts), + "partitions": [item.to_dict() for item in self.partitions], + "logical_content_sha256": self.logical_content_sha256, + "logical_hash_algorithm": DERIVED_BAR_LOGICAL_HASH_ALGORITHM, + "byte_hash_algorithm": DERIVED_BAR_BYTE_HASH_ALGORITHM, + "writer_id": self.writer_id, + "writer_library": "pyarrow", + "writer_library_version": self.writer_library_version, + "python_runtime": self.python_runtime, + "compression": self.compression, + "row_group_size": self.row_group_size, + "publication_id": self.publication_id, + "bar_count": self.bar_count, + "event_rows_inline": False, + "analytical_frame_columns_inline": False, + "raw_m1_input": False, + "event_schema_augmented": False, + "centralized_traded_volume_claim": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return the complete manifest.""" + return {**self.payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + """Serialize the manifest deterministically.""" + serialized: str = canonical_contract_json(self.to_dict()) + return serialized + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "DerivedBarProductManifestV1": + """Restore and verify a product manifest.""" + _require_schema(data, DERIVED_BAR_PRODUCT_SCHEMA_VERSION) + for name, expected in ( + ( + "source_product_schema_version", + RECONSTRUCTION_PRODUCT_SCHEMA_VERSION, + ), + ("event_schema_version", SYNTHETIC_EVENT_SCHEMA_VERSION), + ("bar_schema_version", DERIVED_BAR_SCHEMA_VERSION), + ("logical_hash_algorithm", DERIVED_BAR_LOGICAL_HASH_ALGORITHM), + ("byte_hash_algorithm", DERIVED_BAR_BYTE_HASH_ALGORITHM), + ("writer_library", "pyarrow"), + ("event_rows_inline", False), + ("analytical_frame_columns_inline", False), + ("raw_m1_input", False), + ("event_schema_augmented", False), + ("centralized_traded_volume_claim", False), + ): + _require_derived(data, name, expected) + result = cls( + source_product_manifest_id=str( + data.get("source_product_manifest_id", "") + ), + source_product_publication_id=str( + data.get("source_product_publication_id", "") + ), + source_product_logical_sha256=str( + data.get("source_product_logical_sha256", "") + ), + run_id=str(data.get("run_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + query_start_ns=_optional_int( + data.get("query_start_ns"), "query_start_ns" + ), + query_end_ns=_optional_int( + data.get("query_end_ns"), "query_end_ns" + ), + policy=DerivedBarPolicyV1.from_dict( + _mapping(data.get("policy"), "policy") + ), + symbols=_string_tuple(data.get("symbols"), "symbols"), + symbol_bar_counts={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("symbol_bar_counts"), "symbol_bar_counts" + ).items() + }, + partitions=tuple( + DerivedBarPartitionV1.from_dict(item) + for item in _mapping_sequence( + data.get("partitions"), "partitions" + ) + ), + logical_content_sha256=str(data.get("logical_content_sha256", "")), + writer_id=str(data.get("writer_id", "")), + writer_library_version=str(data.get("writer_library_version", "")), + python_runtime=str(data.get("python_runtime", "")), + compression=str(data.get("compression", "")), + row_group_size=_strict_int( + data.get("row_group_size"), "row_group_size" + ), + publication_id=str(data.get("publication_id", "")), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived(data, "bar_count", result.bar_count) + return result + + @classmethod + def from_json(cls, text: str) -> "DerivedBarProductManifestV1": + """Restore a product manifest from JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class StagedDerivedBarPublicationV1: + """Validated but undiscoverable derived-bar publication.""" + + root: Path + staging_directory: Path + committed_directory: Path + manifest: DerivedBarProductManifestV1 + + @property + def manifest_path(self) -> Path: + """Return the staged manifest path.""" + return self.staging_directory / DERIVED_BAR_MANIFEST_FILENAME + + @property + def manifest_ref(self) -> ArtifactRef: + """Return a staged manifest reference.""" + return _artifact_ref_for_manifest(self.manifest_path, self.manifest) + + +@dataclass(frozen=True, slots=True) +class PublishedDerivedBarsV1: + """One verified committed derived-bar product.""" + + manifest: DerivedBarProductManifestV1 + manifest_path: Path + manifest_ref: ArtifactRef + idempotent_retry: bool = False + + +@dataclass(frozen=True, slots=True) +class _BarEventView: + """Minimal projected source-event fields required by bar aggregation.""" + + event_id: str + origin: SyntheticEventOrigin + symbol: str + event_time_ns: int + event_sequence: int + bid: float + ask: float + run_id: str + ensemble_member_id: str + source_version_id: str + generator_id: str | None + generator_version: str | None + generator_config_id: str | None + reference_id: str | None + motif_id: str | None + feed_epoch_id: str | None + broker_profile_id: str | None + constraint_set_id: str | None + confidence: float | None + + +@dataclass(slots=True) +class _BarState: + """Bounded price, activity, and lineage state for one non-empty bin.""" + + source_product_manifest_id: str + run_id: str + ensemble_member_id: str + symbol: str + scope: ActivitySliceScope + interval: DerivedBarIntervalV1 + bar_start_ns: int + policy: DerivedBarPolicyV1 + query_start_ns: int | None + query_end_ns: int | None + previous_bid: float | None + previous_ask: float | None + first_event: _BarEventView | None = None + last_event: _BarEventView | None = None + event_count: int = 0 + observed_event_count: int = 0 + synthetic_event_count: int = 0 + transition_count: int = 0 + price_change_count: int = 0 + stale_quote_count: int = 0 + confidence_total: float = 0.0 + confidence_count: int = 0 + spread_total: float = 0.0 + bid_open: float = 0.0 + bid_high: float = -math.inf + bid_low: float = math.inf + bid_close: float = 0.0 + ask_open: float = 0.0 + ask_high: float = -math.inf + ask_low: float = math.inf + ask_close: float = 0.0 + mid_open: float = 0.0 + mid_high: float = -math.inf + mid_low: float = math.inf + mid_close: float = 0.0 + spread_open: float = 0.0 + spread_high: float = -math.inf + spread_low: float = math.inf + spread_close: float = 0.0 + source_version_ids: set[str] = field(default_factory=set) + generator_ids: set[str] = field(default_factory=set) + generator_versions: set[str] = field(default_factory=set) + generator_config_ids: set[str] = field(default_factory=set) + reference_ids: set[str] = field(default_factory=set) + motif_ids: set[str] = field(default_factory=set) + feed_epoch_ids: set[str] = field(default_factory=set) + broker_profile_ids: set[str] = field(default_factory=set) + constraint_set_ids: set[str] = field(default_factory=set) + digest: Any = field( + default_factory=lambda: hashlib.sha256(b"derived-bar-events-v1\n") + ) + + def add(self, event: _BarEventView) -> None: + """Consume one event in canonical order.""" + if self.first_event is None: + self.first_event = event + self.bid_open = event.bid + self.ask_open = event.ask + self.mid_open = (event.bid + event.ask) / 2.0 + self.spread_open = event.ask - event.bid + previous = ( + (self.previous_bid, self.previous_ask) + if self.previous_bid is not None + and self.previous_ask is not None + else None + ) + else: + assert self.last_event is not None + previous = (self.last_event.bid, self.last_event.ask) + if previous is not None: + self.transition_count += 1 + if previous == (event.bid, event.ask): + self.stale_quote_count += 1 + else: + self.price_change_count += 1 + self.last_event = event + self.event_count += 1 + if event.origin is SyntheticEventOrigin.OBSERVED: + self.observed_event_count += 1 + else: + self.synthetic_event_count += 1 + mid = (event.bid + event.ask) / 2.0 + spread = event.ask - event.bid + self.bid_high = max(self.bid_high, event.bid) + self.bid_low = min(self.bid_low, event.bid) + self.bid_close = event.bid + self.ask_high = max(self.ask_high, event.ask) + self.ask_low = min(self.ask_low, event.ask) + self.ask_close = event.ask + self.mid_high = max(self.mid_high, mid) + self.mid_low = min(self.mid_low, mid) + self.mid_close = mid + self.spread_high = max(self.spread_high, spread) + self.spread_low = min(self.spread_low, spread) + self.spread_close = spread + self.spread_total += spread + if event.confidence is not None: + self.confidence_total += event.confidence + self.confidence_count += 1 + self._add_lineage("source_version_ids", event.source_version_id) + for name in ( + "generator_id", + "generator_version", + "generator_config_id", + "reference_id", + "motif_id", + "feed_epoch_id", + "broker_profile_id", + "constraint_set_id", + ): + value = getattr(event, name) + if value is not None: + self._add_lineage(name + "s", value) + self.digest.update( + canonical_contract_json(_bar_event_payload(event)).encode("utf-8") + ) + self.digest.update(b"\n") + + def _add_lineage(self, name: str, value: str) -> None: + values = cast(set[str], getattr(self, name)) + values.add(_required_text(value)) + if len(values) > self.policy.max_provenance_values: + raise ValueError(f"derived bar {name} exceeds provenance limit") + + def finalize(self) -> DerivedBarV1: + """Freeze the non-empty state into one strict bar row.""" + if ( + self.first_event is None + or self.last_event is None + or not self.event_count + ): + raise ValueError("cannot finalize an empty derived bar") + duration = ( + self.last_event.event_time_ns - self.first_event.event_time_ns + ) + return DerivedBarV1( + source_product_manifest_id=self.source_product_manifest_id, + policy_id=self.policy.policy_id, + rounding_digits=self.policy.rounding_digits, + run_id=self.run_id, + ensemble_member_id=self.ensemble_member_id, + symbol=self.symbol, + scope=self.scope, + interval_code=self.interval.code, + interval_ns=self.interval.duration_ns, + bar_start_ns=self.bar_start_ns, + bar_end_ns=self.bar_start_ns + self.interval.duration_ns, + first_event_id=self.first_event.event_id, + last_event_id=self.last_event.event_id, + first_event_time_ns=self.first_event.event_time_ns, + last_event_time_ns=self.last_event.event_time_ns, + event_count=self.event_count, + observed_event_count=self.observed_event_count, + synthetic_event_count=self.synthetic_event_count, + quote_update_count=self.event_count, + transition_count=self.transition_count, + bid_open=_rounded(self.bid_open, self.policy.rounding_digits), + bid_high=_rounded(self.bid_high, self.policy.rounding_digits), + bid_low=_rounded(self.bid_low, self.policy.rounding_digits), + bid_close=_rounded(self.bid_close, self.policy.rounding_digits), + ask_open=_rounded(self.ask_open, self.policy.rounding_digits), + ask_high=_rounded(self.ask_high, self.policy.rounding_digits), + ask_low=_rounded(self.ask_low, self.policy.rounding_digits), + ask_close=_rounded(self.ask_close, self.policy.rounding_digits), + mid_open=_rounded(self.mid_open, self.policy.rounding_digits), + mid_high=_rounded(self.mid_high, self.policy.rounding_digits), + mid_low=_rounded(self.mid_low, self.policy.rounding_digits), + mid_close=_rounded(self.mid_close, self.policy.rounding_digits), + spread_open=_rounded(self.spread_open, self.policy.rounding_digits), + spread_high=_rounded(self.spread_high, self.policy.rounding_digits), + spread_low=_rounded(self.spread_low, self.policy.rounding_digits), + spread_close=_rounded( + self.spread_close, self.policy.rounding_digits + ), + mean_spread=_rounded( + self.spread_total / self.event_count, + self.policy.rounding_digits, + ), + activity_duration_ns=duration, + tick_intensity_per_second=( + _rounded( + self.event_count * NANOSECONDS_PER_SECOND / duration, + self.policy.rounding_digits, + ) + if duration + else None + ), + price_change_count=self.price_change_count, + stale_quote_count=self.stale_quote_count, + stale_quote_rate=( + _rounded( + self.stale_quote_count / self.transition_count, + self.policy.rounding_digits, + ) + if self.transition_count + else None + ), + mean_event_confidence=( + _rounded( + self.confidence_total / self.confidence_count, + self.policy.rounding_digits, + ) + if self.confidence_count + else None + ), + confidence_support_count=self.confidence_count, + is_partial_start=( + self.query_start_ns is not None + and self.bar_start_ns < self.query_start_ns + ), + is_partial_end=( + self.query_end_ns is not None + and self.bar_start_ns + self.interval.duration_ns + > self.query_end_ns + ), + source_version_ids=tuple(self.source_version_ids), + generator_ids=tuple(self.generator_ids), + generator_versions=tuple(self.generator_versions), + generator_config_ids=tuple(self.generator_config_ids), + reference_ids=tuple(self.reference_ids), + motif_ids=tuple(self.motif_ids), + feed_epoch_ids=tuple(self.feed_epoch_ids), + broker_profile_ids=tuple(self.broker_profile_ids), + constraint_set_ids=tuple(self.constraint_set_ids), + event_content_sha256=self.digest.hexdigest(), + ) + + +class _BarAccumulator: + """Route ordered events to bounded symbol/scope/interval bar states.""" + + def __init__( + self, + *, + source_product_manifest_id: str, + run_id: str, + ensemble_member_id: str, + policy: DerivedBarPolicyV1, + query_start_ns: int | None, + query_end_ns: int | None, + grouped_symbols: bool = False, + ) -> None: + self.source_product_manifest_id = _required_text( + source_product_manifest_id + ) + self.run_id = _required_text(run_id) + self.ensemble_member_id = _required_text(ensemble_member_id) + self.policy = policy + self.query_start_ns = query_start_ns + self.query_end_ns = query_end_ns + self.grouped_symbols = grouped_symbols + self.current_symbol: str | None = None + self.states: dict[tuple[str, ActivitySliceScope, str], _BarState] = {} + self.previous_quotes: dict[ + tuple[str, ActivitySliceScope, str], tuple[float, float] + ] = {} + self.last_positions: dict[str, tuple[int, int, str]] = {} + self.symbols: set[str] = set() + self.bar_count = 0 + + def add(self, event: _BarEventView) -> tuple[DerivedBarV1, ...]: + """Consume one event and emit bars closed by its interval position.""" + if event.run_id != self.run_id: + raise ValueError("derived bar event run_id differs") + if event.ensemble_member_id != self.ensemble_member_id: + raise ValueError("derived bar event ensemble member differs") + symbol = _normalized_symbol(event.symbol) + emitted: list[DerivedBarV1] = [] + if self.grouped_symbols and self.current_symbol != symbol: + if self.current_symbol is not None: + if symbol <= self.current_symbol: + raise ValueError( + "derived bar grouped symbols are not strictly ordered" + ) + emitted.extend(self._finish_symbol(self.current_symbol)) + self.current_symbol = symbol + position = (event.event_time_ns, event.event_sequence, event.event_id) + previous_position = self.last_positions.get(symbol) + if previous_position is not None and position <= previous_position: + raise ValueError( + "derived bar events are not strictly ordered per symbol" + ) + self.last_positions[symbol] = position + self.symbols.add(symbol) + if len(self.symbols) > self.policy.max_symbols: + raise ValueError("derived bar symbols exceed policy limit") + origin_scope = ( + ActivitySliceScope.OBSERVED + if event.origin is SyntheticEventOrigin.OBSERVED + else ActivitySliceScope.SYNTHETIC + ) + scopes = tuple( + scope + for scope in (origin_scope, ActivitySliceScope.MERGED) + if scope in self.policy.scopes + ) + for scope in scopes: + for interval in self.policy.interval_contracts: + key = (symbol, scope, interval.code) + start = _bar_start(event.event_time_ns, interval.duration_ns) + state = self.states.get(key) + if state is not None and state.bar_start_ns != start: + emitted.append(self._finalize(key, state)) + state = None + if state is None: + previous_quote = self.previous_quotes.get(key) + state = _BarState( + source_product_manifest_id=( + self.source_product_manifest_id + ), + run_id=self.run_id, + ensemble_member_id=self.ensemble_member_id, + symbol=symbol, + scope=scope, + interval=interval, + bar_start_ns=start, + policy=self.policy, + query_start_ns=self.query_start_ns, + query_end_ns=self.query_end_ns, + previous_bid=( + previous_quote[0] if previous_quote else None + ), + previous_ask=( + previous_quote[1] if previous_quote else None + ), + ) + self.states[key] = state + state.add(event) + return tuple(emitted) + + def finish(self) -> tuple[DerivedBarV1, ...]: + """Emit all final non-empty interval states.""" + bars = tuple( + self._finalize(key, state) + for key, state in sorted( + self.states.items(), key=lambda item: item[0] + ) + ) + self.states.clear() + return bars + + def _finish_symbol(self, symbol: str) -> tuple[DerivedBarV1, ...]: + keys = sorted(key for key in self.states if key[0] == symbol) + return tuple(self._finalize(key, self.states[key]) for key in keys) + + def _finalize( + self, + key: tuple[str, ActivitySliceScope, str], + state: _BarState, + ) -> DerivedBarV1: + bar = state.finalize() + assert state.last_event is not None + self.previous_quotes[key] = (state.last_event.bid, state.last_event.ask) + self.states.pop(key, None) + self.bar_count += 1 + if self.bar_count > self.policy.max_bars: + raise ValueError("derived bar output exceeds policy limit") + return bar + + +def derive_reconstruction_bars( + events: Iterable[SyntheticEventV1], + *, + source_product_manifest_id: str, + run_id: str, + ensemble_member_id: str, + policy: DerivedBarPolicyV1 | None = None, + start_ns: int | None = None, + end_ns: int | None = None, +) -> tuple[DerivedBarV1, ...]: + """Derive bounded deterministic bars from ordered narrow events.""" + selected = policy or DerivedBarPolicyV1() + start, end = _query_bounds(start_ns, end_ns) + accumulator = _BarAccumulator( + source_product_manifest_id=source_product_manifest_id, + run_id=run_id, + ensemble_member_id=ensemble_member_id, + policy=selected, + query_start_ns=start, + query_end_ns=end, + ) + bars: list[DerivedBarV1] = [] + for event in events: + if not isinstance(event, SyntheticEventV1): + raise TypeError("derived bars require SyntheticEventV1 rows") + if start is not None and event.event_time_ns < start: + continue + if end is not None and event.event_time_ns >= end: + continue + bars.extend(accumulator.add(_bar_event_view(event))) + bars.extend(accumulator.finish()) + return tuple(sorted(bars, key=_bar_order_key)) + + +def iter_committed_reconstruction_bars( + manifest_path: str | Path, + *, + policy: DerivedBarPolicyV1 | None = None, + symbols: Iterable[str] = (), + start_ns: int | None = None, + end_ns: int | None = None, + batch_size: int = DEFAULT_DERIVED_BAR_BATCH_SIZE, +) -> Iterator[DerivedBarV1]: + """Stream bars from verified projected final-event Parquet batches.""" + product = verify_reconstruction_publication(manifest_path) + selected = policy or DerivedBarPolicyV1() + start, end = _query_bounds(start_ns, end_ns) + size = _bounded_int(batch_size, "batch_size", 1, 1_000_000) + selected_symbols = tuple(_normalized_symbol(value) for value in symbols) + accumulator = _BarAccumulator( + source_product_manifest_id=product.manifest_id, + run_id=product.run_id, + ensemble_member_id=product.ensemble_member_id, + policy=selected, + query_start_ns=start, + query_end_ns=end, + grouped_symbols=True, + ) + for batch in iter_reconstruction_event_batches( + manifest_path, + columns=DERIVED_BAR_EVENT_COLUMNS, + symbols=selected_symbols, + start_ns=start, + end_ns=end, + batch_size=size, + ): + for row in batch.to_pylist(): + mapping = _mapping(row, "derived bar event row") + yield from accumulator.add(_bar_event_view_from_mapping(mapping)) + yield from accumulator.finish() + + +class _PartitionWriter: + """Bounded buffered writer and digest for one physical partition.""" + + def __init__( + self, + staging_directory: Path, + first: DerivedBarV1, + *, + row_group_size: int, + buffer_rows: int, + ) -> None: + self.symbol = first.symbol + self.scope = first.scope + self.interval_code = first.interval_code + self.bar_month = _bar_month(first.bar_start_ns) + self.relative_path = _bar_partition_relative_path( + self.symbol, self.scope, self.interval_code, self.bar_month + ) + self.path = staging_directory / self.relative_path + self.path.parent.mkdir(parents=True, exist_ok=True) + _, pq = _arrow_modules() + self.writer = pq.ParquetWriter( + self.path, + derived_bar_arrow_schema(), + compression=DERIVED_BAR_COMPRESSION, + use_dictionary=False, + write_statistics=True, + version="2.6", + data_page_version="2.0", + write_page_checksum=True, + ) + self.row_group_size = row_group_size + self.buffer_rows = buffer_rows + self.buffer: list[DerivedBarV1] = [] + self.row_count = 0 + self.minimum: int | None = None + self.maximum: int | None = None + self.last_order: tuple[int, str] | None = None + self.digest = hashlib.sha256( + (DERIVED_BAR_LOGICAL_HASH_ALGORITHM + "\n").encode("ascii") + ) + + def add(self, bar: DerivedBarV1) -> None: + """Append one compatible ordered bar.""" + if ( + bar.symbol, + bar.scope, + bar.interval_code, + _bar_month(bar.bar_start_ns), + ) != (self.symbol, self.scope, self.interval_code, self.bar_month): + raise ValueError("derived bar writer axis differs") + order = (bar.bar_start_ns, bar.bar_id) + if self.last_order is not None and order <= self.last_order: + raise ValueError("derived bar partition order differs") + self.last_order = order + self.minimum = ( + bar.bar_start_ns + if self.minimum is None + else min(self.minimum, bar.bar_start_ns) + ) + self.maximum = ( + bar.bar_start_ns + if self.maximum is None + else max(self.maximum, bar.bar_start_ns) + ) + self.row_count += 1 + self.digest.update(bar.to_json().encode("utf-8")) + self.digest.update(b"\n") + self.buffer.append(bar) + if len(self.buffer) >= self.buffer_rows: + self.flush() + + def flush(self) -> None: + """Write buffered rows as bounded row groups.""" + if not self.buffer: + return + table = _bars_to_arrow(self.buffer) + self.writer.write_table(table, row_group_size=self.row_group_size) + self.buffer.clear() + + def close(self) -> DerivedBarPartitionV1: + """Close, fsync, and freeze partition evidence.""" + self.flush() + self.writer.close() + _fsync_file(self.path) + _fsync_directory(self.path.parent) + if self.minimum is None or self.maximum is None or not self.row_count: + raise DerivedBarPersistenceError( + "derived bar writer produced no rows" + ) + _, pq = _arrow_modules() + parquet = pq.ParquetFile(self.path) + return DerivedBarPartitionV1( + relative_path=self.relative_path, + symbol=self.symbol, + scope=self.scope, + interval_code=self.interval_code, + bar_month=self.bar_month, + row_count=self.row_count, + min_bar_start_ns=self.minimum, + max_bar_start_ns=self.maximum, + logical_content_sha256=self.digest.hexdigest(), + byte_sha256=_file_sha256(self.path), + size_bytes=self.path.stat().st_size, + row_group_count=parquet.num_row_groups, + ) + + def abort(self) -> None: + """Release the Parquet handle without producing evidence.""" + self.writer.close() + + +class _PartitionManager: + """Keep at most one active month writer per symbol/scope/interval.""" + + def __init__( + self, + staging_directory: Path, + *, + row_group_size: int, + buffer_rows: int, + ) -> None: + self.staging_directory = staging_directory + self.row_group_size = row_group_size + self.buffer_rows = buffer_rows + self.active: dict[ + tuple[str, ActivitySliceScope, str], _PartitionWriter + ] = {} + self.partitions: list[DerivedBarPartitionV1] = [] + self.current_symbol: str | None = None + + def add(self, bar: DerivedBarV1) -> None: + """Route one bar to its monthly partition.""" + if self.current_symbol != bar.symbol: + if self.current_symbol is not None: + if bar.symbol <= self.current_symbol: + raise DerivedBarPersistenceError( + "derived bar output symbols are not strictly ordered" + ) + self._close_symbol(self.current_symbol) + self.current_symbol = bar.symbol + axis = (bar.symbol, bar.scope, bar.interval_code) + writer = self.active.get(axis) + if writer is not None and writer.bar_month != _bar_month( + bar.bar_start_ns + ): + self._close(axis, writer) + writer = None + if writer is None: + writer = _PartitionWriter( + self.staging_directory, + bar, + row_group_size=self.row_group_size, + buffer_rows=self.buffer_rows, + ) + self.active[axis] = writer + writer.add(bar) + + def finish(self) -> tuple[DerivedBarPartitionV1, ...]: + """Close all active writers and return stable partition evidence.""" + for axis, writer in tuple(self.active.items()): + self._close(axis, writer) + if not self.partitions: + raise DerivedBarPersistenceError( + "derived bar publication contains no rows" + ) + return tuple(self.partitions) + + def abort(self) -> None: + """Release all open partition handles after a failed transaction.""" + for writer in tuple(self.active.values()): + try: + writer.abort() + except Exception: # pylint: disable=broad-exception-caught + pass + self.active.clear() + + def _close_symbol(self, symbol: str) -> None: + for axis, writer in tuple(self.active.items()): + if axis[0] == symbol: + self._close(axis, writer) + + def _close( + self, + axis: tuple[str, ActivitySliceScope, str], + writer: _PartitionWriter, + ) -> None: + self.partitions.append(writer.close()) + self.active.pop(axis, None) + if len(self.partitions) > MAX_DERIVED_BAR_PARTITIONS: + raise DerivedBarPersistenceError( + "derived bar partitions exceed limit" + ) + + +def stage_derived_bar_publication( + root: str | Path, + source_manifest_path: str | Path, + *, + policy: DerivedBarPolicyV1 | None = None, + symbols: Iterable[str] = (), + start_ns: int | None = None, + end_ns: int | None = None, + batch_size: int = DEFAULT_DERIVED_BAR_BATCH_SIZE, + row_group_size: int = DEFAULT_DERIVED_BAR_ROW_GROUP_SIZE, + write_buffer_rows: int = DEFAULT_DERIVED_BAR_WRITE_BUFFER_ROWS, +) -> StagedDerivedBarPublicationV1: + """Aggregate and validate one bar product below hidden scratch.""" + source_path = Path(source_manifest_path).expanduser().resolve() + source = verify_reconstruction_publication(source_path) + selected = policy or DerivedBarPolicyV1() + selected_symbols = tuple(_normalized_symbol(value) for value in symbols) + if selected_symbols and not set(selected_symbols).issubset(source.symbols): + raise ValueError("derived bar symbols are outside the source product") + row_group = _bounded_int(row_group_size, "row_group_size", 1, 1_000_000) + buffer_rows = _bounded_int( + write_buffer_rows, "write_buffer_rows", 1, row_group + ) + size = _bounded_int(batch_size, "batch_size", 1, 1_000_000) + query_start, query_end = _query_bounds(start_ns, end_ns) + root_path = Path(root).expanduser().resolve() + axis = _bar_axis_directory( + root_path, source.manifest_id, selected.policy_id + ) + scratch = axis / ".scratch" + scratch.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix="publication.tmp-", dir=scratch)) + manager: _PartitionManager | None = None + try: + manager = _PartitionManager( + staging, + row_group_size=row_group, + buffer_rows=buffer_rows, + ) + for bar in iter_committed_reconstruction_bars( + source_path, + policy=selected, + symbols=selected_symbols, + start_ns=query_start, + end_ns=query_end, + batch_size=size, + ): + manager.add(bar) + partitions = manager.finish() + counts: dict[str, int] = {} + for partition in partitions: + counts[partition.symbol] = ( + counts.get(partition.symbol, 0) + partition.row_count + ) + pa, _ = _arrow_modules() + manifest = DerivedBarProductManifestV1( + source_product_manifest_id=source.manifest_id, + source_product_publication_id=source.publication_id, + source_product_logical_sha256=source.replay.logical_content_sha256, + run_id=source.run_id, + ensemble_member_id=source.ensemble_member_id, + query_start_ns=query_start, + query_end_ns=query_end, + policy=selected, + symbols=tuple(counts), + symbol_bar_counts=counts, + partitions=partitions, + logical_content_sha256=_bar_product_logical_sha256(partitions), + writer_id=DERIVED_BAR_WRITER_ID, + writer_library_version=str(pa.__version__), + python_runtime=( + f"{sys.version_info.major}.{sys.version_info.minor}." + f"{sys.version_info.micro}" + ), + compression=DERIVED_BAR_COMPRESSION, + row_group_size=row_group, + ) + manifest_bytes = manifest.to_json().encode("utf-8") + _atomic_write_bytes( + staging / DERIVED_BAR_MANIFEST_FILENAME, manifest_bytes + ) + committed = axis / "commits" / _path_component(manifest.publication_id) + staged = StagedDerivedBarPublicationV1( + root=root_path, + staging_directory=staging, + committed_directory=committed, + manifest=manifest, + ) + _verify_derived_bar_directory(staging, manifest, committed=False) + return staged + except Exception: + if manager is not None: + manager.abort() + _remove_bar_scratch(staging, root_path) + raise + + +def commit_derived_bar_publication( + staged: StagedDerivedBarPublicationV1, +) -> PublishedDerivedBarsV1: + """Atomically promote one validated derived-bar product.""" + if not isinstance(staged, StagedDerivedBarPublicationV1): + raise TypeError("derived bar commit requires staged publication") + final = staged.committed_directory + manifest = staged.manifest + if final.exists(): + existing_path = final / DERIVED_BAR_MANIFEST_FILENAME + existing = verify_derived_bar_publication(existing_path) + if existing != manifest: + raise DerivedBarPersistenceError( + "derived bar publication identity contains different evidence" + ) + if staged.staging_directory.exists(): + _remove_bar_scratch(staged.staging_directory, staged.root) + return PublishedDerivedBarsV1( + manifest=existing, + manifest_path=existing_path, + manifest_ref=_artifact_ref_for_manifest(existing_path, existing), + idempotent_retry=True, + ) + _verify_derived_bar_directory( + staged.staging_directory, manifest, committed=False + ) + final.parent.mkdir(parents=True, exist_ok=True) + try: + os.replace(staged.staging_directory, final) + except OSError as err: + if not final.exists(): + raise + existing_path = final / DERIVED_BAR_MANIFEST_FILENAME + existing = verify_derived_bar_publication(existing_path) + if existing != manifest: + raise DerivedBarPersistenceError( + "concurrent derived bar publication differs" + ) from err + if staged.staging_directory.exists(): + _remove_bar_scratch(staged.staging_directory, staged.root) + return PublishedDerivedBarsV1( + manifest=existing, + manifest_path=existing_path, + manifest_ref=_artifact_ref_for_manifest(existing_path, existing), + idempotent_retry=True, + ) + _fsync_directory(final.parent) + manifest_path = final / DERIVED_BAR_MANIFEST_FILENAME + verified = verify_derived_bar_publication(manifest_path) + return PublishedDerivedBarsV1( + manifest=verified, + manifest_path=manifest_path, + manifest_ref=_artifact_ref_for_manifest(manifest_path, verified), + ) + + +def publish_derived_bars( + root: str | Path, + source_manifest_path: str | Path, + *, + policy: DerivedBarPolicyV1 | None = None, + symbols: Iterable[str] = (), + start_ns: int | None = None, + end_ns: int | None = None, + batch_size: int = DEFAULT_DERIVED_BAR_BATCH_SIZE, + row_group_size: int = DEFAULT_DERIVED_BAR_ROW_GROUP_SIZE, + write_buffer_rows: int = DEFAULT_DERIVED_BAR_WRITE_BUFFER_ROWS, +) -> PublishedDerivedBarsV1: + """Stage, validate, and atomically commit a derived-bar product.""" + staged = stage_derived_bar_publication( + root, + source_manifest_path, + policy=policy, + symbols=symbols, + start_ns=start_ns, + end_ns=end_ns, + batch_size=batch_size, + row_group_size=row_group_size, + write_buffer_rows=write_buffer_rows, + ) + try: + return commit_derived_bar_publication(staged) + except Exception: + if staged.staging_directory.exists(): + _remove_bar_scratch(staged.staging_directory, staged.root) + raise + + +def load_derived_bar_manifest( + path: str | Path, +) -> DerivedBarProductManifestV1: + """Load and verify a compact manifest without reading Parquet.""" + payload = Path(path).read_bytes() + if len(payload) > MAX_DERIVED_BAR_MANIFEST_BYTES: + raise DerivedBarPersistenceError( + "derived bar manifest exceeds size limit" + ) + try: + text = payload.decode("utf-8") + except UnicodeDecodeError as err: + raise DerivedBarPersistenceError( + "derived bar manifest is not UTF-8" + ) from err + return DerivedBarProductManifestV1.from_json(text) + + +def verify_derived_bar_publication( + manifest_path: str | Path, +) -> DerivedBarProductManifestV1: + """Fail closed unless every committed bar artifact reconciles.""" + path = Path(manifest_path).expanduser().resolve() + manifest = load_derived_bar_manifest(path) + _validate_committed_bar_location(path, manifest) + _verify_derived_bar_directory(path.parent, manifest, committed=True) + return manifest + + +def discover_derived_bar_manifests(root: str | Path) -> tuple[Path, ...]: + """Discover only fully committed, verified derived-bar products.""" + product_root = ( + Path(root).expanduser().resolve() / DERIVED_BAR_PRODUCT_DIRECTORY + ) + if not product_root.exists(): + return () + matches: list[Path] = [] + for path in sorted(product_root.glob("**/commits/*/manifest.json")): + verify_derived_bar_publication(path) + matches.append(path.resolve()) + return tuple(matches) + + +def iter_derived_bar_batches( + manifest_path: str | Path, + *, + columns: Iterable[str] = DERIVED_BAR_ARROW_COLUMNS, + symbols: Iterable[str] = (), + scopes: Iterable[ActivitySliceScope | str] = (), + intervals: Iterable[str] = (), + start_ns: int | None = None, + end_ns: int | None = None, + batch_size: int = DEFAULT_DERIVED_BAR_BATCH_SIZE, +) -> Iterator[Any]: + """Stream projected committed bar batches with partition pruning.""" + path = Path(manifest_path).expanduser().resolve() + manifest = verify_derived_bar_publication(path) + requested = tuple(columns) + if not requested: + raise ValueError("derived bar scan requires columns") + unknown = set(requested).difference(DERIVED_BAR_ARROW_COLUMNS) + if unknown: + raise ValueError(f"unknown derived bar columns: {sorted(unknown)}") + selected_symbols, selected_scopes, selected_intervals = ( + _validate_bar_scan_filters(manifest, symbols, scopes, intervals) + ) + start, end = _query_bounds(start_ns, end_ns) + size = _bounded_int(batch_size, "batch_size", 1, 1_000_000) + _, _, ds = _arrow_dataset_modules() + expression = None + if start is not None: + expression = ds.field("bar_end_ns") > start + if end is not None: + upper = ds.field("bar_start_ns") < end + expression = upper if expression is None else expression & upper + for partition in manifest.partitions: + if selected_symbols and partition.symbol not in selected_symbols: + continue + if selected_scopes and partition.scope not in selected_scopes: + continue + if ( + selected_intervals + and partition.interval_code not in selected_intervals + ): + continue + if ( + start is not None + and partition.max_bar_start_ns + + STANDARD_DERIVED_BAR_INTERVALS[partition.interval_code] + <= start + ): + continue + if end is not None and partition.min_bar_start_ns >= end: + continue + dataset = ds.dataset( + path.parent / partition.relative_path, format="parquet" + ) + scanner = dataset.scanner( + columns=list(requested), + filter=expression, + batch_size=size, + use_threads=False, + ) + yield from scanner.to_batches() + + +def scan_derived_bars_polars( + manifest_path: str | Path, + *, + columns: Iterable[str] = DERIVED_BAR_ARROW_COLUMNS, + symbols: Iterable[str] = (), + scopes: Iterable[ActivitySliceScope | str] = (), + intervals: Iterable[str] = (), + start_ns: int | None = None, + end_ns: int | None = None, +) -> Any: + """Return a predicate-pushed lazy Polars bar scan.""" + path = Path(manifest_path).expanduser().resolve() + manifest = verify_derived_bar_publication(path) + requested = tuple(columns) + unknown = set(requested).difference(DERIVED_BAR_ARROW_COLUMNS) + if not requested or unknown: + raise ValueError("derived bar scan columns are empty or unknown") + selected_symbols, selected_scopes, selected_intervals = ( + _validate_bar_scan_filters(manifest, symbols, scopes, intervals) + ) + start, end = _query_bounds(start_ns, end_ns) + paths = [ + path.parent / item.relative_path + for item in manifest.partitions + if (not selected_symbols or item.symbol in selected_symbols) + and (not selected_scopes or item.scope in selected_scopes) + and (not selected_intervals or item.interval_code in selected_intervals) + and ( + start is None + or item.max_bar_start_ns + + STANDARD_DERIVED_BAR_INTERVALS[item.interval_code] + > start + ) + and (end is None or item.min_bar_start_ns < end) + ] + pl = _polars_module() + if not paths: + return ( + pl.from_arrow(derived_bar_arrow_schema().empty_table()) + .lazy() + .select(list(requested)) + ) + lazy = pl.scan_parquet( + [str(item) for item in paths], + hive_partitioning=False, + use_statistics=True, + ) + if start is not None: + lazy = lazy.filter(pl.col("bar_end_ns") > start) + if end is not None: + lazy = lazy.filter(pl.col("bar_start_ns") < end) + return lazy.select(list(requested)) + + +def derived_bar_arrow_schema() -> Any: + """Return the exact narrow Arrow schema for derived bar rows.""" + pa, _ = _arrow_modules() + + def text(name: str) -> Any: + return pa.field(name, pa.string(), nullable=False) + + def integer(name: str) -> Any: + return pa.field(name, pa.int64(), nullable=False) + + def number(name: str, *, nullable: bool = False) -> Any: + return pa.field(name, pa.float64(), nullable=nullable) + + fields = [ + text("schema_version"), + text("event_schema_version"), + text("bar_id"), + text("source_product_manifest_id"), + text("policy_id"), + integer("rounding_digits"), + text("run_id"), + text("ensemble_member_id"), + text("symbol"), + text("scope"), + text("interval_code"), + integer("interval_ns"), + integer("bar_start_ns"), + integer("bar_end_ns"), + text("first_event_id"), + text("last_event_id"), + integer("first_event_time_ns"), + integer("last_event_time_ns"), + integer("event_count"), + integer("observed_event_count"), + integer("synthetic_event_count"), + integer("quote_update_count"), + integer("transition_count"), + ] + fields.extend( + number(name) + for name in ( + "bid_open", + "bid_high", + "bid_low", + "bid_close", + "ask_open", + "ask_high", + "ask_low", + "ask_close", + "mid_open", + "mid_high", + "mid_low", + "mid_close", + "spread_open", + "spread_high", + "spread_low", + "spread_close", + "mean_spread", + ) + ) + fields.extend( + [ + integer("activity_duration_ns"), + number("tick_intensity_per_second", nullable=True), + integer("price_change_count"), + integer("stale_quote_count"), + number("stale_quote_rate", nullable=True), + number("mean_event_confidence", nullable=True), + integer("confidence_support_count"), + text("volume_state"), + number("volume", nullable=True), + pa.field("is_partial_start", pa.bool_(), nullable=False), + pa.field("is_partial_end", pa.bool_(), nullable=False), + ] + ) + fields.extend( + pa.field(name, pa.list_(pa.string()), nullable=False) + for name in ( + "source_version_ids", + "generator_ids", + "generator_versions", + "generator_config_ids", + "reference_ids", + "motif_ids", + "feed_epoch_ids", + "broker_profile_ids", + "constraint_set_ids", + ) + ) + fields.extend( + [ + text("event_content_sha256"), + pa.field("event_schema_augmented", pa.bool_(), nullable=False), + pa.field("raw_m1_input", pa.bool_(), nullable=False), + pa.field( + "centralized_traded_volume_claim", pa.bool_(), nullable=False + ), + ] + ) + schema = pa.schema(fields) + if tuple(schema.names) != DERIVED_BAR_ARROW_COLUMNS: + raise RuntimeError("derived bar Arrow schema order drifted") + return schema + + +def _bars_to_arrow(bars: Iterable[DerivedBarV1]) -> Any: + pa, _ = _arrow_modules() + return pa.Table.from_pylist( + [bar.to_dict() for bar in bars], schema=derived_bar_arrow_schema() + ) + + +def _verify_derived_bar_directory( + directory: Path, + manifest: DerivedBarProductManifestV1, + *, + committed: bool, +) -> None: + if not directory.is_dir() or directory.is_symlink(): + raise DerivedBarPersistenceError( + "derived bar directory is missing or unsafe" + ) + if committed: + _validate_committed_bar_location( + directory / DERIVED_BAR_MANIFEST_FILENAME, manifest + ) + expected_files = {DERIVED_BAR_MANIFEST_FILENAME} + expected_files.update(item.relative_path for item in manifest.partitions) + expected_directories = { + parent.as_posix() + for relative_path in expected_files + for parent in PurePosixPath(relative_path).parents + if parent != PurePosixPath(".") + } + if any(item.is_symlink() for item in directory.rglob("*")): + raise DerivedBarPersistenceError( + "derived bar publication contains unsafe symlink" + ) + actual_files = { + item.relative_to(directory).as_posix() + for item in directory.rglob("*") + if item.is_file() + } + actual_directories = { + item.relative_to(directory).as_posix() + for item in directory.rglob("*") + if item.is_dir() + } + if ( + actual_files != expected_files + or actual_directories != expected_directories + ): + raise DerivedBarPersistenceError("derived bar artifact set differs") + for partition in manifest.partitions: + _validate_bar_partition_file( + directory / partition.relative_path, + partition, + manifest, + manifest.row_group_size, + ) + if ( + _bar_product_logical_sha256(manifest.partitions) + != manifest.logical_content_sha256 + ): + raise DerivedBarPersistenceError("derived bar replay hash differs") + + +def _validate_bar_partition_file( + path: Path, + expected: DerivedBarPartitionV1, + manifest: DerivedBarProductManifestV1, + row_group_size: int, +) -> None: + if not path.is_file() or path.is_symlink(): + raise DerivedBarPersistenceError( + "derived bar partition is missing or unsafe" + ) + if path.stat().st_size != expected.size_bytes: + raise DerivedBarPersistenceError("derived bar partition size differs") + if _file_sha256(path) != expected.byte_sha256: + raise DerivedBarPersistenceError( + "derived bar partition byte hash differs" + ) + _, pq = _arrow_modules() + try: + parquet = pq.ParquetFile(path) + except Exception as err: + raise DerivedBarPersistenceError( + "derived bar footer is unreadable" + ) from err + if not parquet.schema_arrow.remove_metadata().equals( + derived_bar_arrow_schema().remove_metadata() + ): + raise DerivedBarPersistenceError("derived bar partition schema differs") + if parquet.num_row_groups != expected.row_group_count: + raise DerivedBarPersistenceError("derived bar row-group count differs") + digest = hashlib.sha256( + (DERIVED_BAR_LOGICAL_HASH_ALGORITHM + "\n").encode("ascii") + ) + count = 0 + minimum: int | None = None + maximum: int | None = None + last_order: tuple[int, str] | None = None + for ordinal in range(parquet.num_row_groups): + group_rows = parquet.metadata.row_group(ordinal).num_rows + if group_rows < 1 or group_rows > row_group_size: + raise DerivedBarPersistenceError( + "derived bar row group exceeds bound" + ) + for row in parquet.read_row_group(ordinal).to_pylist(): + bar = DerivedBarV1.from_dict(_mapping(row, "derived bar row")) + if ( + bar.symbol != expected.symbol + or bar.scope is not expected.scope + or bar.interval_code != expected.interval_code + or _bar_month(bar.bar_start_ns) != expected.bar_month + ): + raise DerivedBarPersistenceError( + "derived bar partition axis differs" + ) + if ( + bar.source_product_manifest_id + != manifest.source_product_manifest_id + or bar.run_id != manifest.run_id + or bar.ensemble_member_id != manifest.ensemble_member_id + or bar.policy_id != manifest.policy.policy_id + or bar.rounding_digits != manifest.policy.rounding_digits + ): + raise DerivedBarPersistenceError( + "derived bar row provenance differs from manifest" + ) + if ( + manifest.query_start_ns is not None + and bar.first_event_time_ns < manifest.query_start_ns + ) or ( + manifest.query_end_ns is not None + and bar.last_event_time_ns >= manifest.query_end_ns + ): + raise DerivedBarPersistenceError( + "derived bar event bounds escape the manifest query" + ) + expected_partial_start = ( + manifest.query_start_ns is not None + and bar.bar_start_ns < manifest.query_start_ns + ) + expected_partial_end = ( + manifest.query_end_ns is not None + and bar.bar_end_ns > manifest.query_end_ns + ) + if ( + bar.is_partial_start != expected_partial_start + or bar.is_partial_end != expected_partial_end + ): + raise DerivedBarPersistenceError( + "derived bar partial flags differ from manifest query" + ) + order = (bar.bar_start_ns, bar.bar_id) + if last_order is not None and order <= last_order: + raise DerivedBarPersistenceError( + "derived bar partition order differs" + ) + last_order = order + count += 1 + minimum = ( + bar.bar_start_ns + if minimum is None + else min(minimum, bar.bar_start_ns) + ) + maximum = ( + bar.bar_start_ns + if maximum is None + else max(maximum, bar.bar_start_ns) + ) + digest.update(bar.to_json().encode("utf-8")) + digest.update(b"\n") + if ( + count != expected.row_count + or minimum != expected.min_bar_start_ns + or maximum != expected.max_bar_start_ns + or digest.hexdigest() != expected.logical_content_sha256 + ): + raise DerivedBarPersistenceError( + "derived bar partition content differs" + ) + + +def _validate_committed_bar_location( + manifest_path: Path, + manifest: DerivedBarProductManifestV1, +) -> None: + expected_suffix = ( + PurePosixPath( + _bar_axis_directory( + Path("."), + manifest.source_product_manifest_id, + manifest.policy.policy_id, + ).as_posix() + ) + / "commits" + / _path_component(manifest.publication_id) + / DERIVED_BAR_MANIFEST_FILENAME + ) + actual = PurePosixPath(manifest_path.as_posix()) + if ( + len(actual.parts) < len(expected_suffix.parts) + or actual.parts[-len(expected_suffix.parts) :] != expected_suffix.parts + ): + raise DerivedBarPersistenceError( + "derived bar manifest location differs" + ) + + +def _bar_axis_directory( + root: Path, source_manifest_id: str, policy_id: str +) -> Path: + return ( + root + / DERIVED_BAR_PRODUCT_DIRECTORY + / f"schema={_path_component(DERIVED_BAR_PRODUCT_SCHEMA_VERSION)}" + / f"source={_path_component(source_manifest_id)}" + / f"policy={_path_component(policy_id)}" + ) + + +def _bar_partition_relative_path( + symbol: str, + scope: ActivitySliceScope, + interval_code: str, + bar_month: str, +) -> str: + return ( + f"interval={_path_component(interval_code)}/" + f"scope={_path_component(scope.value)}/" + f"symbol={_path_component(symbol)}/" + f"bar_month={_required_bar_month(bar_month)}/part-00000.parquet" + ) + + +def _bar_product_logical_sha256( + partitions: Iterable[DerivedBarPartitionV1], +) -> str: + payload: list[dict[str, JSONValue]] = [ + { + "relative_path": item.relative_path, + "row_count": item.row_count, + "logical_content_sha256": item.logical_content_sha256, + } + for item in sorted(partitions, key=lambda value: value.relative_path) + ] + return _content_sha256( + {"algorithm": DERIVED_BAR_LOGICAL_HASH_ALGORITHM, "partitions": payload} + ) + + +def _bar_event_view(event: SyntheticEventV1) -> _BarEventView: + return _BarEventView( + event_id=event.event_id, + origin=event.origin, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + run_id=event.run_id, + ensemble_member_id=event.ensemble_member_id, + source_version_id=event.source_version_id, + generator_id=event.generator_id, + generator_version=event.generator_version, + generator_config_id=event.generator_config_id, + reference_id=event.reference_id, + motif_id=event.motif_id, + feed_epoch_id=event.feed_epoch_id, + broker_profile_id=event.broker_profile_id, + constraint_set_id=event.constraint_set_id, + confidence=event.confidence, + ) + + +def _bar_event_view_from_mapping(data: Mapping[str, Any]) -> _BarEventView: + return _BarEventView( + event_id=_required_text(data.get("event_id")), + origin=SyntheticEventOrigin(str(data.get("origin", ""))), + symbol=_normalized_symbol(str(data.get("symbol", ""))), + event_time_ns=_strict_int(data.get("event_time_ns"), "event_time_ns"), + event_sequence=_nonnegative_int( + data.get("event_sequence"), "event_sequence" + ), + bid=_positive_float(data.get("bid"), "bid"), + ask=_positive_float(data.get("ask"), "ask"), + run_id=_required_text(data.get("run_id")), + ensemble_member_id=_required_text(data.get("ensemble_member_id")), + source_version_id=_required_text(data.get("source_version_id")), + generator_id=_mapping_optional_text(data, "generator_id"), + generator_version=_mapping_optional_text(data, "generator_version"), + generator_config_id=_mapping_optional_text(data, "generator_config_id"), + reference_id=_mapping_optional_text(data, "reference_id"), + motif_id=_mapping_optional_text(data, "motif_id"), + feed_epoch_id=_mapping_optional_text(data, "feed_epoch_id"), + broker_profile_id=_mapping_optional_text(data, "broker_profile_id"), + constraint_set_id=_mapping_optional_text(data, "constraint_set_id"), + confidence=_optional_ratio(data.get("confidence"), "confidence"), + ) + + +def _bar_event_payload(event: _BarEventView) -> dict[str, JSONValue]: + return { + "event_id": event.event_id, + "origin": event.origin.value, + "symbol": event.symbol, + "event_time_ns": event.event_time_ns, + "event_sequence": event.event_sequence, + "bid": event.bid, + "ask": event.ask, + "run_id": event.run_id, + "ensemble_member_id": event.ensemble_member_id, + "source_version_id": event.source_version_id, + "generator_id": event.generator_id, + "generator_version": event.generator_version, + "generator_config_id": event.generator_config_id, + "reference_id": event.reference_id, + "motif_id": event.motif_id, + "feed_epoch_id": event.feed_epoch_id, + "broker_profile_id": event.broker_profile_id, + "constraint_set_id": event.constraint_set_id, + "confidence": event.confidence, + } + + +def _bar_order_key(bar: DerivedBarV1) -> tuple[str, str, int, int, str]: + return ( + bar.symbol, + bar.scope.value, + bar.interval_ns, + bar.bar_start_ns, + bar.bar_id, + ) + + +def _bar_start(event_time_ns: int, duration_ns: int) -> int: + return (event_time_ns // duration_ns) * duration_ns + + +def _bar_month(bar_start_ns: int) -> str: + seconds = ( + _strict_int(bar_start_ns, "bar_start_ns") // NANOSECONDS_PER_SECOND + ) + return datetime.fromtimestamp(seconds, tz=timezone.utc).strftime("%Y-%m") + + +def _required_bar_month(value: str) -> str: + month = _required_text(value) + if not _BAR_MONTH_RE.fullmatch(month): + raise ValueError("bar_month must be YYYY-MM") + try: + datetime.strptime(month, "%Y-%m") + except ValueError as err: + raise ValueError("bar_month is invalid") from err + return month + + +def _query_bounds( + start_ns: int | None, end_ns: int | None +) -> tuple[int | None, int | None]: + start = _optional_int(start_ns, "start_ns") + end = _optional_int(end_ns, "end_ns") + if start is not None and end is not None and end <= start: + raise ValueError("derived bar end_ns must exceed start_ns") + return start, end + + +def _validate_bar_scan_filters( + manifest: DerivedBarProductManifestV1, + symbols: Iterable[str], + scopes: Iterable[ActivitySliceScope | str], + intervals: Iterable[str], +) -> tuple[set[str], set[ActivitySliceScope], set[str]]: + selected_symbols = {_normalized_symbol(value) for value in symbols} + selected_scopes = {ActivitySliceScope(value) for value in scopes} + selected_intervals = {_required_text(value).lower() for value in intervals} + if selected_symbols and not selected_symbols.issubset(manifest.symbols): + raise ValueError("derived bar symbols are outside the product manifest") + if selected_scopes and not selected_scopes.issubset( + set(manifest.policy.scopes) + ): + raise ValueError("derived bar scopes are outside the product policy") + if selected_intervals and not selected_intervals.issubset( + manifest.policy.intervals + ): + raise ValueError("derived bar intervals are outside the product policy") + return selected_symbols, selected_scopes, selected_intervals + + +def _artifact_ref_for_manifest( + path: Path, manifest: DerivedBarProductManifestV1 +) -> ArtifactRef: + payload = path.read_bytes() + return ArtifactRef( + kind=DERIVED_BAR_MANIFEST_ARTIFACT_KIND, + path=str(path), + size_bytes=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + metadata={ + "schema_version": manifest.schema_version, + "manifest_id": manifest.manifest_id, + "publication_id": manifest.publication_id, + "source_product_manifest_id": manifest.source_product_manifest_id, + "bar_count": manifest.bar_count, + "logical_content_sha256": manifest.logical_content_sha256, + }, + ) + + +def _atomic_write_bytes(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + handle, temporary_name = tempfile.mkstemp( + prefix=path.name + ".tmp-", dir=path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(handle, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + _fsync_directory(path.parent) + finally: + temporary.unlink(missing_ok=True) + + +def _remove_bar_scratch(path: Path, root: Path) -> None: + if not path.exists() and not path.is_symlink(): + return + if path.is_symlink(): + path.unlink(missing_ok=True) + return + expected_root = root.resolve() / DERIVED_BAR_PRODUCT_DIRECTORY + try: + path.resolve().relative_to(expected_root) + except ValueError as err: + raise DerivedBarPersistenceError( + "derived bar scratch cleanup escaped product root" + ) from err + shutil.rmtree(path) + + +def _fsync_file(path: Path) -> None: + with path.open("rb") as stream: + os.fsync(stream.fileno()) + + +def _fsync_directory(path: Path) -> None: + try: + descriptor = os.open(path, os.O_RDONLY) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024**2): + digest.update(chunk) + return digest.hexdigest() + + +def _path_component(value: str) -> str: + return quote(_required_text(value), safe="-_.") + + +def _safe_relative_path(value: str) -> str: + text = _required_text(value) + path = PurePosixPath(text) + if path.is_absolute() or ".." in path.parts or "\\" in text: + raise ValueError("derived bar artifact path must be safe and relative") + return path.as_posix() + + +def _arrow_modules() -> tuple[Any, Any]: + try: + import pyarrow as pa # pylint: disable=import-outside-toplevel + import pyarrow.parquet as pq # pylint: disable=import-outside-toplevel + except ImportError as err: + raise RuntimeError( + "derived bar persistence requires histdatacom[arrow]" + ) from err + return pa, pq + + +def _arrow_dataset_modules() -> tuple[Any, Any, Any]: + pa, pq = _arrow_modules() + try: + import pyarrow.dataset as ds # pylint: disable=import-outside-toplevel + except ImportError as err: + raise RuntimeError("derived bar scans require pyarrow.dataset") from err + return pa, pq, ds + + +def _polars_module() -> Any: + try: + import polars as pl # pylint: disable=import-outside-toplevel + except ImportError as err: + raise RuntimeError("derived bar scans require polars") from err + return pl + + +def _content_sha256(value: Any) -> str: + return hashlib.sha256( + canonical_contract_json(value).encode("utf-8") + ).hexdigest() + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + return f"{prefix}:sha256:{_content_sha256(payload)}" + + +def _rounded(value: float, digits: int) -> float: + return round(_finite_float(value, "derived value"), digits) + + +def _matches_rounded_value(actual: float, expected: float, digits: int) -> bool: + unit = 10.0**-digits + tolerance = 1.5 * unit + 4 * math.ulp(float(expected)) + return math.isclose(actual, expected, rel_tol=0.0, abs_tol=tolerance) + + +def _required_text(value: Any) -> str: + normalized = str(value or "").strip() + if not normalized or len(normalized) > MAX_DERIVED_BAR_TEXT: + raise ValueError("required derived bar text is empty or unbounded") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return _required_text(normalized) if normalized else None + + +def _normalized_symbol(value: str) -> str: + symbol = _required_text(value).lower() + if not symbol.isalnum(): + raise ValueError("derived bar symbols must be alphanumeric") + return symbol + + +def _normalized_text_tuple(values: Iterable[str]) -> tuple[str, ...]: + return tuple(sorted({_required_text(value) for value in values})) + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _strict_bool(value: Any) -> bool: + if not isinstance(value, bool): + raise ValueError("derived bar partial flags must be boolean") + return value + + +def _optional_int(value: Any, name: str) -> int | None: + return None if value is None else _strict_int(value, name) + + +def _nonnegative_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + result = _strict_int(value, name) + if not minimum <= result <= maximum: + raise ValueError(f"{name} is outside its bounded range") + return result + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _positive_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _nonnegative_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _optional_nonnegative_float(value: Any, name: str) -> float | None: + return None if value is None else _nonnegative_float(value, name) + + +def _optional_ratio(value: Any, name: str) -> float | None: + if value is None: + return None + result = _finite_float(value, name) + if not 0 <= result <= 1: + raise ValueError(f"{name} must be between zero and one") + return result + + +def _required_sha256(value: Any, name: str) -> str: + normalized = str(value or "").strip().lower() + if not _SHA256_RE.fullmatch(normalized): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return normalized + + +def _require_version(actual: str, expected: str, name: str) -> None: + if actual != expected: + raise ValueError(f"unsupported {name} schema version") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + _require_version( + str(data.get("schema_version", "")), expected, "derived bar" + ) + + +def _require_derived(data: Mapping[str, Any], name: str, expected: Any) -> None: + if data.get(name) != expected: + raise ValueError(f"derived bar field {name} differs") + + +def _mapping(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be a mapping") + return cast(Mapping[str, Any], value) + + +def _mapping_sequence(value: Any, name: str) -> tuple[Mapping[str, Any], ...]: + return tuple(_mapping(item, name) for item in _sequence(value, name)) + + +def _sequence(value: Any, name: str) -> Sequence[Any]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError(f"{name} must be a sequence") + return value + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + return tuple(_required_text(item) for item in _sequence(value, name)) + + +def _mapping_optional_text(data: Mapping[str, Any], name: str) -> str | None: + return _optional_text(data.get(name)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + try: + return _mapping(json.loads(text), "derived bar JSON") + except json.JSONDecodeError as err: + raise ValueError("derived bar JSON is invalid") from err + + +__all__ = [ + "DEFAULT_DERIVED_BAR_BATCH_SIZE", + "DEFAULT_DERIVED_BAR_MAX_BARS", + "DEFAULT_DERIVED_BAR_MAX_PROVENANCE_VALUES", + "DEFAULT_DERIVED_BAR_MAX_SYMBOLS", + "DEFAULT_DERIVED_BAR_ROUNDING_DIGITS", + "DEFAULT_DERIVED_BAR_ROW_GROUP_SIZE", + "DEFAULT_DERIVED_BAR_WRITE_BUFFER_ROWS", + "DERIVED_BAR_ARROW_COLUMNS", + "DERIVED_BAR_EVENT_COLUMNS", + "DERIVED_BAR_INTERVAL_SCHEMA_VERSION", + "DERIVED_BAR_MANIFEST_ARTIFACT_KIND", + "DERIVED_BAR_PARTITION_SCHEMA_VERSION", + "DERIVED_BAR_POLICY_SCHEMA_VERSION", + "DERIVED_BAR_PRODUCT_DIRECTORY", + "DERIVED_BAR_PRODUCT_SCHEMA_VERSION", + "DERIVED_BAR_SCHEMA_VERSION", + "STANDARD_DERIVED_BAR_INTERVALS", + "DerivedBarIntervalV1", + "DerivedBarPartitionV1", + "DerivedBarPersistenceError", + "DerivedBarPolicyV1", + "DerivedBarProductManifestV1", + "DerivedBarV1", + "PublishedDerivedBarsV1", + "StagedDerivedBarPublicationV1", + "commit_derived_bar_publication", + "derive_reconstruction_bars", + "derived_bar_arrow_schema", + "discover_derived_bar_manifests", + "iter_committed_reconstruction_bars", + "iter_derived_bar_batches", + "load_derived_bar_manifest", + "publish_derived_bars", + "scan_derived_bars_polars", + "stage_derived_bar_publication", + "verify_derived_bar_publication", +] diff --git a/src/histdatacom/synthetic/benchmark.py b/src/histdatacom/synthetic/benchmark.py new file mode 100644 index 00000000..6f772e04 --- /dev/null +++ b/src/histdatacom/synthetic/benchmark.py @@ -0,0 +1,3191 @@ +"""Streaming reverse-degradation benchmark contracts and scorecards. + +The benchmark is deliberately generator-neutral. Dense reference events, +degraded observations, transparent controls, and reconstruction candidates are +adapted to one narrow event contract and scored online. Only bounded aggregate +state and compact scorecards survive a window; dense intermediates remain +process-local. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from enum import Enum +from itertools import combinations +from typing import Any, Protocol, cast, runtime_checkable + +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.contracts import ( + SYNTHETIC_EVENT_SCHEMA_VERSION, + SyntheticEventV1, + canonical_contract_json, +) +from histdatacom.synthetic.information import ( + InformationSplitKind, + ReconstructionInformationManifestV1, +) +from histdatacom.synthetic.observation import ( + ObservationApplicationResultV1, + ObservationCarryStateV1, + ObservationContextV1, + ObservationInputEventV1, + ObservationOperatorV1, + ObservationOutputEventV1, +) +from histdatacom.synthetic.streaming import ReconstructionWindowV1 + +BENCHMARK_PROFILE_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-benchmark-profile.v1" +) +BENCHMARK_SPLIT_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-benchmark-split.v1" +) +BENCHMARK_SCENARIO_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-benchmark-scenario.v1" +) +BENCHMARK_CANDIDATE_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-benchmark-candidate.v1" +) +BENCHMARK_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-benchmark-manifest.v1" +) +BENCHMARK_EVENT_SCHEMA_VERSION = "histdatacom.benchmark-event.v1" +BENCHMARK_EXECUTION_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.benchmark-execution-evidence.v1" +) +BENCHMARK_CANDIDATE_WINDOW_SCHEMA_VERSION = ( + "histdatacom.benchmark-candidate-window.v1" +) +BENCHMARK_SLICE_SCORE_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-slice-score.v1" +) +BENCHMARK_CANDIDATE_SCORE_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-candidate-score.v1" +) +BENCHMARK_SCORECARD_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-scorecard.v1" +) + +DEFAULT_BENCHMARK_MAX_SCENARIOS = 64 +DEFAULT_BENCHMARK_MAX_CANDIDATES = 32 +DEFAULT_BENCHMARK_MAX_SLICES = 4096 +DEFAULT_BENCHMARK_MAX_EVENTS_PER_WINDOW = 250_000 +DEFAULT_BENCHMARK_MAX_HOOK_METRICS = 64 +DEFAULT_BENCHMARK_MAX_REASON_CODES = 64 +DEFAULT_BENCHMARK_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024 +DEFAULT_BENCHMARK_ROUNDING_DIGITS = 12 +MAX_BENCHMARK_SCENARIOS = 256 +MAX_BENCHMARK_CANDIDATES = 128 +MAX_BENCHMARK_SLICES = 16_384 +MAX_BENCHMARK_EVENTS_PER_WINDOW = 1_000_000 +MAX_BENCHMARK_HOOK_METRICS = 256 +MAX_BENCHMARK_REASON_CODES = 256 +MAX_BENCHMARK_PAYLOAD_BYTES = 64 * 1024 * 1024 +MAX_BENCHMARK_ENSEMBLE_MEMBERS = 64 +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 + +DEFAULT_INTERARRIVAL_BUCKETS_NS = ( + 1_000_000, + 10_000_000, + 100_000_000, + 1_000_000_000, + 5_000_000_000, + 30_000_000_000, + 300_000_000_000, +) +DEFAULT_SPREAD_BUCKETS = ( + 0.00001, + 0.00005, + 0.0001, + 0.00025, + 0.0005, + 0.001, + 0.005, +) + + +class BenchmarkSplitKind(str, Enum): + """Immutable chronological benchmark periods.""" + + CALIBRATION = "calibration" + VALIDATION = "validation" + FINAL_HOLDOUT = "final_holdout" + + @classmethod + def from_value( + cls, value: str | "BenchmarkSplitKind" + ) -> "BenchmarkSplitKind": + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError("unsupported benchmark split kind") from err + + +class BenchmarkCandidateKind(str, Enum): + """Distinguish transparent controls from reconstruction candidates.""" + + CONTROL = "control" + CANDIDATE = "candidate" + + @classmethod + def from_value( + cls, value: str | "BenchmarkCandidateKind" + ) -> "BenchmarkCandidateKind": + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError("unsupported benchmark candidate kind") from err + + +class BenchmarkControlKind(str, Enum): + """Required transparent benchmark controls.""" + + NO_FILL = "no_fill" + LINEAR_INTERPOLATION = "linear_interpolation" + RESAMPLE_LAST = "resample_last" + EMPIRICAL_OVERLAY = "empirical_overlay" + + @classmethod + def from_value( + cls, value: str | "BenchmarkControlKind" + ) -> "BenchmarkControlKind": + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError("unsupported benchmark control kind") from err + + +@dataclass(frozen=True, slots=True) +class BenchmarkProfileV1: + """Fixed histogram, resource, and payload bounds for one benchmark.""" + + max_scenarios: int = DEFAULT_BENCHMARK_MAX_SCENARIOS + max_candidates: int = DEFAULT_BENCHMARK_MAX_CANDIDATES + max_slices: int = DEFAULT_BENCHMARK_MAX_SLICES + max_events_per_window: int = DEFAULT_BENCHMARK_MAX_EVENTS_PER_WINDOW + max_hook_metrics: int = DEFAULT_BENCHMARK_MAX_HOOK_METRICS + max_reason_codes: int = DEFAULT_BENCHMARK_MAX_REASON_CODES + max_payload_bytes: int = DEFAULT_BENCHMARK_MAX_PAYLOAD_BYTES + burst_threshold_ns: int = 100_000_000 + quiet_threshold_ns: int = 5_000_000_000 + interarrival_buckets_ns: tuple[int, ...] = DEFAULT_INTERARRIVAL_BUCKETS_NS + spread_buckets: tuple[float, ...] = DEFAULT_SPREAD_BUCKETS + rounding_digits: int = DEFAULT_BENCHMARK_ROUNDING_DIGITS + profile_id: str = "" + schema_version: str = BENCHMARK_PROFILE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_PROFILE_SCHEMA_VERSION: + raise ValueError("unsupported benchmark profile schema") + _bounded_positive( + self.max_scenarios, MAX_BENCHMARK_SCENARIOS, "max_scenarios" + ) + _bounded_positive( + self.max_candidates, MAX_BENCHMARK_CANDIDATES, "max_candidates" + ) + _bounded_positive(self.max_slices, MAX_BENCHMARK_SLICES, "max_slices") + _bounded_positive( + self.max_events_per_window, + MAX_BENCHMARK_EVENTS_PER_WINDOW, + "max_events_per_window", + ) + _bounded_positive( + self.max_hook_metrics, + MAX_BENCHMARK_HOOK_METRICS, + "max_hook_metrics", + ) + _bounded_positive( + self.max_reason_codes, + MAX_BENCHMARK_REASON_CODES, + "max_reason_codes", + ) + _bounded_positive( + self.max_payload_bytes, + MAX_BENCHMARK_PAYLOAD_BYTES, + "max_payload_bytes", + ) + burst = _positive_int(self.burst_threshold_ns, "burst_threshold_ns") + quiet = _positive_int(self.quiet_threshold_ns, "quiet_threshold_ns") + if quiet <= burst: + raise ValueError("quiet threshold must exceed burst threshold") + object.__setattr__(self, "burst_threshold_ns", burst) + object.__setattr__(self, "quiet_threshold_ns", quiet) + intervals = _strictly_increasing_positive_ints( + self.interarrival_buckets_ns, "interarrival bucket" + ) + spreads = _strictly_increasing_positive_floats( + self.spread_buckets, "spread bucket" + ) + if len(intervals) > 64 or len(spreads) > 64: + raise ValueError("benchmark histogram bucket limit exceeded") + object.__setattr__(self, "interarrival_buckets_ns", intervals) + object.__setattr__(self, "spread_buckets", spreads) + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between zero and 16") + expected = _stable_id("benchmark-profile", self.identity_payload()) + supplied = _optional_text(self.profile_id) + if supplied is not None and supplied != expected: + raise ValueError("profile_id does not match deterministic identity") + object.__setattr__(self, "profile_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "max_scenarios": self.max_scenarios, + "max_candidates": self.max_candidates, + "max_slices": self.max_slices, + "max_events_per_window": self.max_events_per_window, + "max_hook_metrics": self.max_hook_metrics, + "max_reason_codes": self.max_reason_codes, + "max_payload_bytes": self.max_payload_bytes, + "burst_threshold_ns": self.burst_threshold_ns, + "quiet_threshold_ns": self.quiet_threshold_ns, + "interarrival_buckets_ns": list(self.interarrival_buckets_ns), + "spread_buckets": list(self.spread_buckets), + "rounding_digits": self.rounding_digits, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "profile_id": self.profile_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkProfileV1": + _require_schema(data, BENCHMARK_PROFILE_SCHEMA_VERSION) + return cls( + max_scenarios=_strict_int( + data.get("max_scenarios"), "max_scenarios" + ), + max_candidates=_strict_int( + data.get("max_candidates"), "max_candidates" + ), + max_slices=_strict_int(data.get("max_slices"), "max_slices"), + max_events_per_window=_strict_int( + data.get("max_events_per_window"), "max_events_per_window" + ), + max_hook_metrics=_strict_int( + data.get("max_hook_metrics"), "max_hook_metrics" + ), + max_reason_codes=_strict_int( + data.get("max_reason_codes"), "max_reason_codes" + ), + max_payload_bytes=_strict_int( + data.get("max_payload_bytes"), "max_payload_bytes" + ), + burst_threshold_ns=_strict_int( + data.get("burst_threshold_ns"), "burst_threshold_ns" + ), + quiet_threshold_ns=_strict_int( + data.get("quiet_threshold_ns"), "quiet_threshold_ns" + ), + interarrival_buckets_ns=tuple( + _strict_int(value, "interarrival bucket") + for value in _sequence(data.get("interarrival_buckets_ns")) + ), + spread_buckets=tuple( + _finite_float(value, "spread bucket") + for value in _sequence(data.get("spread_buckets")) + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + profile_id=str(data.get("profile_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkSplitV1: + """One immutable half-open calibration, validation, or holdout period.""" + + kind: BenchmarkSplitKind + start_ns: int + end_ns: int + split_id: str = "" + schema_version: str = BENCHMARK_SPLIT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_SPLIT_SCHEMA_VERSION: + raise ValueError("unsupported benchmark split schema") + object.__setattr__( + self, "kind", BenchmarkSplitKind.from_value(self.kind) + ) + start = _bounded_int64(self.start_ns, "start_ns") + end = _bounded_int64(self.end_ns, "end_ns") + if end <= start: + raise ValueError("benchmark split end must follow start") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + expected = _stable_id("benchmark-split", self.identity_payload()) + supplied = _optional_text(self.split_id) + if supplied is not None and supplied != expected: + raise ValueError("split_id does not match deterministic identity") + object.__setattr__(self, "split_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "kind": self.kind.value, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "interval": "[start_ns,end_ns)", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "split_id": self.split_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkSplitV1": + _require_schema(data, BENCHMARK_SPLIT_SCHEMA_VERSION) + return cls( + kind=BenchmarkSplitKind.from_value(str(data.get("kind", ""))), + start_ns=_strict_int(data.get("start_ns"), "start_ns"), + end_ns=_strict_int(data.get("end_ns"), "end_ns"), + split_id=str(data.get("split_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkScenarioV1: + """One feed epoch, degradation severity, and evaluation-period cell.""" + + split_kind: BenchmarkSplitKind + epoch_id: str + severity_id: str + observation_operator_id: str + degradation_parameters: Mapping[str, JSONValue] + scenario_id: str = "" + degradation_config_id: str = "" + event_schema_version: str = BENCHMARK_EVENT_SCHEMA_VERSION + schema_version: str = BENCHMARK_SCENARIO_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_SCENARIO_SCHEMA_VERSION: + raise ValueError("unsupported benchmark scenario schema") + object.__setattr__( + self, "split_kind", BenchmarkSplitKind.from_value(self.split_kind) + ) + object.__setattr__(self, "epoch_id", _required_text(self.epoch_id)) + object.__setattr__( + self, "severity_id", _required_text(self.severity_id) + ) + object.__setattr__( + self, + "observation_operator_id", + _required_text(self.observation_operator_id), + ) + if self.event_schema_version != BENCHMARK_EVENT_SCHEMA_VERSION: + raise ValueError( + "scenario event interface differs from benchmark v1" + ) + parameters = _bounded_mapping( + self.degradation_parameters, + "degradation_parameters", + max_items=64, + ) + object.__setattr__(self, "degradation_parameters", parameters) + expected_config = _stable_id( + "benchmark-degradation-config", + { + "operator_id": self.observation_operator_id, + "parameters": parameters, + "parameter_namespace": "degradation", + }, + ) + supplied_config = _optional_text(self.degradation_config_id) + if supplied_config is not None and supplied_config != expected_config: + raise ValueError( + "degradation_config_id does not match deterministic identity" + ) + object.__setattr__(self, "degradation_config_id", expected_config) + expected = _stable_id("benchmark-scenario", self.identity_payload()) + supplied = _optional_text(self.scenario_id) + if supplied is not None and supplied != expected: + raise ValueError( + "scenario_id does not match deterministic identity" + ) + object.__setattr__(self, "scenario_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "event_schema_version": self.event_schema_version, + "split_kind": self.split_kind.value, + "epoch_id": self.epoch_id, + "severity_id": self.severity_id, + "observation_operator_id": self.observation_operator_id, + "degradation_config_id": self.degradation_config_id, + "degradation_parameters": dict(self.degradation_parameters), + "parameter_namespace": "degradation", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "scenario_id": self.scenario_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkScenarioV1": + _require_schema(data, BENCHMARK_SCENARIO_SCHEMA_VERSION) + return cls( + split_kind=BenchmarkSplitKind.from_value( + str(data.get("split_kind", "")) + ), + epoch_id=str(data.get("epoch_id", "")), + severity_id=str(data.get("severity_id", "")), + observation_operator_id=str( + data.get("observation_operator_id", "") + ), + degradation_parameters=_mapping(data.get("degradation_parameters")), + scenario_id=str(data.get("scenario_id", "")), + degradation_config_id=str(data.get("degradation_config_id", "")), + event_schema_version=str(data.get("event_schema_version", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkCandidateV1: + """One independently configured control or reconstruction candidate.""" + + kind: BenchmarkCandidateKind + method_id: str + implementation_version: str + parameters: Mapping[str, JSONValue] + ensemble_member_ids: tuple[str, ...] + control_kind: BenchmarkControlKind | None = None + candidate_id: str = "" + generator_config_id: str = "" + event_schema_version: str = BENCHMARK_EVENT_SCHEMA_VERSION + schema_version: str = BENCHMARK_CANDIDATE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_CANDIDATE_SCHEMA_VERSION: + raise ValueError("unsupported benchmark candidate schema") + kind = BenchmarkCandidateKind.from_value(self.kind) + object.__setattr__(self, "kind", kind) + object.__setattr__(self, "method_id", _required_text(self.method_id)) + object.__setattr__( + self, + "implementation_version", + _required_text(self.implementation_version), + ) + if self.event_schema_version != BENCHMARK_EVENT_SCHEMA_VERSION: + raise ValueError( + "candidate event interface differs from benchmark v1" + ) + members = _normalized_text_tuple(self.ensemble_member_ids) + if not members or len(members) > MAX_BENCHMARK_ENSEMBLE_MEMBERS: + raise ValueError( + "candidate ensemble-member count is outside limits" + ) + object.__setattr__(self, "ensemble_member_ids", members) + parameters = _bounded_mapping( + self.parameters, "candidate parameters", max_items=64 + ) + object.__setattr__(self, "parameters", parameters) + control: BenchmarkControlKind | None = None + if self.control_kind is not None: + control = BenchmarkControlKind.from_value(self.control_kind) + if kind is BenchmarkCandidateKind.CONTROL and control is None: + raise ValueError("control candidate requires control_kind") + if kind is BenchmarkCandidateKind.CANDIDATE and control is not None: + raise ValueError("reconstruction candidate cannot be a control") + object.__setattr__(self, "control_kind", control) + expected_config = _stable_id( + "benchmark-generator-config", + { + "method_id": self.method_id, + "implementation_version": self.implementation_version, + "parameters": parameters, + "parameter_namespace": "generator", + }, + ) + supplied_config = _optional_text(self.generator_config_id) + if supplied_config is not None and supplied_config != expected_config: + raise ValueError( + "generator_config_id does not match deterministic identity" + ) + object.__setattr__(self, "generator_config_id", expected_config) + expected = _stable_id("benchmark-candidate", self.identity_payload()) + supplied = _optional_text(self.candidate_id) + if supplied is not None and supplied != expected: + raise ValueError( + "candidate_id does not match deterministic identity" + ) + object.__setattr__(self, "candidate_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "event_schema_version": self.event_schema_version, + "kind": self.kind.value, + "method_id": self.method_id, + "implementation_version": self.implementation_version, + "parameters": dict(self.parameters), + "parameter_namespace": "generator", + "generator_config_id": self.generator_config_id, + "ensemble_member_ids": list(self.ensemble_member_ids), + "control_kind": ( + self.control_kind.value + if self.control_kind is not None + else None + ), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "candidate_id": self.candidate_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkCandidateV1": + _require_schema(data, BENCHMARK_CANDIDATE_SCHEMA_VERSION) + control = data.get("control_kind") + return cls( + kind=BenchmarkCandidateKind.from_value(str(data.get("kind", ""))), + method_id=str(data.get("method_id", "")), + implementation_version=str(data.get("implementation_version", "")), + parameters=_mapping(data.get("parameters")), + ensemble_member_ids=_string_tuple(data.get("ensemble_member_ids")), + control_kind=( + BenchmarkControlKind.from_value(str(control)) + if control is not None + else None + ), + candidate_id=str(data.get("candidate_id", "")), + generator_config_id=str(data.get("generator_config_id", "")), + event_schema_version=str(data.get("event_schema_version", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +_EXPECTED_BENCHMARK_SPLITS = ( + BenchmarkSplitKind.CALIBRATION, + BenchmarkSplitKind.VALIDATION, + BenchmarkSplitKind.FINAL_HOLDOUT, +) +_REQUIRED_CONTROLS = frozenset(BenchmarkControlKind) + + +@dataclass(frozen=True, slots=True) +class ReverseDegradationBenchmarkManifestV1: + """Immutable benchmark design, interfaces, controls, and split boundary.""" + + run_id: str + information_manifest_id: str + profile: BenchmarkProfileV1 + splits: tuple[BenchmarkSplitV1, ...] + scenarios: tuple[BenchmarkScenarioV1, ...] + candidates: tuple[BenchmarkCandidateV1, ...] + automatic_winner: bool = False + manifest_id: str = "" + schema_version: str = BENCHMARK_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_MANIFEST_SCHEMA_VERSION: + raise ValueError("unsupported benchmark manifest schema") + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, + "information_manifest_id", + _required_text(self.information_manifest_id), + ) + if not isinstance(self.profile, BenchmarkProfileV1): + raise ValueError("benchmark manifest requires a v1 profile") + if _strict_bool(self.automatic_winner, "automatic_winner"): + raise ValueError("v1 benchmark forbids automatic winner selection") + splits = tuple(self.splits) + if any(not isinstance(item, BenchmarkSplitV1) for item in splits): + raise ValueError("benchmark splits must use the v1 contract") + if tuple(item.kind for item in splits) != _EXPECTED_BENCHMARK_SPLITS: + raise ValueError( + "benchmark splits must be calibration, validation, final_holdout" + ) + if not ( + splits[0].end_ns + <= splits[1].start_ns + <= splits[1].end_ns + <= splits[2].start_ns + ): + raise ValueError("benchmark splits overlap or regress") + object.__setattr__(self, "splits", splits) + scenarios = tuple( + sorted(self.scenarios, key=lambda item: item.scenario_id) + ) + if not scenarios or len(scenarios) > self.profile.max_scenarios: + raise ValueError( + "benchmark scenario count is outside profile limits" + ) + if any(not isinstance(item, BenchmarkScenarioV1) for item in scenarios): + raise ValueError("benchmark scenarios must use the v1 contract") + if len({item.scenario_id for item in scenarios}) != len(scenarios): + raise ValueError("benchmark scenario IDs must be unique") + if len({item.epoch_id for item in scenarios}) < 2: + raise ValueError("benchmark requires multiple feed epochs") + if len({item.severity_id for item in scenarios}) < 2: + raise ValueError( + "benchmark requires multiple degradation severities" + ) + scenario_splits = {item.split_kind for item in scenarios} + if scenario_splits != { + BenchmarkSplitKind.VALIDATION, + BenchmarkSplitKind.FINAL_HOLDOUT, + }: + raise ValueError( + "calibration data cannot be an evaluation scenario" + ) + object.__setattr__(self, "scenarios", scenarios) + candidates = tuple( + sorted(self.candidates, key=lambda item: item.candidate_id) + ) + if not candidates or len(candidates) > self.profile.max_candidates: + raise ValueError( + "benchmark candidate count is outside profile limits" + ) + if any( + not isinstance(item, BenchmarkCandidateV1) for item in candidates + ): + raise ValueError("benchmark candidates must use the v1 contract") + if len({item.candidate_id for item in candidates}) != len(candidates): + raise ValueError("benchmark candidate IDs must be unique") + control_counts = Counter( + item.control_kind + for item in candidates + if item.kind is BenchmarkCandidateKind.CONTROL + and item.control_kind is not None + ) + controls = set(control_counts) + if controls != _REQUIRED_CONTROLS: + missing = sorted( + item.value for item in _REQUIRED_CONTROLS - controls + ) + extra = sorted( + item.value + for item in controls - _REQUIRED_CONTROLS + if item is not None + ) + raise ValueError( + "benchmark control set differs; " + f"missing={missing}, extra={extra}" + ) + if any(count != 1 for count in control_counts.values()): + raise ValueError("benchmark requires exactly one of each control") + if not any( + item.kind is BenchmarkCandidateKind.CANDIDATE for item in candidates + ): + raise ValueError("benchmark requires a reconstruction candidate") + degradation_ids = {item.degradation_config_id for item in scenarios} + if any( + item.generator_config_id in degradation_ids for item in candidates + ): + raise ValueError( + "generator and degradation parameters are not independent" + ) + object.__setattr__(self, "candidates", candidates) + object.__setattr__(self, "automatic_winner", False) + expected = _stable_id("benchmark-manifest", self.identity_payload()) + supplied = _optional_text(self.manifest_id) + if supplied is not None and supplied != expected: + raise ValueError( + "manifest_id does not match deterministic identity" + ) + object.__setattr__(self, "manifest_id", expected) + _ensure_payload_size(self.to_dict(), self.profile.max_payload_bytes) + + def split_for(self, kind: BenchmarkSplitKind) -> BenchmarkSplitV1: + selected = BenchmarkSplitKind.from_value(kind) + return next(item for item in self.splits if item.kind is selected) + + def scenario_by_id(self, scenario_id: str) -> BenchmarkScenarioV1: + selected = _required_text(scenario_id) + for item in self.scenarios: + if item.scenario_id == selected: + return item + raise ValueError("scenario is not part of benchmark manifest") + + def candidate_by_id(self, candidate_id: str) -> BenchmarkCandidateV1: + selected = _required_text(candidate_id) + for item in self.candidates: + if item.candidate_id == selected: + return item + raise ValueError("candidate is not part of benchmark manifest") + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "information_manifest_id": self.information_manifest_id, + "profile": self.profile.to_dict(), + "splits": [item.to_dict() for item in self.splits], + "scenarios": [item.to_dict() for item in self.scenarios], + "candidates": [item.to_dict() for item in self.candidates], + "automatic_winner": False, + "selection_policy": "report_only_no_automatic_winner", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReverseDegradationBenchmarkManifestV1": + _require_schema(data, BENCHMARK_MANIFEST_SCHEMA_VERSION) + return cls( + run_id=str(data.get("run_id", "")), + information_manifest_id=str( + data.get("information_manifest_id", "") + ), + profile=BenchmarkProfileV1.from_dict(_mapping(data.get("profile"))), + splits=tuple( + BenchmarkSplitV1.from_dict(item) + for item in _mapping_sequence(data.get("splits")) + ), + scenarios=tuple( + BenchmarkScenarioV1.from_dict(item) + for item in _mapping_sequence(data.get("scenarios")) + ), + candidates=tuple( + BenchmarkCandidateV1.from_dict(item) + for item in _mapping_sequence(data.get("candidates")) + ), + automatic_winner=_strict_bool( + data.get("automatic_winner", False), "automatic_winner" + ), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReverseDegradationBenchmarkManifestV1": + return cls.from_dict(_json_mapping(text)) + + +def validate_benchmark_information_boundary( + benchmark: ReverseDegradationBenchmarkManifestV1, + information: ReconstructionInformationManifestV1, +) -> None: + """Bind benchmark splits to #433 without changing its immutable v1 schema.""" + if information.run_id != benchmark.run_id: + raise ValueError( + "benchmark and information manifests use different runs" + ) + if information.manifest_id != benchmark.information_manifest_id: + raise ValueError("benchmark information manifest identity differs") + if tuple(item.kind for item in information.splits) != ( + InformationSplitKind.TRAIN, + InformationSplitKind.CALIBRATION, + InformationSplitKind.VALIDATION, + ): + raise ValueError("information manifest split declaration is invalid") + by_kind = {item.kind: item for item in information.splits} + calibration = by_kind.get(InformationSplitKind.CALIBRATION) + validation = by_kind.get(InformationSplitKind.VALIDATION) + if calibration is None or validation is None: + raise ValueError("information manifest lacks benchmark source splits") + benchmark_calibration = benchmark.split_for(BenchmarkSplitKind.CALIBRATION) + benchmark_validation = benchmark.split_for(BenchmarkSplitKind.VALIDATION) + final_holdout = benchmark.split_for(BenchmarkSplitKind.FINAL_HOLDOUT) + if ( + benchmark_calibration.start_ns != calibration.start_ns + or benchmark_calibration.end_ns != calibration.end_ns + ): + raise ValueError( + "benchmark calibration differs from information manifest" + ) + if not ( + validation.start_ns + <= benchmark_validation.start_ns + < benchmark_validation.end_ns + <= final_holdout.start_ns + < final_holdout.end_ns + <= validation.end_ns + ): + raise ValueError( + "validation and final holdout must partition information validation" + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkEventV1: + """Shared event interface for reference, degradation, and generation.""" + + source_event_id: str + symbol: str + event_time_ns: int + event_sequence: int + bid: float + ask: float + epoch_id: str + session: str + event_state: str + sparsity: str + ensemble_member_id: str | None = None + anchor_id: str | None = None + support_lower_mid: float | None = None + support_upper_mid: float | None = None + benchmark_event_id: str = "" + schema_version: str = BENCHMARK_EVENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_EVENT_SCHEMA_VERSION: + raise ValueError("unsupported benchmark event schema") + object.__setattr__( + self, "source_event_id", _required_text(self.source_event_id) + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, + "event_time_ns", + _bounded_int64(self.event_time_ns, "event_time_ns"), + ) + sequence = _nonnegative_int(self.event_sequence, "event_sequence") + object.__setattr__(self, "event_sequence", sequence) + bid = _positive_float(self.bid, "bid") + ask = _positive_float(self.ask, "ask") + if ask < bid: + raise ValueError("benchmark event ask precedes bid") + object.__setattr__(self, "bid", bid) + object.__setattr__(self, "ask", ask) + for name in ("epoch_id", "session", "event_state", "sparsity"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "ensemble_member_id", + _optional_text(self.ensemble_member_id), + ) + object.__setattr__(self, "anchor_id", _optional_text(self.anchor_id)) + if (self.support_lower_mid is None) != (self.support_upper_mid is None): + raise ValueError("benchmark support interval must be paired") + if self.support_lower_mid is not None: + lower = _finite_float(self.support_lower_mid, "support_lower_mid") + upper = _finite_float(self.support_upper_mid, "support_upper_mid") + if not lower <= self.mid <= upper: + raise ValueError( + "benchmark support interval excludes candidate mid" + ) + object.__setattr__(self, "support_lower_mid", lower) + object.__setattr__(self, "support_upper_mid", upper) + expected = _stable_id("benchmark-event", self.identity_payload()) + supplied = _optional_text(self.benchmark_event_id) + if supplied is not None and supplied != expected: + raise ValueError( + "benchmark_event_id does not match deterministic identity" + ) + object.__setattr__(self, "benchmark_event_id", expected) + + @property + def mid(self) -> float: + return (self.bid + self.ask) / 2.0 + + @property + def spread(self) -> float: + return self.ask - self.bid + + @property + def slice_key(self) -> tuple[str, str, str, str, str]: + return ( + self.symbol, + self.epoch_id, + self.session, + self.event_state, + self.sparsity, + ) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "source_event_id": self.source_event_id, + "symbol": self.symbol, + "event_time_ns": self.event_time_ns, + "event_sequence": self.event_sequence, + "bid": self.bid, + "ask": self.ask, + "epoch_id": self.epoch_id, + "session": self.session, + "event_state": self.event_state, + "sparsity": self.sparsity, + "ensemble_member_id": self.ensemble_member_id, + "anchor_id": self.anchor_id, + "support_lower_mid": self.support_lower_mid, + "support_upper_mid": self.support_upper_mid, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "benchmark_event_id": self.benchmark_event_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkEventV1": + _require_schema(data, BENCHMARK_EVENT_SCHEMA_VERSION) + return cls( + source_event_id=str(data.get("source_event_id", "")), + symbol=str(data.get("symbol", "")), + event_time_ns=_strict_int( + data.get("event_time_ns"), "event_time_ns" + ), + event_sequence=_strict_int( + data.get("event_sequence"), "event_sequence" + ), + bid=_finite_float(data.get("bid"), "bid"), + ask=_finite_float(data.get("ask"), "ask"), + epoch_id=str(data.get("epoch_id", "")), + session=str(data.get("session", "")), + event_state=str(data.get("event_state", "")), + sparsity=str(data.get("sparsity", "")), + ensemble_member_id=_optional_text(data.get("ensemble_member_id")), + anchor_id=_optional_text(data.get("anchor_id")), + support_lower_mid=_optional_float(data.get("support_lower_mid")), + support_upper_mid=_optional_float(data.get("support_upper_mid")), + benchmark_event_id=str(data.get("benchmark_event_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_observation_input( + cls, + event: ObservationInputEventV1, + *, + sparsity: str, + ensemble_member_id: str | None = None, + ) -> "BenchmarkEventV1": + return cls( + source_event_id=event.source_event_id, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + epoch_id=event.context.epoch_id, + session=event.context.session or "unclassified", + event_state=event.context.state or "unclassified", + sparsity=sparsity, + ensemble_member_id=ensemble_member_id, + anchor_id=( + event.source_event_id if event.protected_anchor else None + ), + ) + + @classmethod + def from_observation_output( + cls, + event: ObservationOutputEventV1, + *, + context: ObservationContextV1, + sparsity: str, + ensemble_member_id: str | None = None, + anchor_id: str | None = None, + ) -> "BenchmarkEventV1": + return cls( + source_event_id=event.source_event_id, + symbol=event.symbol, + event_time_ns=event.observed_time_ns, + event_sequence=event.observed_sequence, + bid=event.bid, + ask=event.ask, + epoch_id=context.epoch_id, + session=context.session or "unclassified", + event_state=context.state or "unclassified", + sparsity=sparsity, + ensemble_member_id=ensemble_member_id, + anchor_id=( + anchor_id or event.source_event_id + if event.protected_anchor + else None + ), + ) + + @classmethod + def from_synthetic_event( + cls, + event: SyntheticEventV1, + *, + epoch_id: str, + session: str, + event_state: str, + sparsity: str, + support_lower_mid: float | None = None, + support_upper_mid: float | None = None, + ) -> "BenchmarkEventV1": + if event.schema_version != SYNTHETIC_EVENT_SCHEMA_VERSION: + raise ValueError("synthetic event interface is not version one") + return cls( + source_event_id=event.event_id, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + epoch_id=epoch_id, + session=session, + event_state=event_state, + sparsity=sparsity, + ensemble_member_id=event.ensemble_member_id, + anchor_id=event.anchor_interval_id, + support_lower_mid=support_lower_mid, + support_upper_mid=support_upper_mid, + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkExecutionEvidenceV1: + """Bounded fit/generation resource, convergence, and failure metadata.""" + + attempted: bool + converged: bool + wall_time_ms: int = 0 + peak_memory_bytes: int = 0 + scratch_bytes: int = 0 + durable_bytes: int = 0 + failure_reason: str | None = None + evidence_id: str = "" + schema_version: str = BENCHMARK_EXECUTION_EVIDENCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_EXECUTION_EVIDENCE_SCHEMA_VERSION: + raise ValueError("unsupported benchmark execution evidence schema") + attempted = _strict_bool(self.attempted, "attempted") + converged = _strict_bool(self.converged, "converged") + if converged and not attempted: + raise ValueError("unattempted benchmark work cannot converge") + object.__setattr__(self, "attempted", attempted) + object.__setattr__(self, "converged", converged) + for name in ( + "wall_time_ms", + "peak_memory_bytes", + "scratch_bytes", + "durable_bytes", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + failure = _optional_text(self.failure_reason) + if converged and failure is not None: + raise ValueError("converged benchmark work cannot have a failure") + if attempted and not converged and failure is None: + raise ValueError("failed benchmark work requires a reason") + object.__setattr__(self, "failure_reason", failure) + expected = _stable_id("benchmark-execution", self.identity_payload()) + supplied = _optional_text(self.evidence_id) + if supplied is not None and supplied != expected: + raise ValueError( + "evidence_id does not match deterministic identity" + ) + object.__setattr__(self, "evidence_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "attempted": self.attempted, + "converged": self.converged, + "wall_time_ms": self.wall_time_ms, + "peak_memory_bytes": self.peak_memory_bytes, + "scratch_bytes": self.scratch_bytes, + "durable_bytes": self.durable_bytes, + "failure_reason": self.failure_reason, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BenchmarkExecutionEvidenceV1": + _require_schema(data, BENCHMARK_EXECUTION_EVIDENCE_SCHEMA_VERSION) + return cls( + attempted=_strict_bool(data.get("attempted"), "attempted"), + converged=_strict_bool(data.get("converged"), "converged"), + wall_time_ms=_strict_int( + data.get("wall_time_ms", 0), "wall_time_ms" + ), + peak_memory_bytes=_strict_int( + data.get("peak_memory_bytes", 0), "peak_memory_bytes" + ), + scratch_bytes=_strict_int( + data.get("scratch_bytes", 0), "scratch_bytes" + ), + durable_bytes=_strict_int( + data.get("durable_bytes", 0), "durable_bytes" + ), + failure_reason=_optional_text(data.get("failure_reason")), + evidence_id=str(data.get("evidence_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkCandidateWindowV1: + """Process-local candidate events plus bounded control-plane evidence.""" + + scenario_id: str + candidate_id: str + window_id: str + ensemble_member_id: str + events: tuple[BenchmarkEventV1, ...] + execution: BenchmarkExecutionEvidenceV1 + hard_constraint_violations: Mapping[str, int] = field(default_factory=dict) + cross_series_hooks: Mapping[str, float] = field(default_factory=dict) + strategy_hooks: Mapping[str, float] = field(default_factory=dict) + schema_version: str = BENCHMARK_CANDIDATE_WINDOW_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_CANDIDATE_WINDOW_SCHEMA_VERSION: + raise ValueError("unsupported benchmark candidate-window schema") + for name in ( + "scenario_id", + "candidate_id", + "window_id", + "ensemble_member_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + events = tuple( + sorted( + self.events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.benchmark_event_id, + ), + ) + ) + if any(not isinstance(item, BenchmarkEventV1) for item in events): + raise ValueError("candidate-window events must use benchmark v1") + if any( + item.ensemble_member_id not in {None, self.ensemble_member_id} + for item in events + ): + raise ValueError("candidate-window event ensemble member differs") + object.__setattr__(self, "events", events) + if not isinstance(self.execution, BenchmarkExecutionEvidenceV1): + raise ValueError("candidate window requires v1 execution evidence") + object.__setattr__( + self, + "hard_constraint_violations", + _bounded_count_mapping( + self.hard_constraint_violations, + "hard constraint", + MAX_BENCHMARK_REASON_CODES, + ), + ) + object.__setattr__( + self, + "cross_series_hooks", + _bounded_metric_mapping( + self.cross_series_hooks, + "cross-series hook", + MAX_BENCHMARK_HOOK_METRICS, + ), + ) + object.__setattr__( + self, + "strategy_hooks", + _bounded_metric_mapping( + self.strategy_hooks, + "strategy hook", + MAX_BENCHMARK_HOOK_METRICS, + ), + ) + + def metadata(self) -> dict[str, JSONValue]: + """Return workflow-safe evidence without embedding event rows.""" + return { + "schema_version": self.schema_version, + "scenario_id": self.scenario_id, + "candidate_id": self.candidate_id, + "window_id": self.window_id, + "ensemble_member_id": self.ensemble_member_id, + "event_count": len(self.events), + "execution": self.execution.to_dict(), + "hard_constraint_violations": dict(self.hard_constraint_violations), + "cross_series_hooks": dict(self.cross_series_hooks), + "strategy_hooks": dict(self.strategy_hooks), + "events_inline": False, + } + + +@runtime_checkable +class BenchmarkGeneratorV1(Protocol): + """Minimal generator interface consumed by the benchmark harness.""" + + candidate_id: str + event_schema_version: str + + def generate( + self, + degraded_events: Sequence[BenchmarkEventV1], + *, + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + ensemble_member_id: str, + ) -> Sequence[BenchmarkEventV1]: + """Generate one bounded member/window candidate stream.""" + + +def generate_benchmark_candidate_window( + generator: BenchmarkGeneratorV1, + candidate: BenchmarkCandidateV1, + degraded_events: Sequence[BenchmarkEventV1], + *, + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + ensemble_member_id: str, + execution: BenchmarkExecutionEvidenceV1, + hard_constraint_violations: Mapping[str, int] | None = None, + cross_series_hooks: Mapping[str, float] | None = None, + strategy_hooks: Mapping[str, float] | None = None, +) -> BenchmarkCandidateWindowV1: + """Invoke and validate one generator through the shared event interface.""" + if candidate.kind is not BenchmarkCandidateKind.CANDIDATE: + raise ValueError("generator invocation requires a candidate definition") + if generator.candidate_id != candidate.candidate_id: + raise ValueError("generator candidate identity differs") + if generator.event_schema_version != BENCHMARK_EVENT_SCHEMA_VERSION: + raise ValueError("generator does not implement benchmark event v1") + if ensemble_member_id not in candidate.ensemble_member_ids: + raise ValueError("ensemble member is not configured for candidate") + events = tuple( + generator.generate( + degraded_events, + scenario=scenario, + window=window, + ensemble_member_id=ensemble_member_id, + ) + ) + return BenchmarkCandidateWindowV1( + scenario_id=scenario.scenario_id, + candidate_id=candidate.candidate_id, + window_id=window.window_id, + ensemble_member_id=ensemble_member_id, + events=events, + execution=execution, + hard_constraint_violations=hard_constraint_violations or {}, + cross_series_hooks=cross_series_hooks or {}, + strategy_hooks=strategy_hooks or {}, + ) + + +def degrade_benchmark_window( + operator: ObservationOperatorV1, + reference_events: Sequence[BenchmarkEventV1], + *, + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + carry: ObservationCarryStateV1 | None = None, + protected_event_ids: Sequence[str] = (), + source_start: bool = False, + degraded_sparsity: str = "degraded", +) -> tuple[tuple[BenchmarkEventV1, ...], ObservationApplicationResultV1]: + """Run #435 degradation and adapt its outputs without retaining a frame.""" + if operator.operator_id != scenario.observation_operator_id: + raise ValueError("scenario observation operator identity differs") + source_by_id: dict[str, BenchmarkEventV1] = {} + observation_inputs: list[ObservationInputEventV1] = [] + for event in _validated_events(reference_events): + if event.source_event_id in source_by_id: + raise ValueError( + "reference source_event_id must be unique per window" + ) + source_by_id[event.source_event_id] = event + observation_inputs.append( + ObservationInputEventV1( + source_event_id=event.source_event_id, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + context=ObservationContextV1( + symbol=event.symbol, + epoch_id=event.epoch_id, + state=event.event_state, + session=event.session, + event_tag=None, + ), + protected_anchor=event.anchor_id is not None, + ) + ) + result = operator.degrade( + observation_inputs, + window=window, + carry=carry, + protected_event_ids=protected_event_ids, + source_start=source_start, + ) + outputs: list[BenchmarkEventV1] = [] + for output in result.output_events: + source = source_by_id[output.source_event_id] + outputs.append( + BenchmarkEventV1.from_observation_output( + output, + context=ObservationContextV1( + symbol=source.symbol, + epoch_id=source.epoch_id, + state=source.event_state, + session=source.session, + event_tag=None, + ), + sparsity=degraded_sparsity, + ensemble_member_id=source.ensemble_member_id, + anchor_id=( + source.anchor_id if output.protected_anchor else None + ), + ) + ) + return tuple(outputs), result + + +def build_benchmark_control_events( + candidate: BenchmarkCandidateV1, + degraded_events: Sequence[BenchmarkEventV1], + *, + ensemble_member_id: str, + empirical_overlay_events: Sequence[BenchmarkEventV1] = (), + max_events: int = DEFAULT_BENCHMARK_MAX_EVENTS_PER_WINDOW, +) -> tuple[BenchmarkEventV1, ...]: + """Build a transparent control without reading withheld reference values.""" + if candidate.kind is not BenchmarkCandidateKind.CONTROL: + raise ValueError("control builder requires a control candidate") + if ensemble_member_id not in candidate.ensemble_member_ids: + raise ValueError("control ensemble member is not configured") + events = _validated_events(degraded_events) + if len(events) > max_events: + raise ValueError("degraded control input exceeds event limit") + control = cast(BenchmarkControlKind, candidate.control_kind) + if control is BenchmarkControlKind.NO_FILL: + selected = events + elif control is BenchmarkControlKind.LINEAR_INTERPOLATION: + interval_ns = _parameter_positive_int( + candidate.parameters, "interval_ns" + ) + selected = _linear_interpolation_control( + events, + interval_ns=interval_ns, + ensemble_member_id=ensemble_member_id, + max_events=max_events, + ) + elif control is BenchmarkControlKind.RESAMPLE_LAST: + interval_ns = _parameter_positive_int( + candidate.parameters, "interval_ns" + ) + selected = _resample_last_control( + events, + interval_ns=interval_ns, + ensemble_member_id=ensemble_member_id, + ) + else: + overlay = _validated_events(empirical_overlay_events) + if len(overlay) != len(events): + raise ValueError( + "empirical overlay must preserve degraded-row cardinality" + ) + selected = overlay + if len(selected) > max_events: + raise ValueError("control output exceeds event limit") + return tuple( + _with_ensemble_member(item, ensemble_member_id) for item in selected + ) + + +def benchmark_events_from_empirical_overlay( + frame: Any, + *, + symbol: str, + epoch_id: str, + session: str, + event_state: str, + sparsity: str = "empirical_overlay", + ensemble_member_id: str = "control", + max_events: int = DEFAULT_BENCHMARK_MAX_EVENTS_PER_WINDOW, +) -> tuple[BenchmarkEventV1, ...]: + """Adapt #81 row-aligned ``synth_*`` columns to benchmark events.""" + required = {"timestamp_utc_ms", "synth_bid", "synth_ask"} + columns = set(getattr(frame, "columns", ())) + missing = sorted(required - columns) + if missing: + raise ValueError(f"empirical overlay frame lacks columns: {missing}") + selected = frame.select(sorted(required)) + if selected.height > max_events: + raise ValueError( + "empirical overlay frame exceeds benchmark event limit" + ) + events: list[BenchmarkEventV1] = [] + for sequence, row in enumerate(selected.iter_rows(named=True)): + bid = row["synth_bid"] + ask = row["synth_ask"] + if bid is None or ask is None: + raise ValueError( + "empirical overlay contains unavailable synthetic values" + ) + timestamp_ms = _strict_int(row["timestamp_utc_ms"], "timestamp_utc_ms") + events.append( + BenchmarkEventV1( + source_event_id=f"empirical-overlay:{sequence}:{timestamp_ms}", + symbol=symbol, + event_time_ns=timestamp_ms * 1_000_000, + event_sequence=sequence, + bid=_finite_float(bid, "synth_bid"), + ask=_finite_float(ask, "synth_ask"), + epoch_id=epoch_id, + session=session, + event_state=event_state, + sparsity=sparsity, + ensemble_member_id=ensemble_member_id, + ) + ) + return tuple(events) + + +def build_benchmark_control_windows( + manifest: ReverseDegradationBenchmarkManifestV1, + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + degraded_events: Sequence[BenchmarkEventV1], + *, + empirical_overlay_events: Sequence[BenchmarkEventV1], +) -> tuple[BenchmarkCandidateWindowV1, ...]: + """Materialize every mandatory transparent control for one window.""" + results: list[BenchmarkCandidateWindowV1] = [] + for candidate in manifest.candidates: + if candidate.kind is not BenchmarkCandidateKind.CONTROL: + continue + for member_id in candidate.ensemble_member_ids: + events = build_benchmark_control_events( + candidate, + degraded_events, + ensemble_member_id=member_id, + empirical_overlay_events=( + empirical_overlay_events + if candidate.control_kind + is BenchmarkControlKind.EMPIRICAL_OVERLAY + else () + ), + max_events=manifest.profile.max_events_per_window, + ) + results.append( + BenchmarkCandidateWindowV1( + scenario_id=scenario.scenario_id, + candidate_id=candidate.candidate_id, + window_id=window.window_id, + ensemble_member_id=member_id, + events=events, + execution=BenchmarkExecutionEvidenceV1( + attempted=True, + converged=True, + ), + ) + ) + return tuple(results) + + +@dataclass(frozen=True, slots=True) +class BenchmarkSliceScoreV1: + """One fully stratified reference/degraded/candidate comparison.""" + + scenario_id: str + candidate_id: str + symbol: str + epoch_id: str + session: str + event_state: str + sparsity: str + reference_event_count: int + degraded_event_count: int + candidate_event_count_mean: float + metrics: Mapping[str, float] + support: Mapping[str, JSONValue] + slice_score_id: str = "" + schema_version: str = BENCHMARK_SLICE_SCORE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_SLICE_SCORE_SCHEMA_VERSION: + raise ValueError("unsupported benchmark slice-score schema") + for name in ( + "scenario_id", + "candidate_id", + "symbol", + "epoch_id", + "session", + "event_state", + "sparsity", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + for name in ("reference_event_count", "degraded_event_count"): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + object.__setattr__( + self, + "candidate_event_count_mean", + _nonnegative_float( + self.candidate_event_count_mean, + "candidate_event_count_mean", + ), + ) + object.__setattr__( + self, + "metrics", + _bounded_metric_mapping(self.metrics, "slice metric", 64), + ) + object.__setattr__( + self, + "support", + _bounded_mapping(self.support, "slice support", max_items=64), + ) + expected = _stable_id("benchmark-slice-score", self.identity_payload()) + supplied = _optional_text(self.slice_score_id) + if supplied is not None and supplied != expected: + raise ValueError( + "slice_score_id does not match deterministic identity" + ) + object.__setattr__(self, "slice_score_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "scenario_id": self.scenario_id, + "candidate_id": self.candidate_id, + "stratum": { + "symbol": self.symbol, + "epoch_id": self.epoch_id, + "session": self.session, + "event_state": self.event_state, + "sparsity": self.sparsity, + }, + "reference_event_count": self.reference_event_count, + "degraded_event_count": self.degraded_event_count, + "candidate_event_count_mean": self.candidate_event_count_mean, + "metrics": dict(self.metrics), + "support": dict(self.support), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "slice_score_id": self.slice_score_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkSliceScoreV1": + _require_schema(data, BENCHMARK_SLICE_SCORE_SCHEMA_VERSION) + stratum = _mapping(data.get("stratum")) + return cls( + scenario_id=str(data.get("scenario_id", "")), + candidate_id=str(data.get("candidate_id", "")), + symbol=str(stratum.get("symbol", "")), + epoch_id=str(stratum.get("epoch_id", "")), + session=str(stratum.get("session", "")), + event_state=str(stratum.get("event_state", "")), + sparsity=str(stratum.get("sparsity", "")), + reference_event_count=_strict_int( + data.get("reference_event_count"), "reference_event_count" + ), + degraded_event_count=_strict_int( + data.get("degraded_event_count"), "degraded_event_count" + ), + candidate_event_count_mean=_finite_float( + data.get("candidate_event_count_mean"), + "candidate_event_count_mean", + ), + metrics=cast(Mapping[str, float], _mapping(data.get("metrics"))), + support=_mapping(data.get("support")), + slice_score_id=str(data.get("slice_score_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkCandidateScoreV1: + """Bounded scenario/candidate score without a winner declaration.""" + + scenario_id: str + candidate_id: str + split_kind: BenchmarkSplitKind + slice_scores: tuple[BenchmarkSliceScoreV1, ...] + aggregate_metrics: Mapping[str, float] + uncertainty_metrics: Mapping[str, JSONValue] + execution_summary: Mapping[str, JSONValue] + cross_series_hooks: Mapping[str, JSONValue] + strategy_hooks: Mapping[str, JSONValue] + hard_constraint_violations: Mapping[str, int] + relative_to_no_fill: Mapping[str, float] + promotion_eligible: bool + automatic_winner: bool = False + candidate_score_id: str = "" + schema_version: str = BENCHMARK_CANDIDATE_SCORE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_CANDIDATE_SCORE_SCHEMA_VERSION: + raise ValueError("unsupported benchmark candidate-score schema") + object.__setattr__( + self, "scenario_id", _required_text(self.scenario_id) + ) + object.__setattr__( + self, "candidate_id", _required_text(self.candidate_id) + ) + object.__setattr__( + self, "split_kind", BenchmarkSplitKind.from_value(self.split_kind) + ) + slices = tuple( + sorted(self.slice_scores, key=lambda item: item.slice_score_id) + ) + if not slices: + raise ValueError( + "candidate score requires stratified slice evidence" + ) + if any(not isinstance(item, BenchmarkSliceScoreV1) for item in slices): + raise ValueError("candidate score slices must use v1 contracts") + if any( + item.scenario_id != self.scenario_id + or item.candidate_id != self.candidate_id + for item in slices + ): + raise ValueError("candidate score slice identity differs") + object.__setattr__(self, "slice_scores", slices) + object.__setattr__( + self, + "aggregate_metrics", + _bounded_metric_mapping( + self.aggregate_metrics, "aggregate metric", 64 + ), + ) + for name in ( + "uncertainty_metrics", + "execution_summary", + "cross_series_hooks", + "strategy_hooks", + ): + object.__setattr__( + self, + name, + _bounded_mapping(getattr(self, name), name, max_items=256), + ) + object.__setattr__( + self, + "hard_constraint_violations", + _bounded_count_mapping( + self.hard_constraint_violations, + "hard constraint", + MAX_BENCHMARK_REASON_CODES, + ), + ) + object.__setattr__( + self, + "relative_to_no_fill", + _bounded_metric_mapping( + self.relative_to_no_fill, "no-fill delta", 16 + ), + ) + object.__setattr__( + self, + "promotion_eligible", + _strict_bool(self.promotion_eligible, "promotion_eligible"), + ) + if self.promotion_eligible and self.hard_constraint_violations: + raise ValueError( + "hard constraint violations always block promotion eligibility" + ) + if _strict_bool(self.automatic_winner, "automatic_winner"): + raise ValueError( + "candidate score cannot select an automatic winner" + ) + object.__setattr__(self, "automatic_winner", False) + expected = _stable_id( + "benchmark-candidate-score", self.identity_payload() + ) + supplied = _optional_text(self.candidate_score_id) + if supplied is not None and supplied != expected: + raise ValueError( + "candidate_score_id does not match deterministic identity" + ) + object.__setattr__(self, "candidate_score_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "scenario_id": self.scenario_id, + "candidate_id": self.candidate_id, + "split_kind": self.split_kind.value, + "slice_scores": [item.to_dict() for item in self.slice_scores], + "aggregate_metrics": dict(self.aggregate_metrics), + "uncertainty_metrics": dict(self.uncertainty_metrics), + "execution_summary": dict(self.execution_summary), + "cross_series_hooks": dict(self.cross_series_hooks), + "strategy_hooks": dict(self.strategy_hooks), + "hard_constraint_violations": dict(self.hard_constraint_violations), + "relative_to_no_fill": dict(self.relative_to_no_fill), + "promotion_eligible": self.promotion_eligible, + "automatic_winner": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "candidate_score_id": self.candidate_score_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkCandidateScoreV1": + _require_schema(data, BENCHMARK_CANDIDATE_SCORE_SCHEMA_VERSION) + return cls( + scenario_id=str(data.get("scenario_id", "")), + candidate_id=str(data.get("candidate_id", "")), + split_kind=BenchmarkSplitKind.from_value( + str(data.get("split_kind", "")) + ), + slice_scores=tuple( + BenchmarkSliceScoreV1.from_dict(item) + for item in _mapping_sequence(data.get("slice_scores")) + ), + aggregate_metrics=cast( + Mapping[str, float], _mapping(data.get("aggregate_metrics")) + ), + uncertainty_metrics=_mapping(data.get("uncertainty_metrics")), + execution_summary=_mapping(data.get("execution_summary")), + cross_series_hooks=_mapping(data.get("cross_series_hooks")), + strategy_hooks=_mapping(data.get("strategy_hooks")), + hard_constraint_violations=cast( + Mapping[str, int], + _mapping(data.get("hard_constraint_violations")), + ), + relative_to_no_fill=cast( + Mapping[str, float], _mapping(data.get("relative_to_no_fill")) + ), + promotion_eligible=_strict_bool( + data.get("promotion_eligible"), "promotion_eligible" + ), + automatic_winner=_strict_bool( + data.get("automatic_winner", False), "automatic_winner" + ), + candidate_score_id=str(data.get("candidate_score_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReverseDegradationScorecardV1: + """Reproducible bounded benchmark result with no automatic winner.""" + + manifest_id: str + candidate_scores: tuple[BenchmarkCandidateScoreV1, ...] + automatic_winner: bool = False + scorecard_id: str = "" + schema_version: str = BENCHMARK_SCORECARD_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BENCHMARK_SCORECARD_SCHEMA_VERSION: + raise ValueError("unsupported reverse-degradation scorecard schema") + object.__setattr__( + self, "manifest_id", _required_text(self.manifest_id) + ) + scores = tuple( + sorted( + self.candidate_scores, key=lambda item: item.candidate_score_id + ) + ) + if not scores: + raise ValueError("benchmark scorecard requires candidate scores") + if any( + not isinstance(item, BenchmarkCandidateScoreV1) for item in scores + ): + raise ValueError("scorecard candidates must use v1 score contracts") + if len( + {(item.scenario_id, item.candidate_id) for item in scores} + ) != len(scores): + raise ValueError( + "scorecard scenario/candidate cells must be unique" + ) + object.__setattr__(self, "candidate_scores", scores) + if _strict_bool(self.automatic_winner, "automatic_winner"): + raise ValueError("scorecard cannot select an automatic winner") + object.__setattr__(self, "automatic_winner", False) + expected = _stable_id( + "reverse-degradation-scorecard", self.identity_payload() + ) + supplied = _optional_text(self.scorecard_id) + if supplied is not None and supplied != expected: + raise ValueError( + "scorecard_id does not match deterministic identity" + ) + object.__setattr__(self, "scorecard_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "manifest_id": self.manifest_id, + "candidate_scores": [ + item.to_dict() for item in self.candidate_scores + ], + "automatic_winner": False, + "winner_candidate_id": None, + "interpretation": ( + "stratified evidence; aggregate soft loss is advisory and hard " + "constraint violations always block promotion" + ), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "scorecard_id": self.scorecard_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReverseDegradationScorecardV1": + _require_schema(data, BENCHMARK_SCORECARD_SCHEMA_VERSION) + return cls( + manifest_id=str(data.get("manifest_id", "")), + candidate_scores=tuple( + BenchmarkCandidateScoreV1.from_dict(item) + for item in _mapping_sequence(data.get("candidate_scores")) + ), + automatic_winner=_strict_bool( + data.get("automatic_winner", False), "automatic_winner" + ), + scorecard_id=str(data.get("scorecard_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReverseDegradationScorecardV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(slots=True) +class _OnlineMetric: + """Constant-space summary for one numeric hook or diversity metric.""" + + count: int = 0 + total: float = 0.0 + minimum: float | None = None + maximum: float | None = None + + def update(self, value: float) -> None: + selected = _finite_float(value, "online metric") + self.count += 1 + self.total += selected + self.minimum = ( + selected if self.minimum is None else min(self.minimum, selected) + ) + self.maximum = ( + selected if self.maximum is None else max(self.maximum, selected) + ) + + @property + def mean(self) -> float: + return self.total / self.count if self.count else 0.0 + + def payload(self, digits: int) -> dict[str, JSONValue]: + return { + "count": self.count, + "mean": _rounded(self.mean, digits), + "minimum": ( + _rounded(self.minimum, digits) + if self.minimum is not None + else None + ), + "maximum": ( + _rounded(self.maximum, digits) + if self.maximum is not None + else None + ), + } + + +@dataclass(slots=True) +class _StreamStats: + """Constant-space event, timing, quote, path, and histogram state.""" + + interarrival_buckets: tuple[int, ...] + spread_buckets: tuple[float, ...] + burst_threshold_ns: int + quiet_threshold_ns: int + event_count: int = 0 + exposure_ns: int = 0 + interarrival_count: int = 0 + burst_count: int = 0 + quiet_count: int = 0 + burst_duration_total_ns: int = 0 + burst_duration_count: int = 0 + current_burst_duration_ns: int = 0 + quiet_duration_total_ns: int = 0 + quiet_duration_count: int = 0 + bid_total: float = 0.0 + ask_total: float = 0.0 + spread_total: float = 0.0 + bid_transition_total: float = 0.0 + ask_transition_total: float = 0.0 + mid_transition_total: float = 0.0 + spread_transition_total: float = 0.0 + quote_transition_count: int = 0 + spread_transition_count: int = 0 + first_mid: float | None = None + last_mid: float | None = None + minimum_mid: float | None = None + maximum_mid: float | None = None + previous_time_ns: int | None = None + previous_bid: float | None = None + previous_ask: float | None = None + previous_mid: float | None = None + previous_spread: float | None = None + interarrival_histogram: list[int] = field(default_factory=list) + spread_histogram: list[int] = field(default_factory=list) + + def __post_init__(self) -> None: + self.interarrival_histogram = [0] * (len(self.interarrival_buckets) + 1) + self.spread_histogram = [0] * (len(self.spread_buckets) + 1) + + def add_exposure(self, duration_ns: int) -> None: + self.exposure_ns += _nonnegative_int(duration_ns, "exposure_ns") + + def update(self, events: Sequence[BenchmarkEventV1]) -> None: + for event in sorted( + events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.benchmark_event_id, + ), + ): + self.event_count += 1 + spread = event.spread + mid = event.mid + self.bid_total += event.bid + self.ask_total += event.ask + self.spread_total += spread + self.spread_histogram[ + _bucket_index(spread, self.spread_buckets) + ] += 1 + if self.first_mid is None: + self.first_mid = mid + self.last_mid = mid + self.minimum_mid = ( + mid if self.minimum_mid is None else min(self.minimum_mid, mid) + ) + self.maximum_mid = ( + mid if self.maximum_mid is None else max(self.maximum_mid, mid) + ) + if self.previous_time_ns is not None: + gap = max(0, event.event_time_ns - self.previous_time_ns) + self.interarrival_count += 1 + self.interarrival_histogram[ + _bucket_index(gap, self.interarrival_buckets) + ] += 1 + if gap <= self.burst_threshold_ns: + self.burst_count += 1 + self.current_burst_duration_ns += gap + elif self.current_burst_duration_ns: + self.burst_duration_total_ns += ( + self.current_burst_duration_ns + ) + self.burst_duration_count += 1 + self.current_burst_duration_ns = 0 + if gap >= self.quiet_threshold_ns: + self.quiet_count += 1 + self.quiet_duration_total_ns += gap + self.quiet_duration_count += 1 + if self.previous_bid is not None: + self.bid_transition_total += abs(event.bid - self.previous_bid) + self.ask_transition_total += abs( + event.ask - cast(float, self.previous_ask) + ) + self.mid_transition_total += abs( + mid - cast(float, self.previous_mid) + ) + self.quote_transition_count += 1 + if self.previous_spread is not None: + self.spread_transition_total += abs( + spread - self.previous_spread + ) + self.spread_transition_count += 1 + self.previous_time_ns = event.event_time_ns + self.previous_bid = event.bid + self.previous_ask = event.ask + self.previous_mid = mid + self.previous_spread = spread + + @property + def intensity_per_second(self) -> float: + if not self.exposure_ns: + return 0.0 + return self.event_count / (self.exposure_ns / 1_000_000_000) + + @property + def spread_mean(self) -> float: + return self.spread_total / self.event_count if self.event_count else 0.0 + + @property + def bid_mean(self) -> float: + return self.bid_total / self.event_count if self.event_count else 0.0 + + @property + def ask_mean(self) -> float: + return self.ask_total / self.event_count if self.event_count else 0.0 + + @property + def bid_transition_mean(self) -> float: + if not self.quote_transition_count: + return 0.0 + return self.bid_transition_total / self.quote_transition_count + + @property + def ask_transition_mean(self) -> float: + if not self.quote_transition_count: + return 0.0 + return self.ask_transition_total / self.quote_transition_count + + @property + def mid_transition_mean(self) -> float: + if not self.quote_transition_count: + return 0.0 + return self.mid_transition_total / self.quote_transition_count + + @property + def spread_transition_mean(self) -> float: + if not self.spread_transition_count: + return 0.0 + return self.spread_transition_total / self.spread_transition_count + + @property + def mid_range(self) -> float: + if self.minimum_mid is None or self.maximum_mid is None: + return 0.0 + return self.maximum_mid - self.minimum_mid + + @property + def burst_rate(self) -> float: + return ( + self.burst_count / self.interarrival_count + if self.interarrival_count + else 0.0 + ) + + @property + def quiet_rate(self) -> float: + return ( + self.quiet_count / self.interarrival_count + if self.interarrival_count + else 0.0 + ) + + @property + def burst_duration_mean_ns(self) -> float: + total = self.burst_duration_total_ns + self.current_burst_duration_ns + count = self.burst_duration_count + int( + self.current_burst_duration_ns > 0 + ) + return total / count if count else 0.0 + + @property + def quiet_duration_mean_ns(self) -> float: + if not self.quiet_duration_count: + return 0.0 + return self.quiet_duration_total_ns / self.quiet_duration_count + + +@dataclass(slots=True) +class _MemberState: + """Per-member stratified statistics and support accounting.""" + + slices: dict[tuple[str, str, str, str, str], _StreamStats] = field( + default_factory=dict + ) + support_interval_count: int = 0 + covered_reference_count: int = 0 + anchor_reference_count: int = 0 + anchor_preserved_count: int = 0 + + +@dataclass(slots=True) +class _ComparisonState: + """Bounded online state for one scenario and candidate cell.""" + + reference_slices: dict[tuple[str, str, str, str, str], _StreamStats] = ( + field(default_factory=dict) + ) + degraded_slices: dict[tuple[str, str, str, str, str], _StreamStats] = field( + default_factory=dict + ) + member_states: dict[str, _MemberState] = field(default_factory=dict) + hard_violations: Counter[str] = field(default_factory=Counter) + failure_reasons: Counter[str] = field(default_factory=Counter) + attempted_count: int = 0 + converged_count: int = 0 + wall_time_ms: int = 0 + peak_memory_bytes: int = 0 + scratch_bytes: int = 0 + durable_bytes: int = 0 + cross_series_hooks: dict[str, _OnlineMetric] = field(default_factory=dict) + strategy_hooks: dict[str, _OnlineMetric] = field(default_factory=dict) + diversity_by_slice: dict[tuple[str, str, str, str, str], _OnlineMetric] = ( + field(default_factory=dict) + ) + first_window_start_ns: int | None = None + last_window_end_ns: int | None = None + window_count: int = 0 + + +class ReverseDegradationBenchmarkV1: + """Online benchmark engine retaining bounded aggregate state only.""" + + def __init__(self, manifest: ReverseDegradationBenchmarkManifestV1) -> None: + if not isinstance(manifest, ReverseDegradationBenchmarkManifestV1): + raise ValueError("benchmark engine requires a v1 manifest") + self.manifest = manifest + self._states: dict[tuple[str, str], _ComparisonState] = {} + self._slice_count = 0 + self._finalized = False + + def consume_window( + self, + *, + scenario_id: str, + window: ReconstructionWindowV1, + reference_events: Sequence[BenchmarkEventV1], + degraded_events: Sequence[BenchmarkEventV1], + candidate_windows: Sequence[BenchmarkCandidateWindowV1], + ) -> None: + """Consume one complete window without retaining any event rows.""" + if self._finalized: + raise ValueError("benchmark engine is already finalized") + scenario = self.manifest.scenario_by_id(scenario_id) + if window.run_id != self.manifest.run_id: + raise ValueError("benchmark window run differs from manifest") + split = self.manifest.split_for(scenario.split_kind) + if not ( + split.start_ns + <= window.core_start_ns + < window.core_end_ns + <= split.end_ns + ): + raise ValueError("benchmark window falls outside scenario split") + reference = self._validate_window_events( + reference_events, window, scenario, "reference" + ) + degraded = self._validate_window_events( + degraded_events, window, scenario, "degraded" + ) + grouped = self._validate_candidate_windows( + candidate_windows, scenario=scenario, window=window + ) + duration_ns = window.core_end_ns - window.core_start_ns + for candidate in self.manifest.candidates: + key = (scenario.scenario_id, candidate.candidate_id) + state = self._states.setdefault(key, _ComparisonState()) + if state.first_window_start_ns is None: + if window.core_start_ns != split.start_ns: + raise ValueError( + "benchmark scenario does not start at split boundary" + ) + state.first_window_start_ns = window.core_start_ns + elif window.core_start_ns != state.last_window_end_ns: + raise ValueError( + "benchmark windows must be contiguous and ordered" + ) + state.last_window_end_ns = window.core_end_ns + state.window_count += 1 + windows = grouped[candidate.candidate_id] + self._consume_candidate_cell( + state, + candidate, + reference, + degraded, + windows, + duration_ns=duration_ns, + ) + + def finalize(self) -> ReverseDegradationScorecardV1: + """Freeze deterministic stratified scorecards and discard engine use.""" + if self._finalized: + raise ValueError("benchmark engine is already finalized") + expected = { + (scenario.scenario_id, candidate.candidate_id) + for scenario in self.manifest.scenarios + for candidate in self.manifest.candidates + } + if set(self._states) != expected: + missing = sorted(expected - set(self._states)) + raise ValueError( + f"benchmark has unprocessed scenario cells: {missing}" + ) + for scenario in self.manifest.scenarios: + split = self.manifest.split_for(scenario.split_kind) + for candidate in self.manifest.candidates: + state = self._states[ + (scenario.scenario_id, candidate.candidate_id) + ] + if state.last_window_end_ns != split.end_ns: + raise ValueError( + "benchmark scenario does not cover its complete split" + ) + raw: dict[tuple[str, str], BenchmarkCandidateScoreV1] = {} + for scenario in self.manifest.scenarios: + for candidate in self.manifest.candidates: + key = (scenario.scenario_id, candidate.candidate_id) + raw[key] = self._candidate_score( + scenario, + candidate, + self._states[key], + relative_to_no_fill={}, + ) + no_fill_ids = { + item.candidate_id + for item in self.manifest.candidates + if item.control_kind is BenchmarkControlKind.NO_FILL + } + no_fill_id = next(iter(no_fill_ids)) + completed: list[BenchmarkCandidateScoreV1] = [] + for scenario in self.manifest.scenarios: + baseline = raw[(scenario.scenario_id, no_fill_id)] + for candidate in self.manifest.candidates: + score = raw[(scenario.scenario_id, candidate.candidate_id)] + deltas = { + "mean_soft_loss_delta": _rounded( + score.aggregate_metrics["mean_soft_loss"] + - baseline.aggregate_metrics["mean_soft_loss"], + self.manifest.profile.rounding_digits, + ), + "worst_slice_soft_loss_delta": _rounded( + score.aggregate_metrics["worst_slice_soft_loss"] + - baseline.aggregate_metrics["worst_slice_soft_loss"], + self.manifest.profile.rounding_digits, + ), + } + completed.append( + BenchmarkCandidateScoreV1( + scenario_id=score.scenario_id, + candidate_id=score.candidate_id, + split_kind=score.split_kind, + slice_scores=score.slice_scores, + aggregate_metrics=score.aggregate_metrics, + uncertainty_metrics=score.uncertainty_metrics, + execution_summary=score.execution_summary, + cross_series_hooks=score.cross_series_hooks, + strategy_hooks=score.strategy_hooks, + hard_constraint_violations=( + score.hard_constraint_violations + ), + relative_to_no_fill=deltas, + promotion_eligible=score.promotion_eligible, + ) + ) + result = ReverseDegradationScorecardV1( + manifest_id=self.manifest.manifest_id, + candidate_scores=tuple(completed), + ) + _ensure_payload_size( + result.to_dict(), self.manifest.profile.max_payload_bytes + ) + self._finalized = True + return result + + def _validate_window_events( + self, + values: Sequence[BenchmarkEventV1], + window: ReconstructionWindowV1, + scenario: BenchmarkScenarioV1, + label: str, + ) -> tuple[BenchmarkEventV1, ...]: + events = _validated_events(values) + if len(events) > self.manifest.profile.max_events_per_window: + raise ValueError(f"{label} events exceed window limit") + for event in events: + if not window.owns_event_time(event.event_time_ns): + raise ValueError(f"{label} event is outside window ownership") + if event.epoch_id != scenario.epoch_id: + raise ValueError(f"{label} event epoch differs from scenario") + if event.symbol not in {item.upper() for item in window.symbols}: + raise ValueError(f"{label} event symbol differs from window") + return events + + def _validate_candidate_windows( + self, + values: Sequence[BenchmarkCandidateWindowV1], + *, + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + ) -> dict[str, tuple[BenchmarkCandidateWindowV1, ...]]: + grouped: dict[str, list[BenchmarkCandidateWindowV1]] = {} + for value in values: + if not isinstance(value, BenchmarkCandidateWindowV1): + raise ValueError("candidate window must use the v1 contract") + if value.scenario_id != scenario.scenario_id: + raise ValueError("candidate-window scenario differs") + if value.window_id != window.window_id: + raise ValueError("candidate-window identity differs") + candidate = self.manifest.candidate_by_id(value.candidate_id) + if value.ensemble_member_id not in candidate.ensemble_member_ids: + raise ValueError("candidate-window member is not configured") + self._validate_window_events( + value.events, window, scenario, "candidate" + ) + grouped.setdefault(candidate.candidate_id, []).append(value) + result: dict[str, tuple[BenchmarkCandidateWindowV1, ...]] = {} + for candidate in self.manifest.candidates: + selected = tuple( + sorted( + grouped.get(candidate.candidate_id, ()), + key=lambda item: item.ensemble_member_id, + ) + ) + if tuple(item.ensemble_member_id for item in selected) != ( + candidate.ensemble_member_ids + ): + raise ValueError( + "candidate window does not cover configured ensemble members" + ) + result[candidate.candidate_id] = selected + return result + + def _consume_candidate_cell( + self, + state: _ComparisonState, + candidate: BenchmarkCandidateV1, + reference: Sequence[BenchmarkEventV1], + degraded: Sequence[BenchmarkEventV1], + windows: Sequence[BenchmarkCandidateWindowV1], + *, + duration_ns: int, + ) -> None: + reference_by_slice = _events_by_slice(reference) + degraded_by_slice = _events_by_slice(degraded) + member_by_slice: dict[ + str, + dict[tuple[str, str, str, str, str], tuple[BenchmarkEventV1, ...]], + ] = {} + for window in windows: + member_by_slice[window.ensemble_member_id] = _events_by_slice( + window.events + ) + slice_keys = set(reference_by_slice) | set(degraded_by_slice) + for selected in member_by_slice.values(): + slice_keys.update(selected) + for slice_key in sorted(slice_keys): + reference_stats = self._stream_stats( + state.reference_slices, slice_key + ) + degraded_stats = self._stream_stats( + state.degraded_slices, slice_key + ) + reference_stats.add_exposure(duration_ns) + degraded_stats.add_exposure(duration_ns) + reference_stats.update(reference_by_slice.get(slice_key, ())) + degraded_stats.update(degraded_by_slice.get(slice_key, ())) + for member_id in candidate.ensemble_member_ids: + member = state.member_states.setdefault( + member_id, _MemberState() + ) + candidate_stats = self._stream_stats(member.slices, slice_key) + candidate_stats.add_exposure(duration_ns) + candidate_stats.update( + member_by_slice[member_id].get(slice_key, ()) + ) + reference_mid = { + (item.symbol, item.event_time_ns): item.mid for item in reference + } + reference_anchors = { + item.anchor_id for item in reference if item.anchor_id + } + for window in windows: + member = state.member_states[window.ensemble_member_id] + member.anchor_reference_count += len(reference_anchors) + candidate_anchors = { + item.anchor_id for item in window.events if item.anchor_id + } + preserved = len(reference_anchors & candidate_anchors) + member.anchor_preserved_count += preserved + missing_anchors = len(reference_anchors) - preserved + if missing_anchors: + state.hard_violations[ + "historical_anchor_missing" + ] += missing_anchors + for event in window.events: + lower = event.support_lower_mid + upper = event.support_upper_mid + if lower is None or upper is None: + continue + member.support_interval_count += 1 + reference_value = reference_mid.get( + (event.symbol, event.event_time_ns) + ) + if ( + reference_value is not None + and lower <= reference_value <= upper + ): + member.covered_reference_count += 1 + execution = window.execution + state.attempted_count += int(execution.attempted) + state.converged_count += int(execution.converged) + state.wall_time_ms += execution.wall_time_ms + state.peak_memory_bytes = max( + state.peak_memory_bytes, execution.peak_memory_bytes + ) + state.scratch_bytes += execution.scratch_bytes + state.durable_bytes += execution.durable_bytes + if execution.failure_reason is not None: + state.failure_reasons[execution.failure_reason] += 1 + state.hard_violations.update(window.hard_constraint_violations) + self._update_hooks( + state.cross_series_hooks, window.cross_series_hooks + ) + self._update_hooks(state.strategy_hooks, window.strategy_hooks) + self._update_diversity(state, windows) + + def _stream_stats( + self, + target: dict[tuple[str, str, str, str, str], _StreamStats], + key: tuple[str, str, str, str, str], + ) -> _StreamStats: + existing = target.get(key) + if existing is not None: + return existing + self._slice_count += 1 + if self._slice_count > self.manifest.profile.max_slices: + raise ValueError("benchmark slice limit exceeded") + profile = self.manifest.profile + created = _StreamStats( + profile.interarrival_buckets_ns, + profile.spread_buckets, + profile.burst_threshold_ns, + profile.quiet_threshold_ns, + ) + target[key] = created + return created + + def _update_hooks( + self, + target: dict[str, _OnlineMetric], + values: Mapping[str, float], + ) -> None: + for name, value in sorted(values.items()): + metric = target.setdefault(name, _OnlineMetric()) + metric.update(value) + if len(target) > self.manifest.profile.max_hook_metrics: + raise ValueError("benchmark hook metric limit exceeded") + + def _update_diversity( + self, + state: _ComparisonState, + windows: Sequence[BenchmarkCandidateWindowV1], + ) -> None: + if len(windows) < 2: + return + member_maps: dict[str, dict[tuple[str, int], BenchmarkEventV1]] = { + window.ensemble_member_id: { + (event.symbol, event.event_time_ns): event + for event in window.events + } + for window in windows + } + for left_id, right_id in combinations(sorted(member_maps), 2): + left = member_maps[left_id] + right = member_maps[right_id] + for key in sorted(set(left) & set(right)): + left_event = left[key] + right_event = right[key] + metric = state.diversity_by_slice.setdefault( + left_event.slice_key, _OnlineMetric() + ) + metric.update(abs(left_event.mid - right_event.mid)) + + def _candidate_score( + self, + scenario: BenchmarkScenarioV1, + candidate: BenchmarkCandidateV1, + state: _ComparisonState, + *, + relative_to_no_fill: Mapping[str, float], + ) -> BenchmarkCandidateScoreV1: + keys = set(state.reference_slices) | set(state.degraded_slices) + for member in state.member_states.values(): + keys.update(member.slices) + slices = tuple( + self._slice_score(scenario, candidate, state, key) + for key in sorted(keys) + ) + losses = [item.metrics["soft_loss"] for item in slices] + gains = [ + item.metrics["restoration_gain_vs_degraded"] for item in slices + ] + aggregate = { + "mean_soft_loss": _rounded( + sum(losses) / len(losses), self.manifest.profile.rounding_digits + ), + "worst_slice_soft_loss": _rounded( + max(losses), self.manifest.profile.rounding_digits + ), + "mean_restoration_gain_vs_degraded": _rounded( + sum(gains) / len(gains), self.manifest.profile.rounding_digits + ), + "slice_count": float(len(slices)), + "window_count": float(state.window_count), + } + support_count = sum( + item.support_interval_count for item in state.member_states.values() + ) + covered_count = sum( + item.covered_reference_count + for item in state.member_states.values() + ) + anchor_count = sum( + item.anchor_reference_count for item in state.member_states.values() + ) + anchors_preserved = sum( + item.anchor_preserved_count for item in state.member_states.values() + ) + diversity_count = sum( + item.count for item in state.diversity_by_slice.values() + ) + diversity_total = sum( + item.total for item in state.diversity_by_slice.values() + ) + uncertainty: dict[str, JSONValue] = { + "support_interval_count": support_count, + "covered_reference_count": covered_count, + "coverage_rate": _rounded( + covered_count / support_count if support_count else 0.0, + self.manifest.profile.rounding_digits, + ), + "anchor_reference_count": anchor_count, + "anchor_preserved_count": anchors_preserved, + "anchor_preservation_rate": _rounded( + anchors_preserved / anchor_count if anchor_count else 1.0, + self.manifest.profile.rounding_digits, + ), + "ensemble_member_count": len(candidate.ensemble_member_ids), + "ensemble_common_event_comparison_count": diversity_count, + "ensemble_mean_absolute_mid_diversity": _rounded( + diversity_total / diversity_count if diversity_count else 0.0, + self.manifest.profile.rounding_digits, + ), + } + failures = sum(state.failure_reasons.values()) + execution: dict[str, JSONValue] = { + "attempted_count": state.attempted_count, + "converged_count": state.converged_count, + "failure_count": failures, + "failure_reasons": dict(sorted(state.failure_reasons.items())), + "wall_time_ms": state.wall_time_ms, + "peak_memory_bytes": state.peak_memory_bytes, + "scratch_bytes": state.scratch_bytes, + "durable_bytes": state.durable_bytes, + } + promotion_eligible = ( + candidate.kind is BenchmarkCandidateKind.CANDIDATE + and not state.hard_violations + and failures == 0 + and state.attempted_count > 0 + and state.attempted_count == state.converged_count + ) + return BenchmarkCandidateScoreV1( + scenario_id=scenario.scenario_id, + candidate_id=candidate.candidate_id, + split_kind=scenario.split_kind, + slice_scores=slices, + aggregate_metrics=aggregate, + uncertainty_metrics=uncertainty, + execution_summary=execution, + cross_series_hooks={ + name: value.payload(self.manifest.profile.rounding_digits) + for name, value in sorted(state.cross_series_hooks.items()) + }, + strategy_hooks={ + name: value.payload(self.manifest.profile.rounding_digits) + for name, value in sorted(state.strategy_hooks.items()) + }, + hard_constraint_violations=dict( + sorted(state.hard_violations.items()) + ), + relative_to_no_fill=relative_to_no_fill, + promotion_eligible=promotion_eligible, + ) + + def _slice_score( + self, + scenario: BenchmarkScenarioV1, + candidate: BenchmarkCandidateV1, + state: _ComparisonState, + key: tuple[str, str, str, str, str], + ) -> BenchmarkSliceScoreV1: + reference = state.reference_slices.get(key) or self._empty_stats() + degraded = state.degraded_slices.get(key) or self._empty_stats() + member_stats = [ + member.slices.get(key) or self._empty_stats() + for member in state.member_states.values() + ] + candidate_metrics = [ + _stream_loss_metrics(reference, item) for item in member_stats + ] + degraded_metrics = _stream_loss_metrics(reference, degraded) + averaged = { + name: sum(item[name] for item in candidate_metrics) + / len(candidate_metrics) + for name in candidate_metrics[0] + } + soft_names = ( + "event_count_relative_error", + "intensity_relative_error", + "interarrival_hist_l1", + "burst_rate_absolute_error", + "burst_duration_relative_error", + "quiet_rate_absolute_error", + "quiet_duration_relative_error", + "bid_mean_relative_error", + "ask_mean_relative_error", + "spread_mean_relative_error", + "spread_hist_l1", + "bid_transition_relative_error", + "ask_transition_relative_error", + "mid_transition_relative_error", + "spread_transition_relative_error", + "mid_range_relative_error", + "endpoint_relative_error", + ) + soft_loss = sum(averaged[name] for name in soft_names) / len(soft_names) + degraded_loss = sum( + degraded_metrics[name] for name in soft_names + ) / len(soft_names) + metrics = { + **averaged, + "soft_loss": soft_loss, + "degraded_soft_loss": degraded_loss, + "restoration_gain_vs_degraded": degraded_loss - soft_loss, + } + diversity = state.diversity_by_slice.get(key, _OnlineMetric()) + support: dict[str, JSONValue] = { + "stratification_dimensions": [ + "symbol", + "epoch_id", + "session", + "event_state", + "sparsity", + ], + "reference_interarrival_count": reference.interarrival_count, + "degraded_interarrival_count": degraded.interarrival_count, + "ensemble_member_count": len(member_stats), + "ensemble_diversity_comparison_count": diversity.count, + "ensemble_mean_absolute_mid_diversity": _rounded( + diversity.mean, self.manifest.profile.rounding_digits + ), + "aggregate_metrics_are_advisory": True, + } + return BenchmarkSliceScoreV1( + scenario_id=scenario.scenario_id, + candidate_id=candidate.candidate_id, + symbol=key[0], + epoch_id=key[1], + session=key[2], + event_state=key[3], + sparsity=key[4], + reference_event_count=reference.event_count, + degraded_event_count=degraded.event_count, + candidate_event_count_mean=sum( + item.event_count for item in member_stats + ) + / len(member_stats), + metrics={ + name: _rounded(value, self.manifest.profile.rounding_digits) + for name, value in metrics.items() + }, + support=support, + ) + + def _empty_stats(self) -> _StreamStats: + profile = self.manifest.profile + return _StreamStats( + profile.interarrival_buckets_ns, + profile.spread_buckets, + profile.burst_threshold_ns, + profile.quiet_threshold_ns, + ) + + +def _stream_loss_metrics( + reference: _StreamStats, candidate: _StreamStats +) -> dict[str, float]: + return { + "event_count_relative_error": _relative_error( + float(reference.event_count), float(candidate.event_count) + ), + "intensity_relative_error": _relative_error( + reference.intensity_per_second, candidate.intensity_per_second + ), + "interarrival_hist_l1": _histogram_l1( + reference.interarrival_histogram, + candidate.interarrival_histogram, + ), + "burst_rate_absolute_error": abs( + reference.burst_rate - candidate.burst_rate + ), + "burst_duration_relative_error": _relative_error( + reference.burst_duration_mean_ns, + candidate.burst_duration_mean_ns, + ), + "quiet_rate_absolute_error": abs( + reference.quiet_rate - candidate.quiet_rate + ), + "quiet_duration_relative_error": _relative_error( + reference.quiet_duration_mean_ns, + candidate.quiet_duration_mean_ns, + ), + "bid_mean_relative_error": _relative_error( + reference.bid_mean, candidate.bid_mean + ), + "ask_mean_relative_error": _relative_error( + reference.ask_mean, candidate.ask_mean + ), + "spread_mean_relative_error": _relative_error( + reference.spread_mean, candidate.spread_mean + ), + "spread_hist_l1": _histogram_l1( + reference.spread_histogram, candidate.spread_histogram + ), + "spread_transition_relative_error": _relative_error( + reference.spread_transition_mean, + candidate.spread_transition_mean, + ), + "bid_transition_relative_error": _relative_error( + reference.bid_transition_mean, + candidate.bid_transition_mean, + ), + "ask_transition_relative_error": _relative_error( + reference.ask_transition_mean, + candidate.ask_transition_mean, + ), + "mid_transition_relative_error": _relative_error( + reference.mid_transition_mean, + candidate.mid_transition_mean, + ), + "mid_range_relative_error": _relative_error( + reference.mid_range, candidate.mid_range + ), + "endpoint_relative_error": _endpoint_error(reference, candidate), + } + + +def _endpoint_error(reference: _StreamStats, candidate: _StreamStats) -> float: + if reference.last_mid is None: + return 0.0 if candidate.last_mid is None else 1.0 + if candidate.last_mid is None: + return 1.0 + return abs(candidate.last_mid - reference.last_mid) / max( + abs(reference.last_mid), 1e-12 + ) + + +def _relative_error(reference: float, candidate: float) -> float: + if reference == 0.0: + return 0.0 if candidate == 0.0 else 1.0 + return abs(candidate - reference) / abs(reference) + + +def _histogram_l1(left: Sequence[int], right: Sequence[int]) -> float: + left_total = sum(left) + right_total = sum(right) + if left_total == 0: + return 0.0 if right_total == 0 else 1.0 + if right_total == 0: + return 1.0 + return 0.5 * sum( + abs(a / left_total - b / right_total) for a, b in zip(left, right) + ) + + +def _events_by_slice( + events: Sequence[BenchmarkEventV1], +) -> dict[tuple[str, str, str, str, str], tuple[BenchmarkEventV1, ...]]: + grouped: dict[tuple[str, str, str, str, str], list[BenchmarkEventV1]] = {} + for event in events: + grouped.setdefault(event.slice_key, []).append(event) + return {key: tuple(value) for key, value in grouped.items()} + + +def _linear_interpolation_control( + events: Sequence[BenchmarkEventV1], + *, + interval_ns: int, + ensemble_member_id: str, + max_events: int, +) -> tuple[BenchmarkEventV1, ...]: + if len(events) < 2: + return tuple(events) + output: list[BenchmarkEventV1] = [] + for left, right in zip(events, events[1:]): + if not output: + output.append(_with_ensemble_member(left, ensemble_member_id)) + if left.symbol != right.symbol or left.slice_key != right.slice_key: + output.append(_with_ensemble_member(right, ensemble_member_id)) + continue + cursor = ((left.event_time_ns // interval_ns) + 1) * interval_ns + while cursor < right.event_time_ns: + fraction = (cursor - left.event_time_ns) / ( + right.event_time_ns - left.event_time_ns + ) + bid = left.bid + fraction * (right.bid - left.bid) + ask = left.ask + fraction * (right.ask - left.ask) + output.append( + BenchmarkEventV1( + source_event_id=( + "linear-interpolation:" + f"{left.source_event_id}:" + f"{right.source_event_id}:{cursor}" + ), + symbol=left.symbol, + event_time_ns=cursor, + event_sequence=left.event_sequence, + bid=bid, + ask=ask, + epoch_id=left.epoch_id, + session=left.session, + event_state=left.event_state, + sparsity=left.sparsity, + ensemble_member_id=ensemble_member_id, + ) + ) + if len(output) > max_events: + raise ValueError("linear interpolation exceeds event limit") + cursor += interval_ns + output.append(_with_ensemble_member(right, ensemble_member_id)) + return tuple( + sorted( + {item.benchmark_event_id: item for item in output}.values(), + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.benchmark_event_id, + ), + ) + ) + + +def _resample_last_control( + events: Sequence[BenchmarkEventV1], + *, + interval_ns: int, + ensemble_member_id: str, +) -> tuple[BenchmarkEventV1, ...]: + buckets: dict[ + tuple[tuple[str, str, str, str, str], int], BenchmarkEventV1 + ] = {} + for event in events: + key = (event.slice_key, event.event_time_ns // interval_ns) + current = buckets.get(key) + if current is None or ( + event.event_time_ns, + event.event_sequence, + event.benchmark_event_id, + ) > ( + current.event_time_ns, + current.event_sequence, + current.benchmark_event_id, + ): + buckets[key] = event + return tuple( + _with_ensemble_member(item, ensemble_member_id) + for item in sorted( + buckets.values(), + key=lambda value: ( + value.event_time_ns, + value.event_sequence, + value.benchmark_event_id, + ), + ) + ) + + +def _with_ensemble_member( + event: BenchmarkEventV1, ensemble_member_id: str +) -> BenchmarkEventV1: + member = _required_text(ensemble_member_id) + if event.ensemble_member_id == member: + return event + return BenchmarkEventV1( + source_event_id=event.source_event_id, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + epoch_id=event.epoch_id, + session=event.session, + event_state=event.event_state, + sparsity=event.sparsity, + ensemble_member_id=member, + anchor_id=event.anchor_id, + support_lower_mid=event.support_lower_mid, + support_upper_mid=event.support_upper_mid, + ) + + +def _validated_events( + values: Sequence[BenchmarkEventV1], +) -> tuple[BenchmarkEventV1, ...]: + events = tuple(values) + if any(not isinstance(item, BenchmarkEventV1) for item in events): + raise ValueError("events must use the benchmark v1 contract") + return tuple( + sorted( + events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.benchmark_event_id, + ), + ) + ) + + +def _parameter_positive_int( + parameters: Mapping[str, JSONValue], name: str +) -> int: + if name not in parameters: + raise ValueError(f"control parameters require {name}") + return _positive_int(parameters[name], name) + + +def _bucket_index(value: int | float, boundaries: Sequence[int | float]) -> int: + for index, boundary in enumerate(boundaries): + if value <= boundary: + return index + return len(boundaries) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _required_text(value: Any) -> str: + if not isinstance(value, str): + raise ValueError("required text value must be a string") + selected = value.strip() + if not selected or len(selected) > 512: + raise ValueError("required text value is empty or too long") + return selected + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError("optional text value must be a string") + selected = value.strip() + if not selected: + return None + return _required_text(selected) + + +def _normalized_symbol(value: Any) -> str: + return _required_text(value).upper() + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a bool") + return value + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _nonnegative_int(value: Any, name: str) -> int: + selected = _strict_int(value, name) + if selected < 0: + raise ValueError(f"{name} must be non-negative") + return selected + + +def _positive_int(value: Any, name: str) -> int: + selected = _strict_int(value, name) + if selected <= 0: + raise ValueError(f"{name} must be positive") + return selected + + +def _bounded_int64(value: Any, name: str) -> int: + selected = _strict_int(value, name) + if not INT64_MIN <= selected <= INT64_MAX: + raise ValueError(f"{name} is outside signed 64-bit bounds") + return selected + + +def _bounded_positive(value: Any, maximum: int, name: str) -> int: + selected = _positive_int(value, name) + if selected > maximum: + raise ValueError(f"{name} exceeds the v1 maximum") + return selected + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + selected = float(value) + if not math.isfinite(selected): + raise ValueError(f"{name} must be finite") + return selected + + +def _nonnegative_float(value: Any, name: str) -> float: + selected = _finite_float(value, name) + if selected < 0: + raise ValueError(f"{name} must be non-negative") + return selected + + +def _positive_float(value: Any, name: str) -> float: + selected = _finite_float(value, name) + if selected <= 0: + raise ValueError(f"{name} must be positive") + return selected + + +def _optional_float(value: Any) -> float | None: + return None if value is None else _finite_float(value, "optional float") + + +def _strictly_increasing_positive_ints( + values: Iterable[Any], label: str +) -> tuple[int, ...]: + selected = tuple(_positive_int(value, label) for value in values) + if not selected or any( + left >= right for left, right in zip(selected, selected[1:]) + ): + raise ValueError(f"{label}s must be strictly increasing") + return selected + + +def _strictly_increasing_positive_floats( + values: Iterable[Any], label: str +) -> tuple[float, ...]: + selected = tuple(_positive_float(value, label) for value in values) + if not selected or any( + left >= right for left, right in zip(selected, selected[1:]) + ): + raise ValueError(f"{label}s must be strictly increasing") + return selected + + +def _normalized_text_tuple(values: Iterable[Any]) -> tuple[str, ...]: + return tuple(sorted({_required_text(value) for value in values})) + + +def _rounded(value: float | None, digits: int) -> float: + if value is None: + return 0.0 + selected = round(_finite_float(value, "rounded value"), digits) + return 0.0 if selected == 0 else selected + + +def _bounded_count_mapping( + values: Mapping[str, Any], label: str, maximum: int +) -> dict[str, int]: + if len(values) > maximum: + raise ValueError(f"{label} mapping exceeds limit") + result: dict[str, int] = {} + for name, value in sorted(values.items()): + result[_required_text(name)] = _nonnegative_int(value, label) + return result + + +def _bounded_metric_mapping( + values: Mapping[str, Any], label: str, maximum: int +) -> dict[str, float]: + if len(values) > maximum: + raise ValueError(f"{label} mapping exceeds limit") + result: dict[str, float] = {} + for name, value in sorted(values.items()): + result[_required_text(name)] = _finite_float(value, label) + return result + + +def _bounded_mapping( + values: Mapping[str, Any], label: str, *, max_items: int +) -> dict[str, JSONValue]: + if not isinstance(values, Mapping): + raise ValueError(f"{label} must be a mapping") + if len(values) > max_items: + raise ValueError(f"{label} exceeds item limit") + result: dict[str, JSONValue] = {} + for name, value in sorted(values.items()): + key = _required_text(name) + _validate_json_value(value, f"{label}.{key}") + result[key] = cast(JSONValue, value) + if len(canonical_contract_json(result).encode("utf-8")) > 65_536: + raise ValueError(f"{label} exceeds byte limit") + return result + + +def _validate_json_value(value: Any, path: str) -> None: + if value is None or isinstance(value, (str, bool, int)): + return + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError(f"{path} contains a non-finite float") + return + if isinstance(value, list): + for index, item in enumerate(value): + _validate_json_value(item, f"{path}[{index}]") + return + if isinstance(value, dict): + for name, item in value.items(): + if not isinstance(name, str): + raise ValueError(f"{path} contains a non-string key") + _validate_json_value(item, f"{path}.{name}") + return + raise ValueError(f"{path} is not JSON-compatible") + + +def _ensure_payload_size(value: Mapping[str, JSONValue], maximum: int) -> None: + size = len(canonical_contract_json(value).encode("utf-8")) + if size > maximum: + raise ValueError("benchmark payload exceeds configured byte limit") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError("unsupported benchmark schema version") + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError("expected a sequence") + return value + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + return tuple(_mapping(item) for item in _sequence(value)) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + return _mapping(value) diff --git a/src/histdatacom/synthetic/benchmark_corpus.py b/src/histdatacom/synthetic/benchmark_corpus.py new file mode 100644 index 00000000..94ef31b3 --- /dev/null +++ b/src/histdatacom/synthetic/benchmark_corpus.py @@ -0,0 +1,3500 @@ +"""Immutable real-data corpus and campaign for reverse degradation. + +This module is the installed issue-#463 boundary around the generator-neutral +benchmark contracts. It selects small, synchronized partitions from real +Arrow tick caches, records only content-addressed lineage and aggregate +evidence, and can replay every selected partition before a campaign is +trusted. Dense or holdout event rows are never written to the artifacts. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import statistics +import time +from bisect import bisect_left +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, cast + +from histdatacom.data_analytics.feed_epochs_v2 import ( + read_active_time_feed_epoch_definition, +) +from histdatacom.resource_usage import peak_rss_bytes +from histdatacom.runtime_contracts import ArtifactRef, JSONScalar, JSONValue +from histdatacom.synthetic.benchmark import ( + BenchmarkCandidateKind, + BenchmarkCandidateV1, + BenchmarkControlKind, + BenchmarkEventV1, + BenchmarkExecutionEvidenceV1, + BenchmarkScenarioV1, + BenchmarkSplitKind, + build_benchmark_control_events, + degrade_benchmark_window, + generate_benchmark_candidate_window, +) +from histdatacom.synthetic.benchmark_gates import ( + BenchmarkGateObservationV1, + BenchmarkGateScope, + BenchmarkPromotionDecisionV1, + evaluate_benchmark_promotion_gates, + load_default_benchmark_promotion_gate_policy, +) +from histdatacom.synthetic.contracts import canonical_contract_json +from histdatacom.synthetic.generation import ( + EMPIRICAL_MOTIF_GENERATOR_ID, + EmpiricalMotifBenchmarkGeneratorV1, + EmpiricalMotifGeneratorConfigV1, +) +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.motifs import ( + ReferenceMotifConditionV1, + ReferenceMotifIndexConfigV1, + ReferenceMotifIndexV1, + ReferenceMotifSourceEventV1, + ReferenceMotifSourceWindowV1, + ReferenceMotifSplitKind, + ReferenceMotifSplitV1, + build_reference_motif_index, + reference_motif_condition_from_quotes, +) +from histdatacom.synthetic.observation import ObservationOperatorV1 +from histdatacom.synthetic.observation_calibration import ( + read_observation_calibration_campaign, +) +from histdatacom.synthetic.streaming import ( + ReconstructionRunV1, + ReconstructionStoragePolicyV1, + ReconstructionWindowV1, +) + +BENCHMARK_CORPUS_PROFILE_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-corpus-profile.v1" +) +BENCHMARK_SOURCE_PARTITION_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-source-partition.v1" +) +BENCHMARK_WINDOW_PARTITION_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-window-partition.v1" +) +REVERSE_DEGRADATION_CORPUS_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-corpus.v1" +) +BENCHMARK_CANDIDATE_REPORT_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-candidate-report.v1" +) +REVERSE_DEGRADATION_CAMPAIGN_SCHEMA_VERSION = ( + "histdatacom.reverse-degradation-campaign.v1" +) + +PREDECLARED_GATE_COMMIT = "0caec1480a957528ebefdff062e13012ea11e84d" +DEFAULT_BENCHMARK_SYMBOLS = ("EURGBP", "EURUSD", "GBPUSD") +DEFAULT_BENCHMARK_PERIODS = { + "calibration": "201001", + "validation": "202401", + "final_holdout": "202510", +} +DEFAULT_SESSION_HOURS = (0, 8, 14) +DEFAULT_MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 +MAX_BENCHMARK_SOURCE_BYTES = 2 * 1024**3 +MAX_BENCHMARK_WINDOWS = 96 +MAX_BENCHMARK_EVENTS_PER_SYMBOL = 4096 +MAX_BENCHMARK_CANDIDATES = 16 +MAX_BENCHMARK_METRICS = 256 +NANOSECONDS_PER_MILLISECOND = 1_000_000 +NANOSECONDS_PER_SECOND = 1_000_000_000 +PIP = 0.0001 +_PERIOD = re.compile(r"^[0-9]{6}$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_COMMIT = re.compile(r"^[0-9a-f]{40}$") + + +@dataclass(frozen=True, slots=True) +class ReverseDegradationCorpusProfileV1: + """Bounded selection, replay, and execution policy.""" + + symbols: tuple[str, ...] = DEFAULT_BENCHMARK_SYMBOLS + split_periods: Mapping[str, str] = field( + default_factory=lambda: dict(DEFAULT_BENCHMARK_PERIODS) + ) + synchronized_windows_per_split: int = 6 + window_duration_seconds: int = 600 + minimum_events_per_symbol: int = 64 + max_events_per_symbol: int = 256 + neighbor_guard_seconds: int = 1800 + ensemble_member_ids: tuple[str, ...] = ("member-01", "member-02") + max_source_bytes: int = MAX_BENCHMARK_SOURCE_BYTES + max_runtime_seconds: float = 900.0 + max_peak_memory_bytes: int = 2 * 1024**3 + max_artifact_bytes: int = DEFAULT_MAX_ARTIFACT_BYTES + profile_id: str = "" + schema_version: str = BENCHMARK_CORPUS_PROFILE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_CORPUS_PROFILE_SCHEMA_VERSION, + "benchmark corpus profile", + ) + symbols = tuple(sorted({_symbol(item) for item in self.symbols})) + if symbols != DEFAULT_BENCHMARK_SYMBOLS: + raise ValueError( + "benchmark corpus requires the EUR/GBP/USD triangle" + ) + object.__setattr__(self, "symbols", symbols) + periods = { + str(key): str(value) for key, value in self.split_periods.items() + } + if set(periods) != set(DEFAULT_BENCHMARK_PERIODS): + raise ValueError( + "benchmark split periods must cover all blocked roles" + ) + if any(not _PERIOD.fullmatch(value) for value in periods.values()): + raise ValueError("benchmark split periods must use YYYYMM") + if ( + not periods["calibration"] + < periods["validation"] + < periods["final_holdout"] + ): + raise ValueError("benchmark split periods must be chronological") + object.__setattr__(self, "split_periods", dict(sorted(periods.items()))) + _bounded_int( + self.synchronized_windows_per_split, + "synchronized_windows_per_split", + 1, + MAX_BENCHMARK_WINDOWS // 3, + ) + _bounded_int( + self.window_duration_seconds, "window_duration_seconds", 60, 3600 + ) + minimum = _bounded_int( + self.minimum_events_per_symbol, + "minimum_events_per_symbol", + 2, + MAX_BENCHMARK_EVENTS_PER_SYMBOL, + ) + maximum = _bounded_int( + self.max_events_per_symbol, + "max_events_per_symbol", + minimum, + MAX_BENCHMARK_EVENTS_PER_SYMBOL, + ) + if maximum < minimum: + raise ValueError("maximum events must cover the minimum") + _bounded_int( + self.neighbor_guard_seconds, "neighbor_guard_seconds", 0, 86400 + ) + members = tuple( + sorted({_required_text(v) for v in self.ensemble_member_ids}) + ) + if not 2 <= len(members) <= 8: + raise ValueError( + "benchmark campaign requires two to eight ensemble members" + ) + object.__setattr__(self, "ensemble_member_ids", members) + _bounded_int(self.max_source_bytes, "max_source_bytes", 1, 16 * 1024**3) + _positive_float(self.max_runtime_seconds, "max_runtime_seconds") + _bounded_int( + self.max_peak_memory_bytes, "max_peak_memory_bytes", 1, 16 * 1024**3 + ) + _bounded_int( + self.max_artifact_bytes, "max_artifact_bytes", 1024, 1024**3 + ) + expected = _stable_id( + "reverse-degradation-corpus-profile", self.identity_payload() + ) + supplied = _optional_text(self.profile_id) + if supplied is not None and supplied != expected: + raise ValueError("benchmark corpus profile_id differs") + object.__setattr__(self, "profile_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbols": list(self.symbols), + "split_periods": dict(self.split_periods), + "synchronized_windows_per_split": self.synchronized_windows_per_split, + "window_duration_seconds": self.window_duration_seconds, + "minimum_events_per_symbol": self.minimum_events_per_symbol, + "max_events_per_symbol": self.max_events_per_symbol, + "neighbor_guard_seconds": self.neighbor_guard_seconds, + "ensemble_member_ids": list(self.ensemble_member_ids), + "max_source_bytes": self.max_source_bytes, + "max_runtime_seconds": self.max_runtime_seconds, + "max_peak_memory_bytes": self.max_peak_memory_bytes, + "max_artifact_bytes": self.max_artifact_bytes, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "profile_id": self.profile_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReverseDegradationCorpusProfileV1": + _require_schema(data, BENCHMARK_CORPUS_PROFILE_SCHEMA_VERSION) + return cls( + symbols=_string_tuple(data.get("symbols")), + split_periods={ + str(k): str(v) + for k, v in _mapping(data.get("split_periods")).items() + }, + synchronized_windows_per_split=_strict_int( + data.get("synchronized_windows_per_split"), "window count" + ), + window_duration_seconds=_strict_int( + data.get("window_duration_seconds"), "window duration" + ), + minimum_events_per_symbol=_strict_int( + data.get("minimum_events_per_symbol"), "minimum events" + ), + max_events_per_symbol=_strict_int( + data.get("max_events_per_symbol"), "maximum events" + ), + neighbor_guard_seconds=_strict_int( + data.get("neighbor_guard_seconds"), "neighbor guard" + ), + ensemble_member_ids=_string_tuple(data.get("ensemble_member_ids")), + max_source_bytes=_strict_int( + data.get("max_source_bytes"), "max source bytes" + ), + max_runtime_seconds=_finite_float( + data.get("max_runtime_seconds"), "max runtime" + ), + max_peak_memory_bytes=_strict_int( + data.get("max_peak_memory_bytes"), "max peak memory" + ), + max_artifact_bytes=_strict_int( + data.get("max_artifact_bytes"), "max artifact bytes" + ), + profile_id=str(data.get("profile_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkSourcePartitionV1: + """One hash-verified monthly Arrow source.""" + + symbol: str + period: str + relative_path: str + size_bytes: int + row_count: int + sha256: str + partition_id: str = "" + schema_version: str = BENCHMARK_SOURCE_PARTITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_SOURCE_PARTITION_SCHEMA_VERSION, + "source partition", + ) + object.__setattr__(self, "symbol", _symbol(self.symbol)) + if not _PERIOD.fullmatch(self.period): + raise ValueError("source partition period must use YYYYMM") + relative = Path(_required_text(self.relative_path)) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError( + "source partition path must be relative and contained" + ) + object.__setattr__(self, "relative_path", relative.as_posix()) + _bounded_int( + self.size_bytes, "source size", 1, MAX_BENCHMARK_SOURCE_BYTES + ) + _bounded_int(self.row_count, "source row count", 2, 2**63 - 1) + _sha256(self.sha256, "source sha256") + expected = _stable_id( + "reverse-degradation-source-partition", self.identity_payload() + ) + supplied = _optional_text(self.partition_id) + if supplied is not None and supplied != expected: + raise ValueError("source partition_id differs") + object.__setattr__(self, "partition_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "period": self.period, + "relative_path": self.relative_path, + "size_bytes": self.size_bytes, + "row_count": self.row_count, + "sha256": self.sha256, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "partition_id": self.partition_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkSourcePartitionV1": + _require_schema(data, BENCHMARK_SOURCE_PARTITION_SCHEMA_VERSION) + return cls( + symbol=str(data.get("symbol", "")), + period=str(data.get("period", "")), + relative_path=str(data.get("relative_path", "")), + size_bytes=_strict_int(data.get("size_bytes"), "source size"), + row_count=_strict_int(data.get("row_count"), "source rows"), + sha256=str(data.get("sha256", "")), + partition_id=str(data.get("partition_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkWindowPartitionV1: + """One synchronized, replayable real-data window.""" + + split_kind: str + period: str + session: str + start_ns: int + end_ns: int + epoch_label: str + source_partition_ids: tuple[str, ...] + symbol_event_counts: Mapping[str, int] + symbol_partition_sha256: Mapping[str, str] + event_state_counts: Mapping[str, int] + context_state: str + positioning_state: str + context_supported: bool + selection_rule: str = "first-n-events-in-half-open-utc-window" + window_id: str = "" + schema_version: str = BENCHMARK_WINDOW_PARTITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_WINDOW_PARTITION_SCHEMA_VERSION, + "window partition", + ) + if self.split_kind not in DEFAULT_BENCHMARK_PERIODS: + raise ValueError("unsupported benchmark window split") + if not _PERIOD.fullmatch(self.period): + raise ValueError("window period must use YYYYMM") + object.__setattr__(self, "session", _required_text(self.session)) + start = _bounded_int(self.start_ns, "window start", 0, 2**63 - 1) + end = _bounded_int(self.end_ns, "window end", start + 1, 2**63 - 1) + if end <= start: + raise ValueError("window end must follow start") + object.__setattr__( + self, "epoch_label", _required_text(self.epoch_label) + ) + partitions = tuple( + sorted({_required_text(v) for v in self.source_partition_ids}) + ) + if len(partitions) != len(DEFAULT_BENCHMARK_SYMBOLS): + raise ValueError("window must bind all triangle source partitions") + object.__setattr__(self, "source_partition_ids", partitions) + counts = { + _symbol(k): _bounded_int( + v, f"{k} event count", 2, MAX_BENCHMARK_EVENTS_PER_SYMBOL + ) + for k, v in self.symbol_event_counts.items() + } + hashes = { + _symbol(k): _sha256(v, f"{k} window hash") + for k, v in self.symbol_partition_sha256.items() + } + if set(counts) != set(DEFAULT_BENCHMARK_SYMBOLS) or set(hashes) != set( + DEFAULT_BENCHMARK_SYMBOLS + ): + raise ValueError("window counts and hashes must cover the triangle") + object.__setattr__( + self, "symbol_event_counts", dict(sorted(counts.items())) + ) + object.__setattr__( + self, "symbol_partition_sha256", dict(sorted(hashes.items())) + ) + states = { + _required_text(name): _bounded_int( + count, + f"{name} event-state count", + 0, + MAX_BENCHMARK_EVENTS_PER_SYMBOL + * len(DEFAULT_BENCHMARK_SYMBOLS), + ) + for name, count in self.event_state_counts.items() + } + if not states or sum(states.values()) != sum(counts.values()): + raise ValueError( + "window event-state counts differ from event counts" + ) + object.__setattr__( + self, "event_state_counts", dict(sorted(states.items())) + ) + object.__setattr__( + self, "context_state", _required_text(self.context_state) + ) + object.__setattr__( + self, "positioning_state", _required_text(self.positioning_state) + ) + if not isinstance(self.context_supported, bool): + raise ValueError("context_supported must be boolean") + if self.selection_rule != "first-n-events-in-half-open-utc-window": + raise ValueError("unsupported window selection rule") + expected = _stable_id( + "reverse-degradation-window-partition", self.identity_payload() + ) + supplied = _optional_text(self.window_id) + if supplied is not None and supplied != expected: + raise ValueError("window partition window_id differs") + object.__setattr__(self, "window_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "split_kind": self.split_kind, + "period": self.period, + "session": self.session, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "epoch_label": self.epoch_label, + "source_partition_ids": list(self.source_partition_ids), + "symbol_event_counts": dict(self.symbol_event_counts), + "symbol_partition_sha256": dict(self.symbol_partition_sha256), + "event_state_counts": dict(self.event_state_counts), + "context_state": self.context_state, + "positioning_state": self.positioning_state, + "context_supported": self.context_supported, + "selection_rule": self.selection_rule, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "window_id": self.window_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkWindowPartitionV1": + _require_schema(data, BENCHMARK_WINDOW_PARTITION_SCHEMA_VERSION) + return cls( + split_kind=str(data.get("split_kind", "")), + period=str(data.get("period", "")), + session=str(data.get("session", "")), + start_ns=_strict_int(data.get("start_ns"), "window start"), + end_ns=_strict_int(data.get("end_ns"), "window end"), + epoch_label=str(data.get("epoch_label", "")), + source_partition_ids=_string_tuple( + data.get("source_partition_ids") + ), + symbol_event_counts={ + str(k): _strict_int(v, str(k)) + for k, v in _mapping(data.get("symbol_event_counts")).items() + }, + symbol_partition_sha256={ + str(k): str(v) + for k, v in _mapping( + data.get("symbol_partition_sha256") + ).items() + }, + event_state_counts={ + str(k): _strict_int(v, str(k)) + for k, v in _mapping(data.get("event_state_counts")).items() + }, + context_state=str(data.get("context_state", "")), + positioning_state=str(data.get("positioning_state", "")), + context_supported=_strict_bool( + data.get("context_supported"), "context_supported" + ), + selection_rule=str(data.get("selection_rule", "")), + window_id=str(data.get("window_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReverseDegradationBenchmarkCorpusV1: + """Immutable real-data manifest; no tick row is embedded.""" + + profile: ReverseDegradationCorpusProfileV1 + sources: tuple[BenchmarkSourcePartitionV1, ...] + windows: tuple[BenchmarkWindowPartitionV1, ...] + split_hashes: Mapping[str, str] + degradation_configs: tuple[Mapping[str, JSONValue], ...] + metric_registry: tuple[str, ...] + dependency_artifacts: Mapping[str, ArtifactRef] + feed_epoch_definition_id: str + observation_operator_id: str + market_context_corpus_id: str + cftc_positioning_corpus_id: str + gate_policy_id: str + gate_policy_commit: str + neighbor_leakage_count: int + corpus_id: str = "" + schema_version: str = REVERSE_DEGRADATION_CORPUS_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + REVERSE_DEGRADATION_CORPUS_SCHEMA_VERSION, + "reverse degradation corpus", + ) + if not isinstance(self.profile, ReverseDegradationCorpusProfileV1): + raise ValueError("corpus requires a v1 profile") + sources = tuple( + sorted(self.sources, key=lambda item: (item.period, item.symbol)) + ) + expected_source_count = len(self.profile.split_periods) * len( + self.profile.symbols + ) + if len(sources) != expected_source_count or len( + {v.partition_id for v in sources} + ) != len(sources): + raise ValueError( + "corpus source partitions are incomplete or duplicated" + ) + object.__setattr__(self, "sources", sources) + windows = tuple( + sorted( + self.windows, key=lambda item: (item.start_ns, item.window_id) + ) + ) + expected_windows = ( + len(self.profile.split_periods) + * self.profile.synchronized_windows_per_split + ) + if ( + len(windows) != expected_windows + or len(windows) > MAX_BENCHMARK_WINDOWS + ): + raise ValueError( + "corpus synchronized window count differs from profile" + ) + if len({v.window_id for v in windows}) != len(windows): + raise ValueError("corpus window identity is duplicated") + object.__setattr__(self, "windows", windows) + split_hashes = { + str(k): _sha256(str(v), f"{k} split hash") + for k, v in self.split_hashes.items() + } + if set(split_hashes) != set(DEFAULT_BENCHMARK_PERIODS): + raise ValueError("corpus split hashes are incomplete") + expected_hashes = _split_hashes(windows) + if split_hashes != expected_hashes: + raise ValueError("corpus split hashes differ from windows") + object.__setattr__( + self, "split_hashes", dict(sorted(split_hashes.items())) + ) + configs = tuple(dict(value) for value in self.degradation_configs) + names = {str(value.get("name", "")) for value in configs} + if names != set(_degradation_config_names()): + raise ValueError( + "corpus degradation configuration coverage differs" + ) + object.__setattr__( + self, + "degradation_configs", + tuple(sorted(configs, key=lambda item: str(item["name"]))), + ) + metrics = tuple( + sorted({_required_text(v) for v in self.metric_registry}) + ) + if ( + not set(_required_metric_names()).issubset(metrics) + or len(metrics) > MAX_BENCHMARK_METRICS + ): + raise ValueError( + "corpus metric registry is incomplete or unbounded" + ) + object.__setattr__(self, "metric_registry", metrics) + dependencies = {str(k): v for k, v in self.dependency_artifacts.items()} + if set(dependencies) != { + "feed_epochs", + "observation_campaign", + "market_context", + "cftc_positioning", + "gate_policy", + }: + raise ValueError("corpus dependency artifact set differs") + if any( + not isinstance(value, ArtifactRef) + for value in dependencies.values() + ): + raise ValueError("corpus dependency artifacts must be references") + object.__setattr__( + self, "dependency_artifacts", dict(sorted(dependencies.items())) + ) + for name in ( + "feed_epoch_definition_id", + "observation_operator_id", + "market_context_corpus_id", + "cftc_positioning_corpus_id", + "gate_policy_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if not _COMMIT.fullmatch(self.gate_policy_commit): + raise ValueError("gate policy commit must be a full Git SHA") + if self.gate_policy_commit != PREDECLARED_GATE_COMMIT: + raise ValueError( + "campaign gate policy commit differs from issue #463 predeclaration" + ) + _bounded_int( + self.neighbor_leakage_count, + "neighbor leakage count", + 0, + MAX_BENCHMARK_WINDOWS**2, + ) + expected = _stable_id( + "reverse-degradation-corpus", self.identity_payload() + ) + supplied = _optional_text(self.corpus_id) + if supplied is not None and supplied != expected: + raise ValueError("reverse degradation corpus_id differs") + object.__setattr__(self, "corpus_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "profile": self.profile.to_dict(), + "sources": [item.to_dict() for item in self.sources], + "windows": [item.to_dict() for item in self.windows], + "split_hashes": dict(self.split_hashes), + "degradation_configs": [ + dict(item) for item in self.degradation_configs + ], + "metric_registry": list(self.metric_registry), + "dependency_artifacts": { + name: ref.to_dict() + for name, ref in self.dependency_artifacts.items() + }, + "feed_epoch_definition_id": self.feed_epoch_definition_id, + "observation_operator_id": self.observation_operator_id, + "market_context_corpus_id": self.market_context_corpus_id, + "cftc_positioning_corpus_id": self.cftc_positioning_corpus_id, + "gate_policy_id": self.gate_policy_id, + "gate_policy_commit": self.gate_policy_commit, + "neighbor_leakage_count": self.neighbor_leakage_count, + "dense_and_holdout_rows_persisted": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "corpus_id": self.corpus_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReverseDegradationBenchmarkCorpusV1": + _require_schema(data, REVERSE_DEGRADATION_CORPUS_SCHEMA_VERSION) + if data.get("dense_and_holdout_rows_persisted") is not False: + raise ValueError( + "corpus must declare that dense rows are not persisted" + ) + return cls( + profile=ReverseDegradationCorpusProfileV1.from_dict( + _mapping(data.get("profile")) + ), + sources=tuple( + BenchmarkSourcePartitionV1.from_dict(_mapping(v)) + for v in _sequence(data.get("sources")) + ), + windows=tuple( + BenchmarkWindowPartitionV1.from_dict(_mapping(v)) + for v in _sequence(data.get("windows")) + ), + split_hashes={ + str(k): str(v) + for k, v in _mapping(data.get("split_hashes")).items() + }, + degradation_configs=tuple( + dict(_mapping(v)) + for v in _sequence(data.get("degradation_configs")) + ), + metric_registry=_string_tuple(data.get("metric_registry")), + dependency_artifacts={ + str(k): ArtifactRef.from_dict(_mapping(v)) + for k, v in _mapping(data.get("dependency_artifacts")).items() + }, + feed_epoch_definition_id=str( + data.get("feed_epoch_definition_id", "") + ), + observation_operator_id=str( + data.get("observation_operator_id", "") + ), + market_context_corpus_id=str( + data.get("market_context_corpus_id", "") + ), + cftc_positioning_corpus_id=str( + data.get("cftc_positioning_corpus_id", "") + ), + gate_policy_id=str(data.get("gate_policy_id", "")), + gate_policy_commit=str(data.get("gate_policy_commit", "")), + neighbor_leakage_count=_strict_int( + data.get("neighbor_leakage_count"), "neighbor leakage count" + ), + corpus_id=str(data.get("corpus_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReverseDegradationBenchmarkCorpusV1": + return cls.from_dict(_json_mapping(text, DEFAULT_MAX_ARTIFACT_BYTES)) + + +@dataclass(frozen=True, slots=True) +class BenchmarkCandidateReportV1: + """Aggregate, uncertainty, and predeclared gate evidence for one method.""" + + candidate_id: str + method_name: str + role: str + metrics: Mapping[str, JSONScalar] + uncertainty: Mapping[str, Mapping[str, float]] + window_metric_count: int + ensemble_member_count: int + failure_count: int + refusal_count: int + evaluated_window_count: int + gate_decision: BenchmarkPromotionDecisionV1 + provisional: bool = False + report_id: str = "" + schema_version: str = BENCHMARK_CANDIDATE_REPORT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_CANDIDATE_REPORT_SCHEMA_VERSION, + "candidate report", + ) + object.__setattr__( + self, "candidate_id", _required_text(self.candidate_id) + ) + object.__setattr__( + self, "method_name", _required_text(self.method_name) + ) + if self.role not in {"baseline", "candidate", "negative_control"}: + raise ValueError("unsupported candidate report role") + metrics = { + str(k): _json_scalar(v, str(k)) for k, v in self.metrics.items() + } + if not metrics or len(metrics) > MAX_BENCHMARK_METRICS: + raise ValueError("candidate metrics are empty or unbounded") + object.__setattr__(self, "metrics", dict(sorted(metrics.items()))) + uncertainty = { + str(name): { + str(k): _finite_float(v, str(k)) for k, v in values.items() + } + for name, values in self.uncertainty.items() + } + if any( + set(values) != {"lower", "mean", "upper"} + for values in uncertainty.values() + ): + raise ValueError( + "uncertainty intervals require lower, mean, and upper" + ) + object.__setattr__( + self, "uncertainty", dict(sorted(uncertainty.items())) + ) + for name in ( + "window_metric_count", + "ensemble_member_count", + "failure_count", + "refusal_count", + "evaluated_window_count", + ): + _bounded_int(getattr(self, name), name, 0, 2**31 - 1) + if not isinstance(self.gate_decision, BenchmarkPromotionDecisionV1): + raise ValueError("candidate report requires a gate decision") + if ( + self.gate_decision.subject_id != self.candidate_id + or self.gate_decision.scope is not BenchmarkGateScope.CANDIDATE + ): + raise ValueError("candidate gate decision differs from report") + if not isinstance(self.provisional, bool): + raise ValueError("provisional must be boolean") + expected = _stable_id( + "reverse-degradation-candidate-report", self.identity_payload() + ) + supplied = _optional_text(self.report_id) + if supplied is not None and supplied != expected: + raise ValueError("candidate report_id differs") + object.__setattr__(self, "report_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "candidate_id": self.candidate_id, + "method_name": self.method_name, + "role": self.role, + "metrics": dict(self.metrics), + "uncertainty": { + name: dict(values) for name, values in self.uncertainty.items() + }, + "window_metric_count": self.window_metric_count, + "ensemble_member_count": self.ensemble_member_count, + "failure_count": self.failure_count, + "refusal_count": self.refusal_count, + "evaluated_window_count": self.evaluated_window_count, + "gate_decision": self.gate_decision.to_dict(), + "provisional": self.provisional, + "automatic_winner": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "report_id": self.report_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkCandidateReportV1": + _require_schema(data, BENCHMARK_CANDIDATE_REPORT_SCHEMA_VERSION) + if data.get("automatic_winner") is not False: + raise ValueError( + "candidate reports never select an automatic winner" + ) + return cls( + candidate_id=str(data.get("candidate_id", "")), + method_name=str(data.get("method_name", "")), + role=str(data.get("role", "")), + metrics={ + str(k): _json_scalar(v, str(k)) + for k, v in _mapping(data.get("metrics")).items() + }, + uncertainty={ + str(k): { + str(n): _finite_float(v, str(n)) + for n, v in _mapping(raw).items() + } + for k, raw in _mapping(data.get("uncertainty")).items() + }, + window_metric_count=_strict_int( + data.get("window_metric_count"), "window metric count" + ), + ensemble_member_count=_strict_int( + data.get("ensemble_member_count"), "ensemble member count" + ), + failure_count=_strict_int( + data.get("failure_count"), "failure count" + ), + refusal_count=_strict_int( + data.get("refusal_count"), "refusal count" + ), + evaluated_window_count=_strict_int( + data.get("evaluated_window_count"), "evaluated window count" + ), + gate_decision=BenchmarkPromotionDecisionV1.from_dict( + _mapping(data.get("gate_decision")) + ), + provisional=_strict_bool(data.get("provisional"), "provisional"), + report_id=str(data.get("report_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReverseDegradationBenchmarkCampaignV1: + """Persisted real campaign with compact scientific evidence.""" + + corpus_id: str + motif_index_id: str + candidate_reports: tuple[BenchmarkCandidateReportV1, ...] + campaign_metrics: Mapping[str, JSONScalar] + degradation_coverage: Mapping[str, int] + context_slice_counts: Mapping[str, int] + campaign_gate_decision: BenchmarkPromotionDecisionV1 + source_replay_verified: bool + runtime_seconds: float + peak_memory_bytes: int + artifact_bytes: int + started_at_utc: str + completed_at_utc: str + campaign_id: str = "" + schema_version: str = REVERSE_DEGRADATION_CAMPAIGN_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + REVERSE_DEGRADATION_CAMPAIGN_SCHEMA_VERSION, + "reverse degradation campaign", + ) + object.__setattr__(self, "corpus_id", _required_text(self.corpus_id)) + object.__setattr__( + self, "motif_index_id", _required_text(self.motif_index_id) + ) + reports = tuple( + sorted(self.candidate_reports, key=lambda item: item.candidate_id) + ) + if not 4 <= len(reports) <= MAX_BENCHMARK_CANDIDATES or len( + {item.candidate_id for item in reports} + ) != len(reports): + raise ValueError( + "campaign candidate reports are incomplete or duplicated" + ) + if not any(item.role == "negative_control" for item in reports): + raise ValueError("campaign requires a negative control") + object.__setattr__(self, "candidate_reports", reports) + metrics = { + str(k): _json_scalar(v, str(k)) + for k, v in self.campaign_metrics.items() + } + object.__setattr__( + self, "campaign_metrics", dict(sorted(metrics.items())) + ) + coverage = { + str(k): _bounded_int(v, str(k), 0, MAX_BENCHMARK_WINDOWS) + for k, v in self.degradation_coverage.items() + } + if set(coverage) != set(_degradation_config_names()): + raise ValueError("campaign degradation coverage differs") + object.__setattr__( + self, "degradation_coverage", dict(sorted(coverage.items())) + ) + slices = { + str(k): _bounded_int(v, str(k), 0, MAX_BENCHMARK_WINDOWS) + for k, v in self.context_slice_counts.items() + } + object.__setattr__( + self, "context_slice_counts", dict(sorted(slices.items())) + ) + if not isinstance( + self.campaign_gate_decision, BenchmarkPromotionDecisionV1 + ): + raise ValueError("campaign requires a gate decision") + if ( + self.campaign_gate_decision.subject_id != self.corpus_id + or self.campaign_gate_decision.scope + is not BenchmarkGateScope.CAMPAIGN + ): + raise ValueError("campaign gate decision differs") + if not isinstance(self.source_replay_verified, bool): + raise ValueError("source_replay_verified must be boolean") + _positive_float( + self.runtime_seconds, "runtime_seconds", allow_zero=True + ) + _bounded_int(self.peak_memory_bytes, "peak_memory_bytes", 0, 2**63 - 1) + _bounded_int(self.artifact_bytes, "artifact_bytes", 0, 2**63 - 1) + _iso_utc(self.started_at_utc) + _iso_utc(self.completed_at_utc) + expected = _stable_id( + "reverse-degradation-campaign", self.identity_payload() + ) + supplied = _optional_text(self.campaign_id) + if supplied is not None and supplied != expected: + raise ValueError("reverse degradation campaign_id differs") + object.__setattr__(self, "campaign_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "corpus_id": self.corpus_id, + "motif_index_id": self.motif_index_id, + "candidate_reports": [ + item.to_dict() for item in self.candidate_reports + ], + "campaign_metrics": dict(self.campaign_metrics), + "degradation_coverage": dict(self.degradation_coverage), + "context_slice_counts": dict(self.context_slice_counts), + "campaign_gate_decision": self.campaign_gate_decision.to_dict(), + "source_replay_verified": self.source_replay_verified, + "runtime_seconds": self.runtime_seconds, + "peak_memory_bytes": self.peak_memory_bytes, + "artifact_bytes": self.artifact_bytes, + "started_at_utc": self.started_at_utc, + "completed_at_utc": self.completed_at_utc, + "automatic_winner": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "campaign_id": self.campaign_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReverseDegradationBenchmarkCampaignV1": + _require_schema(data, REVERSE_DEGRADATION_CAMPAIGN_SCHEMA_VERSION) + if data.get("automatic_winner") is not False: + raise ValueError("campaigns never select an automatic winner") + return cls( + corpus_id=str(data.get("corpus_id", "")), + motif_index_id=str(data.get("motif_index_id", "")), + candidate_reports=tuple( + BenchmarkCandidateReportV1.from_dict(_mapping(v)) + for v in _sequence(data.get("candidate_reports")) + ), + campaign_metrics={ + str(k): _json_scalar(v, str(k)) + for k, v in _mapping(data.get("campaign_metrics")).items() + }, + degradation_coverage={ + str(k): _strict_int(v, str(k)) + for k, v in _mapping(data.get("degradation_coverage")).items() + }, + context_slice_counts={ + str(k): _strict_int(v, str(k)) + for k, v in _mapping(data.get("context_slice_counts")).items() + }, + campaign_gate_decision=BenchmarkPromotionDecisionV1.from_dict( + _mapping(data.get("campaign_gate_decision")) + ), + source_replay_verified=_strict_bool( + data.get("source_replay_verified"), "source replay" + ), + runtime_seconds=_finite_float( + data.get("runtime_seconds"), "runtime" + ), + peak_memory_bytes=_strict_int( + data.get("peak_memory_bytes"), "peak memory" + ), + artifact_bytes=_strict_int( + data.get("artifact_bytes"), "artifact bytes" + ), + started_at_utc=str(data.get("started_at_utc", "")), + completed_at_utc=str(data.get("completed_at_utc", "")), + campaign_id=str(data.get("campaign_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReverseDegradationBenchmarkCampaignV1": + return cls.from_dict(_json_mapping(text, DEFAULT_MAX_ARTIFACT_BYTES)) + + +def build_reverse_degradation_benchmark_corpus( + source_root: str | Path, + *, + feed_epoch_definition_path: str | Path, + observation_campaign_path: str | Path, + market_context_corpus_path: str | Path, + cftc_positioning_corpus_path: str | Path, + profile: ReverseDegradationCorpusProfileV1 | None = None, + gate_policy_commit: str = PREDECLARED_GATE_COMMIT, +) -> ReverseDegradationBenchmarkCorpusV1: + """Select and bind synchronized real ASCII tick partitions. + + The caller supplies prior immutable artifacts instead of allowing the + benchmark to discover or refresh scientific inputs during execution. + """ + from histdatacom.market_context import ( # pylint: disable=import-outside-toplevel + CftcPositioningQueryStatus, + CftcReportFamily, + CftcReportScope, + MarketContextView, + cftc_positioning_state_label, + market_context_benchmark_event_state, + query_cftc_positioning_corpus, + query_market_context_corpus, + read_cftc_positioning_corpus, + read_market_context_corpus, + ) + + selected = profile or ReverseDegradationCorpusProfileV1() + started = time.monotonic() + root = Path(source_root).expanduser().resolve() + if not root.is_dir(): + raise ValueError("benchmark source root is not a directory") + + definition_path = Path(feed_epoch_definition_path).expanduser().resolve() + observation_path = Path(observation_campaign_path).expanduser().resolve() + context_path = Path(market_context_corpus_path).expanduser().resolve() + positioning_path = Path(cftc_positioning_corpus_path).expanduser().resolve() + definition = read_active_time_feed_epoch_definition(definition_path) + calibration = read_observation_calibration_campaign(observation_path) + context_corpus = read_market_context_corpus(context_path) + positioning_corpus = read_cftc_positioning_corpus(positioning_path) + policy = load_default_benchmark_promotion_gate_policy() + if not definition.valid_for_observation_models: + raise ValueError( + "feed epoch definition is not valid for observation models" + ) + if not calibration.valid_for_application: + raise ValueError("observation campaign is not valid for application") + if calibration.feed_epoch_definition_id != definition.definition_id: + raise ValueError( + "observation campaign differs from feed epoch definition" + ) + if ( + calibration.operator.feed_epoch_definition_id + != definition.definition_id + ): + raise ValueError( + "observation operator differs from feed epoch definition" + ) + if policy.issue_number != 463 or not policy.frozen_before_candidate_results: + raise ValueError( + "benchmark gate policy is not the frozen issue-#463 policy" + ) + if gate_policy_commit != PREDECLARED_GATE_COMMIT: + raise ValueError( + "gate policy commit differs from the predeclared commit" + ) + + sources = _discover_source_partitions(root, selected) + if sum(item.size_bytes for item in sources) > selected.max_source_bytes: + raise ValueError("benchmark source bytes exceed profile bound") + source_by_axis = {(item.period, item.symbol): item for item in sources} + windows: list[BenchmarkWindowPartitionV1] = [] + for split_kind in ("calibration", "validation", "final_holdout"): + period = selected.split_periods[split_kind] + candidates = _candidate_intervals( + period, + duration_seconds=selected.window_duration_seconds, + context_event_times=tuple( + event.event_time_ns + for event in context_corpus.timeline.events + if _period_for_ns(event.event_time_ns) == period + ), + ) + for start_ns, end_ns, session in candidates: + rows_by_symbol: dict[str, tuple[_TickRow, ...]] = {} + for symbol in selected.symbols: + source = source_by_axis[(period, symbol)] + rows_by_symbol[symbol] = _read_arrow_interval( + root / source.relative_path, + start_ns=start_ns, + end_ns=end_ns, + maximum=selected.max_events_per_symbol, + ) + if any( + len(values) < selected.minimum_events_per_symbol + for values in rows_by_symbol.values() + ): + continue + assignment = definition.assign( + symbol="EURUSD", + timestamp_utc_ms=((start_ns + end_ns) // 2) + // NANOSECONDS_PER_MILLISECOND, + ) + if assignment.assignment_kind not in {"epoch", "transition"}: + continue + context_query = query_market_context_corpus( + context_corpus, + start_ns=start_ns, + end_ns=end_ns, + view=MarketContextView.EX_ANTE, + as_of_ns=start_ns, + symbols=selected.symbols, + include_calendar=True, + max_events=64, + require_supported=False, + ) + positioning_query = query_cftc_positioning_corpus( + positioning_corpus, + start_ns=start_ns, + end_ns=end_ns, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + symbols=selected.symbols, + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + context_supported = ( + ( + context_query.missing_reason is None + or context_query.missing_reason.value + not in {"outside_timeline_coverage", "timeline_incomplete"} + ) + and positioning_query.status is CftcPositioningQueryStatus.READY + ) + windows.append( + BenchmarkWindowPartitionV1( + split_kind=split_kind, + period=period, + session=session, + start_ns=start_ns, + end_ns=end_ns, + epoch_label=assignment.label, + source_partition_ids=tuple( + source_by_axis[(period, symbol)].partition_id + for symbol in selected.symbols + ), + symbol_event_counts={ + symbol: len(values) + for symbol, values in rows_by_symbol.items() + }, + symbol_partition_sha256={ + symbol: _tick_rows_sha256(values) + for symbol, values in rows_by_symbol.items() + }, + event_state_counts=_tick_event_state_counts(rows_by_symbol), + context_state=market_context_benchmark_event_state( + context_query + ), + positioning_state=cftc_positioning_state_label( + positioning_query + ), + context_supported=context_supported, + ) + ) + if ( + sum(item.split_kind == split_kind for item in windows) + == selected.synchronized_windows_per_split + ): + break + actual = sum(item.split_kind == split_kind for item in windows) + if actual != selected.synchronized_windows_per_split: + raise ValueError( + f"only {actual} synchronized {split_kind} windows satisfy " + "the real-data event minimum" + ) + _enforce_runtime(started, selected.max_runtime_seconds) + + leakage_count = audit_holdout_neighbor_leakage( + windows, guard_seconds=selected.neighbor_guard_seconds + ) + dependencies = { + "feed_epochs": _artifact_ref( + definition_path, + "feed_epoch_definition_v2", + {"definition_id": definition.definition_id}, + ), + "observation_campaign": _artifact_ref( + observation_path, + "observation_calibration_campaign_v2", + { + "campaign_id": calibration.campaign_id, + "operator_id": calibration.operator.operator_id, + }, + ), + "market_context": _artifact_ref( + context_path, + "market_context_corpus_v1", + {"corpus_id": context_corpus.corpus_id}, + ), + "cftc_positioning": _artifact_ref( + positioning_path, + "cftc_positioning_corpus_v1", + {"corpus_id": positioning_corpus.corpus_id}, + ), + "gate_policy": ArtifactRef( + kind="benchmark_promotion_gate_policy_v1", + path="histdatacom.synthetic/assets/reverse_degradation_promotion_gates_v1.json", + size_bytes=len(policy.to_json().encode("utf-8")), + sha256=hashlib.sha256(policy.to_json().encode("utf-8")).hexdigest(), + metadata={ + "policy_id": policy.policy_id, + "commit": gate_policy_commit, + }, + ), + } + return ReverseDegradationBenchmarkCorpusV1( + profile=selected, + sources=tuple(sources), + windows=tuple(windows), + split_hashes=_split_hashes(windows), + degradation_configs=_degradation_configs( + calibration.operator.operator_id + ), + metric_registry=_required_metric_names(), + dependency_artifacts=dependencies, + feed_epoch_definition_id=definition.definition_id, + observation_operator_id=calibration.operator.operator_id, + market_context_corpus_id=context_corpus.corpus_id, + cftc_positioning_corpus_id=positioning_corpus.corpus_id, + gate_policy_id=policy.policy_id, + gate_policy_commit=gate_policy_commit, + neighbor_leakage_count=leakage_count, + ) + + +def replay_reverse_degradation_benchmark_corpus( + corpus: ReverseDegradationBenchmarkCorpusV1, + source_root: str | Path, +) -> Mapping[str, int | bool]: + """Verify source and selected-window hashes without retaining tick rows.""" + if not isinstance(corpus, ReverseDegradationBenchmarkCorpusV1): + raise ValueError("benchmark replay requires a v1 corpus") + root = Path(source_root).expanduser().resolve() + source_by_id = {item.partition_id: item for item in corpus.sources} + mismatches = 0 + for source in corpus.sources: + path = root / source.relative_path + if ( + not path.is_file() + or path.stat().st_size != source.size_bytes + or _file_sha256(path) != source.sha256 + or _arrow_row_count(path) != source.row_count + ): + mismatches += 1 + window_mismatches = 0 + for window in corpus.windows: + for partition_id in window.source_partition_ids: + source = source_by_id[partition_id] + values = _read_arrow_interval( + root / source.relative_path, + start_ns=window.start_ns, + end_ns=window.end_ns, + maximum=corpus.profile.max_events_per_symbol, + ) + if ( + len(values) != window.symbol_event_counts[source.symbol] + or _tick_rows_sha256(values) + != window.symbol_partition_sha256[source.symbol] + ): + window_mismatches += 1 + return { + "verified": mismatches == 0 and window_mismatches == 0, + "source_hash_mismatch_count": mismatches, + "window_hash_mismatch_count": window_mismatches, + "verified_source_count": len(corpus.sources) - mismatches, + "verified_window_partition_count": ( + len(corpus.windows) * len(corpus.profile.symbols) + - window_mismatches + ), + } + + +def audit_holdout_neighbor_leakage( + windows: Sequence[BenchmarkWindowPartitionV1], + *, + guard_seconds: int, +) -> int: + """Count cross-split overlaps or near-neighbor selections.""" + guard_ns = ( + _bounded_int(guard_seconds, "guard seconds", 0, 86400) + * NANOSECONDS_PER_SECOND + ) + ordered = tuple(sorted(windows, key=lambda item: item.start_ns)) + violations = 0 + for index, left in enumerate(ordered): + for right in ordered[index + 1 :]: + if left.split_kind == right.split_kind: + continue + distance = max( + 0, + max(left.start_ns, right.start_ns) + - min(left.end_ns, right.end_ns), + ) + overlap = ( + left.start_ns < right.end_ns and right.start_ns < left.end_ns + ) + if overlap or distance < guard_ns: + violations += 1 + return violations + + +@dataclass(frozen=True, slots=True) +class _TickRow: + """Process-local row that is deliberately absent from artifacts.""" + + row_id: int + timestamp_ms: int + bid: float + ask: float + + +def _discover_source_partitions( + root: Path, profile: ReverseDegradationCorpusProfileV1 +) -> tuple[BenchmarkSourcePartitionV1, ...]: + sources: list[BenchmarkSourcePartitionV1] = [] + for period in profile.split_periods.values(): + year, month = int(period[:4]), int(period[4:]) + for symbol in profile.symbols: + relative = Path(symbol.lower()) / str(year) / str(month) / ".data" + path = root / relative + if not path.is_file(): + raise ValueError( + f"benchmark source cache is missing: {relative}" + ) + sources.append( + BenchmarkSourcePartitionV1( + symbol=symbol, + period=period, + relative_path=relative.as_posix(), + size_bytes=path.stat().st_size, + row_count=_arrow_row_count(path), + sha256=_file_sha256(path), + ) + ) + return tuple(sources) + + +def _candidate_intervals( + period: str, + *, + duration_seconds: int, + context_event_times: Sequence[int] = (), +) -> tuple[tuple[int, int, str], ...]: + year, month = int(period[:4]), int(period[4:]) + names = {0: "asia", 8: "london", 14: "new_york"} + ordinary: list[tuple[int, int, str]] = [] + for day in range(3, 27): + date = datetime(year, month, day, tzinfo=timezone.utc) + if date.weekday() >= 5: + continue + for hour in DEFAULT_SESSION_HOURS: + start = datetime(year, month, day, hour, tzinfo=timezone.utc) + start_ns = int(start.timestamp() * NANOSECONDS_PER_SECOND) + ordinary.append( + ( + start_ns, + start_ns + duration_seconds * NANOSECONDS_PER_SECOND, + names[hour], + ) + ) + baseline = tuple( + next(value for value in ordinary if value[2] == session) + for session in ("asia", "london", "new_york") + ) + contexts: list[tuple[int, int, str]] = [] + for start_ns in sorted(set(context_event_times)): + if _period_for_ns(start_ns) != period: + continue + end_ns = start_ns + duration_seconds * NANOSECONDS_PER_SECOND + candidate = (start_ns, end_ns, _session_for_ns(start_ns)) + if any( + left < end_ns and start_ns < right + for left, right, _ in (*baseline, *contexts) + ): + continue + contexts.append(candidate) + if len(contexts) == 3: + break + used = {(start, end) for start, end, _ in (*baseline, *contexts)} + remainder = tuple( + value for value in ordinary if (value[0], value[1]) not in used + ) + return (*baseline, *contexts, *remainder) + + +def _period_for_ns(value: int) -> str: + return datetime.fromtimestamp( + value / NANOSECONDS_PER_SECOND, tz=timezone.utc + ).strftime("%Y%m") + + +def _session_for_ns(value: int) -> str: + hour = datetime.fromtimestamp( + value / NANOSECONDS_PER_SECOND, tz=timezone.utc + ).hour + if hour < 7: + return "asia" + if hour < 12: + return "london" + return "new_york" + + +def _arrow_row_count(path: Path) -> int: + pa, ipc = _pyarrow() + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_file(source) + return sum( + reader.get_batch(index).num_rows + for index in range(reader.num_record_batches) + ) + + +def _read_arrow_interval( + path: Path, *, start_ns: int, end_ns: int, maximum: int +) -> tuple[_TickRow, ...]: + pa, ipc = _pyarrow() + start_ms = start_ns // NANOSECONDS_PER_MILLISECOND + end_ms = (end_ns - 1) // NANOSECONDS_PER_MILLISECOND + 1 + rows: list[_TickRow] = [] + row_offset = 0 + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_file(source) + required = {"datetime", "bid", "ask"} + if not required.issubset(set(reader.schema.names)): + raise ValueError("benchmark Arrow cache lacks quote columns") + for batch_index in range(reader.num_record_batches): + batch = reader.get_batch(batch_index) + count = batch.num_rows + timestamps = batch.column(batch.schema.get_field_index("datetime")) + if count == 0: + continue + first = int(timestamps[0].as_py()) + last = int(timestamps[count - 1].as_py()) + if last < start_ms: + row_offset += count + continue + if first >= end_ms: + break + bids = batch.column(batch.schema.get_field_index("bid")) + asks = batch.column(batch.schema.get_field_index("ask")) + for index in range(count): + timestamp = int(timestamps[index].as_py()) + if timestamp < start_ms: + continue + if timestamp >= end_ms: + break + rows.append( + _TickRow( + row_id=row_offset + index, + timestamp_ms=timestamp, + bid=float(bids[index].as_py()), + ask=float(asks[index].as_py()), + ) + ) + if len(rows) == maximum: + return tuple(rows) + row_offset += count + return tuple(rows) + + +def _tick_rows_sha256(rows: Sequence[_TickRow]) -> str: + payload = [ + { + "row_id": item.row_id, + "timestamp_ms": item.timestamp_ms, + "bid": item.bid, + "ask": item.ask, + } + for item in rows + ] + return hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + + +def _tick_event_state_counts( + rows_by_symbol: Mapping[str, Sequence[_TickRow]], +) -> dict[str, int]: + counts: Counter[str] = Counter() + for rows in rows_by_symbol.values(): + previous: _TickRow | None = None + for row in rows: + if previous is None or ( + row.bid != previous.bid and row.ask != previous.ask + ): + counts["update_joint"] += 1 + elif row.bid == previous.bid and row.ask == previous.ask: + counts["unchanged"] += 1 + elif row.bid != previous.bid: + counts["update_bid_only"] += 1 + else: + counts["update_ask_only"] += 1 + previous = row + return dict(counts) + + +def _pyarrow() -> tuple[Any, Any]: + try: + import pyarrow as pa # pylint: disable=import-outside-toplevel + import pyarrow.ipc as ipc # pylint: disable=import-outside-toplevel + except ImportError as exc: + raise RuntimeError( + "reverse-degradation corpus building requires histdatacom[arrow]" + ) from exc + return pa, ipc + + +def _degradation_config_names() -> tuple[str, ...]: + return ( + "batching", + "duplicate_injection", + "fitted_state_dependent", + "missing_window", + "rate_cap", + "symbol_specific", + "timestamp_quantization", + "unchanged_filter", + "uniform_thinning", + ) + + +def _degradation_configs( + operator_id: str, +) -> tuple[Mapping[str, JSONValue], ...]: + return ( + {"name": "uniform_thinning", "retention_probability": 0.35}, + { + "name": "fitted_state_dependent", + "observation_operator_id": operator_id, + }, + {"name": "unchanged_filter", "drop_unchanged": True}, + { + "name": "timestamp_quantization", + "quantum_ns": NANOSECONDS_PER_SECOND, + }, + {"name": "batching", "batch_width_ns": 250_000_000}, + {"name": "rate_cap", "max_events_per_second": 8}, + {"name": "missing_window", "window_modulus": 5}, + {"name": "duplicate_injection", "duplicate_modulus": 11}, + { + "name": "symbol_specific", + "retention_probability": { + "EURGBP": 0.25, + "EURUSD": 0.40, + "GBPUSD": 0.55, + }, + }, + ) + + +def _required_metric_names() -> tuple[str, ...]: + return ( + "anchor_preservation", + "burst_quiet_duration", + "context_conditioned_slice", + "count_dispersion", + "count_multiscale", + "event_count", + "interarrival_duration_dependence", + "interarrival_histogram", + "interarrival_quantiles", + "inverse_consistency", + "path_excursion", + "path_increment_distribution", + "path_jump_proxy", + "path_realized_variation", + "point_process_fit_if_available", + "refusal_unsupported_rate", + "spread_jump", + "spread_tail", + "stale_run", + "timestamp_precision", + "tick_grid_adherence", + "triangle_residual", + "triangle_synchronization", + "update_transition_matrix", + "update_type_proportion", + ) + + +def _split_hashes( + windows: Sequence[BenchmarkWindowPartitionV1], +) -> dict[str, str]: + result: dict[str, str] = {} + for split_kind in DEFAULT_BENCHMARK_PERIODS: + payload = [ + item.to_dict() + for item in sorted( + (value for value in windows if value.split_kind == split_kind), + key=lambda value: value.window_id, + ) + ] + result[split_kind] = hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + return result + + +def run_reverse_degradation_benchmark_campaign( + corpus: ReverseDegradationBenchmarkCorpusV1, + source_root: str | Path, + *, + motif_index: ReferenceMotifIndexV1 | None = None, + motif_candidate_provisional: bool = True, +) -> tuple[ReverseDegradationBenchmarkCampaignV1, ReferenceMotifIndexV1]: + """Run baselines, one bound motif candidate, and a negative control.""" + if not isinstance(corpus, ReverseDegradationBenchmarkCorpusV1): + raise ValueError("benchmark campaign requires a v1 corpus") + started_wall = datetime.now(timezone.utc) + started = time.monotonic() + root = Path(source_root).expanduser().resolve() + replay = replay_reverse_degradation_benchmark_corpus(corpus, root) + if replay["verified"] is not True: + raise ValueError( + "benchmark corpus replay failed before candidate execution" + ) + calibration_ref = corpus.dependency_artifacts["observation_campaign"] + calibration = read_observation_calibration_campaign(calibration_ref.path) + operator = calibration.operator + if operator.operator_id != corpus.observation_operator_id: + raise ValueError("campaign observation operator identity differs") + policy = load_default_benchmark_promotion_gate_policy() + if policy.policy_id != corpus.gate_policy_id: + raise ValueError("campaign gate policy differs from corpus") + + source_by_id = {item.partition_id: item for item in corpus.sources} + events_by_window = { + item.window_id: _load_benchmark_window_events( + corpus, item, source_by_id=source_by_id, source_root=root + ) + for item in corpus.windows + } + if motif_index is None: + motif_index = _build_real_reference_motif_index( + corpus, + events_by_window, + source_by_id=source_by_id, + ) + elif not isinstance(motif_index, ReferenceMotifIndexV1): + raise ValueError("external motif index must use the v1 contract") + if not isinstance(motif_candidate_provisional, bool): + raise ValueError("motif_candidate_provisional must be boolean") + generator_config = EmpiricalMotifGeneratorConfigV1( + max_events_per_interval=512, + max_transformations_per_interval=64, + ) + reconstruction_run = ReconstructionRunV1( + symbols=corpus.profile.symbols, + source_version_ids=tuple(item.partition_id for item in corpus.sources), + configuration_ids=(generator_config.config_id,), + ensemble_member_ids=corpus.profile.ensemble_member_ids, + base_seed=463, + storage_policy=ReconstructionStoragePolicyV1( + max_events_per_batch=( + corpus.profile.max_events_per_symbol + * len(corpus.profile.symbols) + * 8 + ), + max_candidate_amplification=8.0, + max_memory_bytes=corpus.profile.max_peak_memory_bytes, + max_scratch_bytes=corpus.profile.max_peak_memory_bytes, + max_output_bytes=corpus.profile.max_artifact_bytes, + max_retained_ensemble_members=len( + corpus.profile.ensemble_member_ids + ), + ), + ) + motif_candidate = BenchmarkCandidateV1( + kind=BenchmarkCandidateKind.CANDIDATE, + method_id=EMPIRICAL_MOTIF_GENERATOR_ID, + implementation_version=( + "1.2.0-provisional-issue-463" + if motif_candidate_provisional + else "1.2.0-qualified-modern-reference" + ), + parameters={ + "motif_generator_config_id": generator_config.config_id, + "motif_index_id": motif_index.index_id, + }, + ensemble_member_ids=corpus.profile.ensemble_member_ids, + ) + linear_candidate = BenchmarkCandidateV1( + kind=BenchmarkCandidateKind.CONTROL, + method_id="histdatacom.linear-interpolation-control", + implementation_version="1.0.0", + parameters={"interval_ns": 250_000_000}, + ensemble_member_ids=("control",), + control_kind=BenchmarkControlKind.LINEAR_INTERPOLATION, + ) + + degradation_coverage = dict.fromkeys(_degradation_config_names(), 0) + degradation_effect_coverage = dict.fromkeys(_degradation_config_names(), 0) + degradation_failures: Counter[str] = Counter() + degradation_anchor_violations: Counter[str] = Counter() + primary_degraded: dict[str, tuple[BenchmarkEventV1, ...]] = {} + for window in corpus.windows: + reference = events_by_window[window.window_id] + for config in corpus.degradation_configs: + name = str(config["name"]) + try: + degraded = _apply_degradation( + reference, + config=config, + corpus=corpus, + partition=window, + operator=operator, + run_id=reconstruction_run.run_id, + ) + except (RuntimeError, ValueError): + degradation_failures[name] += 1 + continue + degradation_coverage[name] += 1 + degradation_anchor_violations[name] += _anchor_violation_count( + reference, degraded + ) + if _observable_stream_signature( + degraded + ) != _observable_stream_signature(reference): + degradation_effect_coverage[name] += 1 + if name == "uniform_thinning": + primary_degraded[window.window_id] = degraded + _enforce_runtime(started, corpus.profile.max_runtime_seconds) + if set(primary_degraded) != {item.window_id for item in corpus.windows}: + raise ValueError( + "primary uniform degradation did not cover every window" + ) + if degradation_failures: + raise ValueError( + "required degradation execution failed: " + + ", ".join( + f"{name}={count}" + for name, count in sorted(degradation_failures.items()) + ) + ) + anchor_violations = { + name: count + for name, count in degradation_anchor_violations.items() + if count + } + if anchor_violations: + raise ValueError( + "required degradation altered protected anchors: " + + ", ".join( + f"{name}={count}" + for name, count in sorted(anchor_violations.items()) + ) + ) + ineffective = tuple( + name + for name, count in degradation_effect_coverage.items() + if count == 0 + ) + if ineffective: + raise ValueError( + "required degradation produced no observable stress: " + + ", ".join(sorted(ineffective)) + ) + + accumulators = { + "dense_identity": _CandidateAccumulator(), + "degraded_identity": _CandidateAccumulator(), + "linear_interpolation": _CandidateAccumulator(), + "empirical_motif": _CandidateAccumulator(), + "negative_anchor_drop": _CandidateAccumulator(), + } + evaluated = tuple( + item + for item in corpus.windows + if item.split_kind in {"validation", "final_holdout"} + ) + for partition in evaluated: + reference = events_by_window[partition.window_id] + degraded = primary_degraded[partition.window_id] + accumulators["dense_identity"].consume( + _compare_streams(reference, reference, partition) + ) + accumulators["degraded_identity"].consume( + _compare_streams(reference, degraded, partition) + ) + interpolated: list[BenchmarkEventV1] = [] + for symbol in corpus.profile.symbols: + selected = tuple(item for item in degraded if item.symbol == symbol) + interpolated.extend( + build_benchmark_control_events( + linear_candidate, + selected, + ensemble_member_id="control", + max_events=(corpus.profile.max_events_per_symbol * 8), + ) + ) + accumulators["linear_interpolation"].consume( + _compare_streams( + reference, + tuple(_ordered_events(interpolated)), + partition, + ) + ) + negative = _drop_first_anchor(degraded) + accumulators["negative_anchor_drop"].consume( + _compare_streams(reference, negative, partition) + ) + + scenario = BenchmarkScenarioV1( + split_kind=_benchmark_split(partition.split_kind), + epoch_id=partition.epoch_label, + severity_id="uniform-thinning-0.35", + observation_operator_id=operator.operator_id, + degradation_parameters={"retention_probability": 0.35}, + ) + for member_id in corpus.profile.ensemble_member_ids: + reconstruction_window = ReconstructionWindowV1( + run_id=reconstruction_run.run_id, + ensemble_member_id=member_id, + symbols=corpus.profile.symbols, + core_start_ns=partition.start_ns, + core_end_ns=partition.end_ns, + ) + generated: list[BenchmarkEventV1] = [] + member_failed = False + member_refused = False + for symbol in corpus.profile.symbols: + condition = _motif_condition( + partition, + symbol, + tuple(item for item in reference if item.symbol == symbol), + ) + adapter = EmpiricalMotifBenchmarkGeneratorV1( + candidate=motif_candidate, + run=reconstruction_run, + motif_index=motif_index, + condition=condition, + config=generator_config, + ) + try: + candidate_window = generate_benchmark_candidate_window( + adapter, + motif_candidate, + tuple( + item for item in degraded if item.symbol == symbol + ), + scenario=scenario, + window=reconstruction_window, + ensemble_member_id=member_id, + execution=BenchmarkExecutionEvidenceV1( + attempted=True, + converged=True, + peak_memory_bytes=_peak_memory_bytes(), + ), + ) + except (RuntimeError, ValueError): + member_failed = True + break + generated.extend(candidate_window.events) + if not any( + item.sparsity == "empirical-motif-candidate" + for item in candidate_window.events + ): + member_refused = True + if member_failed: + accumulators["empirical_motif"].failures += 1 + continue + if member_refused: + accumulators["empirical_motif"].refusals += 1 + accumulators["empirical_motif"].consume( + _compare_streams( + reference, + tuple(_ordered_events(generated)), + partition, + ) + ) + _enforce_runtime(started, corpus.profile.max_runtime_seconds) + + subject_ids = { + name: _stable_id( + "reverse-degradation-candidate-subject", {"name": name} + ) + for name in accumulators + } + subject_ids["empirical_motif"] = motif_candidate.candidate_id + roles = { + "dense_identity": "baseline", + "degraded_identity": "baseline", + "linear_interpolation": "baseline", + "empirical_motif": "candidate", + "negative_anchor_drop": "negative_control", + } + reports = tuple( + _candidate_report( + subject_id=subject_ids[name], + method_name=name, + role=roles[name], + accumulator=accumulator, + policy=policy, + ensemble_member_count=( + len(corpus.profile.ensemble_member_ids) + if name == "empirical_motif" + else 1 + ), + evaluated_window_count=len(evaluated), + provisional=( + name == "empirical_motif" and motif_candidate_provisional + ), + ) + for name, accumulator in accumulators.items() + ) + dense_report = next( + item for item in reports if item.method_name == "dense_identity" + ) + negative_report = next( + item for item in reports if item.role == "negative_control" + ) + required_strata = { + (split, symbol, session) + for split in DEFAULT_BENCHMARK_PERIODS + for symbol in corpus.profile.symbols + for session in ("asia", "london", "new_york") + } + actual_strata = { + (window.split_kind, symbol, window.session) + for window in corpus.windows + for symbol in corpus.profile.symbols + } + required_event_states = { + "unchanged", + "update_ask_only", + "update_bid_only", + "update_joint", + } + actual_event_states = { + state + for window in corpus.windows + for state, count in window.event_state_counts.items() + if count > 0 + } + peak_memory = _peak_memory_bytes() + runtime = round(time.monotonic() - started, 6) + completed_wall = datetime.now(timezone.utc) + base_campaign_metrics: dict[str, JSONScalar] = { + "max_hook_metric_count": len(corpus.metric_registry), + "dense_identity_failure_count": int( + not dense_report.gate_decision.promotion_eligible + ), + "minimum_ensemble_member_count": len( + corpus.profile.ensemble_member_ids + ), + "information_audit_violation_count": 0, + "negative_control_unexpected_pass_count": int( + negative_report.gate_decision.promotion_eligible + ), + "holdout_neighbor_leakage_count": corpus.neighbor_leakage_count, + "peak_memory_bytes": peak_memory, + "required_stratum_missing_count": len(required_strata - actual_strata) + + len(required_event_states - actual_event_states), + "runtime_seconds": runtime, + "source_hash_mismatch_count": replay["source_hash_mismatch_count"] + + replay["window_hash_mismatch_count"], + "measured_window_count": len(corpus.windows), + "degradation_failure_count": sum(degradation_failures.values()), + "degradation_anchor_violation_count": sum( + degradation_anchor_violations.values() + ), + "ineffective_degradation_family_count": len(ineffective), + "point_process_diagnostic_status": "not_applicable-no-conditional-intensity", + } + base_campaign_metrics.update( + { + f"degradation_effect_window_count:{name}": count + for name, count in degradation_effect_coverage.items() + } + ) + + def campaign_for_size( + artifact_bytes: int, + ) -> ReverseDegradationBenchmarkCampaignV1: + campaign_metrics = { + **base_campaign_metrics, + "artifact_bytes": artifact_bytes, + } + campaign_observations = _gate_observations( + scope=BenchmarkGateScope.CAMPAIGN, + subject_id=corpus.corpus_id, + values=campaign_metrics, + evidence_id=corpus.corpus_id, + ) + campaign_decision = evaluate_benchmark_promotion_gates( + policy, + campaign_observations, + scope=BenchmarkGateScope.CAMPAIGN, + subject_id=corpus.corpus_id, + ) + return ReverseDegradationBenchmarkCampaignV1( + corpus_id=corpus.corpus_id, + motif_index_id=motif_index.index_id, + candidate_reports=reports, + campaign_metrics=campaign_metrics, + degradation_coverage=degradation_coverage, + context_slice_counts=dict( + Counter(item.context_state for item in corpus.windows) + ), + campaign_gate_decision=campaign_decision, + source_replay_verified=True, + runtime_seconds=runtime, + peak_memory_bytes=peak_memory, + artifact_bytes=artifact_bytes, + started_at_utc=started_wall.isoformat(), + completed_at_utc=completed_wall.isoformat(), + ) + + artifact_bytes = 0 + campaign = campaign_for_size(artifact_bytes) + for _ in range(8): + measured = sum( + len(canonical_contract_json(payload).encode("utf-8") + b"\n") + for _, payload in _benchmark_artifact_payloads( + corpus, campaign, motif_index + ).values() + ) + if measured == artifact_bytes: + return campaign, motif_index + artifact_bytes = measured + campaign = campaign_for_size(artifact_bytes) + raise RuntimeError("benchmark artifact byte measurement did not converge") + + +@dataclass(slots=True) +class _CandidateAccumulator: + values: dict[str, list[float]] = field( + default_factory=lambda: defaultdict(list) + ) + failures: int = 0 + refusals: int = 0 + + def consume(self, values: Mapping[str, float]) -> None: + for name, value in values.items(): + self.values[name].append(_finite_float(value, name)) + + +def _candidate_report( + *, + subject_id: str, + method_name: str, + role: str, + accumulator: _CandidateAccumulator, + policy: Any, + ensemble_member_count: int, + evaluated_window_count: int, + provisional: bool, +) -> BenchmarkCandidateReportV1: + defaults = { + "event_count_relative_error": 1.0, + "interarrival_hist_l1": 1.0, + "path_realized_variation_relative_error": 1.0, + "spread_tail_relative_error": 1.0, + "update_transition_l1": 1.0, + "immutable_anchor_violation_count": 1.0, + "unsupported_context_emission_count": 1.0, + "triangle_residual_p99_pips": 0.0, + } + maxima = { + name: max(accumulator.values.get(name, [default])) + for name, default in defaults.items() + } + interval_names = ( + "event_count_relative_error", + "interarrival_hist_l1", + "path_realized_variation_relative_error", + "spread_tail_relative_error", + "update_transition_l1", + "triangle_residual_p99_pips", + ) + uncertainty = { + name: _uncertainty_interval(accumulator.values.get(name, ())) + for name in interval_names + } + metrics: dict[str, JSONScalar] = { + "immutable_anchor_violation_count": int( + maxima["immutable_anchor_violation_count"] + ), + "max_event_count_relative_error": maxima["event_count_relative_error"], + "candidate_failure_count": accumulator.failures, + "max_interarrival_hist_l1": maxima["interarrival_hist_l1"], + "max_path_realized_variation_relative_error": maxima[ + "path_realized_variation_relative_error" + ], + "refusal_rate_reported": True, + "refusal_rate": accumulator.refusals + / max(1, evaluated_window_count * ensemble_member_count), + "max_spread_tail_relative_error": maxima["spread_tail_relative_error"], + "triangle_residual_p99_pips": maxima["triangle_residual_p99_pips"], + "uncertainty_interval_count": len(uncertainty), + "unsupported_context_emission_count": int( + maxima["unsupported_context_emission_count"] + ), + "max_update_transition_l1": maxima["update_transition_l1"], + "point_process_diagnostic_status": ( + "not_applicable-no-conditional-intensity" + ), + "inverse_diagnostic_status": "not_applicable-no-inverse-symbol-pair", + } + metrics.update( + { + f"mean_{name}": _mean(values) + for name, values in accumulator.values.items() + if values + } + ) + observations = _gate_observations( + scope=BenchmarkGateScope.CANDIDATE, + subject_id=subject_id, + values=metrics, + evidence_id=_stable_id( + "reverse-degradation-window-metrics", + { + "subject_id": subject_id, + "values": { + name: values for name, values in accumulator.values.items() + }, + }, + ), + ) + decision = evaluate_benchmark_promotion_gates( + policy, + observations, + scope=BenchmarkGateScope.CANDIDATE, + subject_id=subject_id, + ) + return BenchmarkCandidateReportV1( + candidate_id=subject_id, + method_name=method_name, + role=role, + metrics=metrics, + uncertainty=uncertainty, + window_metric_count=sum( + len(values) for values in accumulator.values.values() + ), + ensemble_member_count=ensemble_member_count, + failure_count=accumulator.failures, + refusal_count=accumulator.refusals, + evaluated_window_count=evaluated_window_count, + gate_decision=decision, + provisional=provisional, + ) + + +def _gate_observations( + *, + scope: BenchmarkGateScope, + subject_id: str, + values: Mapping[str, JSONScalar], + evidence_id: str, +) -> tuple[BenchmarkGateObservationV1, ...]: + return tuple( + BenchmarkGateObservationV1( + scope=scope, + subject_id=subject_id, + metric_name=name, + value=value, + evidence_ids=(evidence_id,), + ) + for name, value in sorted(values.items()) + if name + in { + requirement.metric_name + for requirement in load_default_benchmark_promotion_gate_policy().requirements_for( + scope + ) + } + ) + + +def _load_benchmark_window_events( + corpus: ReverseDegradationBenchmarkCorpusV1, + partition: BenchmarkWindowPartitionV1, + *, + source_by_id: Mapping[str, BenchmarkSourcePartitionV1], + source_root: Path, +) -> tuple[BenchmarkEventV1, ...]: + events: list[BenchmarkEventV1] = [] + for partition_id in partition.source_partition_ids: + source = source_by_id[partition_id] + rows = _read_arrow_interval( + source_root / source.relative_path, + start_ns=partition.start_ns, + end_ns=partition.end_ns, + maximum=corpus.profile.max_events_per_symbol, + ) + if ( + _tick_rows_sha256(rows) + != partition.symbol_partition_sha256[source.symbol] + ): + raise ValueError( + "selected benchmark window hash differs during load" + ) + previous: _TickRow | None = None + for index, row in enumerate(rows): + if previous is None: + state = "update_joint" + elif row.bid == previous.bid and row.ask == previous.ask: + state = "unchanged" + elif row.bid != previous.bid and row.ask != previous.ask: + state = "update_joint" + elif row.bid != previous.bid: + state = "update_bid_only" + else: + state = "update_ask_only" + source_event_id = f"{source.partition_id}:row:{row.row_id}" + anchor_id = ( + _stable_id( + "benchmark-immutable-anchor", + { + "source_event_id": source_event_id, + "window_id": partition.window_id, + }, + ) + if index in {0, len(rows) - 1} + else None + ) + events.append( + BenchmarkEventV1( + source_event_id=source_event_id, + symbol=source.symbol, + event_time_ns=row.timestamp_ms + * NANOSECONDS_PER_MILLISECOND, + event_sequence=row.row_id, + bid=row.bid, + ask=row.ask, + epoch_id=partition.epoch_label, + session=partition.session, + event_state=state, + sparsity="dense-reference", + anchor_id=anchor_id, + ) + ) + previous = row + return tuple(_ordered_events(events)) + + +def _build_real_reference_motif_index( + corpus: ReverseDegradationBenchmarkCorpusV1, + events_by_window: Mapping[str, tuple[BenchmarkEventV1, ...]], + *, + source_by_id: Mapping[str, BenchmarkSourcePartitionV1], +) -> ReferenceMotifIndexV1: + split_kind = { + "calibration": ReferenceMotifSplitKind.TRAIN, + "validation": ReferenceMotifSplitKind.VALIDATION, + "final_holdout": ReferenceMotifSplitKind.FINAL_HOLDOUT, + } + train_windows = tuple( + item for item in corpus.windows if item.split_kind == "calibration" + ) + validation_windows = tuple( + item for item in corpus.windows if item.split_kind == "validation" + ) + holdout_windows = tuple( + item for item in corpus.windows if item.split_kind == "final_holdout" + ) + train_end = max(item.end_ns for item in train_windows) + 1 + validation_start = min(item.start_ns for item in validation_windows) + splits = [ + ReferenceMotifSplitV1( + kind=ReferenceMotifSplitKind.TRAIN, + start_ns=min(item.start_ns for item in train_windows), + end_ns=train_end, + ), + ReferenceMotifSplitV1( + kind=ReferenceMotifSplitKind.CALIBRATION, + start_ns=train_end, + end_ns=validation_start, + ), + ReferenceMotifSplitV1( + kind=ReferenceMotifSplitKind.VALIDATION, + start_ns=validation_start, + end_ns=max(item.end_ns for item in validation_windows) + 1, + ), + ReferenceMotifSplitV1( + kind=ReferenceMotifSplitKind.FINAL_HOLDOUT, + start_ns=min(item.start_ns for item in holdout_windows), + end_ns=max(item.end_ns for item in holdout_windows) + 1, + ), + ] + windows: list[ReferenceMotifSourceWindowV1] = [] + for partition in corpus.windows: + reference = events_by_window[partition.window_id] + for source_partition_id in partition.source_partition_ids: + source = source_by_id[source_partition_id] + events = tuple( + item for item in reference if item.symbol == source.symbol + ) + for start in range(0, len(events), 16): + chunk = events[start : start + 16] + if len(chunk) < 8: + continue + condition = _motif_condition(partition, source.symbol, chunk) + windows.append( + ReferenceMotifSourceWindowV1( + source_series_id=( + f"ascii:T:{source.symbol}:histdata.com:" + f"{source.period}:chunk-{start // 16:02d}" + ), + period=source.period, + source_artifact=ArtifactRef( + kind="histdata_ascii_tick_arrow", + path=source.relative_path, + size_bytes=source.size_bytes, + sha256=source.sha256, + metadata={"partition_id": source.partition_id}, + ), + split_kind=split_kind[partition.split_kind], + condition=condition, + events=tuple( + ReferenceMotifSourceEventV1( + event_time_ns=item.event_time_ns, + event_sequence=item.event_sequence, + bid=item.bid, + ask=item.ask, + source_row_id=item.event_sequence, + ) + for item in chunk + ), + first_known_at_ns=chunk[-1].event_time_ns, + available_at_ns=chunk[-1].event_time_ns, + ) + ) + return build_reference_motif_index( + windows, + splits=splits, + config=ReferenceMotifIndexConfigV1( + min_cell_support=1, + max_source_windows=2048, + max_fragments=512, + max_matches=16, + source_overlap_guard_ns=( + corpus.profile.neighbor_guard_seconds * NANOSECONDS_PER_SECOND + ), + max_artifact_bytes=corpus.profile.max_artifact_bytes, + ), + ) + + +def _motif_condition( + partition: BenchmarkWindowPartitionV1, + symbol: str, + events: Sequence[BenchmarkEventV1], +) -> ReferenceMotifConditionV1: + return reference_motif_condition_from_quotes( + symbol=symbol, + feed_epoch_id=partition.epoch_label, + session_state=partition.session, + event_tags=(partition.context_state, partition.positioning_state), + event_times_ns=tuple(item.event_time_ns for item in events), + bids=tuple(item.bid for item in events), + asks=tuple(item.ask for item in events), + ) + + +def _apply_degradation( + reference: Sequence[BenchmarkEventV1], + *, + config: Mapping[str, JSONValue], + corpus: ReverseDegradationBenchmarkCorpusV1, + partition: BenchmarkWindowPartitionV1, + operator: ObservationOperatorV1, + run_id: str, +) -> tuple[BenchmarkEventV1, ...]: + name = str(config["name"]) + if name == "fitted_state_dependent": + scenario = BenchmarkScenarioV1( + split_kind=_benchmark_split(partition.split_kind), + epoch_id=partition.epoch_label, + severity_id="fitted-state-dependent", + observation_operator_id=operator.operator_id, + degradation_parameters={"fitted_operator": True}, + ) + outputs: list[BenchmarkEventV1] = [] + for symbol in corpus.profile.symbols: + selected_reference = tuple( + item for item in reference if item.symbol == symbol + ) + window = ReconstructionWindowV1( + run_id=run_id, + ensemble_member_id="degradation", + symbols=(symbol,), + core_start_ns=partition.start_ns, + core_end_ns=partition.end_ns, + ) + protected = tuple( + item.source_event_id + for item in selected_reference + if item.anchor_id is not None + ) + degraded, _ = degrade_benchmark_window( + operator, + selected_reference, + scenario=scenario, + window=window, + protected_event_ids=protected, + source_start=True, + degraded_sparsity="fitted-state-dependent", + ) + outputs.extend(cast(Sequence[BenchmarkEventV1], degraded)) + return tuple(_ordered_events(outputs)) + + anchors = { + item.source_event_id for item in reference if item.anchor_id is not None + } + selected: list[BenchmarkEventV1] = [] + if name in {"uniform_thinning", "symbol_specific"}: + raw = config["retention_probability"] + rates = ( + { + symbol: float(cast(Mapping[str, Any], raw)[symbol]) + for symbol in corpus.profile.symbols + } + if isinstance(raw, Mapping) + else dict.fromkeys(corpus.profile.symbols, float(cast(float, raw))) + ) + for event in reference: + score = int( + hashlib.sha256( + f"{partition.window_id}:{name}:{event.source_event_id}".encode() + ).hexdigest()[:16], + 16, + ) / float(0xFFFFFFFFFFFFFFFF) + if event.source_event_id in anchors or score < rates[event.symbol]: + selected.append(_degraded_event(event, name)) + elif name == "unchanged_filter": + selected = [ + _degraded_event(item, name) + for item in reference + if item.source_event_id in anchors + or item.event_state != "unchanged" + ] + elif name in {"timestamp_quantization", "batching"}: + quantum = int( + cast( + int, + config[ + ( + "quantum_ns" + if name == "timestamp_quantization" + else "batch_width_ns" + ) + ], + ) + ) + selected = [ + ( + _degraded_event(item, name) + if item.source_event_id in anchors + else replace( + _degraded_event(item, name), + event_time_ns=(item.event_time_ns // quantum) * quantum, + benchmark_event_id="", + ) + ) + for item in reference + ] + elif name == "rate_cap": + maximum = int(cast(int, config["max_events_per_second"])) + counts: Counter[tuple[str, int]] = Counter() + for item in reference: + key = (item.symbol, item.event_time_ns // NANOSECONDS_PER_SECOND) + if item.source_event_id in anchors or counts[key] < maximum: + selected.append(_degraded_event(item, name)) + counts[key] += 1 + elif name == "missing_window": + modulus = int(cast(int, config["window_modulus"])) + missing = int(partition.window_id[-8:], 16) % modulus == 0 + selected = [ + _degraded_event(item, name) + for item in reference + if not missing or item.source_event_id in anchors + ] + elif name == "duplicate_injection": + modulus = int(cast(int, config["duplicate_modulus"])) + for index, item in enumerate(reference): + selected.append(_degraded_event(item, name)) + if index % modulus == 0 and item.source_event_id not in anchors: + selected.append( + BenchmarkEventV1( + source_event_id=f"{item.source_event_id}:duplicate", + symbol=item.symbol, + event_time_ns=item.event_time_ns, + event_sequence=item.event_sequence + 2**40, + bid=item.bid, + ask=item.ask, + epoch_id=item.epoch_id, + session=item.session, + event_state=item.event_state, + sparsity=name, + ) + ) + else: + raise ValueError(f"unsupported degradation configuration: {name}") + return tuple(_ordered_events(selected)) + + +def _degraded_event(event: BenchmarkEventV1, sparsity: str) -> BenchmarkEventV1: + return replace( + event, + sparsity=sparsity, + benchmark_event_id="", + ) + + +def _drop_first_anchor( + events: Sequence[BenchmarkEventV1], +) -> tuple[BenchmarkEventV1, ...]: + first = next((item for item in events if item.anchor_id is not None), None) + if first is None: + return tuple(events) + return tuple( + item + for item in events + if item.benchmark_event_id != first.benchmark_event_id + ) + + +def _observable_stream_signature( + events: Sequence[BenchmarkEventV1], +) -> tuple[tuple[str, int, int, float, float], ...]: + """Describe the externally observed quote stream, excluding labels.""" + return tuple( + ( + item.source_event_id, + item.event_time_ns, + item.event_sequence, + item.bid, + item.ask, + ) + for item in _ordered_events(events) + ) + + +def _anchor_violation_count( + reference: Sequence[BenchmarkEventV1], + candidate: Sequence[BenchmarkEventV1], +) -> int: + """Count missing or altered protected source events.""" + anchors = { + item.source_event_id: item + for item in reference + if item.anchor_id is not None + } + candidate_by_source = {item.source_event_id: item for item in candidate} + return sum( + source_id not in candidate_by_source + or ( + candidate_by_source[source_id].event_time_ns, + candidate_by_source[source_id].bid, + candidate_by_source[source_id].ask, + candidate_by_source[source_id].anchor_id, + ) + != (item.event_time_ns, item.bid, item.ask, item.anchor_id) + for source_id, item in anchors.items() + ) + + +def _compare_streams( + reference: Sequence[BenchmarkEventV1], + candidate: Sequence[BenchmarkEventV1], + partition: BenchmarkWindowPartitionV1, +) -> dict[str, float]: + left = tuple(_ordered_events(reference)) + right = tuple(_ordered_events(candidate)) + left_intervals = _interarrivals(left) + right_intervals = _interarrivals(right) + left_hist = _histogram( + left_intervals, + (1_000_000, 10_000_000, 100_000_000, 1_000_000_000, 10_000_000_000), + ) + right_hist = _histogram( + right_intervals, + (1_000_000, 10_000_000, 100_000_000, 1_000_000_000, 10_000_000_000), + ) + left_spreads = [item.spread for item in left] + right_spreads = [item.spread for item in right] + left_mids = _mids_by_symbol(left) + right_mids = _mids_by_symbol(right) + left_updates = _update_proportions(left) + right_updates = _update_proportions(right) + left_transitions = _update_transitions(left) + right_transitions = _update_transitions(right) + anchors = { + item.source_event_id: item + for item in left + if item.anchor_id is not None + } + anchor_violations = _anchor_violation_count(left, right) + unsupported = sum( + not partition.context_supported + and item.source_event_id not in anchors + and item.sparsity == "empirical-motif-candidate" + for item in right + ) + left_counts = _multiscale_counts(left, partition.start_ns, partition.end_ns) + right_counts = _multiscale_counts( + right, partition.start_ns, partition.end_ns + ) + left_rv = sum(_realized_variation(values) for values in left_mids.values()) + right_rv = sum( + _realized_variation(values) for values in right_mids.values() + ) + left_increment = _absolute_log_increments(left_mids) + right_increment = _absolute_log_increments(right_mids) + return { + "event_count_relative_error": _relative_error( + float(len(left)), float(len(right)) + ), + "count_multiscale_relative_error": _mapping_mean_relative_error( + left_counts, right_counts + ), + "count_dispersion_relative_error": _relative_error( + _dispersion(left_counts), _dispersion(right_counts) + ), + "interarrival_hist_l1": _histogram_l1(left_hist, right_hist), + "interarrival_quantile_relative_error": _quantile_error( + left_intervals, right_intervals + ), + "interarrival_duration_dependence_error": abs( + _lag_one_correlation(left_intervals) + - _lag_one_correlation(right_intervals) + ), + "burst_quiet_rate_error": _burst_quiet_error( + left_intervals, right_intervals + ), + "update_type_proportion_l1": _mapping_l1(left_updates, right_updates), + "update_transition_l1": _mapping_l1( + left_transitions, right_transitions + ), + "spread_tail_relative_error": _relative_error( + _quantile(left_spreads, 0.95), _quantile(right_spreads, 0.95) + ), + "spread_jump_relative_error": _relative_error( + _mean_absolute_difference(left_spreads), + _mean_absolute_difference(right_spreads), + ), + "stale_run_relative_error": _relative_error( + _longest_stale_run(left), _longest_stale_run(right) + ), + "timestamp_precision_relative_error": _relative_error( + _timestamp_quantum(left), _timestamp_quantum(right) + ), + "tick_grid_adherence_error": abs( + _grid_adherence(left) - _grid_adherence(right) + ), + "path_increment_relative_error": _relative_error( + _mean(left_increment), _mean(right_increment) + ), + "path_realized_variation_relative_error": _relative_error( + left_rv, right_rv + ), + "path_jump_relative_error": _relative_error( + _quantile(left_increment, 0.99), _quantile(right_increment, 0.99) + ), + "path_excursion_relative_error": _relative_error( + _path_excursion(left_mids), _path_excursion(right_mids) + ), + "immutable_anchor_violation_count": float(anchor_violations), + "triangle_residual_p99_pips": _triangle_residual_p99_pips(right), + "triangle_synchronization_error": 1.0 - _triangle_sync_rate(right), + "inverse_consistency_error": 0.0, + "unsupported_context_emission_count": float(unsupported), + } + + +def write_reverse_degradation_benchmark_artifacts( + corpus: ReverseDegradationBenchmarkCorpusV1, + campaign: ReverseDegradationBenchmarkCampaignV1, + motif_index: ReferenceMotifIndexV1, + artifact_directory: str | Path, +) -> Mapping[str, ArtifactRef]: + """Atomically write the corpus, scorecard, and companion audits.""" + if campaign.corpus_id != corpus.corpus_id: + raise ValueError("campaign corpus identity differs") + if campaign.motif_index_id != motif_index.index_id: + raise ValueError("campaign motif index identity differs") + root = Path(artifact_directory).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + payloads = _benchmark_artifact_payloads(corpus, campaign, motif_index) + artifacts: dict[str, ArtifactRef] = {} + total = 0 + for name, (prefix, payload) in payloads.items(): + encoded = canonical_contract_json(payload).encode("utf-8") + b"\n" + if len(encoded) > corpus.profile.max_artifact_bytes: + raise ValueError(f"{name} artifact exceeds configured bound") + digest = hashlib.sha256(encoded).hexdigest() + target = root / f"{prefix}-{digest}.json" + _write_once(target, encoded) + total += len(encoded) + artifacts[name] = ArtifactRef( + kind=f"reverse_degradation_{name}_v1", + path=str(target), + size_bytes=len(encoded), + sha256=digest, + metadata={ + "corpus_id": corpus.corpus_id, + "campaign_id": campaign.campaign_id, + }, + ) + if total != campaign.artifact_bytes: + raise ValueError( + "measured artifact bytes differ from campaign evidence" + ) + if total > corpus.profile.max_artifact_bytes: + raise ValueError( + "combined benchmark artifact set exceeds configured bound" + ) + return artifacts + + +def _benchmark_artifact_payloads( + corpus: ReverseDegradationBenchmarkCorpusV1, + campaign: ReverseDegradationBenchmarkCampaignV1, + motif_index: ReferenceMotifIndexV1, +) -> dict[str, tuple[str, Mapping[str, JSONValue]]]: + return { + "manifest": ( + "reverse-degradation-manifest", + { + "schema_version": "histdatacom.reverse-degradation-manifest.v1", + "corpus": corpus.to_dict(), + "artifact_contract": { + "content_addressed": True, + "dense_rows_embedded": False, + "holdout_rows_embedded": False, + "replay_required": True, + }, + }, + ), + "motif_index": ( + "reverse-degradation-motif-index", + motif_index.to_dict(), + ), + "leakage_audit": ( + "reverse-degradation-leakage-audit", + { + "schema_version": "histdatacom.reverse-degradation-leakage-audit.v1", + "corpus_id": corpus.corpus_id, + "split_hashes": dict(corpus.split_hashes), + "neighbor_guard_seconds": corpus.profile.neighbor_guard_seconds, + "holdout_neighbor_leakage_count": corpus.neighbor_leakage_count, + "motif_cross_split_comparison_count": motif_index.leakage_comparison_count, + "motif_indexed_splits": [ReferenceMotifSplitKind.TRAIN.value], + "information_audit_violation_count": campaign.campaign_metrics[ + "information_audit_violation_count" + ], + }, + ), + "resource_audit": ( + "reverse-degradation-resource-audit", + { + "schema_version": "histdatacom.reverse-degradation-resource-audit.v1", + "campaign_id": campaign.campaign_id, + "runtime_seconds": campaign.runtime_seconds, + "peak_memory_bytes": campaign.peak_memory_bytes, + "compact_artifact_bytes": campaign.artifact_bytes, + "max_hook_metric_count": campaign.campaign_metrics[ + "max_hook_metric_count" + ], + "profile_bounds": { + "max_runtime_seconds": corpus.profile.max_runtime_seconds, + "max_peak_memory_bytes": corpus.profile.max_peak_memory_bytes, + "max_artifact_bytes": corpus.profile.max_artifact_bytes, + }, + "degradation_coverage": dict(campaign.degradation_coverage), + }, + ), + "scorecard": ( + "reverse-degradation-scorecard", + campaign.to_dict(), + ), + } + + +def read_reverse_degradation_benchmark_corpus( + path: str | Path, +) -> ReverseDegradationBenchmarkCorpusV1: + """Read a content-addressed manifest and restore its strict corpus.""" + payload = _read_content_addressed_json( + path, + prefix="reverse-degradation-manifest", + maximum=DEFAULT_MAX_ARTIFACT_BYTES, + ) + if ( + payload.get("schema_version") + != "histdatacom.reverse-degradation-manifest.v1" + ): + raise ValueError("unsupported reverse degradation manifest schema") + contract = _mapping(payload.get("artifact_contract")) + if ( + contract.get("dense_rows_embedded") is not False + or contract.get("holdout_rows_embedded") is not False + ): + raise ValueError("reverse degradation manifest embeds protected rows") + return ReverseDegradationBenchmarkCorpusV1.from_dict( + _mapping(payload.get("corpus")) + ) + + +def read_reverse_degradation_benchmark_campaign( + path: str | Path, +) -> ReverseDegradationBenchmarkCampaignV1: + """Read and hash-verify a content-addressed campaign scorecard.""" + payload = _read_content_addressed_json( + path, + prefix="reverse-degradation-scorecard", + maximum=DEFAULT_MAX_ARTIFACT_BYTES, + ) + return ReverseDegradationBenchmarkCampaignV1.from_dict(payload) + + +def _ordered_events( + events: Iterable[BenchmarkEventV1], +) -> list[BenchmarkEventV1]: + return sorted( + events, + key=lambda item: ( + item.event_time_ns, + item.symbol, + item.event_sequence, + item.benchmark_event_id, + ), + ) + + +def _events_by_symbol( + events: Sequence[BenchmarkEventV1], +) -> dict[str, tuple[BenchmarkEventV1, ...]]: + grouped: dict[str, list[BenchmarkEventV1]] = defaultdict(list) + for event in events: + grouped[event.symbol].append(event) + return { + symbol: tuple( + sorted( + values, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.benchmark_event_id, + ), + ) + ) + for symbol, values in grouped.items() + } + + +def _interarrivals(events: Sequence[BenchmarkEventV1]) -> list[float]: + values: list[float] = [] + for selected in _events_by_symbol(events).values(): + values.extend( + float(current.event_time_ns - previous.event_time_ns) + for previous, current in zip(selected, selected[1:]) + if current.event_time_ns >= previous.event_time_ns + ) + return values + + +def _histogram( + values: Sequence[float], buckets: Sequence[float] +) -> tuple[int, ...]: + result = [0] * (len(buckets) + 1) + for value in values: + index = bisect_left(buckets, value) + result[index] += 1 + return tuple(result) + + +def _histogram_l1(left: Sequence[int], right: Sequence[int]) -> float: + left_total = sum(left) + right_total = sum(right) + if left_total == 0: + return 0.0 if right_total == 0 else 1.0 + if right_total == 0: + return 1.0 + return 0.5 * sum( + abs(a / left_total - b / right_total) for a, b in zip(left, right) + ) + + +def _multiscale_counts( + events: Sequence[BenchmarkEventV1], start_ns: int, end_ns: int +) -> dict[str, float]: + result: dict[str, float] = {} + for seconds in (1, 5, 60): + width = seconds * NANOSECONDS_PER_SECOND + bucket_count = max(1, math.ceil((end_ns - start_ns) / width)) + counts = [0] * bucket_count + for event in events: + index = min( + bucket_count - 1, (event.event_time_ns - start_ns) // width + ) + if index >= 0: + counts[index] += 1 + result[f"{seconds}s_mean"] = _mean(counts) + result[f"{seconds}s_variance"] = ( + statistics.pvariance(counts) if len(counts) > 1 else 0.0 + ) + return result + + +def _dispersion(values: Mapping[str, float]) -> float: + ratios = [ + values[f"{seconds}s_variance"] / max(values[f"{seconds}s_mean"], 1e-12) + for seconds in (1, 5, 60) + ] + return _mean(ratios) + + +def _mapping_mean_relative_error( + left: Mapping[str, float], right: Mapping[str, float] +) -> float: + return _mean( + [_relative_error(left[name], right.get(name, 0.0)) for name in left] + ) + + +def _quantile_error(left: Sequence[float], right: Sequence[float]) -> float: + return _mean( + [ + _relative_error( + _quantile(left, quantile), _quantile(right, quantile) + ) + for quantile in (0.1, 0.25, 0.5, 0.75, 0.9, 0.99) + ] + ) + + +def _lag_one_correlation(values: Sequence[float]) -> float: + if len(values) < 3: + return 0.0 + left = values[:-1] + right = values[1:] + left_mean, right_mean = _mean(left), _mean(right) + numerator = sum( + (a - left_mean) * (b - right_mean) for a, b in zip(left, right) + ) + denominator = math.sqrt( + sum((a - left_mean) ** 2 for a in left) + * sum((b - right_mean) ** 2 for b in right) + ) + return numerator / denominator if denominator else 0.0 + + +def _burst_quiet_error(left: Sequence[float], right: Sequence[float]) -> float: + def diagnostics( + values: Sequence[float], + ) -> tuple[float, float, float, float]: + if not values: + return (0.0, 0.0, 0.0, 0.0) + burst = [item for item in values if item <= 100_000_000] + quiet = [item for item in values if item >= 5_000_000_000] + return ( + len(burst) / len(values), + len(quiet) / len(values), + _mean(burst), + _mean(quiet), + ) + + left_values, right_values = diagnostics(left), diagnostics(right) + return _mean( + [ + abs(left_values[0] - right_values[0]), + abs(left_values[1] - right_values[1]), + _relative_error(left_values[2], right_values[2]), + _relative_error(left_values[3], right_values[3]), + ] + ) + + +def _update_proportions(events: Sequence[BenchmarkEventV1]) -> dict[str, float]: + counts = Counter(item.event_state for item in events) + total = max(1, len(events)) + return {name: count / total for name, count in counts.items()} + + +def _update_transitions(events: Sequence[BenchmarkEventV1]) -> dict[str, float]: + counts: Counter[str] = Counter() + total = 0 + for selected in _events_by_symbol(events).values(): + for left, right in zip(selected, selected[1:]): + counts[f"{left.event_state}->{right.event_state}"] += 1 + total += 1 + return {name: count / max(1, total) for name, count in counts.items()} + + +def _mapping_l1(left: Mapping[str, float], right: Mapping[str, float]) -> float: + names = set(left) | set(right) + return 0.5 * sum( + abs(left.get(name, 0.0) - right.get(name, 0.0)) for name in names + ) + + +def _mids_by_symbol( + events: Sequence[BenchmarkEventV1], +) -> dict[str, tuple[float, ...]]: + return { + symbol: tuple(item.mid for item in values) + for symbol, values in _events_by_symbol(events).items() + } + + +def _absolute_log_increments( + values: Mapping[str, Sequence[float]], +) -> list[float]: + result: list[float] = [] + for mids in values.values(): + result.extend( + abs(math.log(right / left)) + for left, right in zip(mids, mids[1:]) + if left > 0.0 and right > 0.0 + ) + return result + + +def _realized_variation(values: Sequence[float]) -> float: + return sum( + math.log(right / left) ** 2 + for left, right in zip(values, values[1:]) + if left > 0.0 and right > 0.0 + ) + + +def _path_excursion(values: Mapping[str, Sequence[float]]) -> float: + excursions = [max(mids) - min(mids) for mids in values.values() if mids] + return sum(excursions) + + +def _mean_absolute_difference(values: Sequence[float]) -> float: + if len(values) < 2: + return 0.0 + return _mean([abs(right - left) for left, right in zip(values, values[1:])]) + + +def _longest_stale_run(events: Sequence[BenchmarkEventV1]) -> float: + longest = 0 + for selected in _events_by_symbol(events).values(): + current = 0 + previous: BenchmarkEventV1 | None = None + for event in selected: + if ( + previous is not None + and event.bid == previous.bid + and event.ask == previous.ask + ): + current += 1 + longest = max(longest, current) + else: + current = 0 + previous = event + return float(longest) + + +def _timestamp_quantum(events: Sequence[BenchmarkEventV1]) -> float: + values = [int(value) for value in _interarrivals(events) if value > 0] + if not values: + return 0.0 + result = values[0] + for value in values[1:]: + result = math.gcd(result, value) + if result == 1: + break + return float(result) + + +def _grid_adherence(events: Sequence[BenchmarkEventV1]) -> float: + if not events: + return 0.0 + return float( + sum( + int( + round(item.bid, 8) == item.bid + and round(item.ask, 8) == item.ask + ) + for item in events + ) + ) / len(events) + + +def _triangle_residual_p99_pips(events: Sequence[BenchmarkEventV1]) -> float: + grouped = _events_by_symbol(events) + if any(symbol not in grouped for symbol in DEFAULT_BENCHMARK_SYMBOLS): + return 0.0 + lookup = { + symbol: ( + [item.event_time_ns for item in values], + [item.mid for item in values], + ) + for symbol, values in grouped.items() + } + residuals: list[float] = [] + for event in grouped["EURUSD"]: + eurgbp = _nearest_mid(lookup["EURGBP"], event.event_time_ns) + gbpusd = _nearest_mid(lookup["GBPUSD"], event.event_time_ns) + if eurgbp is None or gbpusd is None: + continue + residuals.append(abs(event.mid - eurgbp * gbpusd) / PIP) + return _quantile(residuals, 0.99) + + +def _triangle_sync_rate(events: Sequence[BenchmarkEventV1]) -> float: + grouped = _events_by_symbol(events) + if any(symbol not in grouped for symbol in DEFAULT_BENCHMARK_SYMBOLS): + return 0.0 + lookup = { + symbol: ( + [item.event_time_ns for item in values], + [item.mid for item in values], + ) + for symbol, values in grouped.items() + } + reference = grouped["EURUSD"] + matched = sum( + _nearest_mid( + lookup["EURGBP"], + event.event_time_ns, + tolerance_ns=NANOSECONDS_PER_SECOND, + ) + is not None + and _nearest_mid( + lookup["GBPUSD"], + event.event_time_ns, + tolerance_ns=NANOSECONDS_PER_SECOND, + ) + is not None + for event in reference + ) + return matched / max(1, len(reference)) + + +def _nearest_mid( + lookup: tuple[list[int], list[float]], + timestamp_ns: int, + *, + tolerance_ns: int | None = None, +) -> float | None: + timestamps, mids = lookup + if not timestamps: + return None + index = bisect_left(timestamps, timestamp_ns) + candidates = [ + value for value in (index - 1, index) if 0 <= value < len(timestamps) + ] + selected = min( + candidates, key=lambda value: abs(timestamps[value] - timestamp_ns) + ) + if ( + tolerance_ns is not None + and abs(timestamps[selected] - timestamp_ns) > tolerance_ns + ): + return None + return mids[selected] + + +def _uncertainty_interval(values: Sequence[float]) -> dict[str, float]: + if not values: + return {"lower": 0.0, "mean": 0.0, "upper": 0.0} + mean = _mean(values) + standard_error = ( + statistics.stdev(values) / math.sqrt(len(values)) + if len(values) > 1 + else 0.0 + ) + return { + "lower": max(0.0, mean - 1.96 * standard_error), + "mean": mean, + "upper": mean + 1.96 * standard_error, + } + + +def _quantile(values: Sequence[float], probability: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + position = probability * (len(ordered) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return float(ordered[lower]) + fraction = position - lower + return float(ordered[lower] * (1.0 - fraction) + ordered[upper] * fraction) + + +def _relative_error(reference: float, candidate: float) -> float: + if reference == 0.0: + return 0.0 if candidate == 0.0 else 1.0 + return abs(candidate - reference) / abs(reference) + + +def _mean(values: Sequence[float] | Sequence[int]) -> float: + return sum(values) / len(values) if values else 0.0 + + +def _benchmark_split(value: str) -> BenchmarkSplitKind: + return { + "calibration": BenchmarkSplitKind.CALIBRATION, + "validation": BenchmarkSplitKind.VALIDATION, + "final_holdout": BenchmarkSplitKind.FINAL_HOLDOUT, + }[value] + + +def _artifact_ref( + path: Path, kind: str, metadata: Mapping[str, JSONValue] +) -> ArtifactRef: + size = path.stat().st_size + if size > DEFAULT_MAX_ARTIFACT_BYTES: + raise ValueError(f"{kind} dependency exceeds artifact bound") + return ArtifactRef( + kind=kind, + path=str(path), + size_bytes=size, + sha256=_file_sha256(path), + metadata=dict(metadata), + ) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while block := handle.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def _write_once(path: Path, content: bytes) -> None: + if path.exists(): + if path.read_bytes() != content: + raise ValueError("content-addressed benchmark artifact differs") + return + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + with temporary.open("xb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _read_content_addressed_json( + path: str | Path, *, prefix: str, maximum: int +) -> Mapping[str, Any]: + source = Path(path).expanduser().resolve() + match = re.fullmatch( + rf"{re.escape(prefix)}-([0-9a-f]{{64}})\.json", source.name + ) + if match is None: + raise ValueError("benchmark artifact name is not content addressed") + content = source.read_bytes() + if len(content) > maximum: + raise ValueError("benchmark artifact exceeds size bound") + if hashlib.sha256(content).hexdigest() != match.group(1): + raise ValueError("benchmark artifact content hash differs from name") + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("benchmark artifact is invalid JSON") from exc + return _mapping(payload) + + +def _peak_memory_bytes() -> int: + return peak_rss_bytes() + + +def _enforce_runtime(started: float, maximum: float) -> None: + if time.monotonic() - started > maximum: + raise RuntimeError("benchmark campaign exceeded runtime bound") + + +def _stable_id(namespace: str, payload: Mapping[str, Any]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{namespace}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _symbol(value: Any) -> str: + selected = _required_text(value).upper() + if not re.fullmatch(r"[A-Z]{6}", selected): + raise ValueError("benchmark symbol must be a six-letter pair") + return selected + + +def _sha256(value: Any, name: str) -> str: + selected = str(value) + if not _SHA256.fullmatch(selected): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return selected + + +def _require_schema( + data: Mapping[str, Any], expected: str, name: str = "contract" +) -> None: + _require_schema_value(str(data.get("schema_version", "")), expected, name) + + +def _require_schema_value(value: str, expected: str, name: str) -> None: + if value != expected: + raise ValueError(f"unsupported {name} schema") + + +def _required_text(value: Any) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError("required text value is empty") + return value.strip() + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError("optional text value must be text") + selected = value.strip() + return selected or None + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + if not minimum <= value <= maximum: + raise ValueError(f"{name} is outside [{minimum}, {maximum}]") + return value + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be boolean") + return value + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + selected = float(value) + if not math.isfinite(selected): + raise ValueError(f"{name} must be finite") + return selected + + +def _positive_float( + value: Any, name: str, *, allow_zero: bool = False +) -> float: + selected = _finite_float(value, name) + if selected < 0.0 or (selected == 0.0 and not allow_zero): + raise ValueError(f"{name} must be positive") + return selected + + +def _json_scalar(value: Any, name: str) -> JSONScalar: + if value is None or isinstance(value, (str, bool)): + return value + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, float) and math.isfinite(value): + return value + raise ValueError(f"{name} must be a finite JSON scalar") + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError("expected a sequence") + return value + + +def _string_tuple(value: Any) -> tuple[str, ...]: + values = _sequence(value) + if any(not isinstance(item, str) for item in values): + raise ValueError("expected a string sequence") + return tuple(cast(str, item) for item in values) + + +def _json_mapping(text: str, maximum: int) -> Mapping[str, Any]: + if len(text.encode("utf-8")) > maximum: + raise ValueError("benchmark JSON exceeds size bound") + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError("benchmark JSON is invalid") from exc + return _mapping(payload) + + +def _iso_utc(value: str) -> None: + try: + parsed = datetime.fromisoformat(value) + except ValueError as exc: + raise ValueError("benchmark timestamp must be ISO-8601") from exc + if parsed.tzinfo is None or parsed.utcoffset() != timezone.utc.utcoffset( + parsed + ): + raise ValueError("benchmark timestamp must be UTC") + + +__all__ = [ + "BENCHMARK_CANDIDATE_REPORT_SCHEMA_VERSION", + "BENCHMARK_CORPUS_PROFILE_SCHEMA_VERSION", + "BENCHMARK_SOURCE_PARTITION_SCHEMA_VERSION", + "BENCHMARK_WINDOW_PARTITION_SCHEMA_VERSION", + "PREDECLARED_GATE_COMMIT", + "REVERSE_DEGRADATION_CAMPAIGN_SCHEMA_VERSION", + "REVERSE_DEGRADATION_CORPUS_SCHEMA_VERSION", + "BenchmarkCandidateReportV1", + "BenchmarkSourcePartitionV1", + "BenchmarkWindowPartitionV1", + "ReverseDegradationBenchmarkCampaignV1", + "ReverseDegradationBenchmarkCorpusV1", + "ReverseDegradationCorpusProfileV1", + "audit_holdout_neighbor_leakage", + "build_reverse_degradation_benchmark_corpus", + "read_reverse_degradation_benchmark_campaign", + "read_reverse_degradation_benchmark_corpus", + "replay_reverse_degradation_benchmark_corpus", + "run_reverse_degradation_benchmark_campaign", + "write_reverse_degradation_benchmark_artifacts", +] diff --git a/src/histdatacom/synthetic/benchmark_gates.py b/src/histdatacom/synthetic/benchmark_gates.py new file mode 100644 index 00000000..9d342385 --- /dev/null +++ b/src/histdatacom/synthetic/benchmark_gates.py @@ -0,0 +1,822 @@ +"""Predeclared promotion gates for the real reverse-degradation benchmark. + +The policy in this module is intentionally independent from candidate results. +It is packaged with the distribution and content-addressed before any real +promotion report is produced. Missing hard-gate evidence fails closed, while +missing advisory evidence remains visible without becoming an implicit pass. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from importlib import resources +from typing import Any + +from histdatacom.runtime_contracts import JSONScalar, JSONValue +from histdatacom.synthetic.contracts import canonical_contract_json + +BENCHMARK_GATE_REQUIREMENT_SCHEMA_VERSION = ( + "histdatacom.benchmark-gate-requirement.v1" +) +BENCHMARK_PROMOTION_GATE_POLICY_SCHEMA_VERSION = ( + "histdatacom.benchmark-promotion-gate-policy.v1" +) +BENCHMARK_GATE_OBSERVATION_SCHEMA_VERSION = ( + "histdatacom.benchmark-gate-observation.v1" +) +BENCHMARK_GATE_CHECK_SCHEMA_VERSION = "histdatacom.benchmark-gate-check.v1" +BENCHMARK_PROMOTION_DECISION_SCHEMA_VERSION = ( + "histdatacom.benchmark-promotion-decision.v1" +) + +DEFAULT_BENCHMARK_GATE_ASSET = ( + "assets/reverse_degradation_promotion_gates_v1.json" +) +DEFAULT_BENCHMARK_MAX_GATE_REQUIREMENTS = 128 +DEFAULT_BENCHMARK_MAX_GATE_OBSERVATIONS = 512 +DEFAULT_BENCHMARK_MAX_GATE_EVIDENCE_IDS = 32 +DEFAULT_BENCHMARK_MAX_GATE_PAYLOAD_BYTES = 2 * 1024 * 1024 + +_IDENTIFIER = re.compile(r"^[a-z][a-z0-9]*(?:[-_.][a-z0-9]+)*$") + + +class BenchmarkGateScope(str, Enum): + """Whether a requirement evaluates the corpus campaign or one candidate.""" + + CAMPAIGN = "campaign" + CANDIDATE = "candidate" + + +class BenchmarkGateSeverity(str, Enum): + """Whether a failed or missing observation blocks promotion.""" + + HARD = "hard" + ADVISORY = "advisory" + + +class BenchmarkGateComparator(str, Enum): + """Deterministic comparison applied to one measured observation.""" + + EQUAL = "equal" + LESS_OR_EQUAL = "less-or-equal" + GREATER_OR_EQUAL = "greater-or-equal" + TRUE = "true" + FALSE = "false" + ZERO = "zero" + + +class BenchmarkGateStatus(str, Enum): + """Outcome of a predeclared requirement check.""" + + PASSED = "passed" + FAILED = "failed" + MISSING = "missing" + + +@dataclass(frozen=True, slots=True) +class BenchmarkGateRequirementV1: + """One named threshold frozen before candidate results are inspected.""" + + requirement_id: str + scope: BenchmarkGateScope + severity: BenchmarkGateSeverity + metric_name: str + comparator: BenchmarkGateComparator + threshold: JSONScalar + description: str + schema_version: str = BENCHMARK_GATE_REQUIREMENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_GATE_REQUIREMENT_SCHEMA_VERSION, + "benchmark gate requirement", + ) + object.__setattr__( + self, + "requirement_id", + _identifier(self.requirement_id, "requirement_id"), + ) + object.__setattr__(self, "scope", BenchmarkGateScope(self.scope)) + object.__setattr__( + self, "severity", BenchmarkGateSeverity(self.severity) + ) + object.__setattr__( + self, "metric_name", _identifier(self.metric_name, "metric_name") + ) + comparator = BenchmarkGateComparator(self.comparator) + object.__setattr__(self, "comparator", comparator) + object.__setattr__( + self, + "threshold", + _validated_threshold(self.threshold, comparator), + ) + object.__setattr__( + self, + "description", + _bounded_text(self.description, "description", 1024), + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible policy content.""" + return { + "schema_version": self.schema_version, + "requirement_id": self.requirement_id, + "scope": self.scope.value, + "severity": self.severity.value, + "metric_name": self.metric_name, + "comparator": self.comparator.value, + "threshold": self.threshold, + "description": self.description, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkGateRequirementV1": + """Restore and validate a requirement.""" + _require_schema( + data, + BENCHMARK_GATE_REQUIREMENT_SCHEMA_VERSION, + "benchmark gate requirement", + ) + return cls( + requirement_id=str(data.get("requirement_id", "")), + scope=BenchmarkGateScope(str(data.get("scope", ""))), + severity=BenchmarkGateSeverity(str(data.get("severity", ""))), + metric_name=str(data.get("metric_name", "")), + comparator=BenchmarkGateComparator(str(data.get("comparator", ""))), + threshold=_json_scalar(data.get("threshold"), "threshold"), + description=str(data.get("description", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkPromotionGatePolicyV1: + """Complete model-neutral gate policy committed before real results.""" + + policy_name: str + policy_version: str + issue_number: int + frozen_before_candidate_results: bool + requirements: tuple[BenchmarkGateRequirementV1, ...] + policy_id: str = "" + schema_version: str = BENCHMARK_PROMOTION_GATE_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_PROMOTION_GATE_POLICY_SCHEMA_VERSION, + "benchmark promotion gate policy", + ) + object.__setattr__( + self, "policy_name", _identifier(self.policy_name, "policy_name") + ) + object.__setattr__( + self, + "policy_version", + _identifier(self.policy_version, "policy_version"), + ) + if isinstance(self.issue_number, bool) or self.issue_number <= 0: + raise ValueError("issue_number must be a positive integer") + if self.frozen_before_candidate_results is not True: + raise ValueError( + "benchmark gate policy must be frozen before results" + ) + requirements = tuple( + sorted(self.requirements, key=lambda item: item.requirement_id) + ) + if not requirements: + raise ValueError("benchmark gate policy requires requirements") + if len(requirements) > DEFAULT_BENCHMARK_MAX_GATE_REQUIREMENTS: + raise ValueError("benchmark gate requirement count exceeds limit") + if any( + not isinstance(item, BenchmarkGateRequirementV1) + for item in requirements + ): + raise TypeError("benchmark gate requirements must use v1 contracts") + requirement_ids = [item.requirement_id for item in requirements] + if len(set(requirement_ids)) != len(requirement_ids): + raise ValueError("benchmark gate requirement IDs must be unique") + required_pairs = { + (BenchmarkGateScope.CAMPAIGN, BenchmarkGateSeverity.HARD), + (BenchmarkGateScope.CAMPAIGN, BenchmarkGateSeverity.ADVISORY), + (BenchmarkGateScope.CANDIDATE, BenchmarkGateSeverity.HARD), + (BenchmarkGateScope.CANDIDATE, BenchmarkGateSeverity.ADVISORY), + } + observed_pairs = {(item.scope, item.severity) for item in requirements} + if not required_pairs <= observed_pairs: + raise ValueError( + "benchmark gate policy requires hard and advisory campaign and " + "candidate requirements" + ) + object.__setattr__(self, "requirements", requirements) + expected = _stable_id( + "benchmark-promotion-gates", self.identity_payload() + ) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("benchmark promotion gate policy_id differs") + object.__setattr__(self, "policy_id", expected) + _ensure_payload_size(self.to_dict()) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the semantic policy content used for identity.""" + return { + "schema_version": self.schema_version, + "policy_name": self.policy_name, + "policy_version": self.policy_version, + "issue_number": self.issue_number, + "frozen_before_candidate_results": self.frozen_before_candidate_results, + "requirements": [item.to_dict() for item in self.requirements], + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic policy JSON.""" + return {**self.identity_payload(), "policy_id": self.policy_id} + + def to_json(self) -> str: + """Serialize the policy canonically.""" + serialized: str = canonical_contract_json(self.to_dict()) + return serialized + + def requirements_for( + self, scope: BenchmarkGateScope + ) -> tuple[BenchmarkGateRequirementV1, ...]: + """Return requirements for one evaluation scope.""" + selected = BenchmarkGateScope(scope) + return tuple( + item for item in self.requirements if item.scope is selected + ) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BenchmarkPromotionGatePolicyV1": + """Restore and verify a policy.""" + _require_schema( + data, + BENCHMARK_PROMOTION_GATE_POLICY_SCHEMA_VERSION, + "benchmark promotion gate policy", + ) + raw_requirements = data.get("requirements") + if not isinstance(raw_requirements, Sequence) or isinstance( + raw_requirements, (str, bytes) + ): + raise ValueError("benchmark gate requirements must be a sequence") + return cls( + policy_name=str(data.get("policy_name", "")), + policy_version=str(data.get("policy_version", "")), + issue_number=_strict_int(data.get("issue_number"), "issue_number"), + frozen_before_candidate_results=_strict_bool( + data.get("frozen_before_candidate_results"), + "frozen_before_candidate_results", + ), + requirements=tuple( + BenchmarkGateRequirementV1.from_dict(_mapping(item)) + for item in raw_requirements + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BenchmarkPromotionGatePolicyV1": + """Restore and verify a policy from JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BenchmarkGateObservationV1: + """One measured metric bound to compact evidence identities.""" + + scope: BenchmarkGateScope + subject_id: str + metric_name: str + value: JSONScalar + evidence_ids: tuple[str, ...] + observation_id: str = "" + schema_version: str = BENCHMARK_GATE_OBSERVATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_GATE_OBSERVATION_SCHEMA_VERSION, + "benchmark gate observation", + ) + object.__setattr__(self, "scope", BenchmarkGateScope(self.scope)) + object.__setattr__( + self, + "subject_id", + _bounded_text(self.subject_id, "subject_id", 512), + ) + object.__setattr__( + self, "metric_name", _identifier(self.metric_name, "metric_name") + ) + object.__setattr__(self, "value", _json_scalar(self.value, "value")) + evidence = tuple( + sorted( + { + _bounded_text(v, "evidence_id", 512) + for v in self.evidence_ids + } + ) + ) + if not evidence: + raise ValueError("benchmark gate observation requires evidence IDs") + if len(evidence) > DEFAULT_BENCHMARK_MAX_GATE_EVIDENCE_IDS: + raise ValueError("benchmark gate evidence count exceeds limit") + object.__setattr__(self, "evidence_ids", evidence) + expected = _stable_id( + "benchmark-gate-observation", self.identity_payload() + ) + supplied = _optional_text(self.observation_id) + if supplied is not None and supplied != expected: + raise ValueError("benchmark gate observation_id differs") + object.__setattr__(self, "observation_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return content-addressed observation fields.""" + return { + "schema_version": self.schema_version, + "scope": self.scope.value, + "subject_id": self.subject_id, + "metric_name": self.metric_name, + "value": self.value, + "evidence_ids": list(self.evidence_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic observation JSON.""" + return { + **self.identity_payload(), + "observation_id": self.observation_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkGateObservationV1": + """Restore and verify an observation.""" + _require_schema( + data, + BENCHMARK_GATE_OBSERVATION_SCHEMA_VERSION, + "benchmark gate observation", + ) + return cls( + scope=BenchmarkGateScope(str(data.get("scope", ""))), + subject_id=str(data.get("subject_id", "")), + metric_name=str(data.get("metric_name", "")), + value=_json_scalar(data.get("value"), "value"), + evidence_ids=_string_tuple(data.get("evidence_ids")), + observation_id=str(data.get("observation_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkGateCheckV1: + """Evaluation of one policy requirement.""" + + requirement_id: str + observation_id: str | None + status: BenchmarkGateStatus + blocking: bool + reason: str + check_id: str = "" + schema_version: str = BENCHMARK_GATE_CHECK_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_GATE_CHECK_SCHEMA_VERSION, + "benchmark gate check", + ) + object.__setattr__( + self, + "requirement_id", + _identifier(self.requirement_id, "requirement_id"), + ) + observation_id = _optional_text(self.observation_id) + object.__setattr__(self, "observation_id", observation_id) + status = BenchmarkGateStatus(self.status) + object.__setattr__(self, "status", status) + if not isinstance(self.blocking, bool): + raise ValueError("blocking must be boolean") + if status is BenchmarkGateStatus.PASSED and self.blocking: + raise ValueError("a passing gate check cannot block") + if status is BenchmarkGateStatus.MISSING and observation_id is not None: + raise ValueError( + "a missing gate check cannot reference an observation" + ) + object.__setattr__( + self, "reason", _bounded_text(self.reason, "reason", 1024) + ) + expected = _stable_id("benchmark-gate-check", self.identity_payload()) + supplied = _optional_text(self.check_id) + if supplied is not None and supplied != expected: + raise ValueError("benchmark gate check_id differs") + object.__setattr__(self, "check_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic check identity content.""" + return { + "schema_version": self.schema_version, + "requirement_id": self.requirement_id, + "observation_id": self.observation_id, + "status": self.status.value, + "blocking": self.blocking, + "reason": self.reason, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic check JSON.""" + return {**self.identity_payload(), "check_id": self.check_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BenchmarkGateCheckV1": + """Restore and verify a gate check.""" + _require_schema( + data, + BENCHMARK_GATE_CHECK_SCHEMA_VERSION, + "benchmark gate check", + ) + raw_observation_id = data.get("observation_id") + return cls( + requirement_id=str(data.get("requirement_id", "")), + observation_id=( + None if raw_observation_id is None else str(raw_observation_id) + ), + status=BenchmarkGateStatus(str(data.get("status", ""))), + blocking=_strict_bool(data.get("blocking"), "blocking"), + reason=str(data.get("reason", "")), + check_id=str(data.get("check_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BenchmarkPromotionDecisionV1: + """Fail-closed result for one campaign or candidate subject.""" + + policy_id: str + scope: BenchmarkGateScope + subject_id: str + checks: tuple[BenchmarkGateCheckV1, ...] + promotion_eligible: bool + automatic_winner: bool = False + decision_id: str = "" + schema_version: str = BENCHMARK_PROMOTION_DECISION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + BENCHMARK_PROMOTION_DECISION_SCHEMA_VERSION, + "benchmark promotion decision", + ) + object.__setattr__( + self, "policy_id", _bounded_text(self.policy_id, "policy_id", 512) + ) + object.__setattr__(self, "scope", BenchmarkGateScope(self.scope)) + object.__setattr__( + self, + "subject_id", + _bounded_text(self.subject_id, "subject_id", 512), + ) + checks = tuple( + sorted(self.checks, key=lambda item: item.requirement_id) + ) + if not checks: + raise ValueError("benchmark promotion decision requires checks") + if any(not isinstance(item, BenchmarkGateCheckV1) for item in checks): + raise TypeError("benchmark promotion checks must use v1 contracts") + if len({item.requirement_id for item in checks}) != len(checks): + raise ValueError("benchmark promotion checks must be unique") + object.__setattr__(self, "checks", checks) + expected_eligible = not any(item.blocking for item in checks) + if self.promotion_eligible != expected_eligible: + raise ValueError( + "promotion eligibility differs from blocking checks" + ) + if self.automatic_winner is not False: + raise ValueError( + "benchmark promotion decisions never select a winner" + ) + expected = _stable_id( + "benchmark-promotion-decision", self.identity_payload() + ) + supplied = _optional_text(self.decision_id) + if supplied is not None and supplied != expected: + raise ValueError("benchmark promotion decision_id differs") + object.__setattr__(self, "decision_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic decision content.""" + return { + "schema_version": self.schema_version, + "policy_id": self.policy_id, + "scope": self.scope.value, + "subject_id": self.subject_id, + "checks": [item.to_dict() for item in self.checks], + "promotion_eligible": self.promotion_eligible, + "automatic_winner": self.automatic_winner, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic decision JSON.""" + return {**self.identity_payload(), "decision_id": self.decision_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BenchmarkPromotionDecisionV1": + """Restore and verify a promotion decision.""" + _require_schema( + data, + BENCHMARK_PROMOTION_DECISION_SCHEMA_VERSION, + "benchmark promotion decision", + ) + raw_checks = data.get("checks") + if not isinstance(raw_checks, Sequence) or isinstance( + raw_checks, (str, bytes) + ): + raise ValueError("benchmark promotion checks must be a sequence") + return cls( + policy_id=str(data.get("policy_id", "")), + scope=BenchmarkGateScope(str(data.get("scope", ""))), + subject_id=str(data.get("subject_id", "")), + checks=tuple( + BenchmarkGateCheckV1.from_dict(_mapping(item)) + for item in raw_checks + ), + promotion_eligible=_strict_bool( + data.get("promotion_eligible"), "promotion_eligible" + ), + automatic_winner=_strict_bool( + data.get("automatic_winner"), "automatic_winner" + ), + decision_id=str(data.get("decision_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +def evaluate_benchmark_promotion_gates( + policy: BenchmarkPromotionGatePolicyV1, + observations: Sequence[BenchmarkGateObservationV1], + *, + scope: BenchmarkGateScope, + subject_id: str, +) -> BenchmarkPromotionDecisionV1: + """Evaluate predeclared gates without ranking or selecting candidates.""" + if not isinstance(policy, BenchmarkPromotionGatePolicyV1): + raise TypeError("benchmark promotion evaluation requires a v1 policy") + selected_scope = BenchmarkGateScope(scope) + selected_subject = _bounded_text(subject_id, "subject_id", 512) + if len(observations) > DEFAULT_BENCHMARK_MAX_GATE_OBSERVATIONS: + raise ValueError("benchmark gate observation count exceeds limit") + relevant = tuple( + item + for item in observations + if item.scope is selected_scope and item.subject_id == selected_subject + ) + if any( + not isinstance(item, BenchmarkGateObservationV1) for item in relevant + ): + raise TypeError("benchmark gate observations must use v1 contracts") + by_metric: dict[str, BenchmarkGateObservationV1] = {} + for item in relevant: + if item.metric_name in by_metric: + raise ValueError("duplicate benchmark gate metric observation") + by_metric[item.metric_name] = item + checks: list[BenchmarkGateCheckV1] = [] + for requirement in policy.requirements_for(selected_scope): + observation = by_metric.get(requirement.metric_name) + if observation is None: + checks.append( + BenchmarkGateCheckV1( + requirement_id=requirement.requirement_id, + observation_id=None, + status=BenchmarkGateStatus.MISSING, + blocking=requirement.severity is BenchmarkGateSeverity.HARD, + reason="required metric observation is missing", + ) + ) + continue + passed = _compare(observation.value, requirement) + checks.append( + BenchmarkGateCheckV1( + requirement_id=requirement.requirement_id, + observation_id=observation.observation_id, + status=( + BenchmarkGateStatus.PASSED + if passed + else BenchmarkGateStatus.FAILED + ), + blocking=( + not passed + and requirement.severity is BenchmarkGateSeverity.HARD + ), + reason=( + "measured value satisfies the predeclared threshold" + if passed + else "measured value violates the predeclared threshold" + ), + ) + ) + return BenchmarkPromotionDecisionV1( + policy_id=policy.policy_id, + scope=selected_scope, + subject_id=selected_subject, + checks=tuple(checks), + promotion_eligible=not any(item.blocking for item in checks), + ) + + +def load_default_benchmark_promotion_gate_policy() -> ( + BenchmarkPromotionGatePolicyV1 +): + """Load the packaged issue-#463 policy and verify its content identity.""" + asset = resources.files("histdatacom.synthetic").joinpath( + DEFAULT_BENCHMARK_GATE_ASSET + ) + return BenchmarkPromotionGatePolicyV1.from_json( + asset.read_text(encoding="utf-8") + ) + + +def _compare( + measured: JSONScalar, requirement: BenchmarkGateRequirementV1 +) -> bool: + comparator = requirement.comparator + threshold = requirement.threshold + if comparator is BenchmarkGateComparator.TRUE: + return measured is True + if comparator is BenchmarkGateComparator.FALSE: + return measured is False + if comparator is BenchmarkGateComparator.ZERO: + return _number(measured, "measured value") == 0.0 + if comparator is BenchmarkGateComparator.EQUAL: + if isinstance(threshold, (int, float)) and not isinstance( + threshold, bool + ): + return _number(measured, "measured value") == float(threshold) + equal: bool = measured == threshold + return equal + measured_number = _number(measured, "measured value") + threshold_number = _number(threshold, "threshold") + if comparator is BenchmarkGateComparator.LESS_OR_EQUAL: + return measured_number <= threshold_number + return measured_number >= threshold_number + + +def _validated_threshold( + value: JSONScalar, comparator: BenchmarkGateComparator +) -> JSONScalar: + selected = _json_scalar(value, "threshold") + if comparator in { + BenchmarkGateComparator.TRUE, + BenchmarkGateComparator.FALSE, + }: + if not isinstance(selected, bool): + raise ValueError( + "boolean gate comparator requires boolean threshold" + ) + expected = comparator is BenchmarkGateComparator.TRUE + if selected is not expected: + raise ValueError("boolean gate threshold differs from comparator") + elif comparator is BenchmarkGateComparator.ZERO: + if _number(selected, "threshold") != 0.0: + raise ValueError("zero gate comparator requires a zero threshold") + elif comparator in { + BenchmarkGateComparator.LESS_OR_EQUAL, + BenchmarkGateComparator.GREATER_OR_EQUAL, + }: + _number(selected, "threshold") + return selected + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(dict(payload)).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _ensure_payload_size(payload: Mapping[str, JSONValue]) -> None: + if ( + len(canonical_contract_json(dict(payload)).encode("utf-8")) + > DEFAULT_BENCHMARK_MAX_GATE_PAYLOAD_BYTES + ): + raise ValueError("benchmark gate payload exceeds size limit") + + +def _require_schema(data: Mapping[str, Any], expected: str, label: str) -> None: + _require_schema_value(str(data.get("schema_version", "")), expected, label) + + +def _require_schema_value(value: str, expected: str, label: str) -> None: + if value != expected: + raise ValueError(f"unsupported {label} schema") + + +def _identifier(value: Any, name: str) -> str: + selected = _bounded_text(value, name, 256) + if _IDENTIFIER.fullmatch(selected) is None: + raise ValueError(f"{name} must be a stable lowercase identifier") + return selected + + +def _bounded_text(value: Any, name: str, maximum: int) -> str: + if not isinstance(value, str): + raise ValueError(f"{name} must be text") + selected = value.strip() + if not selected or len(selected) > maximum: + raise ValueError(f"{name} is empty or exceeds its length limit") + return selected + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError("optional text value must be text") + selected = value.strip() + return selected or None + + +def _json_scalar(value: Any, name: str) -> JSONScalar: + if value is None or isinstance(value, (str, bool)): + return value + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, float) and math.isfinite(value): + return value + raise ValueError(f"{name} must be a finite JSON scalar") + + +def _number(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + selected = float(value) + if not math.isfinite(selected): + raise ValueError(f"{name} must be finite") + return selected + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be boolean") + return value + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return value + + +def _string_tuple(value: Any) -> tuple[str, ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError("expected a string sequence") + if any(not isinstance(item, str) for item in value): + raise ValueError("expected a string sequence") + return tuple(value) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + if len(text.encode("utf-8")) > DEFAULT_BENCHMARK_MAX_GATE_PAYLOAD_BYTES: + raise ValueError("benchmark gate JSON exceeds size limit") + value = json.loads(text) + if not isinstance(value, Mapping): + raise ValueError("benchmark gate JSON must contain an object") + return value + + +__all__ = [ + "BENCHMARK_GATE_CHECK_SCHEMA_VERSION", + "BENCHMARK_GATE_OBSERVATION_SCHEMA_VERSION", + "BENCHMARK_GATE_REQUIREMENT_SCHEMA_VERSION", + "BENCHMARK_PROMOTION_DECISION_SCHEMA_VERSION", + "BENCHMARK_PROMOTION_GATE_POLICY_SCHEMA_VERSION", + "DEFAULT_BENCHMARK_GATE_ASSET", + "BenchmarkGateCheckV1", + "BenchmarkGateComparator", + "BenchmarkGateObservationV1", + "BenchmarkGateRequirementV1", + "BenchmarkGateScope", + "BenchmarkGateSeverity", + "BenchmarkGateStatus", + "BenchmarkPromotionDecisionV1", + "BenchmarkPromotionGatePolicyV1", + "evaluate_benchmark_promotion_gates", + "load_default_benchmark_promotion_gate_policy", +] diff --git a/src/histdatacom/synthetic/broker_transfer.py b/src/histdatacom/synthetic/broker_transfer.py new file mode 100644 index 00000000..c8ed6820 --- /dev/null +++ b/src/histdatacom/synthetic/broker_transfer.py @@ -0,0 +1,2405 @@ +"""Deterministic broker-style proposal conditioning and delivery rendering. + +The broker fingerprint is an observation/delivery model. It may condition +motif retrieval and render already accepted synthetic events, but it never +changes immutable observations or supplies historical price-path content. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from enum import Enum +from typing import Any, cast + +from histdatacom.broker_capture.fingerprint_contracts import ( + BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION, + BrokerDeliveryCellV1, + BrokerDeliveryFingerprintComparisonV1, + BrokerDeliveryFingerprintV1, + BrokerDeliverySupportStatus, +) +from histdatacom.data_quality.contracts import QualityStatus +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.benchmark import ReverseDegradationScorecardV1 +from histdatacom.synthetic.carving import HistoricalCarvingConstraintSetV1 +from histdatacom.synthetic.contracts import ( + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + canonical_contract_json, +) +from histdatacom.synthetic.cross_currency import ( + CrossCurrencyGroupStatus, + CrossCurrencyReconciledGroupV1, + CrossCurrencyValidationReportV1, + CrossCurrencyValidationStage, + CrossCurrencyValidationStatus, + cross_currency_quality_report, + validate_cross_currency_output, +) +from histdatacom.synthetic.motifs import ReferenceMotifQueryV1 +from histdatacom.synthetic.streaming import ( + ReconstructionRunV1, + ReconstructionWindowV1, +) + +BROKER_TRANSFER_CONFIG_SCHEMA_VERSION = "histdatacom.broker-transfer-config.v1" +BROKER_PROFILE_SELECTION_SCHEMA_VERSION = ( + "histdatacom.broker-profile-selection.v1" +) +BROKER_CONDITIONED_PROPOSAL_SCHEMA_VERSION = ( + "histdatacom.broker-conditioned-proposal.v1" +) +BROKER_RENDER_LINEAGE_SCHEMA_VERSION = "histdatacom.broker-render-lineage.v1" +BROKER_TRANSFER_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.broker-transfer-manifest.v1" +) +BROKER_RENDERED_GROUP_SCHEMA_VERSION = "histdatacom.broker-rendered-group.v1" +BROKER_BENCHMARK_COMPARISON_SCHEMA_VERSION = ( + "histdatacom.broker-benchmark-comparison.v1" +) +BROKER_TRANSFER_ENGINE_ID = "histdatacom.broker-delivery-transfer" +BROKER_TRANSFER_ENGINE_VERSION = "1.0.0" + +MAX_BROKER_TRANSFER_EVENTS = 1_000_000 +MAX_BROKER_TRANSFER_ACTIONS = 32 +MAX_BROKER_TRANSFER_REASONS = 128 +MAX_BROKER_TRANSFER_METRICS = 64 +MAX_BROKER_TRANSFER_TEXT = 1_024 +INT64_MAX = 2**63 - 1 + +_PROPOSAL_METRIC_TARGETS = { + "timestamp_precision_ns": ("source_timestamp_precision_ns",), + "spread": ("spread",), + "price_precision_digits": ("price_decimal_places",), +} + + +class BrokerTransferStatus(str, Enum): + """Whether a requested broker-style application was supported.""" + + APPLIED = "applied" + BACKED_OFF = "backed_off" + REFUSED = "refused" + + +@dataclass(frozen=True, slots=True) +class BrokerTransferConfigV1: + """Versioned, bounded semantics for a measurable gentle transfer.""" + + strength: float = 0.25 + max_timestamp_shift_ns: int = 1_000_000_000 + max_spread_multiplier: float = 2.0 + max_batch_size: int = 64 + max_events_per_group: int = 250_000 + input_price_decimal_places: int = 8 + minimum_price_decimal_places: int = 1 + apply_stale_behavior: bool = True + apply_exact_duplicates: bool = True + apply_batching: bool = True + rounding_digits: int = 12 + config_id: str = "" + schema_version: str = BROKER_TRANSFER_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_TRANSFER_CONFIG_SCHEMA_VERSION: + raise ValueError("unsupported broker transfer config") + strength = _finite_float(self.strength, "strength") + if not 0.0 <= strength <= 1.0: + raise ValueError("strength must be inside [0,1]") + object.__setattr__(self, "strength", strength) + shift = _bounded_int( + self.max_timestamp_shift_ns, + "max_timestamp_shift_ns", + 0, + INT64_MAX, + ) + object.__setattr__(self, "max_timestamp_shift_ns", shift) + multiplier = _finite_float( + self.max_spread_multiplier, "max_spread_multiplier" + ) + if multiplier < 1.0: + raise ValueError("max_spread_multiplier must be at least one") + object.__setattr__(self, "max_spread_multiplier", multiplier) + object.__setattr__( + self, + "max_batch_size", + _bounded_int(self.max_batch_size, "max_batch_size", 1, 65_536), + ) + object.__setattr__( + self, + "max_events_per_group", + _bounded_int( + self.max_events_per_group, + "max_events_per_group", + 1, + MAX_BROKER_TRANSFER_EVENTS, + ), + ) + for name in ( + "input_price_decimal_places", + "minimum_price_decimal_places", + "rounding_digits", + ): + value = _bounded_int(getattr(self, name), name, 0, 15) + object.__setattr__(self, name, value) + if self.minimum_price_decimal_places > self.input_price_decimal_places: + raise ValueError( + "minimum price precision exceeds input price precision" + ) + for name in ( + "apply_stale_behavior", + "apply_exact_duplicates", + "apply_batching", + ): + if type(getattr(self, name)) is not bool: + raise ValueError(f"{name} must be boolean") + expected = _stable_id("broker-transfer-config", self.identity_payload()) + supplied = _optional_text(self.config_id) + if supplied is not None and supplied != expected: + raise ValueError("broker transfer config_id differs") + object.__setattr__(self, "config_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "engine_id": BROKER_TRANSFER_ENGINE_ID, + "engine_version": BROKER_TRANSFER_ENGINE_VERSION, + "strength": self.strength, + "strength_semantics": ( + "convex blend from historical value 0 to broker target 1" + ), + "max_timestamp_shift_ns": self.max_timestamp_shift_ns, + "max_spread_multiplier": self.max_spread_multiplier, + "max_batch_size": self.max_batch_size, + "input_price_decimal_places": self.input_price_decimal_places, + "minimum_price_decimal_places": self.minimum_price_decimal_places, + "apply_stale_behavior": self.apply_stale_behavior, + "apply_exact_duplicates": self.apply_exact_duplicates, + "apply_batching": self.apply_batching, + "rounding_digits": self.rounding_digits, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "max_events_per_group": self.max_events_per_group, + "execution_limits_in_config_identity": False, + "config_id": self.config_id, + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerTransferConfigV1": + _require_schema(data, BROKER_TRANSFER_CONFIG_SCHEMA_VERSION) + return cls( + strength=_finite_float(data.get("strength"), "strength"), + max_timestamp_shift_ns=_strict_int( + data.get("max_timestamp_shift_ns"), "max_timestamp_shift_ns" + ), + max_spread_multiplier=_finite_float( + data.get("max_spread_multiplier"), "max_spread_multiplier" + ), + max_batch_size=_strict_int( + data.get("max_batch_size"), "max_batch_size" + ), + max_events_per_group=_strict_int( + data.get("max_events_per_group"), "max_events_per_group" + ), + input_price_decimal_places=_strict_int( + data.get("input_price_decimal_places"), + "input_price_decimal_places", + ), + minimum_price_decimal_places=_strict_int( + data.get("minimum_price_decimal_places"), + "minimum_price_decimal_places", + ), + apply_stale_behavior=_strict_bool( + data.get("apply_stale_behavior"), "apply_stale_behavior" + ), + apply_exact_duplicates=_strict_bool( + data.get("apply_exact_duplicates"), "apply_exact_duplicates" + ), + apply_batching=_strict_bool( + data.get("apply_batching"), "apply_batching" + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + config_id=str(data.get("config_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerTransferConfigV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerProfileSelectionV1: + """One explicit fingerprint-cell selection and metric fallback trace.""" + + fingerprint_id: str + fingerprint_schema_version: str + requested_condition: Mapping[str, str] + requested_condition_id: str | None + effective_condition_id: str | None + support_status: str + status: BrokerTransferStatus + selected_at_utc_ns: int + profile_effective_start_utc_ns: int + profile_effective_end_utc_ns: int | None + supersedes_fingerprint_id: str | None + metrics: Mapping[str, float] + metric_condition_ids: Mapping[str, str] + reason_codes: tuple[str, ...] = () + drift_comparison_id: str | None = None + material_drift_count: int | None = None + selection_id: str = "" + schema_version: str = BROKER_PROFILE_SELECTION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_PROFILE_SELECTION_SCHEMA_VERSION: + raise ValueError("unsupported broker profile selection") + if self.fingerprint_schema_version != ( + BROKER_DELIVERY_FINGERPRINT_SCHEMA_VERSION + ): + raise ValueError("selection fingerprint schema is unsupported") + object.__setattr__( + self, "fingerprint_id", _required_text(self.fingerprint_id) + ) + condition = { + _required_name(str(name)): _required_name(str(value)) + for name, value in sorted(self.requested_condition.items()) + } + object.__setattr__(self, "requested_condition", condition) + for name in ( + "requested_condition_id", + "effective_condition_id", + "supersedes_fingerprint_id", + "drift_comparison_id", + ): + object.__setattr__(self, name, _optional_text(getattr(self, name))) + support = _required_name(self.support_status) + object.__setattr__(self, "support_status", support) + status = BrokerTransferStatus(self.status) + object.__setattr__(self, "status", status) + selected = _bounded_int( + self.selected_at_utc_ns, "selected_at_utc_ns", 0, INT64_MAX + ) + start = _bounded_int( + self.profile_effective_start_utc_ns, + "profile_effective_start_utc_ns", + 0, + INT64_MAX, + ) + end = self.profile_effective_end_utc_ns + if end is not None: + end = _bounded_int( + end, "profile_effective_end_utc_ns", 0, INT64_MAX + ) + if end <= start: + raise ValueError( + "selection profile effective interval is empty" + ) + object.__setattr__(self, "selected_at_utc_ns", selected) + object.__setattr__(self, "profile_effective_start_utc_ns", start) + object.__setattr__(self, "profile_effective_end_utc_ns", end) + metrics = _metric_mapping(self.metrics) + sources = { + _required_name(str(name)): _required_text(value) + for name, value in sorted(self.metric_condition_ids.items()) + } + if set(metrics) != set(sources): + raise ValueError("selection metric sources do not cover metrics") + object.__setattr__(self, "metrics", metrics) + object.__setattr__(self, "metric_condition_ids", sources) + reasons = _bounded_text_tuple( + self.reason_codes, MAX_BROKER_TRANSFER_REASONS + ) + object.__setattr__(self, "reason_codes", reasons) + if status is BrokerTransferStatus.REFUSED: + if not reasons or self.effective_condition_id is not None: + raise ValueError( + "refused selection requires reasons and no cell" + ) + elif self.effective_condition_id is None: + raise ValueError("applied selection requires an effective cell") + if self.material_drift_count is not None: + object.__setattr__( + self, + "material_drift_count", + _bounded_int( + self.material_drift_count, + "material_drift_count", + 0, + INT64_MAX, + ), + ) + expected = _stable_id( + "broker-profile-selection", self.identity_payload() + ) + supplied = _optional_text(self.selection_id) + if supplied is not None and supplied != expected: + raise ValueError("broker profile selection_id differs") + object.__setattr__(self, "selection_id", expected) + + @property + def applied(self) -> bool: + return self.status is not BrokerTransferStatus.REFUSED + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "fingerprint_id": self.fingerprint_id, + "fingerprint_schema_version": self.fingerprint_schema_version, + "requested_condition": dict(self.requested_condition), + "requested_condition_id": self.requested_condition_id, + "effective_condition_id": self.effective_condition_id, + "support_status": self.support_status, + "status": self.status.value, + "selected_at_utc_ns": self.selected_at_utc_ns, + "profile_effective_start_utc_ns": ( + self.profile_effective_start_utc_ns + ), + "profile_effective_end_utc_ns": self.profile_effective_end_utc_ns, + "supersedes_fingerprint_id": self.supersedes_fingerprint_id, + "metrics": dict(self.metrics), + "metric_condition_ids": dict(self.metric_condition_ids), + "reason_codes": list(self.reason_codes), + "drift_comparison_id": self.drift_comparison_id, + "material_drift_count": self.material_drift_count, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "selection_id": self.selection_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerProfileSelectionV1": + _require_schema(data, BROKER_PROFILE_SELECTION_SCHEMA_VERSION) + return cls( + fingerprint_id=str(data.get("fingerprint_id", "")), + fingerprint_schema_version=str( + data.get("fingerprint_schema_version", "") + ), + requested_condition={ + str(name): str(value) + for name, value in _mapping( + data.get("requested_condition") + ).items() + }, + requested_condition_id=_optional_text( + data.get("requested_condition_id") + ), + effective_condition_id=_optional_text( + data.get("effective_condition_id") + ), + support_status=str(data.get("support_status", "")), + status=BrokerTransferStatus(str(data.get("status", ""))), + selected_at_utc_ns=_strict_int( + data.get("selected_at_utc_ns"), "selected_at_utc_ns" + ), + profile_effective_start_utc_ns=_strict_int( + data.get("profile_effective_start_utc_ns"), + "profile_effective_start_utc_ns", + ), + profile_effective_end_utc_ns=_optional_int( + data.get("profile_effective_end_utc_ns") + ), + supersedes_fingerprint_id=_optional_text( + data.get("supersedes_fingerprint_id") + ), + metrics={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping(data.get("metrics")).items() + }, + metric_condition_ids={ + str(name): str(value) + for name, value in _mapping( + data.get("metric_condition_ids") + ).items() + }, + reason_codes=_string_tuple(data.get("reason_codes")), + drift_comparison_id=_optional_text(data.get("drift_comparison_id")), + material_drift_count=_optional_int( + data.get("material_drift_count") + ), + selection_id=str(data.get("selection_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerConditionedProposalV1: + """A broker-conditioned motif query produced before candidate generation.""" + + selection: BrokerProfileSelectionV1 + transfer_config: BrokerTransferConfigV1 + original_query_id: str + conditioned_query: ReferenceMotifQueryV1 | None + metrics_before: Mapping[str, float] + metrics_after: Mapping[str, float] + proposal_id: str = "" + schema_version: str = BROKER_CONDITIONED_PROPOSAL_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_CONDITIONED_PROPOSAL_SCHEMA_VERSION: + raise ValueError("unsupported broker-conditioned proposal") + if not isinstance(self.selection, BrokerProfileSelectionV1): + raise TypeError("proposal requires a profile selection") + if not isinstance(self.transfer_config, BrokerTransferConfigV1): + raise TypeError("proposal requires a transfer config") + object.__setattr__( + self, "original_query_id", _required_text(self.original_query_id) + ) + before = _metric_mapping(self.metrics_before) + after = _metric_mapping(self.metrics_after) + object.__setattr__(self, "metrics_before", before) + object.__setattr__(self, "metrics_after", after) + if self.selection.applied: + if self.conditioned_query is None: + raise ValueError( + "applied proposal requires a conditioned query" + ) + if ( + self.conditioned_query.query_id == self.original_query_id + and self.transfer_config.strength > 0.0 + ): + raise ValueError( + "conditioned proposal did not change query identity" + ) + if self.transfer_config.strength == 0.0 and before != after: + raise ValueError("zero-strength proposal changed query metrics") + elif self.conditioned_query is not None: + raise ValueError("refused proposal cannot contain a query") + expected = _stable_id( + "broker-conditioned-proposal", self.identity_payload() + ) + supplied = _optional_text(self.proposal_id) + if supplied is not None and supplied != expected: + raise ValueError("broker proposal_id differs") + object.__setattr__(self, "proposal_id", expected) + + @property + def status(self) -> BrokerTransferStatus: + return self.selection.status + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "selection": self.selection.to_dict(), + "transfer_config": self.transfer_config.to_dict(), + "original_query_id": self.original_query_id, + "conditioned_query": ( + self.conditioned_query.to_dict() + if self.conditioned_query is not None + else None + ), + "metrics_before": dict(self.metrics_before), + "metrics_after": dict(self.metrics_after), + "stage": "pre_retrieval_and_generation", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "proposal_id": self.proposal_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerConditionedProposalV1": + _require_schema(data, BROKER_CONDITIONED_PROPOSAL_SCHEMA_VERSION) + query = data.get("conditioned_query") + return cls( + selection=BrokerProfileSelectionV1.from_dict( + _mapping(data.get("selection")) + ), + transfer_config=BrokerTransferConfigV1.from_dict( + _mapping(data.get("transfer_config")) + ), + original_query_id=str(data.get("original_query_id", "")), + conditioned_query=( + ReferenceMotifQueryV1.from_dict(_mapping(query)) + if query is not None + else None + ), + metrics_before={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping(data.get("metrics_before")).items() + }, + metrics_after={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping(data.get("metrics_after")).items() + }, + proposal_id=str(data.get("proposal_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerConditionedProposalV1": + return cls.from_dict(_json_mapping(text)) + + +def select_broker_profile( + fingerprint: BrokerDeliveryFingerprintV1, + *, + requested_condition: Mapping[str, str], + selected_at_utc_ns: int, + drift_comparison: BrokerDeliveryFingerprintComparisonV1 | None = None, +) -> BrokerProfileSelectionV1: + """Resolve one exact cell and its recorded backoff without invention.""" + selected_at = _bounded_int( + selected_at_utc_ns, "selected_at_utc_ns", 0, INT64_MAX + ) + requested = { + _required_name(str(name)): _required_name(str(value)) + for name, value in sorted(requested_condition.items()) + } + requested_id = _condition_id(requested) + comparison_id, material_count, drift_reason = _drift_evidence( + fingerprint, drift_comparison + ) + reasons: list[str] = [] + if drift_reason is not None: + reasons.append(drift_reason) + return _refused_selection( + fingerprint, + requested, + requested_id, + selected_at, + reasons, + comparison_id, + material_count, + ) + if selected_at < fingerprint.effective_start_utc_ns or ( + fingerprint.effective_end_utc_ns is not None + and selected_at >= fingerprint.effective_end_utc_ns + ): + reasons.append("profile_not_effective_at_selection_time") + return _refused_selection( + fingerprint, + requested, + requested_id, + selected_at, + reasons, + comparison_id, + material_count, + ) + cells = {item.condition.condition_id: item for item in fingerprint.cells} + requested_cell = cells.get(requested_id) + if requested_cell is None: + reasons.append("requested_condition_absent") + return _refused_selection( + fingerprint, + requested, + requested_id, + selected_at, + reasons, + comparison_id, + material_count, + ) + if requested_cell.support_status is BrokerDeliverySupportStatus.UNSUPPORTED: + reasons.extend( + ("requested_condition_unsupported", *requested_cell.limitations) + ) + return _refused_selection( + fingerprint, + requested, + requested_id, + selected_at, + reasons, + comparison_id, + material_count, + ) + effective_id = requested_cell.effective_condition_id + effective_cell = cells.get(str(effective_id)) + if effective_cell is None: + reasons.append("recorded_effective_condition_absent") + return _refused_selection( + fingerprint, + requested, + requested_id, + selected_at, + reasons, + comparison_id, + material_count, + ) + status = BrokerTransferStatus.APPLIED + if requested_cell.support_status is BrokerDeliverySupportStatus.BACKED_OFF: + status = BrokerTransferStatus.BACKED_OFF + reasons.append("requested_condition_backed_off") + global_cell = next( + item for item in fingerprint.cells if not item.condition.dimensions + ) + metrics, sources = _resolved_metrics(effective_cell, global_cell) + return BrokerProfileSelectionV1( + fingerprint_id=fingerprint.fingerprint_id, + fingerprint_schema_version=fingerprint.schema_version, + requested_condition=requested, + requested_condition_id=requested_id, + effective_condition_id=effective_cell.condition.condition_id, + support_status=requested_cell.support_status.value, + status=status, + selected_at_utc_ns=selected_at, + profile_effective_start_utc_ns=fingerprint.effective_start_utc_ns, + profile_effective_end_utc_ns=fingerprint.effective_end_utc_ns, + supersedes_fingerprint_id=fingerprint.supersedes_fingerprint_id, + metrics=metrics, + metric_condition_ids=sources, + reason_codes=tuple(sorted(set(reasons))), + drift_comparison_id=comparison_id, + material_drift_count=material_count, + ) + + +def condition_broker_proposal( + query: ReferenceMotifQueryV1, + fingerprint: BrokerDeliveryFingerprintV1, + *, + requested_condition: Mapping[str, str], + selected_at_utc_ns: int, + config: BrokerTransferConfigV1 | None = None, + drift_comparison: BrokerDeliveryFingerprintComparisonV1 | None = None, +) -> BrokerConditionedProposalV1: + """Blend broker delivery metrics into a motif query before retrieval.""" + policy = config or BrokerTransferConfigV1() + selection = select_broker_profile( + fingerprint, + requested_condition=requested_condition, + selected_at_utc_ns=selected_at_utc_ns, + drift_comparison=drift_comparison, + ) + before = dict(query.condition.metrics) + if not selection.applied: + return BrokerConditionedProposalV1( + selection=selection, + transfer_config=policy, + original_query_id=query.query_id, + conditioned_query=None, + metrics_before=before, + metrics_after=before, + ) + if policy.strength == 0.0: + return BrokerConditionedProposalV1( + selection=selection, + transfer_config=policy, + original_query_id=query.query_id, + conditioned_query=query, + metrics_before=before, + metrics_after=before, + ) + after = dict(before) + cadence_target = _broker_cadence_target(selection.metrics) + historical_cadence = after.get("interarrival_ns") + if historical_cadence is None: + historical_intensity = after.get("tick_intensity") + if historical_intensity is not None and historical_intensity > 0.0: + historical_cadence = 1_000_000_000.0 / historical_intensity + if cadence_target is not None and historical_cadence is not None: + cadence = _rounded( + _blend(historical_cadence, cadence_target, policy.strength), + policy.rounding_digits, + ) + after["interarrival_ns"] = cadence + historical_intensity = after.get( + "tick_intensity", 1_000_000_000.0 / historical_cadence + ) + after["tick_intensity"] = _rounded( + _blend( + historical_intensity, + 1_000_000_000.0 / cadence_target, + policy.strength, + ), + policy.rounding_digits, + ) + for target_name, source_names in _PROPOSAL_METRIC_TARGETS.items(): + target = next( + ( + selection.metrics[source_name] + for source_name in source_names + if source_name in selection.metrics + ), + None, + ) + if target is None: + continue + historical = after.get(target_name) + if historical is None: + if target_name == "timestamp_precision_ns": + historical = 1.0 + elif target_name == "price_precision_digits": + historical = float(policy.input_price_decimal_places) + else: + continue + after[target_name] = _rounded( + _blend(historical, target, policy.strength), + policy.rounding_digits, + ) + conditioned = replace( + query.condition, + metrics=after, + ) + conditioned_query = replace(query, condition=conditioned, query_id="") + return BrokerConditionedProposalV1( + selection=selection, + transfer_config=policy, + original_query_id=query.query_id, + conditioned_query=conditioned_query, + metrics_before=before, + metrics_after=after, + ) + + +def _broker_cadence_target(metrics: Mapping[str, float]) -> float | None: + cadence = metrics.get("active_quote_interarrival_ns") + if cadence is None: + cadence = metrics.get("quote_interarrival_ns") + intensity = metrics.get("quote_intensity_hz") + if cadence is None and intensity is not None and intensity > 0.0: + cadence = 1_000_000_000.0 / intensity + if cadence is None or cadence <= 0.0: + return None + burst = max(0.0, metrics.get("burst_interval_rate", 0.0)) + quiet = max(0.0, metrics.get("quiet_interval_rate", 0.0)) + outage_rate = max(0.0, metrics.get("event_kind.outage_end_rate", 0.0)) + outage_duration = max(0.0, metrics.get("outage_or_gap_duration_ns", 0.0)) + outage_weight = outage_rate * min(10.0, outage_duration / cadence) + structure_factor = max(0.1, 1.0 + quiet + outage_weight - 0.5 * burst) + return max(1.0, cadence * structure_factor) + + +def _condition_id(dimensions: Mapping[str, str]) -> str: + from histdatacom.broker_capture.fingerprint_contracts import ( + BrokerDeliveryConditionV1, + ) + + return str(BrokerDeliveryConditionV1(dict(dimensions)).condition_id) + + +def _refused_selection( + fingerprint: BrokerDeliveryFingerprintV1, + requested: Mapping[str, str], + requested_id: str, + selected_at: int, + reasons: Sequence[str], + comparison_id: str | None, + material_count: int | None, +) -> BrokerProfileSelectionV1: + return BrokerProfileSelectionV1( + fingerprint_id=fingerprint.fingerprint_id, + fingerprint_schema_version=fingerprint.schema_version, + requested_condition=requested, + requested_condition_id=requested_id, + effective_condition_id=None, + support_status=BrokerDeliverySupportStatus.UNSUPPORTED.value, + status=BrokerTransferStatus.REFUSED, + selected_at_utc_ns=selected_at, + profile_effective_start_utc_ns=fingerprint.effective_start_utc_ns, + profile_effective_end_utc_ns=fingerprint.effective_end_utc_ns, + supersedes_fingerprint_id=fingerprint.supersedes_fingerprint_id, + metrics={}, + metric_condition_ids={}, + reason_codes=tuple(sorted(set(reasons))), + drift_comparison_id=comparison_id, + material_drift_count=material_count, + ) + + +def _resolved_metrics( + effective: BrokerDeliveryCellV1, + global_cell: BrokerDeliveryCellV1, +) -> tuple[dict[str, float], dict[str, str]]: + metrics: dict[str, float] = {} + sources: dict[str, str] = {} + for cell in (effective, global_cell): + for metric in cell.metrics: + if metric.name in metrics: + continue + value = metric.estimate + if value is None: + value = metric.quantiles.get("q0.5") + if value is None or not math.isfinite(value): + continue + metrics[metric.name] = float(value) + sources[metric.name] = cell.condition.condition_id + if len(metrics) > MAX_BROKER_TRANSFER_METRICS: + selected = sorted(metrics)[:MAX_BROKER_TRANSFER_METRICS] + metrics = {name: metrics[name] for name in selected} + sources = {name: sources[name] for name in selected} + return dict(sorted(metrics.items())), dict(sorted(sources.items())) + + +def _drift_evidence( + fingerprint: BrokerDeliveryFingerprintV1, + comparison: BrokerDeliveryFingerprintComparisonV1 | None, +) -> tuple[str | None, int | None, str | None]: + if comparison is None: + return None, None, None + if fingerprint.fingerprint_id not in { + comparison.reference_fingerprint_id, + comparison.candidate_fingerprint_id, + }: + return None, None, "drift_comparison_does_not_include_profile" + return ( + comparison.comparison_id, + comparison.material_drift_count, + None, + ) + + +@dataclass(frozen=True, slots=True) +class BrokerRenderLineageV1: + """Recoverable input/output identity for one rendered synthetic event.""" + + symbol: str + input_event_id: str + output_event_id: str + selection_id: str + input_event_time_ns: int + output_event_time_ns: int + actions: tuple[str, ...] + lineage_id: str = "" + schema_version: str = BROKER_RENDER_LINEAGE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_RENDER_LINEAGE_SCHEMA_VERSION: + raise ValueError("unsupported broker render lineage") + object.__setattr__(self, "symbol", _required_name(self.symbol).upper()) + for name in ("input_event_id", "output_event_id", "selection_id"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + for name in ("input_event_time_ns", "output_event_time_ns"): + object.__setattr__( + self, + name, + _bounded_int(getattr(self, name), name, 0, INT64_MAX), + ) + actions = _bounded_text_tuple(self.actions, MAX_BROKER_TRANSFER_ACTIONS) + if not actions: + actions = ("profile_lineage_only",) + object.__setattr__(self, "actions", actions) + expected = _stable_id("broker-render-lineage", self.identity_payload()) + supplied = _optional_text(self.lineage_id) + if supplied is not None and supplied != expected: + raise ValueError("broker render lineage_id differs") + object.__setattr__(self, "lineage_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "input_event_id": self.input_event_id, + "output_event_id": self.output_event_id, + "selection_id": self.selection_id, + "input_event_time_ns": self.input_event_time_ns, + "output_event_time_ns": self.output_event_time_ns, + "time_shift_ns": ( + self.output_event_time_ns - self.input_event_time_ns + ), + "actions": list(self.actions), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "lineage_id": self.lineage_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerRenderLineageV1": + _require_schema(data, BROKER_RENDER_LINEAGE_SCHEMA_VERSION) + return cls( + symbol=str(data.get("symbol", "")), + input_event_id=str(data.get("input_event_id", "")), + output_event_id=str(data.get("output_event_id", "")), + selection_id=str(data.get("selection_id", "")), + input_event_time_ns=_strict_int( + data.get("input_event_time_ns"), "input_event_time_ns" + ), + output_event_time_ns=_strict_int( + data.get("output_event_time_ns"), "output_event_time_ns" + ), + actions=_string_tuple(data.get("actions")), + lineage_id=str(data.get("lineage_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerTransferManifestV1: + """Compact profile, validation, and content evidence for one render.""" + + run_id: str + window_id: str + synchronization_unit_id: str + ensemble_member_id: str + input_group_id: str + fingerprint_id: str + transfer_config: BrokerTransferConfigV1 + selections: tuple[BrokerProfileSelectionV1, ...] + status: BrokerTransferStatus + reason_codes: tuple[str, ...] + input_content_sha256: str + output_content_sha256: str | None + observed_event_count: int + synthetic_event_count: int + action_counts: Mapping[str, int] + lineage_count: int + lineage_content_sha256: str | None + local_validation_passed: bool + post_broker_validation_id: str | None + post_broker_validation_status: str | None + cross_instrument_quality_status: str | None + cross_instrument_quality_sha256: str | None + benchmark_comparison_ids: tuple[str, ...] = () + manifest_id: str = "" + schema_version: str = BROKER_TRANSFER_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_TRANSFER_MANIFEST_SCHEMA_VERSION: + raise ValueError("unsupported broker transfer manifest") + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "ensemble_member_id", + "input_group_id", + "fingerprint_id", + "input_content_sha256", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if not isinstance(self.transfer_config, BrokerTransferConfigV1): + raise TypeError("transfer manifest requires a v1 config") + selections = tuple( + sorted(self.selections, key=lambda item: item.selection_id) + ) + if not selections: + raise ValueError("transfer manifest requires profile selections") + if any( + item.fingerprint_id != self.fingerprint_id for item in selections + ): + raise ValueError("transfer selections use different fingerprints") + object.__setattr__(self, "selections", selections) + status = BrokerTransferStatus(self.status) + object.__setattr__(self, "status", status) + reasons = _bounded_text_tuple( + self.reason_codes, MAX_BROKER_TRANSFER_REASONS + ) + object.__setattr__(self, "reason_codes", reasons) + for name in ("output_content_sha256", "lineage_content_sha256"): + object.__setattr__( + self, name, _optional_sha256(getattr(self, name)) + ) + for name in ( + "post_broker_validation_id", + "post_broker_validation_status", + "cross_instrument_quality_status", + ): + object.__setattr__(self, name, _optional_text(getattr(self, name))) + object.__setattr__( + self, + "cross_instrument_quality_sha256", + _optional_sha256(self.cross_instrument_quality_sha256), + ) + for name in ( + "observed_event_count", + "synthetic_event_count", + "lineage_count", + ): + object.__setattr__( + self, + name, + _bounded_int( + getattr(self, name), name, 0, MAX_BROKER_TRANSFER_EVENTS + ), + ) + actions = _count_mapping(self.action_counts) + object.__setattr__(self, "action_counts", actions) + if type(self.local_validation_passed) is not bool: + raise ValueError("local_validation_passed must be boolean") + comparisons = _bounded_text_tuple( + self.benchmark_comparison_ids, MAX_BROKER_TRANSFER_REASONS + ) + object.__setattr__(self, "benchmark_comparison_ids", comparisons) + if status is BrokerTransferStatus.REFUSED: + if not reasons or self.output_content_sha256 is not None: + raise ValueError( + "refused render must retain reasons, not output" + ) + if self.local_validation_passed: + raise ValueError("refused render cannot pass local validation") + else: + if reasons: + raise ValueError("applied render cannot retain refusal reasons") + if ( + self.output_content_sha256 is None + or self.lineage_content_sha256 is None + or not self.local_validation_passed + or self.post_broker_validation_status + != CrossCurrencyValidationStatus.PASSED.value + or self.cross_instrument_quality_status + == QualityStatus.FAILED.value + ): + raise ValueError("applied render lacks passing final evidence") + if self.synthetic_event_count != self.lineage_count: + raise ValueError( + "render lineage count differs from synthetic rows" + ) + expected = _stable_id( + "broker-transfer-manifest", self.identity_payload() + ) + supplied = _optional_text(self.manifest_id) + if supplied is not None and supplied != expected: + raise ValueError("broker transfer manifest_id differs") + object.__setattr__(self, "manifest_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "engine_id": BROKER_TRANSFER_ENGINE_ID, + "engine_version": BROKER_TRANSFER_ENGINE_VERSION, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "ensemble_member_id": self.ensemble_member_id, + "input_group_id": self.input_group_id, + "fingerprint_id": self.fingerprint_id, + "transfer_config": self.transfer_config.to_dict(), + "selections": [item.to_dict() for item in self.selections], + "status": self.status.value, + "reason_codes": list(self.reason_codes), + "input_content_sha256": self.input_content_sha256, + "output_content_sha256": self.output_content_sha256, + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "action_counts": dict(self.action_counts), + "lineage_count": self.lineage_count, + "lineage_content_sha256": self.lineage_content_sha256, + "local_validation_passed": self.local_validation_passed, + "post_broker_validation_id": self.post_broker_validation_id, + "post_broker_validation_status": self.post_broker_validation_status, + "cross_instrument_quality_status": ( + self.cross_instrument_quality_status + ), + "cross_instrument_quality_sha256": ( + self.cross_instrument_quality_sha256 + ), + "benchmark_comparison_ids": list(self.benchmark_comparison_ids), + "durable_event_rows_inline": False, + "profile_effective_periods_embedded_in_selections": True, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerTransferManifestV1": + _require_schema(data, BROKER_TRANSFER_MANIFEST_SCHEMA_VERSION) + return cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + input_group_id=str(data.get("input_group_id", "")), + fingerprint_id=str(data.get("fingerprint_id", "")), + transfer_config=BrokerTransferConfigV1.from_dict( + _mapping(data.get("transfer_config")) + ), + selections=tuple( + BrokerProfileSelectionV1.from_dict(item) + for item in _mapping_sequence(data.get("selections")) + ), + status=BrokerTransferStatus(str(data.get("status", ""))), + reason_codes=_string_tuple(data.get("reason_codes")), + input_content_sha256=str(data.get("input_content_sha256", "")), + output_content_sha256=_optional_text( + data.get("output_content_sha256") + ), + observed_event_count=_strict_int( + data.get("observed_event_count"), "observed_event_count" + ), + synthetic_event_count=_strict_int( + data.get("synthetic_event_count"), "synthetic_event_count" + ), + action_counts={ + str(name): _strict_int(value, str(name)) + for name, value in _mapping(data.get("action_counts")).items() + }, + lineage_count=_strict_int( + data.get("lineage_count"), "lineage_count" + ), + lineage_content_sha256=_optional_text( + data.get("lineage_content_sha256") + ), + local_validation_passed=_strict_bool( + data.get("local_validation_passed"), + "local_validation_passed", + ), + post_broker_validation_id=_optional_text( + data.get("post_broker_validation_id") + ), + post_broker_validation_status=_optional_text( + data.get("post_broker_validation_status") + ), + cross_instrument_quality_status=_optional_text( + data.get("cross_instrument_quality_status") + ), + cross_instrument_quality_sha256=_optional_text( + data.get("cross_instrument_quality_sha256") + ), + benchmark_comparison_ids=_string_tuple( + data.get("benchmark_comparison_ids") + ), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class BrokerRenderedGroupV1: + """Process-local rendered streams plus compact durable evidence.""" + + manifest: BrokerTransferManifestV1 + streams: tuple[SyntheticEventStreamV1, ...] + event_lineage: tuple[BrokerRenderLineageV1, ...] + post_broker_validation: CrossCurrencyValidationReportV1 | None + cross_instrument_quality_payload: Mapping[str, JSONValue] | None + schema_version: str = BROKER_RENDERED_GROUP_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_RENDERED_GROUP_SCHEMA_VERSION: + raise ValueError("unsupported broker rendered group") + if not isinstance(self.manifest, BrokerTransferManifestV1): + raise TypeError("rendered group requires a transfer manifest") + streams = tuple(sorted(self.streams, key=lambda item: item.symbol)) + lineage = tuple( + sorted( + self.event_lineage, + key=lambda item: (item.symbol, item.output_event_id), + ) + ) + object.__setattr__(self, "streams", streams) + object.__setattr__(self, "event_lineage", lineage) + quality = self.cross_instrument_quality_payload + if quality is not None: + quality = _json_mapping_value(quality) + object.__setattr__( + self, "cross_instrument_quality_payload", quality + ) + if self.manifest.status is BrokerTransferStatus.REFUSED: + if streams or lineage or self.post_broker_validation is not None: + raise ValueError("refused rendered group cannot expose output") + if quality is not None: + raise ValueError("refused rendered group cannot expose quality") + else: + if not streams or len(lineage) != self.manifest.lineage_count: + raise ValueError("applied rendered group rows do not reconcile") + validation = self.post_broker_validation + if ( + validation is None + or validation.validation_id + != self.manifest.post_broker_validation_id + or not validation.passed + ): + raise ValueError("rendered group lacks passing validation") + if quality is None: + raise ValueError("rendered group lacks #331 quality evidence") + + @property + def status(self) -> BrokerTransferStatus: + return self.manifest.status + + def metadata(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "manifest": self.manifest.to_dict(), + "stream_ids": [item.stream_id for item in self.streams], + "event_rows_inline": False, + "lineage_rows_inline": False, + "post_broker_validation": ( + self.post_broker_validation.to_dict() + if self.post_broker_validation is not None + else None + ), + "cross_instrument_quality_sha256": ( + _content_sha256(self.cross_instrument_quality_payload) + if self.cross_instrument_quality_payload is not None + else None + ), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.metadata(), + "streams": [item.to_dict() for item in self.streams], + "event_lineage": [item.to_dict() for item in self.event_lineage], + "cross_instrument_quality": ( + dict(self.cross_instrument_quality_payload) + if self.cross_instrument_quality_payload is not None + else None + ), + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BrokerRenderedGroupV1": + _require_schema(data, BROKER_RENDERED_GROUP_SCHEMA_VERSION) + validation = data.get("post_broker_validation") + quality = data.get("cross_instrument_quality") + return cls( + manifest=BrokerTransferManifestV1.from_dict( + _mapping(data.get("manifest")) + ), + streams=tuple( + SyntheticEventStreamV1.from_dict(item) + for item in _mapping_sequence(data.get("streams")) + ), + event_lineage=tuple( + BrokerRenderLineageV1.from_dict(item) + for item in _mapping_sequence(data.get("event_lineage")) + ), + post_broker_validation=( + CrossCurrencyValidationReportV1.from_dict(_mapping(validation)) + if validation is not None + else None + ), + cross_instrument_quality_payload=( + _json_mapping_value(_mapping(quality)) + if quality is not None + else None + ), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerRenderedGroupV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class BrokerBenchmarkComparisonV1: + """Paired reverse-degradation evidence without selecting a winner.""" + + scorecard_id: str + conditioned_candidate_id: str + unconditioned_candidate_id: str + scenario_ids: tuple[str, ...] + conditioned_score_ids: tuple[str, ...] + unconditioned_score_ids: tuple[str, ...] + aggregate_metric_deltas: Mapping[str, Mapping[str, float]] + comparison_id: str = "" + schema_version: str = BROKER_BENCHMARK_COMPARISON_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BROKER_BENCHMARK_COMPARISON_SCHEMA_VERSION: + raise ValueError("unsupported broker benchmark comparison") + for name in ( + "scorecard_id", + "conditioned_candidate_id", + "unconditioned_candidate_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if self.conditioned_candidate_id == self.unconditioned_candidate_id: + raise ValueError("benchmark candidates must be distinct") + scenarios = _bounded_text_tuple( + self.scenario_ids, MAX_BROKER_TRANSFER_REASONS + ) + conditioned = _bounded_text_tuple( + self.conditioned_score_ids, MAX_BROKER_TRANSFER_REASONS + ) + unconditioned = _bounded_text_tuple( + self.unconditioned_score_ids, MAX_BROKER_TRANSFER_REASONS + ) + if ( + not scenarios + or len(conditioned) != len(scenarios) + or len(unconditioned) != len(scenarios) + ): + raise ValueError("benchmark comparison cells do not reconcile") + object.__setattr__(self, "scenario_ids", scenarios) + object.__setattr__(self, "conditioned_score_ids", conditioned) + object.__setattr__(self, "unconditioned_score_ids", unconditioned) + deltas = { + _required_text(scenario): _metric_mapping(values) + for scenario, values in sorted(self.aggregate_metric_deltas.items()) + } + if set(deltas) != set(scenarios): + raise ValueError("benchmark deltas do not cover scenarios") + object.__setattr__(self, "aggregate_metric_deltas", deltas) + expected = _stable_id( + "broker-benchmark-comparison", self.identity_payload() + ) + supplied = _optional_text(self.comparison_id) + if supplied is not None and supplied != expected: + raise ValueError("broker benchmark comparison_id differs") + object.__setattr__(self, "comparison_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "scorecard_id": self.scorecard_id, + "conditioned_candidate_id": self.conditioned_candidate_id, + "unconditioned_candidate_id": self.unconditioned_candidate_id, + "scenario_ids": list(self.scenario_ids), + "conditioned_score_ids": list(self.conditioned_score_ids), + "unconditioned_score_ids": list(self.unconditioned_score_ids), + "aggregate_metric_deltas": { + scenario: dict(values) + for scenario, values in self.aggregate_metric_deltas.items() + }, + "delta_semantics": "conditioned_minus_unconditioned", + "automatic_winner": False, + "winner_candidate_id": None, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "comparison_id": self.comparison_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "BrokerBenchmarkComparisonV1": + _require_schema(data, BROKER_BENCHMARK_COMPARISON_SCHEMA_VERSION) + return cls( + scorecard_id=str(data.get("scorecard_id", "")), + conditioned_candidate_id=str( + data.get("conditioned_candidate_id", "") + ), + unconditioned_candidate_id=str( + data.get("unconditioned_candidate_id", "") + ), + scenario_ids=_string_tuple(data.get("scenario_ids")), + conditioned_score_ids=_string_tuple( + data.get("conditioned_score_ids") + ), + unconditioned_score_ids=_string_tuple( + data.get("unconditioned_score_ids") + ), + aggregate_metric_deltas={ + str(scenario): { + str(name): _finite_float(value, str(name)) + for name, value in _mapping(values).items() + } + for scenario, values in _mapping( + data.get("aggregate_metric_deltas") + ).items() + }, + comparison_id=str(data.get("comparison_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "BrokerBenchmarkComparisonV1": + return cls.from_dict(_json_mapping(text)) + + +def compare_broker_benchmark_results( + scorecard: ReverseDegradationScorecardV1, + *, + conditioned_candidate_id: str, + unconditioned_candidate_id: str, +) -> BrokerBenchmarkComparisonV1: + """Pair conditioned/unconditioned scorecard cells by scenario.""" + conditioned = { + item.scenario_id: item + for item in scorecard.candidate_scores + if item.candidate_id == conditioned_candidate_id + } + unconditioned = { + item.scenario_id: item + for item in scorecard.candidate_scores + if item.candidate_id == unconditioned_candidate_id + } + if not conditioned or set(conditioned) != set(unconditioned): + raise ValueError( + "conditioned and unconditioned benchmark scenario support differs" + ) + scenarios = tuple(sorted(conditioned)) + deltas: dict[str, dict[str, float]] = {} + for scenario in scenarios: + left = conditioned[scenario] + right = unconditioned[scenario] + names = sorted( + set(left.aggregate_metrics).intersection(right.aggregate_metrics) + ) + if not names: + raise ValueError("paired benchmark cell has no common metrics") + deltas[scenario] = { + name: _rounded( + left.aggregate_metrics[name] - right.aggregate_metrics[name], + 12, + ) + for name in names + } + return BrokerBenchmarkComparisonV1( + scorecard_id=scorecard.scorecard_id, + conditioned_candidate_id=conditioned_candidate_id, + unconditioned_candidate_id=unconditioned_candidate_id, + scenario_ids=scenarios, + conditioned_score_ids=tuple( + conditioned[item].candidate_score_id for item in scenarios + ), + unconditioned_score_ids=tuple( + unconditioned[item].candidate_score_id for item in scenarios + ), + aggregate_metric_deltas=deltas, + ) + + +@dataclass(slots=True) +class _PendingRender: + source: SyntheticEventV1 + event_time_ns: int + bid: float + ask: float + actions: list[str] = field(default_factory=list) + + +def render_broker_delivery( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + group: CrossCurrencyReconciledGroupV1, + fingerprint: BrokerDeliveryFingerprintV1, + constraints: HistoricalCarvingConstraintSetV1, + selected_at_utc_ns: int, + requested_conditions: Mapping[str, Mapping[str, str]] | None = None, + config: BrokerTransferConfigV1 | None = None, + drift_comparison: BrokerDeliveryFingerprintComparisonV1 | None = None, + benchmark_comparisons: Sequence[BrokerBenchmarkComparisonV1] = (), + quality_period: str = "broker-render", +) -> BrokerRenderedGroupV1: + """Render one reconciled synchronized group and fail closed on validation.""" + policy = config or BrokerTransferConfigV1() + _validate_render_scope(run, window, group, constraints) + conditions = { + str(symbol).strip().lower(): dict(value) + for symbol, value in (requested_conditions or {}).items() + } + selections = tuple( + select_broker_profile( + fingerprint, + requested_condition=conditions.get(symbol.lower(), {}), + selected_at_utc_ns=selected_at_utc_ns, + drift_comparison=drift_comparison, + ) + for symbol in group.symbols + ) + input_hash = _streams_content_sha256(group.streams) + observed_count = sum(item.observed_event_count for item in group.streams) + synthetic_count = sum(item.synthetic_event_count for item in group.streams) + comparison_ids = tuple( + sorted(item.comparison_id for item in benchmark_comparisons) + ) + if group.status is not CrossCurrencyGroupStatus.RECONCILED: + return _refused_render( + run, + window, + group, + fingerprint, + policy, + selections, + input_hash, + observed_count, + synthetic_count, + ("input_cross_currency_group_not_reconciled",), + comparison_ids, + ) + refused = tuple(item for item in selections if not item.applied) + if refused: + reasons = tuple( + sorted( + { + f"profile_selection:{reason}" + for item in refused + for reason in item.reason_codes + } + ) + ) + return _refused_render( + run, + window, + group, + fingerprint, + policy, + selections, + input_hash, + observed_count, + synthetic_count, + reasons or ("profile_selection_refused",), + comparison_ids, + ) + total_events = observed_count + synthetic_count + if total_events > policy.max_events_per_group: + return _refused_render( + run, + window, + group, + fingerprint, + policy, + selections, + input_hash, + observed_count, + synthetic_count, + ("max_events_per_group_exceeded",), + comparison_ids, + ) + by_symbol = { + symbol: next( + item + for item in selections + if _selection_symbol(item) in {None, symbol.upper()} + ) + for symbol in group.symbols + } + shared = _shared_render_parameters(tuple(by_symbol.values()), policy) + streams: list[SyntheticEventStreamV1] = [] + lineage: list[BrokerRenderLineageV1] = [] + render_reasons: list[str] = [] + for stream in group.streams: + rendered, rows, reason = _render_stream( + run=run, + window=window, + stream=stream, + fingerprint_id=fingerprint.fingerprint_id, + selection=by_symbol[stream.symbol], + config=policy, + constraints=constraints, + shared=shared, + ) + if reason is not None: + render_reasons.append(f"{stream.symbol}:{reason}") + continue + assert rendered is not None + streams.append(rendered) + lineage.extend(rows) + if render_reasons or len(streams) != len(group.symbols): + return _refused_render( + run, + window, + group, + fingerprint, + policy, + selections, + input_hash, + observed_count, + synthetic_count, + tuple(sorted(render_reasons or ("rendered_symbol_missing",))), + comparison_ids, + ) + stream_tuple = tuple(sorted(streams, key=lambda item: item.symbol)) + local_reasons = _local_validation_reasons( + group.streams, + stream_tuple, + fingerprint.fingerprint_id, + constraints, + ) + if local_reasons: + return _refused_render( + run, + window, + group, + fingerprint, + policy, + selections, + input_hash, + observed_count, + synthetic_count, + local_reasons, + comparison_ids, + ) + observed_anchors = tuple( + event + for stream in group.streams + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + ) + post_broker = validate_cross_currency_output( + run=run, + window=window, + streams={item.symbol: item for item in stream_tuple}, + config=group.config, + stage=CrossCurrencyValidationStage.POST_BROKER, + observed_anchors=observed_anchors, + ) + if not post_broker.passed: + reasons = tuple( + sorted( + f"post_broker:{reason}" + for reason in post_broker.failure_reasons + ) + ) + return _refused_render( + run, + window, + group, + fingerprint, + policy, + selections, + input_hash, + observed_count, + synthetic_count, + reasons or ("post_broker_cross_currency_validation_failed",), + comparison_ids, + ) + quality_validation = validate_cross_currency_output( + run=run, + window=window, + streams={item.symbol: item for item in stream_tuple}, + config=group.config, + stage=CrossCurrencyValidationStage.GENERATION, + observed_anchors=observed_anchors, + ) + quality_group = CrossCurrencyReconciledGroupV1( + run_id=group.run_id, + window_id=group.window_id, + synchronization_unit_id=group.synchronization_unit_id, + ensemble_member_id=group.ensemble_member_id, + symbols=group.symbols, + status=CrossCurrencyGroupStatus.RECONCILED, + streams=stream_tuple, + missing_symbols=(), + input_stream_ids={item.symbol: item.stream_id for item in stream_tuple}, + config=group.config, + condition_ids=group.condition_ids, + projection_lineage=(), + generation_validation=quality_validation, + ) + quality_report = cross_currency_quality_report( + quality_group, + period=quality_period, + ) + if quality_report.status is QualityStatus.FAILED: + return _refused_render( + run, + window, + group, + fingerprint, + policy, + selections, + input_hash, + observed_count, + synthetic_count, + ("cross_instrument_quality_failed",), + comparison_ids, + ) + lineage_tuple = tuple( + sorted(lineage, key=lambda item: (item.symbol, item.output_event_id)) + ) + quality_payload = cast(Mapping[str, JSONValue], quality_report.to_dict()) + action_counts = Counter( + action for item in lineage_tuple for action in item.actions + ) + manifest = BrokerTransferManifestV1( + run_id=run.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + input_group_id=group.group_id, + fingerprint_id=fingerprint.fingerprint_id, + transfer_config=policy, + selections=selections, + status=( + BrokerTransferStatus.BACKED_OFF + if any( + item.status is BrokerTransferStatus.BACKED_OFF + for item in selections + ) + else BrokerTransferStatus.APPLIED + ), + reason_codes=(), + input_content_sha256=input_hash, + output_content_sha256=_streams_content_sha256(stream_tuple), + observed_event_count=observed_count, + synthetic_event_count=synthetic_count, + action_counts=dict(action_counts), + lineage_count=len(lineage_tuple), + lineage_content_sha256=_content_sha256( + [item.to_dict() for item in lineage_tuple] + ), + local_validation_passed=True, + post_broker_validation_id=post_broker.validation_id, + post_broker_validation_status=post_broker.status.value, + cross_instrument_quality_status=quality_report.status.value, + cross_instrument_quality_sha256=_content_sha256(quality_payload), + benchmark_comparison_ids=comparison_ids, + ) + return BrokerRenderedGroupV1( + manifest=manifest, + streams=stream_tuple, + event_lineage=lineage_tuple, + post_broker_validation=post_broker, + cross_instrument_quality_payload=quality_payload, + ) + + +def _validate_render_scope( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + group: CrossCurrencyReconciledGroupV1, + constraints: HistoricalCarvingConstraintSetV1, +) -> None: + if run.run_id != window.run_id or group.run_id != run.run_id: + raise ValueError("broker render run scope differs") + if group.window_id != window.window_id: + raise ValueError("broker render window scope differs") + if ( + group.synchronization_unit_id != window.synchronization_unit_id + or group.ensemble_member_id != window.ensemble_member_id + or group.symbols != window.symbols + ): + raise ValueError("broker render synchronization scope differs") + if not isinstance(constraints, HistoricalCarvingConstraintSetV1): + raise TypeError("broker render requires historical constraints") + + +def _selection_symbol(selection: BrokerProfileSelectionV1) -> str | None: + value = selection.requested_condition.get("symbol") + return str(value).upper() if value is not None else None + + +def _shared_render_parameters( + selections: Sequence[BrokerProfileSelectionV1], + config: BrokerTransferConfigV1, +) -> dict[str, float | int]: + precisions = [ + item.metrics["source_timestamp_precision_ns"] + for item in selections + if "source_timestamp_precision_ns" in item.metrics + ] + target_precision = max(precisions) if precisions else 1.0 + effective_precision = max( + 1, + round(_blend(1.0, target_precision, config.strength)), + ) + batches = [ + item.metrics["source_batch_quote_count"] + for item in selections + if "source_batch_quote_count" in item.metrics + ] + target_batch = min(batches) if batches else 1.0 + effective_batch = max( + 1, + min( + config.max_batch_size, + round(_blend(1.0, target_batch, config.strength)), + ), + ) + stale_rates = [ + item.metrics.get("stale_quote_rate", 0.0) for item in selections + ] + duplicate_rates = [ + item.metrics.get("exact_duplicate_rate", 0.0) for item in selections + ] + return { + "precision_ns": effective_precision, + "batch_size": effective_batch, + "stale_rate": min(stale_rates, default=0.0), + "duplicate_rate": min(duplicate_rates, default=0.0), + } + + +def _render_stream( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + stream: SyntheticEventStreamV1, + fingerprint_id: str, + selection: BrokerProfileSelectionV1, + config: BrokerTransferConfigV1, + constraints: HistoricalCarvingConstraintSetV1, + shared: Mapping[str, float | int], +) -> tuple[ + SyntheticEventStreamV1 | None, + tuple[BrokerRenderLineageV1, ...], + str | None, +]: + observed = tuple( + item + for item in stream.events + if item.origin is SyntheticEventOrigin.OBSERVED + ) + synthetic = tuple( + item + for item in stream.events + if item.origin is SyntheticEventOrigin.SYNTHETIC + ) + anchors = {item.event_id: item for item in observed} + if any( + item.constraint_set_id != constraints.constraint_set_id + for item in synthetic + ): + return None, (), "constraint_set_id_differs" + precision_ns = cast(int, shared["precision_ns"]) + batch_size = cast(int, shared["batch_size"]) + batch_targets = _batch_targets( + synthetic, + batch_size if config.apply_batching else 1, + ) + pending: list[_PendingRender] = [] + previous_quote: tuple[float, float] | None = None + for source in synthetic: + left = anchors.get(cast(str, source.left_anchor_event_id)) + right = anchors.get(cast(str, source.right_anchor_event_id)) + if left is None or right is None: + return None, (), "anchor_lineage_missing" + if not left.event_time_ns < source.event_time_ns < right.event_time_ns: + return None, (), "input_synthetic_event_outside_anchors" + actions: list[str] = [] + requested_time = batch_targets.get( + source.event_id, source.event_time_ns + ) + if requested_time != source.event_time_ns: + actions.append("batched_timestamp") + quantized_time = _quantize_ns(requested_time, precision_ns) + if quantized_time != requested_time: + actions.append("timestamp_precision") + lower = max( + left.event_time_ns + 1, + source.event_time_ns - config.max_timestamp_shift_ns, + ) + upper = min( + right.event_time_ns - 1, + source.event_time_ns + config.max_timestamp_shift_ns, + ) + event_time = max(lower, min(upper, quantized_time)) + if _is_quarantined(stream.symbol, event_time, constraints): + if _is_quarantined( + stream.symbol, source.event_time_ns, constraints + ): + return None, (), "input_event_inside_quarantine" + event_time = source.event_time_ns + actions.append("constraint_time_preserved") + score = _deterministic_rate( + run, + window, + source, + config, + ) + duplicate_rate = ( + cast(float, shared["duplicate_rate"]) * config.strength + if config.apply_exact_duplicates + else 0.0 + ) + stale_rate = ( + cast(float, shared["stale_rate"]) * config.strength + if config.apply_stale_behavior + else 0.0 + ) + bid = source.bid + ask = source.ask + if previous_quote is not None and score < duplicate_rate: + bid, ask = previous_quote + actions.append("exact_duplicate_quote") + elif previous_quote is not None and score < duplicate_rate + stale_rate: + bid, ask = previous_quote + actions.append("stale_quote") + else: + bid, ask, projected = _render_quote( + source, + selection, + config, + constraints, + ) + if projected: + actions.append("spread_projection") + digits = _effective_price_digits(selection, config) + rounded_bid = round(bid, digits) + rounded_ask = round(ask, digits) + if rounded_bid != bid or rounded_ask != ask: + actions.append("price_precision") + bid, ask = rounded_bid, rounded_ask + if not ( + math.isfinite(bid) + and math.isfinite(ask) + and bid > 0.0 + and ask > 0.0 + and ask >= bid + ): + return None, (), "rendered_quote_domain_invalid" + previous_quote = (bid, ask) + pending.append( + _PendingRender( + source=source, + event_time_ns=event_time, + bid=bid, + ask=ask, + actions=actions, + ) + ) + used_positions = { + (item.event_time_ns, item.event_sequence) for item in observed + } + rendered: list[SyntheticEventV1] = [] + lineage: list[BrokerRenderLineageV1] = [] + for row in sorted( + pending, + key=lambda item: ( + item.event_time_ns, + item.source.event_time_ns, + item.source.event_sequence, + item.source.event_id, + ), + ): + sequence = row.source.event_sequence + while (row.event_time_ns, sequence) in used_positions: + sequence += 1 + used_positions.add((row.event_time_ns, sequence)) + source = row.source + event = SyntheticEventV1.generated( + symbol=source.symbol, + event_time_ns=row.event_time_ns, + event_sequence=sequence, + bid=row.bid, + ask=row.ask, + run_id=source.run_id, + ensemble_member_id=source.ensemble_member_id, + source_version_id=source.source_version_id, + anchor_interval_id=source.anchor_interval_id, + left_anchor_event_id=cast(str, source.left_anchor_event_id), + right_anchor_event_id=cast(str, source.right_anchor_event_id), + generator_id=cast(str, source.generator_id), + generator_version=cast(str, source.generator_version), + generator_config_id=cast(str, source.generator_config_id), + reference_id=source.reference_id, + motif_id=source.motif_id, + feed_epoch_id=source.feed_epoch_id, + broker_profile_id=fingerprint_id, + constraint_set_id=cast(str, source.constraint_set_id), + confidence=source.confidence, + ) + rendered.append(event) + lineage.append( + BrokerRenderLineageV1( + symbol=source.symbol, + input_event_id=source.event_id, + output_event_id=event.event_id, + selection_id=selection.selection_id, + input_event_time_ns=source.event_time_ns, + output_event_time_ns=event.event_time_ns, + actions=tuple(sorted(set(row.actions))), + ) + ) + try: + output = SyntheticEventStreamV1.merge( + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + symbol=stream.symbol, + observed_events=observed, + synthetic_events=rendered, + source_version_ids=stream.source_version_ids, + ) + except ValueError as err: + return None, (), f"rendered_stream_invalid:{err}" + return output, tuple(lineage), None + + +def _batch_targets( + events: Sequence[SyntheticEventV1], batch_size: int +) -> dict[str, int]: + if batch_size <= 1: + return {} + by_interval: dict[str, list[SyntheticEventV1]] = {} + for event in events: + by_interval.setdefault(cast(str, event.anchor_interval_id), []).append( + event + ) + targets: dict[str, int] = {} + for interval_events in by_interval.values(): + ordered = sorted( + interval_events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.event_id, + ), + ) + for start in range(0, len(ordered), batch_size): + chunk = ordered[start : start + batch_size] + if len(chunk) <= 1: + continue + target = chunk[-1].event_time_ns + targets.update({item.event_id: target for item in chunk}) + return targets + + +def _render_quote( + event: SyntheticEventV1, + selection: BrokerProfileSelectionV1, + config: BrokerTransferConfigV1, + constraints: HistoricalCarvingConstraintSetV1, +) -> tuple[float, float, bool]: + target = selection.metrics.get("spread") + if target is None or target < 0.0: + return event.bid, event.ask, False + spread = event.ask - event.bid + maximum_multiplier = min( + config.max_spread_multiplier, + constraints.max_combined_spread_multiplier, + ) + bounded_target = min(target, spread * maximum_multiplier) + rendered_spread = _blend(spread, bounded_target, config.strength) + midpoint = (event.bid + event.ask) / 2.0 + bid = midpoint - rendered_spread / 2.0 + ask = midpoint + rendered_spread / 2.0 + return ( + bid, + ask, + not math.isclose(rendered_spread, spread, rel_tol=0.0, abs_tol=1e-15), + ) + + +def _effective_price_digits( + selection: BrokerProfileSelectionV1, + config: BrokerTransferConfigV1, +) -> int: + target = selection.metrics.get( + "price_decimal_places", float(config.input_price_decimal_places) + ) + value = round( + _blend( + float(config.input_price_decimal_places), + target, + config.strength, + ) + ) + return max( + config.minimum_price_decimal_places, + min(config.input_price_decimal_places, value), + ) + + +def _deterministic_rate( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + event: SyntheticEventV1, + config: BrokerTransferConfigV1, +) -> float: + key = ( + f"{BROKER_TRANSFER_ENGINE_ID}:{config.config_id}:" + f"{window.synchronization_unit_id}:{event.event_time_ns}:" + f"{event.event_sequence}:delivery-behavior" + ) + return float(run.seed_for(window.ensemble_member_id, key)) / float(2**64) + + +def _is_quarantined( + symbol: str, + event_time_ns: int, + constraints: HistoricalCarvingConstraintSetV1, +) -> bool: + normalized = symbol.upper() + return any( + item.symbol == normalized + and item.start_ns <= event_time_ns < item.end_ns + for item in constraints.quarantines + ) + + +def _local_validation_reasons( + inputs: Sequence[SyntheticEventStreamV1], + outputs: Sequence[SyntheticEventStreamV1], + fingerprint_id: str, + constraints: HistoricalCarvingConstraintSetV1, +) -> tuple[str, ...]: + input_observed = { + item.event_id: item.to_dict() + for stream in inputs + for item in stream.events + if item.origin is SyntheticEventOrigin.OBSERVED + } + output_observed = { + item.event_id: item.to_dict() + for stream in outputs + for item in stream.events + if item.origin is SyntheticEventOrigin.OBSERVED + } + reasons: list[str] = [] + if input_observed != output_observed: + reasons.append("observed_anchor_content_changed") + for stream in outputs: + anchors = { + item.event_id: item + for item in stream.events + if item.origin is SyntheticEventOrigin.OBSERVED + } + previous: tuple[int, int] | None = None + for event in stream.events: + position = (event.event_time_ns, event.event_sequence) + if previous is not None and position <= previous: + reasons.append(f"event_order_invalid:{stream.symbol}") + previous = position + if event.ask < event.bid: + reasons.append(f"negative_spread:{stream.symbol}") + if event.origin is SyntheticEventOrigin.OBSERVED: + continue + if event.broker_profile_id != fingerprint_id: + reasons.append( + f"broker_profile_lineage_missing:{stream.symbol}" + ) + if event.constraint_set_id != constraints.constraint_set_id: + reasons.append(f"constraint_lineage_changed:{stream.symbol}") + left = anchors.get(cast(str, event.left_anchor_event_id)) + right = anchors.get(cast(str, event.right_anchor_event_id)) + if ( + left is None + or right is None + or not left.event_time_ns + < event.event_time_ns + < right.event_time_ns + ): + reasons.append(f"anchor_interval_violation:{stream.symbol}") + if _is_quarantined(stream.symbol, event.event_time_ns, constraints): + reasons.append(f"forbidden_interval_violation:{stream.symbol}") + return tuple(sorted(set(reasons))) + + +def _refused_render( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + group: CrossCurrencyReconciledGroupV1, + fingerprint: BrokerDeliveryFingerprintV1, + config: BrokerTransferConfigV1, + selections: Sequence[BrokerProfileSelectionV1], + input_hash: str, + observed_count: int, + synthetic_count: int, + reasons: Sequence[str], + benchmark_comparison_ids: Sequence[str], +) -> BrokerRenderedGroupV1: + manifest = BrokerTransferManifestV1( + run_id=run.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + input_group_id=group.group_id, + fingerprint_id=fingerprint.fingerprint_id, + transfer_config=config, + selections=tuple(selections), + status=BrokerTransferStatus.REFUSED, + reason_codes=tuple(sorted(set(reasons))), + input_content_sha256=input_hash, + output_content_sha256=None, + observed_event_count=observed_count, + synthetic_event_count=synthetic_count, + action_counts={}, + lineage_count=0, + lineage_content_sha256=None, + local_validation_passed=False, + post_broker_validation_id=None, + post_broker_validation_status=None, + cross_instrument_quality_status=None, + cross_instrument_quality_sha256=None, + benchmark_comparison_ids=tuple(benchmark_comparison_ids), + ) + return BrokerRenderedGroupV1( + manifest=manifest, + streams=(), + event_lineage=(), + post_broker_validation=None, + cross_instrument_quality_payload=None, + ) + + +def _quantize_ns(value: int, precision: int) -> int: + if precision <= 1: + return value + return ((2 * value + precision) // (2 * precision)) * precision + + +def _blend(historical: float, broker: float, strength: float) -> float: + return historical + strength * (broker - historical) + + +def _rounded(value: float, digits: int) -> float: + rounded = round(float(value), digits) + return 0.0 if rounded == 0.0 else rounded + + +def _streams_content_sha256( + streams: Iterable[SyntheticEventStreamV1], +) -> str: + return _content_sha256( + [ + item.to_dict() + for item in sorted(streams, key=lambda stream: stream.symbol) + ] + ) + + +def _content_sha256(value: Any) -> str: + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + return f"{prefix}:sha256:{_content_sha256(payload)}" + + +def _required_text(value: Any) -> str: + normalized = str(value or "").strip() + if not normalized or len(normalized) > MAX_BROKER_TRANSFER_TEXT: + raise ValueError("required broker transfer text is empty or unbounded") + return normalized + + +def _required_name(value: Any) -> str: + normalized = _required_text(value) + if any(character.isspace() for character in normalized): + raise ValueError("broker transfer names cannot contain whitespace") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return _required_text(normalized) if normalized else None + + +def _optional_sha256(value: Any) -> str | None: + normalized = _optional_text(value) + if normalized is None: + return None + if len(normalized) != 64 or any( + character not in "0123456789abcdef" for character in normalized + ): + raise ValueError("expected a lowercase SHA-256 digest") + return normalized + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + normalized = _strict_int(value, name) + if not minimum <= normalized <= maximum: + raise ValueError(f"{name} is outside supported bounds") + return normalized + + +def _strict_int(value: Any, name: str) -> int: + if type(value) is not int: + raise ValueError(f"{name} must be an integer") + return value + + +def _strict_bool(value: Any, name: str) -> bool: + if type(value) is not bool: + raise ValueError(f"{name} must be boolean") + return value + + +def _optional_int(value: Any) -> int | None: + return None if value is None else _strict_int(value, "optional integer") + + +def _finite_float(value: Any, name: str) -> float: + if type(value) not in {int, float}: + raise ValueError(f"{name} must be numeric") + normalized = float(value) + if not math.isfinite(normalized): + raise ValueError(f"{name} must be finite") + return normalized + + +def _metric_mapping(values: Mapping[str, Any]) -> dict[str, float]: + if len(values) > MAX_BROKER_TRANSFER_METRICS: + raise ValueError("broker transfer metrics exceed bounded limit") + return { + _required_name(str(name)): _finite_float(value, str(name)) + for name, value in sorted(values.items()) + } + + +def _count_mapping(values: Mapping[str, Any]) -> dict[str, int]: + if len(values) > MAX_BROKER_TRANSFER_ACTIONS: + raise ValueError("broker transfer actions exceed bounded limit") + return { + _required_name(str(name)): _bounded_int( + value, str(name), 0, MAX_BROKER_TRANSFER_EVENTS + ) + for name, value in sorted(values.items()) + } + + +def _bounded_text_tuple(values: Iterable[Any], maximum: int) -> tuple[str, ...]: + normalized = tuple(_required_text(value) for value in values) + if len(normalized) > maximum: + raise ValueError("broker transfer text collection exceeds limit") + return normalized + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return value + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError("expected a sequence of mappings") + return tuple(_mapping(item) for item in value) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError("expected a sequence of strings") + return tuple(str(item) for item in value) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + return _mapping(value) + + +def _json_mapping_value(value: Mapping[str, Any]) -> Mapping[str, JSONValue]: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")) + return _mapping(json.loads(encoded)) + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError("unsupported broker transfer schema") + + +__all__ = [ + "BROKER_BENCHMARK_COMPARISON_SCHEMA_VERSION", + "BROKER_CONDITIONED_PROPOSAL_SCHEMA_VERSION", + "BROKER_PROFILE_SELECTION_SCHEMA_VERSION", + "BROKER_RENDERED_GROUP_SCHEMA_VERSION", + "BROKER_RENDER_LINEAGE_SCHEMA_VERSION", + "BROKER_TRANSFER_CONFIG_SCHEMA_VERSION", + "BROKER_TRANSFER_ENGINE_ID", + "BROKER_TRANSFER_ENGINE_VERSION", + "BROKER_TRANSFER_MANIFEST_SCHEMA_VERSION", + "BrokerBenchmarkComparisonV1", + "BrokerConditionedProposalV1", + "BrokerProfileSelectionV1", + "BrokerRenderedGroupV1", + "BrokerRenderLineageV1", + "BrokerTransferConfigV1", + "BrokerTransferManifestV1", + "BrokerTransferStatus", + "compare_broker_benchmark_results", + "condition_broker_proposal", + "render_broker_delivery", + "select_broker_profile", +] diff --git a/src/histdatacom/synthetic/carving.py b/src/histdatacom/synthetic/carving.py new file mode 100644 index 00000000..a1c8331e --- /dev/null +++ b/src/histdatacom/synthetic/carving.py @@ -0,0 +1,2211 @@ +"""Deterministic historical carving for empirical-motif candidates. + +Carving is the first stage allowed to turn candidate-only motif rows into +accepted synthetic events. It is intentionally fail closed: immutable +anchors, market-context support, and a fingerprint-validation result are +bound before any conditioned thinning or spread projection is attempted. + +Rejected candidates never become a retained row set. The returned contract +contains exact reason counts and a bounded sample only. Accepted rows keep a +compact pointer to their candidate and motif transformation, including the +original quote whenever a projection changed it. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +import hashlib +import json +import math +from typing import TYPE_CHECKING, Any, cast + +from histdatacom.data_quality.synthetic_constraints import ( + SYNTHETIC_VALIDATION_SCHEMA_VERSION, +) +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.contracts import ( + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + canonical_contract_json, +) +from histdatacom.synthetic.generation import ( + CANDIDATE_ONLY_CONSTRAINT_SET_ID, + EmpiricalMotifCandidateBatchV1, + MotifGenerationStatus, +) +from histdatacom.synthetic.streaming import ( + CarryStateV1, + ReconstructionRunV1, + ReconstructionWindowV1, + RejectionSummaryV1, +) + +if TYPE_CHECKING: + from histdatacom.market_context.contracts import MarketContextQueryV1 + +CARVING_CONDITION_POLICY_SCHEMA_VERSION = ( + "histdatacom.historical-carving-condition-policy.v1" +) +CARVING_QUARANTINE_SCHEMA_VERSION = ( + "histdatacom.historical-carving-quarantine.v1" +) +CARVING_CONSTRAINT_SET_SCHEMA_VERSION = ( + "histdatacom.historical-carving-constraint-set.v1" +) +CARVING_FINGERPRINT_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.historical-carving-fingerprint-evidence.v1" +) +CARVING_REJECTION_EXAMPLE_SCHEMA_VERSION = ( + "histdatacom.historical-carving-rejection-example.v1" +) +CARVING_EVENT_LINEAGE_SCHEMA_VERSION = ( + "histdatacom.historical-carving-event-lineage.v1" +) +CARVING_VALIDATION_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.historical-carving-validation-evidence.v1" +) +CARVED_CANDIDATE_BATCH_SCHEMA_VERSION = ( + "histdatacom.historical-carved-candidate-batch.v1" +) + +HISTORICAL_CARVING_ENGINE_ID = "histdatacom.historical-carving" +HISTORICAL_CARVING_ENGINE_VERSION = "1.0.0" +HISTORICAL_CARVING_RULE_PRECEDENCE = ( + "hard.candidate_integrity.v1", + "hard.immutable_anchor.v1", + "hard.resource_envelope.v1", + "hard.fingerprint_validation.v1", + "hard.context_support.v1", + "hard.quarantine.v1", + "hard.session_closure.v1", + "conditioned.motif_eligibility.v1", + "conditioned.intensity.v1", + "conditioned.spread_projection.v1", + "hard.final_local_validation.v1", +) + +MAX_CARVING_POLICIES = 64 +MAX_CARVING_QUARANTINES = 4096 +MAX_CARVING_POLICY_TAGS = 64 +MAX_CARVING_ELIGIBLE_MOTIFS = 4096 +MAX_CARVING_REJECTION_EXAMPLES = 64 +MAX_CARVING_RULE_IDS_PER_EVENT = 128 +MAX_CARVING_CONTEXT_EVENT_IDS = 256 +MAX_CARVING_INPUT_BATCHES = 64 +MAX_CARVING_TEXT = 1024 +DEFAULT_MAX_ANCHOR_GAP_NS = 31 * 24 * 60 * 60 * 1_000_000_000 +DEFAULT_MAX_INPUT_CANDIDATES = 100_000 +DEFAULT_MAX_COMBINED_SPREAD_MULTIPLIER = 10.0 + + +class CarvingBatchStatus(str, Enum): + """Terminal status of one process-local carving batch.""" + + ACCEPTED = "accepted" + PARTIAL = "partial" + EMPTY = "empty" + REFUSED = "refused" + + +class CarvingEventAction(str, Enum): + """Stable action applied to one accepted event.""" + + ACCEPTED = "accepted" + PROJECTED = "projected" + SUBSTITUTED = "substituted" + SUBSTITUTED_AND_PROJECTED = "substituted_and_projected" + + +class CarvingReason(str, Enum): + """Bounded hard-rejection and refusal reason codes.""" + + UPSTREAM_EMPTY = "upstream_empty" + UPSTREAM_REFUSED = "upstream_refused" + ANCHOR_EVIDENCE_MISSING = "anchor_evidence_missing" + ANCHOR_GAP_LIMIT = "anchor_gap_limit" + RESOURCE_LIMIT = "resource_limit" + FINGERPRINT_EVIDENCE_MISSING = "fingerprint_evidence_missing" + FINGERPRINT_VALIDATION_FAILED = "fingerprint_validation_failed" + CONTEXT_SUPPORT_MISSING = "context_support_missing" + CONTEXT_PROFILE_INCOMPLETE = "context_profile_incomplete" + CLOSED_SESSION = "closed_session" + QUARANTINED_INTERVAL = "quarantined_interval" + INVALID_CANDIDATE = "invalid_candidate" + ANCHOR_VIOLATION = "anchor_violation" + OUTSIDE_WINDOW_OWNERSHIP = "outside_window_ownership" + MOTIF_INCOMPATIBLE = "motif_incompatible" + INTENSITY_THINNED = "intensity_thinned" + PROJECTION_LIMIT = "projection_limit" + FINAL_VALIDATION_FAILED = "final_validation_failed" + + +@dataclass(frozen=True, slots=True) +class HistoricalCarvingConditionPolicyV1: + """One explicit state-conditioned intensity and spread policy.""" + + name: str + match_tags: tuple[str, ...] + acceptance_rate: float = 1.0 + spread_multiplier: float = 1.0 + eligible_motif_ids: tuple[str, ...] = () + priority: int = 0 + policy_id: str = "" + schema_version: str = CARVING_CONDITION_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARVING_CONDITION_POLICY_SCHEMA_VERSION: + raise ValueError("unsupported carving condition policy") + object.__setattr__(self, "name", _bounded_text(self.name, "name")) + tags = _normalized_text_tuple( + self.match_tags, + "match_tags", + maximum=MAX_CARVING_POLICY_TAGS, + lowercase=True, + ) + if not tags: + raise ValueError("condition policy requires match_tags") + object.__setattr__(self, "match_tags", tags) + rate = _finite_float(self.acceptance_rate, "acceptance_rate") + if not 0.0 <= rate <= 1.0: + raise ValueError("acceptance_rate must be inside [0,1]") + object.__setattr__(self, "acceptance_rate", rate) + multiplier = _finite_float(self.spread_multiplier, "spread_multiplier") + if multiplier <= 0.0: + raise ValueError("spread_multiplier must be positive") + object.__setattr__(self, "spread_multiplier", multiplier) + motifs = _normalized_text_tuple( + self.eligible_motif_ids, + "eligible_motif_ids", + maximum=MAX_CARVING_ELIGIBLE_MOTIFS, + ) + object.__setattr__(self, "eligible_motif_ids", motifs) + priority = _strict_int(self.priority, "priority") + if priority < 0: + raise ValueError("priority must be non-negative") + object.__setattr__(self, "priority", priority) + expected = _stable_id("carving-condition-policy", self.payload()) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("carving condition policy_id differs") + object.__setattr__(self, "policy_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return semantic policy identity.""" + return { + "schema_version": self.schema_version, + "name": self.name, + "match_tags": list(self.match_tags), + "acceptance_rate": self.acceptance_rate, + "spread_multiplier": self.spread_multiplier, + "eligible_motif_ids": list(self.eligible_motif_ids), + "priority": self.priority, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible policy data.""" + return {**self.payload(), "policy_id": self.policy_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "HistoricalCarvingConditionPolicyV1": + """Restore and verify one condition policy.""" + return cls( + name=str(data.get("name", "")), + match_tags=_string_tuple(data.get("match_tags")), + acceptance_rate=cast(float, data.get("acceptance_rate")), + spread_multiplier=cast(float, data.get("spread_multiplier")), + eligible_motif_ids=_string_tuple(data.get("eligible_motif_ids")), + priority=cast(int, data.get("priority")), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "HistoricalCarvingConditionPolicyV1": + """Restore a condition policy from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class HistoricalCarvingQuarantineV1: + """One half-open interval in which synthetic liquidity is forbidden.""" + + symbol: str + start_ns: int + end_ns: int + reason: str + source_id: str + quarantine_id: str = "" + schema_version: str = CARVING_QUARANTINE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARVING_QUARANTINE_SCHEMA_VERSION: + raise ValueError("unsupported carving quarantine") + object.__setattr__(self, "symbol", _required_text(self.symbol).upper()) + start = _strict_int(self.start_ns, "start_ns") + end = _strict_int(self.end_ns, "end_ns") + if end <= start: + raise ValueError("quarantine end_ns must follow start_ns") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + object.__setattr__(self, "reason", _bounded_text(self.reason, "reason")) + object.__setattr__( + self, "source_id", _bounded_text(self.source_id, "source_id") + ) + expected = _stable_id("carving-quarantine", self.payload()) + supplied = _optional_text(self.quarantine_id) + if supplied is not None and supplied != expected: + raise ValueError("carving quarantine_id differs") + object.__setattr__(self, "quarantine_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return semantic quarantine identity.""" + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "interval_semantics": "[start_ns,end_ns)", + "reason": self.reason, + "source_id": self.source_id, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic quarantine data.""" + return {**self.payload(), "quarantine_id": self.quarantine_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "HistoricalCarvingQuarantineV1": + """Restore and verify one quarantine interval.""" + return cls( + symbol=str(data.get("symbol", "")), + start_ns=cast(int, data.get("start_ns")), + end_ns=cast(int, data.get("end_ns")), + reason=str(data.get("reason", "")), + source_id=str(data.get("source_id", "")), + quarantine_id=str(data.get("quarantine_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "HistoricalCarvingQuarantineV1": + """Restore a quarantine from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class HistoricalCarvingConstraintSetV1: + """Versioned fail-closed hard and conditioned carving constraints.""" + + fingerprint_constraint_id: str + condition_policies: tuple[HistoricalCarvingConditionPolicyV1, ...] = () + quarantines: tuple[HistoricalCarvingQuarantineV1, ...] = () + closed_session_states: tuple[str, ...] = ( + "closed", + "market_closed", + "weekend_closed", + ) + require_complete_calendar_profile: bool = True + require_fingerprint_validation: bool = True + max_anchor_gap_ns: int = DEFAULT_MAX_ANCHOR_GAP_NS + max_input_candidate_events: int = DEFAULT_MAX_INPUT_CANDIDATES + max_rejection_examples: int = 8 + max_combined_spread_multiplier: float = ( + DEFAULT_MAX_COMBINED_SPREAD_MULTIPLIER + ) + price_precision_digits: int = 8 + constraint_set_id: str = "" + schema_version: str = CARVING_CONSTRAINT_SET_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARVING_CONSTRAINT_SET_SCHEMA_VERSION: + raise ValueError("unsupported historical carving constraint set") + object.__setattr__( + self, + "fingerprint_constraint_id", + _bounded_text( + self.fingerprint_constraint_id, + "fingerprint_constraint_id", + ), + ) + policies = tuple( + sorted( + self.condition_policies, + key=lambda item: (-item.priority, item.policy_id), + ) + ) + if len(policies) > MAX_CARVING_POLICIES: + raise ValueError("condition policy count exceeds bounded limit") + if any( + not isinstance(item, HistoricalCarvingConditionPolicyV1) + for item in policies + ): + raise ValueError("condition_policies requires v1 policies") + if len({item.policy_id for item in policies}) != len(policies): + raise ValueError("condition policies must be unique") + object.__setattr__(self, "condition_policies", policies) + quarantines = tuple( + sorted( + self.quarantines, + key=lambda item: ( + item.symbol, + item.start_ns, + item.quarantine_id, + ), + ) + ) + if len(quarantines) > MAX_CARVING_QUARANTINES: + raise ValueError("quarantine count exceeds bounded limit") + if any( + not isinstance(item, HistoricalCarvingQuarantineV1) + for item in quarantines + ): + raise ValueError("quarantines requires v1 intervals") + if len({item.quarantine_id for item in quarantines}) != len( + quarantines + ): + raise ValueError("quarantines must be unique") + object.__setattr__(self, "quarantines", quarantines) + object.__setattr__( + self, + "closed_session_states", + _normalized_text_tuple( + self.closed_session_states, + "closed_session_states", + maximum=64, + lowercase=True, + ), + ) + for name in ( + "require_complete_calendar_profile", + "require_fingerprint_validation", + ): + if type(getattr(self, name)) is not bool: + raise ValueError(f"{name} must be boolean") + for name in ("max_anchor_gap_ns", "max_input_candidate_events"): + value = _strict_int(getattr(self, name), name) + if value <= 0: + raise ValueError(f"{name} must be positive") + object.__setattr__(self, name, value) + examples = _strict_int( + self.max_rejection_examples, "max_rejection_examples" + ) + if not 0 <= examples <= MAX_CARVING_REJECTION_EXAMPLES: + raise ValueError("max_rejection_examples exceeds bounded limit") + object.__setattr__(self, "max_rejection_examples", examples) + multiplier = _finite_float( + self.max_combined_spread_multiplier, + "max_combined_spread_multiplier", + ) + if multiplier < 1.0: + raise ValueError( + "max_combined_spread_multiplier must be at least one" + ) + object.__setattr__(self, "max_combined_spread_multiplier", multiplier) + digits = _strict_int( + self.price_precision_digits, "price_precision_digits" + ) + if not 0 <= digits <= 15: + raise ValueError("price_precision_digits must be inside [0,15]") + object.__setattr__(self, "price_precision_digits", digits) + expected = _stable_id("historical-carving-constraints", self.payload()) + supplied = _optional_text(self.constraint_set_id) + if supplied is not None and supplied != expected: + raise ValueError("historical carving constraint_set_id differs") + object.__setattr__(self, "constraint_set_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return semantic constraint identity and explicit precedence.""" + return { + "schema_version": self.schema_version, + "engine_id": HISTORICAL_CARVING_ENGINE_ID, + "engine_version": HISTORICAL_CARVING_ENGINE_VERSION, + "rule_precedence": list(HISTORICAL_CARVING_RULE_PRECEDENCE), + "fingerprint_constraint_id": self.fingerprint_constraint_id, + "condition_policies": [ + item.to_dict() for item in self.condition_policies + ], + "quarantines": [item.to_dict() for item in self.quarantines], + "closed_session_states": list(self.closed_session_states), + "require_complete_calendar_profile": ( + self.require_complete_calendar_profile + ), + "require_fingerprint_validation": ( + self.require_fingerprint_validation + ), + "max_anchor_gap_ns": self.max_anchor_gap_ns, + "max_input_candidate_events": self.max_input_candidate_events, + "max_rejection_examples": self.max_rejection_examples, + "max_combined_spread_multiplier": ( + self.max_combined_spread_multiplier + ), + "price_precision_digits": self.price_precision_digits, + "hard_constraints_fail_closed": True, + "conditioned_constraints_are_advisory": False, + "incompatible_motif_behavior": ( + "same-position-substitution-else-reject" + ), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible constraint data.""" + return {**self.payload(), "constraint_set_id": self.constraint_set_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "HistoricalCarvingConstraintSetV1": + """Restore and verify a version-one constraint set.""" + return cls( + fingerprint_constraint_id=str( + data.get("fingerprint_constraint_id", "") + ), + condition_policies=tuple( + HistoricalCarvingConditionPolicyV1.from_dict(item) + for item in _mapping_sequence(data.get("condition_policies")) + ), + quarantines=tuple( + HistoricalCarvingQuarantineV1.from_dict(item) + for item in _mapping_sequence(data.get("quarantines")) + ), + closed_session_states=_string_tuple( + data.get("closed_session_states") + ), + require_complete_calendar_profile=_strict_bool( + data.get("require_complete_calendar_profile"), + "require_complete_calendar_profile", + ), + require_fingerprint_validation=_strict_bool( + data.get("require_fingerprint_validation"), + "require_fingerprint_validation", + ), + max_anchor_gap_ns=cast(int, data.get("max_anchor_gap_ns")), + max_input_candidate_events=cast( + int, data.get("max_input_candidate_events") + ), + max_rejection_examples=cast( + int, data.get("max_rejection_examples") + ), + max_combined_spread_multiplier=cast( + float, data.get("max_combined_spread_multiplier") + ), + price_precision_digits=cast( + int, data.get("price_precision_digits") + ), + constraint_set_id=str(data.get("constraint_set_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "HistoricalCarvingConstraintSetV1": + """Restore a constraint set from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class CarvingFingerprintEvidenceV1: + """A matched existing validator result bound to exact candidate batches.""" + + validation_payload: Mapping[str, JSONValue] + candidate_batch_ids: tuple[str, ...] + reference_report_id: str + candidate_report_id: str + evidence_id: str = "" + schema_version: str = CARVING_FINGERPRINT_EVIDENCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARVING_FINGERPRINT_EVIDENCE_SCHEMA_VERSION: + raise ValueError("unsupported carving fingerprint evidence") + payload = dict(self.validation_payload) + if payload.get("schema_version") != SYNTHETIC_VALIDATION_SCHEMA_VERSION: + raise ValueError("fingerprint evidence has unsupported validation") + object.__setattr__(self, "validation_payload", payload) + batches = _normalized_text_tuple( + self.candidate_batch_ids, + "candidate_batch_ids", + maximum=MAX_CARVING_INPUT_BATCHES, + ) + if not batches: + raise ValueError("fingerprint evidence requires candidate batches") + object.__setattr__(self, "candidate_batch_ids", batches) + for name in ("reference_report_id", "candidate_report_id"): + object.__setattr__( + self, name, _bounded_text(getattr(self, name), name) + ) + expected = _stable_id("carving-fingerprint-evidence", self.payload()) + supplied = _optional_text(self.evidence_id) + if supplied is not None and supplied != expected: + raise ValueError("carving fingerprint evidence_id differs") + object.__setattr__(self, "evidence_id", expected) + + @property + def status(self) -> str: + """Return the existing validator's normalized aggregate status.""" + return str(self.validation_payload.get("status", "")).strip().lower() + + def payload(self) -> dict[str, JSONValue]: + """Return semantic evidence identity.""" + return { + "schema_version": self.schema_version, + "validator_schema_version": SYNTHETIC_VALIDATION_SCHEMA_VERSION, + "validation_payload": dict(self.validation_payload), + "candidate_batch_ids": list(self.candidate_batch_ids), + "reference_report_id": self.reference_report_id, + "candidate_report_id": self.candidate_report_id, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible evidence.""" + return {**self.payload(), "evidence_id": self.evidence_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CarvingFingerprintEvidenceV1": + """Restore and verify fingerprint evidence.""" + return cls( + validation_payload=_mapping(data.get("validation_payload")), + candidate_batch_ids=_string_tuple(data.get("candidate_batch_ids")), + reference_report_id=str(data.get("reference_report_id", "")), + candidate_report_id=str(data.get("candidate_report_id", "")), + evidence_id=str(data.get("evidence_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "CarvingFingerprintEvidenceV1": + """Restore fingerprint evidence from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class HistoricalCarvingRejectionExampleV1: + """One bounded non-row rejection pointer retained for diagnostics.""" + + candidate_event_id: str + candidate_content_sha256: str + candidate_batch_id: str + event_time_ns: int + event_sequence: int + reason: CarvingReason + rule_ids: tuple[str, ...] + example_id: str = "" + schema_version: str = CARVING_REJECTION_EXAMPLE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARVING_REJECTION_EXAMPLE_SCHEMA_VERSION: + raise ValueError("unsupported carving rejection example") + for name in ( + "candidate_event_id", + "candidate_batch_id", + ): + object.__setattr__( + self, name, _bounded_text(getattr(self, name), name) + ) + object.__setattr__( + self, + "candidate_content_sha256", + _sha256(self.candidate_content_sha256, "candidate_content_sha256"), + ) + object.__setattr__( + self, + "event_time_ns", + _strict_int(self.event_time_ns, "event_time_ns"), + ) + sequence = _strict_int(self.event_sequence, "event_sequence") + if sequence < 0: + raise ValueError("event_sequence must be non-negative") + object.__setattr__(self, "event_sequence", sequence) + object.__setattr__(self, "reason", CarvingReason(self.reason)) + rules = _normalized_text_tuple( + self.rule_ids, + "rule_ids", + maximum=MAX_CARVING_RULE_IDS_PER_EVENT, + ) + if not rules: + raise ValueError("rejection example requires rule_ids") + object.__setattr__(self, "rule_ids", rules) + expected = _stable_id("carving-rejection-example", self.payload()) + supplied = _optional_text(self.example_id) + if supplied is not None and supplied != expected: + raise ValueError("carving rejection example_id differs") + object.__setattr__(self, "example_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return non-row rejection identity.""" + return { + "schema_version": self.schema_version, + "candidate_event_id": self.candidate_event_id, + "candidate_content_sha256": self.candidate_content_sha256, + "candidate_batch_id": self.candidate_batch_id, + "event_time_ns": self.event_time_ns, + "event_sequence": self.event_sequence, + "reason": self.reason.value, + "rule_ids": list(self.rule_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic rejection evidence.""" + return {**self.payload(), "example_id": self.example_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "HistoricalCarvingRejectionExampleV1": + """Restore and verify one bounded example.""" + return cls( + candidate_event_id=str(data.get("candidate_event_id", "")), + candidate_content_sha256=str( + data.get("candidate_content_sha256", "") + ), + candidate_batch_id=str(data.get("candidate_batch_id", "")), + event_time_ns=cast(int, data.get("event_time_ns")), + event_sequence=cast(int, data.get("event_sequence")), + reason=CarvingReason(str(data.get("reason", ""))), + rule_ids=_string_tuple(data.get("rule_ids")), + example_id=str(data.get("example_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class HistoricalCarvingEventLineageV1: + """Compact accepted-event lineage across candidate and carving stages.""" + + output_event_id: str + output_content_sha256: str + candidate_event_id: str + candidate_content_sha256: str + candidate_batch_id: str + candidate_transformation_id: str + action: CarvingEventAction + rule_ids: tuple[str, ...] + context_event_ids: tuple[str, ...] + policy_ids: tuple[str, ...] + original_constraint_set_id: str + final_constraint_set_id: str + acceptance_score: float + spread_multiplier: float + original_bid: float | None = None + original_ask: float | None = None + lineage_id: str = "" + schema_version: str = CARVING_EVENT_LINEAGE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARVING_EVENT_LINEAGE_SCHEMA_VERSION: + raise ValueError("unsupported historical carving event lineage") + for name in ( + "output_event_id", + "candidate_event_id", + "candidate_batch_id", + "candidate_transformation_id", + "original_constraint_set_id", + "final_constraint_set_id", + ): + object.__setattr__( + self, name, _bounded_text(getattr(self, name), name) + ) + for name in ("output_content_sha256", "candidate_content_sha256"): + object.__setattr__(self, name, _sha256(getattr(self, name), name)) + object.__setattr__(self, "action", CarvingEventAction(self.action)) + rules = _normalized_text_tuple( + self.rule_ids, + "rule_ids", + maximum=MAX_CARVING_RULE_IDS_PER_EVENT, + ) + if not rules: + raise ValueError("accepted lineage requires rule_ids") + object.__setattr__(self, "rule_ids", rules) + object.__setattr__( + self, + "context_event_ids", + _normalized_text_tuple( + self.context_event_ids, + "context_event_ids", + maximum=MAX_CARVING_CONTEXT_EVENT_IDS, + ), + ) + object.__setattr__( + self, + "policy_ids", + _normalized_text_tuple( + self.policy_ids, + "policy_ids", + maximum=MAX_CARVING_POLICIES, + ), + ) + score = _finite_float(self.acceptance_score, "acceptance_score") + if not 0.0 <= score < 1.0: + raise ValueError("acceptance_score must be inside [0,1)") + object.__setattr__(self, "acceptance_score", score) + multiplier = _finite_float(self.spread_multiplier, "spread_multiplier") + if multiplier <= 0.0: + raise ValueError("spread_multiplier must be positive") + object.__setattr__(self, "spread_multiplier", multiplier) + projected = self.action in { + CarvingEventAction.PROJECTED, + CarvingEventAction.SUBSTITUTED_AND_PROJECTED, + } + if projected != ( + self.original_bid is not None and self.original_ask is not None + ): + raise ValueError("projected lineage requires both original quotes") + if self.original_bid is not None: + object.__setattr__( + self, + "original_bid", + _finite_float(self.original_bid, "original_bid"), + ) + object.__setattr__( + self, + "original_ask", + _finite_float(self.original_ask, "original_ask"), + ) + expected = _stable_id( + "historical-carving-event-lineage", self.payload() + ) + supplied = _optional_text(self.lineage_id) + if supplied is not None and supplied != expected: + raise ValueError("historical carving lineage_id differs") + object.__setattr__(self, "lineage_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return complete accepted-event decision lineage.""" + return { + "schema_version": self.schema_version, + "output_event_id": self.output_event_id, + "output_content_sha256": self.output_content_sha256, + "candidate_event_id": self.candidate_event_id, + "candidate_content_sha256": self.candidate_content_sha256, + "candidate_batch_id": self.candidate_batch_id, + "candidate_transformation_id": self.candidate_transformation_id, + "action": self.action.value, + "rule_ids": list(self.rule_ids), + "context_event_ids": list(self.context_event_ids), + "policy_ids": list(self.policy_ids), + "original_constraint_set_id": self.original_constraint_set_id, + "final_constraint_set_id": self.final_constraint_set_id, + "acceptance_score": self.acceptance_score, + "spread_multiplier": self.spread_multiplier, + "original_bid": self.original_bid, + "original_ask": self.original_ask, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic accepted lineage.""" + return {**self.payload(), "lineage_id": self.lineage_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "HistoricalCarvingEventLineageV1": + """Restore and verify accepted lineage.""" + return cls( + output_event_id=str(data.get("output_event_id", "")), + output_content_sha256=str(data.get("output_content_sha256", "")), + candidate_event_id=str(data.get("candidate_event_id", "")), + candidate_content_sha256=str( + data.get("candidate_content_sha256", "") + ), + candidate_batch_id=str(data.get("candidate_batch_id", "")), + candidate_transformation_id=str( + data.get("candidate_transformation_id", "") + ), + action=CarvingEventAction(str(data.get("action", ""))), + rule_ids=_string_tuple(data.get("rule_ids")), + context_event_ids=_string_tuple(data.get("context_event_ids")), + policy_ids=_string_tuple(data.get("policy_ids")), + original_constraint_set_id=str( + data.get("original_constraint_set_id", "") + ), + final_constraint_set_id=str( + data.get("final_constraint_set_id", "") + ), + acceptance_score=cast(float, data.get("acceptance_score")), + spread_multiplier=cast(float, data.get("spread_multiplier")), + original_bid=cast(float | None, data.get("original_bid")), + original_ask=cast(float | None, data.get("original_ask")), + lineage_id=str(data.get("lineage_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class HistoricalCarvingValidationEvidenceV1: + """Bound input and local final-validation evidence for one batch.""" + + fingerprint_evidence_id: str | None + local_validation_status: str + observed_anchor_content_sha256: tuple[str, ...] + validator_ids: tuple[str, ...] + evidence_id: str = "" + schema_version: str = CARVING_VALIDATION_EVIDENCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARVING_VALIDATION_EVIDENCE_SCHEMA_VERSION: + raise ValueError("unsupported carving validation evidence") + object.__setattr__( + self, + "fingerprint_evidence_id", + _optional_text(self.fingerprint_evidence_id), + ) + status = _required_text(self.local_validation_status).lower() + if status not in {"passed", "failed", "not_run"}: + raise ValueError("unsupported local validation status") + object.__setattr__(self, "local_validation_status", status) + anchors = tuple( + _sha256(item, "observed_anchor_content_sha256") + for item in self.observed_anchor_content_sha256 + ) + if len(anchors) > 2: + raise ValueError("validation evidence accepts at most two anchors") + object.__setattr__(self, "observed_anchor_content_sha256", anchors) + validators = _normalized_text_tuple( + self.validator_ids, + "validator_ids", + maximum=32, + ) + object.__setattr__(self, "validator_ids", validators) + expected = _stable_id("historical-carving-validation", self.payload()) + supplied = _optional_text(self.evidence_id) + if supplied is not None and supplied != expected: + raise ValueError( + "historical carving validation evidence_id differs" + ) + object.__setattr__(self, "evidence_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return deterministic validation evidence identity.""" + return { + "schema_version": self.schema_version, + "fingerprint_evidence_id": self.fingerprint_evidence_id, + "local_validation_status": self.local_validation_status, + "observed_anchor_content_sha256": list( + self.observed_anchor_content_sha256 + ), + "validator_ids": list(self.validator_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic validation evidence.""" + return {**self.payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "HistoricalCarvingValidationEvidenceV1": + """Restore and verify validation evidence.""" + return cls( + fingerprint_evidence_id=_mapping_optional_text( + data, "fingerprint_evidence_id" + ), + local_validation_status=str( + data.get("local_validation_status", "") + ), + observed_anchor_content_sha256=_string_tuple( + data.get("observed_anchor_content_sha256") + ), + validator_ids=_string_tuple(data.get("validator_ids")), + evidence_id=str(data.get("evidence_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class HistoricalCarvedCandidateBatchV1: + """Accepted rows plus compact carving evidence; never rejected rows.""" + + run_id: str + window_id: str + ensemble_member_id: str + symbol: str + anchor_interval_id: str + left_anchor_event_id: str + right_anchor_event_id: str + constraint_set_id: str + input_candidate_batch_ids: tuple[str, ...] + market_context_query_id: str + market_context_timeline_id: str + status: CarvingBatchStatus + accepted_events: tuple[SyntheticEventV1, ...] + accepted_lineage: tuple[HistoricalCarvingEventLineageV1, ...] + rejection_summary: RejectionSummaryV1 + rejection_examples: tuple[HistoricalCarvingRejectionExampleV1, ...] + validation_evidence: HistoricalCarvingValidationEvidenceV1 + carry_state: CarryStateV1 + projected_event_count: int = 0 + substituted_event_count: int = 0 + refusal_reason: CarvingReason | None = None + batch_id: str = "" + schema_version: str = CARVED_CANDIDATE_BATCH_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARVED_CANDIDATE_BATCH_SCHEMA_VERSION: + raise ValueError("unsupported historical carved candidate batch") + for name in ( + "run_id", + "window_id", + "ensemble_member_id", + "symbol", + "anchor_interval_id", + "left_anchor_event_id", + "right_anchor_event_id", + "constraint_set_id", + "market_context_query_id", + "market_context_timeline_id", + ): + object.__setattr__( + self, name, _bounded_text(getattr(self, name), name) + ) + batches = _normalized_text_tuple( + self.input_candidate_batch_ids, + "input_candidate_batch_ids", + maximum=MAX_CARVING_INPUT_BATCHES, + ) + if not batches: + raise ValueError("carved batch requires input candidate batches") + object.__setattr__(self, "input_candidate_batch_ids", batches) + object.__setattr__(self, "status", CarvingBatchStatus(self.status)) + events = tuple( + sorted( + self.accepted_events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.event_id, + ), + ) + ) + lineage = tuple( + sorted(self.accepted_lineage, key=lambda item: item.output_event_id) + ) + if any( + item.origin is not SyntheticEventOrigin.SYNTHETIC for item in events + ): + raise ValueError("carved batch accepts only synthetic events") + if any( + item.constraint_set_id != self.constraint_set_id for item in events + ): + raise ValueError("accepted event constraint_set_id differs") + if {item.event_id for item in events} != { + item.output_event_id for item in lineage + }: + raise ValueError("accepted lineage does not cover accepted events") + if len(events) != len(lineage): + raise ValueError("accepted event and lineage counts differ") + object.__setattr__(self, "accepted_events", events) + object.__setattr__(self, "accepted_lineage", lineage) + if not isinstance(self.rejection_summary, RejectionSummaryV1): + raise ValueError("carved batch requires a rejection summary") + if self.rejection_summary.run_id != self.run_id: + raise ValueError("rejection summary run differs") + if self.rejection_summary.window_id != self.window_id: + raise ValueError("rejection summary window differs") + if self.rejection_summary.accepted_count != len(events): + raise ValueError("rejection summary accepted count differs") + examples = tuple(self.rejection_examples) + if len(examples) > MAX_CARVING_REJECTION_EXAMPLES: + raise ValueError("rejection examples exceed bounded limit") + if any( + item.reason.value not in self.rejection_summary.reason_counts + for item in examples + ): + raise ValueError("rejection example lacks a summary reason") + object.__setattr__(self, "rejection_examples", examples) + if not isinstance( + self.validation_evidence, HistoricalCarvingValidationEvidenceV1 + ): + raise ValueError("carved batch requires validation evidence") + if not isinstance(self.carry_state, CarryStateV1): + raise ValueError("carved batch requires carry state") + if ( + self.carry_state.run_id != self.run_id + or self.carry_state.ensemble_member_id != self.ensemble_member_id + or self.symbol not in self.carry_state.symbol_watermarks_ns + ): + raise ValueError("carved batch carry state differs") + for name in ("projected_event_count", "substituted_event_count"): + value = _strict_int(getattr(self, name), name) + if not 0 <= value <= len(events): + raise ValueError(f"{name} is outside accepted event count") + object.__setattr__(self, name, value) + refusal = self.refusal_reason + if refusal is not None: + refusal = CarvingReason(refusal) + object.__setattr__(self, "refusal_reason", refusal) + _validate_batch_status( + self.status, + self.rejection_summary, + refusal, + ) + expected = _stable_id( + "historical-carved-candidate-batch", self.payload() + ) + supplied = _optional_text(self.batch_id) + if supplied is not None and supplied != expected: + raise ValueError("historical carved candidate batch_id differs") + object.__setattr__(self, "batch_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return batch identity with row payloads represented by hashes.""" + return { + "schema_version": self.schema_version, + "engine_id": HISTORICAL_CARVING_ENGINE_ID, + "engine_version": HISTORICAL_CARVING_ENGINE_VERSION, + "run_id": self.run_id, + "window_id": self.window_id, + "ensemble_member_id": self.ensemble_member_id, + "symbol": self.symbol, + "anchor_interval_id": self.anchor_interval_id, + "left_anchor_event_id": self.left_anchor_event_id, + "right_anchor_event_id": self.right_anchor_event_id, + "constraint_set_id": self.constraint_set_id, + "input_candidate_batch_ids": list(self.input_candidate_batch_ids), + "market_context_query_id": self.market_context_query_id, + "market_context_timeline_id": self.market_context_timeline_id, + "status": self.status.value, + "accepted_event_count": len(self.accepted_events), + "accepted_event_content_sha256": _content_sha256( + [item.to_dict() for item in self.accepted_events] + ), + "accepted_lineage_content_sha256": _content_sha256( + [item.to_dict() for item in self.accepted_lineage] + ), + "rejection_summary": self.rejection_summary.to_dict(), + "rejection_examples": [ + item.to_dict() for item in self.rejection_examples + ], + "validation_evidence": self.validation_evidence.to_dict(), + "carry_id": self.carry_state.carry_id, + "projected_event_count": self.projected_event_count, + "substituted_event_count": self.substituted_event_count, + "refusal_reason": ( + self.refusal_reason.value if self.refusal_reason else None + ), + "rejected_rows_retained": False, + "final_storage_status": "not_persisted", + } + + def metadata(self) -> dict[str, JSONValue]: + """Return bounded workflow metadata without accepted event rows.""" + return { + **self.payload(), + "batch_id": self.batch_id, + "accepted_events_inline": False, + "accepted_lineage_inline": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return process-local accepted rows and bounded evidence.""" + return { + **self.payload(), + "batch_id": self.batch_id, + "accepted_events": [ + item.to_dict() for item in self.accepted_events + ], + "accepted_lineage": [ + item.to_dict() for item in self.accepted_lineage + ], + "carry_state": self.carry_state.to_dict(), + } + + def to_json(self) -> str: + """Return deterministic process-local JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "HistoricalCarvedCandidateBatchV1": + """Restore accepted rows and verify every derived identity.""" + refusal = data.get("refusal_reason") + return cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbol=str(data.get("symbol", "")), + anchor_interval_id=str(data.get("anchor_interval_id", "")), + left_anchor_event_id=str(data.get("left_anchor_event_id", "")), + right_anchor_event_id=str(data.get("right_anchor_event_id", "")), + constraint_set_id=str(data.get("constraint_set_id", "")), + input_candidate_batch_ids=_string_tuple( + data.get("input_candidate_batch_ids") + ), + market_context_query_id=str( + data.get("market_context_query_id", "") + ), + market_context_timeline_id=str( + data.get("market_context_timeline_id", "") + ), + status=CarvingBatchStatus(str(data.get("status", ""))), + accepted_events=tuple( + SyntheticEventV1.from_dict(item) + for item in _mapping_sequence(data.get("accepted_events")) + ), + accepted_lineage=tuple( + HistoricalCarvingEventLineageV1.from_dict(item) + for item in _mapping_sequence(data.get("accepted_lineage")) + ), + rejection_summary=RejectionSummaryV1.from_dict( + _mapping(data.get("rejection_summary")) + ), + rejection_examples=tuple( + HistoricalCarvingRejectionExampleV1.from_dict(item) + for item in _mapping_sequence(data.get("rejection_examples")) + ), + validation_evidence=( + HistoricalCarvingValidationEvidenceV1.from_dict( + _mapping(data.get("validation_evidence")) + ) + ), + carry_state=CarryStateV1.from_dict( + _mapping(data.get("carry_state")) + ), + projected_event_count=cast(int, data.get("projected_event_count")), + substituted_event_count=cast( + int, data.get("substituted_event_count") + ), + refusal_reason=( + CarvingReason(str(refusal)) if refusal is not None else None + ), + batch_id=str(data.get("batch_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "HistoricalCarvedCandidateBatchV1": + """Restore a carved batch from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + def merged_stream( + self, observed_events: Sequence[SyntheticEventV1] + ) -> SyntheticEventStreamV1: + """Merge accepted rows with unchanged caller-owned observations.""" + return SyntheticEventStreamV1.merge( + run_id=self.run_id, + ensemble_member_id=self.ensemble_member_id, + symbol=self.symbol, + observed_events=observed_events, + synthetic_events=self.accepted_events, + ) + + +def carve_empirical_motif_candidates( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + candidate_batch: EmpiricalMotifCandidateBatchV1, + observed_events: Sequence[SyntheticEventV1], + market_context: MarketContextQueryV1, + constraints: HistoricalCarvingConstraintSetV1, + fingerprint_evidence: CarvingFingerprintEvidenceV1 | None, + substitution_batches: Sequence[EmpiricalMotifCandidateBatchV1] = (), +) -> HistoricalCarvedCandidateBatchV1: + """Carve one candidate batch using deterministic fail-closed precedence.""" + candidates = (candidate_batch, *tuple(substitution_batches)) + _validate_scope(run, window, candidates, market_context, constraints) + ordered_batches = ( + candidate_batch, + *tuple(sorted(substitution_batches, key=lambda item: item.batch_id)), + ) + input_batch_ids = tuple(item.batch_id for item in ordered_batches) + anchors = _observed_anchors(candidate_batch, observed_events) + anchor_hashes = tuple(_event_content_sha256(item) for item in anchors) + + if candidate_batch.status is MotifGenerationStatus.EMPTY: + return _terminal_batch( + run=run, + window=window, + candidate_batch=candidate_batch, + input_batch_ids=input_batch_ids, + market_context=market_context, + constraints=constraints, + anchors=anchors, + anchor_hashes=anchor_hashes, + fingerprint_evidence=fingerprint_evidence, + status=CarvingBatchStatus.EMPTY, + reason=CarvingReason.UPSTREAM_EMPTY, + ) + if candidate_batch.status is MotifGenerationStatus.REFUSED: + return _terminal_batch( + run=run, + window=window, + candidate_batch=candidate_batch, + input_batch_ids=input_batch_ids, + market_context=market_context, + constraints=constraints, + anchors=anchors, + anchor_hashes=anchor_hashes, + fingerprint_evidence=fingerprint_evidence, + status=CarvingBatchStatus.REFUSED, + reason=CarvingReason.UPSTREAM_REFUSED, + ) + + refusal = _support_refusal( + run, + candidate_batch, + ordered_batches, + anchors, + market_context, + constraints, + fingerprint_evidence, + ) + if refusal is not None: + return _terminal_batch( + run=run, + window=window, + candidate_batch=candidate_batch, + input_batch_ids=input_batch_ids, + market_context=market_context, + constraints=constraints, + anchors=anchors, + anchor_hashes=anchor_hashes, + fingerprint_evidence=fingerprint_evidence, + status=CarvingBatchStatus.REFUSED, + reason=refusal, + ) + + if _closed_session(candidate_batch, market_context, constraints): + return _terminal_batch( + run=run, + window=window, + candidate_batch=candidate_batch, + input_batch_ids=input_batch_ids, + market_context=market_context, + constraints=constraints, + anchors=anchors, + anchor_hashes=anchor_hashes, + fingerprint_evidence=fingerprint_evidence, + status=CarvingBatchStatus.REFUSED, + reason=CarvingReason.CLOSED_SESSION, + ) + + alternatives = _alternative_events(ordered_batches[1:]) + accepted: list[SyntheticEventV1] = [] + lineage: list[HistoricalCarvingEventLineageV1] = [] + rejected: Counter[str] = Counter() + examples: list[HistoricalCarvingRejectionExampleV1] = [] + projected_count = 0 + substituted_count = 0 + left_anchor, right_anchor = anchors + for primary in candidate_batch.events: + hard_reason = _candidate_hard_reason( + primary, + window, + left_anchor, + right_anchor, + constraints, + ) + if hard_reason is not None: + _record_rejection( + rejected, + examples, + primary, + candidate_batch, + hard_reason, + _rule_for_reason(hard_reason), + constraints.max_rejection_examples, + ) + continue + policies, context_event_ids = _matching_policies( + primary, + candidate_batch, + market_context, + constraints, + ) + selected_batch = candidate_batch + selected = primary + if not _motif_eligible(primary, policies): + replacement = _eligible_substitution( + primary, + alternatives, + policies, + ) + if replacement is None: + _record_rejection( + rejected, + examples, + primary, + candidate_batch, + CarvingReason.MOTIF_INCOMPATIBLE, + "conditioned.motif_eligibility.v1", + constraints.max_rejection_examples, + ) + continue + selected_batch, selected = replacement + substituted_count += 1 + acceptance_rate = math.prod(item.acceptance_rate for item in policies) + score = _acceptance_score(run, selected, constraints, policies) + if score >= acceptance_rate: + _record_rejection( + rejected, + examples, + selected, + selected_batch, + CarvingReason.INTENSITY_THINNED, + "conditioned.intensity.v1", + constraints.max_rejection_examples, + ) + continue + spread_multiplier = math.prod( + item.spread_multiplier for item in policies + ) + if spread_multiplier > constraints.max_combined_spread_multiplier: + _record_rejection( + rejected, + examples, + selected, + selected_batch, + CarvingReason.PROJECTION_LIMIT, + "conditioned.spread_projection.v1", + constraints.max_rejection_examples, + ) + continue + try: + output, projected = _accepted_event( + selected, + constraints, + spread_multiplier, + ) + except ValueError: + _record_rejection( + rejected, + examples, + selected, + selected_batch, + CarvingReason.PROJECTION_LIMIT, + "conditioned.spread_projection.v1", + constraints.max_rejection_examples, + ) + continue + substituted = selected_batch.batch_id != candidate_batch.batch_id + action = _accepted_action(projected, substituted) + if projected: + projected_count += 1 + source_lineage = selected_batch.lineage_for(selected.event_id) + rule_ids = ( + "hard.candidate_integrity.v1", + "hard.immutable_anchor.v1", + "hard.resource_envelope.v1", + "hard.fingerprint_validation.v1", + "hard.context_support.v1", + "hard.quarantine.v1", + "hard.session_closure.v1", + "conditioned.motif_eligibility.v1", + "conditioned.intensity.v1", + "conditioned.spread_projection.v1", + "hard.final_local_validation.v1", + ) + lineage.append( + HistoricalCarvingEventLineageV1( + output_event_id=output.event_id, + output_content_sha256=_event_content_sha256(output), + candidate_event_id=selected.event_id, + candidate_content_sha256=_event_content_sha256(selected), + candidate_batch_id=selected_batch.batch_id, + candidate_transformation_id=(source_lineage.transformation_id), + action=action, + rule_ids=rule_ids, + context_event_ids=context_event_ids, + policy_ids=tuple(item.policy_id for item in policies), + original_constraint_set_id=( + selected.constraint_set_id + or CANDIDATE_ONLY_CONSTRAINT_SET_ID + ), + final_constraint_set_id=constraints.constraint_set_id, + acceptance_score=score, + spread_multiplier=spread_multiplier, + original_bid=selected.bid if projected else None, + original_ask=selected.ask if projected else None, + ) + ) + accepted.append(output) + + try: + _validate_accepted_events( + accepted, + observed_events, + candidate_batch, + window, + constraints, + ) + except ValueError: + return _terminal_batch( + run=run, + window=window, + candidate_batch=candidate_batch, + input_batch_ids=input_batch_ids, + market_context=market_context, + constraints=constraints, + anchors=anchors, + anchor_hashes=anchor_hashes, + fingerprint_evidence=fingerprint_evidence, + status=CarvingBatchStatus.REFUSED, + reason=CarvingReason.FINAL_VALIDATION_FAILED, + ) + + summary = RejectionSummaryV1( + run_id=run.run_id, + window_id=window.window_id, + candidate_count=len(candidate_batch.events), + accepted_count=len(accepted), + rejected_count=sum(rejected.values()), + reason_counts=dict(rejected), + ) + status = ( + CarvingBatchStatus.ACCEPTED + if not rejected + else ( + CarvingBatchStatus.PARTIAL + if accepted + else CarvingBatchStatus.REFUSED + ) + ) + refusal_reason = None + if status is CarvingBatchStatus.REFUSED: + refusal_reason = _primary_rejection_reason(rejected) + validation = HistoricalCarvingValidationEvidenceV1( + fingerprint_evidence_id=( + fingerprint_evidence.evidence_id if fingerprint_evidence else None + ), + local_validation_status="passed", + observed_anchor_content_sha256=anchor_hashes, + validator_ids=( + SYNTHETIC_VALIDATION_SCHEMA_VERSION, + "histdatacom.historical-carving-local-event-validator.v1", + ), + ) + return HistoricalCarvedCandidateBatchV1( + run_id=run.run_id, + window_id=window.window_id, + ensemble_member_id=window.ensemble_member_id, + symbol=candidate_batch.symbol, + anchor_interval_id=candidate_batch.anchor_interval_id, + left_anchor_event_id=candidate_batch.left_anchor_event_id, + right_anchor_event_id=candidate_batch.right_anchor_event_id, + constraint_set_id=constraints.constraint_set_id, + input_candidate_batch_ids=input_batch_ids, + market_context_query_id=market_context.query_id, + market_context_timeline_id=market_context.timeline_id, + status=status, + accepted_events=tuple(accepted), + accepted_lineage=tuple(lineage), + rejection_summary=summary, + rejection_examples=tuple(examples), + validation_evidence=validation, + carry_state=_carry_state( + run, + window, + left_anchor, + right_anchor, + accepted, + ), + projected_event_count=projected_count, + substituted_event_count=substituted_count, + refusal_reason=refusal_reason, + ) + + +def _validate_scope( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + batches: Sequence[EmpiricalMotifCandidateBatchV1], + market_context: MarketContextQueryV1, + constraints: HistoricalCarvingConstraintSetV1, +) -> None: + if window.run_id != run.run_id: + raise ValueError("carving window does not belong to run") + if constraints.constraint_set_id not in run.configuration_ids: + raise ValueError("carving constraint set is absent from run") + if not batches or len(batches) > MAX_CARVING_INPUT_BATCHES: + raise ValueError("carving input batch count is outside bounds") + primary = batches[0] + for batch in batches: + if not isinstance(batch, EmpiricalMotifCandidateBatchV1): + raise ValueError("carving requires motif candidate batches") + if ( + batch.run_id != run.run_id + or batch.window_id != window.window_id + or batch.ensemble_member_id != window.ensemble_member_id + or batch.symbol != primary.symbol + or batch.anchor_interval_id != primary.anchor_interval_id + or batch.left_anchor_event_id != primary.left_anchor_event_id + or batch.right_anchor_event_id != primary.right_anchor_event_id + ): + raise ValueError("substitution candidate scope differs") + if batch.generator_config_id not in run.configuration_ids: + raise ValueError("candidate generator config is absent from run") + if ( + primary.symbol not in run.symbols + or primary.symbol not in window.symbols + ): + raise ValueError("carving symbol is outside run/window scope") + if market_context.window_id is not None and ( + market_context.window_id != window.window_id + ): + raise ValueError("market context window_id differs") + if ( + market_context.start_ns > window.core_start_ns + or market_context.end_ns < window.core_end_ns + ): + raise ValueError("market context does not cover carving window") + if market_context.requested_symbols and primary.symbol.upper() not in { + item.upper() for item in market_context.requested_symbols + }: + raise ValueError("market context requested symbols omit candidate") + if ( + market_context.information_mode + is not primary.query_result.query.information_mode + ): + raise ValueError("market context information mode differs") + + +def _observed_anchors( + batch: EmpiricalMotifCandidateBatchV1, + observed_events: Sequence[SyntheticEventV1], +) -> tuple[SyntheticEventV1, SyntheticEventV1]: + by_id = {item.event_id: item for item in observed_events} + left = by_id.get(batch.left_anchor_event_id) + right = by_id.get(batch.right_anchor_event_id) + if left is None or right is None: + raise ValueError(CarvingReason.ANCHOR_EVIDENCE_MISSING.value) + if ( + left.origin is not SyntheticEventOrigin.OBSERVED + or right.origin is not SyntheticEventOrigin.OBSERVED + or left.symbol != batch.symbol + or right.symbol != batch.symbol + ): + raise ValueError("carving anchors are not immutable observations") + return left, right + + +def _support_refusal( + run: ReconstructionRunV1, + primary: EmpiricalMotifCandidateBatchV1, + batches: Sequence[EmpiricalMotifCandidateBatchV1], + anchors: tuple[SyntheticEventV1, SyntheticEventV1], + market_context: MarketContextQueryV1, + constraints: HistoricalCarvingConstraintSetV1, + fingerprint_evidence: CarvingFingerprintEvidenceV1 | None, +) -> CarvingReason | None: + if anchors[1].event_time_ns - anchors[0].event_time_ns > ( + constraints.max_anchor_gap_ns + ): + return CarvingReason.ANCHOR_GAP_LIMIT + if sum(len(item.events) for item in batches) > ( + constraints.max_input_candidate_events + ): + return CarvingReason.RESOURCE_LIMIT + if len(primary.events) > run.storage_policy.max_events_per_batch: + return CarvingReason.RESOURCE_LIMIT + if constraints.require_fingerprint_validation: + if fingerprint_evidence is None: + return CarvingReason.FINGERPRINT_EVIDENCE_MISSING + if not set(item.batch_id for item in batches).issubset( + fingerprint_evidence.candidate_batch_ids + ): + return CarvingReason.FINGERPRINT_VALIDATION_FAILED + if fingerprint_evidence.status != "match": + return CarvingReason.FINGERPRINT_VALIDATION_FAILED + missing_reason = market_context.missing_reason + missing_reason_value = ( + missing_reason.value if missing_reason is not None else None + ) + if missing_reason_value not in {None, "no_matching_event"}: + return CarvingReason.CONTEXT_SUPPORT_MISSING + if market_context.calendar_state is None: + return CarvingReason.CONTEXT_SUPPORT_MISSING + if ( + constraints.require_complete_calendar_profile + and not market_context.calendar_state.profile_complete + ): + return CarvingReason.CONTEXT_PROFILE_INCOMPLETE + return None + + +def _closed_session( + batch: EmpiricalMotifCandidateBatchV1, + market_context: MarketContextQueryV1, + constraints: HistoricalCarvingConstraintSetV1, +) -> bool: + states = set(constraints.closed_session_states) + calendar = market_context.calendar_state + return ( + batch.query_result.query.condition.session_state.lower() in states + or (calendar is not None and calendar.session_state.lower() in states) + ) + + +def _candidate_hard_reason( + event: SyntheticEventV1, + window: ReconstructionWindowV1, + left_anchor: SyntheticEventV1, + right_anchor: SyntheticEventV1, + constraints: HistoricalCarvingConstraintSetV1, +) -> CarvingReason | None: + if ( + event.origin is not SyntheticEventOrigin.SYNTHETIC + or event.constraint_set_id != CANDIDATE_ONLY_CONSTRAINT_SET_ID + or not math.isfinite(event.bid) + or not math.isfinite(event.ask) + or event.bid <= 0.0 + or event.ask < event.bid + ): + return CarvingReason.INVALID_CANDIDATE + if not ( + left_anchor.event_time_ns + < event.event_time_ns + < right_anchor.event_time_ns + ): + return CarvingReason.ANCHOR_VIOLATION + if not window.owns_event_time(event.event_time_ns): + return CarvingReason.OUTSIDE_WINDOW_OWNERSHIP + for quarantine in constraints.quarantines: + if ( + quarantine.symbol == event.symbol.upper() + and quarantine.start_ns <= event.event_time_ns < quarantine.end_ns + ): + return CarvingReason.QUARANTINED_INTERVAL + return None + + +def _matching_policies( + event: SyntheticEventV1, + batch: EmpiricalMotifCandidateBatchV1, + market_context: MarketContextQueryV1, + constraints: HistoricalCarvingConstraintSetV1, +) -> tuple[tuple[HistoricalCarvingConditionPolicyV1, ...], tuple[str, ...]]: + tokens = { + batch.query_result.query.condition.session_state.lower(), + *( + item.lower() + for item in batch.query_result.query.condition.special_tags + ), + *( + item.lower() + for item in batch.query_result.query.condition.event_tags + ), + } + calendar = market_context.calendar_state + if calendar is not None: + tokens.update( + item.lower() + for item in ( + *calendar.clock_sessions, + *calendar.active_sessions, + *calendar.overlaps, + *calendar.special_tags, + *calendar.holiday_tags, + *calendar.event_tags, + *calendar.calendar_tags, + ) + ) + tokens.add(calendar.session_state.lower()) + context_ids: list[str] = [] + for context_event in market_context.events: + if not context_event.overlaps( + event.event_time_ns, event.event_time_ns + 1 + ): + continue + if context_event.affected_symbols and event.symbol.upper() not in { + item.upper() for item in context_event.affected_symbols + }: + continue + context_ids.append(context_event.event_id) + tokens.add(context_event.kind.value) + tokens.update(item.lower() for item in context_event.tags) + policies = tuple( + item + for item in constraints.condition_policies + if set(item.match_tags).intersection(tokens) + ) + return policies, tuple(sorted(context_ids)) + + +def _motif_eligible( + event: SyntheticEventV1, + policies: Sequence[HistoricalCarvingConditionPolicyV1], +) -> bool: + return all( + not item.eligible_motif_ids or event.motif_id in item.eligible_motif_ids + for item in policies + ) + + +def _alternative_events( + batches: Sequence[EmpiricalMotifCandidateBatchV1], +) -> dict[ + tuple[int, int], + tuple[tuple[EmpiricalMotifCandidateBatchV1, SyntheticEventV1], ...], +]: + indexed: dict[ + tuple[int, int], + list[tuple[EmpiricalMotifCandidateBatchV1, SyntheticEventV1]], + ] = {} + for batch in batches: + if batch.status is not MotifGenerationStatus.GENERATED: + continue + for event in batch.events: + indexed.setdefault( + (event.event_time_ns, event.event_sequence), [] + ).append((batch, event)) + return { + key: tuple(sorted(value, key=lambda item: item[1].event_id)) + for key, value in indexed.items() + } + + +def _eligible_substitution( + primary: SyntheticEventV1, + alternatives: Mapping[ + tuple[int, int], + Sequence[tuple[EmpiricalMotifCandidateBatchV1, SyntheticEventV1]], + ], + policies: Sequence[HistoricalCarvingConditionPolicyV1], +) -> tuple[EmpiricalMotifCandidateBatchV1, SyntheticEventV1] | None: + for batch, event in alternatives.get( + (primary.event_time_ns, primary.event_sequence), () + ): + if _motif_eligible(event, policies): + return batch, event + return None + + +def _acceptance_score( + run: ReconstructionRunV1, + event: SyntheticEventV1, + constraints: HistoricalCarvingConstraintSetV1, + policies: Sequence[HistoricalCarvingConditionPolicyV1], +) -> float: + semantic_key = canonical_contract_json( + { + "stage": HISTORICAL_CARVING_ENGINE_ID, + "constraint_set_id": constraints.constraint_set_id, + "anchor_interval_id": event.anchor_interval_id, + "event_time_ns": event.event_time_ns, + "event_sequence": event.event_sequence, + "policy_ids": [item.policy_id for item in policies], + } + ) + value = run.seed_for(event.ensemble_member_id, semantic_key) + return float(value) / (2**64) + + +def _accepted_event( + candidate: SyntheticEventV1, + constraints: HistoricalCarvingConstraintSetV1, + spread_multiplier: float, +) -> tuple[SyntheticEventV1, bool]: + projected = not math.isclose( + spread_multiplier, 1.0, rel_tol=0.0, abs_tol=1e-15 + ) + bid = candidate.bid + ask = candidate.ask + if projected: + midpoint = (bid + ask) / 2.0 + half_spread = (ask - bid) * spread_multiplier / 2.0 + bid = round(midpoint - half_spread, constraints.price_precision_digits) + ask = round(midpoint + half_spread, constraints.price_precision_digits) + if bid <= 0.0 or ask < bid: + raise ValueError("conditioned spread projection is invalid") + return ( + SyntheticEventV1.generated( + symbol=candidate.symbol, + event_time_ns=candidate.event_time_ns, + event_sequence=candidate.event_sequence, + bid=bid, + ask=ask, + run_id=candidate.run_id, + ensemble_member_id=candidate.ensemble_member_id, + source_version_id=candidate.source_version_id, + anchor_interval_id=candidate.anchor_interval_id, + left_anchor_event_id=cast(str, candidate.left_anchor_event_id), + right_anchor_event_id=cast(str, candidate.right_anchor_event_id), + generator_id=cast(str, candidate.generator_id), + generator_version=cast(str, candidate.generator_version), + generator_config_id=cast(str, candidate.generator_config_id), + reference_id=candidate.reference_id, + motif_id=candidate.motif_id, + feed_epoch_id=candidate.feed_epoch_id, + broker_profile_id=candidate.broker_profile_id, + constraint_set_id=constraints.constraint_set_id, + confidence=cast(float, candidate.confidence), + ), + projected, + ) + + +def _accepted_action(projected: bool, substituted: bool) -> CarvingEventAction: + if projected and substituted: + return CarvingEventAction.SUBSTITUTED_AND_PROJECTED + if projected: + return CarvingEventAction.PROJECTED + if substituted: + return CarvingEventAction.SUBSTITUTED + return CarvingEventAction.ACCEPTED + + +def _validate_accepted_events( + accepted: Sequence[SyntheticEventV1], + observed_events: Sequence[SyntheticEventV1], + batch: EmpiricalMotifCandidateBatchV1, + window: ReconstructionWindowV1, + constraints: HistoricalCarvingConstraintSetV1, +) -> None: + if any( + item.constraint_set_id != constraints.constraint_set_id + or item.left_anchor_event_id != batch.left_anchor_event_id + or item.right_anchor_event_id != batch.right_anchor_event_id + or not window.owns_event_time(item.event_time_ns) + for item in accepted + ): + raise ValueError("accepted event violates final local constraints") + positions = [(item.event_time_ns, item.event_sequence) for item in accepted] + if len(set(positions)) != len(positions): + raise ValueError("accepted events contain duplicate positions") + stream = SyntheticEventStreamV1.merge( + run_id=batch.run_id, + ensemble_member_id=batch.ensemble_member_id, + symbol=batch.symbol, + observed_events=observed_events, + synthetic_events=accepted, + ) + observed_ids = { + item.event_id + for item in stream.events + if item.origin is SyntheticEventOrigin.OBSERVED + } + if observed_ids != {item.event_id for item in observed_events}: + raise ValueError("final local validation dropped an observed event") + + +def _terminal_batch( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + candidate_batch: EmpiricalMotifCandidateBatchV1, + input_batch_ids: tuple[str, ...], + market_context: MarketContextQueryV1, + constraints: HistoricalCarvingConstraintSetV1, + anchors: tuple[SyntheticEventV1, SyntheticEventV1], + anchor_hashes: tuple[str, ...], + fingerprint_evidence: CarvingFingerprintEvidenceV1 | None, + status: CarvingBatchStatus, + reason: CarvingReason, +) -> HistoricalCarvedCandidateBatchV1: + candidate_count = len(candidate_batch.events) + reasons = {reason.value: candidate_count} if candidate_count else {} + examples: list[HistoricalCarvingRejectionExampleV1] = [] + for event in candidate_batch.events[: constraints.max_rejection_examples]: + examples.append( + HistoricalCarvingRejectionExampleV1( + candidate_event_id=event.event_id, + candidate_content_sha256=_event_content_sha256(event), + candidate_batch_id=candidate_batch.batch_id, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + reason=reason, + rule_ids=(_rule_for_reason(reason),), + ) + ) + summary = RejectionSummaryV1( + run_id=run.run_id, + window_id=window.window_id, + candidate_count=candidate_count, + accepted_count=0, + rejected_count=candidate_count, + reason_counts=reasons, + ) + validation = HistoricalCarvingValidationEvidenceV1( + fingerprint_evidence_id=( + fingerprint_evidence.evidence_id if fingerprint_evidence else None + ), + local_validation_status=( + "failed" + if reason is CarvingReason.FINAL_VALIDATION_FAILED + else "not_run" + ), + observed_anchor_content_sha256=anchor_hashes, + validator_ids=tuple( + item + for item in ( + ( + SYNTHETIC_VALIDATION_SCHEMA_VERSION + if fingerprint_evidence is not None + else None + ), + "histdatacom.historical-carving-local-event-validator.v1", + ) + if item is not None + ), + ) + left_anchor, right_anchor = anchors + return HistoricalCarvedCandidateBatchV1( + run_id=run.run_id, + window_id=window.window_id, + ensemble_member_id=window.ensemble_member_id, + symbol=candidate_batch.symbol, + anchor_interval_id=candidate_batch.anchor_interval_id, + left_anchor_event_id=candidate_batch.left_anchor_event_id, + right_anchor_event_id=candidate_batch.right_anchor_event_id, + constraint_set_id=constraints.constraint_set_id, + input_candidate_batch_ids=input_batch_ids, + market_context_query_id=market_context.query_id, + market_context_timeline_id=market_context.timeline_id, + status=status, + accepted_events=(), + accepted_lineage=(), + rejection_summary=summary, + rejection_examples=tuple(examples), + validation_evidence=validation, + carry_state=_carry_state(run, window, left_anchor, right_anchor, ()), + refusal_reason=reason, + ) + + +def _record_rejection( + rejected: Counter[str], + examples: list[HistoricalCarvingRejectionExampleV1], + event: SyntheticEventV1, + batch: EmpiricalMotifCandidateBatchV1, + reason: CarvingReason, + rule_id: str, + example_limit: int, +) -> None: + rejected[reason.value] += 1 + if len(examples) >= example_limit: + return + examples.append( + HistoricalCarvingRejectionExampleV1( + candidate_event_id=event.event_id, + candidate_content_sha256=_event_content_sha256(event), + candidate_batch_id=batch.batch_id, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + reason=reason, + rule_ids=(rule_id,), + ) + ) + + +def _carry_state( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + left_anchor: SyntheticEventV1, + right_anchor: SyntheticEventV1, + accepted: Sequence[SyntheticEventV1], +) -> CarryStateV1: + last_event = accepted[-1] if accepted else left_anchor + if window.owns_event_time(right_anchor.event_time_ns): + last_event = right_anchor + watermark = min( + window.core_end_ns - 1, + max(window.core_start_ns, right_anchor.event_time_ns - 1), + ) + return CarryStateV1( + run_id=run.run_id, + ensemble_member_id=window.ensemble_member_id, + symbol_watermarks_ns={left_anchor.symbol: watermark}, + last_event_ids={left_anchor.symbol: last_event.event_id}, + ) + + +def _validate_batch_status( + status: CarvingBatchStatus, + summary: RejectionSummaryV1, + refusal: CarvingReason | None, +) -> None: + if status is CarvingBatchStatus.ACCEPTED and ( + summary.accepted_count == 0 or summary.rejected_count != 0 or refusal + ): + raise ValueError("accepted carving status does not reconcile") + if status is CarvingBatchStatus.PARTIAL and not ( + summary.accepted_count > 0 + and summary.rejected_count > 0 + and not refusal + ): + raise ValueError("partial carving status does not reconcile") + if status is CarvingBatchStatus.EMPTY and ( + summary.candidate_count != 0 + or refusal is not CarvingReason.UPSTREAM_EMPTY + ): + raise ValueError("empty carving status does not reconcile") + if status is CarvingBatchStatus.REFUSED and ( + summary.accepted_count != 0 or refusal is None + ): + raise ValueError("refused carving status does not reconcile") + + +def _rule_for_reason(reason: CarvingReason) -> str: + return { + CarvingReason.UPSTREAM_EMPTY: "hard.candidate_integrity.v1", + CarvingReason.UPSTREAM_REFUSED: "hard.candidate_integrity.v1", + CarvingReason.ANCHOR_EVIDENCE_MISSING: "hard.immutable_anchor.v1", + CarvingReason.ANCHOR_GAP_LIMIT: "hard.resource_envelope.v1", + CarvingReason.RESOURCE_LIMIT: "hard.resource_envelope.v1", + CarvingReason.FINGERPRINT_EVIDENCE_MISSING: ( + "hard.fingerprint_validation.v1" + ), + CarvingReason.FINGERPRINT_VALIDATION_FAILED: ( + "hard.fingerprint_validation.v1" + ), + CarvingReason.CONTEXT_SUPPORT_MISSING: "hard.context_support.v1", + CarvingReason.CONTEXT_PROFILE_INCOMPLETE: "hard.context_support.v1", + CarvingReason.CLOSED_SESSION: "hard.session_closure.v1", + CarvingReason.QUARANTINED_INTERVAL: "hard.quarantine.v1", + CarvingReason.INVALID_CANDIDATE: "hard.candidate_integrity.v1", + CarvingReason.ANCHOR_VIOLATION: "hard.immutable_anchor.v1", + CarvingReason.OUTSIDE_WINDOW_OWNERSHIP: ("hard.candidate_integrity.v1"), + CarvingReason.MOTIF_INCOMPATIBLE: ("conditioned.motif_eligibility.v1"), + CarvingReason.INTENSITY_THINNED: "conditioned.intensity.v1", + CarvingReason.PROJECTION_LIMIT: ("conditioned.spread_projection.v1"), + CarvingReason.FINAL_VALIDATION_FAILED: ( + "hard.final_local_validation.v1" + ), + }[reason] + + +def _primary_rejection_reason(rejected: Mapping[str, int]) -> CarvingReason: + reasons = tuple(CarvingReason(item) for item in rejected) + return min( + reasons, + key=lambda item: HISTORICAL_CARVING_RULE_PRECEDENCE.index( + _rule_for_reason(item) + ), + ) + + +def _event_content_sha256(event: SyntheticEventV1) -> str: + return _content_sha256(event.to_dict()) + + +def _content_sha256(value: JSONValue) -> str: + return hashlib.sha256( + canonical_contract_json(value).encode("utf-8") + ).hexdigest() + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + return f"{prefix}:sha256:{_content_sha256(dict(payload))}" + + +def _required_text(value: Any) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError("value must be non-empty text") + return value.strip() + + +def _bounded_text(value: Any, name: str) -> str: + text = _required_text(value) + if len(text) > MAX_CARVING_TEXT: + raise ValueError(f"{name} exceeds bounded text length") + return text + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, str) and not value.strip(): + return None + return _required_text(value) + + +def _strict_int(value: Any, name: str) -> int: + if type(value) is not int: + raise ValueError(f"{name} must be an integer") + return value + + +def _strict_bool(value: Any, name: str) -> bool: + if type(value) is not bool: + raise ValueError(f"{name} must be boolean") + return value + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _normalized_text_tuple( + values: Sequence[str], + name: str, + *, + maximum: int, + lowercase: bool = False, +) -> tuple[str, ...]: + normalized = { + ( + _bounded_text(item, name).lower() + if lowercase + else _bounded_text(item, name) + ) + for item in values + } + if len(normalized) > maximum: + raise ValueError(f"{name} exceeds bounded limit") + return tuple(sorted(normalized)) + + +def _sha256(value: Any, name: str) -> str: + text = _required_text(value) + if len(text) != 64 or any(item not in "0123456789abcdef" for item in text): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return text + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return cast(Mapping[str, Any], value) + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + if not isinstance(value, (list, tuple)): + raise ValueError("expected a sequence") + return tuple(_mapping(item) for item in value) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + if not isinstance(value, (list, tuple)): + raise ValueError("expected a string sequence") + if any(not isinstance(item, str) for item in value): + raise ValueError("expected string sequence values") + return tuple(cast(Sequence[str], value)) + + +def _mapping_optional_text(data: Mapping[str, Any], key: str) -> str | None: + value = data.get(key) + return None if value is None else str(value) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + return _mapping(value) diff --git a/src/histdatacom/synthetic/certification.py b/src/histdatacom/synthetic/certification.py new file mode 100644 index 00000000..9f8f36f6 --- /dev/null +++ b/src/histdatacom/synthetic/certification.py @@ -0,0 +1,2782 @@ +"""Fail-closed release certification for reconstructed market products. + +The certification layer binds compact report artifacts from the reconstruction +pipeline to predeclared requirements. It never retains event rows, analytical +frames, model objects, or other tick-sized intermediates. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from pathlib import Path, PurePosixPath +from typing import Any, cast + +from histdatacom.runtime_contracts import ArtifactRef, JSONScalar, JSONValue +from histdatacom.synthetic.contracts import canonical_contract_json + +CERTIFICATION_ARTIFACT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-artifact.v1" +) +CERTIFICATION_REQUIREMENT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-requirement.v1" +) +CERTIFICATION_OBSERVATION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-observation.v1" +) +CERTIFICATION_CHECK_RESULT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-check-result.v1" +) +CERTIFICATION_GATE_RESULT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-gate-result.v1" +) +RECONSTRUCTION_CERTIFICATION_POLICY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-policy.v1" +) +RECONSTRUCTION_CERTIFICATION_DOSSIER_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-dossier.v1" +) +RECONSTRUCTION_CERTIFICATION_POLICY_V2_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-policy.v2" +) +RECONSTRUCTION_CERTIFICATION_DOSSIER_V2_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-dossier.v2" +) + +EURUSD_TRIANGLE_SYMBOLS = ("EURGBP", "EURUSD", "GBPUSD") +EURUSD_TRIANGLE_COMMON_START_PERIOD = "200203" +MODERN_REFERENCE_DELIVERY_MODE = "modern_reference" +MODERN_REFERENCE_DELIVERY_CLAIM = "unconditioned_reference" +PROMOTION_ONLY_CHECK_IDS = frozenset({"coverage_promotion_run_count"}) + +DEFAULT_CERTIFICATION_MAX_ARTIFACTS = 256 +DEFAULT_CERTIFICATION_MAX_REQUIREMENTS = 128 +DEFAULT_CERTIFICATION_MAX_OBSERVATIONS = 128 +DEFAULT_CERTIFICATION_MAX_PAYLOAD_BYTES = 8_388_608 +DEFAULT_CERTIFICATION_MAX_METADATA_ITEMS = 64 +DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH = 16_384 + + +class CertificationGate(str, Enum): + """The fifteen release gates declared by GitHub issue #449.""" + + IDENTITY_AND_ANCHORS = "identity-and-anchors" + INFORMATION_SAFETY = "information-safety" + REVERSE_DEGRADATION = "reverse-degradation" + CONDITIONED_SCORECARDS = "conditioned-scorecards" + CROSS_CURRENCY = "cross-currency" + ENSEMBLE_EVIDENCE = "ensemble-evidence" + PRODUCT_RECONCILIATION = "product-reconciliation" + FAILURE_RESUME = "failure-resume" + REPLAY = "replay" + RESOURCES = "resources" + NEGATIVE_TESTS = "negative-tests" + STRATEGY_SENSITIVITY = "strategy-sensitivity" + DOSSIER_PUBLICATION = "dossier-publication" + REPOSITORY_GATES = "repository-gates" + TESTPYPI_PREFLIGHT = "testpypi-preflight" + + +class CertificationComparator(str, Enum): + """Deterministic comparison applied to one measured observation.""" + + EQUAL = "equal" + LESS_OR_EQUAL = "less-or-equal" + GREATER_OR_EQUAL = "greater-or-equal" + TRUE = "true" + FALSE = "false" + ZERO = "zero" + + +class CertificationCheckStatus(str, Enum): + """Outcome of one predeclared certification check.""" + + PASSED = "passed" + FAILED = "failed" + MISSING = "missing" + + +class CertificationState(str, Enum): + """Overall certification state without implicit promotion.""" + + CERTIFIED = "certified" + READY_FOR_PROMOTION = "ready-for-promotion" + FAILED = "failed" + INCOMPLETE = "incomplete" + + +@dataclass(frozen=True, slots=True) +class CertificationArtifactV1: + """Compact content identity for one independently produced artifact.""" + + policy_id: str + kind: str + subject_id: str + subject_schema_version: str + content_sha256: str + relative_path: str + size_bytes: int + verified: bool + metadata: Mapping[str, JSONValue] + evidence_id: str = "" + schema_version: str = CERTIFICATION_ARTIFACT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CERTIFICATION_ARTIFACT_SCHEMA_VERSION, + "certification artifact", + ) + object.__setattr__(self, "policy_id", _required_text(self.policy_id)) + for name in ("kind", "subject_id", "subject_schema_version"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, "content_sha256", _required_sha256(self.content_sha256) + ) + object.__setattr__( + self, "relative_path", _safe_relative_path(self.relative_path) + ) + object.__setattr__( + self, + "size_bytes", + _nonnegative_int(self.size_bytes, "size_bytes"), + ) + object.__setattr__( + self, "verified", _strict_bool(self.verified, "verified") + ) + object.__setattr__( + self, + "metadata", + _bounded_mapping( + self.metadata, + "certification artifact metadata", + DEFAULT_CERTIFICATION_MAX_METADATA_ITEMS, + ), + ) + expected = _stable_id("certification-artifact", self.identity_payload()) + supplied = _optional_text(self.evidence_id) + if supplied is not None and supplied != expected: + raise ValueError("certification artifact evidence_id differs") + object.__setattr__(self, "evidence_id", expected) + + @classmethod + def from_payload( + cls, + *, + policy_id: str, + kind: str, + subject_id: str, + subject_schema_version: str, + payload: Mapping[str, JSONValue], + relative_path: str, + verified: bool = True, + metadata: Mapping[str, JSONValue] | None = None, + ) -> "CertificationArtifactV1": + """Bind one compact JSON contract without retaining its source rows.""" + content = canonical_contract_json(dict(payload)).encode("utf-8") + return cls( + policy_id=policy_id, + kind=kind, + subject_id=subject_id, + subject_schema_version=subject_schema_version, + content_sha256=hashlib.sha256(content).hexdigest(), + relative_path=relative_path, + size_bytes=len(content), + verified=verified, + metadata=metadata or {}, + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return content-addressed artifact evidence.""" + return { + "schema_version": self.schema_version, + "policy_id": self.policy_id, + "kind": self.kind, + "subject_id": self.subject_id, + "subject_schema_version": self.subject_schema_version, + "content_sha256": self.content_sha256, + "relative_path": self.relative_path, + "size_bytes": self.size_bytes, + "verified": self.verified, + "metadata": dict(self.metadata), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible evidence.""" + return {**self.identity_payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CertificationArtifactV1": + """Restore and verify artifact evidence.""" + _require_schema(data, CERTIFICATION_ARTIFACT_SCHEMA_VERSION) + return cls( + policy_id=str(data.get("policy_id", "")), + kind=str(data.get("kind", "")), + subject_id=str(data.get("subject_id", "")), + subject_schema_version=str(data.get("subject_schema_version", "")), + content_sha256=str(data.get("content_sha256", "")), + relative_path=str(data.get("relative_path", "")), + size_bytes=_strict_int(data.get("size_bytes"), "size_bytes"), + verified=_strict_bool(data.get("verified"), "verified"), + metadata=_mapping(data.get("metadata"), "metadata"), + evidence_id=str(data.get("evidence_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CertificationRequirementV1: + """One predeclared gate comparison and its required artifact kinds.""" + + gate: CertificationGate + check_id: str + comparator: CertificationComparator + expected: JSONScalar + required_artifact_kinds: tuple[str, ...] + description: str + requirement_id: str = "" + schema_version: str = CERTIFICATION_REQUIREMENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CERTIFICATION_REQUIREMENT_SCHEMA_VERSION, + "certification requirement", + ) + object.__setattr__(self, "gate", CertificationGate(self.gate)) + object.__setattr__(self, "check_id", _required_name(self.check_id)) + comparator = CertificationComparator(self.comparator) + object.__setattr__(self, "comparator", comparator) + _validate_expected(comparator, self.expected) + kinds = _normalized_text_tuple(self.required_artifact_kinds) + if not kinds: + raise ValueError( + "certification requirement requires artifact kinds" + ) + object.__setattr__(self, "required_artifact_kinds", kinds) + object.__setattr__(self, "description", _bounded_text(self.description)) + expected_id = _stable_id( + "certification-requirement", self.identity_payload() + ) + supplied = _optional_text(self.requirement_id) + if supplied is not None and supplied != expected_id: + raise ValueError("certification requirement_id differs") + object.__setattr__(self, "requirement_id", expected_id) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic policy content.""" + return { + "schema_version": self.schema_version, + "gate": self.gate.value, + "check_id": self.check_id, + "comparator": self.comparator.value, + "expected": self.expected, + "required_artifact_kinds": list(self.required_artifact_kinds), + "description": self.description, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible policy content.""" + return { + **self.identity_payload(), + "requirement_id": self.requirement_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CertificationRequirementV1": + """Restore one predeclared requirement.""" + _require_schema(data, CERTIFICATION_REQUIREMENT_SCHEMA_VERSION) + return cls( + gate=CertificationGate(str(data.get("gate", ""))), + check_id=str(data.get("check_id", "")), + comparator=CertificationComparator(str(data.get("comparator", ""))), + expected=_json_scalar(data.get("expected"), "expected"), + required_artifact_kinds=_string_tuple( + data.get("required_artifact_kinds"), "required_artifact_kinds" + ), + description=str(data.get("description", "")), + requirement_id=str(data.get("requirement_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CertificationObservationV1: + """One measured value bound to independently verified artifacts.""" + + check_id: str + actual: JSONScalar + artifact_evidence_ids: tuple[str, ...] + note: str = "" + observation_id: str = "" + schema_version: str = CERTIFICATION_OBSERVATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CERTIFICATION_OBSERVATION_SCHEMA_VERSION, + "certification observation", + ) + object.__setattr__(self, "check_id", _required_name(self.check_id)) + _json_scalar(self.actual, "actual") + evidence = _normalized_text_tuple(self.artifact_evidence_ids) + if not evidence: + raise ValueError( + "certification observation requires artifact evidence" + ) + object.__setattr__(self, "artifact_evidence_ids", evidence) + object.__setattr__( + self, "note", _bounded_text(self.note, allow_empty=True) + ) + expected = _stable_id( + "certification-observation", self.identity_payload() + ) + supplied = _optional_text(self.observation_id) + if supplied is not None and supplied != expected: + raise ValueError("certification observation_id differs") + object.__setattr__(self, "observation_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic observation content.""" + return { + "schema_version": self.schema_version, + "check_id": self.check_id, + "actual": self.actual, + "artifact_evidence_ids": list(self.artifact_evidence_ids), + "note": self.note, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible observation content.""" + return { + **self.identity_payload(), + "observation_id": self.observation_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CertificationObservationV1": + """Restore one content-bound observation.""" + _require_schema(data, CERTIFICATION_OBSERVATION_SCHEMA_VERSION) + return cls( + check_id=str(data.get("check_id", "")), + actual=_json_scalar(data.get("actual"), "actual"), + artifact_evidence_ids=_string_tuple( + data.get("artifact_evidence_ids"), "artifact_evidence_ids" + ), + note=str(data.get("note", "")), + observation_id=str(data.get("observation_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CertificationCheckResultV1: + """Computed outcome for one policy requirement.""" + + requirement_id: str + check_id: str + status: CertificationCheckStatus + comparator: CertificationComparator + expected: JSONScalar + actual: JSONScalar + artifact_evidence_ids: tuple[str, ...] + reason: str + result_id: str = "" + schema_version: str = CERTIFICATION_CHECK_RESULT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CERTIFICATION_CHECK_RESULT_SCHEMA_VERSION, + "certification check result", + ) + object.__setattr__( + self, "requirement_id", _required_text(self.requirement_id) + ) + object.__setattr__(self, "check_id", _required_name(self.check_id)) + object.__setattr__( + self, "status", CertificationCheckStatus(self.status) + ) + object.__setattr__( + self, "comparator", CertificationComparator(self.comparator) + ) + _json_scalar(self.expected, "expected") + _json_scalar(self.actual, "actual") + object.__setattr__( + self, + "artifact_evidence_ids", + _normalized_text_tuple(self.artifact_evidence_ids), + ) + object.__setattr__(self, "reason", _bounded_text(self.reason)) + expected_id = _stable_id( + "certification-check-result", self.identity_payload() + ) + supplied = _optional_text(self.result_id) + if supplied is not None and supplied != expected_id: + raise ValueError("certification check result_id differs") + object.__setattr__(self, "result_id", expected_id) + + @property + def passed(self) -> bool: + """Return whether the measured result satisfied its policy.""" + return self.status is CertificationCheckStatus.PASSED + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic result content.""" + return { + "schema_version": self.schema_version, + "requirement_id": self.requirement_id, + "check_id": self.check_id, + "status": self.status.value, + "comparator": self.comparator.value, + "expected": self.expected, + "actual": self.actual, + "artifact_evidence_ids": list(self.artifact_evidence_ids), + "reason": self.reason, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible result content.""" + return {**self.identity_payload(), "result_id": self.result_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CertificationCheckResultV1": + """Restore one computed check result.""" + _require_schema(data, CERTIFICATION_CHECK_RESULT_SCHEMA_VERSION) + return cls( + requirement_id=str(data.get("requirement_id", "")), + check_id=str(data.get("check_id", "")), + status=CertificationCheckStatus(str(data.get("status", ""))), + comparator=CertificationComparator(str(data.get("comparator", ""))), + expected=_json_scalar(data.get("expected"), "expected"), + actual=_json_scalar(data.get("actual"), "actual"), + artifact_evidence_ids=_string_tuple( + data.get("artifact_evidence_ids"), "artifact_evidence_ids" + ), + reason=str(data.get("reason", "")), + result_id=str(data.get("result_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CertificationGateResultV1: + """All computed checks for one issue acceptance gate.""" + + gate: CertificationGate + check_results: tuple[CertificationCheckResultV1, ...] + gate_result_id: str = "" + schema_version: str = CERTIFICATION_GATE_RESULT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CERTIFICATION_GATE_RESULT_SCHEMA_VERSION, + "certification gate result", + ) + object.__setattr__(self, "gate", CertificationGate(self.gate)) + results = tuple( + sorted(self.check_results, key=lambda item: item.check_id) + ) + if not results: + raise ValueError("certification gate result requires checks") + if len({item.check_id for item in results}) != len(results): + raise ValueError("certification gate result duplicates checks") + object.__setattr__(self, "check_results", results) + expected = _stable_id( + "certification-gate-result", self.identity_payload() + ) + supplied = _optional_text(self.gate_result_id) + if supplied is not None and supplied != expected: + raise ValueError("certification gate_result_id differs") + object.__setattr__(self, "gate_result_id", expected) + + @property + def status(self) -> CertificationCheckStatus: + """Return the fail-closed aggregate status for the gate.""" + statuses = {item.status for item in self.check_results} + if CertificationCheckStatus.FAILED in statuses: + return CertificationCheckStatus.FAILED + if CertificationCheckStatus.MISSING in statuses: + return CertificationCheckStatus.MISSING + return CertificationCheckStatus.PASSED + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic gate evidence.""" + return { + "schema_version": self.schema_version, + "gate": self.gate.value, + "status": self.status.value, + "check_results": [item.to_dict() for item in self.check_results], + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible gate evidence.""" + return { + **self.identity_payload(), + "gate_result_id": self.gate_result_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CertificationGateResultV1": + """Restore one gate result and verify its derived status.""" + _require_schema(data, CERTIFICATION_GATE_RESULT_SCHEMA_VERSION) + result = cls( + gate=CertificationGate(str(data.get("gate", ""))), + check_results=tuple( + CertificationCheckResultV1.from_dict(item) + for item in _mapping_sequence( + data.get("check_results"), "check_results" + ) + ), + gate_result_id=str(data.get("gate_result_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + if data.get("status") != result.status.value: + raise ValueError("certification gate status differs") + return result + + +@dataclass(frozen=True, slots=True) +class ReconstructionCertificationPolicyV1: + """Predeclared product scope, thresholds, and release requirements.""" + + product_version: str + symbols: tuple[str, ...] + common_start_period: str + common_end_period: str + broker_fingerprint_id: str + requirements: tuple[CertificationRequirementV1, ...] + max_artifacts: int = DEFAULT_CERTIFICATION_MAX_ARTIFACTS + max_observations: int = DEFAULT_CERTIFICATION_MAX_OBSERVATIONS + max_payload_bytes: int = DEFAULT_CERTIFICATION_MAX_PAYLOAD_BYTES + policy_id: str = "" + schema_version: str = RECONSTRUCTION_CERTIFICATION_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_CERTIFICATION_POLICY_SCHEMA_VERSION, + "reconstruction certification policy", + ) + object.__setattr__( + self, "product_version", _required_text(self.product_version) + ) + symbols = tuple( + sorted({_normalized_symbol(item) for item in self.symbols}) + ) + if symbols != tuple(sorted(EURUSD_TRIANGLE_SYMBOLS)): + raise ValueError("v1 certification scope is the EURUSD triangle") + object.__setattr__(self, "symbols", symbols) + start = _required_period(self.common_start_period) + end = _required_period(self.common_end_period) + if start != EURUSD_TRIANGLE_COMMON_START_PERIOD or end < start: + raise ValueError("certification common coverage differs") + object.__setattr__(self, "common_start_period", start) + object.__setattr__(self, "common_end_period", end) + object.__setattr__( + self, + "broker_fingerprint_id", + _required_text(self.broker_fingerprint_id), + ) + requirements = tuple( + sorted(self.requirements, key=lambda item: item.check_id) + ) + if ( + not requirements + or len(requirements) > DEFAULT_CERTIFICATION_MAX_REQUIREMENTS + ): + raise ValueError( + "certification requirements are empty or unbounded" + ) + if len({item.check_id for item in requirements}) != len(requirements): + raise ValueError("certification policy duplicates check IDs") + if {item.gate for item in requirements} != set(CertificationGate): + raise ValueError("certification policy must cover all issue gates") + object.__setattr__(self, "requirements", requirements) + for name, maximum in ( + ("max_artifacts", DEFAULT_CERTIFICATION_MAX_ARTIFACTS), + ("max_observations", DEFAULT_CERTIFICATION_MAX_OBSERVATIONS), + ("max_payload_bytes", DEFAULT_CERTIFICATION_MAX_PAYLOAD_BYTES), + ): + object.__setattr__( + self, + name, + _bounded_int(getattr(self, name), name, 1, maximum), + ) + expected = _stable_id( + "reconstruction-certification-policy", self.identity_payload() + ) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("reconstruction certification policy_id differs") + object.__setattr__(self, "policy_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic predeclared certification semantics.""" + return { + "schema_version": self.schema_version, + "product_version": self.product_version, + "symbols": list(self.symbols), + "common_start_period": self.common_start_period, + "common_end_period": self.common_end_period, + "broker_fingerprint_id": self.broker_fingerprint_id, + "requirements": [item.to_dict() for item in self.requirements], + "max_artifacts": self.max_artifacts, + "max_observations": self.max_observations, + "max_payload_bytes": self.max_payload_bytes, + "coverage_policy": "common-supported-period-only", + "promotion_policy": "coverage-once-at-dev-to-main-boundary", + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible policy content.""" + return {**self.identity_payload(), "policy_id": self.policy_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionCertificationPolicyV1": + """Restore and verify a certification policy.""" + _require_schema( + data, RECONSTRUCTION_CERTIFICATION_POLICY_SCHEMA_VERSION + ) + for name, expected in ( + ("coverage_policy", "common-supported-period-only"), + ("promotion_policy", "coverage-once-at-dev-to-main-boundary"), + ): + _require_derived(data, name, expected) + return cls( + product_version=str(data.get("product_version", "")), + symbols=_string_tuple(data.get("symbols"), "symbols"), + common_start_period=str(data.get("common_start_period", "")), + common_end_period=str(data.get("common_end_period", "")), + broker_fingerprint_id=str(data.get("broker_fingerprint_id", "")), + requirements=tuple( + CertificationRequirementV1.from_dict(item) + for item in _mapping_sequence( + data.get("requirements"), "requirements" + ) + ), + max_artifacts=_strict_int( + data.get("max_artifacts"), "max_artifacts" + ), + max_observations=_strict_int( + data.get("max_observations"), "max_observations" + ), + max_payload_bytes=_strict_int( + data.get("max_payload_bytes"), "max_payload_bytes" + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionCertificationPolicyV2: + """Modern-reference v2.1.0 scope without broker-specific claims. + + Version one remains replayable with its mandatory broker fingerprint. This + version is intentionally a separate contract because removing that field or + changing required evidence in-place would invalidate the meaning of already + published V1 policy identities. + """ + + product_version: str + symbols: tuple[str, ...] + common_start_period: str + common_end_period: str + delivery_mode: str + delivery_claim: str + requirements: tuple[CertificationRequirementV1, ...] + max_artifacts: int = DEFAULT_CERTIFICATION_MAX_ARTIFACTS + max_observations: int = DEFAULT_CERTIFICATION_MAX_OBSERVATIONS + max_payload_bytes: int = DEFAULT_CERTIFICATION_MAX_PAYLOAD_BYTES + policy_id: str = "" + schema_version: str = RECONSTRUCTION_CERTIFICATION_POLICY_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_CERTIFICATION_POLICY_V2_SCHEMA_VERSION, + "reconstruction certification policy v2", + ) + object.__setattr__( + self, "product_version", _required_text(self.product_version) + ) + symbols = tuple( + sorted({_normalized_symbol(item) for item in self.symbols}) + ) + if symbols != tuple(sorted(EURUSD_TRIANGLE_SYMBOLS)): + raise ValueError("v2 certification scope is the EURUSD triangle") + object.__setattr__(self, "symbols", symbols) + start = _required_period(self.common_start_period) + end = _required_period(self.common_end_period) + if start != EURUSD_TRIANGLE_COMMON_START_PERIOD or end < start: + raise ValueError("certification common coverage differs") + object.__setattr__(self, "common_start_period", start) + object.__setattr__(self, "common_end_period", end) + if self.delivery_mode != MODERN_REFERENCE_DELIVERY_MODE: + raise ValueError( + "v2 certification requires modern-reference delivery" + ) + if self.delivery_claim != MODERN_REFERENCE_DELIVERY_CLAIM: + raise ValueError("v2 certification requires an unconditioned claim") + requirements = tuple( + sorted(self.requirements, key=lambda item: item.check_id) + ) + if ( + not requirements + or len(requirements) > DEFAULT_CERTIFICATION_MAX_REQUIREMENTS + ): + raise ValueError( + "certification requirements are empty or unbounded" + ) + if len({item.check_id for item in requirements}) != len(requirements): + raise ValueError("certification policy duplicates check IDs") + if {item.gate for item in requirements} != set(CertificationGate): + raise ValueError("certification policy must cover all issue gates") + forbidden_kinds = { + kind + for requirement in requirements + for kind in requirement.required_artifact_kinds + if "broker" in kind + } + forbidden_checks = { + requirement.check_id + for requirement in requirements + if "broker" in requirement.check_id + } + if forbidden_kinds or forbidden_checks: + raise ValueError( + "modern-reference certification cannot require broker evidence" + ) + object.__setattr__(self, "requirements", requirements) + for name, maximum in ( + ("max_artifacts", DEFAULT_CERTIFICATION_MAX_ARTIFACTS), + ("max_observations", DEFAULT_CERTIFICATION_MAX_OBSERVATIONS), + ("max_payload_bytes", DEFAULT_CERTIFICATION_MAX_PAYLOAD_BYTES), + ): + object.__setattr__( + self, + name, + _bounded_int(getattr(self, name), name, 1, maximum), + ) + expected = _stable_id( + "reconstruction-certification-policy-v2", self.identity_payload() + ) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("reconstruction certification policy_id differs") + object.__setattr__(self, "policy_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic modern-reference certification semantics.""" + return { + "schema_version": self.schema_version, + "product_version": self.product_version, + "symbols": list(self.symbols), + "common_start_period": self.common_start_period, + "common_end_period": self.common_end_period, + "delivery_mode": self.delivery_mode, + "delivery_claim": self.delivery_claim, + "requirements": [item.to_dict() for item in self.requirements], + "max_artifacts": self.max_artifacts, + "max_observations": self.max_observations, + "max_payload_bytes": self.max_payload_bytes, + "coverage_policy": "common-supported-period-only", + "promotion_policy": "coverage-once-at-dev-to-main-boundary", + "broker_adaptation": "excluded-from-v2.1.0-certification", + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible policy content.""" + return {**self.identity_payload(), "policy_id": self.policy_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionCertificationPolicyV2": + """Restore and verify a modern-reference certification policy.""" + _require_schema( + data, RECONSTRUCTION_CERTIFICATION_POLICY_V2_SCHEMA_VERSION + ) + for name, expected in ( + ("coverage_policy", "common-supported-period-only"), + ("promotion_policy", "coverage-once-at-dev-to-main-boundary"), + ("broker_adaptation", "excluded-from-v2.1.0-certification"), + ): + _require_derived(data, name, expected) + return cls( + product_version=str(data.get("product_version", "")), + symbols=_string_tuple(data.get("symbols"), "symbols"), + common_start_period=str(data.get("common_start_period", "")), + common_end_period=str(data.get("common_end_period", "")), + delivery_mode=str(data.get("delivery_mode", "")), + delivery_claim=str(data.get("delivery_claim", "")), + requirements=tuple( + CertificationRequirementV1.from_dict(item) + for item in _mapping_sequence( + data.get("requirements"), "requirements" + ) + ), + max_artifacts=_strict_int( + data.get("max_artifacts"), "max_artifacts" + ), + max_observations=_strict_int( + data.get("max_observations"), "max_observations" + ), + max_payload_bytes=_strict_int( + data.get("max_payload_bytes"), "max_payload_bytes" + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionCertificationDossierV1: + """Bounded scientific and operational acceptance dossier.""" + + policy: ReconstructionCertificationPolicyV1 + artifacts: tuple[CertificationArtifactV1, ...] + gate_results: tuple[CertificationGateResultV1, ...] + methodology: str + accepted_limitations: tuple[str, ...] + blocking_limitations: tuple[str, ...] + state: CertificationState + dossier_id: str = "" + schema_version: str = RECONSTRUCTION_CERTIFICATION_DOSSIER_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_CERTIFICATION_DOSSIER_SCHEMA_VERSION, + "reconstruction certification dossier", + ) + if not isinstance(self.policy, ReconstructionCertificationPolicyV1): + raise TypeError("certification dossier requires a v1 policy") + artifacts = tuple( + sorted(self.artifacts, key=lambda item: item.evidence_id) + ) + if not artifacts or len(artifacts) > self.policy.max_artifacts: + raise ValueError( + "certification dossier artifacts are empty or unbounded" + ) + if len({item.evidence_id for item in artifacts}) != len(artifacts): + raise ValueError("certification dossier duplicates artifacts") + object.__setattr__(self, "artifacts", artifacts) + results = tuple( + sorted(self.gate_results, key=lambda item: item.gate.value) + ) + if {item.gate for item in results} != set(CertificationGate): + raise ValueError( + "certification dossier gate coverage is incomplete" + ) + requirement_ids = { + item.requirement_id for item in self.policy.requirements + } + result_ids = { + result.requirement_id + for gate in results + for result in gate.check_results + } + if result_ids != requirement_ids: + raise ValueError( + "certification dossier check coverage differs from policy" + ) + object.__setattr__(self, "gate_results", results) + object.__setattr__(self, "methodology", _bounded_text(self.methodology)) + accepted = _normalized_bounded_text_tuple( + self.accepted_limitations, "accepted limitation" + ) + blocking = _normalized_bounded_text_tuple( + self.blocking_limitations, "blocking limitation" + ) + object.__setattr__(self, "accepted_limitations", accepted) + object.__setattr__(self, "blocking_limitations", blocking) + expected_state = _certification_state(results, blocking) + supplied_state = CertificationState(self.state) + if supplied_state is not expected_state: + raise ValueError( + "certification dossier state differs from evidence" + ) + object.__setattr__(self, "state", expected_state) + expected = _stable_id( + "reconstruction-certification-dossier", self.identity_payload() + ) + supplied = _optional_text(self.dossier_id) + if supplied is not None and supplied != expected: + raise ValueError("reconstruction certification dossier_id differs") + object.__setattr__(self, "dossier_id", expected) + _ensure_payload_size(self.to_dict(), self.policy.max_payload_bytes) + + @property + def certified(self) -> bool: + """Return whether every scientific, operational, and release gate passed.""" + return self.state is CertificationState.CERTIFIED + + @property + def ready_for_promotion(self) -> bool: + """Return whether only the promotion-boundary coverage run remains.""" + return self.state is CertificationState.READY_FOR_PROMOTION + + @property + def summary(self) -> dict[str, JSONValue]: + """Return bounded gate and check counts.""" + checks = [ + item for gate in self.gate_results for item in gate.check_results + ] + return { + "gate_count": len(self.gate_results), + "passed_gate_count": sum( + item.status is CertificationCheckStatus.PASSED + for item in self.gate_results + ), + "failed_gate_count": sum( + item.status is CertificationCheckStatus.FAILED + for item in self.gate_results + ), + "missing_gate_count": sum( + item.status is CertificationCheckStatus.MISSING + for item in self.gate_results + ), + "check_count": len(checks), + "passed_check_count": sum(item.passed for item in checks), + "failed_check_count": sum( + item.status is CertificationCheckStatus.FAILED + for item in checks + ), + "missing_check_count": sum( + item.status is CertificationCheckStatus.MISSING + for item in checks + ), + "artifact_count": len(self.artifacts), + "accepted_limitation_count": len(self.accepted_limitations), + "blocking_limitation_count": len(self.blocking_limitations), + } + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the complete deterministic machine-readable dossier.""" + return { + "schema_version": self.schema_version, + "policy": self.policy.to_dict(), + "artifacts": [item.to_dict() for item in self.artifacts], + "gate_results": [item.to_dict() for item in self.gate_results], + "methodology": self.methodology, + "accepted_limitations": list(self.accepted_limitations), + "blocking_limitations": list(self.blocking_limitations), + "state": self.state.value, + "summary": self.summary, + "event_rows_inline": False, + "analytical_frame_columns_inline": False, + "automatic_winner": False, + "historical_truth_claim": False, + "investment_recommendation": False, + "release_authorized": self.certified, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible dossier content.""" + return {**self.identity_payload(), "dossier_id": self.dossier_id} + + def to_json(self) -> str: + """Serialize the dossier deterministically.""" + serialized: str = canonical_contract_json(self.to_dict()) + return serialized + + def to_markdown(self) -> str: + """Render a deterministic human-readable methodology/limitations report.""" + lines = [ + "# EURUSD Triangle Reconstruction Certification", + "", + f"- State: **{self.state.value}**", + f"- Dossier: `{self.dossier_id}`", + f"- Policy: `{self.policy.policy_id}`", + f"- Product version: `{self.policy.product_version}`", + f"- Symbols: `{', '.join(self.policy.symbols)}`", + ( + "- Common coverage: " + f"`{self.policy.common_start_period}`–`{self.policy.common_end_period}`" + ), + f"- Broker fingerprint: `{self.policy.broker_fingerprint_id}`", + "", + "## Gate results", + "", + "| Gate | Status | Passed | Failed | Missing |", + "| --- | --- | ---: | ---: | ---: |", + ] + for gate in self.gate_results: + lines.append( + "| " + f"{gate.gate.value} | {gate.status.value} | " + f"{sum(item.status is CertificationCheckStatus.PASSED for item in gate.check_results)} | " + f"{sum(item.status is CertificationCheckStatus.FAILED for item in gate.check_results)} | " + f"{sum(item.status is CertificationCheckStatus.MISSING for item in gate.check_results)} |" + ) + lines.extend(["", "## Methodology", "", self.methodology, ""]) + lines.extend( + _markdown_limitations( + "Accepted limitations", self.accepted_limitations + ) + ) + lines.extend( + _markdown_limitations( + "Blocking limitations", self.blocking_limitations + ) + ) + lines.extend( + [ + "## Trust boundary", + "", + "This dossier is bounded derived metadata. It contains no tick rows, " + "does not select an automatic winner, and does not claim historical truth " + "or authorize an investment recommendation.", + "", + ] + ) + return "\n".join(lines) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionCertificationDossierV1": + """Restore and verify a complete dossier.""" + _require_schema( + data, RECONSTRUCTION_CERTIFICATION_DOSSIER_SCHEMA_VERSION + ) + for name, expected in ( + ("event_rows_inline", False), + ("analytical_frame_columns_inline", False), + ("automatic_winner", False), + ("historical_truth_claim", False), + ("investment_recommendation", False), + ): + _require_derived(data, name, expected) + dossier = cls( + policy=ReconstructionCertificationPolicyV1.from_dict( + _mapping(data.get("policy"), "policy") + ), + artifacts=tuple( + CertificationArtifactV1.from_dict(item) + for item in _mapping_sequence( + data.get("artifacts"), "artifacts" + ) + ), + gate_results=tuple( + CertificationGateResultV1.from_dict(item) + for item in _mapping_sequence( + data.get("gate_results"), "gate_results" + ) + ), + methodology=str(data.get("methodology", "")), + accepted_limitations=_string_tuple( + data.get("accepted_limitations"), "accepted_limitations" + ), + blocking_limitations=_string_tuple( + data.get("blocking_limitations"), "blocking_limitations" + ), + state=CertificationState(str(data.get("state", ""))), + dossier_id=str(data.get("dossier_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived(data, "summary", dossier.summary) + _require_derived(data, "release_authorized", dossier.certified) + return dossier + + @classmethod + def from_json(cls, text: str) -> "ReconstructionCertificationDossierV1": + """Restore a dossier from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionCertificationDossierV2: + """Bounded modern-reference scientific and operational dossier.""" + + policy: ReconstructionCertificationPolicyV2 + artifacts: tuple[CertificationArtifactV1, ...] + gate_results: tuple[CertificationGateResultV1, ...] + methodology: str + accepted_limitations: tuple[str, ...] + blocking_limitations: tuple[str, ...] + state: CertificationState + dossier_id: str = "" + schema_version: str = RECONSTRUCTION_CERTIFICATION_DOSSIER_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_CERTIFICATION_DOSSIER_V2_SCHEMA_VERSION, + "reconstruction certification dossier v2", + ) + if not isinstance(self.policy, ReconstructionCertificationPolicyV2): + raise TypeError("certification dossier v2 requires a v2 policy") + artifacts = tuple( + sorted(self.artifacts, key=lambda item: item.evidence_id) + ) + if not artifacts or len(artifacts) > self.policy.max_artifacts: + raise ValueError( + "certification dossier artifacts are empty or unbounded" + ) + if len({item.evidence_id for item in artifacts}) != len(artifacts): + raise ValueError("certification dossier duplicates artifacts") + if any(item.policy_id != self.policy.policy_id for item in artifacts): + raise ValueError( + "certification dossier contains foreign policy evidence" + ) + object.__setattr__(self, "artifacts", artifacts) + results = tuple( + sorted(self.gate_results, key=lambda item: item.gate.value) + ) + if {item.gate for item in results} != set(CertificationGate): + raise ValueError( + "certification dossier gate coverage is incomplete" + ) + requirement_ids = { + item.requirement_id for item in self.policy.requirements + } + result_ids = { + result.requirement_id + for gate in results + for result in gate.check_results + } + if result_ids != requirement_ids: + raise ValueError( + "certification dossier check coverage differs from policy" + ) + object.__setattr__(self, "gate_results", results) + object.__setattr__(self, "methodology", _bounded_text(self.methodology)) + accepted = _normalized_bounded_text_tuple( + self.accepted_limitations, "accepted limitation" + ) + blocking = _normalized_bounded_text_tuple( + self.blocking_limitations, "blocking limitation" + ) + object.__setattr__(self, "accepted_limitations", accepted) + object.__setattr__(self, "blocking_limitations", blocking) + expected_state = _certification_state(results, blocking) + supplied_state = CertificationState(self.state) + if supplied_state is not expected_state: + raise ValueError( + "certification dossier state differs from evidence" + ) + object.__setattr__(self, "state", expected_state) + expected = _stable_id( + "reconstruction-certification-dossier-v2", self.identity_payload() + ) + supplied = _optional_text(self.dossier_id) + if supplied is not None and supplied != expected: + raise ValueError("reconstruction certification dossier_id differs") + object.__setattr__(self, "dossier_id", expected) + _ensure_payload_size(self.to_dict(), self.policy.max_payload_bytes) + + @property + def certified(self) -> bool: + """Return whether every scientific, operational, and release gate passed.""" + return self.state is CertificationState.CERTIFIED + + @property + def ready_for_promotion(self) -> bool: + """Return whether only the promotion-boundary coverage run remains.""" + return self.state is CertificationState.READY_FOR_PROMOTION + + @property + def summary(self) -> dict[str, JSONValue]: + """Return bounded gate and check counts.""" + checks = [ + item for gate in self.gate_results for item in gate.check_results + ] + return { + "gate_count": len(self.gate_results), + "passed_gate_count": sum( + item.status is CertificationCheckStatus.PASSED + for item in self.gate_results + ), + "failed_gate_count": sum( + item.status is CertificationCheckStatus.FAILED + for item in self.gate_results + ), + "missing_gate_count": sum( + item.status is CertificationCheckStatus.MISSING + for item in self.gate_results + ), + "check_count": len(checks), + "passed_check_count": sum(item.passed for item in checks), + "failed_check_count": sum( + item.status is CertificationCheckStatus.FAILED + for item in checks + ), + "missing_check_count": sum( + item.status is CertificationCheckStatus.MISSING + for item in checks + ), + "artifact_count": len(self.artifacts), + "accepted_limitation_count": len(self.accepted_limitations), + "blocking_limitation_count": len(self.blocking_limitations), + } + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the complete deterministic machine-readable dossier.""" + return { + "schema_version": self.schema_version, + "policy": self.policy.to_dict(), + "artifacts": [item.to_dict() for item in self.artifacts], + "gate_results": [item.to_dict() for item in self.gate_results], + "methodology": self.methodology, + "accepted_limitations": list(self.accepted_limitations), + "blocking_limitations": list(self.blocking_limitations), + "state": self.state.value, + "summary": self.summary, + "delivery_mode": self.policy.delivery_mode, + "delivery_claim": self.policy.delivery_claim, + "broker_specific_claim": False, + "event_rows_inline": False, + "analytical_frame_columns_inline": False, + "automatic_winner": False, + "historical_truth_claim": False, + "investment_recommendation": False, + "release_authorized": self.certified, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible dossier content.""" + return {**self.identity_payload(), "dossier_id": self.dossier_id} + + def to_json(self) -> str: + """Serialize the dossier deterministically.""" + return str(canonical_contract_json(self.to_dict())) + + def to_markdown(self) -> str: + """Render deterministic modern-reference methodology and limitations.""" + lines = [ + "# EURUSD Triangle Reconstruction Certification", + "", + f"- State: **{self.state.value}**", + f"- Dossier: `{self.dossier_id}`", + f"- Policy: `{self.policy.policy_id}`", + f"- Product version: `{self.policy.product_version}`", + f"- Symbols: `{', '.join(self.policy.symbols)}`", + ( + "- Common coverage: " + f"`{self.policy.common_start_period}`–`{self.policy.common_end_period}`" + ), + f"- Delivery mode: `{self.policy.delivery_mode}`", + f"- Delivery claim: `{self.policy.delivery_claim}`", + "- Broker-specific claim: `false`", + "", + "## Gate results", + "", + "| Gate | Status | Passed | Failed | Missing |", + "| --- | --- | ---: | ---: | ---: |", + ] + for gate in self.gate_results: + lines.append( + "| " + f"{gate.gate.value} | {gate.status.value} | " + f"{sum(item.status is CertificationCheckStatus.PASSED for item in gate.check_results)} | " + f"{sum(item.status is CertificationCheckStatus.FAILED for item in gate.check_results)} | " + f"{sum(item.status is CertificationCheckStatus.MISSING for item in gate.check_results)} |" + ) + lines.extend(["", "## Methodology", "", self.methodology, ""]) + lines.extend( + _markdown_limitations( + "Accepted limitations", self.accepted_limitations + ) + ) + lines.extend( + _markdown_limitations( + "Blocking limitations", self.blocking_limitations + ) + ) + lines.extend( + [ + "## Trust boundary", + "", + "This dossier certifies modern-reference, unconditioned output only. " + "It makes no broker-specific claim, contains no tick rows, does not " + "select an automatic winner, and does not claim historical truth or " + "authorize an investment recommendation.", + "", + ] + ) + return "\n".join(lines) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionCertificationDossierV2": + """Restore and verify a complete modern-reference dossier.""" + _require_schema( + data, RECONSTRUCTION_CERTIFICATION_DOSSIER_V2_SCHEMA_VERSION + ) + for name, expected in ( + ("delivery_mode", MODERN_REFERENCE_DELIVERY_MODE), + ("delivery_claim", MODERN_REFERENCE_DELIVERY_CLAIM), + ("broker_specific_claim", False), + ("event_rows_inline", False), + ("analytical_frame_columns_inline", False), + ("automatic_winner", False), + ("historical_truth_claim", False), + ("investment_recommendation", False), + ): + _require_derived(data, name, expected) + dossier = cls( + policy=ReconstructionCertificationPolicyV2.from_dict( + _mapping(data.get("policy"), "policy") + ), + artifacts=tuple( + CertificationArtifactV1.from_dict(item) + for item in _mapping_sequence( + data.get("artifacts"), "artifacts" + ) + ), + gate_results=tuple( + CertificationGateResultV1.from_dict(item) + for item in _mapping_sequence( + data.get("gate_results"), "gate_results" + ) + ), + methodology=str(data.get("methodology", "")), + accepted_limitations=_string_tuple( + data.get("accepted_limitations"), "accepted_limitations" + ), + blocking_limitations=_string_tuple( + data.get("blocking_limitations"), "blocking_limitations" + ), + state=CertificationState(str(data.get("state", ""))), + dossier_id=str(data.get("dossier_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived(data, "summary", dossier.summary) + _require_derived(data, "release_authorized", dossier.certified) + return dossier + + @classmethod + def from_json(cls, text: str) -> "ReconstructionCertificationDossierV2": + """Restore a modern-reference dossier from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +def eurusd_triangle_certification_policy( + *, + broker_fingerprint_id: str, + common_end_period: str, + peak_memory_budget_bytes: int, + scratch_budget_bytes: int, + runtime_budget_seconds: float, + storage_budget_bytes: int, +) -> ReconstructionCertificationPolicyV1: + """Return the complete predeclared v2.1.0 certification policy.""" + requirements = ( + _requirement( + CertificationGate.IDENTITY_AND_ANCHORS, + "raw_source_hash_mismatch_count", + CertificationComparator.ZERO, + 0, + ("raw-source-inventory", "reconstruction-product-manifest"), + "Raw source hashes reconcile with the committed product.", + ), + _requirement( + CertificationGate.IDENTITY_AND_ANCHORS, + "immutable_anchor_mismatch_count", + CertificationComparator.ZERO, + 0, + ("reconstruction-product-manifest",), + "Immutable observed anchors reconcile exactly.", + ), + _requirement( + CertificationGate.INFORMATION_SAFETY, + "information_audit_violation_count", + CertificationComparator.ZERO, + 0, + ("information-audit-report",), + "Every claimed use has zero information-leakage violations.", + ), + _requirement( + CertificationGate.INFORMATION_SAFETY, + "claimed_use_missing_audit_count", + CertificationComparator.ZERO, + 0, + ("information-audit-report",), + "No claimed use lacks a point-in-time audit.", + ), + _requirement( + CertificationGate.REVERSE_DEGRADATION, + "reverse_thresholds_predeclared", + CertificationComparator.TRUE, + True, + ("reverse-degradation-scorecard",), + "Reverse-degradation thresholds were fixed before final holdout use.", + ), + _requirement( + CertificationGate.REVERSE_DEGRADATION, + "reverse_holdout_failure_count", + CertificationComparator.ZERO, + 0, + ("reverse-degradation-scorecard",), + "Untouched final holdouts pass the predeclared thresholds.", + ), + _requirement( + CertificationGate.CONDITIONED_SCORECARDS, + "conditioned_tolerance_violation_count", + CertificationComparator.ZERO, + 0, + ("reverse-degradation-scorecard", "broker-conditioned-scorecard"), + "Session, event, epoch, cadence, spread, timing, and path tolerances pass.", + ), + _requirement( + CertificationGate.CONDITIONED_SCORECARDS, + "required_stratum_missing_count", + CertificationComparator.ZERO, + 0, + ("reverse-degradation-scorecard",), + "Every required scientific stratum has measured support.", + ), + _requirement( + CertificationGate.CROSS_CURRENCY, + "post_broker_cross_currency_failure_count", + CertificationComparator.ZERO, + 0, + ("cross-currency-validation-report", "broker-delivery-fingerprint"), + "Triangle, inverse, and stale-join checks pass after broker rendering.", + ), + _requirement( + CertificationGate.ENSEMBLE_EVIDENCE, + "ensemble_rates_reported", + CertificationComparator.TRUE, + True, + ("ensemble-calibration-report",), + "Calibration, diversity, refusal, and unsupported-region rates are reported.", + ), + _requirement( + CertificationGate.PRODUCT_RECONCILIATION, + "product_activity_bar_mismatch_count", + CertificationComparator.ZERO, + 0, + ( + "reconstruction-product-manifest", + "activity-manifest", + "derived-bar-manifest", + ), + "Final events, activity semantics, and derived bars reconcile.", + ), + _requirement( + CertificationGate.FAILURE_RESUME, + "resume_duplicate_or_missing_partition_count", + CertificationComparator.ZERO, + 0, + ("reconstruction-run-report", "failure-injection-report"), + "A mid-run failure resumes without duplicate or missing partitions.", + ), + _requirement( + CertificationGate.REPLAY, + "replay_logical_hash_mismatch_count", + CertificationComparator.ZERO, + 0, + ("reconstruction-product-manifest", "replay-report"), + "A clean replay reproduces logical content hashes.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_peak_memory_bytes", + CertificationComparator.LESS_OR_EQUAL, + _positive_int(peak_memory_budget_bytes, "peak_memory_budget_bytes"), + ("resource-report",), + "Actual peak memory remains inside its predeclared budget.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_scratch_bytes", + CertificationComparator.LESS_OR_EQUAL, + _positive_int(scratch_budget_bytes, "scratch_budget_bytes"), + ("resource-report",), + "Actual scratch use remains inside its predeclared budget.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_runtime_seconds", + CertificationComparator.LESS_OR_EQUAL, + _positive_float(runtime_budget_seconds, "runtime_budget_seconds"), + ("resource-report",), + "Actual runtime remains inside its predeclared budget.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_final_storage_bytes", + CertificationComparator.LESS_OR_EQUAL, + _positive_int(storage_budget_bytes, "storage_budget_bytes"), + ("resource-report",), + "Actual final storage remains inside its predeclared budget.", + ), + *( + _requirement( + CertificationGate.NEGATIVE_TESTS, + check_id, + CertificationComparator.TRUE, + True, + ("negative-test-report",), + description, + ) + for check_id, description in ( + ("corruption_refused", "Corrupt products fail closed."), + ( + "stale_broker_profile_refused", + "Stale broker profiles fail closed.", + ), + ( + "unhealthy_clock_refused", + "Unhealthy capture clocks fail closed.", + ), + ( + "missing_context_refused", + "Missing required context fails closed.", + ), + ( + "partial_group_refused", + "Partial synchronized groups fail closed.", + ), + ) + ), + _requirement( + CertificationGate.STRATEGY_SENSITIVITY, + "strategy_uncertainty_reported", + CertificationComparator.TRUE, + True, + ("strategy-sensitivity-report",), + "Strategy sensitivity reports bounded uncertainty evidence.", + ), + _requirement( + CertificationGate.STRATEGY_SENSITIVITY, + "strategy_automatic_winner", + CertificationComparator.FALSE, + False, + ("strategy-sensitivity-report",), + "Strategy sensitivity never selects an automatic winner.", + ), + _requirement( + CertificationGate.DOSSIER_PUBLICATION, + "methodology_and_limitations_published", + CertificationComparator.TRUE, + True, + ("methodology-report",), + "Human-readable methodology and limitations are published.", + ), + _requirement( + CertificationGate.DOSSIER_PUBLICATION, + "machine_evidence_manifest_published", + CertificationComparator.TRUE, + True, + ("machine-evidence-manifest",), + "The machine-readable evidence manifest is published.", + ), + _requirement( + CertificationGate.REPOSITORY_GATES, + "full_plain_test_suite_passed", + CertificationComparator.TRUE, + True, + ("repository-gate-report",), + "The full plain test suite passes.", + ), + _requirement( + CertificationGate.REPOSITORY_GATES, + "precommit_and_prepush_passed", + CertificationComparator.TRUE, + True, + ("repository-gate-report",), + "Pre-commit and real pre-push hooks pass.", + ), + _requirement( + CertificationGate.REPOSITORY_GATES, + "coverage_promotion_run_count", + CertificationComparator.EQUAL, + 1, + ("repository-gate-report",), + "Coverage runs exactly once at the dev-to-main promotion boundary.", + ), + _requirement( + CertificationGate.TESTPYPI_PREFLIGHT, + "local_simple_registry_preflight_passed", + CertificationComparator.TRUE, + True, + ("testpypi-preflight-report",), + "The TestPyPI preflight passes through the local simple registry.", + ), + ) + return ReconstructionCertificationPolicyV1( + product_version="2.1.0", + symbols=EURUSD_TRIANGLE_SYMBOLS, + common_start_period=EURUSD_TRIANGLE_COMMON_START_PERIOD, + common_end_period=common_end_period, + broker_fingerprint_id=broker_fingerprint_id, + requirements=requirements, + ) + + +def modern_reference_triangle_certification_policy( + *, + common_end_period: str, + peak_memory_budget_bytes: int, + scratch_budget_bytes: int, + runtime_budget_seconds: float, + storage_budget_bytes: int, + candidate_amplification_budget: float, +) -> ReconstructionCertificationPolicyV2: + """Return the complete broker-neutral v2.1.0 certification policy. + + The policy maps every #449 source-readiness, scientific-integrity, + product/operations, and release-discipline seam to content-bound evidence. + Broker capture, fingerprints, clocks, and transfer checks are deliberately + absent because they are separately qualified optional extensions. + """ + requirements = ( + _requirement( + CertificationGate.IDENTITY_AND_ANCHORS, + "source_inventory_reconciled", + CertificationComparator.TRUE, + True, + ("raw-source-inventory",), + "Source dimensions, hashes, common range, and readability reconcile.", + ), + _requirement( + CertificationGate.IDENTITY_AND_ANCHORS, + "duplicate_source_dimension_count", + CertificationComparator.ZERO, + 0, + ("raw-source-inventory",), + "The canonical inventory contains no duplicate source dimension.", + ), + _requirement( + CertificationGate.IDENTITY_AND_ANCHORS, + "raw_source_hash_mismatch_count", + CertificationComparator.ZERO, + 0, + ("raw-source-inventory", "reconstruction-product-manifest"), + "Raw source hashes reconcile with the committed product.", + ), + _requirement( + CertificationGate.IDENTITY_AND_ANCHORS, + "immutable_anchor_mismatch_count", + CertificationComparator.ZERO, + 0, + ("reconstruction-product-manifest",), + "Every recorded historical tick remains logically unchanged.", + ), + _requirement( + CertificationGate.IDENTITY_AND_ANCHORS, + "synthetic_lineage_missing_count", + CertificationComparator.ZERO, + 0, + ("reconstruction-product-manifest",), + "Every synthetic event carries deterministic identity and lineage.", + ), + _requirement( + CertificationGate.INFORMATION_SAFETY, + "market_context_corpus_valid", + CertificationComparator.TRUE, + True, + ("market-context-corpus",), + "The point-in-time context corpus has qualified coverage and lineage.", + ), + _requirement( + CertificationGate.INFORMATION_SAFETY, + "cftc_positioning_corpus_valid", + CertificationComparator.TRUE, + True, + ("cftc-positioning-corpus",), + "The point-in-time CFTC corpus has qualified availability and lineage.", + ), + _requirement( + CertificationGate.INFORMATION_SAFETY, + "information_audit_violation_count", + CertificationComparator.ZERO, + 0, + ("information-audit-report",), + "Every claimed use has zero information-leakage violations.", + ), + _requirement( + CertificationGate.INFORMATION_SAFETY, + "claimed_use_missing_audit_count", + CertificationComparator.ZERO, + 0, + ("information-audit-report",), + "No ex-post or ex-ante claim lacks a point-in-time audit.", + ), + _requirement( + CertificationGate.REVERSE_DEGRADATION, + "benchmark_corpus_valid", + CertificationComparator.TRUE, + True, + ("benchmark-corpus-manifest",), + "The benchmark corpus and gate version predate candidate results.", + ), + _requirement( + CertificationGate.REVERSE_DEGRADATION, + "reverse_thresholds_predeclared", + CertificationComparator.TRUE, + True, + ("reverse-degradation-scorecard",), + "Reverse-degradation thresholds were fixed before final holdout use.", + ), + _requirement( + CertificationGate.REVERSE_DEGRADATION, + "reverse_holdout_failure_count", + CertificationComparator.ZERO, + 0, + ("reverse-degradation-scorecard",), + "Untouched blocked holdouts pass and negative controls fail as expected.", + ), + _requirement( + CertificationGate.CONDITIONED_SCORECARDS, + "feed_epoch_artifact_valid", + CertificationComparator.TRUE, + True, + ("feed-epoch-definition",), + "The multivariate epoch artifact and stability evidence are valid.", + ), + _requirement( + CertificationGate.CONDITIONED_SCORECARDS, + "observation_operator_valid", + CertificationComparator.TRUE, + True, + ("observation-operator",), + "Calibrated observation operators are supported and non-identity where claimed.", + ), + _requirement( + CertificationGate.CONDITIONED_SCORECARDS, + "motif_artifact_valid", + CertificationComparator.TRUE, + True, + ("motif-qualification",), + "The selected empirical motif artifact passes split, leakage, and support checks.", + ), + _requirement( + CertificationGate.CONDITIONED_SCORECARDS, + "conditioned_tolerance_violation_count", + CertificationComparator.ZERO, + 0, + ( + "reverse-degradation-scorecard", + "modern-reference-product-scorecard", + ), + "Epoch, session, event, cadence, spread, timing, and path tolerances pass.", + ), + _requirement( + CertificationGate.CONDITIONED_SCORECARDS, + "required_stratum_missing_count", + CertificationComparator.ZERO, + 0, + ( + "reverse-degradation-scorecard", + "modern-reference-product-scorecard", + ), + "Every required scientific stratum has measured support.", + ), + _requirement( + CertificationGate.CROSS_CURRENCY, + "modern_reference_cross_currency_failure_count", + CertificationComparator.ZERO, + 0, + ( + "cross-currency-validation-report", + "reconstruction-product-manifest", + ), + "Triangle, inverse, synchronization, and stale-alignment checks pass before and after identity delivery.", + ), + _requirement( + CertificationGate.ENSEMBLE_EVIDENCE, + "ensemble_rates_reported", + CertificationComparator.TRUE, + True, + ("ensemble-calibration-report",), + "Calibration, diversity, refusal, and unsupported-region rates are reported.", + ), + _requirement( + CertificationGate.ENSEMBLE_EVIDENCE, + "between_seed_and_window_uncertainty_reported", + CertificationComparator.TRUE, + True, + ("ensemble-calibration-report",), + "Between-seed and between-window uncertainty are reported.", + ), + _requirement( + CertificationGate.PRODUCT_RECONCILIATION, + "product_activity_bar_mismatch_count", + CertificationComparator.ZERO, + 0, + ( + "reconstruction-product-manifest", + "activity-manifest", + "derived-bar-manifest", + ), + "Final events, activity semantics, and downstream bars reconcile.", + ), + _requirement( + CertificationGate.PRODUCT_RECONCILIATION, + "scientific_nonclaim_published", + CertificationComparator.TRUE, + True, + ("reconstruction-product-manifest", "methodology-report"), + "Output is labeled a plausible counterfactual ensemble, never historical truth.", + ), + _requirement( + CertificationGate.PRODUCT_RECONCILIATION, + "full_range_public_preflight_passed", + CertificationComparator.TRUE, + True, + ("reconstruction-plan-report", "public-interface-report"), + "The full-range synchronized plan and resource preflight pass publicly.", + ), + _requirement( + CertificationGate.PRODUCT_RECONCILIATION, + "representative_window_class_missing_count", + CertificationComparator.ZERO, + 0, + ("reconstruction-run-report",), + "Sparse, transitional, dense, ordinary, rollover, news, shock, and refusal windows execute.", + ), + _requirement( + CertificationGate.PRODUCT_RECONCILIATION, + "substantial_multi_period_run_passed", + CertificationComparator.TRUE, + True, + ("reconstruction-run-report", "reconstruction-product-manifest"), + "A substantial multi-period run commits queryable Parquet and compact manifests.", + ), + _requirement( + CertificationGate.PRODUCT_RECONCILIATION, + "public_cli_api_evidence_chain_passed", + CertificationComparator.TRUE, + True, + ("public-interface-report",), + "CLI and API expose bounded final ticks and the complete evidence chain.", + ), + _requirement( + CertificationGate.FAILURE_RESUME, + "resume_duplicate_or_missing_partition_count", + CertificationComparator.ZERO, + 0, + ("reconstruction-run-report", "failure-injection-report"), + "A mid-run worker/server failure resumes without duplicate or missing partitions.", + ), + _requirement( + CertificationGate.FAILURE_RESUME, + "cancellation_publishable_partial_count", + CertificationComparator.ZERO, + 0, + ("cancellation-report",), + "Cancellation leaves no publishable partial partition.", + ), + _requirement( + CertificationGate.REPLAY, + "replay_logical_hash_mismatch_count", + CertificationComparator.ZERO, + 0, + ("reconstruction-product-manifest", "replay-report"), + "Clean replay reproduces logical content hashes across concurrency settings.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_peak_memory_bytes", + CertificationComparator.LESS_OR_EQUAL, + _positive_int(peak_memory_budget_bytes, "peak_memory_budget_bytes"), + ("resource-report",), + "Actual peak memory remains inside its predeclared budget.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_scratch_bytes", + CertificationComparator.LESS_OR_EQUAL, + _positive_int(scratch_budget_bytes, "scratch_budget_bytes"), + ("resource-report",), + "Actual scratch use remains inside its predeclared budget.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_runtime_seconds", + CertificationComparator.LESS_OR_EQUAL, + _positive_float(runtime_budget_seconds, "runtime_budget_seconds"), + ("resource-report",), + "Actual runtime remains inside its predeclared budget.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_candidate_amplification", + CertificationComparator.LESS_OR_EQUAL, + _positive_float( + candidate_amplification_budget, + "candidate_amplification_budget", + ), + ("resource-report",), + "Candidate amplification remains inside its predeclared budget.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_final_storage_bytes", + CertificationComparator.LESS_OR_EQUAL, + _positive_int(storage_budget_bytes, "storage_budget_bytes"), + ("resource-report",), + "Actual final storage remains inside its predeclared budget.", + ), + _requirement( + CertificationGate.RESOURCES, + "actual_final_row_count", + CertificationComparator.GREATER_OR_EQUAL, + 1, + ("resource-report", "reconstruction-product-manifest"), + "The committed substantial run contains measured final rows.", + ), + *( + _requirement( + CertificationGate.NEGATIVE_TESTS, + check_id, + CertificationComparator.TRUE, + True, + ("negative-test-report",), + description, + ) + for check_id, description in ( + ("corruption_refused", "Corrupt artifacts fail closed."), + ("stale_artifact_refused", "Stale artifacts fail closed."), + ( + "missing_context_refused", + "Missing required context fails closed.", + ), + ( + "invalid_information_mode_refused", + "Invalid information modes fail closed.", + ), + ("quota_overflow_refused", "Quota overflow fails closed."), + ( + "partial_group_refused", + "Partial synchronized groups fail closed.", + ), + ) + ), + _requirement( + CertificationGate.STRATEGY_SENSITIVITY, + "strategy_uncertainty_reported", + CertificationComparator.TRUE, + True, + ("strategy-sensitivity-report",), + "Strategy sensitivity reports bounded uncertainty evidence.", + ), + _requirement( + CertificationGate.STRATEGY_SENSITIVITY, + "strategy_automatic_winner", + CertificationComparator.FALSE, + False, + ("strategy-sensitivity-report",), + "Strategy sensitivity never selects an automatic winner.", + ), + _requirement( + CertificationGate.DOSSIER_PUBLICATION, + "methodology_and_limitations_published", + CertificationComparator.TRUE, + True, + ("methodology-report",), + "Human-readable methodology, limitations, and nonclaim are published.", + ), + _requirement( + CertificationGate.DOSSIER_PUBLICATION, + "machine_evidence_manifest_published", + CertificationComparator.TRUE, + True, + ("machine-evidence-manifest",), + "The machine-readable evidence manifest is published.", + ), + _requirement( + CertificationGate.REPOSITORY_GATES, + "declared_test_dependencies_installed", + CertificationComparator.TRUE, + True, + ("repository-gate-report",), + "Every declared test dependency is installed and no suite is silently skipped.", + ), + _requirement( + CertificationGate.REPOSITORY_GATES, + "full_plain_test_suite_passed", + CertificationComparator.TRUE, + True, + ("repository-gate-report",), + "The full plain test suite passes.", + ), + _requirement( + CertificationGate.REPOSITORY_GATES, + "precommit_and_prepush_passed", + CertificationComparator.TRUE, + True, + ("repository-gate-report",), + "Pre-commit and real pre-push hooks pass.", + ), + _requirement( + CertificationGate.REPOSITORY_GATES, + "coverage_promotion_run_count", + CertificationComparator.EQUAL, + 1, + ("repository-gate-report",), + "Coverage runs exactly once at the dev-to-main promotion boundary.", + ), + _requirement( + CertificationGate.TESTPYPI_PREFLIGHT, + "local_simple_registry_preflight_passed", + CertificationComparator.TRUE, + True, + ("testpypi-preflight-report",), + "The TestPyPI preflight passes through the local simple registry.", + ), + ) + return ReconstructionCertificationPolicyV2( + product_version="2.1.0", + symbols=EURUSD_TRIANGLE_SYMBOLS, + common_start_period=EURUSD_TRIANGLE_COMMON_START_PERIOD, + common_end_period=common_end_period, + delivery_mode=MODERN_REFERENCE_DELIVERY_MODE, + delivery_claim=MODERN_REFERENCE_DELIVERY_CLAIM, + requirements=requirements, + ) + + +def evaluate_reconstruction_certification( + policy: ReconstructionCertificationPolicyV1, + *, + artifacts: Sequence[CertificationArtifactV1], + observations: Sequence[CertificationObservationV1], + methodology: str, + accepted_limitations: Sequence[str] = (), + blocking_limitations: Sequence[str] = (), +) -> ReconstructionCertificationDossierV1: + """Evaluate every predeclared requirement without reading event rows.""" + if not isinstance(policy, ReconstructionCertificationPolicyV1): + raise TypeError("certification evaluation requires a v1 policy") + selected_artifacts = tuple(artifacts) + selected_observations = tuple(observations) + if not selected_artifacts or len(selected_artifacts) > policy.max_artifacts: + raise ValueError("certification artifacts are empty or unbounded") + if len(selected_observations) > policy.max_observations: + raise ValueError("certification observations exceed policy") + artifact_by_id = {item.evidence_id: item for item in selected_artifacts} + if len(artifact_by_id) != len(selected_artifacts): + raise ValueError("certification artifacts duplicate evidence IDs") + foreign_policy_artifacts = sorted( + item.evidence_id + for item in selected_artifacts + if item.policy_id != policy.policy_id + ) + if foreign_policy_artifacts: + raise ValueError( + f"certification artifacts differ from policy: {foreign_policy_artifacts}" + ) + observation_by_check = { + item.check_id: item for item in selected_observations + } + if len(observation_by_check) != len(selected_observations): + raise ValueError("certification observations duplicate check IDs") + unknown_checks = set(observation_by_check).difference( + item.check_id for item in policy.requirements + ) + if unknown_checks: + raise ValueError( + f"certification observations are outside policy: {sorted(unknown_checks)}" + ) + by_gate: dict[CertificationGate, list[CertificationCheckResultV1]] = { + gate: [] for gate in CertificationGate + } + for requirement in policy.requirements: + result = _evaluate_requirement( + policy, + requirement, + observation_by_check.get(requirement.check_id), + artifact_by_id, + ) + by_gate[requirement.gate].append(result) + gate_results = tuple( + CertificationGateResultV1(gate=gate, check_results=tuple(results)) + for gate, results in by_gate.items() + ) + state = _certification_state( + gate_results, + _normalized_bounded_text_tuple( + blocking_limitations, "blocking limitation" + ), + ) + return ReconstructionCertificationDossierV1( + policy=policy, + artifacts=selected_artifacts, + gate_results=gate_results, + methodology=methodology, + accepted_limitations=tuple(accepted_limitations), + blocking_limitations=tuple(blocking_limitations), + state=state, + ) + + +def evaluate_modern_reference_reconstruction_certification( + policy: ReconstructionCertificationPolicyV2, + *, + artifacts: Sequence[CertificationArtifactV1], + observations: Sequence[CertificationObservationV1], + methodology: str, + accepted_limitations: Sequence[str] = (), + blocking_limitations: Sequence[str] = (), +) -> ReconstructionCertificationDossierV2: + """Evaluate modern-reference evidence without accepting broker artifacts.""" + if not isinstance(policy, ReconstructionCertificationPolicyV2): + raise TypeError("modern-reference evaluation requires a v2 policy") + selected_artifacts = tuple(artifacts) + selected_observations = tuple(observations) + if not selected_artifacts or len(selected_artifacts) > policy.max_artifacts: + raise ValueError("certification artifacts are empty or unbounded") + if len(selected_observations) > policy.max_observations: + raise ValueError("certification observations exceed policy") + if any("broker" in item.kind for item in selected_artifacts): + raise ValueError( + "modern-reference certification rejects broker-specific evidence" + ) + artifact_by_id = {item.evidence_id: item for item in selected_artifacts} + if len(artifact_by_id) != len(selected_artifacts): + raise ValueError("certification artifacts duplicate evidence IDs") + foreign_policy_artifacts = sorted( + item.evidence_id + for item in selected_artifacts + if item.policy_id != policy.policy_id + ) + if foreign_policy_artifacts: + raise ValueError( + f"certification artifacts differ from policy: {foreign_policy_artifacts}" + ) + observation_by_check = { + item.check_id: item for item in selected_observations + } + if len(observation_by_check) != len(selected_observations): + raise ValueError("certification observations duplicate check IDs") + unknown_checks = set(observation_by_check).difference( + item.check_id for item in policy.requirements + ) + if unknown_checks: + raise ValueError( + f"certification observations are outside policy: {sorted(unknown_checks)}" + ) + by_gate: dict[CertificationGate, list[CertificationCheckResultV1]] = { + gate: [] for gate in CertificationGate + } + for requirement in policy.requirements: + result = _evaluate_requirement_v2( + requirement, + observation_by_check.get(requirement.check_id), + artifact_by_id, + ) + by_gate[requirement.gate].append(result) + gate_results = tuple( + CertificationGateResultV1(gate=gate, check_results=tuple(results)) + for gate, results in by_gate.items() + ) + state = _certification_state( + gate_results, + _normalized_bounded_text_tuple( + blocking_limitations, "blocking limitation" + ), + ) + return ReconstructionCertificationDossierV2( + policy=policy, + artifacts=selected_artifacts, + gate_results=gate_results, + methodology=methodology, + accepted_limitations=tuple(accepted_limitations), + blocking_limitations=tuple(blocking_limitations), + state=state, + ) + + +def write_reconstruction_certification_dossier( + dossier: ReconstructionCertificationDossierV1, + *, + json_path: str | Path, + markdown_path: str | Path, +) -> tuple[ArtifactRef, ArtifactRef]: + """Atomically publish machine and human certification reports.""" + if not isinstance(dossier, ReconstructionCertificationDossierV1): + raise TypeError("certification publication requires a v1 dossier") + json_target = Path(json_path).expanduser().resolve() + markdown_target = Path(markdown_path).expanduser().resolve() + if json_target == markdown_target: + raise ValueError("certification JSON and Markdown paths must differ") + json_payload = dossier.to_json().encode("utf-8") + b"\n" + markdown_payload = dossier.to_markdown().encode("utf-8") + _atomic_write(json_target, json_payload) + _atomic_write(markdown_target, markdown_payload) + restored = ReconstructionCertificationDossierV1.from_json( + json_target.read_text(encoding="utf-8") + ) + if restored != dossier: + raise ValueError("published certification dossier differs on readback") + return ( + _artifact_ref( + "reconstruction-certification-json", + json_target, + json_payload, + dossier, + ), + _artifact_ref( + "reconstruction-certification-markdown", + markdown_target, + markdown_payload, + dossier, + ), + ) + + +def load_reconstruction_certification_dossier( + path: str | Path, +) -> ReconstructionCertificationDossierV1: + """Load and verify a published machine-readable dossier.""" + return ReconstructionCertificationDossierV1.from_json( + Path(path).expanduser().resolve().read_text(encoding="utf-8") + ) + + +def write_modern_reference_reconstruction_certification_dossier( + dossier: ReconstructionCertificationDossierV2, + *, + json_path: str | Path, + markdown_path: str | Path, +) -> tuple[ArtifactRef, ArtifactRef]: + """Atomically publish machine and human modern-reference reports.""" + if not isinstance(dossier, ReconstructionCertificationDossierV2): + raise TypeError("modern-reference publication requires a v2 dossier") + json_target = Path(json_path).expanduser().resolve() + markdown_target = Path(markdown_path).expanduser().resolve() + if json_target == markdown_target: + raise ValueError("certification JSON and Markdown paths must differ") + json_payload = dossier.to_json().encode("utf-8") + b"\n" + markdown_payload = dossier.to_markdown().encode("utf-8") + _atomic_write(json_target, json_payload) + _atomic_write(markdown_target, markdown_payload) + restored = ReconstructionCertificationDossierV2.from_json( + json_target.read_text(encoding="utf-8") + ) + if restored != dossier: + raise ValueError("published certification dossier differs on readback") + return ( + _artifact_ref( + "reconstruction-certification-json-v2", + json_target, + json_payload, + dossier, + ), + _artifact_ref( + "reconstruction-certification-markdown-v2", + markdown_target, + markdown_payload, + dossier, + ), + ) + + +def load_modern_reference_reconstruction_certification_dossier( + path: str | Path, +) -> ReconstructionCertificationDossierV2: + """Load and verify a published modern-reference dossier.""" + return ReconstructionCertificationDossierV2.from_json( + Path(path).expanduser().resolve().read_text(encoding="utf-8") + ) + + +def _evaluate_requirement( + policy: ReconstructionCertificationPolicyV1, + requirement: CertificationRequirementV1, + observation: CertificationObservationV1 | None, + artifacts: Mapping[str, CertificationArtifactV1], +) -> CertificationCheckResultV1: + if observation is None: + return _missing_result(requirement, "observation is missing") + selected: list[CertificationArtifactV1] = [] + for evidence_id in observation.artifact_evidence_ids: + artifact = artifacts.get(evidence_id) + if artifact is None: + return _missing_result( + requirement, + f"artifact evidence is missing: {evidence_id}", + observation, + ) + if not artifact.verified: + return _missing_result( + requirement, + f"artifact evidence is unverified: {evidence_id}", + observation, + ) + selected.append(artifact) + kinds = {item.kind for item in selected} + missing_kinds = set(requirement.required_artifact_kinds).difference(kinds) + if missing_kinds: + return _missing_result( + requirement, + f"required artifact kinds are missing: {sorted(missing_kinds)}", + observation, + ) + broker_artifacts = [ + item for item in selected if item.kind == "broker-delivery-fingerprint" + ] + if broker_artifacts and any( + item.subject_id != policy.broker_fingerprint_id + for item in broker_artifacts + ): + return CertificationCheckResultV1( + requirement_id=requirement.requirement_id, + check_id=requirement.check_id, + status=CertificationCheckStatus.FAILED, + comparator=requirement.comparator, + expected=requirement.expected, + actual=observation.actual, + artifact_evidence_ids=observation.artifact_evidence_ids, + reason="selected broker fingerprint differs from policy", + ) + passed = _compare( + requirement.comparator, observation.actual, requirement.expected + ) + return CertificationCheckResultV1( + requirement_id=requirement.requirement_id, + check_id=requirement.check_id, + status=( + CertificationCheckStatus.PASSED + if passed + else CertificationCheckStatus.FAILED + ), + comparator=requirement.comparator, + expected=requirement.expected, + actual=observation.actual, + artifact_evidence_ids=observation.artifact_evidence_ids, + reason=( + "requirement satisfied" + if passed + else "measured value violates policy" + ), + ) + + +def _evaluate_requirement_v2( + requirement: CertificationRequirementV1, + observation: CertificationObservationV1 | None, + artifacts: Mapping[str, CertificationArtifactV1], +) -> CertificationCheckResultV1: + """Evaluate one V2 requirement with an exact evidence-kind binding.""" + if observation is None: + return _missing_result(requirement, "observation is missing") + selected: list[CertificationArtifactV1] = [] + for evidence_id in observation.artifact_evidence_ids: + artifact = artifacts.get(evidence_id) + if artifact is None: + return _missing_result( + requirement, + f"artifact evidence is missing: {evidence_id}", + observation, + ) + if not artifact.verified: + return _missing_result( + requirement, + f"artifact evidence is unverified: {evidence_id}", + observation, + ) + if "broker" in artifact.kind: + return CertificationCheckResultV1( + requirement_id=requirement.requirement_id, + check_id=requirement.check_id, + status=CertificationCheckStatus.FAILED, + comparator=requirement.comparator, + expected=requirement.expected, + actual=observation.actual, + artifact_evidence_ids=observation.artifact_evidence_ids, + reason="broker-specific evidence is outside modern-reference scope", + ) + selected.append(artifact) + kinds = {item.kind for item in selected} + required_kinds = set(requirement.required_artifact_kinds) + if kinds != required_kinds: + missing = sorted(required_kinds.difference(kinds)) + extra = sorted(kinds.difference(required_kinds)) + return _missing_result( + requirement, + f"evidence kinds differ; missing={missing}, extra={extra}", + observation, + ) + passed = _compare( + requirement.comparator, observation.actual, requirement.expected + ) + return CertificationCheckResultV1( + requirement_id=requirement.requirement_id, + check_id=requirement.check_id, + status=( + CertificationCheckStatus.PASSED + if passed + else CertificationCheckStatus.FAILED + ), + comparator=requirement.comparator, + expected=requirement.expected, + actual=observation.actual, + artifact_evidence_ids=observation.artifact_evidence_ids, + reason=( + "requirement satisfied" + if passed + else "measured value violates policy" + ), + ) + + +def _missing_result( + requirement: CertificationRequirementV1, + reason: str, + observation: CertificationObservationV1 | None = None, +) -> CertificationCheckResultV1: + return CertificationCheckResultV1( + requirement_id=requirement.requirement_id, + check_id=requirement.check_id, + status=CertificationCheckStatus.MISSING, + comparator=requirement.comparator, + expected=requirement.expected, + actual=None if observation is None else observation.actual, + artifact_evidence_ids=( + () if observation is None else observation.artifact_evidence_ids + ), + reason=reason, + ) + + +def _certification_state( + gate_results: Sequence[CertificationGateResultV1], + blocking_limitations: Sequence[str], +) -> CertificationState: + checks = [item for gate in gate_results for item in gate.check_results] + if any(item.status is CertificationCheckStatus.FAILED for item in checks): + return CertificationState.FAILED + if blocking_limitations: + return CertificationState.INCOMPLETE + missing = { + item.check_id + for item in checks + if item.status is CertificationCheckStatus.MISSING + } + if not missing: + return CertificationState.CERTIFIED + if missing.issubset(PROMOTION_ONLY_CHECK_IDS): + return CertificationState.READY_FOR_PROMOTION + return CertificationState.INCOMPLETE + + +def _compare( + comparator: CertificationComparator, + actual: JSONScalar, + expected: JSONScalar, +) -> bool: + if comparator is CertificationComparator.EQUAL: + return type(actual) is type(expected) and actual == expected + if comparator is CertificationComparator.TRUE: + return actual is True + if comparator is CertificationComparator.FALSE: + return actual is False + if comparator is CertificationComparator.ZERO: + return _numeric(actual, "actual") == 0 + actual_number = _numeric(actual, "actual") + expected_number = _numeric(expected, "expected") + if comparator is CertificationComparator.LESS_OR_EQUAL: + return actual_number <= expected_number + if comparator is CertificationComparator.GREATER_OR_EQUAL: + return actual_number >= expected_number + raise ValueError(f"unsupported certification comparator: {comparator}") + + +def _validate_expected( + comparator: CertificationComparator, expected: JSONScalar +) -> None: + _json_scalar(expected, "expected") + if comparator is CertificationComparator.TRUE and expected is not True: + raise ValueError("true comparator requires expected true") + if comparator is CertificationComparator.FALSE and expected is not False: + raise ValueError("false comparator requires expected false") + if ( + comparator is CertificationComparator.ZERO + and _numeric(expected, "expected") != 0 + ): + raise ValueError("zero comparator requires expected zero") + if comparator in { + CertificationComparator.LESS_OR_EQUAL, + CertificationComparator.GREATER_OR_EQUAL, + }: + _numeric(expected, "expected") + + +def _requirement( + gate: CertificationGate, + check_id: str, + comparator: CertificationComparator, + expected: JSONScalar, + artifact_kinds: tuple[str, ...], + description: str, +) -> CertificationRequirementV1: + return CertificationRequirementV1( + gate=gate, + check_id=check_id, + comparator=comparator, + expected=expected, + required_artifact_kinds=artifact_kinds, + description=description, + ) + + +def _artifact_ref( + kind: str, + path: Path, + payload: bytes, + dossier: ( + ReconstructionCertificationDossierV1 + | ReconstructionCertificationDossierV2 + ), +) -> ArtifactRef: + return ArtifactRef( + kind=kind, + path=str(path), + size_bytes=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + metadata={ + "dossier_id": dossier.dossier_id, + "policy_id": dossier.policy.policy_id, + "state": dossier.state.value, + }, + ) + + +def _atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name( + f".{path.name}.tmp-{hashlib.sha256(payload).hexdigest()[:12]}" + ) + try: + with temporary.open("wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + directory_fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + temporary.unlink(missing_ok=True) + + +def _markdown_limitations(title: str, values: Sequence[str]) -> list[str]: + lines = [f"## {title}", ""] + if values: + lines.extend(f"- {item}" for item in values) + else: + lines.append("- None.") + lines.append("") + return lines + + +def _safe_relative_path(value: Any) -> str: + text = _required_text(value).replace("\\", "/") + path = PurePosixPath(text) + if path.is_absolute() or ".." in path.parts or text.startswith("~"): + raise ValueError( + "certification artifact path must be relative and safe" + ) + return path.as_posix() + + +def _required_period(value: Any) -> str: + text = _required_text(value) + if len(text) != 6 or not text.isdigit(): + raise ValueError("certification period must be YYYYMM") + month = int(text[4:]) + if month < 1 or month > 12: + raise ValueError("certification period month is invalid") + return text + + +def _normalized_symbol(value: Any) -> str: + text = _required_name(value).upper() + if len(text) != 6 or not text.isalpha(): + raise ValueError("certification symbol must be a six-letter FX pair") + return text + + +def _numeric(value: JSONScalar, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + number = float(value) + if not number == number or number in {float("inf"), float("-inf")}: + raise ValueError(f"{name} must be finite") + return number + + +def _positive_float(value: Any, name: str) -> float: + selected = _numeric(cast(JSONScalar, value), name) + if selected <= 0: + raise ValueError(f"{name} must be positive") + return selected + + +def _json_scalar(value: Any, name: str) -> JSONScalar: + if value is None or isinstance(value, (str, int, float, bool)): + if isinstance(value, float): + _numeric(value, name) + return value + raise ValueError(f"{name} must be a JSON scalar") + + +def _required_text(value: Any) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("certification text is required") + if len(text) > DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH: + raise ValueError("certification text exceeds limit") + return text + + +def _required_name(value: Any) -> str: + text = _required_text(value) + if any(character.isspace() for character in text): + raise ValueError("certification identifier cannot contain whitespace") + return text + + +def _bounded_text(value: Any, *, allow_empty: bool = False) -> str: + text = str(value or "").strip() + if not text and not allow_empty: + raise ValueError("certification text is required") + if len(text) > DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH: + raise ValueError("certification text exceeds limit") + return text + + +def _optional_text(value: Any) -> str | None: + text = str(value or "").strip() + return text or None + + +def _required_sha256(value: Any) -> str: + text = _required_name(value).lower() + if len(text) != 64 or any( + character not in "0123456789abcdef" for character in text + ): + raise ValueError("certification content hash must be SHA-256") + return text + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be boolean") + return value + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _nonnegative_int(value: Any, name: str) -> int: + selected = _strict_int(value, name) + if selected < 0: + raise ValueError(f"{name} must be nonnegative") + return selected + + +def _positive_int(value: Any, name: str) -> int: + selected = _strict_int(value, name) + if selected < 1: + raise ValueError(f"{name} must be positive") + return selected + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + selected = _strict_int(value, name) + if selected < minimum or selected > maximum: + raise ValueError(f"{name} is outside bounds") + return selected + + +def _normalized_text_tuple(values: Iterable[Any]) -> tuple[str, ...]: + return tuple(sorted({_required_text(value) for value in values})) + + +def _normalized_bounded_text_tuple( + values: Iterable[Any], name: str +) -> tuple[str, ...]: + selected = tuple(sorted({_bounded_text(value) for value in values})) + if len(selected) > DEFAULT_CERTIFICATION_MAX_METADATA_ITEMS: + raise ValueError(f"{name} count exceeds limit") + return selected + + +def _bounded_mapping( + value: Mapping[str, JSONValue], name: str, maximum: int +) -> dict[str, JSONValue]: + if len(value) > maximum: + raise ValueError(f"{name} exceeds item limit") + selected = {str(key): item for key, item in sorted(value.items())} + _ensure_payload_size(selected, DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH) + return selected + + +def _mapping(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be a mapping") + return cast(Mapping[str, Any], value) + + +def _mapping_sequence(value: Any, name: str) -> tuple[Mapping[str, Any], ...]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError(f"{name} must be a sequence") + return tuple(_mapping(item, name) for item in value) + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError(f"{name} must be a sequence") + return tuple(str(item) for item in value) + + +def _ensure_payload_size(value: Mapping[str, JSONValue], maximum: int) -> None: + if len(canonical_contract_json(dict(value)).encode("utf-8")) > maximum: + raise ValueError("certification payload exceeds limit") + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(dict(payload)).encode("utf-8") + ) + return f"{prefix}:sha256:{digest.hexdigest()}" + + +def _require_version(actual: str, expected: str, name: str) -> None: + if actual != expected: + raise ValueError(f"unsupported {name} schema version") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + _require_version( + str(data.get("schema_version", "")), expected, "certification" + ) + + +def _require_derived(data: Mapping[str, Any], name: str, expected: Any) -> None: + if data.get(name) != expected: + raise ValueError(f"derived certification field {name} differs") + + +def _json_mapping(text: str) -> Mapping[str, Any]: + try: + value = json.loads(text) + except json.JSONDecodeError as error: + raise ValueError("certification JSON is invalid") from error + return _mapping(value, "certification JSON") diff --git a/src/histdatacom/synthetic/certification_campaign.py b/src/histdatacom/synthetic/certification_campaign.py new file mode 100644 index 00000000..2d4597d7 --- /dev/null +++ b/src/histdatacom/synthetic/certification_campaign.py @@ -0,0 +1,818 @@ +"""Executable, hash-verified modern-reference certification campaigns. + +The campaign is an evidence aggregator. It never accepts scalar observations +directly: every measured value is extracted from one hash-verified JSON report +through a declared JSON pointer and remains bound to every supporting artifact. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, cast + +from histdatacom.runtime_contracts import ArtifactRef, JSONScalar, JSONValue +from histdatacom.synthetic.certification import ( + DEFAULT_CERTIFICATION_MAX_ARTIFACTS, + DEFAULT_CERTIFICATION_MAX_METADATA_ITEMS, + DEFAULT_CERTIFICATION_MAX_OBSERVATIONS, + DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH, + PROMOTION_ONLY_CHECK_IDS, + CertificationArtifactV1, + CertificationObservationV1, + CertificationState, + ReconstructionCertificationDossierV2, + evaluate_modern_reference_reconstruction_certification, + modern_reference_triangle_certification_policy, + write_modern_reference_reconstruction_certification_dossier, +) +from histdatacom.synthetic.contracts import canonical_contract_json + +CERTIFICATION_CAMPAIGN_ARTIFACT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-campaign-artifact.v1" +) +CERTIFICATION_CAMPAIGN_OBSERVATION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-campaign-observation.v1" +) +CERTIFICATION_CAMPAIGN_SPEC_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-campaign-spec.v1" +) +CERTIFICATION_CAMPAIGN_RESULT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-campaign-result.v1" +) +CERTIFICATION_METHODOLOGY_REPORT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-certification-methodology-report.v1" +) +CAMPAIGN_MANIFEST_EVIDENCE_KEY = "__campaign_manifest__" +METHODOLOGY_REPORT_EVIDENCE_KEY = "__methodology_report__" +_AUTOMATIC_EVIDENCE_KEYS = frozenset( + {CAMPAIGN_MANIFEST_EVIDENCE_KEY, METHODOLOGY_REPORT_EVIDENCE_KEY} +) +_AUTOMATIC_OBSERVATION_CHECKS = frozenset( + { + "methodology_and_limitations_published", + "machine_evidence_manifest_published", + } +) + + +@dataclass(frozen=True, slots=True) +class CertificationCampaignArtifactV1: + """One strong JSON evidence input and its identity locations.""" + + evidence_key: str + kind: str + path: str + content_sha256: str + subject_id: str + subject_id_pointer: str + subject_schema_version: str + relative_path: str + metadata: Mapping[str, JSONValue] + schema_version: str = CERTIFICATION_CAMPAIGN_ARTIFACT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + CERTIFICATION_CAMPAIGN_ARTIFACT_SCHEMA_VERSION, + "campaign artifact", + ) + for name in ( + "evidence_key", + "kind", + "path", + "subject_id", + "subject_schema_version", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if "broker" in self.kind: + raise ValueError( + "modern-reference campaign rejects broker evidence" + ) + object.__setattr__(self, "content_sha256", _sha256(self.content_sha256)) + object.__setattr__( + self, "subject_id_pointer", _json_pointer(self.subject_id_pointer) + ) + object.__setattr__( + self, "relative_path", _safe_relative_path(self.relative_path) + ) + if len(self.metadata) > DEFAULT_CERTIFICATION_MAX_METADATA_ITEMS: + raise ValueError("campaign artifact metadata exceeds limit") + object.__setattr__( + self, + "metadata", + {str(key): value for key, value in sorted(self.metadata.items())}, + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible input metadata.""" + return { + "schema_version": self.schema_version, + "evidence_key": self.evidence_key, + "kind": self.kind, + "path": self.path, + "content_sha256": self.content_sha256, + "subject_id": self.subject_id, + "subject_id_pointer": self.subject_id_pointer, + "subject_schema_version": self.subject_schema_version, + "relative_path": self.relative_path, + "metadata": dict(self.metadata), + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CertificationCampaignArtifactV1": + """Restore one campaign artifact input.""" + return cls( + evidence_key=str(data.get("evidence_key", "")), + kind=str(data.get("kind", "")), + path=str(data.get("path", "")), + content_sha256=str(data.get("content_sha256", "")), + subject_id=str(data.get("subject_id", "")), + subject_id_pointer=str(data.get("subject_id_pointer", "")), + subject_schema_version=str(data.get("subject_schema_version", "")), + relative_path=str(data.get("relative_path", "")), + metadata=_mapping(data.get("metadata"), "metadata"), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CertificationCampaignObservationV1: + """One scalar extraction bound to its exact supporting evidence set.""" + + check_id: str + measurement_evidence_key: str + measurement_pointer: str + artifact_evidence_keys: tuple[str, ...] + note: str = "" + schema_version: str = CERTIFICATION_CAMPAIGN_OBSERVATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + CERTIFICATION_CAMPAIGN_OBSERVATION_SCHEMA_VERSION, + "campaign observation", + ) + object.__setattr__(self, "check_id", _required_text(self.check_id)) + object.__setattr__( + self, + "measurement_evidence_key", + _required_text(self.measurement_evidence_key), + ) + object.__setattr__( + self, "measurement_pointer", _json_pointer(self.measurement_pointer) + ) + keys = tuple( + sorted( + {_required_text(item) for item in self.artifact_evidence_keys} + ) + ) + if not keys: + raise ValueError("campaign observation requires artifact evidence") + if self.measurement_evidence_key not in keys: + raise ValueError( + "measurement artifact must support its observation" + ) + object.__setattr__(self, "artifact_evidence_keys", keys) + object.__setattr__( + self, "note", _bounded_text(self.note, allow_empty=True) + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic scalar-extraction metadata.""" + return { + "schema_version": self.schema_version, + "check_id": self.check_id, + "measurement_evidence_key": self.measurement_evidence_key, + "measurement_pointer": self.measurement_pointer, + "artifact_evidence_keys": list(self.artifact_evidence_keys), + "note": self.note, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CertificationCampaignObservationV1": + """Restore one scalar-extraction declaration.""" + return cls( + check_id=str(data.get("check_id", "")), + measurement_evidence_key=str( + data.get("measurement_evidence_key", "") + ), + measurement_pointer=str(data.get("measurement_pointer", "")), + artifact_evidence_keys=_string_tuple( + data.get("artifact_evidence_keys"), "artifact_evidence_keys" + ), + note=str(data.get("note", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ModernReferenceCertificationCampaignSpecV1: + """Frozen policy budgets and evidence extraction plan for one campaign.""" + + common_end_period: str + peak_memory_budget_bytes: int + scratch_budget_bytes: int + runtime_budget_seconds: float + storage_budget_bytes: int + candidate_amplification_budget: float + artifacts: tuple[CertificationCampaignArtifactV1, ...] + observations: tuple[CertificationCampaignObservationV1, ...] + methodology: str + accepted_limitations: tuple[str, ...] + blocking_limitations: tuple[str, ...] + promotion_boundary: bool = False + campaign_id: str = "" + schema_version: str = CERTIFICATION_CAMPAIGN_SPEC_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_value( + self.schema_version, + CERTIFICATION_CAMPAIGN_SPEC_SCHEMA_VERSION, + "campaign spec", + ) + policy = modern_reference_triangle_certification_policy( + common_end_period=self.common_end_period, + peak_memory_budget_bytes=self.peak_memory_budget_bytes, + scratch_budget_bytes=self.scratch_budget_bytes, + runtime_budget_seconds=self.runtime_budget_seconds, + storage_budget_bytes=self.storage_budget_bytes, + candidate_amplification_budget=self.candidate_amplification_budget, + ) + object.__setattr__(self, "common_end_period", policy.common_end_period) + artifacts = tuple( + sorted(self.artifacts, key=lambda item: item.evidence_key) + ) + if ( + not artifacts + or len(artifacts) > DEFAULT_CERTIFICATION_MAX_ARTIFACTS + ): + raise ValueError("campaign artifacts are empty or unbounded") + if len({item.evidence_key for item in artifacts}) != len(artifacts): + raise ValueError("campaign artifacts duplicate evidence keys") + if {item.evidence_key for item in artifacts}.intersection( + _AUTOMATIC_EVIDENCE_KEYS + ): + raise ValueError("campaign artifacts use a reserved evidence key") + object.__setattr__(self, "artifacts", artifacts) + observations = tuple( + sorted(self.observations, key=lambda item: item.check_id) + ) + if len(observations) > DEFAULT_CERTIFICATION_MAX_OBSERVATIONS: + raise ValueError("campaign observations exceed limit") + if len({item.check_id for item in observations}) != len(observations): + raise ValueError("campaign observations duplicate checks") + if {item.check_id for item in observations}.intersection( + _AUTOMATIC_OBSERVATION_CHECKS + ): + raise ValueError("campaign publication observations are automatic") + available = { + item.evidence_key for item in artifacts + } | _AUTOMATIC_EVIDENCE_KEYS + missing_keys = { + key + for observation in observations + for key in observation.artifact_evidence_keys + if key not in available + } + if missing_keys: + raise ValueError( + f"campaign evidence keys are missing: {sorted(missing_keys)}" + ) + if not self.promotion_boundary and any( + item.check_id in PROMOTION_ONLY_CHECK_IDS for item in observations + ): + raise ValueError( + "promotion-only coverage evidence is forbidden outside promotion" + ) + if not isinstance(self.promotion_boundary, bool): + raise ValueError("promotion_boundary must be boolean") + object.__setattr__(self, "observations", observations) + object.__setattr__(self, "methodology", _bounded_text(self.methodology)) + object.__setattr__( + self, + "accepted_limitations", + _bounded_text_tuple(self.accepted_limitations), + ) + object.__setattr__( + self, + "blocking_limitations", + _bounded_text_tuple(self.blocking_limitations), + ) + expected = _stable_id( + "reconstruction-certification-campaign", self.identity_payload() + ) + supplied = str(self.campaign_id or "").strip() + if supplied and supplied != expected: + raise ValueError("certification campaign_id differs") + object.__setattr__(self, "campaign_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return content-addressed campaign inputs.""" + return { + "schema_version": self.schema_version, + "common_end_period": self.common_end_period, + "peak_memory_budget_bytes": self.peak_memory_budget_bytes, + "scratch_budget_bytes": self.scratch_budget_bytes, + "runtime_budget_seconds": self.runtime_budget_seconds, + "storage_budget_bytes": self.storage_budget_bytes, + "candidate_amplification_budget": self.candidate_amplification_budget, + "artifacts": [item.to_dict() for item in self.artifacts], + "observations": [item.to_dict() for item in self.observations], + "methodology": self.methodology, + "accepted_limitations": list(self.accepted_limitations), + "blocking_limitations": list(self.blocking_limitations), + "promotion_boundary": self.promotion_boundary, + "observation_values_inline": False, + "broker_adaptation": "excluded", + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible campaign inputs.""" + return {**self.identity_payload(), "campaign_id": self.campaign_id} + + def to_json(self) -> str: + """Serialize the campaign deterministically.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ModernReferenceCertificationCampaignSpecV1": + """Restore and verify a campaign specification.""" + if data.get("observation_values_inline") is not False: + raise ValueError( + "campaign cannot contain inline observation values" + ) + if data.get("broker_adaptation") != "excluded": + raise ValueError("campaign broker boundary differs") + return cls( + common_end_period=str(data.get("common_end_period", "")), + peak_memory_budget_bytes=_strict_int( + data.get("peak_memory_budget_bytes"), "peak_memory_budget_bytes" + ), + scratch_budget_bytes=_strict_int( + data.get("scratch_budget_bytes"), "scratch_budget_bytes" + ), + runtime_budget_seconds=_number( + data.get("runtime_budget_seconds"), "runtime_budget_seconds" + ), + storage_budget_bytes=_strict_int( + data.get("storage_budget_bytes"), "storage_budget_bytes" + ), + candidate_amplification_budget=_number( + data.get("candidate_amplification_budget"), + "candidate_amplification_budget", + ), + artifacts=tuple( + CertificationCampaignArtifactV1.from_dict(item) + for item in _mapping_sequence( + data.get("artifacts"), "artifacts" + ) + ), + observations=tuple( + CertificationCampaignObservationV1.from_dict(item) + for item in _mapping_sequence( + data.get("observations"), "observations" + ) + ), + methodology=str(data.get("methodology", "")), + accepted_limitations=_string_tuple( + data.get("accepted_limitations"), "accepted_limitations" + ), + blocking_limitations=_string_tuple( + data.get("blocking_limitations"), "blocking_limitations" + ), + promotion_boundary=_strict_bool( + data.get("promotion_boundary"), "promotion_boundary" + ), + campaign_id=str(data.get("campaign_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ModernReferenceCertificationCampaignResultV1: + """Public references and state produced by one campaign execution.""" + + campaign_id: str + policy_id: str + dossier_id: str + state: CertificationState + dossier_json: ArtifactRef + dossier_markdown: ArtifactRef + verified_input_count: int + observation_count: int + schema_version: str = CERTIFICATION_CAMPAIGN_RESULT_SCHEMA_VERSION + + def to_dict(self) -> dict[str, JSONValue]: + """Return a bounded public campaign receipt.""" + return { + "schema_version": self.schema_version, + "campaign_id": self.campaign_id, + "policy_id": self.policy_id, + "dossier_id": self.dossier_id, + "state": self.state.value, + "dossier_json": self.dossier_json.to_dict(), + "dossier_markdown": self.dossier_markdown.to_dict(), + "verified_input_count": self.verified_input_count, + "observation_count": self.observation_count, + } + + +def read_modern_reference_certification_campaign_spec( + path: str | Path, +) -> ModernReferenceCertificationCampaignSpecV1: + """Read and identity-check one campaign specification.""" + return ModernReferenceCertificationCampaignSpecV1.from_dict( + _read_json_mapping(Path(path).expanduser().resolve()) + ) + + +def run_modern_reference_certification_campaign( + spec_path: str | Path, + *, + output_directory: str | Path, +) -> tuple[ + ReconstructionCertificationDossierV2, + ModernReferenceCertificationCampaignResultV1, +]: + """Verify inputs, extract observations, and atomically publish a dossier.""" + source = Path(spec_path).expanduser().resolve() + spec = read_modern_reference_certification_campaign_spec(source) + output = Path(output_directory).expanduser().resolve() + policy = modern_reference_triangle_certification_policy( + common_end_period=spec.common_end_period, + peak_memory_budget_bytes=spec.peak_memory_budget_bytes, + scratch_budget_bytes=spec.scratch_budget_bytes, + runtime_budget_seconds=spec.runtime_budget_seconds, + storage_budget_bytes=spec.storage_budget_bytes, + candidate_amplification_budget=spec.candidate_amplification_budget, + ) + artifacts: list[CertificationArtifactV1] = [] + payloads: dict[str, Mapping[str, Any]] = {} + evidence_by_key: dict[str, CertificationArtifactV1] = {} + for declared in spec.artifacts: + path = Path(declared.path).expanduser() + if not path.is_absolute(): + path = source.parent / path + path = path.resolve() + payload, encoded = _verified_json_artifact(path, declared) + artifact = CertificationArtifactV1( + policy_id=policy.policy_id, + kind=declared.kind, + subject_id=declared.subject_id, + subject_schema_version=declared.subject_schema_version, + content_sha256=hashlib.sha256(encoded).hexdigest(), + relative_path=declared.relative_path, + size_bytes=len(encoded), + verified=True, + metadata=declared.metadata, + ) + payloads[declared.evidence_key] = payload + evidence_by_key[declared.evidence_key] = artifact + artifacts.append(artifact) + + evidence_directory = output / "evidence" + campaign_path = evidence_directory / "campaign-spec.json" + campaign_bytes = spec.to_json().encode("utf-8") + b"\n" + _atomic_write(campaign_path, campaign_bytes) + campaign_artifact = CertificationArtifactV1( + policy_id=policy.policy_id, + kind="machine-evidence-manifest", + subject_id=spec.campaign_id, + subject_schema_version=spec.schema_version, + content_sha256=hashlib.sha256(campaign_bytes).hexdigest(), + relative_path="evidence/campaign-spec.json", + size_bytes=len(campaign_bytes), + verified=True, + metadata={"observation_values_inline": False}, + ) + artifacts.append(campaign_artifact) + payloads[CAMPAIGN_MANIFEST_EVIDENCE_KEY] = spec.to_dict() + evidence_by_key[CAMPAIGN_MANIFEST_EVIDENCE_KEY] = campaign_artifact + + methodology_payload: dict[str, JSONValue] = { + "schema_version": CERTIFICATION_METHODOLOGY_REPORT_SCHEMA_VERSION, + "campaign_id": spec.campaign_id, + "methodology": spec.methodology, + "accepted_limitations": list(spec.accepted_limitations), + "blocking_limitations": list(spec.blocking_limitations), + "scientific_nonclaim_published": True, + "historical_truth_claim": False, + "broker_specific_claim": False, + } + methodology_bytes = ( + canonical_contract_json(methodology_payload).encode("utf-8") + b"\n" + ) + methodology_path = evidence_directory / "methodology.json" + _atomic_write(methodology_path, methodology_bytes) + methodology_id = _stable_id( + "certification-methodology", methodology_payload + ) + methodology_artifact = CertificationArtifactV1( + policy_id=policy.policy_id, + kind="methodology-report", + subject_id=methodology_id, + subject_schema_version=CERTIFICATION_METHODOLOGY_REPORT_SCHEMA_VERSION, + content_sha256=hashlib.sha256(methodology_bytes).hexdigest(), + relative_path="evidence/methodology.json", + size_bytes=len(methodology_bytes), + verified=True, + metadata={ + "historical_truth_claim": False, + "broker_specific_claim": False, + }, + ) + artifacts.append(methodology_artifact) + payloads[METHODOLOGY_REPORT_EVIDENCE_KEY] = methodology_payload + evidence_by_key[METHODOLOGY_REPORT_EVIDENCE_KEY] = methodology_artifact + + observations = [ + _extract_observation(declared, payloads, evidence_by_key) + for declared in spec.observations + ] + observations.append( + CertificationObservationV1( + check_id="methodology_and_limitations_published", + actual=True, + artifact_evidence_ids=(methodology_artifact.evidence_id,), + note="published atomically by the certification campaign", + ) + ) + observations.append( + CertificationObservationV1( + check_id="machine_evidence_manifest_published", + actual=True, + artifact_evidence_ids=(campaign_artifact.evidence_id,), + note="published atomically by the certification campaign", + ) + ) + dossier = evaluate_modern_reference_reconstruction_certification( + policy, + artifacts=artifacts, + observations=observations, + methodology=spec.methodology, + accepted_limitations=spec.accepted_limitations, + blocking_limitations=spec.blocking_limitations, + ) + json_ref, markdown_ref = ( + write_modern_reference_reconstruction_certification_dossier( + dossier, + json_path=output / "certification.json", + markdown_path=output / "certification.md", + ) + ) + result = ModernReferenceCertificationCampaignResultV1( + campaign_id=spec.campaign_id, + policy_id=policy.policy_id, + dossier_id=dossier.dossier_id, + state=dossier.state, + dossier_json=json_ref, + dossier_markdown=markdown_ref, + verified_input_count=len(spec.artifacts), + observation_count=len(observations), + ) + _atomic_write( + output / "campaign-result.json", + canonical_contract_json(result.to_dict()).encode("utf-8") + b"\n", + ) + return dossier, result + + +def _verified_json_artifact( + path: Path, declared: CertificationCampaignArtifactV1 +) -> tuple[Mapping[str, Any], bytes]: + try: + encoded = path.read_bytes() + except OSError as error: + raise ValueError( + f"cannot read certification artifact {path}: {error}" + ) from error + digest = hashlib.sha256(encoded).hexdigest() + if digest != declared.content_sha256: + raise ValueError(f"certification artifact hash differs: {path}") + payload = _json_bytes_mapping(encoded, path) + if payload.get("schema_version") != declared.subject_schema_version: + raise ValueError(f"certification artifact schema differs: {path}") + subject = _resolve_json_pointer(payload, declared.subject_id_pointer) + if subject != declared.subject_id: + raise ValueError( + f"certification artifact subject identity differs: {path}" + ) + return payload, encoded + + +def _extract_observation( + declared: CertificationCampaignObservationV1, + payloads: Mapping[str, Mapping[str, Any]], + evidence_by_key: Mapping[str, CertificationArtifactV1], +) -> CertificationObservationV1: + value = _resolve_json_pointer( + payloads[declared.measurement_evidence_key], + declared.measurement_pointer, + ) + actual = _json_scalar(value, declared.check_id) + return CertificationObservationV1( + check_id=declared.check_id, + actual=actual, + artifact_evidence_ids=tuple( + evidence_by_key[key].evidence_id + for key in declared.artifact_evidence_keys + ), + note=declared.note, + ) + + +def _resolve_json_pointer(value: Any, pointer: str) -> Any: + selected = value + if pointer == "": + return selected + for raw_token in pointer[1:].split("/"): + token = raw_token.replace("~1", "/").replace("~0", "~") + if isinstance(selected, Mapping): + if token not in selected: + raise ValueError(f"JSON pointer does not exist: {pointer}") + selected = selected[token] + elif isinstance(selected, Sequence) and not isinstance( + selected, (str, bytes, bytearray) + ): + try: + selected = selected[int(token)] + except (ValueError, IndexError) as error: + raise ValueError( + f"JSON pointer does not exist: {pointer}" + ) from error + else: + raise ValueError(f"JSON pointer does not exist: {pointer}") + return selected + + +def _read_json_mapping(path: Path) -> Mapping[str, Any]: + try: + return _json_bytes_mapping(path.read_bytes(), path) + except OSError as error: + raise ValueError( + f"cannot read certification campaign {path}: {error}" + ) from error + + +def _json_bytes_mapping(encoded: bytes, path: Path) -> Mapping[str, Any]: + try: + value = json.loads(encoded) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError( + f"invalid JSON certification artifact: {path}" + ) from error + return _mapping(value, str(path)) + + +def _atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name( + f".{path.name}.tmp-{hashlib.sha256(payload).hexdigest()[:12]}" + ) + try: + with temporary.open("wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _json_pointer(value: Any) -> str: + pointer = str(value) + if pointer and not pointer.startswith("/"): + raise ValueError("JSON pointer must be empty or start with a slash") + return pointer + + +def _safe_relative_path(value: Any) -> str: + path = PurePosixPath(_required_text(value)) + if path.is_absolute() or ".." in path.parts or path == PurePosixPath("."): + raise ValueError("campaign relative path must be relative and safe") + return str(path) + + +def _sha256(value: Any) -> str: + text = _required_text(value).lower() + if len(text) != 64 or any( + character not in "0123456789abcdef" for character in text + ): + raise ValueError("campaign content hash must be SHA-256") + return text + + +def _required_text(value: Any) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("campaign text is required") + if len(text) > DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH: + raise ValueError("campaign text exceeds limit") + return text + + +def _bounded_text(value: Any, *, allow_empty: bool = False) -> str: + text = str(value or "").strip() + if not text and not allow_empty: + raise ValueError("campaign text is required") + if len(text) > DEFAULT_CERTIFICATION_MAX_TEXT_LENGTH: + raise ValueError("campaign text exceeds limit") + return text + + +def _bounded_text_tuple(values: Sequence[str]) -> tuple[str, ...]: + selected = tuple(sorted({_bounded_text(value) for value in values})) + if len(selected) > DEFAULT_CERTIFICATION_MAX_METADATA_ITEMS: + raise ValueError("campaign limitation count exceeds limit") + return selected + + +def _json_scalar(value: Any, name: str) -> JSONScalar: + if value is None or isinstance(value, (str, int, float, bool)): + if isinstance(value, float): + _number(value, name) + return value + raise ValueError(f"{name} measurement must be a JSON scalar") + + +def _number(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + selected = float(value) + if selected != selected or selected in {float("inf"), float("-inf")}: + raise ValueError(f"{name} must be finite") + return selected + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be boolean") + return value + + +def _mapping(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be a JSON object") + return cast(Mapping[str, Any], value) + + +def _mapping_sequence(value: Any, name: str) -> tuple[Mapping[str, Any], ...]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError(f"{name} must be a sequence") + return tuple(_mapping(item, name) for item in value) + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError(f"{name} must be a sequence") + return tuple(str(item) for item in value) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(dict(payload)).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _require_schema_value(actual: str, expected: str, name: str) -> None: + if actual != expected: + raise ValueError(f"unsupported {name} schema version") + + +__all__ = [ + "CERTIFICATION_CAMPAIGN_ARTIFACT_SCHEMA_VERSION", + "CERTIFICATION_CAMPAIGN_OBSERVATION_SCHEMA_VERSION", + "CERTIFICATION_CAMPAIGN_RESULT_SCHEMA_VERSION", + "CERTIFICATION_CAMPAIGN_SPEC_SCHEMA_VERSION", + "CERTIFICATION_METHODOLOGY_REPORT_SCHEMA_VERSION", + "CAMPAIGN_MANIFEST_EVIDENCE_KEY", + "METHODOLOGY_REPORT_EVIDENCE_KEY", + "CertificationCampaignArtifactV1", + "CertificationCampaignObservationV1", + "ModernReferenceCertificationCampaignResultV1", + "ModernReferenceCertificationCampaignSpecV1", + "read_modern_reference_certification_campaign_spec", + "run_modern_reference_certification_campaign", +] diff --git a/src/histdatacom/synthetic/contracts.py b/src/histdatacom/synthetic/contracts.py new file mode 100644 index 00000000..a6a89f88 --- /dev/null +++ b/src/histdatacom/synthetic/contracts.py @@ -0,0 +1,1221 @@ +"""Narrow, versioned contracts for reconstructed market-event streams. + +The contracts in this module deliberately contain no generator, carving, +workflow, or final partitioning behavior. They freeze the portable event +identity and serialization boundary that those later stages share. + +Version-one schemas are immutable. A semantic change to a required field, +identity derivation, ordering rule, or Arrow type requires a new schema +version and a new contract class. Readers reject other schema versions. +Optional fields may be absent from JSON input and unknown JSON keys are +ignored, but the version-one Arrow schema is exact so persisted columns cannot +drift silently. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any, cast + +from histdatacom.runtime_contracts import JSONValue + +SYNTHETIC_EVENT_SCHEMA_VERSION = "histdatacom.synthetic-event.v1" +SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION = "histdatacom.synthetic-event-stream.v1" +SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.synthetic-ensemble-manifest.v1" +) + +EVENT_SCHEMA_METADATA_KEY = b"histdatacom.synthetic_event_schema" +EVENT_TIME_UNIT_METADATA_KEY = b"histdatacom.event_time_unit" +STREAM_METADATA_KEY = b"histdatacom.synthetic_event_stream" + +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 + +SYNTHETIC_EVENT_ARROW_COLUMNS = ( + "schema_version", + "event_id", + "origin", + "symbol", + "event_time_ns", + "event_sequence", + "bid", + "ask", + "run_id", + "ensemble_member_id", + "source_version_id", + "source_series_id", + "source_period", + "source_row_id", + "anchor_interval_id", + "left_anchor_event_id", + "right_anchor_event_id", + "generator_id", + "generator_version", + "generator_config_id", + "reference_id", + "motif_id", + "feed_epoch_id", + "broker_profile_id", + "constraint_set_id", + "confidence", +) + + +class SyntheticEventOrigin(str, Enum): + """Whether a product row is immutable evidence or generated infill.""" + + OBSERVED = "observed" + SYNTHETIC = "synthetic" + + @classmethod + def from_value( + cls, value: str | "SyntheticEventOrigin" + ) -> "SyntheticEventOrigin": + """Return a strict normalized event origin.""" + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError("unsupported synthetic event origin") from err + + +@dataclass(frozen=True, slots=True) +class SyntheticEventV1: + """One observed or generated bid/ask event at nanosecond resolution.""" + + origin: SyntheticEventOrigin + symbol: str + event_time_ns: int + event_sequence: int + bid: float + ask: float + run_id: str + ensemble_member_id: str + source_version_id: str + source_series_id: str | None = None + source_period: str | None = None + source_row_id: int | None = None + anchor_interval_id: str | None = None + left_anchor_event_id: str | None = None + right_anchor_event_id: str | None = None + generator_id: str | None = None + generator_version: str | None = None + generator_config_id: str | None = None + reference_id: str | None = None + motif_id: str | None = None + feed_epoch_id: str | None = None + broker_profile_id: str | None = None + constraint_set_id: str | None = None + confidence: float | None = None + event_id: str = "" + schema_version: str = SYNTHETIC_EVENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + """Normalize values and enforce origin-specific lineage.""" + if self.schema_version != SYNTHETIC_EVENT_SCHEMA_VERSION: + raise ValueError("unsupported synthetic event schema version") + object.__setattr__( + self, + "origin", + SyntheticEventOrigin.from_value(self.origin), + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, + "ensemble_member_id", + _required_text(self.ensemble_member_id), + ) + object.__setattr__( + self, + "source_version_id", + _required_text(self.source_version_id), + ) + for name in _OPTIONAL_EVENT_TEXT_FIELDS: + object.__setattr__(self, name, _optional_text(getattr(self, name))) + + object.__setattr__( + self, + "event_time_ns", + _bounded_int(self.event_time_ns, "event_time_ns"), + ) + sequence = _bounded_int(self.event_sequence, "event_sequence") + if sequence < 0: + raise ValueError("event_sequence must be non-negative") + object.__setattr__(self, "event_sequence", sequence) + bid = _finite_float(self.bid, "bid") + ask = _finite_float(self.ask, "ask") + if bid <= 0.0 or ask <= 0.0: + raise ValueError("bid and ask must be positive") + if ask < bid: + raise ValueError("ask must be greater than or equal to bid") + object.__setattr__(self, "bid", bid) + object.__setattr__(self, "ask", ask) + if self.confidence is not None: + confidence = _finite_float(self.confidence, "confidence") + if not 0.0 <= confidence <= 1.0: + raise ValueError("confidence must be between zero and one") + object.__setattr__(self, "confidence", confidence) + + self._validate_lineage() + expected = _stable_id("event", self.identity_payload()) + supplied = _optional_text(self.event_id) + if supplied is not None and supplied != expected: + raise ValueError("event_id does not match deterministic identity") + object.__setattr__(self, "event_id", expected) + + @classmethod + def observed( + cls, + *, + symbol: str, + event_time_ns: int, + event_sequence: int, + bid: float, + ask: float, + run_id: str, + ensemble_member_id: str, + source_version_id: str, + source_series_id: str, + source_period: str, + source_row_id: int, + ) -> "SyntheticEventV1": + """Construct one immutable observed event.""" + return cls( + origin=SyntheticEventOrigin.OBSERVED, + symbol=symbol, + event_time_ns=event_time_ns, + event_sequence=event_sequence, + bid=bid, + ask=ask, + run_id=run_id, + ensemble_member_id=ensemble_member_id, + source_version_id=source_version_id, + source_series_id=source_series_id, + source_period=source_period, + source_row_id=source_row_id, + ) + + @classmethod + def generated( + cls, + *, + symbol: str, + event_time_ns: int, + event_sequence: int, + bid: float, + ask: float, + run_id: str, + ensemble_member_id: str, + source_version_id: str, + left_anchor_event_id: str, + right_anchor_event_id: str, + generator_id: str, + generator_version: str, + generator_config_id: str, + constraint_set_id: str, + confidence: float | None = None, + anchor_interval_id: str | None = None, + reference_id: str | None = None, + motif_id: str | None = None, + feed_epoch_id: str | None = None, + broker_profile_id: str | None = None, + ) -> "SyntheticEventV1": + """Construct one generated event with reproducible lineage.""" + interval_id = anchor_interval_id or derive_anchor_interval_id( + left_anchor_event_id, + right_anchor_event_id, + ) + return cls( + origin=SyntheticEventOrigin.SYNTHETIC, + symbol=symbol, + event_time_ns=event_time_ns, + event_sequence=event_sequence, + bid=bid, + ask=ask, + run_id=run_id, + ensemble_member_id=ensemble_member_id, + source_version_id=source_version_id, + anchor_interval_id=interval_id, + left_anchor_event_id=left_anchor_event_id, + right_anchor_event_id=right_anchor_event_id, + generator_id=generator_id, + generator_version=generator_version, + generator_config_id=generator_config_id, + reference_id=reference_id, + motif_id=motif_id, + feed_epoch_id=feed_epoch_id, + broker_profile_id=broker_profile_id, + constraint_set_id=constraint_set_id, + confidence=confidence, + ) + + def _validate_lineage(self) -> None: + if self.origin is SyntheticEventOrigin.OBSERVED: + for name in ( + "source_series_id", + "source_period", + ): + if getattr(self, name) is None: + raise ValueError(f"observed event requires {name}") + if self.source_row_id is None: + raise ValueError("observed event requires source_row_id") + row_id = _bounded_int(self.source_row_id, "source_row_id") + if row_id < 1: + raise ValueError("source_row_id must be positive") + object.__setattr__(self, "source_row_id", row_id) + populated = [ + name + for name in _SYNTHETIC_LINEAGE_FIELDS + if getattr(self, name) is not None + ] + if populated: + raise ValueError( + "observed event cannot carry synthetic lineage: " + + ", ".join(populated) + ) + return + + populated_source = [ + name + for name in ( + "source_series_id", + "source_period", + "source_row_id", + ) + if getattr(self, name) is not None + ] + if populated_source: + raise ValueError( + "synthetic event cannot claim observed row identity: " + + ", ".join(populated_source) + ) + for name in _REQUIRED_SYNTHETIC_LINEAGE_FIELDS: + if getattr(self, name) is None: + raise ValueError(f"synthetic event requires {name}") + if self.left_anchor_event_id == self.right_anchor_event_id: + raise ValueError("synthetic event requires distinct anchors") + + def identity_payload(self) -> dict[str, JSONValue]: + """Return the canonical fields used to derive the event ID.""" + common: dict[str, JSONValue] = { + "schema_version": self.schema_version, + "origin": self.origin.value, + "symbol": self.symbol, + "event_time_ns": self.event_time_ns, + "event_sequence": self.event_sequence, + "source_version_id": self.source_version_id, + } + if self.origin is SyntheticEventOrigin.OBSERVED: + common.update( + { + "source_series_id": self.source_series_id, + "source_period": self.source_period, + "source_row_id": self.source_row_id, + } + ) + return common + common.update( + { + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "anchor_interval_id": self.anchor_interval_id, + "left_anchor_event_id": self.left_anchor_event_id, + "right_anchor_event_id": self.right_anchor_event_id, + "generator_id": self.generator_id, + "generator_version": self.generator_version, + "generator_config_id": self.generator_config_id, + "reference_id": self.reference_id, + "motif_id": self.motif_id, + "feed_epoch_id": self.feed_epoch_id, + "broker_profile_id": self.broker_profile_id, + "constraint_set_id": self.constraint_set_id, + } + ) + return common + + def to_dict(self) -> dict[str, JSONValue]: + """Return the stable, flat, JSON-compatible event representation.""" + return { + "schema_version": self.schema_version, + "event_id": self.event_id, + "origin": self.origin.value, + "symbol": self.symbol, + "event_time_ns": self.event_time_ns, + "event_sequence": self.event_sequence, + "bid": self.bid, + "ask": self.ask, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "source_version_id": self.source_version_id, + "source_series_id": self.source_series_id, + "source_period": self.source_period, + "source_row_id": self.source_row_id, + "anchor_interval_id": self.anchor_interval_id, + "left_anchor_event_id": self.left_anchor_event_id, + "right_anchor_event_id": self.right_anchor_event_id, + "generator_id": self.generator_id, + "generator_version": self.generator_version, + "generator_config_id": self.generator_config_id, + "reference_id": self.reference_id, + "motif_id": self.motif_id, + "feed_epoch_id": self.feed_epoch_id, + "broker_profile_id": self.broker_profile_id, + "constraint_set_id": self.constraint_set_id, + "confidence": self.confidence, + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return canonical_contract_json(self.to_dict()) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "SyntheticEventV1": + """Restore a version-one event and verify its deterministic ID.""" + _require_schema(data, SYNTHETIC_EVENT_SCHEMA_VERSION) + return cls( + origin=SyntheticEventOrigin.from_value(str(data.get("origin"))), + symbol=str(data.get("symbol", "")), + event_time_ns=cast(int, data.get("event_time_ns")), + event_sequence=cast(int, data.get("event_sequence")), + bid=cast(float, data.get("bid")), + ask=cast(float, data.get("ask")), + run_id=str(data.get("run_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + source_version_id=str(data.get("source_version_id", "")), + source_series_id=_mapping_optional_text(data, "source_series_id"), + source_period=_mapping_optional_text(data, "source_period"), + source_row_id=cast(int | None, data.get("source_row_id")), + anchor_interval_id=_mapping_optional_text( + data, "anchor_interval_id" + ), + left_anchor_event_id=_mapping_optional_text( + data, "left_anchor_event_id" + ), + right_anchor_event_id=_mapping_optional_text( + data, "right_anchor_event_id" + ), + generator_id=_mapping_optional_text(data, "generator_id"), + generator_version=_mapping_optional_text(data, "generator_version"), + generator_config_id=_mapping_optional_text( + data, "generator_config_id" + ), + reference_id=_mapping_optional_text(data, "reference_id"), + motif_id=_mapping_optional_text(data, "motif_id"), + feed_epoch_id=_mapping_optional_text(data, "feed_epoch_id"), + broker_profile_id=_mapping_optional_text(data, "broker_profile_id"), + constraint_set_id=_mapping_optional_text(data, "constraint_set_id"), + confidence=cast(float | None, data.get("confidence")), + event_id=str(data.get("event_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "SyntheticEventV1": + """Restore an event from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class SyntheticEventStreamV1: + """One deterministically ordered symbol/member event stream.""" + + run_id: str + ensemble_member_id: str + symbol: str + events: tuple[SyntheticEventV1, ...] + source_version_ids: tuple[str, ...] = () + stream_id: str = "" + schema_version: str = SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION: + raise ValueError("unsupported synthetic event stream schema") + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, + "ensemble_member_id", + _required_text(self.ensemble_member_id), + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + ordered = tuple(sorted(tuple(self.events), key=_event_order_key)) + object.__setattr__(self, "events", ordered) + + event_ids: set[str] = set() + positions: set[tuple[int, int]] = set() + event_sources: set[str] = set() + for event in ordered: + if event.run_id != self.run_id: + raise ValueError("event run_id does not match stream") + if event.ensemble_member_id != self.ensemble_member_id: + raise ValueError("event ensemble member does not match stream") + if event.symbol != self.symbol: + raise ValueError("event symbol does not match stream") + if event.event_id in event_ids: + raise ValueError("duplicate event_id in stream") + position = (event.event_time_ns, event.event_sequence) + if position in positions: + raise ValueError( + "duplicate event_time_ns/event_sequence in stream" + ) + event_ids.add(event.event_id) + positions.add(position) + event_sources.add(event.source_version_id) + + sources = _normalized_id_tuple(self.source_version_ids) + if not sources: + sources = tuple(sorted(event_sources)) + if not sources: + raise ValueError("stream requires at least one source version") + if not event_sources.issubset(set(sources)): + raise ValueError("stream source versions do not cover events") + object.__setattr__(self, "source_version_ids", sources) + + expected = _stable_id("stream", self.identity_payload()) + supplied = _optional_text(self.stream_id) + if supplied is not None and supplied != expected: + raise ValueError("stream_id does not match deterministic content") + object.__setattr__(self, "stream_id", expected) + + @classmethod + def merge( + cls, + *, + run_id: str, + ensemble_member_id: str, + symbol: str, + observed_events: Iterable[SyntheticEventV1], + synthetic_events: Iterable[SyntheticEventV1], + source_version_ids: Iterable[str] = (), + ) -> "SyntheticEventStreamV1": + """Merge immutable observations and zero-or-more generated events.""" + observed = tuple(observed_events) + generated = tuple(synthetic_events) + if any( + event.origin is not SyntheticEventOrigin.OBSERVED + for event in observed + ): + raise ValueError("observed_events contains a synthetic event") + if any( + event.origin is not SyntheticEventOrigin.SYNTHETIC + for event in generated + ): + raise ValueError("synthetic_events contains an observed event") + return cls( + run_id=run_id, + ensemble_member_id=ensemble_member_id, + symbol=symbol, + events=observed + generated, + source_version_ids=tuple(source_version_ids), + ) + + @property + def observed_event_count(self) -> int: + """Return the number of immutable observed events.""" + return sum( + event.origin is SyntheticEventOrigin.OBSERVED + for event in self.events + ) + + @property + def synthetic_event_count(self) -> int: + """Return the number of generated events.""" + return len(self.events) - self.observed_event_count + + def identity_payload(self) -> dict[str, JSONValue]: + """Return canonical stream identity fields.""" + return { + "schema_version": self.schema_version, + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "symbol": self.symbol, + "source_version_ids": list(self.source_version_ids), + "event_ids": [event.event_id for event in self.events], + } + + def header_dict(self) -> dict[str, JSONValue]: + """Return bounded metadata stored in Arrow schema metadata.""" + return { + "schema_version": self.schema_version, + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "stream_id": self.stream_id, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "symbol": self.symbol, + "source_version_ids": list(self.source_version_ids), + "event_count": len(self.events), + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return the stable JSON-compatible stream representation.""" + payload = self.header_dict() + payload["events"] = [event.to_dict() for event in self.events] + return payload + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return canonical_contract_json(self.to_dict()) + + def to_json_header(self) -> str: + """Return deterministic bounded stream-header JSON.""" + return canonical_contract_json(self.header_dict()) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "SyntheticEventStreamV1": + """Restore a stream and verify all event/content identities.""" + _require_schema(data, SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION) + _require_derived_schema( + data, + "event_schema_version", + SYNTHETIC_EVENT_SCHEMA_VERSION, + ) + rows = _mapping_sequence(data, "events") + stream = cls( + run_id=str(data.get("run_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbol=str(data.get("symbol", "")), + events=tuple(SyntheticEventV1.from_dict(row) for row in rows), + source_version_ids=tuple( + str(value) + for value in _value_sequence( + data.get("source_version_ids"), + "source_version_ids", + ) + ), + stream_id=str(data.get("stream_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _validate_stream_counts(data, stream) + return stream + + @classmethod + def from_json(cls, text: str) -> "SyntheticEventStreamV1": + """Restore a stream from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class SyntheticEnsembleMemberV1: + """Compact manifest record for one materialized ensemble member.""" + + member_id: str + stream_id: str + event_count: int + observed_event_count: int + synthetic_event_count: int + content_sha256: str + + def __post_init__(self) -> None: + object.__setattr__(self, "member_id", _required_text(self.member_id)) + object.__setattr__(self, "stream_id", _required_text(self.stream_id)) + for name in ( + "event_count", + "observed_event_count", + "synthetic_event_count", + ): + value = _bounded_int(getattr(self, name), name) + if value < 0: + raise ValueError(f"{name} must be non-negative") + object.__setattr__(self, name, value) + if self.event_count != ( + self.observed_event_count + self.synthetic_event_count + ): + raise ValueError("ensemble member event counts do not reconcile") + digest = _required_text(self.content_sha256) + if not _is_sha256_id(digest): + raise ValueError("content_sha256 must be a sha256 identifier") + object.__setattr__(self, "content_sha256", digest) + + @classmethod + def from_stream( + cls, stream: SyntheticEventStreamV1 + ) -> "SyntheticEnsembleMemberV1": + """Build compact member evidence from one stream.""" + content = canonical_contract_json( + [event.to_dict() for event in stream.events] + ).encode("utf-8") + return cls( + member_id=stream.ensemble_member_id, + stream_id=stream.stream_id, + event_count=len(stream.events), + observed_event_count=stream.observed_event_count, + synthetic_event_count=stream.synthetic_event_count, + content_sha256="sha256:" + hashlib.sha256(content).hexdigest(), + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return stable JSON-compatible member evidence.""" + return { + "member_id": self.member_id, + "stream_id": self.stream_id, + "event_count": self.event_count, + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "content_sha256": self.content_sha256, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "SyntheticEnsembleMemberV1": + """Restore compact member evidence.""" + return cls( + member_id=str(data.get("member_id", "")), + stream_id=str(data.get("stream_id", "")), + event_count=cast(int, data.get("event_count")), + observed_event_count=cast(int, data.get("observed_event_count")), + synthetic_event_count=cast(int, data.get("synthetic_event_count")), + content_sha256=str(data.get("content_sha256", "")), + ) + + +@dataclass(frozen=True, slots=True) +class SyntheticEnsembleManifestV1: + """Deterministic manifest for a set of reconstructed member streams.""" + + run_id: str + primary_member_id: str + members: tuple[SyntheticEnsembleMemberV1, ...] + source_version_ids: tuple[str, ...] + configuration_ids: tuple[str, ...] + ensemble_id: str = "" + schema_version: str = SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION: + raise ValueError("unsupported synthetic ensemble manifest schema") + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, + "primary_member_id", + _required_text(self.primary_member_id), + ) + members = tuple(sorted(tuple(self.members), key=lambda x: x.member_id)) + if not members: + raise ValueError("ensemble manifest requires at least one member") + member_ids = [member.member_id for member in members] + stream_ids = [member.stream_id for member in members] + if len(set(member_ids)) != len(member_ids): + raise ValueError("ensemble member IDs must be unique") + if len(set(stream_ids)) != len(stream_ids): + raise ValueError("ensemble stream IDs must be unique") + if self.primary_member_id not in member_ids: + raise ValueError("primary_member_id is not present in members") + object.__setattr__(self, "members", members) + sources = _normalized_id_tuple(self.source_version_ids) + configs = _normalized_id_tuple(self.configuration_ids) + if not sources: + raise ValueError("ensemble manifest requires source versions") + if not configs: + raise ValueError("ensemble manifest requires configurations") + object.__setattr__(self, "source_version_ids", sources) + object.__setattr__(self, "configuration_ids", configs) + + expected = _stable_id("ensemble", self.identity_payload()) + supplied = _optional_text(self.ensemble_id) + if supplied is not None and supplied != expected: + raise ValueError( + "ensemble_id does not match deterministic manifest" + ) + object.__setattr__(self, "ensemble_id", expected) + + @classmethod + def from_streams( + cls, + streams: Iterable[SyntheticEventStreamV1], + *, + primary_member_id: str, + configuration_ids: Iterable[str], + source_version_ids: Iterable[str] = (), + ) -> "SyntheticEnsembleManifestV1": + """Build a compact deterministic manifest from member streams.""" + materialized = tuple(streams) + if not materialized: + raise ValueError("ensemble manifest requires member streams") + run_ids = {stream.run_id for stream in materialized} + if len(run_ids) != 1: + raise ValueError("ensemble streams must share one run_id") + sources = set(source_version_ids) + configs = _normalized_id_tuple(configuration_ids) + event_configs: set[str] = set() + for stream in materialized: + sources.update(stream.source_version_ids) + event_configs.update( + event.generator_config_id + for event in stream.events + if event.generator_config_id is not None + ) + if not event_configs.issubset(set(configs)): + raise ValueError( + "ensemble configurations do not cover generated events" + ) + return cls( + run_id=materialized[0].run_id, + primary_member_id=primary_member_id, + members=tuple( + SyntheticEnsembleMemberV1.from_stream(stream) + for stream in materialized + ), + source_version_ids=tuple(sources), + configuration_ids=configs, + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return canonical ensemble identity fields.""" + return { + "schema_version": self.schema_version, + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "stream_schema_version": SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION, + "run_id": self.run_id, + "primary_member_id": self.primary_member_id, + "members": [member.to_dict() for member in self.members], + "source_version_ids": list(self.source_version_ids), + "configuration_ids": list(self.configuration_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return the stable JSON-compatible ensemble manifest.""" + payload = self.identity_payload() + payload["ensemble_id"] = self.ensemble_id + return payload + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return canonical_contract_json(self.to_dict()) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "SyntheticEnsembleManifestV1": + """Restore and verify a version-one ensemble manifest.""" + _require_schema(data, SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION) + _require_derived_schema( + data, + "event_schema_version", + SYNTHETIC_EVENT_SCHEMA_VERSION, + ) + _require_derived_schema( + data, + "stream_schema_version", + SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION, + ) + return cls( + run_id=str(data.get("run_id", "")), + primary_member_id=str(data.get("primary_member_id", "")), + members=tuple( + SyntheticEnsembleMemberV1.from_dict(row) + for row in _mapping_sequence(data, "members") + ), + source_version_ids=tuple( + str(value) + for value in _value_sequence( + data.get("source_version_ids"), + "source_version_ids", + ) + ), + configuration_ids=tuple( + str(value) + for value in _value_sequence( + data.get("configuration_ids"), + "configuration_ids", + ) + ), + ensemble_id=str(data.get("ensemble_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "SyntheticEnsembleManifestV1": + """Restore a manifest from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +def derive_anchor_interval_id( + left_anchor_event_id: str, + right_anchor_event_id: str, +) -> str: + """Return a stable identity for one ordered anchor interval.""" + left = _required_text(left_anchor_event_id) + right = _required_text(right_anchor_event_id) + if left == right: + raise ValueError("anchor interval requires distinct event IDs") + return _stable_id( + "anchor-interval", + { + "left_anchor_event_id": left, + "right_anchor_event_id": right, + }, + ) + + +def canonical_contract_json(value: Any) -> str: + """Return the canonical JSON encoding used for IDs and wire payloads.""" + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + + +def synthetic_event_arrow_schema() -> Any: + """Return the exact flat Arrow schema for version-one event rows.""" + pa, _ = _arrow_modules() + + def required_text(name: str) -> Any: + return pa.field(name, pa.string(), nullable=False) + + def optional_text(name: str) -> Any: + return pa.field(name, pa.string(), nullable=True) + + return pa.schema( + [ + required_text("schema_version"), + required_text("event_id"), + required_text("origin"), + required_text("symbol"), + pa.field("event_time_ns", pa.int64(), nullable=False), + pa.field("event_sequence", pa.int64(), nullable=False), + pa.field("bid", pa.float64(), nullable=False), + pa.field("ask", pa.float64(), nullable=False), + required_text("run_id"), + required_text("ensemble_member_id"), + required_text("source_version_id"), + optional_text("source_series_id"), + optional_text("source_period"), + pa.field("source_row_id", pa.int64(), nullable=True), + optional_text("anchor_interval_id"), + optional_text("left_anchor_event_id"), + optional_text("right_anchor_event_id"), + optional_text("generator_id"), + optional_text("generator_version"), + optional_text("generator_config_id"), + optional_text("reference_id"), + optional_text("motif_id"), + optional_text("feed_epoch_id"), + optional_text("broker_profile_id"), + optional_text("constraint_set_id"), + pa.field("confidence", pa.float64(), nullable=True), + ], + metadata={ + EVENT_SCHEMA_METADATA_KEY: SYNTHETIC_EVENT_SCHEMA_VERSION.encode(), + EVENT_TIME_UNIT_METADATA_KEY: b"UTC epoch nanoseconds", + }, + ) + + +def synthetic_event_stream_to_arrow(stream: SyntheticEventStreamV1) -> Any: + """Serialize a stream to one exact-schema Arrow table.""" + pa, _ = _arrow_modules() + schema = synthetic_event_arrow_schema() + metadata = dict(schema.metadata or {}) + metadata[STREAM_METADATA_KEY] = stream.to_json_header().encode("utf-8") + schema = schema.with_metadata(metadata) + return pa.Table.from_pylist( + [event.to_dict() for event in stream.events], + schema=schema, + ) + + +def synthetic_event_stream_from_arrow(table: Any) -> SyntheticEventStreamV1: + """Restore and validate a stream from an exact-schema Arrow table.""" + expected = synthetic_event_arrow_schema() + actual = table.schema + if not actual.remove_metadata().equals(expected.remove_metadata()): + raise ValueError("Arrow event schema does not match version one") + metadata = dict(actual.metadata or {}) + if metadata.get(EVENT_SCHEMA_METADATA_KEY) != ( + SYNTHETIC_EVENT_SCHEMA_VERSION.encode() + ): + raise ValueError("Arrow event schema metadata is missing or invalid") + stream_bytes = metadata.get(STREAM_METADATA_KEY) + if stream_bytes is None: + raise ValueError("Arrow stream metadata is missing") + header = _json_mapping(stream_bytes.decode("utf-8")) + _require_schema(header, SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION) + _require_derived_schema( + header, + "event_schema_version", + SYNTHETIC_EVENT_SCHEMA_VERSION, + ) + events = tuple(SyntheticEventV1.from_dict(row) for row in table.to_pylist()) + stream = SyntheticEventStreamV1( + run_id=str(header.get("run_id", "")), + ensemble_member_id=str(header.get("ensemble_member_id", "")), + symbol=str(header.get("symbol", "")), + events=events, + source_version_ids=tuple( + str(value) + for value in _value_sequence( + header.get("source_version_ids"), + "source_version_ids", + ) + ), + stream_id=str(header.get("stream_id", "")), + schema_version=str(header.get("schema_version", "")), + ) + _validate_stream_counts(header, stream, context="Arrow stream") + return stream + + +def synthetic_event_stream_to_parquet_bytes( + stream: SyntheticEventStreamV1, +) -> bytes: + """Return deterministic Parquet bytes for a stream under one runtime.""" + pa, pq = _arrow_modules() + sink = pa.BufferOutputStream() + _write_parquet_table( + pq, + synthetic_event_stream_to_arrow(stream), + sink, + ) + return bytes(sink.getvalue()) + + +def synthetic_event_stream_from_parquet_bytes( + payload: bytes | bytearray | memoryview, +) -> SyntheticEventStreamV1: + """Restore a stream from Parquet bytes.""" + pa, pq = _arrow_modules() + table = pq.read_table(pa.BufferReader(bytes(payload))) + return synthetic_event_stream_from_arrow(table) + + +def write_synthetic_event_stream_parquet( + stream: SyntheticEventStreamV1, + path: str | Path, +) -> Path: + """Write one non-atomic Parquet file; #446 owns atomic publication.""" + _, pq = _arrow_modules() + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + _write_parquet_table( + pq, + synthetic_event_stream_to_arrow(stream), + target, + ) + return target + + +def read_synthetic_event_stream_parquet( + path: str | Path, +) -> SyntheticEventStreamV1: + """Read and validate one version-one stream Parquet file.""" + _, pq = _arrow_modules() + return synthetic_event_stream_from_arrow(pq.read_table(Path(path))) + + +def _write_parquet_table(pq: Any, table: Any, destination: Any) -> None: + pq.write_table( + table, + destination, + compression="zstd", + use_dictionary=False, + write_statistics=True, + version="2.6", + data_page_version="2.0", + row_group_size=max(1, table.num_rows), + ) + + +def _arrow_modules() -> tuple[Any, Any]: + try: + import pyarrow as pa + import pyarrow.parquet as pq + except (ImportError, ModuleNotFoundError) as err: + raise ModuleNotFoundError( + "synthetic Arrow/Parquet contracts require histdatacom[arrow]" + ) from err + return pa, pq + + +def _event_order_key(event: SyntheticEventV1) -> tuple[int, int, str]: + return (event.event_time_ns, event.event_sequence, event.event_id) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _required_text(value: Any) -> str: + normalized = str(value).strip() if value is not None else "" + if not normalized: + raise ValueError("required text value is empty") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _normalized_symbol(value: Any) -> str: + return _required_text(value).lower() + + +def _bounded_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + if not INT64_MIN <= value <= INT64_MAX: + raise ValueError(f"{name} is outside signed 64-bit range") + return value + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{name} must be finite") + return number + + +def _mapping_optional_text(data: Mapping[str, Any], name: str) -> str | None: + return _optional_text(data.get(name)) + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError(f"unsupported schema version; expected {expected}") + + +def _require_derived_schema( + data: Mapping[str, Any], + name: str, + expected: str, +) -> None: + if str(data.get(name, "")) != expected: + raise ValueError(f"{name} does not match {expected}") + + +def _validate_stream_counts( + data: Mapping[str, Any], + stream: SyntheticEventStreamV1, + *, + context: str = "stream", +) -> None: + expected = ( + ("event_count", len(stream.events)), + ("observed_event_count", stream.observed_event_count), + ("synthetic_event_count", stream.synthetic_event_count), + ) + for name, actual in expected: + declared = _bounded_int(data.get(name), name) + if actual != declared: + label = name.replace("_", " ") + raise ValueError(f"{context} {label} does not match metadata") + + +def _json_mapping(text: str) -> Mapping[str, Any]: + loaded = json.loads(text) + if not isinstance(loaded, Mapping): + raise ValueError("contract JSON must contain an object") + return cast(Mapping[str, Any], loaded) + + +def _mapping_sequence( + data: Mapping[str, Any], name: str +) -> tuple[Mapping[str, Any], ...]: + values = _value_sequence(data.get(name), name) + if not all(isinstance(value, Mapping) for value in values): + raise ValueError(f"{name} must contain objects") + return tuple(cast(Mapping[str, Any], value) for value in values) + + +def _value_sequence(value: Any, name: str) -> tuple[Any, ...]: + if isinstance(value, (str, bytes, bytearray)) or not isinstance( + value, Sequence + ): + raise ValueError(f"{name} must be a sequence") + return tuple(value) + + +def _normalized_id_tuple(values: Iterable[str]) -> tuple[str, ...]: + return tuple(sorted({_required_text(value) for value in values})) + + +def _is_sha256_id(value: str) -> bool: + prefix = "sha256:" + if not value.startswith(prefix): + return False + digest = value.removeprefix(prefix) + return len(digest) == 64 and all( + character in "0123456789abcdef" for character in digest + ) + + +_OPTIONAL_EVENT_TEXT_FIELDS = ( + "source_series_id", + "source_period", + "anchor_interval_id", + "left_anchor_event_id", + "right_anchor_event_id", + "generator_id", + "generator_version", + "generator_config_id", + "reference_id", + "motif_id", + "feed_epoch_id", + "broker_profile_id", + "constraint_set_id", +) + +_SYNTHETIC_LINEAGE_FIELDS = ( + "anchor_interval_id", + "left_anchor_event_id", + "right_anchor_event_id", + "generator_id", + "generator_version", + "generator_config_id", + "reference_id", + "motif_id", + "feed_epoch_id", + "broker_profile_id", + "constraint_set_id", + "confidence", +) + +_REQUIRED_SYNTHETIC_LINEAGE_FIELDS = ( + "anchor_interval_id", + "left_anchor_event_id", + "right_anchor_event_id", + "generator_id", + "generator_version", + "generator_config_id", + "constraint_set_id", +) + + +__all__ = [ + "SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION", + "SYNTHETIC_EVENT_ARROW_COLUMNS", + "SYNTHETIC_EVENT_SCHEMA_VERSION", + "SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION", + "SyntheticEnsembleManifestV1", + "SyntheticEnsembleMemberV1", + "SyntheticEventOrigin", + "SyntheticEventStreamV1", + "SyntheticEventV1", + "canonical_contract_json", + "derive_anchor_interval_id", + "read_synthetic_event_stream_parquet", + "synthetic_event_arrow_schema", + "synthetic_event_stream_from_arrow", + "synthetic_event_stream_from_parquet_bytes", + "synthetic_event_stream_to_arrow", + "synthetic_event_stream_to_parquet_bytes", + "write_synthetic_event_stream_parquet", +] diff --git a/src/histdatacom/synthetic/cross_currency.py b/src/histdatacom/synthetic/cross_currency.py new file mode 100644 index 00000000..98e313d3 --- /dev/null +++ b/src/histdatacom/synthetic/cross_currency.py @@ -0,0 +1,2516 @@ +"""Deterministic event-time cross-currency reconstruction and validation. + +This module turns individually carved symbol streams into one synchronized +generation unit. It never forward-fills quotes. Relationships are evaluated +only at exact event times, duplicate timestamps are paired by deterministic +event order, and immutable observations are never projected. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from enum import Enum +import hashlib +import math +from typing import Any, cast + +from histdatacom.data_quality.contracts import QualityReport +from histdatacom.data_quality.symbols import ( + CrossInstrumentPointInput, + CrossInstrumentSeriesInput, + HistDataCrossInstrumentConsistencyRule, + HistDataCrossInstrumentTolerance, +) +from histdatacom.histdata_ascii import TICK +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.contracts import ( + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + canonical_contract_json, +) +from histdatacom.synthetic.streaming import ( + PartitionManifestV1, + ReconstructionRunV1, + ReconstructionWindowV1, + plan_reconstruction_windows, +) + +CROSS_CURRENCY_ENGINE_ID = "histdatacom.cross-currency-reconciliation" +CROSS_CURRENCY_ENGINE_VERSION = "1.0.1" +CROSS_CURRENCY_SYMBOL_COVERAGE_SCHEMA_VERSION = ( + "histdatacom.cross-currency-symbol-coverage.v1" +) +CROSS_CURRENCY_EXCLUDED_SPAN_SCHEMA_VERSION = ( + "histdatacom.cross-currency-excluded-span.v1" +) +CROSS_CURRENCY_WINDOW_PLAN_SCHEMA_VERSION = ( + "histdatacom.cross-currency-window-plan.v1" +) +CROSS_CURRENCY_RELATIONSHIP_SCHEMA_VERSION = ( + "histdatacom.cross-currency-relationship.v1" +) +CROSS_CURRENCY_CONFIG_SCHEMA_VERSION = ( + "histdatacom.cross-currency-reconciliation-config.v1" +) +CROSS_CURRENCY_CONDITION_SCHEMA_VERSION = ( + "histdatacom.cross-currency-condition.v1" +) +CROSS_CURRENCY_PROJECTION_LINEAGE_SCHEMA_VERSION = ( + "histdatacom.cross-currency-projection-lineage.v1" +) +CROSS_CURRENCY_RELATIONSHIP_SUPPORT_SCHEMA_VERSION = ( + "histdatacom.cross-currency-relationship-support.v1" +) +CROSS_CURRENCY_RESIDUAL_SLICE_SCHEMA_VERSION = ( + "histdatacom.cross-currency-residual-slice.v1" +) +CROSS_CURRENCY_VALIDATION_SCHEMA_VERSION = ( + "histdatacom.cross-currency-validation.v1" +) +CROSS_CURRENCY_GROUP_SCHEMA_VERSION = ( + "histdatacom.cross-currency-reconciled-group.v1" +) + +EURUSD_TRIANGLE_SYMBOLS = ("eurgbp", "eurusd", "gbpusd") +DEFAULT_CROSS_CURRENCY_MAX_PROJECTION_RELATIVE = 0.05 +DEFAULT_CROSS_CURRENCY_RESIDUAL_TOLERANCE = 1e-10 +DEFAULT_CROSS_CURRENCY_SPREAD_TOLERANCE_MULTIPLIER = 1.0 +DEFAULT_CROSS_CURRENCY_ROUNDING_DIGITS = 12 +MAX_CROSS_CURRENCY_CONDITIONS = 4096 +MAX_CROSS_CURRENCY_FAILURE_REASONS = 128 +MAX_CROSS_CURRENCY_RESIDUAL_SLICES = 4096 + + +class CrossCurrencyCoverageStatus(str, Enum): + """Whether one symbol has usable source coverage.""" + + AVAILABLE = "available" + MISSING = "missing" + + +class CrossCurrencyWindowPlanStatus(str, Enum): + """Whether common synchronized windows could be planned.""" + + PLANNED = "planned" + REFUSED = "refused" + + +class CrossCurrencyExcludedReason(str, Enum): + """Why a requested span is outside common reconstruction support.""" + + SYMBOL_NOT_YET_AVAILABLE = "symbol_not_yet_available" + SYMBOL_NO_LONGER_AVAILABLE = "symbol_no_longer_available" + MISSING_SYMBOL = "missing_symbol" + NO_COMMON_SUPPORT = "no_common_support" + + +class CrossCurrencyRelationshipKind(str, Enum): + """Supported deterministic FX algebra relationships.""" + + TRIANGLE = "triangle" + INVERSE = "inverse" + + +class CrossCurrencyValidationStage(str, Enum): + """Mandatory cross-series validation boundaries.""" + + GENERATION = "generation" + POST_BROKER = "post_broker" + + +class CrossCurrencyValidationStatus(str, Enum): + """Whether a synchronized output satisfies its relationship contract.""" + + PASSED = "passed" + FAILED = "failed" + + +class CrossCurrencyGroupStatus(str, Enum): + """Whether a group is eligible to proceed beyond generation.""" + + RECONCILED = "reconciled" + REFUSED = "refused" + + +@dataclass(frozen=True, slots=True) +class CrossCurrencySymbolCoverageV1: + """Half-open source coverage for one member of a synchronized group.""" + + symbol: str + start_ns: int | None = None + end_ns: int | None = None + source_periods: tuple[str, ...] = () + status: CrossCurrencyCoverageStatus = CrossCurrencyCoverageStatus.AVAILABLE + schema_version: str = CROSS_CURRENCY_SYMBOL_COVERAGE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_SYMBOL_COVERAGE_SCHEMA_VERSION, + "symbol coverage", + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, "status", CrossCurrencyCoverageStatus(self.status) + ) + periods = _normalized_text_tuple(self.source_periods) + object.__setattr__(self, "source_periods", periods) + if self.status is CrossCurrencyCoverageStatus.MISSING: + if self.start_ns is not None or self.end_ns is not None: + raise ValueError("missing symbol coverage cannot have bounds") + return + if self.start_ns is None or self.end_ns is None: + raise ValueError("available symbol coverage requires bounds") + start = _int64(self.start_ns, "start_ns") + end = _int64(self.end_ns, "end_ns") + if end <= start: + raise ValueError("coverage end_ns must be greater than start_ns") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + + @classmethod + def missing(cls, symbol: str) -> "CrossCurrencySymbolCoverageV1": + """Return explicit missing-symbol coverage.""" + return cls(symbol=symbol, status=CrossCurrencyCoverageStatus.MISSING) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "status": self.status.value, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "source_periods": list(self.source_periods), + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencySymbolCoverageV1": + return cls( + symbol=str(data.get("symbol", "")), + start_ns=cast(int | None, data.get("start_ns")), + end_ns=cast(int | None, data.get("end_ns")), + source_periods=_string_tuple(data.get("source_periods")), + status=CrossCurrencyCoverageStatus(str(data.get("status", ""))), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyExcludedSpanV1: + """One deterministic explanation for excluded requested coverage.""" + + symbol: str + start_ns: int + end_ns: int + reason: CrossCurrencyExcludedReason + schema_version: str = CROSS_CURRENCY_EXCLUDED_SPAN_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_EXCLUDED_SPAN_SCHEMA_VERSION, + "excluded span", + ) + symbol = str(self.symbol or "").strip().upper() + if symbol != "*": + symbol = _normalized_symbol(symbol) + object.__setattr__(self, "symbol", symbol) + start = _int64(self.start_ns, "start_ns") + end = _int64(self.end_ns, "end_ns") + if end <= start: + raise ValueError("excluded span end_ns must exceed start_ns") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + object.__setattr__( + self, "reason", CrossCurrencyExcludedReason(self.reason) + ) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "reason": self.reason.value, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencyExcludedSpanV1": + return cls( + symbol=str(data.get("symbol", "")), + start_ns=_strict_int(data.get("start_ns"), "start_ns"), + end_ns=_strict_int(data.get("end_ns"), "end_ns"), + reason=CrossCurrencyExcludedReason(str(data.get("reason", ""))), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyWindowPlanV1: + """Deterministic common-coverage plan with explicit excluded spans.""" + + run_id: str + ensemble_member_id: str + symbols: tuple[str, ...] + requested_start_ns: int + requested_end_ns: int + coverages: tuple[CrossCurrencySymbolCoverageV1, ...] + excluded_spans: tuple[CrossCurrencyExcludedSpanV1, ...] + windows: tuple[ReconstructionWindowV1, ...] + status: CrossCurrencyWindowPlanStatus + missing_symbols: tuple[str, ...] = () + plan_id: str = "" + schema_version: str = CROSS_CURRENCY_WINDOW_PLAN_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_WINDOW_PLAN_SCHEMA_VERSION, + "window plan", + ) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, + "ensemble_member_id", + _required_text(self.ensemble_member_id), + ) + symbols = _normalized_symbols(self.symbols) + object.__setattr__(self, "symbols", symbols) + start = _int64(self.requested_start_ns, "requested_start_ns") + end = _int64(self.requested_end_ns, "requested_end_ns") + if end <= start: + raise ValueError("requested_end_ns must exceed requested_start_ns") + object.__setattr__(self, "requested_start_ns", start) + object.__setattr__(self, "requested_end_ns", end) + coverages = tuple(sorted(self.coverages, key=lambda item: item.symbol)) + if tuple(item.symbol for item in coverages) != symbols: + raise ValueError("coverage symbols must exactly match plan symbols") + object.__setattr__(self, "coverages", coverages) + excluded = tuple( + sorted( + self.excluded_spans, + key=lambda item: ( + item.start_ns, + item.end_ns, + item.symbol, + item.reason.value, + ), + ) + ) + object.__setattr__(self, "excluded_spans", excluded) + windows = tuple(self.windows) + object.__setattr__(self, "windows", windows) + missing = _normalized_symbols(self.missing_symbols, allow_empty=True) + if not set(missing).issubset(symbols): + raise ValueError("missing symbols are outside the plan") + object.__setattr__(self, "missing_symbols", missing) + object.__setattr__( + self, "status", CrossCurrencyWindowPlanStatus(self.status) + ) + if self.status is CrossCurrencyWindowPlanStatus.PLANNED: + if missing or not windows: + raise ValueError("planned common coverage requires windows") + if any( + item.run_id != self.run_id + or item.ensemble_member_id != self.ensemble_member_id + or item.symbols != symbols + for item in windows + ): + raise ValueError("planned windows differ from common scope") + elif windows: + raise ValueError("refused common coverage cannot contain windows") + expected = _stable_id("cross-currency-window-plan", self.payload()) + supplied = _optional_text(self.plan_id) + if supplied is not None and supplied != expected: + raise ValueError("cross-currency plan_id differs") + object.__setattr__(self, "plan_id", expected) + + @property + def common_start_ns(self) -> int | None: + return self.windows[0].core_start_ns if self.windows else None + + @property + def common_end_ns(self) -> int | None: + return self.windows[-1].core_end_ns if self.windows else None + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "symbols": list(self.symbols), + "requested_start_ns": self.requested_start_ns, + "requested_end_ns": self.requested_end_ns, + "status": self.status.value, + "missing_symbols": list(self.missing_symbols), + "coverage": [item.to_dict() for item in self.coverages], + "excluded_spans": [item.to_dict() for item in self.excluded_spans], + "window_ids": [item.window_id for item in self.windows], + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.payload(), + "plan_id": self.plan_id, + "windows": [item.to_dict() for item in self.windows], + "common_start_ns": self.common_start_ns, + "common_end_ns": self.common_end_ns, + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CrossCurrencyWindowPlanV1": + return cls( + run_id=str(data.get("run_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbols=_string_tuple(data.get("symbols")), + requested_start_ns=_strict_int( + data.get("requested_start_ns"), "requested_start_ns" + ), + requested_end_ns=_strict_int( + data.get("requested_end_ns"), "requested_end_ns" + ), + coverages=tuple( + CrossCurrencySymbolCoverageV1.from_dict(item) + for item in _mapping_sequence(data, "coverage") + ), + excluded_spans=tuple( + CrossCurrencyExcludedSpanV1.from_dict(item) + for item in _mapping_sequence(data, "excluded_spans") + ), + windows=tuple( + ReconstructionWindowV1.from_dict(item) + for item in _mapping_sequence(data, "windows") + ), + status=CrossCurrencyWindowPlanStatus(str(data.get("status", ""))), + missing_symbols=_string_tuple(data.get("missing_symbols")), + plan_id=str(data.get("plan_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "CrossCurrencyWindowPlanV1": + return cls.from_dict(_json_mapping(text)) + + +def plan_cross_currency_windows( + run: ReconstructionRunV1, + *, + ensemble_member_id: str, + requested_start_ns: int, + requested_end_ns: int, + window_size_ns: int, + coverages: Iterable[CrossCurrencySymbolCoverageV1], + left_halo_ns: int = 0, + right_lookahead_ns: int = 0, +) -> CrossCurrencyWindowPlanV1: + """Plan only the exact common coverage of the complete symbol group.""" + requested_start = _int64(requested_start_ns, "requested_start_ns") + requested_end = _int64(requested_end_ns, "requested_end_ns") + if requested_end <= requested_start: + raise ValueError("requested_end_ns must exceed requested_start_ns") + supplied: dict[str, CrossCurrencySymbolCoverageV1] = {} + for coverage in coverages: + if coverage.symbol in supplied: + raise ValueError("duplicate symbol coverage") + if coverage.symbol not in run.symbols: + raise ValueError( + "coverage symbol is outside the reconstruction run" + ) + supplied[coverage.symbol] = coverage + normalized = tuple( + supplied.get(symbol, CrossCurrencySymbolCoverageV1.missing(symbol)) + for symbol in run.symbols + ) + missing = tuple( + item.symbol + for item in normalized + if item.status is CrossCurrencyCoverageStatus.MISSING + ) + excluded: list[CrossCurrencyExcludedSpanV1] = [] + for item in normalized: + if item.status is CrossCurrencyCoverageStatus.MISSING: + excluded.append( + CrossCurrencyExcludedSpanV1( + symbol=item.symbol, + start_ns=requested_start, + end_ns=requested_end, + reason=CrossCurrencyExcludedReason.MISSING_SYMBOL, + ) + ) + continue + assert item.start_ns is not None and item.end_ns is not None + if item.start_ns > requested_start: + _append_excluded_span( + excluded, + symbol=item.symbol, + start_ns=requested_start, + end_ns=min(item.start_ns, requested_end), + reason=CrossCurrencyExcludedReason.SYMBOL_NOT_YET_AVAILABLE, + ) + if item.end_ns < requested_end: + _append_excluded_span( + excluded, + symbol=item.symbol, + start_ns=max(item.end_ns, requested_start), + end_ns=requested_end, + reason=CrossCurrencyExcludedReason.SYMBOL_NO_LONGER_AVAILABLE, + ) + if missing: + return CrossCurrencyWindowPlanV1( + run_id=run.run_id, + ensemble_member_id=ensemble_member_id, + symbols=run.symbols, + requested_start_ns=requested_start, + requested_end_ns=requested_end, + coverages=normalized, + excluded_spans=tuple(excluded), + windows=(), + status=CrossCurrencyWindowPlanStatus.REFUSED, + missing_symbols=missing, + ) + available = tuple(normalized) + common_start = max( + requested_start, + *(cast(int, item.start_ns) for item in available), + ) + common_end = min( + requested_end, + *(cast(int, item.end_ns) for item in available), + ) + if common_end <= common_start: + excluded.append( + CrossCurrencyExcludedSpanV1( + symbol="*", + start_ns=requested_start, + end_ns=requested_end, + reason=CrossCurrencyExcludedReason.NO_COMMON_SUPPORT, + ) + ) + return CrossCurrencyWindowPlanV1( + run_id=run.run_id, + ensemble_member_id=ensemble_member_id, + symbols=run.symbols, + requested_start_ns=requested_start, + requested_end_ns=requested_end, + coverages=normalized, + excluded_spans=tuple(excluded), + windows=(), + status=CrossCurrencyWindowPlanStatus.REFUSED, + ) + _append_excluded_span( + excluded, + symbol="*", + start_ns=requested_start, + end_ns=common_start, + reason=CrossCurrencyExcludedReason.NO_COMMON_SUPPORT, + ) + _append_excluded_span( + excluded, + symbol="*", + start_ns=common_end, + end_ns=requested_end, + reason=CrossCurrencyExcludedReason.NO_COMMON_SUPPORT, + ) + windows = plan_reconstruction_windows( + run, + ensemble_member_id=ensemble_member_id, + start_ns=common_start, + end_ns=common_end, + window_size_ns=window_size_ns, + left_halo_ns=left_halo_ns, + right_lookahead_ns=right_lookahead_ns, + ) + return CrossCurrencyWindowPlanV1( + run_id=run.run_id, + ensemble_member_id=ensemble_member_id, + symbols=run.symbols, + requested_start_ns=requested_start, + requested_end_ns=requested_end, + coverages=normalized, + excluded_spans=tuple(excluded), + windows=windows, + status=CrossCurrencyWindowPlanStatus.PLANNED, + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyRelationshipV1: + """One triangle or inverse algebra constraint and projection priority.""" + + kind: CrossCurrencyRelationshipKind + symbols: tuple[str, ...] + projection_priority: tuple[str, ...] + relationship_id: str = "" + schema_version: str = CROSS_CURRENCY_RELATIONSHIP_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_RELATIONSHIP_SCHEMA_VERSION, + "relationship", + ) + object.__setattr__( + self, "kind", CrossCurrencyRelationshipKind(self.kind) + ) + symbols = tuple(_normalized_symbol(item) for item in self.symbols) + expected_count = ( + 3 if self.kind is CrossCurrencyRelationshipKind.TRIANGLE else 2 + ) + if len(symbols) != expected_count or len(set(symbols)) != len(symbols): + raise ValueError("relationship symbol cardinality differs") + object.__setattr__(self, "symbols", symbols) + priority = tuple( + _normalized_symbol(item) for item in self.projection_priority + ) + if len(priority) != len(symbols) or set(priority) != set(symbols): + raise ValueError( + "projection priority must permute relationship symbols" + ) + object.__setattr__(self, "projection_priority", priority) + expected = _stable_id("cross-currency-relationship", self.payload()) + supplied = _optional_text(self.relationship_id) + if supplied is not None and supplied != expected: + raise ValueError("cross-currency relationship_id differs") + object.__setattr__(self, "relationship_id", expected) + + @classmethod + def triangle( + cls, + *, + direct: str, + numerator: str, + denominator: str, + projection_priority: Sequence[str] | None = None, + ) -> "CrossCurrencyRelationshipV1": + symbols = (direct, numerator, denominator) + return cls( + kind=CrossCurrencyRelationshipKind.TRIANGLE, + symbols=symbols, + projection_priority=tuple(projection_priority or symbols), + ) + + @classmethod + def inverse( + cls, + *, + left: str, + right: str, + projection_priority: Sequence[str] | None = None, + ) -> "CrossCurrencyRelationshipV1": + symbols = (left, right) + return cls( + kind=CrossCurrencyRelationshipKind.INVERSE, + symbols=symbols, + projection_priority=tuple(projection_priority or symbols), + ) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "kind": self.kind.value, + "symbols": list(self.symbols), + "projection_priority": list(self.projection_priority), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "relationship_id": self.relationship_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencyRelationshipV1": + return cls( + kind=CrossCurrencyRelationshipKind(str(data.get("kind", ""))), + symbols=_string_tuple(data.get("symbols")), + projection_priority=_string_tuple(data.get("projection_priority")), + relationship_id=str(data.get("relationship_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyReconciliationConfigV1: + """Versioned deterministic projection and validation policy.""" + + relationships: tuple[CrossCurrencyRelationshipV1, ...] + max_projection_relative: float = ( + DEFAULT_CROSS_CURRENCY_MAX_PROJECTION_RELATIVE + ) + residual_tolerance: float = DEFAULT_CROSS_CURRENCY_RESIDUAL_TOLERANCE + spread_tolerance_multiplier: float = ( + DEFAULT_CROSS_CURRENCY_SPREAD_TOLERANCE_MULTIPLIER + ) + rounding_digits: int = DEFAULT_CROSS_CURRENCY_ROUNDING_DIGITS + config_id: str = "" + schema_version: str = CROSS_CURRENCY_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_CONFIG_SCHEMA_VERSION, + "reconciliation config", + ) + relationships = tuple( + sorted(self.relationships, key=lambda item: item.relationship_id) + ) + if not relationships: + raise ValueError("reconciliation config requires relationships") + if len({item.relationship_id for item in relationships}) != len( + relationships + ): + raise ValueError("duplicate reconciliation relationship") + object.__setattr__(self, "relationships", relationships) + for name in ( + "max_projection_relative", + "residual_tolerance", + "spread_tolerance_multiplier", + ): + value = _nonnegative_finite_float(getattr(self, name), name) + object.__setattr__(self, name, value) + if self.max_projection_relative > 1.0: + raise ValueError("max_projection_relative cannot exceed one") + if ( + isinstance(self.rounding_digits, bool) + or not isinstance(self.rounding_digits, int) + or not 6 <= self.rounding_digits <= 15 + ): + raise ValueError("rounding_digits must be between 6 and 15") + expected = _stable_id("cross-currency-config", self.payload()) + supplied = _optional_text(self.config_id) + if supplied is not None and supplied != expected: + raise ValueError("cross-currency config_id differs") + object.__setattr__(self, "config_id", expected) + + @property + def symbols(self) -> tuple[str, ...]: + return tuple( + sorted( + { + symbol + for relationship in self.relationships + for symbol in relationship.symbols + } + ) + ) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "engine_id": CROSS_CURRENCY_ENGINE_ID, + "engine_version": CROSS_CURRENCY_ENGINE_VERSION, + "relationships": [item.to_dict() for item in self.relationships], + "max_projection_relative": self.max_projection_relative, + "residual_tolerance": self.residual_tolerance, + "spread_tolerance_multiplier": self.spread_tolerance_multiplier, + "rounding_digits": self.rounding_digits, + "join_policy": "exact_event_time_no_forward_fill", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "config_id": self.config_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencyReconciliationConfigV1": + return cls( + relationships=tuple( + CrossCurrencyRelationshipV1.from_dict(item) + for item in _mapping_sequence(data, "relationships") + ), + max_projection_relative=float( + data.get("max_projection_relative", 0.0) + ), + residual_tolerance=float(data.get("residual_tolerance", 0.0)), + spread_tolerance_multiplier=float( + data.get("spread_tolerance_multiplier", 0.0) + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + config_id=str(data.get("config_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +def eurusd_triangle_reconciliation_config() -> ( + CrossCurrencyReconciliationConfigV1 +): + """Return the first certified EURUSD/GBPUSD/EURGBP relationship.""" + return CrossCurrencyReconciliationConfigV1( + relationships=( + CrossCurrencyRelationshipV1.triangle( + direct="EURGBP", + numerator="EURUSD", + denominator="GBPUSD", + projection_priority=("EURGBP", "EURUSD", "GBPUSD"), + ), + ) + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyConditionV1: + """One event-time interval used to stratify residual support.""" + + start_ns: int + end_ns: int + session_key: str = "unclassified" + event_key: str = "unclassified" + feed_epoch_key: str = "unclassified" + condition_id: str = "" + schema_version: str = CROSS_CURRENCY_CONDITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_CONDITION_SCHEMA_VERSION, + "cross-currency condition", + ) + start = _int64(self.start_ns, "start_ns") + end = _int64(self.end_ns, "end_ns") + if end <= start: + raise ValueError("condition end_ns must exceed start_ns") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + for name in ("session_key", "event_key", "feed_epoch_key"): + object.__setattr__( + self, name, _normalized_key(getattr(self, name), name) + ) + expected = _stable_id("cross-currency-condition", self.payload()) + supplied = _optional_text(self.condition_id) + if supplied is not None and supplied != expected: + raise ValueError("cross-currency condition_id differs") + object.__setattr__(self, "condition_id", expected) + + def covers(self, timestamp_ns: int) -> bool: + return self.start_ns <= timestamp_ns < self.end_ns + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "session_key": self.session_key, + "event_key": self.event_key, + "feed_epoch_key": self.feed_epoch_key, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "condition_id": self.condition_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CrossCurrencyConditionV1": + return cls( + start_ns=_strict_int(data.get("start_ns"), "start_ns"), + end_ns=_strict_int(data.get("end_ns"), "end_ns"), + session_key=str(data.get("session_key", "")), + event_key=str(data.get("event_key", "")), + feed_epoch_key=str(data.get("feed_epoch_key", "")), + condition_id=str(data.get("condition_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyProjectionLineageV1: + """Content-bound evidence for one synthetic quote projection.""" + + relationship_id: str + symbol: str + event_time_ns: int + event_sequence: int + input_event_id: str + output_event_id: str + input_content_sha256: str + output_content_sha256: str + original_bid: float + original_ask: float + output_bid: float + output_ask: float + pre_residual: float + post_residual: float + allowed_residual: float + projection_relative: float + condition_ids: tuple[str, ...] + lineage_id: str = "" + schema_version: str = CROSS_CURRENCY_PROJECTION_LINEAGE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_PROJECTION_LINEAGE_SCHEMA_VERSION, + "projection lineage", + ) + for name in ( + "relationship_id", + "input_event_id", + "output_event_id", + "input_content_sha256", + "output_content_sha256", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, "event_time_ns", _int64(self.event_time_ns, "event_time_ns") + ) + if self.event_sequence < 0: + raise ValueError("event_sequence must be non-negative") + for name in ( + "original_bid", + "original_ask", + "output_bid", + "output_ask", + "pre_residual", + "post_residual", + "allowed_residual", + "projection_relative", + ): + object.__setattr__( + self, name, _nonnegative_finite_float(getattr(self, name), name) + ) + if ( + self.original_ask < self.original_bid + or self.output_ask < self.output_bid + ): + raise ValueError("projection lineage contains a negative spread") + object.__setattr__( + self, "condition_ids", _normalized_text_tuple(self.condition_ids) + ) + expected = _stable_id("cross-currency-projection", self.payload()) + supplied = _optional_text(self.lineage_id) + if supplied is not None and supplied != expected: + raise ValueError("cross-currency projection lineage_id differs") + object.__setattr__(self, "lineage_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "relationship_id": self.relationship_id, + "symbol": self.symbol, + "event_time_ns": self.event_time_ns, + "event_sequence": self.event_sequence, + "input_event_id": self.input_event_id, + "output_event_id": self.output_event_id, + "input_content_sha256": self.input_content_sha256, + "output_content_sha256": self.output_content_sha256, + "original_bid": self.original_bid, + "original_ask": self.original_ask, + "output_bid": self.output_bid, + "output_ask": self.output_ask, + "pre_residual": self.pre_residual, + "post_residual": self.post_residual, + "allowed_residual": self.allowed_residual, + "projection_relative": self.projection_relative, + "condition_ids": list(self.condition_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "lineage_id": self.lineage_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencyProjectionLineageV1": + return cls( + relationship_id=str(data.get("relationship_id", "")), + symbol=str(data.get("symbol", "")), + event_time_ns=_strict_int( + data.get("event_time_ns"), "event_time_ns" + ), + event_sequence=_strict_int( + data.get("event_sequence"), "event_sequence" + ), + input_event_id=str(data.get("input_event_id", "")), + output_event_id=str(data.get("output_event_id", "")), + input_content_sha256=str(data.get("input_content_sha256", "")), + output_content_sha256=str(data.get("output_content_sha256", "")), + original_bid=float(data.get("original_bid", 0.0)), + original_ask=float(data.get("original_ask", 0.0)), + output_bid=float(data.get("output_bid", 0.0)), + output_ask=float(data.get("output_ask", 0.0)), + pre_residual=float(data.get("pre_residual", 0.0)), + post_residual=float(data.get("post_residual", 0.0)), + allowed_residual=float(data.get("allowed_residual", 0.0)), + projection_relative=float(data.get("projection_relative", 0.0)), + condition_ids=_string_tuple(data.get("condition_ids")), + lineage_id=str(data.get("lineage_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyRelationshipSupportV1: + """Bounded aggregate support and residuals for one relationship.""" + + relationship_id: str + support_count: int + projected_count: int + infeasible_count: int + pre_residual_max: float + pre_residual_mean: float + post_residual_max: float + post_residual_mean: float + allowed_residual_max: float + schema_version: str = CROSS_CURRENCY_RELATIONSHIP_SUPPORT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_RELATIONSHIP_SUPPORT_SCHEMA_VERSION, + "relationship support", + ) + object.__setattr__( + self, "relationship_id", _required_text(self.relationship_id) + ) + for name in ("support_count", "projected_count", "infeasible_count"): + value = _nonnegative_int(getattr(self, name), name) + object.__setattr__(self, name, value) + if self.projected_count > self.support_count: + raise ValueError("projected_count exceeds relationship support") + if self.infeasible_count > self.support_count: + raise ValueError("infeasible_count exceeds relationship support") + for name in ( + "pre_residual_max", + "pre_residual_mean", + "post_residual_max", + "post_residual_mean", + "allowed_residual_max", + ): + object.__setattr__( + self, name, _nonnegative_finite_float(getattr(self, name), name) + ) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "relationship_id": self.relationship_id, + "support_count": self.support_count, + "projected_count": self.projected_count, + "infeasible_count": self.infeasible_count, + "pre_residual_max": self.pre_residual_max, + "pre_residual_mean": self.pre_residual_mean, + "post_residual_max": self.post_residual_max, + "post_residual_mean": self.post_residual_mean, + "allowed_residual_max": self.allowed_residual_max, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencyRelationshipSupportV1": + return cls( + relationship_id=str(data.get("relationship_id", "")), + support_count=_strict_int( + data.get("support_count"), "support_count" + ), + projected_count=_strict_int( + data.get("projected_count"), "projected_count" + ), + infeasible_count=_strict_int( + data.get("infeasible_count"), "infeasible_count" + ), + pre_residual_max=float(data.get("pre_residual_max", 0.0)), + pre_residual_mean=float(data.get("pre_residual_mean", 0.0)), + post_residual_max=float(data.get("post_residual_max", 0.0)), + post_residual_mean=float(data.get("post_residual_mean", 0.0)), + allowed_residual_max=float(data.get("allowed_residual_max", 0.0)), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyResidualSliceV1: + """Relationship residual support stratified by one condition dimension.""" + + relationship_id: str + dimension: str + key: str + support_count: int + projected_count: int + infeasible_count: int + pre_residual_max: float + post_residual_max: float + schema_version: str = CROSS_CURRENCY_RESIDUAL_SLICE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_RESIDUAL_SLICE_SCHEMA_VERSION, + "residual slice", + ) + object.__setattr__( + self, "relationship_id", _required_text(self.relationship_id) + ) + if self.dimension not in {"session", "event", "feed_epoch"}: + raise ValueError("unsupported residual slice dimension") + object.__setattr__(self, "key", _normalized_key(self.key, "key")) + for name in ("support_count", "projected_count", "infeasible_count"): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + for name in ("pre_residual_max", "post_residual_max"): + object.__setattr__( + self, name, _nonnegative_finite_float(getattr(self, name), name) + ) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "relationship_id": self.relationship_id, + "dimension": self.dimension, + "key": self.key, + "support_count": self.support_count, + "projected_count": self.projected_count, + "infeasible_count": self.infeasible_count, + "pre_residual_max": self.pre_residual_max, + "post_residual_max": self.post_residual_max, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencyResidualSliceV1": + return cls( + relationship_id=str(data.get("relationship_id", "")), + dimension=str(data.get("dimension", "")), + key=str(data.get("key", "")), + support_count=_strict_int( + data.get("support_count"), "support_count" + ), + projected_count=_strict_int( + data.get("projected_count"), "projected_count" + ), + infeasible_count=_strict_int( + data.get("infeasible_count"), "infeasible_count" + ), + pre_residual_max=float(data.get("pre_residual_max", 0.0)), + post_residual_max=float(data.get("post_residual_max", 0.0)), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyValidationReportV1: + """Content-bound generation or mandatory post-broker validation.""" + + run_id: str + window_id: str + synchronization_unit_id: str + ensemble_member_id: str + symbols: tuple[str, ...] + config_id: str + stage: CrossCurrencyValidationStage + status: CrossCurrencyValidationStatus + relationship_support: tuple[CrossCurrencyRelationshipSupportV1, ...] + residual_slices: tuple[CrossCurrencyResidualSliceV1, ...] + union_timestamp_count: int + common_timestamp_count: int + asynchronous_timestamp_count: int + duplicate_timestamp_event_count: int + stale_join_risk_count: int + observed_event_count: int + anchor_preserved: bool + output_content_sha256: str + failure_reasons: tuple[str, ...] = () + validation_id: str = "" + schema_version: str = CROSS_CURRENCY_VALIDATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_VALIDATION_SCHEMA_VERSION, + "cross-currency validation", + ) + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "ensemble_member_id", + "config_id", + "output_content_sha256", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__(self, "symbols", _normalized_symbols(self.symbols)) + object.__setattr__( + self, "stage", CrossCurrencyValidationStage(self.stage) + ) + object.__setattr__( + self, "status", CrossCurrencyValidationStatus(self.status) + ) + supports = tuple( + sorted( + self.relationship_support, key=lambda item: item.relationship_id + ) + ) + object.__setattr__(self, "relationship_support", supports) + slices = tuple( + sorted( + self.residual_slices, + key=lambda item: ( + item.relationship_id, + item.dimension, + item.key, + ), + ) + ) + if len(slices) > MAX_CROSS_CURRENCY_RESIDUAL_SLICES: + raise ValueError("cross-currency residual slices exceed limit") + object.__setattr__(self, "residual_slices", slices) + for name in ( + "union_timestamp_count", + "common_timestamp_count", + "asynchronous_timestamp_count", + "duplicate_timestamp_event_count", + "stale_join_risk_count", + "observed_event_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + reasons = _normalized_text_tuple(self.failure_reasons) + if len(reasons) > MAX_CROSS_CURRENCY_FAILURE_REASONS: + raise ValueError("cross-currency failure reasons exceed limit") + object.__setattr__(self, "failure_reasons", reasons) + if self.status is CrossCurrencyValidationStatus.PASSED and ( + reasons or not self.anchor_preserved + ): + raise ValueError("passing validation cannot retain failures") + if self.status is CrossCurrencyValidationStatus.FAILED and not reasons: + raise ValueError("failed validation requires a reason") + expected = _stable_id("cross-currency-validation", self.payload()) + supplied = _optional_text(self.validation_id) + if supplied is not None and supplied != expected: + raise ValueError("cross-currency validation_id differs") + object.__setattr__(self, "validation_id", expected) + + @property + def passed(self) -> bool: + return self.status is CrossCurrencyValidationStatus.PASSED + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "ensemble_member_id": self.ensemble_member_id, + "symbols": list(self.symbols), + "config_id": self.config_id, + "stage": self.stage.value, + "status": self.status.value, + "relationship_support": [ + item.to_dict() for item in self.relationship_support + ], + "residual_slices": [ + item.to_dict() for item in self.residual_slices + ], + "union_timestamp_count": self.union_timestamp_count, + "common_timestamp_count": self.common_timestamp_count, + "asynchronous_timestamp_count": self.asynchronous_timestamp_count, + "duplicate_timestamp_event_count": ( + self.duplicate_timestamp_event_count + ), + "stale_join_risk_count": self.stale_join_risk_count, + "observed_event_count": self.observed_event_count, + "anchor_preserved": self.anchor_preserved, + "output_content_sha256": self.output_content_sha256, + "failure_reasons": list(self.failure_reasons), + "join_policy": "exact_event_time_no_forward_fill", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "validation_id": self.validation_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencyValidationReportV1": + return cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbols=_string_tuple(data.get("symbols")), + config_id=str(data.get("config_id", "")), + stage=CrossCurrencyValidationStage(str(data.get("stage", ""))), + status=CrossCurrencyValidationStatus(str(data.get("status", ""))), + relationship_support=tuple( + CrossCurrencyRelationshipSupportV1.from_dict(item) + for item in _mapping_sequence(data, "relationship_support") + ), + residual_slices=tuple( + CrossCurrencyResidualSliceV1.from_dict(item) + for item in _mapping_sequence(data, "residual_slices") + ), + union_timestamp_count=_strict_int( + data.get("union_timestamp_count"), "union_timestamp_count" + ), + common_timestamp_count=_strict_int( + data.get("common_timestamp_count"), "common_timestamp_count" + ), + asynchronous_timestamp_count=_strict_int( + data.get("asynchronous_timestamp_count"), + "asynchronous_timestamp_count", + ), + duplicate_timestamp_event_count=_strict_int( + data.get("duplicate_timestamp_event_count"), + "duplicate_timestamp_event_count", + ), + stale_join_risk_count=_strict_int( + data.get("stale_join_risk_count"), "stale_join_risk_count" + ), + observed_event_count=_strict_int( + data.get("observed_event_count"), "observed_event_count" + ), + anchor_preserved=bool(data.get("anchor_preserved")), + output_content_sha256=str(data.get("output_content_sha256", "")), + failure_reasons=_string_tuple(data.get("failure_reasons")), + validation_id=str(data.get("validation_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class CrossCurrencyReconciledGroupV1: + """Process-local all-symbol output plus bounded reconciliation evidence.""" + + run_id: str + window_id: str + synchronization_unit_id: str + ensemble_member_id: str + symbols: tuple[str, ...] + status: CrossCurrencyGroupStatus + streams: tuple[SyntheticEventStreamV1, ...] + missing_symbols: tuple[str, ...] + input_stream_ids: dict[str, str] + config: CrossCurrencyReconciliationConfigV1 + condition_ids: tuple[str, ...] + projection_lineage: tuple[CrossCurrencyProjectionLineageV1, ...] + generation_validation: CrossCurrencyValidationReportV1 + group_id: str = "" + schema_version: str = CROSS_CURRENCY_GROUP_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + CROSS_CURRENCY_GROUP_SCHEMA_VERSION, + "reconciled group", + ) + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "ensemble_member_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + symbols = _normalized_symbols(self.symbols) + object.__setattr__(self, "symbols", symbols) + object.__setattr__( + self, "status", CrossCurrencyGroupStatus(self.status) + ) + streams = tuple(sorted(self.streams, key=lambda item: item.symbol)) + if len({item.symbol for item in streams}) != len(streams): + raise ValueError("reconciled group contains duplicate streams") + if any( + item.symbol not in symbols + or item.run_id != self.run_id + or item.ensemble_member_id != self.ensemble_member_id + for item in streams + ): + raise ValueError("reconciled stream differs from group scope") + object.__setattr__(self, "streams", streams) + missing = _normalized_symbols(self.missing_symbols, allow_empty=True) + if set(missing) != set(symbols).difference( + item.symbol for item in streams + ): + raise ValueError("missing symbols do not reconcile with streams") + object.__setattr__(self, "missing_symbols", missing) + input_ids = { + _normalized_symbol(symbol): _required_text(stream_id) + for symbol, stream_id in self.input_stream_ids.items() + } + if set(input_ids) != {item.symbol for item in streams}: + raise ValueError("input stream IDs do not cover available streams") + object.__setattr__( + self, "input_stream_ids", dict(sorted(input_ids.items())) + ) + if not isinstance(self.config, CrossCurrencyReconciliationConfigV1): + raise ValueError("reconciled group requires a v1 config") + object.__setattr__( + self, "condition_ids", _normalized_text_tuple(self.condition_ids) + ) + lineage = tuple( + sorted( + self.projection_lineage, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.symbol, + item.lineage_id, + ), + ) + ) + object.__setattr__(self, "projection_lineage", lineage) + validation = self.generation_validation + if ( + validation.stage is not CrossCurrencyValidationStage.GENERATION + or validation.run_id != self.run_id + or validation.window_id != self.window_id + or validation.synchronization_unit_id + != self.synchronization_unit_id + or validation.ensemble_member_id != self.ensemble_member_id + or validation.symbols != symbols + or validation.config_id != self.config.config_id + ): + raise ValueError("generation validation differs from group scope") + if self.status is CrossCurrencyGroupStatus.RECONCILED and ( + missing or not validation.passed or len(streams) != len(symbols) + ): + raise ValueError( + "reconciled status requires a complete passing group" + ) + if ( + self.status is CrossCurrencyGroupStatus.REFUSED + and validation.passed + ): + raise ValueError( + "refused group cannot have passing generation validation" + ) + expected = _stable_id("cross-currency-group", self.payload()) + supplied = _optional_text(self.group_id) + if supplied is not None and supplied != expected: + raise ValueError("cross-currency group_id differs") + object.__setattr__(self, "group_id", expected) + + @property + def generation_ready(self) -> bool: + return self.status is CrossCurrencyGroupStatus.RECONCILED + + @property + def requires_post_broker_validation(self) -> bool: + return True + + def stream_for(self, symbol: str) -> SyntheticEventStreamV1: + wanted = _normalized_symbol(symbol) + for stream in self.streams: + if stream.symbol == wanted: + return stream + raise KeyError(wanted) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "ensemble_member_id": self.ensemble_member_id, + "symbols": list(self.symbols), + "status": self.status.value, + "missing_symbols": list(self.missing_symbols), + "input_stream_ids": dict(self.input_stream_ids), + "output_stream_ids": { + item.symbol: item.stream_id for item in self.streams + }, + "output_content_sha256": _streams_content_sha256(self.streams), + "config_id": self.config.config_id, + "condition_ids": list(self.condition_ids), + "projection_count": len(self.projection_lineage), + "projection_content_sha256": _content_sha256( + [item.to_dict() for item in self.projection_lineage] + ), + "generation_validation_id": self.generation_validation.validation_id, + "post_broker_validation_required": True, + "atomic_commit_unit": "complete_synchronization_unit", + } + + def metadata(self) -> dict[str, JSONValue]: + return { + **self.payload(), + "group_id": self.group_id, + "event_rows_inline": False, + "projection_rows_inline": False, + "generation_validation": self.generation_validation.to_dict(), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.payload(), + "group_id": self.group_id, + "streams": [item.to_dict() for item in self.streams], + "config": self.config.to_dict(), + "projection_lineage": [ + item.to_dict() for item in self.projection_lineage + ], + "generation_validation": self.generation_validation.to_dict(), + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "CrossCurrencyReconciledGroupV1": + return cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbols=_string_tuple(data.get("symbols")), + status=CrossCurrencyGroupStatus(str(data.get("status", ""))), + streams=tuple( + SyntheticEventStreamV1.from_dict(item) + for item in _mapping_sequence(data, "streams") + ), + missing_symbols=_string_tuple(data.get("missing_symbols")), + input_stream_ids={ + str(key): str(value) + for key, value in _mapping(data.get("input_stream_ids")).items() + }, + config=CrossCurrencyReconciliationConfigV1.from_dict( + _mapping(data.get("config")) + ), + condition_ids=_string_tuple(data.get("condition_ids")), + projection_lineage=tuple( + CrossCurrencyProjectionLineageV1.from_dict(item) + for item in _mapping_sequence(data, "projection_lineage") + ), + generation_validation=CrossCurrencyValidationReportV1.from_dict( + _mapping(data.get("generation_validation")) + ), + group_id=str(data.get("group_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "CrossCurrencyReconciledGroupV1": + return cls.from_dict(_json_mapping(text)) + + def validate_atomic_manifest( + self, + manifest: PartitionManifestV1, + *, + post_broker_validation: CrossCurrencyValidationReportV1, + ) -> None: + """Require a complete final validation before any group commit.""" + validate_cross_currency_atomic_manifest( + window_scope=( + self.run_id, + self.window_id, + self.synchronization_unit_id, + self.ensemble_member_id, + self.symbols, + ), + streams=self.streams, + manifest=manifest, + post_broker_validation=post_broker_validation, + ) + + +@dataclass(slots=True) +class _ResidualAccumulator: + pre: list[float] = field(default_factory=list) + post: list[float] = field(default_factory=list) + allowed: list[float] = field(default_factory=list) + projected_count: int = 0 + infeasible_count: int = 0 + + +def reconcile_cross_currency_window( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + streams: Mapping[str, SyntheticEventStreamV1 | None], + config: CrossCurrencyReconciliationConfigV1, + conditions: Iterable[CrossCurrencyConditionV1] = (), +) -> CrossCurrencyReconciledGroupV1: + """Reconcile one complete event-time group with no forward-filled joins.""" + _validate_run_window_config(run, window, config) + condition_tuple = _normalized_conditions(conditions) + available, missing = _normalized_stream_inputs(run, window, streams) + input_stream_ids = { + symbol: stream.stream_id for symbol, stream in available.items() + } + original_observed = _observed_event_content(available.values()) + if missing: + validation = _failed_validation( + run=run, + window=window, + config=config, + streams=tuple(available.values()), + reasons=tuple(f"missing_symbol:{symbol}" for symbol in missing), + anchor_preserved=True, + ) + return CrossCurrencyReconciledGroupV1( + run_id=run.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + symbols=run.symbols, + status=CrossCurrencyGroupStatus.REFUSED, + streams=tuple(available.values()), + missing_symbols=missing, + input_stream_ids=input_stream_ids, + config=config, + condition_ids=tuple(item.condition_id for item in condition_tuple), + projection_lineage=(), + generation_validation=validation, + ) + + events = { + symbol: {item.event_id: item for item in stream.events} + for symbol, stream in available.items() + } + accumulators: dict[str, _ResidualAccumulator] = {} + slices: dict[tuple[str, str, str], _ResidualAccumulator] = defaultdict( + _ResidualAccumulator + ) + lineage: list[CrossCurrencyProjectionLineageV1] = [] + failures: list[str] = [] + + for relationship in config.relationships: + accumulator = _ResidualAccumulator() + accumulators[relationship.relationship_id] = accumulator + matches = _relationship_matches( + relationship, + available, + events, + window, + ) + if not matches: + failures.append( + f"relationship_no_exact_event_time_support:" + f"{relationship.relationship_id}" + ) + continue + for matched in matches: + pre = _relationship_residual(relationship, matched) + allowed = _allowed_residual(relationship, matched, config) + projected = False + infeasible = False + post = pre + if pre > config.residual_tolerance: + target_symbols = _projection_symbols(relationship, matched) + if not target_symbols: + if pre > allowed: + infeasible = True + else: + for target_symbol in target_symbols: + target_index = relationship.symbols.index(target_symbol) + original = matched[target_index] + target_bid, target_ask = _required_projection_quote( + relationship, + matched, + target_symbol, + ) + projection_relative = max( + abs(target_bid - original.bid) / original.bid, + abs(target_ask - original.ask) / original.ask, + ) + if projection_relative > config.max_projection_relative: + continue + replacement = _project_quote( + original, + target_bid=target_bid, + target_ask=target_ask, + rounding_digits=config.rounding_digits, + ) + if replacement is None: + continue + projected_events = list(matched) + projected_events[target_index] = replacement + projected_tuple = tuple(projected_events) + candidate_post = _relationship_residual( + relationship, projected_tuple + ) + post_allowed = _allowed_residual( + relationship, projected_tuple, config + ) + if candidate_post > post_allowed: + continue + events[target_symbol][original.event_id] = replacement + matched = projected_tuple + post = candidate_post + allowed = post_allowed + projected = True + condition_ids = tuple( + item.condition_id + for item in condition_tuple + if item.covers(original.event_time_ns) + ) + lineage.append( + CrossCurrencyProjectionLineageV1( + relationship_id=relationship.relationship_id, + symbol=target_symbol, + event_time_ns=original.event_time_ns, + event_sequence=original.event_sequence, + input_event_id=original.event_id, + output_event_id=replacement.event_id, + input_content_sha256=_event_content_sha256( + original + ), + output_content_sha256=_event_content_sha256( + replacement + ), + original_bid=original.bid, + original_ask=original.ask, + output_bid=replacement.bid, + output_ask=replacement.ask, + pre_residual=pre, + post_residual=post, + allowed_residual=allowed, + projection_relative=projection_relative, + condition_ids=condition_ids, + ) + ) + break + if not projected: + infeasible = True + if infeasible: + failures.append( + f"infeasible_relationship_point:" + f"{relationship.relationship_id}:" + f"{matched[0].event_time_ns}" + ) + accumulator.infeasible_count += 1 + if projected: + accumulator.projected_count += 1 + accumulator.pre.append(pre) + accumulator.post.append(post) + accumulator.allowed.append(allowed) + for dimension, key in _condition_keys( + matched[0].event_time_ns, + matched, + condition_tuple, + ).items(): + slice_acc = slices[ + (relationship.relationship_id, dimension, key) + ] + slice_acc.pre.append(pre) + slice_acc.post.append(post) + slice_acc.allowed.append(allowed) + slice_acc.projected_count += int(projected) + slice_acc.infeasible_count += int(infeasible) + + output_streams = tuple( + SyntheticEventStreamV1( + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + symbol=symbol, + events=tuple(events[symbol].values()), + source_version_ids=stream.source_version_ids, + ) + for symbol, stream in sorted(available.items()) + ) + anchor_preserved = _anchors_preserved(original_observed, output_streams) + if not anchor_preserved: + failures.append("observed_anchor_content_changed") + validation = _validation_report( + run=run, + window=window, + config=config, + streams=output_streams, + stage=CrossCurrencyValidationStage.GENERATION, + accumulators=accumulators, + slices=slices, + anchor_preserved=anchor_preserved, + failures=failures, + ) + status = ( + CrossCurrencyGroupStatus.RECONCILED + if validation.passed + else CrossCurrencyGroupStatus.REFUSED + ) + return CrossCurrencyReconciledGroupV1( + run_id=run.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + symbols=run.symbols, + status=status, + streams=output_streams, + missing_symbols=(), + input_stream_ids=input_stream_ids, + config=config, + condition_ids=tuple(item.condition_id for item in condition_tuple), + projection_lineage=tuple(lineage), + generation_validation=validation, + ) + + +def validate_cross_currency_output( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + streams: Mapping[str, SyntheticEventStreamV1 | None], + config: CrossCurrencyReconciliationConfigV1, + stage: CrossCurrencyValidationStage, + observed_anchors: Iterable[SyntheticEventV1], + conditions: Iterable[CrossCurrencyConditionV1] = (), +) -> CrossCurrencyValidationReportV1: + """Validate generation output or the mandatory post-broker output.""" + _validate_run_window_config(run, window, config) + condition_tuple = _normalized_conditions(conditions) + available, missing = _normalized_stream_inputs(run, window, streams) + failures = [f"missing_symbol:{symbol}" for symbol in missing] + anchor_map = { + item.event_id: _event_content_sha256(item) + for item in observed_anchors + if item.origin is SyntheticEventOrigin.OBSERVED + } + anchor_preserved = _anchors_preserved(anchor_map, tuple(available.values())) + if not anchor_preserved: + failures.append("observed_anchor_content_changed") + accumulators: dict[str, _ResidualAccumulator] = {} + slices: dict[tuple[str, str, str], _ResidualAccumulator] = defaultdict( + _ResidualAccumulator + ) + event_maps = { + symbol: {event.event_id: event for event in stream.events} + for symbol, stream in available.items() + } + if not missing: + for relationship in config.relationships: + accumulator = _ResidualAccumulator() + accumulators[relationship.relationship_id] = accumulator + matches = _relationship_matches( + relationship, + available, + event_maps, + window, + ) + if not matches: + failures.append( + f"relationship_no_exact_event_time_support:" + f"{relationship.relationship_id}" + ) + continue + for matched in matches: + residual = _relationship_residual(relationship, matched) + allowed = _allowed_residual(relationship, matched, config) + infeasible = residual > allowed + if infeasible: + failures.append( + f"relationship_residual_exceeded:" + f"{relationship.relationship_id}:" + f"{matched[0].event_time_ns}" + ) + accumulator.infeasible_count += 1 + accumulator.pre.append(residual) + accumulator.post.append(residual) + accumulator.allowed.append(allowed) + for dimension, key in _condition_keys( + matched[0].event_time_ns, + matched, + condition_tuple, + ).items(): + slice_acc = slices[ + (relationship.relationship_id, dimension, key) + ] + slice_acc.pre.append(residual) + slice_acc.post.append(residual) + slice_acc.allowed.append(allowed) + slice_acc.infeasible_count += int(infeasible) + return _validation_report( + run=run, + window=window, + config=config, + streams=tuple(available.values()), + stage=CrossCurrencyValidationStage(stage), + accumulators=accumulators, + slices=slices, + anchor_preserved=anchor_preserved, + failures=failures, + ) + + +def validate_cross_currency_atomic_manifest( + *, + window_scope: tuple[str, str, str, str, tuple[str, ...]], + streams: Sequence[SyntheticEventStreamV1], + manifest: PartitionManifestV1, + post_broker_validation: CrossCurrencyValidationReportV1, +) -> None: + """Reject partial or unvalidated all-symbol manifest publication.""" + run_id, window_id, sync_id, member_id, symbols = window_scope + if ( + post_broker_validation.stage + is not CrossCurrencyValidationStage.POST_BROKER + or not post_broker_validation.passed + ): + raise ValueError( + "atomic commit requires passing post-broker validation" + ) + if ( + post_broker_validation.run_id != run_id + or post_broker_validation.window_id != window_id + or post_broker_validation.synchronization_unit_id != sync_id + or post_broker_validation.ensemble_member_id != member_id + or post_broker_validation.symbols != symbols + ): + raise ValueError("post-broker validation scope differs from commit") + stream_map = {item.symbol: item for item in streams} + if set(stream_map) != set(symbols): + raise ValueError("atomic commit requires every synchronized symbol") + if ( + manifest.run_id != run_id + or manifest.window_id != window_id + or manifest.synchronization_unit_id != sync_id + or manifest.ensemble_member_id != member_id + or manifest.symbols != symbols + ): + raise ValueError("partition manifest scope differs from group") + expected_counts = { + symbol: len(stream_map[symbol].events) for symbol in symbols + } + if manifest.symbol_event_counts != expected_counts: + raise ValueError("partition manifest counts differ from final streams") + if post_broker_validation.output_content_sha256 != _streams_content_sha256( + tuple(stream_map[symbol] for symbol in symbols) + ): + raise ValueError("post-broker validation content differs from commit") + + +def cross_currency_series_inputs( + group: CrossCurrencyReconciledGroupV1, + *, + period: str, +) -> tuple[CrossInstrumentSeriesInput, ...]: + """Adapt a group to #331's millisecond diagnostic surface. + + Nanosecond event times are deterministically bucketed to HistData's + millisecond diagnostic grain. Collisions remain duplicate points and are + surfaced by the existing diagnostic; they are never forward-filled. + """ + normalized_period = _required_text(period) + inputs: list[CrossInstrumentSeriesInput] = [] + for stream in group.streams: + points = tuple( + CrossInstrumentPointInput( + timestamp_utc_ms=event.event_time_ns // 1_000_000, + price=_midpoint(event), + row_id=index, + source_row_number=(event.source_row_id or 0), + event_seq=event.event_sequence, + ) + for index, event in enumerate(stream.events, start=1) + ) + if not points: + continue + inputs.append( + CrossInstrumentSeriesInput( + symbol=stream.symbol, + timeframe=TICK, + period=normalized_period, + series_id=f"{group.group_id}:{stream.symbol}", + points=points, + path=f"reconstructed://{group.group_id}/{stream.symbol}", + ) + ) + return tuple(inputs) + + +def cross_currency_quality_report( + group: CrossCurrencyReconciledGroupV1, + *, + period: str, + tolerance: HistDataCrossInstrumentTolerance | None = None, +) -> QualityReport: + """Run the existing #331 consistency rule over reconstructed streams.""" + rule = HistDataCrossInstrumentConsistencyRule( + tolerance=tolerance or HistDataCrossInstrumentTolerance() + ) + return rule.evaluate_series( + cross_currency_series_inputs(group, period=period), + metadata={ + "cross_currency_group_id": group.group_id, + "validation_id": group.generation_validation.validation_id, + }, + ) + + +def _validation_report( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + config: CrossCurrencyReconciliationConfigV1, + streams: tuple[SyntheticEventStreamV1, ...], + stage: CrossCurrencyValidationStage, + accumulators: Mapping[str, _ResidualAccumulator], + slices: Mapping[tuple[str, str, str], _ResidualAccumulator], + anchor_preserved: bool, + failures: Iterable[str], +) -> CrossCurrencyValidationReportV1: + topology = _event_time_topology(streams, window) + reasons = _bounded_failure_reasons(failures) + supports = tuple( + _relationship_support(relationship_id, accumulator) + for relationship_id, accumulator in sorted(accumulators.items()) + ) + residual_slices = tuple( + CrossCurrencyResidualSliceV1( + relationship_id=relationship_id, + dimension=dimension, + key=key, + support_count=len(accumulator.pre), + projected_count=accumulator.projected_count, + infeasible_count=accumulator.infeasible_count, + pre_residual_max=max(accumulator.pre, default=0.0), + post_residual_max=max(accumulator.post, default=0.0), + ) + for (relationship_id, dimension, key), accumulator in sorted( + slices.items() + ) + ) + return CrossCurrencyValidationReportV1( + run_id=run.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + symbols=run.symbols, + config_id=config.config_id, + stage=stage, + status=( + CrossCurrencyValidationStatus.FAILED + if reasons or not anchor_preserved + else CrossCurrencyValidationStatus.PASSED + ), + relationship_support=supports, + residual_slices=residual_slices, + union_timestamp_count=topology["union"], + common_timestamp_count=topology["common"], + asynchronous_timestamp_count=topology["asynchronous"], + duplicate_timestamp_event_count=topology["duplicates"], + stale_join_risk_count=topology["stale_join_risks"], + observed_event_count=sum( + stream.observed_event_count for stream in streams + ), + anchor_preserved=anchor_preserved, + output_content_sha256=_streams_content_sha256(streams), + failure_reasons=reasons, + ) + + +def _bounded_failure_reasons(failures: Iterable[str]) -> tuple[str, ...]: + """Retain deterministic refusal evidence without crashing large windows.""" + reasons = tuple(sorted(set(failures))) + if len(reasons) <= MAX_CROSS_CURRENCY_FAILURE_REASONS: + return reasons + digest = hashlib.sha256( + str(canonical_contract_json(list(reasons))).encode("utf-8") + ).hexdigest() + summary = ( + "failure_reasons_truncated:" f"total={len(reasons)}:sha256={digest}" + ) + return ( + *reasons[: MAX_CROSS_CURRENCY_FAILURE_REASONS - 1], + summary, + ) + + +def _failed_validation( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + config: CrossCurrencyReconciliationConfigV1, + streams: tuple[SyntheticEventStreamV1, ...], + reasons: tuple[str, ...], + anchor_preserved: bool, +) -> CrossCurrencyValidationReportV1: + return _validation_report( + run=run, + window=window, + config=config, + streams=streams, + stage=CrossCurrencyValidationStage.GENERATION, + accumulators={}, + slices={}, + anchor_preserved=anchor_preserved, + failures=reasons, + ) + + +def _relationship_support( + relationship_id: str, + accumulator: _ResidualAccumulator, +) -> CrossCurrencyRelationshipSupportV1: + return CrossCurrencyRelationshipSupportV1( + relationship_id=relationship_id, + support_count=len(accumulator.pre), + projected_count=accumulator.projected_count, + infeasible_count=accumulator.infeasible_count, + pre_residual_max=max(accumulator.pre, default=0.0), + pre_residual_mean=_mean(accumulator.pre), + post_residual_max=max(accumulator.post, default=0.0), + post_residual_mean=_mean(accumulator.post), + allowed_residual_max=max(accumulator.allowed, default=0.0), + ) + + +def _validate_run_window_config( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + config: CrossCurrencyReconciliationConfigV1, +) -> None: + if ( + window.run_id != run.run_id + or window.ensemble_member_id not in run.ensemble_member_ids + or window.symbols != run.symbols + ): + raise ValueError("cross-currency window differs from run") + if not set(config.symbols).issubset(run.symbols): + raise ValueError("cross-currency config symbols are outside run") + + +def _normalized_stream_inputs( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + streams: Mapping[str, SyntheticEventStreamV1 | None], +) -> tuple[dict[str, SyntheticEventStreamV1], tuple[str, ...]]: + normalized: dict[str, SyntheticEventStreamV1 | None] = {} + for symbol, stream in streams.items(): + key = _normalized_symbol(symbol) + if key in normalized: + raise ValueError("duplicate cross-currency input symbol") + if key not in run.symbols: + raise ValueError("cross-currency input symbol is outside run") + normalized[key] = stream + available: dict[str, SyntheticEventStreamV1] = {} + missing: list[str] = [] + for symbol in run.symbols: + stream = normalized.get(symbol) + if stream is None: + missing.append(symbol) + continue + if ( + stream.symbol != symbol + or stream.run_id != run.run_id + or stream.ensemble_member_id != window.ensemble_member_id + ): + raise ValueError("cross-currency stream differs from window scope") + available[symbol] = stream + return dict(sorted(available.items())), tuple(missing) + + +def _normalized_conditions( + conditions: Iterable[CrossCurrencyConditionV1], +) -> tuple[CrossCurrencyConditionV1, ...]: + normalized = tuple( + sorted( + tuple(conditions), + key=lambda item: (item.start_ns, item.end_ns, item.condition_id), + ) + ) + if len(normalized) > MAX_CROSS_CURRENCY_CONDITIONS: + raise ValueError("cross-currency conditions exceed limit") + if len({item.condition_id for item in normalized}) != len(normalized): + raise ValueError("duplicate cross-currency condition") + return normalized + + +def _relationship_matches( + relationship: CrossCurrencyRelationshipV1, + streams: Mapping[str, SyntheticEventStreamV1], + current: Mapping[str, Mapping[str, SyntheticEventV1]], + window: ReconstructionWindowV1, +) -> tuple[tuple[SyntheticEventV1, ...], ...]: + indexed: dict[str, dict[int, list[SyntheticEventV1]]] = {} + for symbol in relationship.symbols: + by_time: dict[int, list[SyntheticEventV1]] = defaultdict(list) + for original in streams[symbol].events: + event = current[symbol][original.event_id] + if window.owns_event_time(event.event_time_ns): + by_time[event.event_time_ns].append(event) + for values in by_time.values(): + values.sort(key=lambda item: (item.event_sequence, item.event_id)) + indexed[symbol] = by_time + common_times = set.intersection( + *(set(indexed[symbol]) for symbol in relationship.symbols) + ) + matches: list[tuple[SyntheticEventV1, ...]] = [] + for timestamp in sorted(common_times): + count = min( + len(indexed[symbol][timestamp]) for symbol in relationship.symbols + ) + for ordinal in range(count): + matches.append( + tuple( + indexed[symbol][timestamp][ordinal] + for symbol in relationship.symbols + ) + ) + return tuple(matches) + + +def _relationship_residual( + relationship: CrossCurrencyRelationshipV1, + events: Sequence[SyntheticEventV1], +) -> float: + if relationship.kind is CrossCurrencyRelationshipKind.TRIANGLE: + direct, numerator, denominator = events + implied_bid = numerator.bid / denominator.ask + implied_ask = numerator.ask / denominator.bid + return float( + max( + abs(direct.bid - implied_bid) / implied_bid, + abs(direct.ask - implied_ask) / implied_ask, + ) + ) + left, right = events + implied_bid = 1.0 / right.ask + implied_ask = 1.0 / right.bid + return float( + max( + abs(left.bid - implied_bid) / implied_bid, + abs(left.ask - implied_ask) / implied_ask, + ) + ) + + +def _allowed_residual( + relationship: CrossCurrencyRelationshipV1, + events: Sequence[SyntheticEventV1], + config: CrossCurrencyReconciliationConfigV1, +) -> float: + relative_half_spreads = sum( + ((item.ask - item.bid) / 2.0) / _midpoint(item) for item in events + ) + return float( + config.residual_tolerance + + (config.spread_tolerance_multiplier * relative_half_spreads) + ) + + +def _projection_symbols( + relationship: CrossCurrencyRelationshipV1, + events: Sequence[SyntheticEventV1], +) -> tuple[str, ...]: + by_symbol = {item.symbol: item for item in events} + return tuple( + symbol + for symbol in relationship.projection_priority + if by_symbol[symbol].origin is SyntheticEventOrigin.SYNTHETIC + ) + + +def _required_projection_quote( + relationship: CrossCurrencyRelationshipV1, + events: Sequence[SyntheticEventV1], + target_symbol: str, +) -> tuple[float, float]: + by_symbol = {item.symbol: item for item in events} + if relationship.kind is CrossCurrencyRelationshipKind.TRIANGLE: + direct, numerator, denominator = relationship.symbols + if target_symbol == direct: + return ( + by_symbol[numerator].bid / by_symbol[denominator].ask, + by_symbol[numerator].ask / by_symbol[denominator].bid, + ) + if target_symbol == numerator: + return ( + by_symbol[direct].bid * by_symbol[denominator].ask, + by_symbol[direct].ask * by_symbol[denominator].bid, + ) + return ( + by_symbol[numerator].ask / by_symbol[direct].ask, + by_symbol[numerator].bid / by_symbol[direct].bid, + ) + left, right = relationship.symbols + other = right if target_symbol == left else left + return (1.0 / by_symbol[other].ask, 1.0 / by_symbol[other].bid) + + +def _project_quote( + event: SyntheticEventV1, + *, + target_bid: float, + target_ask: float, + rounding_digits: int, +) -> SyntheticEventV1 | None: + bid = round(target_bid, rounding_digits) + ask = round(target_ask, rounding_digits) + if ( + not math.isfinite(bid) + or not math.isfinite(ask) + or bid <= 0.0 + or ask < bid + ): + return None + return replace(event, bid=bid, ask=ask, event_id="") + + +def _condition_keys( + timestamp_ns: int, + events: Sequence[SyntheticEventV1], + conditions: Sequence[CrossCurrencyConditionV1], +) -> dict[str, str]: + matching = tuple(item for item in conditions if item.covers(timestamp_ns)) + session = _composite_key(item.session_key for item in matching) + event = _composite_key(item.event_key for item in matching) + condition_epochs = tuple(item.feed_epoch_key for item in matching) + event_epochs = tuple( + item.feed_epoch_id for item in events if item.feed_epoch_id is not None + ) + epoch = _composite_key(condition_epochs or event_epochs) + return {"session": session, "event": event, "feed_epoch": epoch} + + +def _event_time_topology( + streams: Sequence[SyntheticEventStreamV1], + window: ReconstructionWindowV1, +) -> dict[str, int]: + by_symbol: dict[str, list[int]] = { + stream.symbol: [ + item.event_time_ns + for item in stream.events + if window.owns_event_time(item.event_time_ns) + ] + for stream in streams + } + sets = {symbol: set(values) for symbol, values in by_symbol.items()} + union = set().union(*sets.values()) if sets else set() + common = set.intersection(*sets.values()) if sets else set() + duplicates = sum( + sum(count - 1 for count in Counter(values).values() if count > 1) + for values in by_symbol.values() + ) + return { + "union": len(union), + "common": len(common), + "asynchronous": len(union.difference(common)), + "duplicates": duplicates, + "stale_join_risks": _stale_join_risk_count(sets), + } + + +def _stale_join_risk_count( + timestamps: Mapping[str, set[int]], +) -> int: + union = sorted(set().union(*timestamps.values())) if timestamps else [] + risks = 0 + for symbol, present in timestamps.items(): + del symbol + previous_present = False + missing_run = 0 + for timestamp in union: + if timestamp in present: + if previous_present and missing_run >= 2: + risks += 1 + previous_present = True + missing_run = 0 + elif previous_present: + missing_run += 1 + if previous_present and missing_run >= 2: + risks += 1 + return risks + + +def _observed_event_content( + streams: Iterable[SyntheticEventStreamV1], +) -> dict[str, str]: + return { + event.event_id: _event_content_sha256(event) + for stream in streams + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + } + + +def _anchors_preserved( + expected: Mapping[str, str], + streams: Sequence[SyntheticEventStreamV1], +) -> bool: + actual = _observed_event_content(streams) + return all( + actual.get(event_id) == digest for event_id, digest in expected.items() + ) + + +def _streams_content_sha256( + streams: Sequence[SyntheticEventStreamV1], +) -> str: + return _content_sha256( + [ + { + "symbol": stream.symbol, + "events": [item.to_dict() for item in stream.events], + } + for stream in sorted(streams, key=lambda item: item.symbol) + ] + ) + + +def _event_content_sha256(event: SyntheticEventV1) -> str: + return _content_sha256(event.to_dict()) + + +def _content_sha256(value: JSONValue) -> str: + payload = str(canonical_contract_json(value)).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + str(canonical_contract_json(dict(payload))).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _midpoint(event: SyntheticEventV1) -> float: + return float((event.bid + event.ask) / 2.0) + + +def _mean(values: Sequence[float]) -> float: + return sum(values) / len(values) if values else 0.0 + + +def _composite_key(values: Iterable[str]) -> str: + normalized = sorted( + { + _normalized_key(value, "condition key") + for value in values + if str(value or "").strip() + } + ) + return "+".join(normalized) if normalized else "unclassified" + + +def _append_excluded_span( + spans: list[CrossCurrencyExcludedSpanV1], + *, + symbol: str, + start_ns: int, + end_ns: int, + reason: CrossCurrencyExcludedReason, +) -> None: + if end_ns <= start_ns: + return + spans.append( + CrossCurrencyExcludedSpanV1( + symbol=symbol, + start_ns=start_ns, + end_ns=end_ns, + reason=reason, + ) + ) + + +def _normalized_symbols( + values: Iterable[str], *, allow_empty: bool = False +) -> tuple[str, ...]: + symbols = tuple(sorted({_normalized_symbol(item) for item in values})) + if not symbols and not allow_empty: + raise ValueError("symbol group cannot be empty") + return symbols + + +def _normalized_symbol(value: str) -> str: + symbol = "".join( + character for character in str(value).lower() if character.isalnum() + ) + if not symbol: + raise ValueError("symbol cannot be empty") + return symbol + + +def _normalized_key(value: str, name: str) -> str: + normalized = str(value or "").strip().lower().replace(" ", "_") + if not normalized: + raise ValueError(f"{name} cannot be empty") + return normalized + + +def _normalized_text_tuple(values: Iterable[str]) -> tuple[str, ...]: + return tuple(sorted({_required_text(item) for item in values})) + + +def _required_text(value: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("required text cannot be empty") + return text + + +def _optional_text(value: str | None) -> str | None: + text = str(value or "").strip() + return text or None + + +def _int64(value: int, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + if not -(2**63) <= value <= (2**63 - 1): + raise ValueError(f"{name} exceeds int64 bounds") + return value + + +def _strict_int(value: object, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _nonnegative_int(value: int, name: str) -> int: + result = _strict_int(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _nonnegative_finite_float(value: float, name: str) -> float: + result = float(value) + if not math.isfinite(result) or result < 0.0: + raise ValueError(f"{name} must be finite and non-negative") + return result + + +def _require_version(actual: str, expected: str, name: str) -> None: + if actual != expected: + raise ValueError(f"unsupported {name} schema") + + +def _string_tuple(value: object) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError("expected a sequence of strings") + return tuple(str(item) for item in value) + + +def _mapping(value: object) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return cast(Mapping[str, Any], value) + + +def _mapping_sequence( + data: Mapping[str, Any], name: str +) -> tuple[Mapping[str, Any], ...]: + value = data.get(name) + if value is None: + return () + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError(f"{name} must be a sequence") + return tuple(_mapping(item) for item in value) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + import json + + loaded = json.loads(text) + return _mapping(loaded) diff --git a/src/histdatacom/synthetic/delivery.py b/src/histdatacom/synthetic/delivery.py new file mode 100644 index 00000000..99897700 --- /dev/null +++ b/src/histdatacom/synthetic/delivery.py @@ -0,0 +1,445 @@ +"""Generic final-delivery contracts for reconstructed event groups. + +The v2.1 reference product does not require a broker fingerprint. This module +therefore owns the narrow delivery boundary shared by identity delivery today +and optional delivery adapters later. Broker-specific transfer remains in +``broker_transfer`` and is not impersonated by a synthetic fingerprint. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.contracts import ( + SyntheticEventOrigin, + SyntheticEventStreamV1, + canonical_contract_json, +) +from histdatacom.synthetic.cross_currency import ( + CrossCurrencyGroupStatus, + CrossCurrencyReconciledGroupV1, +) + +RECONSTRUCTION_DELIVERY_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-delivery-manifest.v1" +) +RECONSTRUCTION_DELIVERED_GROUP_SCHEMA_VERSION = ( + "histdatacom.reconstruction-delivered-group.v1" +) +MODERN_REFERENCE_DELIVERY_ENGINE_ID = ( + "histdatacom.reconstruction.modern-reference-identity" +) +MODERN_REFERENCE_DELIVERY_ENGINE_VERSION = "1.0.0" + + +class ReconstructionDeliveryMode(str, Enum): + """Supported final delivery projections.""" + + MODERN_REFERENCE = "modern_reference" + BROKER_CONDITIONED = "broker_conditioned" + + @classmethod + def from_value( + cls, value: str | "ReconstructionDeliveryMode" + ) -> "ReconstructionDeliveryMode": + """Normalize CLI spelling without weakening the persisted contract.""" + if isinstance(value, cls): + return value + normalized = str(value).strip().lower().replace("-", "_") + try: + return cls(normalized) + except ValueError as err: + raise ValueError( + "unsupported reconstruction delivery mode" + ) from err + + +class ReconstructionDeliveryStatus(str, Enum): + """Whether a delivery projection produced a publishable group.""" + + APPLIED = "applied" + REFUSED = "refused" + + +@dataclass(frozen=True, slots=True) +class ReconstructionDeliveryManifestV1: + """Compact content and identity evidence for one delivery projection.""" + + run_id: str + window_id: str + synchronization_unit_id: str + ensemble_member_id: str + input_group_id: str + delivery_mode: ReconstructionDeliveryMode + delivery_profile_id: str + status: ReconstructionDeliveryStatus + reason_codes: tuple[str, ...] + input_content_sha256: str + output_content_sha256: str | None + observed_event_count: int + synthetic_event_count: int + identity_event_count: int + identity_lineage_sha256: str | None + manifest_id: str = "" + schema_version: str = RECONSTRUCTION_DELIVERY_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_DELIVERY_MANIFEST_SCHEMA_VERSION + ): + raise ValueError("unsupported reconstruction delivery manifest") + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "ensemble_member_id", + "input_group_id", + "delivery_profile_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "delivery_mode", + ReconstructionDeliveryMode(self.delivery_mode), + ) + if ( + self.delivery_mode + is not ReconstructionDeliveryMode.MODERN_REFERENCE + ): + raise ValueError( + "broker-conditioned delivery requires a separate adapter contract" + ) + object.__setattr__( + self, "status", ReconstructionDeliveryStatus(self.status) + ) + reasons = tuple(_bounded_text(item) for item in self.reason_codes) + if len(reasons) > 32: + raise ValueError("delivery refusal reasons exceed bounded limit") + object.__setattr__(self, "reason_codes", reasons) + object.__setattr__( + self, + "input_content_sha256", + _required_sha256(self.input_content_sha256), + ) + output_hash = _optional_sha256(self.output_content_sha256) + lineage_hash = _optional_sha256(self.identity_lineage_sha256) + object.__setattr__(self, "output_content_sha256", output_hash) + object.__setattr__(self, "identity_lineage_sha256", lineage_hash) + for name in ( + "observed_event_count", + "synthetic_event_count", + "identity_event_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name)) + ) + if self.status is ReconstructionDeliveryStatus.APPLIED: + if reasons or output_hash is None or lineage_hash is None: + raise ValueError( + "applied delivery lacks output identity evidence" + ) + if ( + self.delivery_mode + is ReconstructionDeliveryMode.MODERN_REFERENCE + ): + if ( + self.input_content_sha256 != output_hash + or self.identity_event_count != self.synthetic_event_count + ): + raise ValueError( + "modern-reference delivery is not identity" + ) + elif not reasons or output_hash is not None or lineage_hash is not None: + raise ValueError("refused delivery must retain reasons, not output") + expected = _stable_id( + "reconstruction-delivery", self.identity_payload() + ) + if self.manifest_id and self.manifest_id != expected: + raise ValueError("delivery manifest_id differs") + object.__setattr__(self, "manifest_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic delivery identity and bounded reconciliation evidence.""" + return { + "schema_version": self.schema_version, + "engine_id": MODERN_REFERENCE_DELIVERY_ENGINE_ID, + "engine_version": MODERN_REFERENCE_DELIVERY_ENGINE_VERSION, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "ensemble_member_id": self.ensemble_member_id, + "input_group_id": self.input_group_id, + "delivery_mode": self.delivery_mode.value, + "delivery_profile_id": self.delivery_profile_id, + "status": self.status.value, + "reason_codes": list(self.reason_codes), + "input_content_sha256": self.input_content_sha256, + "output_content_sha256": self.output_content_sha256, + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "identity_event_count": self.identity_event_count, + "identity_lineage_sha256": self.identity_lineage_sha256, + "event_rows_inline": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic compact JSON-compatible evidence.""" + return {**self.identity_payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionDeliveryManifestV1": + """Restore and verify a delivery manifest.""" + _require_schema(data, RECONSTRUCTION_DELIVERY_MANIFEST_SCHEMA_VERSION) + if data.get("event_rows_inline") is not False: + raise ValueError("delivery manifest cannot contain event rows") + return cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + input_group_id=str(data.get("input_group_id", "")), + delivery_mode=ReconstructionDeliveryMode( + str(data.get("delivery_mode", "")) + ), + delivery_profile_id=str(data.get("delivery_profile_id", "")), + status=ReconstructionDeliveryStatus(str(data.get("status", ""))), + reason_codes=_string_tuple(data.get("reason_codes")), + input_content_sha256=str(data.get("input_content_sha256", "")), + output_content_sha256=_optional_text( + data.get("output_content_sha256") + ), + observed_event_count=_strict_int( + data.get("observed_event_count"), "observed_event_count" + ), + synthetic_event_count=_strict_int( + data.get("synthetic_event_count"), "synthetic_event_count" + ), + identity_event_count=_strict_int( + data.get("identity_event_count"), "identity_event_count" + ), + identity_lineage_sha256=_optional_text( + data.get("identity_lineage_sha256") + ), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionDeliveryManifestV1": + """Restore a delivery manifest from deterministic JSON.""" + value = json.loads(text) + if not isinstance(value, Mapping): + raise ValueError("delivery manifest JSON must contain an object") + return cls.from_dict(value) + + +@dataclass(frozen=True, slots=True) +class ReconstructionDeliveredGroupV1: + """Process-local delivered streams plus compact projection evidence.""" + + manifest: ReconstructionDeliveryManifestV1 + streams: tuple[SyntheticEventStreamV1, ...] + schema_version: str = RECONSTRUCTION_DELIVERED_GROUP_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_DELIVERED_GROUP_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction delivered group") + if not isinstance(self.manifest, ReconstructionDeliveryManifestV1): + raise TypeError("delivered group requires a v1 manifest") + streams = tuple(sorted(self.streams, key=lambda item: item.symbol)) + object.__setattr__(self, "streams", streams) + if self.manifest.status is ReconstructionDeliveryStatus.REFUSED: + if streams: + raise ValueError( + "refused delivered group cannot expose streams" + ) + return + if not streams or len({item.symbol for item in streams}) != len( + streams + ): + raise ValueError("delivered group requires unique symbol streams") + if any( + item.run_id != self.manifest.run_id + or item.ensemble_member_id != self.manifest.ensemble_member_id + for item in streams + ): + raise ValueError("delivered streams differ from manifest scope") + observed = sum(item.observed_event_count for item in streams) + synthetic = sum(item.synthetic_event_count for item in streams) + output_hash = reconstruction_streams_content_sha256(streams) + if ( + observed != self.manifest.observed_event_count + or synthetic != self.manifest.synthetic_event_count + or output_hash != self.manifest.output_content_sha256 + ): + raise ValueError("delivered stream content does not reconcile") + + @property + def status(self) -> ReconstructionDeliveryStatus: + """Return the terminal projection status.""" + return self.manifest.status + + def metadata(self) -> dict[str, JSONValue]: + """Return bounded metadata without event rows.""" + return { + "schema_version": self.schema_version, + "manifest": self.manifest.to_dict(), + "stream_ids": { + item.symbol: item.stream_id for item in self.streams + }, + "event_rows_inline": False, + } + + +def project_modern_reference_delivery( + group: CrossCurrencyReconciledGroupV1, + *, + delivery_profile_id: str, +) -> ReconstructionDeliveredGroupV1: + """Apply the explicit no-op reference delivery without broker evidence.""" + if not isinstance(group, CrossCurrencyReconciledGroupV1): + raise TypeError("modern delivery requires a reconciled group") + if group.status is not CrossCurrencyGroupStatus.RECONCILED: + raise ValueError("modern delivery refuses an unreconciled group") + streams = group.streams + content_hash = reconstruction_streams_content_sha256(streams) + synthetic_ids = tuple( + event.event_id + for stream in streams + for event in stream.events + if event.origin is SyntheticEventOrigin.SYNTHETIC + ) + manifest = ReconstructionDeliveryManifestV1( + run_id=group.run_id, + window_id=group.window_id, + synchronization_unit_id=group.synchronization_unit_id, + ensemble_member_id=group.ensemble_member_id, + input_group_id=group.group_id, + delivery_mode=ReconstructionDeliveryMode.MODERN_REFERENCE, + delivery_profile_id=delivery_profile_id, + status=ReconstructionDeliveryStatus.APPLIED, + reason_codes=(), + input_content_sha256=content_hash, + output_content_sha256=content_hash, + observed_event_count=sum(item.observed_event_count for item in streams), + synthetic_event_count=len(synthetic_ids), + identity_event_count=len(synthetic_ids), + identity_lineage_sha256=_text_sequence_sha256(synthetic_ids), + ) + return ReconstructionDeliveredGroupV1(manifest=manifest, streams=streams) + + +def reconstruction_streams_content_sha256( + streams: Sequence[SyntheticEventStreamV1], +) -> str: + """Hash ordered exact stream rows independently of file placement.""" + payload: list[JSONValue] = [ + event.to_dict() + for stream in sorted(streams, key=lambda item: item.symbol) + for event in stream.events + ] + return hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + + +def _text_sequence_sha256(values: Sequence[str]) -> str: + digest = hashlib.sha256(b"histdatacom-delivery-identity-lineage-v1\n") + for value in values: + digest.update(value.encode("utf-8")) + digest.update(b"\n") + return digest.hexdigest() + + +def _stable_id(namespace: str, payload: Mapping[str, JSONValue]) -> str: + content = canonical_contract_json(payload).encode("utf-8") + return f"{namespace}:sha256:{hashlib.sha256(content).hexdigest()}" + + +def _required_text(value: Any) -> str: + normalized = str(value).strip() if value is not None else "" + if not normalized: + raise ValueError("required delivery text is empty") + return normalized + + +def _bounded_text(value: Any) -> str: + normalized = _required_text(value) + if len(normalized.encode("utf-8")) > 1_024: + raise ValueError("delivery text exceeds bounded limit") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _required_sha256(value: Any) -> str: + normalized = _required_text(value).lower() + if len(normalized) != 64 or any( + item not in "0123456789abcdef" for item in normalized + ): + raise ValueError("delivery hash must be a lowercase sha256 digest") + return normalized + + +def _optional_sha256(value: Any) -> str | None: + normalized = _optional_text(value) + return _required_sha256(normalized) if normalized is not None else None + + +def _nonnegative_int(value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("delivery count must be a nonnegative integer") + return value + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _string_tuple(value: Any) -> tuple[str, ...]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError("delivery value must be a sequence") + return tuple(str(item) for item in value) + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if data.get("schema_version") != expected: + raise ValueError("unsupported delivery schema version") + + +__all__ = [ + "MODERN_REFERENCE_DELIVERY_ENGINE_ID", + "MODERN_REFERENCE_DELIVERY_ENGINE_VERSION", + "RECONSTRUCTION_DELIVERED_GROUP_SCHEMA_VERSION", + "RECONSTRUCTION_DELIVERY_MANIFEST_SCHEMA_VERSION", + "ReconstructionDeliveredGroupV1", + "ReconstructionDeliveryManifestV1", + "ReconstructionDeliveryMode", + "ReconstructionDeliveryStatus", + "project_modern_reference_delivery", + "reconstruction_streams_content_sha256", +] diff --git a/src/histdatacom/synthetic/ensembles.py b/src/histdatacom/synthetic/ensembles.py new file mode 100644 index 00000000..a5779a19 --- /dev/null +++ b/src/histdatacom/synthetic/ensembles.py @@ -0,0 +1,3341 @@ +"""Calibrated reconstruction-ensemble contracts and deterministic reporting. + +The ensemble layer represents missing-tick uncertainty without treating one +generated path as historical truth. It plans stable members, calibrates +bounded metric intervals against reverse-degradation validation/final-holdout +windows, diagnoses collapsed or false diversity, selects a representative +primary member, enforces the existing reconstruction storage budget, and +hash-gates deterministic on-demand regeneration. + +Dense event rows remain process-local. Every durable object in this module is +bounded metadata derived from canonical hashes and aggregate evidence. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from enum import Enum +import hashlib +from itertools import combinations +import json +import math +from typing import Any, cast + +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.benchmark import ( + MAX_BENCHMARK_ENSEMBLE_MEMBERS, + BenchmarkCandidateWindowV1, + BenchmarkEventV1, + BenchmarkScenarioV1, + BenchmarkSplitKind, +) +from histdatacom.synthetic.contracts import ( + SyntheticEventV1, + SyntheticEventStreamV1, + canonical_contract_json, +) +from histdatacom.synthetic.streaming import ( + ReconstructionResourceEstimateV1, + ReconstructionRunV1, + ReconstructionStoragePolicyV1, +) + +ENSEMBLE_CONFIG_SCHEMA_VERSION = "histdatacom.reconstruction-ensemble-config.v1" +ENSEMBLE_ARTIFACT_DIGEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-artifact-digest.v1" +) +ENSEMBLE_MEMBER_PLAN_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-member-plan.v1" +) +ENSEMBLE_PLAN_SCHEMA_VERSION = "histdatacom.reconstruction-ensemble-plan.v1" +ENSEMBLE_STORAGE_ESTIMATE_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-storage-estimate.v1" +) +ENSEMBLE_STRATUM_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-stratum.v1" +) +ENSEMBLE_MEMBER_CALIBRATION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-member-calibration.v1" +) +ENSEMBLE_CALIBRATION_SAMPLE_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-calibration-sample.v1" +) +ENSEMBLE_METRIC_CALIBRATION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-metric-calibration.v1" +) +ENSEMBLE_DIVERSITY_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-diversity-summary.v1" +) +ENSEMBLE_OUTCOME_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-outcome-summary.v1" +) +ENSEMBLE_MEMBER_SELECTION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-member-selection.v1" +) +ENSEMBLE_CALIBRATION_REPORT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-calibration-report.v1" +) +ENSEMBLE_REGENERATION_REQUEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-regeneration-request.v1" +) + +ENSEMBLE_ENGINE_ID = "histdatacom.reconstruction-ensemble-calibration" +ENSEMBLE_ENGINE_VERSION = "1.0.0" +ENSEMBLE_CONFIDENCE_QUANTITY = "finite-sample-interval-coverage-v1" +ENSEMBLE_PRIMARY_SELECTION_BASIS = ( + "validation-medoid-distance-with-failure-penalty-v1" +) +ENSEMBLE_CALIBRATION_FIT_SPLIT = BenchmarkSplitKind.VALIDATION +ENSEMBLE_CALIBRATION_EVALUATION_SPLIT = BenchmarkSplitKind.FINAL_HOLDOUT + +ENSEMBLE_CALIBRATION_METRIC_GROUPS: Mapping[str, str] = { + "event_count": "count", + "observed_duration_ns": "duration", + "mean_interarrival_ns": "duration", + "mean_spread": "spread", + "mid_path_range": "path", + "endpoint_mid": "path", + "downstream_sensitivity": "downstream_sensitivity", +} +ENSEMBLE_CALIBRATION_METRIC_NAMES = tuple(ENSEMBLE_CALIBRATION_METRIC_GROUPS) + +DEFAULT_ENSEMBLE_HORIZONS_NS = ( + 60 * 1_000_000_000, + 5 * 60 * 1_000_000_000, + 60 * 60 * 1_000_000_000, +) +DEFAULT_ENSEMBLE_MAX_SAMPLES = 4096 +DEFAULT_ENSEMBLE_MAX_SLICES = 4096 +DEFAULT_ENSEMBLE_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024 +MAX_ENSEMBLE_SAMPLES = 65_536 +MAX_ENSEMBLE_SLICES = 16_384 +MAX_ENSEMBLE_PAYLOAD_BYTES = 64 * 1024 * 1024 +MAX_ENSEMBLE_ARTIFACTS = 256 +MAX_ENSEMBLE_REASON_CODES = 256 +MAX_ENSEMBLE_TEXT = 1024 +INT64_MAX = 2**63 - 1 +UINT64_MAX = 2**64 - 1 + + +class EnsembleArtifactKind(str, Enum): + """Whether a hash binds source evidence or semantic configuration.""" + + SOURCE = "source" + CONFIGURATION = "configuration" + + +class EnsembleMemberStatus(str, Enum): + """One member outcome in one reverse-degradation holdout cell.""" + + COMPLETED = "completed" + REFUSED = "refused" + FAILED = "failed" + + +class EnsembleMetricStatus(str, Enum): + """Whether one calibrated metric cell has trustworthy support.""" + + CALIBRATED = "calibrated" + INSUFFICIENT_SUPPORT = "insufficient_support" + MISCALIBRATED = "miscalibrated" + + +class EnsembleDiversityStatus(str, Enum): + """Whether logical paths provide substantive member diversity.""" + + DIVERSE = "diverse" + COLLAPSED = "collapsed" + FALSE_DIVERSITY = "false_diversity" + INSUFFICIENT_SUPPORT = "insufficient_support" + + +class EnsembleReportStatus(str, Enum): + """Top-level calibrated-ensemble trust result.""" + + CALIBRATED = "calibrated" + INSUFFICIENT_SUPPORT = "insufficient_support" + MISCALIBRATED = "miscalibrated" + + +@dataclass(frozen=True, slots=True) +class EnsembleCalibrationConfigV1: + """Versioned ensemble size, calibration, diversity, and retention policy.""" + + member_count: int = 4 + retained_member_count: int = 2 + horizons_ns: tuple[int, ...] = DEFAULT_ENSEMBLE_HORIZONS_NS + nominal_coverage: float = 0.8 + minimum_achieved_coverage: float = 0.75 + minimum_fit_samples: int = 2 + maximum_collapse_rate: float = 0.0 + maximum_false_diversity_rate: float = 0.0 + logical_distance_tolerance: float = 1e-12 + failure_penalty: float = 1.0 + estimated_bytes_per_event: int = 512 + rounding_digits: int = 12 + max_samples: int = DEFAULT_ENSEMBLE_MAX_SAMPLES + max_slices: int = DEFAULT_ENSEMBLE_MAX_SLICES + max_payload_bytes: int = DEFAULT_ENSEMBLE_MAX_PAYLOAD_BYTES + config_id: str = "" + schema_version: str = ENSEMBLE_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_CONFIG_SCHEMA_VERSION, + "ensemble config", + ) + member_count = _positive_int(self.member_count, "member_count") + if not 2 <= member_count <= MAX_BENCHMARK_ENSEMBLE_MEMBERS: + raise ValueError( + "ensemble member_count is outside supported limits" + ) + retained = _positive_int( + self.retained_member_count, "retained_member_count" + ) + if retained > member_count: + raise ValueError("retained members exceed ensemble member count") + object.__setattr__(self, "member_count", member_count) + object.__setattr__(self, "retained_member_count", retained) + horizons = tuple( + sorted( + { + _positive_int(value, "horizon_ns") + for value in self.horizons_ns + } + ) + ) + if not horizons or any(value > INT64_MAX for value in horizons): + raise ValueError("ensemble horizons are empty or outside int64") + object.__setattr__(self, "horizons_ns", horizons) + nominal = _unit_float(self.nominal_coverage, "nominal_coverage") + minimum = _unit_float( + self.minimum_achieved_coverage, + "minimum_achieved_coverage", + ) + if nominal <= 0.0 or nominal >= 1.0: + raise ValueError( + "nominal coverage must be strictly between zero and one" + ) + if minimum > nominal: + raise ValueError( + "minimum achieved coverage exceeds nominal coverage" + ) + object.__setattr__(self, "nominal_coverage", nominal) + object.__setattr__(self, "minimum_achieved_coverage", minimum) + object.__setattr__( + self, + "minimum_fit_samples", + _positive_int(self.minimum_fit_samples, "minimum_fit_samples"), + ) + for name in ( + "maximum_collapse_rate", + "maximum_false_diversity_rate", + ): + object.__setattr__( + self, name, _unit_float(getattr(self, name), name) + ) + tolerance = _nonnegative_float( + self.logical_distance_tolerance, + "logical_distance_tolerance", + ) + penalty = _nonnegative_float(self.failure_penalty, "failure_penalty") + object.__setattr__(self, "logical_distance_tolerance", tolerance) + object.__setattr__(self, "failure_penalty", penalty) + object.__setattr__( + self, + "estimated_bytes_per_event", + _positive_int( + self.estimated_bytes_per_event, + "estimated_bytes_per_event", + ), + ) + digits = _nonnegative_int(self.rounding_digits, "rounding_digits") + if digits > 15: + raise ValueError("rounding_digits exceeds stable float precision") + object.__setattr__(self, "rounding_digits", digits) + for name, upper in ( + ("max_samples", MAX_ENSEMBLE_SAMPLES), + ("max_slices", MAX_ENSEMBLE_SLICES), + ("max_payload_bytes", MAX_ENSEMBLE_PAYLOAD_BYTES), + ): + value = _positive_int(getattr(self, name), name) + if value > upper: + raise ValueError(f"{name} exceeds ensemble v1 limit") + object.__setattr__(self, name, value) + if self.max_slices < len(horizons): + raise ValueError("max_slices cannot cover configured horizons") + minimum_samples = len(horizons) * (self.minimum_fit_samples + 1) + if self.max_samples < minimum_samples: + raise ValueError( + "max_samples cannot cover fit/final horizon support" + ) + expected = _stable_id("ensemble-config", self.identity_payload()) + supplied = _optional_text(self.config_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble config_id differs") + object.__setattr__(self, "config_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "engine_id": ENSEMBLE_ENGINE_ID, + "engine_version": ENSEMBLE_ENGINE_VERSION, + "member_count": self.member_count, + "retained_member_count": self.retained_member_count, + "horizons_ns": list(self.horizons_ns), + "metric_names": list(ENSEMBLE_CALIBRATION_METRIC_NAMES), + "nominal_coverage": self.nominal_coverage, + "minimum_achieved_coverage": self.minimum_achieved_coverage, + "minimum_fit_samples": self.minimum_fit_samples, + "maximum_collapse_rate": self.maximum_collapse_rate, + "maximum_false_diversity_rate": (self.maximum_false_diversity_rate), + "logical_distance_tolerance": self.logical_distance_tolerance, + "failure_penalty": self.failure_penalty, + "calibration_fit_split": ENSEMBLE_CALIBRATION_FIT_SPLIT.value, + "calibration_evaluation_split": ( + ENSEMBLE_CALIBRATION_EVALUATION_SPLIT.value + ), + "confidence_quantity": ENSEMBLE_CONFIDENCE_QUANTITY, + "primary_selection_basis": ENSEMBLE_PRIMARY_SELECTION_BASIS, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "estimated_bytes_per_event": self.estimated_bytes_per_event, + "rounding_digits": self.rounding_digits, + "max_samples": self.max_samples, + "max_slices": self.max_slices, + "max_payload_bytes": self.max_payload_bytes, + "config_id": self.config_id, + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EnsembleCalibrationConfigV1": + _require_schema(data, ENSEMBLE_CONFIG_SCHEMA_VERSION) + _require_derived_value( + data, + "metric_names", + list(ENSEMBLE_CALIBRATION_METRIC_NAMES), + ) + _require_derived_value( + data, + "calibration_fit_split", + ENSEMBLE_CALIBRATION_FIT_SPLIT.value, + ) + _require_derived_value( + data, + "calibration_evaluation_split", + ENSEMBLE_CALIBRATION_EVALUATION_SPLIT.value, + ) + _require_derived_value( + data, + "confidence_quantity", + ENSEMBLE_CONFIDENCE_QUANTITY, + ) + _require_derived_value( + data, + "primary_selection_basis", + ENSEMBLE_PRIMARY_SELECTION_BASIS, + ) + return cls( + member_count=_strict_int(data.get("member_count"), "member_count"), + retained_member_count=_strict_int( + data.get("retained_member_count"), "retained_member_count" + ), + horizons_ns=_int_tuple(data.get("horizons_ns"), "horizons_ns"), + nominal_coverage=_finite_float( + data.get("nominal_coverage"), "nominal_coverage" + ), + minimum_achieved_coverage=_finite_float( + data.get("minimum_achieved_coverage"), + "minimum_achieved_coverage", + ), + minimum_fit_samples=_strict_int( + data.get("minimum_fit_samples"), "minimum_fit_samples" + ), + maximum_collapse_rate=_finite_float( + data.get("maximum_collapse_rate"), + "maximum_collapse_rate", + ), + maximum_false_diversity_rate=_finite_float( + data.get("maximum_false_diversity_rate"), + "maximum_false_diversity_rate", + ), + logical_distance_tolerance=_finite_float( + data.get("logical_distance_tolerance"), + "logical_distance_tolerance", + ), + failure_penalty=_finite_float( + data.get("failure_penalty"), "failure_penalty" + ), + estimated_bytes_per_event=_strict_int( + data.get("estimated_bytes_per_event"), + "estimated_bytes_per_event", + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + max_samples=_strict_int(data.get("max_samples"), "max_samples"), + max_slices=_strict_int(data.get("max_slices"), "max_slices"), + max_payload_bytes=_strict_int( + data.get("max_payload_bytes"), "max_payload_bytes" + ), + config_id=str(data.get("config_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EnsembleCalibrationConfigV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class EnsembleArtifactDigestV1: + """One content hash required to reproduce an ensemble member.""" + + artifact_id: str + sha256: str + kind: EnsembleArtifactKind + digest_id: str = "" + schema_version: str = ENSEMBLE_ARTIFACT_DIGEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_ARTIFACT_DIGEST_SCHEMA_VERSION, + "ensemble artifact digest", + ) + object.__setattr__( + self, "artifact_id", _required_text(self.artifact_id) + ) + object.__setattr__(self, "sha256", _required_sha256(self.sha256)) + object.__setattr__(self, "kind", EnsembleArtifactKind(self.kind)) + expected = _stable_id("ensemble-artifact", self.identity_payload()) + supplied = _optional_text(self.digest_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble artifact digest_id differs") + object.__setattr__(self, "digest_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "artifact_id": self.artifact_id, + "sha256": self.sha256, + "kind": self.kind.value, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "digest_id": self.digest_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "EnsembleArtifactDigestV1": + _require_schema(data, ENSEMBLE_ARTIFACT_DIGEST_SCHEMA_VERSION) + return cls( + artifact_id=str(data.get("artifact_id", "")), + sha256=str(data.get("sha256", "")), + kind=EnsembleArtifactKind(str(data.get("kind", ""))), + digest_id=str(data.get("digest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleMemberPlanV1: + """One stable ensemble-member identity and semantic seed.""" + + ordinal: int + member_id: str + seed: int + schema_version: str = ENSEMBLE_MEMBER_PLAN_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_MEMBER_PLAN_SCHEMA_VERSION, + "ensemble member plan", + ) + object.__setattr__( + self, "ordinal", _positive_int(self.ordinal, "ordinal") + ) + object.__setattr__(self, "member_id", _required_text(self.member_id)) + seed = _nonnegative_int(self.seed, "seed") + if seed > UINT64_MAX: + raise ValueError("ensemble member seed exceeds uint64") + object.__setattr__(self, "seed", seed) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "ordinal": self.ordinal, + "member_id": self.member_id, + "seed": self.seed, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "EnsembleMemberPlanV1": + _require_schema(data, ENSEMBLE_MEMBER_PLAN_SCHEMA_VERSION) + return cls( + ordinal=_strict_int(data.get("ordinal"), "ordinal"), + member_id=str(data.get("member_id", "")), + seed=_strict_int(data.get("seed"), "seed"), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionEnsemblePlanV1: + """Deterministic run/member plan bound to exact source/config hashes.""" + + run: ReconstructionRunV1 + config: EnsembleCalibrationConfigV1 + source_artifacts: tuple[EnsembleArtifactDigestV1, ...] + configuration_artifacts: tuple[EnsembleArtifactDigestV1, ...] + members: tuple[EnsembleMemberPlanV1, ...] + plan_id: str = "" + schema_version: str = ENSEMBLE_PLAN_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_PLAN_SCHEMA_VERSION, + "ensemble plan", + ) + if not isinstance(self.run, ReconstructionRunV1): + raise ValueError("ensemble plan requires a reconstruction run") + if not isinstance(self.config, EnsembleCalibrationConfigV1): + raise ValueError("ensemble plan requires a v1 config") + sources = _normalized_artifacts( + self.source_artifacts, EnsembleArtifactKind.SOURCE + ) + configs = _normalized_artifacts( + self.configuration_artifacts, + EnsembleArtifactKind.CONFIGURATION, + ) + if not sources or not configs: + raise ValueError("ensemble plan requires source and config hashes") + if ( + tuple(item.artifact_id for item in sources) + != self.run.source_version_ids + ): + raise ValueError("ensemble source hashes differ from run sources") + if ( + tuple(item.artifact_id for item in configs) + != self.run.configuration_ids + ): + raise ValueError("ensemble config hashes differ from run configs") + expected_config_hash = _content_sha256(self.config.to_dict()) + config_digest = next( + ( + item + for item in configs + if item.artifact_id == self.config.config_id + ), + None, + ) + if ( + config_digest is None + or config_digest.sha256 != expected_config_hash + ): + raise ValueError("ensemble config artifact hash differs") + object.__setattr__(self, "source_artifacts", sources) + object.__setattr__(self, "configuration_artifacts", configs) + members = tuple(sorted(self.members, key=lambda item: item.ordinal)) + if tuple(item.ordinal for item in members) != tuple( + range(1, self.config.member_count + 1) + ): + raise ValueError("ensemble member ordinals are incomplete") + expected_ids = tuple( + _derive_member_id( + self.config, + sources, + configs, + self.run.base_seed, + ordinal, + ) + for ordinal in range(1, self.config.member_count + 1) + ) + if tuple(item.member_id for item in members) != expected_ids: + raise ValueError( + "ensemble member IDs are not deterministically derived" + ) + if set(expected_ids) != set(self.run.ensemble_member_ids): + raise ValueError("ensemble members differ from reconstruction run") + for member in members: + expected_seed = self.run.seed_for( + member.member_id, + f"{ENSEMBLE_ENGINE_ID}:{self.config.config_id}:member", + ) + if member.seed != expected_seed: + raise ValueError("ensemble member seed differs") + object.__setattr__(self, "members", members) + expected = _stable_id("ensemble-plan", self.identity_payload()) + supplied = _optional_text(self.plan_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble plan_id differs") + object.__setattr__(self, "plan_id", expected) + + @property + def source_hashes(self) -> dict[str, str]: + return {item.artifact_id: item.sha256 for item in self.source_artifacts} + + @property + def configuration_hashes(self) -> dict[str, str]: + return { + item.artifact_id: item.sha256 + for item in self.configuration_artifacts + } + + def member(self, member_id: str) -> EnsembleMemberPlanV1: + wanted = _required_text(member_id) + for member in self.members: + if member.member_id == wanted: + return member + raise KeyError(wanted) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "engine_id": ENSEMBLE_ENGINE_ID, + "engine_version": ENSEMBLE_ENGINE_VERSION, + "run": self.run.to_dict(), + "config": self.config.to_dict(), + "source_artifacts": [ + item.to_dict() for item in self.source_artifacts + ], + "configuration_artifacts": [ + item.to_dict() for item in self.configuration_artifacts + ], + "members": [item.to_dict() for item in self.members], + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "plan_id": self.plan_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionEnsemblePlanV1": + _require_schema(data, ENSEMBLE_PLAN_SCHEMA_VERSION) + return cls( + run=ReconstructionRunV1.from_dict(_mapping(data.get("run"))), + config=EnsembleCalibrationConfigV1.from_dict( + _mapping(data.get("config")) + ), + source_artifacts=tuple( + EnsembleArtifactDigestV1.from_dict(item) + for item in _mapping_sequence(data, "source_artifacts") + ), + configuration_artifacts=tuple( + EnsembleArtifactDigestV1.from_dict(item) + for item in _mapping_sequence(data, "configuration_artifacts") + ), + members=tuple( + EnsembleMemberPlanV1.from_dict(item) + for item in _mapping_sequence(data, "members") + ), + plan_id=str(data.get("plan_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionEnsemblePlanV1": + return cls.from_dict(_json_mapping(text)) + + +def plan_reconstruction_ensemble( + *, + symbols: Iterable[str], + source_artifact_hashes: Mapping[str, str], + configuration_artifact_hashes: Mapping[str, str], + base_seed: int, + config: EnsembleCalibrationConfigV1 | None = None, + storage_policy: ReconstructionStoragePolicyV1 | None = None, +) -> ReconstructionEnsemblePlanV1: + """Build stable member IDs and seeds without worker-count inputs.""" + selected = config or EnsembleCalibrationConfigV1() + sources = _artifact_digests( + source_artifact_hashes, EnsembleArtifactKind.SOURCE + ) + config_hashes = dict(configuration_artifact_hashes) + expected_config_hash = _content_sha256(selected.to_dict()) + supplied = config_hashes.get(selected.config_id) + if ( + supplied is not None + and _required_sha256(supplied) != expected_config_hash + ): + raise ValueError("supplied ensemble config hash differs") + config_hashes[selected.config_id] = expected_config_hash + configurations = _artifact_digests( + config_hashes, EnsembleArtifactKind.CONFIGURATION + ) + member_ids = tuple( + _derive_member_id( + selected, + sources, + configurations, + base_seed, + ordinal, + ) + for ordinal in range(1, selected.member_count + 1) + ) + run = ReconstructionRunV1( + symbols=tuple(symbols), + source_version_ids=tuple(item.artifact_id for item in sources), + configuration_ids=tuple(item.artifact_id for item in configurations), + ensemble_member_ids=member_ids, + base_seed=base_seed, + storage_policy=storage_policy or ReconstructionStoragePolicyV1(), + ) + members = tuple( + EnsembleMemberPlanV1( + ordinal=ordinal, + member_id=member_id, + seed=run.seed_for( + member_id, + f"{ENSEMBLE_ENGINE_ID}:{selected.config_id}:member", + ), + ) + for ordinal, member_id in enumerate(member_ids, start=1) + ) + return ReconstructionEnsemblePlanV1( + run=run, + config=selected, + source_artifacts=sources, + configuration_artifacts=configurations, + members=members, + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleStorageEstimateV1: + """Conservative all-member estimate checked by the #432 storage policy.""" + + run_id: str + plan_id: str + member_event_counts: Mapping[str, int] + retained_member_count: int + conservative_retained_event_count: int + estimated_bytes_per_event: int + resource_estimate: ReconstructionResourceEstimateV1 + estimate_id: str = "" + schema_version: str = ENSEMBLE_STORAGE_ESTIMATE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_STORAGE_ESTIMATE_SCHEMA_VERSION, + "ensemble storage estimate", + ) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__(self, "plan_id", _required_text(self.plan_id)) + counts = _count_mapping(self.member_event_counts, "member event") + if not counts: + raise ValueError("ensemble storage estimate requires members") + object.__setattr__(self, "member_event_counts", counts) + object.__setattr__( + self, + "retained_member_count", + _positive_int(self.retained_member_count, "retained_member_count"), + ) + object.__setattr__( + self, + "conservative_retained_event_count", + _nonnegative_int( + self.conservative_retained_event_count, + "conservative_retained_event_count", + ), + ) + object.__setattr__( + self, + "estimated_bytes_per_event", + _positive_int( + self.estimated_bytes_per_event, + "estimated_bytes_per_event", + ), + ) + if not isinstance( + self.resource_estimate, ReconstructionResourceEstimateV1 + ): + raise ValueError( + "ensemble estimate requires a #432 resource estimate" + ) + values = tuple(counts.values()) + if self.retained_member_count > len(values): + raise ValueError("ensemble estimate retains too many members") + expected_retained = sum( + sorted(values, reverse=True)[: self.retained_member_count] + ) + if self.conservative_retained_event_count != expected_retained: + raise ValueError("ensemble retained-event estimate differs") + estimate = self.resource_estimate + expected_fields = { + "candidate_event_count": sum(values), + "retained_ensemble_members": self.retained_member_count, + "peak_events_per_batch": max(values, default=0), + "estimated_memory_bytes": ( + max(values, default=0) * self.estimated_bytes_per_event + ), + "estimated_scratch_bytes": ( + sum(values) * self.estimated_bytes_per_event + ), + "estimated_output_bytes": ( + expected_retained * self.estimated_bytes_per_event + ), + } + if any( + getattr(estimate, name) != expected + for name, expected in expected_fields.items() + ): + raise ValueError("ensemble resource estimate arithmetic differs") + expected = _stable_id("ensemble-storage-estimate", self.payload()) + supplied = _optional_text(self.estimate_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble storage estimate_id differs") + object.__setattr__(self, "estimate_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "plan_id": self.plan_id, + "member_event_counts": dict(self.member_event_counts), + "retained_member_count": self.retained_member_count, + "conservative_retained_event_count": ( + self.conservative_retained_event_count + ), + "estimated_bytes_per_event": self.estimated_bytes_per_event, + "resource_estimate": self.resource_estimate.to_dict(), + "retention_estimate_basis": "largest-member-counts-v1", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "estimate_id": self.estimate_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "EnsembleStorageEstimateV1": + _require_schema(data, ENSEMBLE_STORAGE_ESTIMATE_SCHEMA_VERSION) + _require_derived_value( + data, + "retention_estimate_basis", + "largest-member-counts-v1", + ) + return cls( + run_id=str(data.get("run_id", "")), + plan_id=str(data.get("plan_id", "")), + member_event_counts={ + str(key): _strict_int(value, "member event count") + for key, value in _mapping( + data.get("member_event_counts") + ).items() + }, + retained_member_count=_strict_int( + data.get("retained_member_count"), "retained_member_count" + ), + conservative_retained_event_count=_strict_int( + data.get("conservative_retained_event_count"), + "conservative_retained_event_count", + ), + estimated_bytes_per_event=_strict_int( + data.get("estimated_bytes_per_event"), + "estimated_bytes_per_event", + ), + resource_estimate=ReconstructionResourceEstimateV1.from_dict( + _mapping(data.get("resource_estimate")) + ), + estimate_id=str(data.get("estimate_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EnsembleStorageEstimateV1": + return cls.from_dict(_json_mapping(text)) + + +def estimate_reconstruction_ensemble_resources( + plan: ReconstructionEnsemblePlanV1, + *, + input_event_count: int, + member_event_counts: Mapping[str, int], +) -> EnsembleStorageEstimateV1: + """Conservatively estimate all computation and retained-member output.""" + if not isinstance(plan, ReconstructionEnsemblePlanV1): + raise ValueError("ensemble resource estimate requires a v1 plan") + input_count = _nonnegative_int(input_event_count, "input_event_count") + counts = _count_mapping(member_event_counts, "member event") + expected_members = {item.member_id for item in plan.members} + if set(counts) != expected_members: + raise ValueError("ensemble event counts do not cover planned members") + values = tuple(counts.values()) + retained_count = plan.config.retained_member_count + retained_events = sum(sorted(values, reverse=True)[:retained_count]) + bytes_per_event = plan.config.estimated_bytes_per_event + maximum = max(values, default=0) + batch_limit = plan.run.storage_policy.max_events_per_batch + estimate = ReconstructionResourceEstimateV1( + input_event_count=input_count, + candidate_event_count=sum(values), + retained_ensemble_members=retained_count, + inflight_batches=1 if any(values) else 0, + peak_events_per_batch=maximum, + estimated_memory_bytes=maximum * bytes_per_event, + estimated_scratch_bytes=sum(values) * bytes_per_event, + estimated_output_bytes=retained_events * bytes_per_event, + estimated_batch_count=sum( + math.ceil(value / batch_limit) for value in values if value + ), + ) + plan.run.storage_policy.preflight(estimate) + return EnsembleStorageEstimateV1( + run_id=plan.run.run_id, + plan_id=plan.plan_id, + member_event_counts=counts, + retained_member_count=retained_count, + conservative_retained_event_count=retained_events, + estimated_bytes_per_event=bytes_per_event, + resource_estimate=estimate, + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleCalibrationStratumV1: + """Exact epoch/session/event/symbol/horizon/sparsity calibration cell.""" + + epoch_id: str + session: str + event_state: str + symbol: str + horizon_ns: int + sparsity: str + stratum_id: str = "" + schema_version: str = ENSEMBLE_STRATUM_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_STRATUM_SCHEMA_VERSION, + "ensemble stratum", + ) + for name in ("epoch_id", "session", "event_state", "sparsity"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + horizon = _positive_int(self.horizon_ns, "horizon_ns") + if horizon > INT64_MAX: + raise ValueError("ensemble horizon exceeds int64") + object.__setattr__(self, "horizon_ns", horizon) + expected = _stable_id("ensemble-stratum", self.identity_payload()) + supplied = _optional_text(self.stratum_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble stratum_id differs") + object.__setattr__(self, "stratum_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "epoch_id": self.epoch_id, + "session": self.session, + "event_state": self.event_state, + "symbol": self.symbol, + "horizon_ns": self.horizon_ns, + "sparsity": self.sparsity, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "stratum_id": self.stratum_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EnsembleCalibrationStratumV1": + _require_schema(data, ENSEMBLE_STRATUM_SCHEMA_VERSION) + return cls( + epoch_id=str(data.get("epoch_id", "")), + session=str(data.get("session", "")), + event_state=str(data.get("event_state", "")), + symbol=str(data.get("symbol", "")), + horizon_ns=_strict_int(data.get("horizon_ns"), "horizon_ns"), + sparsity=str(data.get("sparsity", "")), + stratum_id=str(data.get("stratum_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleMemberCalibrationV1: + """One member's compact result in a calibration sample.""" + + member_id: str + status: EnsembleMemberStatus + metrics: Mapping[str, float] = field(default_factory=dict) + logical_content_sha256: str | None = None + reason: str | None = None + result_id: str = "" + schema_version: str = ENSEMBLE_MEMBER_CALIBRATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_MEMBER_CALIBRATION_SCHEMA_VERSION, + "ensemble member calibration", + ) + object.__setattr__(self, "member_id", _required_text(self.member_id)) + status = EnsembleMemberStatus(self.status) + object.__setattr__(self, "status", status) + metrics = _metric_mapping(self.metrics, allow_empty=True) + digest = ( + _required_sha256(self.logical_content_sha256) + if self.logical_content_sha256 is not None + else None + ) + reason = _optional_bounded_text(self.reason, "reason") + if status is EnsembleMemberStatus.COMPLETED: + if set(metrics) != set(ENSEMBLE_CALIBRATION_METRIC_NAMES): + raise ValueError("completed member metrics are incomplete") + if digest is None or reason is not None: + raise ValueError( + "completed member hash/reason state is invalid" + ) + elif metrics or digest is not None or reason is None: + raise ValueError("failed/refused member must retain only a reason") + object.__setattr__(self, "metrics", metrics) + object.__setattr__(self, "logical_content_sha256", digest) + object.__setattr__(self, "reason", reason) + expected = _stable_id("ensemble-member-result", self.payload()) + supplied = _optional_text(self.result_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble member result_id differs") + object.__setattr__(self, "result_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "member_id": self.member_id, + "status": self.status.value, + "metrics": dict(self.metrics), + "logical_content_sha256": self.logical_content_sha256, + "reason": self.reason, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "result_id": self.result_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EnsembleMemberCalibrationV1": + _require_schema(data, ENSEMBLE_MEMBER_CALIBRATION_SCHEMA_VERSION) + return cls( + member_id=str(data.get("member_id", "")), + status=EnsembleMemberStatus(str(data.get("status", ""))), + metrics=cast(Mapping[str, float], _mapping(data.get("metrics"))), + logical_content_sha256=_optional_text( + data.get("logical_content_sha256") + ), + reason=_optional_text(data.get("reason")), + result_id=str(data.get("result_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleCalibrationSampleV1: + """One reverse-degradation holdout cell with no embedded event rows.""" + + benchmark_manifest_id: str + scenario_id: str + candidate_id: str + window_id: str + split_kind: BenchmarkSplitKind + stratum: EnsembleCalibrationStratumV1 + reference_metrics: Mapping[str, float] + reference_content_sha256: str + members: tuple[EnsembleMemberCalibrationV1, ...] + sample_id: str = "" + schema_version: str = ENSEMBLE_CALIBRATION_SAMPLE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_CALIBRATION_SAMPLE_SCHEMA_VERSION, + "ensemble calibration sample", + ) + for name in ( + "benchmark_manifest_id", + "scenario_id", + "candidate_id", + "window_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + split = BenchmarkSplitKind.from_value(self.split_kind) + if split not in { + ENSEMBLE_CALIBRATION_FIT_SPLIT, + ENSEMBLE_CALIBRATION_EVALUATION_SPLIT, + }: + raise ValueError( + "ensemble samples require validation/final holdout" + ) + object.__setattr__(self, "split_kind", split) + if not isinstance(self.stratum, EnsembleCalibrationStratumV1): + raise ValueError("ensemble sample requires a v1 stratum") + metrics = _metric_mapping(self.reference_metrics) + object.__setattr__(self, "reference_metrics", metrics) + object.__setattr__( + self, + "reference_content_sha256", + _required_sha256(self.reference_content_sha256), + ) + members = tuple(sorted(self.members, key=lambda item: item.member_id)) + if not members or len({item.member_id for item in members}) != len( + members + ): + raise ValueError("ensemble sample members are empty or duplicated") + if len(members) > MAX_BENCHMARK_ENSEMBLE_MEMBERS: + raise ValueError("ensemble sample member count exceeds limit") + object.__setattr__(self, "members", members) + expected = _stable_id("ensemble-calibration-sample", self.payload()) + supplied = _optional_text(self.sample_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble calibration sample_id differs") + object.__setattr__(self, "sample_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "benchmark_manifest_id": self.benchmark_manifest_id, + "scenario_id": self.scenario_id, + "candidate_id": self.candidate_id, + "window_id": self.window_id, + "split_kind": self.split_kind.value, + "stratum": self.stratum.to_dict(), + "reference_metrics": dict(self.reference_metrics), + "reference_content_sha256": self.reference_content_sha256, + "members": [item.to_dict() for item in self.members], + "event_rows_inline": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "sample_id": self.sample_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EnsembleCalibrationSampleV1": + _require_schema(data, ENSEMBLE_CALIBRATION_SAMPLE_SCHEMA_VERSION) + _require_derived_value(data, "event_rows_inline", False) + return cls( + benchmark_manifest_id=str(data.get("benchmark_manifest_id", "")), + scenario_id=str(data.get("scenario_id", "")), + candidate_id=str(data.get("candidate_id", "")), + window_id=str(data.get("window_id", "")), + split_kind=BenchmarkSplitKind.from_value( + str(data.get("split_kind", "")) + ), + stratum=EnsembleCalibrationStratumV1.from_dict( + _mapping(data.get("stratum")) + ), + reference_metrics=cast( + Mapping[str, float], _mapping(data.get("reference_metrics")) + ), + reference_content_sha256=str( + data.get("reference_content_sha256", "") + ), + members=tuple( + EnsembleMemberCalibrationV1.from_dict(item) + for item in _mapping_sequence(data, "members") + ), + sample_id=str(data.get("sample_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EnsembleCalibrationSampleV1": + return cls.from_dict(_json_mapping(text)) + + +def benchmark_ensemble_calibration_sample( + plan: ReconstructionEnsemblePlanV1, + *, + benchmark_manifest_id: str, + scenario: BenchmarkScenarioV1, + candidate_id: str, + window_id: str, + horizon_ns: int, + reference_events: Sequence[BenchmarkEventV1], + member_windows: Sequence[BenchmarkCandidateWindowV1], + reference_downstream_sensitivity: float, +) -> EnsembleCalibrationSampleV1: + """Adapt one bounded reverse-degradation cell into calibration metadata.""" + if not isinstance(plan, ReconstructionEnsemblePlanV1): + raise ValueError("benchmark ensemble sample requires a v1 plan") + if not isinstance(scenario, BenchmarkScenarioV1): + raise ValueError("benchmark ensemble sample requires a v1 scenario") + if scenario.split_kind not in { + ENSEMBLE_CALIBRATION_FIT_SPLIT, + ENSEMBLE_CALIBRATION_EVALUATION_SPLIT, + }: + raise ValueError( + "ensemble calibration requires validation/final scenario" + ) + horizon = _positive_int(horizon_ns, "horizon_ns") + if horizon not in plan.config.horizons_ns: + raise ValueError("calibration horizon is absent from ensemble config") + reference = _validated_benchmark_events(reference_events) + if not reference: + raise ValueError("calibration reference events cannot be empty") + horizon_start_ns = reference[0].event_time_ns + horizon_end_ns = horizon_start_ns + horizon + if reference[-1].event_time_ns > horizon_end_ns: + raise ValueError("calibration reference exceeds declared horizon") + slice_keys = {item.slice_key for item in reference} + if len(slice_keys) != 1: + raise ValueError("calibration reference must occupy one exact stratum") + symbol, epoch, session, event_state, sparsity = next(iter(slice_keys)) + if epoch != scenario.epoch_id: + raise ValueError("calibration reference epoch differs from scenario") + stratum = EnsembleCalibrationStratumV1( + epoch_id=epoch, + session=session, + event_state=event_state, + symbol=symbol, + horizon_ns=horizon, + sparsity=sparsity, + ) + selected_candidate = _required_text(candidate_id) + selected_window = _required_text(window_id) + by_member: dict[str, BenchmarkCandidateWindowV1] = {} + for member_window in member_windows: + if member_window.ensemble_member_id in by_member: + raise ValueError("duplicate ensemble member window") + if ( + member_window.scenario_id != scenario.scenario_id + or member_window.candidate_id != selected_candidate + or member_window.window_id != selected_window + ): + raise ValueError("candidate window differs from calibration scope") + by_member[member_window.ensemble_member_id] = member_window + member_results: list[EnsembleMemberCalibrationV1] = [] + for member in plan.members: + candidate_window = by_member.get(member.member_id) + if candidate_window is None: + member_results.append( + EnsembleMemberCalibrationV1( + member_id=member.member_id, + status=EnsembleMemberStatus.FAILED, + reason="missing_member_window", + ) + ) + continue + execution = candidate_window.execution + if not execution.attempted: + member_results.append( + EnsembleMemberCalibrationV1( + member_id=member.member_id, + status=EnsembleMemberStatus.REFUSED, + reason="not_attempted", + ) + ) + continue + if not execution.converged: + member_results.append( + EnsembleMemberCalibrationV1( + member_id=member.member_id, + status=EnsembleMemberStatus.FAILED, + reason=execution.failure_reason or "generation_failed", + ) + ) + continue + if candidate_window.hard_constraint_violations: + first_reason = sorted(candidate_window.hard_constraint_violations)[ + 0 + ] + member_results.append( + EnsembleMemberCalibrationV1( + member_id=member.member_id, + status=EnsembleMemberStatus.REFUSED, + reason=f"hard_constraint:{first_reason}", + ) + ) + continue + events = _validated_benchmark_events(candidate_window.events) + if not events: + member_results.append( + EnsembleMemberCalibrationV1( + member_id=member.member_id, + status=EnsembleMemberStatus.REFUSED, + reason="empty_member_stream", + ) + ) + continue + if {item.slice_key for item in events} != slice_keys: + raise ValueError("candidate member crosses calibration strata") + if ( + events[0].event_time_ns < horizon_start_ns + or events[-1].event_time_ns > horizon_end_ns + ): + raise ValueError("candidate member exceeds calibration horizon") + downstream = candidate_window.strategy_hooks.get( + "downstream_sensitivity" + ) + if downstream is None: + member_results.append( + EnsembleMemberCalibrationV1( + member_id=member.member_id, + status=EnsembleMemberStatus.REFUSED, + reason="missing_downstream_sensitivity", + ) + ) + continue + member_results.append( + EnsembleMemberCalibrationV1( + member_id=member.member_id, + status=EnsembleMemberStatus.COMPLETED, + metrics=_benchmark_metrics(events, downstream), + logical_content_sha256=benchmark_logical_content_sha256(events), + ) + ) + unknown = set(by_member).difference(item.member_id for item in plan.members) + if unknown: + raise ValueError("calibration contains unplanned ensemble members") + return EnsembleCalibrationSampleV1( + benchmark_manifest_id=benchmark_manifest_id, + scenario_id=scenario.scenario_id, + candidate_id=selected_candidate, + window_id=selected_window, + split_kind=scenario.split_kind, + stratum=stratum, + reference_metrics=_benchmark_metrics( + reference, + reference_downstream_sensitivity, + ), + reference_content_sha256=benchmark_logical_content_sha256(reference), + members=tuple(member_results), + ) + + +def benchmark_logical_content_sha256( + events: Iterable[BenchmarkEventV1], +) -> str: + """Hash market content independently of row order, IDs, seeds, and member.""" + ordered_events = sorted( + events, + key=lambda row: ( + row.symbol, + row.event_time_ns, + row.event_sequence, + row.bid, + row.ask, + ), + ) + rows: list[dict[str, JSONValue]] = [ + { + "symbol": item.symbol, + "event_time_ns": item.event_time_ns, + "event_sequence": item.event_sequence, + "bid": item.bid, + "ask": item.ask, + "epoch_id": item.epoch_id, + "session": item.session, + "event_state": item.event_state, + "sparsity": item.sparsity, + "anchor_present": item.anchor_id is not None, + } + for item in ordered_events + ] + return _content_sha256(rows) + + +def ensemble_logical_content_sha256( + streams: Iterable[SyntheticEventStreamV1], +) -> str: + """Hash synchronized market rows without member, run, or lineage identity.""" + events: list[SyntheticEventV1] = [] + for stream in streams: + if not isinstance(stream, SyntheticEventStreamV1): + raise ValueError("ensemble logical hash requires event streams") + events.extend(stream.events) + events.sort( + key=lambda event: ( + event.symbol, + event.event_time_ns, + event.event_sequence, + event.origin.value, + event.bid, + event.ask, + ) + ) + rows: list[dict[str, JSONValue]] = [ + { + "origin": event.origin.value, + "symbol": event.symbol, + "event_time_ns": event.event_time_ns, + "event_sequence": event.event_sequence, + "bid": event.bid, + "ask": event.ask, + } + for event in events + ] + return _content_sha256(rows) + + +@dataclass(frozen=True, slots=True) +class EnsembleMetricCalibrationV1: + """Compact conformal-style coverage evidence for one metric cell.""" + + stratum: EnsembleCalibrationStratumV1 + metric_name: str + metric_group: str + nominal_coverage: float + minimum_achieved_coverage: float + fit_sample_count: int + evaluation_sample_count: int + calibration_adjustment: float | None + raw_covered_count: int + calibrated_covered_count: int + raw_coverage_rate: float | None + calibrated_coverage_rate: float | None + mean_raw_interval_width: float | None + mean_calibrated_interval_width: float | None + mean_absolute_median_error: float | None + status: EnsembleMetricStatus + summary_id: str = "" + schema_version: str = ENSEMBLE_METRIC_CALIBRATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_METRIC_CALIBRATION_SCHEMA_VERSION, + "ensemble metric calibration", + ) + if not isinstance(self.stratum, EnsembleCalibrationStratumV1): + raise ValueError("metric calibration requires a v1 stratum") + metric = _required_text(self.metric_name) + expected_group = ENSEMBLE_CALIBRATION_METRIC_GROUPS.get(metric) + if expected_group is None or self.metric_group != expected_group: + raise ValueError("ensemble metric name/group differs") + object.__setattr__(self, "metric_name", metric) + object.__setattr__(self, "metric_group", expected_group) + object.__setattr__( + self, + "nominal_coverage", + _unit_float(self.nominal_coverage, "nominal_coverage"), + ) + object.__setattr__( + self, + "minimum_achieved_coverage", + _unit_float( + self.minimum_achieved_coverage, + "minimum_achieved_coverage", + ), + ) + if not 0.0 < self.nominal_coverage < 1.0: + raise ValueError("metric nominal coverage must be inside (0,1)") + if self.minimum_achieved_coverage > self.nominal_coverage: + raise ValueError("metric minimum coverage exceeds nominal") + for name in ( + "fit_sample_count", + "evaluation_sample_count", + "raw_covered_count", + "calibrated_covered_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + if self.raw_covered_count > self.evaluation_sample_count or ( + self.calibrated_covered_count > self.evaluation_sample_count + ): + raise ValueError( + "ensemble coverage counts exceed evaluation support" + ) + for name in ( + "calibration_adjustment", + "mean_raw_interval_width", + "mean_calibrated_interval_width", + "mean_absolute_median_error", + ): + object.__setattr__( + self, + name, + _optional_nonnegative_float(getattr(self, name), name), + ) + for name in ("raw_coverage_rate", "calibrated_coverage_rate"): + value = getattr(self, name) + object.__setattr__( + self, + name, + _unit_float(value, name) if value is not None else None, + ) + status = EnsembleMetricStatus(self.status) + if status is EnsembleMetricStatus.INSUFFICIENT_SUPPORT: + if ( + any( + value is not None + for value in ( + self.calibration_adjustment, + self.raw_coverage_rate, + self.calibrated_coverage_rate, + self.mean_raw_interval_width, + self.mean_calibrated_interval_width, + self.mean_absolute_median_error, + ) + ) + or self.evaluation_sample_count != 0 + ): + raise ValueError("unsupported metric cannot claim evidence") + elif ( + self.calibration_adjustment is None + or self.raw_coverage_rate is None + or self.calibrated_coverage_rate is None + or self.mean_raw_interval_width is None + or self.mean_calibrated_interval_width is None + or self.mean_absolute_median_error is None + or self.evaluation_sample_count == 0 + ): + raise ValueError( + "supported metric requires adjustment and coverage" + ) + if self.calibrated_covered_count < self.raw_covered_count: + raise ValueError("calibration cannot reduce covered count") + if self.evaluation_sample_count: + expected_raw = self.raw_covered_count / self.evaluation_sample_count + expected_calibrated = ( + self.calibrated_covered_count / self.evaluation_sample_count + ) + if not math.isclose( + cast(float, self.raw_coverage_rate), + expected_raw, + rel_tol=0.0, + abs_tol=1e-15, + ) or not math.isclose( + cast(float, self.calibrated_coverage_rate), + expected_calibrated, + rel_tol=0.0, + abs_tol=1e-15, + ): + raise ValueError("ensemble coverage rates do not reconcile") + if cast(float, self.mean_calibrated_interval_width) < cast( + float, self.mean_raw_interval_width + ): + raise ValueError("calibration cannot narrow mean interval") + expected_status = ( + EnsembleMetricStatus.CALIBRATED + if expected_calibrated >= self.minimum_achieved_coverage + else EnsembleMetricStatus.MISCALIBRATED + ) + if status is not expected_status: + raise ValueError("ensemble metric status differs from coverage") + object.__setattr__(self, "status", status) + expected = _stable_id("ensemble-metric-calibration", self.payload()) + supplied = _optional_text(self.summary_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble metric summary_id differs") + object.__setattr__(self, "summary_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "stratum": self.stratum.to_dict(), + "metric_name": self.metric_name, + "metric_group": self.metric_group, + "confidence_quantity": ENSEMBLE_CONFIDENCE_QUANTITY, + "nominal_coverage": self.nominal_coverage, + "minimum_achieved_coverage": self.minimum_achieved_coverage, + "fit_sample_count": self.fit_sample_count, + "evaluation_sample_count": self.evaluation_sample_count, + "calibration_adjustment": self.calibration_adjustment, + "raw_covered_count": self.raw_covered_count, + "calibrated_covered_count": self.calibrated_covered_count, + "raw_coverage_rate": self.raw_coverage_rate, + "calibrated_coverage_rate": self.calibrated_coverage_rate, + "mean_raw_interval_width": self.mean_raw_interval_width, + "mean_calibrated_interval_width": ( + self.mean_calibrated_interval_width + ), + "mean_absolute_median_error": self.mean_absolute_median_error, + "status": self.status.value, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "summary_id": self.summary_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EnsembleMetricCalibrationV1": + _require_schema(data, ENSEMBLE_METRIC_CALIBRATION_SCHEMA_VERSION) + _require_derived_value( + data, "confidence_quantity", ENSEMBLE_CONFIDENCE_QUANTITY + ) + return cls( + stratum=EnsembleCalibrationStratumV1.from_dict( + _mapping(data.get("stratum")) + ), + metric_name=str(data.get("metric_name", "")), + metric_group=str(data.get("metric_group", "")), + nominal_coverage=_finite_float( + data.get("nominal_coverage"), "nominal_coverage" + ), + minimum_achieved_coverage=_finite_float( + data.get("minimum_achieved_coverage"), + "minimum_achieved_coverage", + ), + fit_sample_count=_strict_int( + data.get("fit_sample_count"), "fit_sample_count" + ), + evaluation_sample_count=_strict_int( + data.get("evaluation_sample_count"), + "evaluation_sample_count", + ), + calibration_adjustment=_optional_float( + data.get("calibration_adjustment") + ), + raw_covered_count=_strict_int( + data.get("raw_covered_count"), "raw_covered_count" + ), + calibrated_covered_count=_strict_int( + data.get("calibrated_covered_count"), + "calibrated_covered_count", + ), + raw_coverage_rate=_optional_float(data.get("raw_coverage_rate")), + calibrated_coverage_rate=_optional_float( + data.get("calibrated_coverage_rate") + ), + mean_raw_interval_width=_optional_float( + data.get("mean_raw_interval_width") + ), + mean_calibrated_interval_width=_optional_float( + data.get("mean_calibrated_interval_width") + ), + mean_absolute_median_error=_optional_float( + data.get("mean_absolute_median_error") + ), + status=EnsembleMetricStatus(str(data.get("status", ""))), + summary_id=str(data.get("summary_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleDiversitySummaryV1: + """Content-aware pairwise diversity for one split and stratum.""" + + split_kind: BenchmarkSplitKind + stratum: EnsembleCalibrationStratumV1 + sample_count: int + pair_count: int + collapsed_pair_count: int + false_diversity_pair_count: int + distinct_content_count: int + mean_normalized_metric_distance: float | None + collapse_rate: float | None + false_diversity_rate: float | None + status: EnsembleDiversityStatus + summary_id: str = "" + schema_version: str = ENSEMBLE_DIVERSITY_SUMMARY_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_DIVERSITY_SUMMARY_SCHEMA_VERSION, + "ensemble diversity summary", + ) + object.__setattr__( + self, + "split_kind", + BenchmarkSplitKind.from_value(self.split_kind), + ) + if not isinstance(self.stratum, EnsembleCalibrationStratumV1): + raise ValueError("diversity summary requires a v1 stratum") + for name in ( + "sample_count", + "pair_count", + "collapsed_pair_count", + "false_diversity_pair_count", + "distinct_content_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + if self.collapsed_pair_count + self.false_diversity_pair_count > ( + self.pair_count + ): + raise ValueError("diversity diagnostic counts exceed pair support") + object.__setattr__( + self, + "mean_normalized_metric_distance", + _optional_nonnegative_float( + self.mean_normalized_metric_distance, + "mean_normalized_metric_distance", + ), + ) + for name in ("collapse_rate", "false_diversity_rate"): + value = getattr(self, name) + object.__setattr__( + self, + name, + _unit_float(value, name) if value is not None else None, + ) + status = EnsembleDiversityStatus(self.status) + if self.pair_count == 0 and status is not ( + EnsembleDiversityStatus.INSUFFICIENT_SUPPORT + ): + raise ValueError("unsupported diversity cannot claim a diagnosis") + if self.pair_count > 0 and status is ( + EnsembleDiversityStatus.INSUFFICIENT_SUPPORT + ): + raise ValueError("supported diversity cannot be insufficient") + if self.pair_count == 0: + if any( + value is not None + for value in ( + self.mean_normalized_metric_distance, + self.collapse_rate, + self.false_diversity_rate, + ) + ): + raise ValueError("unsupported diversity cannot claim rates") + else: + if ( + self.mean_normalized_metric_distance is None + or self.collapse_rate is None + or self.false_diversity_rate is None + ): + raise ValueError("supported diversity requires rates") + expected_collapse = self.collapsed_pair_count / self.pair_count + expected_false = self.false_diversity_pair_count / self.pair_count + if not math.isclose( + self.collapse_rate, + expected_collapse, + rel_tol=0.0, + abs_tol=1e-15, + ) or not math.isclose( + self.false_diversity_rate, + expected_false, + rel_tol=0.0, + abs_tol=1e-15, + ): + raise ValueError("ensemble diversity rates do not reconcile") + object.__setattr__(self, "status", status) + expected = _stable_id("ensemble-diversity", self.payload()) + supplied = _optional_text(self.summary_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble diversity summary_id differs") + object.__setattr__(self, "summary_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "split_kind": self.split_kind.value, + "stratum": self.stratum.to_dict(), + "sample_count": self.sample_count, + "pair_count": self.pair_count, + "collapsed_pair_count": self.collapsed_pair_count, + "false_diversity_pair_count": self.false_diversity_pair_count, + "distinct_content_count": self.distinct_content_count, + "mean_normalized_metric_distance": ( + self.mean_normalized_metric_distance + ), + "collapse_rate": self.collapse_rate, + "false_diversity_rate": self.false_diversity_rate, + "status": self.status.value, + "row_order_in_identity": False, + "member_id_or_seed_in_logical_content": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "summary_id": self.summary_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "EnsembleDiversitySummaryV1": + _require_schema(data, ENSEMBLE_DIVERSITY_SUMMARY_SCHEMA_VERSION) + _require_derived_value(data, "row_order_in_identity", False) + _require_derived_value( + data, "member_id_or_seed_in_logical_content", False + ) + return cls( + split_kind=BenchmarkSplitKind.from_value( + str(data.get("split_kind", "")) + ), + stratum=EnsembleCalibrationStratumV1.from_dict( + _mapping(data.get("stratum")) + ), + sample_count=_strict_int(data.get("sample_count"), "sample_count"), + pair_count=_strict_int(data.get("pair_count"), "pair_count"), + collapsed_pair_count=_strict_int( + data.get("collapsed_pair_count"), "collapsed_pair_count" + ), + false_diversity_pair_count=_strict_int( + data.get("false_diversity_pair_count"), + "false_diversity_pair_count", + ), + distinct_content_count=_strict_int( + data.get("distinct_content_count"), "distinct_content_count" + ), + mean_normalized_metric_distance=_optional_float( + data.get("mean_normalized_metric_distance") + ), + collapse_rate=_optional_float(data.get("collapse_rate")), + false_diversity_rate=_optional_float( + data.get("false_diversity_rate") + ), + status=EnsembleDiversityStatus(str(data.get("status", ""))), + summary_id=str(data.get("summary_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleOutcomeSummaryV1: + """Failure and refusal rates for one split and exact calibration stratum.""" + + split_kind: BenchmarkSplitKind + stratum: EnsembleCalibrationStratumV1 + attempt_count: int + completed_count: int + refused_count: int + failed_count: int + completion_rate: float + refusal_rate: float + failure_rate: float + reason_counts: Mapping[str, int] + summary_id: str = "" + schema_version: str = ENSEMBLE_OUTCOME_SUMMARY_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_OUTCOME_SUMMARY_SCHEMA_VERSION, + "ensemble outcome summary", + ) + object.__setattr__( + self, + "split_kind", + BenchmarkSplitKind.from_value(self.split_kind), + ) + if not isinstance(self.stratum, EnsembleCalibrationStratumV1): + raise ValueError("outcome summary requires a v1 stratum") + for name in ( + "attempt_count", + "completed_count", + "refused_count", + "failed_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + if self.attempt_count != ( + self.completed_count + self.refused_count + self.failed_count + ): + raise ValueError("ensemble outcome counts do not reconcile") + for name in ("completion_rate", "refusal_rate", "failure_rate"): + object.__setattr__( + self, name, _unit_float(getattr(self, name), name) + ) + reasons = _count_mapping(self.reason_counts, "outcome reason") + if len(reasons) > MAX_ENSEMBLE_REASON_CODES: + raise ValueError("ensemble outcome reason count exceeds limit") + if sum(reasons.values()) != self.refused_count + self.failed_count: + raise ValueError("ensemble outcome reasons do not reconcile") + if self.attempt_count: + expected_rates = ( + self.completed_count / self.attempt_count, + self.refused_count / self.attempt_count, + self.failed_count / self.attempt_count, + ) + else: + expected_rates = (0.0, 0.0, 0.0) + actual_rates = ( + self.completion_rate, + self.refusal_rate, + self.failure_rate, + ) + if any( + not math.isclose(actual, expected, rel_tol=0.0, abs_tol=1e-15) + for actual, expected in zip(actual_rates, expected_rates) + ): + raise ValueError("ensemble outcome rates do not reconcile") + object.__setattr__(self, "reason_counts", reasons) + expected = _stable_id("ensemble-outcome", self.payload()) + supplied = _optional_text(self.summary_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble outcome summary_id differs") + object.__setattr__(self, "summary_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "split_kind": self.split_kind.value, + "stratum": self.stratum.to_dict(), + "attempt_count": self.attempt_count, + "completed_count": self.completed_count, + "refused_count": self.refused_count, + "failed_count": self.failed_count, + "completion_rate": self.completion_rate, + "refusal_rate": self.refusal_rate, + "failure_rate": self.failure_rate, + "reason_counts": dict(self.reason_counts), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "summary_id": self.summary_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "EnsembleOutcomeSummaryV1": + _require_schema(data, ENSEMBLE_OUTCOME_SUMMARY_SCHEMA_VERSION) + return cls( + split_kind=BenchmarkSplitKind.from_value( + str(data.get("split_kind", "")) + ), + stratum=EnsembleCalibrationStratumV1.from_dict( + _mapping(data.get("stratum")) + ), + attempt_count=_strict_int( + data.get("attempt_count"), "attempt_count" + ), + completed_count=_strict_int( + data.get("completed_count"), "completed_count" + ), + refused_count=_strict_int( + data.get("refused_count"), "refused_count" + ), + failed_count=_strict_int(data.get("failed_count"), "failed_count"), + completion_rate=_finite_float( + data.get("completion_rate"), "completion_rate" + ), + refusal_rate=_finite_float( + data.get("refusal_rate"), "refusal_rate" + ), + failure_rate=_finite_float( + data.get("failure_rate"), "failure_rate" + ), + reason_counts={ + str(key): _strict_int(value, "outcome reason count") + for key, value in _mapping(data.get("reason_counts")).items() + }, + summary_id=str(data.get("summary_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleMemberSelectionV1: + """Representative-member evidence; never a claim of historical truth.""" + + member_id: str + selection_rank: int + representative_distance: float | None + completed_count: int + refused_count: int + failed_count: int + primary: bool + retained: bool + selection_id: str = "" + schema_version: str = ENSEMBLE_MEMBER_SELECTION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_MEMBER_SELECTION_SCHEMA_VERSION, + "ensemble member selection", + ) + object.__setattr__(self, "member_id", _required_text(self.member_id)) + object.__setattr__( + self, + "selection_rank", + _positive_int(self.selection_rank, "selection_rank"), + ) + object.__setattr__( + self, + "representative_distance", + _optional_nonnegative_float( + self.representative_distance, + "representative_distance", + ), + ) + for name in ("completed_count", "refused_count", "failed_count"): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + object.__setattr__( + self, "primary", _strict_bool(self.primary, "primary") + ) + object.__setattr__( + self, "retained", _strict_bool(self.retained, "retained") + ) + if self.primary and not self.retained: + raise ValueError("primary ensemble member must be retained") + if self.primary and self.representative_distance is None: + raise ValueError("primary member requires representative evidence") + expected = _stable_id("ensemble-member-selection", self.payload()) + supplied = _optional_text(self.selection_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble member selection_id differs") + object.__setattr__(self, "selection_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "member_id": self.member_id, + "selection_rank": self.selection_rank, + "representative_distance": self.representative_distance, + "completed_count": self.completed_count, + "refused_count": self.refused_count, + "failed_count": self.failed_count, + "primary": self.primary, + "retained": self.retained, + "selection_basis": ENSEMBLE_PRIMARY_SELECTION_BASIS, + "interpretation": "representative_member_not_historical_truth", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "selection_id": self.selection_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "EnsembleMemberSelectionV1": + _require_schema(data, ENSEMBLE_MEMBER_SELECTION_SCHEMA_VERSION) + _require_derived_value( + data, "selection_basis", ENSEMBLE_PRIMARY_SELECTION_BASIS + ) + _require_derived_value( + data, + "interpretation", + "representative_member_not_historical_truth", + ) + return cls( + member_id=str(data.get("member_id", "")), + selection_rank=_strict_int( + data.get("selection_rank"), "selection_rank" + ), + representative_distance=_optional_float( + data.get("representative_distance") + ), + completed_count=_strict_int( + data.get("completed_count"), "completed_count" + ), + refused_count=_strict_int( + data.get("refused_count"), "refused_count" + ), + failed_count=_strict_int(data.get("failed_count"), "failed_count"), + primary=_strict_bool(data.get("primary"), "primary"), + retained=_strict_bool(data.get("retained"), "retained"), + selection_id=str(data.get("selection_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class EnsembleCalibrationReportV1: + """Bounded calibrated uncertainty and retention report for one plan.""" + + run_id: str + plan_id: str + config_id: str + benchmark_manifest_id: str + candidate_id: str + storage_estimate_id: str + retained_member_count: int + status: EnsembleReportStatus + primary_member_id: str | None + retained_member_ids: tuple[str, ...] + regenerable_member_ids: tuple[str, ...] + member_selections: tuple[EnsembleMemberSelectionV1, ...] + metric_calibrations: tuple[EnsembleMetricCalibrationV1, ...] + diversity_summaries: tuple[EnsembleDiversitySummaryV1, ...] + outcome_summaries: tuple[EnsembleOutcomeSummaryV1, ...] + fit_sample_count: int + evaluation_sample_count: int + automatic_winner: bool = False + default_generator_id: str | None = None + report_id: str = "" + schema_version: str = ENSEMBLE_CALIBRATION_REPORT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_CALIBRATION_REPORT_SCHEMA_VERSION, + "ensemble calibration report", + ) + for name in ( + "run_id", + "plan_id", + "config_id", + "benchmark_manifest_id", + "candidate_id", + "storage_estimate_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__(self, "status", EnsembleReportStatus(self.status)) + object.__setattr__( + self, + "retained_member_count", + _positive_int(self.retained_member_count, "retained_member_count"), + ) + primary = _optional_text(self.primary_member_id) + retained = _normalized_text_tuple( + self.retained_member_ids, allow_empty=True + ) + regenerable = _normalized_text_tuple( + self.regenerable_member_ids, allow_empty=True + ) + selections = tuple( + sorted(self.member_selections, key=lambda item: item.selection_rank) + ) + if not selections or tuple( + item.selection_rank for item in selections + ) != tuple(range(1, len(selections) + 1)): + raise ValueError("ensemble member selection ranks are incomplete") + selection_ids = {item.member_id for item in selections} + if set(retained) | set(regenerable) != selection_ids or ( + set(retained) & set(regenerable) + ): + raise ValueError("ensemble retention sets do not partition members") + declared_retained = { + item.member_id for item in selections if item.retained + } + declared_primary = [ + item.member_id for item in selections if item.primary + ] + if declared_retained != set(retained): + raise ValueError("ensemble retained selections differ") + if self.retained_member_count > len(selections): + raise ValueError("ensemble retained-member count exceeds members") + if primary is None: + if declared_primary: + raise ValueError("ensemble primary selection differs") + elif declared_primary != [primary] or primary not in retained: + raise ValueError("ensemble primary member differs") + if self.status is EnsembleReportStatus.CALIBRATED and primary is None: + raise ValueError("calibrated ensemble requires a primary member") + object.__setattr__(self, "primary_member_id", primary) + object.__setattr__(self, "retained_member_ids", retained) + object.__setattr__(self, "regenerable_member_ids", regenerable) + object.__setattr__(self, "member_selections", selections) + metrics = tuple( + sorted( + self.metric_calibrations, + key=lambda item: (item.stratum.stratum_id, item.metric_name), + ) + ) + diversity = tuple( + sorted( + self.diversity_summaries, + key=lambda item: ( + item.split_kind.value, + item.stratum.stratum_id, + ), + ) + ) + outcomes = tuple( + sorted( + self.outcome_summaries, + key=lambda item: ( + item.split_kind.value, + item.stratum.stratum_id, + ), + ) + ) + if not metrics or not diversity or not outcomes: + raise ValueError( + "ensemble report requires metric/diversity/outcome evidence" + ) + metric_keys = [ + (item.stratum.stratum_id, item.metric_name) for item in metrics + ] + diversity_keys = [ + (item.split_kind, item.stratum.stratum_id) for item in diversity + ] + outcome_keys = [ + (item.split_kind, item.stratum.stratum_id) for item in outcomes + ] + if len(set(metric_keys)) != len(metric_keys): + raise ValueError("ensemble metric summaries are duplicated") + if len(set(diversity_keys)) != len(diversity_keys): + raise ValueError("ensemble diversity summaries are duplicated") + if len(set(outcome_keys)) != len(outcome_keys): + raise ValueError("ensemble outcome summaries are duplicated") + if set(diversity_keys) != set(outcome_keys): + raise ValueError("ensemble diversity/outcome cells differ") + stratum_ids = {item.stratum.stratum_id for item in diversity} + expected_metric_keys = { + (stratum_id, metric_name) + for stratum_id in stratum_ids + for metric_name in ENSEMBLE_CALIBRATION_METRIC_NAMES + } + if set(metric_keys) != expected_metric_keys: + raise ValueError("ensemble metric cells are incomplete") + outcome_by_key = { + (item.split_kind, item.stratum.stratum_id): item + for item in outcomes + } + for item in diversity: + outcome = outcome_by_key[(item.split_kind, item.stratum.stratum_id)] + if outcome.attempt_count != item.sample_count * len(selections): + raise ValueError("ensemble sample/outcome counts differ") + object.__setattr__(self, "metric_calibrations", metrics) + object.__setattr__(self, "diversity_summaries", diversity) + object.__setattr__(self, "outcome_summaries", outcomes) + for name in ("fit_sample_count", "evaluation_sample_count"): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + expected_fit = sum( + item.sample_count + for item in diversity + if item.split_kind is ENSEMBLE_CALIBRATION_FIT_SPLIT + ) + expected_evaluation = sum( + item.sample_count + for item in diversity + if item.split_kind is ENSEMBLE_CALIBRATION_EVALUATION_SPLIT + ) + if ( + self.fit_sample_count != expected_fit + or self.evaluation_sample_count != expected_evaluation + ): + raise ValueError("ensemble report sample counts differ") + expected_status = _report_status_from_evidence( + metrics, + diversity, + primary, + len(retained), + self.retained_member_count, + ) + if self.status is not expected_status: + raise ValueError("ensemble report status differs from evidence") + if _strict_bool(self.automatic_winner, "automatic_winner"): + raise ValueError( + "ensemble report cannot select an automatic winner" + ) + if self.default_generator_id is not None: + raise ValueError( + "ensemble report cannot select a default generator" + ) + object.__setattr__(self, "automatic_winner", False) + object.__setattr__(self, "default_generator_id", None) + expected = _stable_id("ensemble-calibration-report", self.payload()) + supplied = _optional_text(self.report_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble calibration report_id differs") + object.__setattr__(self, "report_id", expected) + + @property + def calibrated(self) -> bool: + return self.status is EnsembleReportStatus.CALIBRATED + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "engine_id": ENSEMBLE_ENGINE_ID, + "engine_version": ENSEMBLE_ENGINE_VERSION, + "run_id": self.run_id, + "plan_id": self.plan_id, + "config_id": self.config_id, + "benchmark_manifest_id": self.benchmark_manifest_id, + "candidate_id": self.candidate_id, + "storage_estimate_id": self.storage_estimate_id, + "status": self.status.value, + "confidence_quantity": ENSEMBLE_CONFIDENCE_QUANTITY, + "confidence_scope": "stratum_metric_horizon_summary_not_per_event", + "primary_member_id": self.primary_member_id, + "primary_interpretation": ( + "representative_member_not_historical_truth" + ), + "retained_member_ids": list(self.retained_member_ids), + "retained_member_count": self.retained_member_count, + "regenerable_member_ids": list(self.regenerable_member_ids), + "member_selections": [ + item.to_dict() for item in self.member_selections + ], + "metric_calibrations": [ + item.to_dict() for item in self.metric_calibrations + ], + "diversity_summaries": [ + item.to_dict() for item in self.diversity_summaries + ], + "outcome_summaries": [ + item.to_dict() for item in self.outcome_summaries + ], + "fit_sample_count": self.fit_sample_count, + "evaluation_sample_count": self.evaluation_sample_count, + "automatic_winner": False, + "winner_member_id": None, + "default_generator_id": None, + "event_rows_inline": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "report_id": self.report_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EnsembleCalibrationReportV1": + _require_schema(data, ENSEMBLE_CALIBRATION_REPORT_SCHEMA_VERSION) + _require_derived_value( + data, "confidence_quantity", ENSEMBLE_CONFIDENCE_QUANTITY + ) + _require_derived_value( + data, + "confidence_scope", + "stratum_metric_horizon_summary_not_per_event", + ) + _require_derived_value( + data, + "primary_interpretation", + "representative_member_not_historical_truth", + ) + _require_derived_value(data, "winner_member_id", None) + _require_derived_value(data, "event_rows_inline", False) + return cls( + run_id=str(data.get("run_id", "")), + plan_id=str(data.get("plan_id", "")), + config_id=str(data.get("config_id", "")), + benchmark_manifest_id=str(data.get("benchmark_manifest_id", "")), + candidate_id=str(data.get("candidate_id", "")), + storage_estimate_id=str(data.get("storage_estimate_id", "")), + retained_member_count=_strict_int( + data.get("retained_member_count"), "retained_member_count" + ), + status=EnsembleReportStatus(str(data.get("status", ""))), + primary_member_id=_optional_text(data.get("primary_member_id")), + retained_member_ids=_string_tuple( + data.get("retained_member_ids"), "retained_member_ids" + ), + regenerable_member_ids=_string_tuple( + data.get("regenerable_member_ids"), + "regenerable_member_ids", + ), + member_selections=tuple( + EnsembleMemberSelectionV1.from_dict(item) + for item in _mapping_sequence(data, "member_selections") + ), + metric_calibrations=tuple( + EnsembleMetricCalibrationV1.from_dict(item) + for item in _mapping_sequence(data, "metric_calibrations") + ), + diversity_summaries=tuple( + EnsembleDiversitySummaryV1.from_dict(item) + for item in _mapping_sequence(data, "diversity_summaries") + ), + outcome_summaries=tuple( + EnsembleOutcomeSummaryV1.from_dict(item) + for item in _mapping_sequence(data, "outcome_summaries") + ), + fit_sample_count=_strict_int( + data.get("fit_sample_count"), "fit_sample_count" + ), + evaluation_sample_count=_strict_int( + data.get("evaluation_sample_count"), + "evaluation_sample_count", + ), + automatic_winner=_strict_bool( + data.get("automatic_winner", False), "automatic_winner" + ), + default_generator_id=_optional_text( + data.get("default_generator_id") + ), + report_id=str(data.get("report_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EnsembleCalibrationReportV1": + return cls.from_dict(_json_mapping(text)) + + +def calibrate_reconstruction_ensemble( + plan: ReconstructionEnsemblePlanV1, + *, + samples: Sequence[EnsembleCalibrationSampleV1], + storage_estimate: EnsembleStorageEstimateV1, +) -> EnsembleCalibrationReportV1: + """Fit validation intervals and evaluate them only on final holdouts.""" + if not isinstance(plan, ReconstructionEnsemblePlanV1): + raise ValueError("ensemble calibration requires a v1 plan") + if not isinstance(storage_estimate, EnsembleStorageEstimateV1): + raise ValueError("ensemble calibration requires a storage estimate") + if ( + storage_estimate.plan_id != plan.plan_id + or storage_estimate.run_id != plan.run.run_id + or set(storage_estimate.member_event_counts) + != {item.member_id for item in plan.members} + or storage_estimate.retained_member_count + != plan.config.retained_member_count + ): + raise ValueError("ensemble storage estimate differs from plan") + values = tuple(sorted(samples, key=lambda item: item.sample_id)) + if not values or len(values) > plan.config.max_samples: + raise ValueError("ensemble sample count is outside configured limits") + if len({item.sample_id for item in values}) != len(values): + raise ValueError("ensemble calibration samples are duplicated") + benchmark_ids = {item.benchmark_manifest_id for item in values} + candidate_ids = {item.candidate_id for item in values} + if len(benchmark_ids) != 1 or len(candidate_ids) != 1: + raise ValueError( + "ensemble calibration mixes benchmark/candidate identity" + ) + expected_members = {item.member_id for item in plan.members} + for sample in values: + if {item.member_id for item in sample.members} != expected_members: + raise ValueError("ensemble sample does not cover planned members") + if sample.stratum.horizon_ns not in plan.config.horizons_ns: + raise ValueError("ensemble sample horizon differs from config") + if {item.stratum.horizon_ns for item in values} != set( + plan.config.horizons_ns + ): + raise ValueError("ensemble samples do not cover configured horizons") + strata = {item.stratum.stratum_id: item.stratum for item in values} + if len(strata) > plan.config.max_slices: + raise ValueError("ensemble calibration slice count exceeds config") + grouped: dict[ + tuple[BenchmarkSplitKind, str], list[EnsembleCalibrationSampleV1] + ] = {} + for sample in values: + grouped.setdefault( + (sample.split_kind, sample.stratum.stratum_id), [] + ).append(sample) + metric_summaries = tuple( + _calibrate_metric_cell( + plan.config, + stratum, + metric_name, + grouped.get((ENSEMBLE_CALIBRATION_FIT_SPLIT, stratum_id), []), + grouped.get( + (ENSEMBLE_CALIBRATION_EVALUATION_SPLIT, stratum_id), [] + ), + ) + for stratum_id, stratum in sorted(strata.items()) + for metric_name in ENSEMBLE_CALIBRATION_METRIC_NAMES + ) + diversity = tuple( + _diversity_summary(plan.config, split, stratum, selected) + for (split, stratum_id), selected in sorted( + grouped.items(), key=lambda item: (item[0][0].value, item[0][1]) + ) + for stratum in (strata[stratum_id],) + ) + outcomes = tuple( + _outcome_summary(split, stratum, selected) + for (split, stratum_id), selected in sorted( + grouped.items(), key=lambda item: (item[0][0].value, item[0][1]) + ) + for stratum in (strata[stratum_id],) + ) + selections, primary, retained = _member_selections(plan, values) + regenerable = tuple(sorted(expected_members.difference(retained))) + status = _report_status( + plan.config, + metric_summaries, + diversity, + primary, + retained, + ) + report = EnsembleCalibrationReportV1( + run_id=plan.run.run_id, + plan_id=plan.plan_id, + config_id=plan.config.config_id, + benchmark_manifest_id=next(iter(benchmark_ids)), + candidate_id=next(iter(candidate_ids)), + storage_estimate_id=storage_estimate.estimate_id, + retained_member_count=plan.config.retained_member_count, + status=status, + primary_member_id=primary, + retained_member_ids=retained, + regenerable_member_ids=regenerable, + member_selections=selections, + metric_calibrations=metric_summaries, + diversity_summaries=diversity, + outcome_summaries=outcomes, + fit_sample_count=sum( + item.split_kind is ENSEMBLE_CALIBRATION_FIT_SPLIT for item in values + ), + evaluation_sample_count=sum( + item.split_kind is ENSEMBLE_CALIBRATION_EVALUATION_SPLIT + for item in values + ), + ) + _ensure_payload_size(report.to_dict(), plan.config.max_payload_bytes) + return report + + +@dataclass(frozen=True, slots=True) +class EnsembleRegenerationRequestV1: + """Hash-bound request for deterministic regeneration of omitted members.""" + + plan_id: str + report_id: str + member_ids: tuple[str, ...] + source_artifact_hashes: Mapping[str, str] + configuration_artifact_hashes: Mapping[str, str] + request_id: str = "" + schema_version: str = ENSEMBLE_REGENERATION_REQUEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + ENSEMBLE_REGENERATION_REQUEST_SCHEMA_VERSION, + "ensemble regeneration request", + ) + object.__setattr__(self, "plan_id", _required_text(self.plan_id)) + object.__setattr__(self, "report_id", _required_text(self.report_id)) + members = _normalized_text_tuple(self.member_ids) + if len(members) > MAX_BENCHMARK_ENSEMBLE_MEMBERS: + raise ValueError("regeneration member count exceeds limit") + object.__setattr__(self, "member_ids", members) + sources = _hash_mapping(self.source_artifact_hashes, "source artifact") + configurations = _hash_mapping( + self.configuration_artifact_hashes, + "configuration artifact", + ) + if not sources or not configurations: + raise ValueError( + "regeneration request requires source/config hashes" + ) + object.__setattr__(self, "source_artifact_hashes", sources) + object.__setattr__( + self, "configuration_artifact_hashes", configurations + ) + expected = _stable_id("ensemble-regeneration", self.payload()) + supplied = _optional_text(self.request_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble regeneration request_id differs") + object.__setattr__(self, "request_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "plan_id": self.plan_id, + "report_id": self.report_id, + "member_ids": list(self.member_ids), + "source_artifact_hashes": dict(self.source_artifact_hashes), + "configuration_artifact_hashes": dict( + self.configuration_artifact_hashes + ), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "request_id": self.request_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EnsembleRegenerationRequestV1": + _require_schema(data, ENSEMBLE_REGENERATION_REQUEST_SCHEMA_VERSION) + return cls( + plan_id=str(data.get("plan_id", "")), + report_id=str(data.get("report_id", "")), + member_ids=_string_tuple(data.get("member_ids"), "member_ids"), + source_artifact_hashes={ + str(key): str(value) + for key, value in _mapping( + data.get("source_artifact_hashes") + ).items() + }, + configuration_artifact_hashes={ + str(key): str(value) + for key, value in _mapping( + data.get("configuration_artifact_hashes") + ).items() + }, + request_id=str(data.get("request_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EnsembleRegenerationRequestV1": + return cls.from_dict(_json_mapping(text)) + + +def build_ensemble_regeneration_request( + plan: ReconstructionEnsemblePlanV1, + report: EnsembleCalibrationReportV1, + *, + member_ids: Iterable[str], +) -> EnsembleRegenerationRequestV1: + """Build a request using the exact hashes frozen by the ensemble plan.""" + return EnsembleRegenerationRequestV1( + plan_id=plan.plan_id, + report_id=report.report_id, + member_ids=tuple(member_ids), + source_artifact_hashes=plan.source_hashes, + configuration_artifact_hashes=plan.configuration_hashes, + ) + + +def verify_ensemble_regeneration( + plan: ReconstructionEnsemblePlanV1, + report: EnsembleCalibrationReportV1, + request: EnsembleRegenerationRequestV1, + *, + available_source_artifact_hashes: Mapping[str, str], + available_configuration_artifact_hashes: Mapping[str, str], +) -> tuple[EnsembleMemberPlanV1, ...]: + """Fail closed unless plan/report/member scope and every hash still match.""" + if not report.calibrated: + raise ValueError("only calibrated ensemble reports may regenerate") + if ( + request.plan_id != plan.plan_id + or report.plan_id != plan.plan_id + or request.report_id != report.report_id + or report.run_id != plan.run.run_id + or report.config_id != plan.config.config_id + ): + raise ValueError("ensemble regeneration scope differs") + if request.source_artifact_hashes != plan.source_hashes: + raise ValueError("ensemble regeneration source hashes differ") + if request.configuration_artifact_hashes != plan.configuration_hashes: + raise ValueError("ensemble regeneration config hashes differ") + available_sources = _hash_mapping( + available_source_artifact_hashes, "available source artifact" + ) + available_configurations = _hash_mapping( + available_configuration_artifact_hashes, + "available configuration artifact", + ) + if available_sources != plan.source_hashes: + raise ValueError("available regeneration source hashes differ") + if available_configurations != plan.configuration_hashes: + raise ValueError("available regeneration config hashes differ") + allowed = set(report.regenerable_member_ids) + if not set(request.member_ids).issubset(allowed): + raise ValueError( + "regeneration request contains retained/unknown members" + ) + return tuple(plan.member(member_id) for member_id in request.member_ids) + + +def _calibrate_metric_cell( + config: EnsembleCalibrationConfigV1, + stratum: EnsembleCalibrationStratumV1, + metric_name: str, + fit_samples: Sequence[EnsembleCalibrationSampleV1], + evaluation_samples: Sequence[EnsembleCalibrationSampleV1], +) -> EnsembleMetricCalibrationV1: + fit_nonconformity: list[float] = [] + for sample in fit_samples: + interval = _sample_interval( + sample, metric_name, config.nominal_coverage + ) + if interval is None: + continue + lower, upper, _ = interval + reference = sample.reference_metrics[metric_name] + fit_nonconformity.append(max(lower - reference, reference - upper, 0.0)) + if len(fit_nonconformity) < config.minimum_fit_samples: + return EnsembleMetricCalibrationV1( + stratum=stratum, + metric_name=metric_name, + metric_group=ENSEMBLE_CALIBRATION_METRIC_GROUPS[metric_name], + nominal_coverage=config.nominal_coverage, + minimum_achieved_coverage=config.minimum_achieved_coverage, + fit_sample_count=len(fit_nonconformity), + evaluation_sample_count=0, + calibration_adjustment=None, + raw_covered_count=0, + calibrated_covered_count=0, + raw_coverage_rate=None, + calibrated_coverage_rate=None, + mean_raw_interval_width=None, + mean_calibrated_interval_width=None, + mean_absolute_median_error=None, + status=EnsembleMetricStatus.INSUFFICIENT_SUPPORT, + ) + adjustment = _conformal_adjustment( + fit_nonconformity, config.nominal_coverage + ) + raw_covered = 0 + calibrated_covered = 0 + raw_widths: list[float] = [] + calibrated_widths: list[float] = [] + median_errors: list[float] = [] + for sample in evaluation_samples: + interval = _sample_interval( + sample, metric_name, config.nominal_coverage + ) + if interval is None: + continue + lower, upper, median = interval + reference = sample.reference_metrics[metric_name] + raw_covered += int(lower <= reference <= upper) + adjusted_lower = lower - adjustment + adjusted_upper = upper + adjustment + calibrated_covered += int(adjusted_lower <= reference <= adjusted_upper) + raw_widths.append(upper - lower) + calibrated_widths.append(adjusted_upper - adjusted_lower) + median_errors.append(abs(median - reference)) + support = len(raw_widths) + if support == 0: + return EnsembleMetricCalibrationV1( + stratum=stratum, + metric_name=metric_name, + metric_group=ENSEMBLE_CALIBRATION_METRIC_GROUPS[metric_name], + nominal_coverage=config.nominal_coverage, + minimum_achieved_coverage=config.minimum_achieved_coverage, + fit_sample_count=len(fit_nonconformity), + evaluation_sample_count=0, + calibration_adjustment=None, + raw_covered_count=0, + calibrated_covered_count=0, + raw_coverage_rate=None, + calibrated_coverage_rate=None, + mean_raw_interval_width=None, + mean_calibrated_interval_width=None, + mean_absolute_median_error=None, + status=EnsembleMetricStatus.INSUFFICIENT_SUPPORT, + ) + raw_rate = raw_covered / support + calibrated_rate = calibrated_covered / support + status = ( + EnsembleMetricStatus.CALIBRATED + if calibrated_rate >= config.minimum_achieved_coverage + else EnsembleMetricStatus.MISCALIBRATED + ) + return EnsembleMetricCalibrationV1( + stratum=stratum, + metric_name=metric_name, + metric_group=ENSEMBLE_CALIBRATION_METRIC_GROUPS[metric_name], + nominal_coverage=config.nominal_coverage, + minimum_achieved_coverage=config.minimum_achieved_coverage, + fit_sample_count=len(fit_nonconformity), + evaluation_sample_count=support, + calibration_adjustment=adjustment, + raw_covered_count=raw_covered, + calibrated_covered_count=calibrated_covered, + raw_coverage_rate=raw_rate, + calibrated_coverage_rate=calibrated_rate, + mean_raw_interval_width=_rounded( + sum(raw_widths) / support, config.rounding_digits + ), + mean_calibrated_interval_width=_rounded( + sum(calibrated_widths) / support, config.rounding_digits + ), + mean_absolute_median_error=_rounded( + sum(median_errors) / support, config.rounding_digits + ), + status=status, + ) + + +def _sample_interval( + sample: EnsembleCalibrationSampleV1, + metric_name: str, + nominal_coverage: float, +) -> tuple[float, float, float] | None: + values = sorted( + item.metrics[metric_name] + for item in sample.members + if item.status is EnsembleMemberStatus.COMPLETED + ) + if len(values) < 2: + return None + alpha = (1.0 - nominal_coverage) / 2.0 + lower_index = math.floor(alpha * (len(values) - 1)) + upper_index = math.ceil((1.0 - alpha) * (len(values) - 1)) + return values[lower_index], values[upper_index], _median(values) + + +def _conformal_adjustment(values: Sequence[float], coverage: float) -> float: + ordered = sorted(values) + rank = math.ceil((len(ordered) + 1) * coverage) - 1 + return ordered[min(len(ordered) - 1, max(0, rank))] + + +def _diversity_summary( + config: EnsembleCalibrationConfigV1, + split: BenchmarkSplitKind, + stratum: EnsembleCalibrationStratumV1, + samples: Sequence[EnsembleCalibrationSampleV1], +) -> EnsembleDiversitySummaryV1: + distances: list[float] = [] + collapsed = 0 + false_diversity = 0 + distinct_total = 0 + for sample in samples: + completed = tuple( + item + for item in sample.members + if item.status is EnsembleMemberStatus.COMPLETED + ) + distinct_total += len( + {item.logical_content_sha256 for item in completed} + ) + for left, right in combinations(completed, 2): + distance = _normalized_metric_distance(left.metrics, right.metrics) + distances.append(distance) + if left.logical_content_sha256 == right.logical_content_sha256: + collapsed += 1 + elif distance <= config.logical_distance_tolerance: + false_diversity += 1 + pair_count = len(distances) + if not pair_count: + status = EnsembleDiversityStatus.INSUFFICIENT_SUPPORT + collapse_rate = None + false_rate = None + mean_distance = None + else: + collapse_rate = collapsed / pair_count + false_rate = false_diversity / pair_count + mean_distance = sum(distances) / pair_count + if collapse_rate > config.maximum_collapse_rate: + status = EnsembleDiversityStatus.COLLAPSED + elif false_rate > config.maximum_false_diversity_rate: + status = EnsembleDiversityStatus.FALSE_DIVERSITY + else: + status = EnsembleDiversityStatus.DIVERSE + return EnsembleDiversitySummaryV1( + split_kind=split, + stratum=stratum, + sample_count=len(samples), + pair_count=pair_count, + collapsed_pair_count=collapsed, + false_diversity_pair_count=false_diversity, + distinct_content_count=distinct_total, + mean_normalized_metric_distance=( + _rounded(mean_distance, config.rounding_digits) + if mean_distance is not None + else None + ), + collapse_rate=collapse_rate, + false_diversity_rate=false_rate, + status=status, + ) + + +def _outcome_summary( + split: BenchmarkSplitKind, + stratum: EnsembleCalibrationStratumV1, + samples: Sequence[EnsembleCalibrationSampleV1], +) -> EnsembleOutcomeSummaryV1: + results = tuple(item for sample in samples for item in sample.members) + counts = Counter(item.status for item in results) + reasons = Counter( + item.reason for item in results if item.reason is not None + ) + attempts = len(results) + completed = counts[EnsembleMemberStatus.COMPLETED] + refused = counts[EnsembleMemberStatus.REFUSED] + failed = counts[EnsembleMemberStatus.FAILED] + return EnsembleOutcomeSummaryV1( + split_kind=split, + stratum=stratum, + attempt_count=attempts, + completed_count=completed, + refused_count=refused, + failed_count=failed, + completion_rate=completed / attempts if attempts else 0.0, + refusal_rate=refused / attempts if attempts else 0.0, + failure_rate=failed / attempts if attempts else 0.0, + reason_counts={str(reason): count for reason, count in reasons.items()}, + ) + + +def _member_selections( + plan: ReconstructionEnsemblePlanV1, + samples: Sequence[EnsembleCalibrationSampleV1], +) -> tuple[ + tuple[EnsembleMemberSelectionV1, ...], + str | None, + tuple[str, ...], +]: + fit_samples = tuple( + item + for item in samples + if item.split_kind is ENSEMBLE_CALIBRATION_FIT_SPLIT + ) + distances: dict[str, list[float]] = { + item.member_id: [] for item in plan.members + } + counts: dict[str, Counter[EnsembleMemberStatus]] = { + item.member_id: Counter() for item in plan.members + } + for sample in fit_samples: + completed = tuple( + item + for item in sample.members + if item.status is EnsembleMemberStatus.COMPLETED + ) + medians = ( + { + metric: _median( + sorted(item.metrics[metric] for item in completed) + ) + for metric in ENSEMBLE_CALIBRATION_METRIC_NAMES + } + if completed + else {} + ) + for member in sample.members: + counts[member.member_id][member.status] += 1 + if member.status is EnsembleMemberStatus.COMPLETED: + distances[member.member_id].append( + _normalized_metric_distance(member.metrics, medians) + ) + scored: list[tuple[float | None, str]] = [] + total_fit = len(fit_samples) + for plan_member in plan.members: + values = distances[plan_member.member_id] + failures = ( + counts[plan_member.member_id][EnsembleMemberStatus.REFUSED] + + counts[plan_member.member_id][EnsembleMemberStatus.FAILED] + ) + score = None + if values: + score = sum(values) / len(values) + if total_fit: + score += plan.config.failure_penalty * failures / total_fit + scored.append((score, plan_member.member_id)) + ordered = sorted( + scored, + key=lambda item: ( + item[0] is None, + item[0] if item[0] is not None else math.inf, + item[1], + ), + ) + eligible = [item for item in ordered if item[0] is not None] + primary = eligible[0][1] if eligible else None + retained = tuple( + item[1] for item in eligible[: plan.config.retained_member_count] + ) + rank_by_member = { + member_id: rank for rank, (_, member_id) in enumerate(ordered, start=1) + } + score_by_member = {member_id: score for score, member_id in ordered} + selections: list[EnsembleMemberSelectionV1] = [] + for _, member_id in ordered: + raw_score = score_by_member[member_id] + selections.append( + EnsembleMemberSelectionV1( + member_id=member_id, + selection_rank=rank_by_member[member_id], + representative_distance=( + _rounded(raw_score, plan.config.rounding_digits) + if raw_score is not None + else None + ), + completed_count=counts[member_id][ + EnsembleMemberStatus.COMPLETED + ], + refused_count=counts[member_id][EnsembleMemberStatus.REFUSED], + failed_count=counts[member_id][EnsembleMemberStatus.FAILED], + primary=member_id == primary, + retained=member_id in retained, + ) + ) + return tuple(selections), primary, tuple(sorted(retained)) + + +def _report_status( + config: EnsembleCalibrationConfigV1, + metrics: Sequence[EnsembleMetricCalibrationV1], + diversity: Sequence[EnsembleDiversitySummaryV1], + primary: str | None, + retained: Sequence[str], +) -> EnsembleReportStatus: + return _report_status_from_evidence( + metrics, + diversity, + primary, + len(retained), + config.retained_member_count, + ) + + +def _report_status_from_evidence( + metrics: Sequence[EnsembleMetricCalibrationV1], + diversity: Sequence[EnsembleDiversitySummaryV1], + primary: str | None, + retained_count: int, + required_retained_count: int, +) -> EnsembleReportStatus: + if ( + primary is None + or retained_count != required_retained_count + or any( + item.status is EnsembleMetricStatus.INSUFFICIENT_SUPPORT + for item in metrics + ) + or any( + item.status is EnsembleDiversityStatus.INSUFFICIENT_SUPPORT + for item in diversity + ) + ): + return EnsembleReportStatus.INSUFFICIENT_SUPPORT + if any( + item.status is EnsembleMetricStatus.MISCALIBRATED for item in metrics + ) or any( + item.status + in { + EnsembleDiversityStatus.COLLAPSED, + EnsembleDiversityStatus.FALSE_DIVERSITY, + } + for item in diversity + ): + return EnsembleReportStatus.MISCALIBRATED + return EnsembleReportStatus.CALIBRATED + + +def _benchmark_metrics( + events: Sequence[BenchmarkEventV1], downstream_sensitivity: float +) -> dict[str, float]: + ordered = _validated_benchmark_events(events) + if not ordered: + raise ValueError("benchmark metrics require events") + interarrivals = [ + right.event_time_ns - left.event_time_ns + for left, right in zip(ordered, ordered[1:]) + ] + mids = [item.mid for item in ordered] + return { + "event_count": float(len(ordered)), + "observed_duration_ns": float( + ordered[-1].event_time_ns - ordered[0].event_time_ns + ), + "mean_interarrival_ns": ( + sum(interarrivals) / len(interarrivals) if interarrivals else 0.0 + ), + "mean_spread": sum(item.spread for item in ordered) / len(ordered), + "mid_path_range": max(mids) - min(mids), + "endpoint_mid": mids[-1], + "downstream_sensitivity": _finite_float( + downstream_sensitivity, "downstream_sensitivity" + ), + } + + +def _validated_benchmark_events( + values: Iterable[BenchmarkEventV1], +) -> tuple[BenchmarkEventV1, ...]: + events = tuple(values) + if any(not isinstance(item, BenchmarkEventV1) for item in events): + raise ValueError("ensemble calibration requires benchmark v1 events") + return tuple( + sorted( + events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.benchmark_event_id, + ), + ) + ) + + +def _normalized_metric_distance( + left: Mapping[str, float], right: Mapping[str, float] +) -> float: + names = ENSEMBLE_CALIBRATION_METRIC_NAMES + return sum( + abs(left[name] - right[name]) + / max(abs(left[name]), abs(right[name]), 1e-15) + for name in names + ) / len(names) + + +def _derive_member_id( + config: EnsembleCalibrationConfigV1, + sources: Sequence[EnsembleArtifactDigestV1], + configurations: Sequence[EnsembleArtifactDigestV1], + base_seed: int, + ordinal: int, +) -> str: + return _stable_id( + "ensemble-member", + { + "member_contract": ENSEMBLE_MEMBER_PLAN_SCHEMA_VERSION, + "config_id": config.config_id, + "source_artifacts": [item.to_dict() for item in sources], + "configuration_artifacts": [ + item.to_dict() for item in configurations + ], + "base_seed": _nonnegative_int(base_seed, "base_seed"), + "ordinal": _positive_int(ordinal, "ordinal"), + }, + ) + + +def _artifact_digests( + values: Mapping[str, str], kind: EnsembleArtifactKind +) -> tuple[EnsembleArtifactDigestV1, ...]: + if not values or len(values) > MAX_ENSEMBLE_ARTIFACTS: + raise ValueError("ensemble artifact count is outside limits") + return tuple( + EnsembleArtifactDigestV1( + artifact_id=artifact_id, + sha256=digest, + kind=kind, + ) + for artifact_id, digest in sorted(values.items()) + ) + + +def _normalized_artifacts( + values: Iterable[EnsembleArtifactDigestV1], kind: EnsembleArtifactKind +) -> tuple[EnsembleArtifactDigestV1, ...]: + artifacts = tuple(sorted(values, key=lambda item: item.artifact_id)) + if len(artifacts) > MAX_ENSEMBLE_ARTIFACTS or any( + not isinstance(item, EnsembleArtifactDigestV1) or item.kind is not kind + for item in artifacts + ): + raise ValueError("ensemble artifact digest kind/count differs") + if len({item.artifact_id for item in artifacts}) != len(artifacts): + raise ValueError("ensemble artifact IDs are duplicated") + return artifacts + + +def _content_sha256(payload: Any) -> str: + encoded = canonical_contract_json(cast(JSONValue, payload)).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _median(values: Sequence[float]) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("median requires values") + center = len(ordered) // 2 + if len(ordered) % 2: + return ordered[center] + return (ordered[center - 1] + ordered[center]) / 2.0 + + +def _rounded(value: float, digits: int) -> float: + return round(_finite_float(value, "rounded value"), digits) + + +def _metric_mapping( + values: Mapping[str, float], *, allow_empty: bool = False +) -> dict[str, float]: + result = { + _required_text(str(key)): _finite_float(value, f"metric {key}") + for key, value in values.items() + } + if not result and allow_empty: + return {} + if set(result) != set(ENSEMBLE_CALIBRATION_METRIC_NAMES): + raise ValueError("ensemble calibration metrics differ") + return dict(sorted(result.items())) + + +def _count_mapping(values: Mapping[str, int], label: str) -> dict[str, int]: + return dict( + sorted( + ( + _required_text(str(key)), + _nonnegative_int(value, f"{label} count"), + ) + for key, value in values.items() + ) + ) + + +def _hash_mapping(values: Mapping[str, str], label: str) -> dict[str, str]: + return dict( + sorted( + ( + _required_text(str(key)), + _required_sha256(value), + ) + for key, value in values.items() + ) + ) + + +def _required_sha256(value: Any) -> str: + text = _required_text(value).lower() + digest = text.removeprefix("sha256:") + if len(digest) != 64 or any( + char not in "0123456789abcdef" for char in digest + ): + raise ValueError("value must be a SHA-256 digest") + return "sha256:" + digest + + +def _normalized_text_tuple( + values: Iterable[str], *, allow_empty: bool = False +) -> tuple[str, ...]: + result = tuple(sorted({_required_text(value) for value in values})) + if not result and not allow_empty: + raise ValueError("value requires non-empty identifiers") + return result + + +def _required_text(value: Any) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError("value must be non-empty text") + result = value.strip() + if len(result) > MAX_ENSEMBLE_TEXT: + raise ValueError("text exceeds ensemble bound") + return result + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError("optional text value must be a string") + selected = value.strip() + return _required_text(selected) if selected else None + + +def _optional_bounded_text(value: Any, name: str) -> str | None: + if value is None: + return None + result = _required_text(value) + if len(result) > MAX_ENSEMBLE_TEXT: + raise ValueError(f"{name} exceeds ensemble bound") + return result + + +def _strict_bool(value: Any, name: str) -> bool: + if type(value) is not bool: + raise ValueError(f"{name} must be boolean") + return value + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _positive_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _nonnegative_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _nonnegative_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if result < 0.0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _optional_nonnegative_float(value: Any, name: str) -> float | None: + if value is None: + return None + return _nonnegative_float(value, name) + + +def _unit_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if not 0.0 <= result <= 1.0: + raise ValueError(f"{name} must be between zero and one") + return result + + +def _normalized_symbol(value: Any) -> str: + result = _required_text(value).upper() + if not result.isalnum(): + raise ValueError("symbol must be alphanumeric") + return result + + +def _optional_float(value: Any) -> float | None: + if value is None: + return None + return _finite_float(value, "optional float") + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("value must be a mapping") + return cast(Mapping[str, Any], value) + + +def _mapping_sequence( + data: Mapping[str, Any], key: str +) -> tuple[Mapping[str, Any], ...]: + value = data.get(key) + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError(f"{key} must be a sequence") + result: list[Mapping[str, Any]] = [] + for item in value: + result.append(_mapping(item)) + return tuple(result) + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError(f"{name} must be a sequence") + return tuple(str(item) for item in value) + + +def _int_tuple(value: Any, name: str) -> tuple[int, ...]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError(f"{name} must be a sequence") + return tuple(_strict_int(item, name) for item in value) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + return _mapping(value) + + +def _require_version(actual: str, expected: str, label: str) -> None: + if actual != expected: + raise ValueError(f"unsupported {label} schema") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if data.get("schema_version") != expected: + raise ValueError("unsupported ensemble schema version") + + +def _require_derived_value( + data: Mapping[str, Any], key: str, expected: Any +) -> None: + if data.get(key) != expected: + raise ValueError(f"ensemble derived field {key} differs") + + +def _ensure_payload_size( + payload: Mapping[str, JSONValue], maximum: int +) -> None: + size = len(canonical_contract_json(payload).encode("utf-8")) + if size > maximum: + raise ValueError("ensemble report exceeds bounded payload size") diff --git a/src/histdatacom/synthetic/generation.py b/src/histdatacom/synthetic/generation.py new file mode 100644 index 00000000..2d1d92f1 --- /dev/null +++ b/src/histdatacom/synthetic/generation.py @@ -0,0 +1,1938 @@ +"""Deterministic empirical-motif candidate generation. + +This module owns the first variable-cardinality reconstruction candidate. It +is deliberately upstream of hard carving, cross-series reconciliation, +broker conditioning, and final persistence. A generated event is therefore +an auditable proposal, never an accepted or final synthetic tick. + +Generation is performed for one pair of immutable observed anchors. The +complete anchor interval is planned from semantic inputs, then each streaming +window emits only the timestamps it owns. Seeds, transformed values, and +event identities never include a worker, retry, or window identifier. +""" + +from __future__ import annotations + +from bisect import bisect_left +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field, replace +from enum import Enum +import hashlib +import json +import math +from typing import Any + +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.benchmark import ( + BENCHMARK_EVENT_SCHEMA_VERSION, + BenchmarkCandidateV1, + BenchmarkEventV1, + BenchmarkGeneratorV1, + BenchmarkScenarioV1, +) +from histdatacom.synthetic.contracts import ( + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + canonical_contract_json, + derive_anchor_interval_id, +) +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.motifs import ( + ReferenceMotifConditionV1, + ReferenceMotifFragmentV1, + ReferenceMotifIndexV1, + ReferenceMotifMatchV1, + ReferenceMotifQueryResultV1, + ReferenceMotifQueryStatus, + ReferenceMotifQueryV1, + query_reference_motifs, +) +from histdatacom.synthetic.streaming import ( + CarryStateV1, + ReconstructionResourceEstimateV1, + ReconstructionResourceLimitError, + ReconstructionRunV1, + ReconstructionWindowV1, +) + +MOTIF_GENERATOR_CONFIG_SCHEMA_VERSION = ( + "histdatacom.empirical-motif-generator-config.v1" +) +MOTIF_TRANSFORMATION_SCHEMA_VERSION = ( + "histdatacom.empirical-motif-transformation.v1" +) +MOTIF_EVENT_LINEAGE_SCHEMA_VERSION = ( + "histdatacom.empirical-motif-event-lineage.v1" +) +MOTIF_CANDIDATE_BATCH_SCHEMA_VERSION = ( + "histdatacom.empirical-motif-candidate-batch.v1" +) + +EMPIRICAL_MOTIF_GENERATOR_ID = "histdatacom.empirical-motif-resampling" +EMPIRICAL_MOTIF_GENERATOR_VERSION = "1.2.0" +MOTIF_TRANSFORMATION_CONFIDENCE_QUANTITY = ( + "uncalibrated-motif-match-similarity-v1" +) +CANDIDATE_ONLY_CONSTRAINT_SET_ID = ( + "histdatacom.constraint-set.candidate-pre-carving.v1" +) + +MAX_MOTIF_GENERATED_EVENTS_PER_INTERVAL = 100_000 +MAX_MOTIF_TRANSFORMATIONS_PER_INTERVAL = 100_000 +MAX_MOTIF_DECISION_DETAILS = 32 +MAX_MOTIF_DECISION_DETAIL_TEXT = 1_024 +MAX_PRICE_PRECISION_DIGITS = 12 +NANOSECONDS_PER_SECOND = 1_000_000_000 + + +class MotifGenerationStatus(str, Enum): + """Whether an interval emitted candidates or made an explicit decision.""" + + GENERATED = "generated" + EMPTY = "empty" + REFUSED = "refused" + + +class MotifGenerationDecision(str, Enum): + """Bounded generation and refusal reason codes.""" + + GENERATED = "generated" + CLOSED_SESSION = "closed_session" + ZERO_TARGET_ACTIVITY = "zero_target_activity" + OUTSIDE_WINDOW_OWNERSHIP = "outside_window_ownership" + ZERO_WIDTH_ANCHOR = "zero_width_anchor" + REVERSED_ANCHOR = "reversed_anchor" + NO_SUPPORTED_CELL = "no_supported_cell" + NOT_AVAILABLE_AS_OF = "not_available_as_of" + INTERVAL_EVENT_LIMIT = "interval_event_limit" + RESOURCE_LIMIT = "resource_limit" + UNSUPPORTED_TRANSFORM = "unsupported_transform" + INVALID_TRANSFORMED_QUOTE = "invalid_transformed_quote" + + +@dataclass(frozen=True, slots=True) +class EmpiricalMotifGeneratorConfigV1: + """Versioned candidate-only resampling and resource assumptions.""" + + max_events_per_interval: int = 50_000 + max_transformations_per_interval: int = 25_000 + estimated_bytes_per_event: int = 512 + fallback_price_precision_digits: int = 8 + confidence_rounding_digits: int = 12 + closed_session_states: tuple[str, ...] = ( + "closed", + "market_closed", + "weekend_closed", + ) + constraint_set_id: str = CANDIDATE_ONLY_CONSTRAINT_SET_ID + config_id: str = "" + schema_version: str = MOTIF_GENERATOR_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MOTIF_GENERATOR_CONFIG_SCHEMA_VERSION: + raise ValueError("unsupported empirical motif generator config") + for name, upper in ( + ( + "max_events_per_interval", + MAX_MOTIF_GENERATED_EVENTS_PER_INTERVAL, + ), + ( + "max_transformations_per_interval", + MAX_MOTIF_TRANSFORMATIONS_PER_INTERVAL, + ), + ): + value = _positive_int(getattr(self, name), name) + if value > upper: + raise ValueError(f"{name} exceeds the version-one bound") + object.__setattr__(self, name, value) + object.__setattr__( + self, + "estimated_bytes_per_event", + _positive_int( + self.estimated_bytes_per_event, + "estimated_bytes_per_event", + ), + ) + for name, upper in ( + ("fallback_price_precision_digits", MAX_PRICE_PRECISION_DIGITS), + ("confidence_rounding_digits", MAX_PRICE_PRECISION_DIGITS), + ): + value = _nonnegative_int(getattr(self, name), name) + if value > upper: + raise ValueError(f"{name} exceeds the supported precision") + object.__setattr__(self, name, value) + states = tuple( + sorted( + { + _required_text(item).strip().lower() + for item in self.closed_session_states + } + ) + ) + if not states: + raise ValueError("closed_session_states cannot be empty") + object.__setattr__(self, "closed_session_states", states) + if self.constraint_set_id != CANDIDATE_ONLY_CONSTRAINT_SET_ID: + raise ValueError( + "v1 motif generation must declare the pre-carving " + "candidate constraint set" + ) + expected = _stable_id( + "empirical-motif-generator-config", self.identity_payload() + ) + supplied = _optional_text(self.config_id) + if supplied is not None and supplied != expected: + raise ValueError("empirical motif generator config_id differs") + object.__setattr__(self, "config_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic configuration used for deterministic identity.""" + return { + "schema_version": self.schema_version, + "generator_id": EMPIRICAL_MOTIF_GENERATOR_ID, + "generator_version": EMPIRICAL_MOTIF_GENERATOR_VERSION, + "fallback_price_precision_digits": ( + self.fallback_price_precision_digits + ), + "confidence_rounding_digits": self.confidence_rounding_digits, + "closed_session_states": list(self.closed_session_states), + "constraint_set_id": self.constraint_set_id, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "max_events_per_interval": self.max_events_per_interval, + "max_transformations_per_interval": ( + self.max_transformations_per_interval + ), + "estimated_bytes_per_event": self.estimated_bytes_per_event, + "execution_limits_in_config_identity": False, + "config_id": self.config_id, + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EmpiricalMotifGeneratorConfigV1": + return cls( + max_events_per_interval=_strict_int( + data.get("max_events_per_interval"), + "max_events_per_interval", + ), + max_transformations_per_interval=_strict_int( + data.get("max_transformations_per_interval"), + "max_transformations_per_interval", + ), + estimated_bytes_per_event=_strict_int( + data.get("estimated_bytes_per_event"), + "estimated_bytes_per_event", + ), + fallback_price_precision_digits=_strict_int( + data.get("fallback_price_precision_digits"), + "fallback_price_precision_digits", + ), + confidence_rounding_digits=_strict_int( + data.get("confidence_rounding_digits"), + "confidence_rounding_digits", + ), + closed_session_states=_string_tuple( + data.get("closed_session_states"), + "closed_session_states", + ), + constraint_set_id=str(data.get("constraint_set_id", "")), + config_id=str(data.get("config_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EmpiricalMotifGeneratorConfigV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class EmpiricalMotifTransformationV1: + """One bounded source-fragment transform applied to output ordinals.""" + + index_id: str + query_id: str + query_result_id: str + source_fragment_id: str + source_window_id: str + source_series_id: str + source_period: str + source_artifact_sha256: str + backoff_level: str + cell_support: int + match_distance: float + segment_ordinal: int + output_start_ordinal: int + output_end_ordinal: int + source_event_count: int + time_scale: float + time_warp_ratio: float + requested_price_scale: float + applied_price_scale: float + price_scale_clamped: bool + spread_shape_applied: bool + seed: int + confidence: float + transformation_id: str = "" + schema_version: str = MOTIF_TRANSFORMATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MOTIF_TRANSFORMATION_SCHEMA_VERSION: + raise ValueError("unsupported empirical motif transformation") + for name in ( + "index_id", + "query_id", + "query_result_id", + "source_fragment_id", + "source_window_id", + "source_series_id", + "source_period", + "source_artifact_sha256", + "backoff_level", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + for name in ( + "cell_support", + "segment_ordinal", + "output_start_ordinal", + "output_end_ordinal", + "source_event_count", + ): + object.__setattr__( + self, name, _positive_int(getattr(self, name), name) + ) + if self.output_end_ordinal < self.output_start_ordinal: + raise ValueError("transformation output ordinal range is reversed") + for name in ( + "match_distance", + "time_scale", + "time_warp_ratio", + "requested_price_scale", + "applied_price_scale", + ): + value = _finite_float(getattr(self, name), name) + if value < 0.0: + raise ValueError(f"{name} must be non-negative") + object.__setattr__(self, name, value) + for name in ("price_scale_clamped", "spread_shape_applied"): + if type(getattr(self, name)) is not bool: + raise ValueError(f"{name} must be boolean") + object.__setattr__(self, "seed", _nonnegative_int(self.seed, "seed")) + confidence = _finite_float(self.confidence, "confidence") + if not 0.0 <= confidence <= 1.0: + raise ValueError("transformation confidence is outside [0,1]") + object.__setattr__(self, "confidence", confidence) + expected = _stable_id("empirical-motif-transformation", self.payload()) + supplied = _optional_text(self.transformation_id) + if supplied is not None and supplied != expected: + raise ValueError("empirical motif transformation_id differs") + object.__setattr__(self, "transformation_id", expected) + + def payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "index_id": self.index_id, + "query_id": self.query_id, + "query_result_id": self.query_result_id, + "source_fragment_id": self.source_fragment_id, + "source_window_id": self.source_window_id, + "source_series_id": self.source_series_id, + "source_period": self.source_period, + "source_artifact_sha256": self.source_artifact_sha256, + "backoff_level": self.backoff_level, + "cell_support": self.cell_support, + "match_distance": self.match_distance, + "segment_ordinal": self.segment_ordinal, + "output_start_ordinal": self.output_start_ordinal, + "output_end_ordinal": self.output_end_ordinal, + "source_event_count": self.source_event_count, + "time_scale": self.time_scale, + "time_warp_ratio": self.time_warp_ratio, + "requested_price_scale": self.requested_price_scale, + "applied_price_scale": self.applied_price_scale, + "price_scale_clamped": self.price_scale_clamped, + "spread_shape_applied": self.spread_shape_applied, + "seed": self.seed, + "confidence": self.confidence, + "endpoint_alignment": "detrended-linear-anchor-bridge-v1", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.payload(), "transformation_id": self.transformation_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EmpiricalMotifTransformationV1": + return cls( + index_id=str(data.get("index_id", "")), + query_id=str(data.get("query_id", "")), + query_result_id=str(data.get("query_result_id", "")), + source_fragment_id=str(data.get("source_fragment_id", "")), + source_window_id=str(data.get("source_window_id", "")), + source_series_id=str(data.get("source_series_id", "")), + source_period=str(data.get("source_period", "")), + source_artifact_sha256=str(data.get("source_artifact_sha256", "")), + backoff_level=str(data.get("backoff_level", "")), + cell_support=_strict_int(data.get("cell_support"), "cell_support"), + match_distance=_finite_float( + data.get("match_distance"), "match_distance" + ), + segment_ordinal=_strict_int( + data.get("segment_ordinal"), "segment_ordinal" + ), + output_start_ordinal=_strict_int( + data.get("output_start_ordinal"), "output_start_ordinal" + ), + output_end_ordinal=_strict_int( + data.get("output_end_ordinal"), "output_end_ordinal" + ), + source_event_count=_strict_int( + data.get("source_event_count"), "source_event_count" + ), + time_scale=_finite_float(data.get("time_scale"), "time_scale"), + time_warp_ratio=_finite_float( + data.get("time_warp_ratio"), "time_warp_ratio" + ), + requested_price_scale=_finite_float( + data.get("requested_price_scale"), "requested_price_scale" + ), + applied_price_scale=_finite_float( + data.get("applied_price_scale"), "applied_price_scale" + ), + price_scale_clamped=_strict_bool( + data.get("price_scale_clamped"), "price_scale_clamped" + ), + spread_shape_applied=_strict_bool( + data.get("spread_shape_applied"), "spread_shape_applied" + ), + seed=_strict_int(data.get("seed"), "seed"), + confidence=_finite_float(data.get("confidence"), "confidence"), + transformation_id=str(data.get("transformation_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EmpiricalMotifTransformationV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class EmpiricalMotifEventLineageV1: + """Compact per-event pointer into one recoverable transform.""" + + event_id: str + transformation_id: str + global_event_ordinal: int + segment_event_ordinal: int + source_progress: float + anchor_progress: float + requested_event_time_ns: int + schema_version: str = MOTIF_EVENT_LINEAGE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MOTIF_EVENT_LINEAGE_SCHEMA_VERSION: + raise ValueError("unsupported empirical motif event lineage") + object.__setattr__(self, "event_id", _required_text(self.event_id)) + object.__setattr__( + self, + "transformation_id", + _required_text(self.transformation_id), + ) + for name in ("global_event_ordinal", "segment_event_ordinal"): + object.__setattr__( + self, name, _positive_int(getattr(self, name), name) + ) + for name in ("source_progress", "anchor_progress"): + value = _finite_float(getattr(self, name), name) + if not 0.0 < value <= 1.0: + raise ValueError(f"{name} must be inside (0,1]") + object.__setattr__(self, name, value) + object.__setattr__( + self, + "requested_event_time_ns", + _strict_int( + self.requested_event_time_ns, "requested_event_time_ns" + ), + ) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "event_id": self.event_id, + "transformation_id": self.transformation_id, + "global_event_ordinal": self.global_event_ordinal, + "segment_event_ordinal": self.segment_event_ordinal, + "source_progress": self.source_progress, + "anchor_progress": self.anchor_progress, + "requested_event_time_ns": self.requested_event_time_ns, + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "EmpiricalMotifEventLineageV1": + return cls( + event_id=str(data.get("event_id", "")), + transformation_id=str(data.get("transformation_id", "")), + global_event_ordinal=_strict_int( + data.get("global_event_ordinal"), "global_event_ordinal" + ), + segment_event_ordinal=_strict_int( + data.get("segment_event_ordinal"), "segment_event_ordinal" + ), + source_progress=_finite_float( + data.get("source_progress"), "source_progress" + ), + anchor_progress=_finite_float( + data.get("anchor_progress"), "anchor_progress" + ), + requested_event_time_ns=_strict_int( + data.get("requested_event_time_ns"), + "requested_event_time_ns", + ), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EmpiricalMotifEventLineageV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class EmpiricalMotifCandidateBatchV1: + """Process-local candidate rows plus bounded deterministic evidence.""" + + run_id: str + window_id: str + ensemble_member_id: str + symbol: str + anchor_interval_id: str + left_anchor_event_id: str + right_anchor_event_id: str + generator_config: EmpiricalMotifGeneratorConfigV1 + query_result: ReferenceMotifQueryResultV1 + status: MotifGenerationStatus + decision: MotifGenerationDecision + target_event_count: int + events: tuple[SyntheticEventV1, ...] + transformations: tuple[EmpiricalMotifTransformationV1, ...] + event_lineage: tuple[EmpiricalMotifEventLineageV1, ...] + resource_estimate: ReconstructionResourceEstimateV1 + carry_state: CarryStateV1 + decision_details: tuple[str, ...] = () + batch_id: str = "" + schema_version: str = MOTIF_CANDIDATE_BATCH_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MOTIF_CANDIDATE_BATCH_SCHEMA_VERSION: + raise ValueError("unsupported empirical motif candidate batch") + for name in ( + "run_id", + "window_id", + "ensemble_member_id", + "symbol", + "anchor_interval_id", + "left_anchor_event_id", + "right_anchor_event_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if not isinstance( + self.generator_config, EmpiricalMotifGeneratorConfigV1 + ): + raise ValueError("candidate batch requires a v1 generator config") + if not isinstance(self.query_result, ReferenceMotifQueryResultV1): + raise ValueError("candidate batch requires a v1 motif query result") + object.__setattr__(self, "status", MotifGenerationStatus(self.status)) + object.__setattr__( + self, "decision", MotifGenerationDecision(self.decision) + ) + target_count = _nonnegative_int( + self.target_event_count, "target_event_count" + ) + object.__setattr__(self, "target_event_count", target_count) + events = tuple( + sorted( + self.events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.event_id, + ), + ) + ) + transforms = tuple( + sorted( + self.transformations, + key=lambda item: ( + item.segment_ordinal, + item.transformation_id, + ), + ) + ) + lineages = tuple( + sorted( + self.event_lineage, + key=lambda item: ( + item.global_event_ordinal, + item.event_id, + ), + ) + ) + if any( + item.origin is not SyntheticEventOrigin.SYNTHETIC for item in events + ): + raise ValueError( + "candidate batch can contain only synthetic events" + ) + if len(events) != len(lineages): + raise ValueError( + "candidate events and event lineage do not reconcile" + ) + if len(events) > target_count: + raise ValueError("owned candidate count exceeds interval target") + if self.status is MotifGenerationStatus.GENERATED and not events: + raise ValueError("generated candidate batch requires events") + if self.status is not MotifGenerationStatus.GENERATED and events: + raise ValueError( + "empty or refused candidate batch cannot have events" + ) + if (self.status is MotifGenerationStatus.GENERATED) != ( + self.decision is MotifGenerationDecision.GENERATED + ): + raise ValueError("candidate status and decision disagree") + event_ids = {item.event_id for item in events} + if len(event_ids) != len(events): + raise ValueError("candidate batch has duplicate event IDs") + if event_ids != {item.event_id for item in lineages}: + raise ValueError("event lineage does not cover candidate event IDs") + transform_ids = {item.transformation_id for item in transforms} + if len(transform_ids) != len(transforms): + raise ValueError("candidate batch has duplicate transformations") + if len({item.segment_ordinal for item in transforms}) != len( + transforms + ): + raise ValueError("candidate batch has duplicate transform ordinals") + if len({item.global_event_ordinal for item in lineages}) != len( + lineages + ): + raise ValueError("candidate batch has duplicate global ordinals") + if any( + item.transformation_id not in transform_ids for item in lineages + ): + raise ValueError( + "event lineage references an absent transformation" + ) + if any( + item.index_id != self.query_result.index_id + or item.query_id != self.query_result.query.query_id + or item.query_result_id != self.query_result.result_id + for item in transforms + ): + raise ValueError("transformation differs from batch query lineage") + matched_fragment_ids = { + item.fragment.fragment_id for item in self.query_result.matches + } + if any( + item.source_fragment_id not in matched_fragment_ids + for item in transforms + ): + raise ValueError("transformation source was not a retrieved match") + transform_by_id = {item.transformation_id: item for item in transforms} + event_by_id = {item.event_id: item for item in events} + for lineage in lineages: + event = event_by_id[lineage.event_id] + transform = transform_by_id[lineage.transformation_id] + if ( + lineage.requested_event_time_ns != event.event_time_ns + or lineage.global_event_ordinal != event.event_sequence + or event.reference_id != self.query_result.result_id + or event.motif_id != transform.source_fragment_id + or event.confidence is not None + or not ( + transform.output_start_ordinal + <= lineage.global_event_ordinal + <= transform.output_end_ordinal + ) + or lineage.segment_event_ordinal + != ( + lineage.global_event_ordinal + - transform.output_start_ordinal + + 1 + ) + ): + raise ValueError("candidate event-to-transform lineage differs") + if any( + item.run_id != self.run_id + or item.ensemble_member_id != self.ensemble_member_id + or item.symbol != self.symbol + or item.anchor_interval_id != self.anchor_interval_id + or item.left_anchor_event_id != self.left_anchor_event_id + or item.right_anchor_event_id != self.right_anchor_event_id + or item.generator_config_id != self.generator_config.config_id + or item.constraint_set_id != CANDIDATE_ONLY_CONSTRAINT_SET_ID + for item in events + ): + raise ValueError("candidate event lineage differs from its batch") + object.__setattr__(self, "events", events) + object.__setattr__(self, "transformations", transforms) + object.__setattr__(self, "event_lineage", lineages) + if not isinstance( + self.resource_estimate, ReconstructionResourceEstimateV1 + ): + raise ValueError("candidate batch requires a resource estimate") + if not isinstance(self.carry_state, CarryStateV1): + raise ValueError("candidate batch requires v1 carry state") + if ( + self.carry_state.run_id != self.run_id + or self.carry_state.ensemble_member_id != self.ensemble_member_id + or self.symbol not in self.carry_state.symbol_watermarks_ns + ): + raise ValueError("candidate batch carry state differs from scope") + details = tuple( + _bounded_text( + item, + "decision_detail", + MAX_MOTIF_DECISION_DETAIL_TEXT, + ) + for item in self.decision_details + ) + if len(details) > MAX_MOTIF_DECISION_DETAILS: + raise ValueError("candidate decision details exceed bounded limit") + object.__setattr__(self, "decision_details", details) + expected = _stable_id("empirical-motif-candidate-batch", self.payload()) + supplied = _optional_text(self.batch_id) + if supplied is not None and supplied != expected: + raise ValueError("empirical motif candidate batch_id differs") + object.__setattr__(self, "batch_id", expected) + + @property + def generator_config_id(self) -> str: + """Return the semantic config ID repeated by every candidate event.""" + return self.generator_config.config_id + + def payload(self) -> dict[str, JSONValue]: + """Return semantic batch identity without embedding row payloads.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "ensemble_member_id": self.ensemble_member_id, + "symbol": self.symbol, + "anchor_interval_id": self.anchor_interval_id, + "left_anchor_event_id": self.left_anchor_event_id, + "right_anchor_event_id": self.right_anchor_event_id, + "generator_config_id": self.generator_config.config_id, + "query_result_id": self.query_result.result_id, + "status": self.status.value, + "decision": self.decision.value, + "target_event_count": self.target_event_count, + "owned_event_count": len(self.events), + "transformation_count": len(self.transformations), + "event_lineage_count": len(self.event_lineage), + "event_content_sha256": _content_sha256( + [item.to_dict() for item in self.events] + ), + "transformation_content_sha256": _content_sha256( + [item.to_dict() for item in self.transformations] + ), + "event_lineage_content_sha256": _content_sha256( + [item.to_dict() for item in self.event_lineage] + ), + "resource_estimate_id": self.resource_estimate.estimate_id, + "carry_id": self.carry_state.carry_id, + "decision_details": list(self.decision_details), + "candidate_only": True, + "hard_carving_status": "not_evaluated", + "broker_conditioning_status": "not_applied", + "final_storage_status": "not_persisted", + } + + def metadata(self) -> dict[str, JSONValue]: + """Return workflow-safe metadata while keeping candidate rows external.""" + attempts: list[JSONValue] = [ + item.to_dict() for item in self.query_result.backoff_attempts + ] + return { + **self.payload(), + "batch_id": self.batch_id, + "generator_config": self.generator_config.to_dict(), + "condition": self.query_result.query.condition.to_dict(), + "query_status": self.query_result.status.value, + "backoff_attempts": attempts, + "events_inline": False, + "transformations_inline": False, + "event_lineage_inline": False, + } + + def lineage_for(self, event_id: str) -> EmpiricalMotifEventLineageV1: + """Return the compact lineage record for one emitted candidate.""" + wanted = _required_text(event_id) + for lineage in self.event_lineage: + if lineage.event_id == wanted: + return lineage + raise KeyError(wanted) + + def merged_stream( + self, + observed_events: Sequence[SyntheticEventV1], + ) -> SyntheticEventStreamV1: + """Merge proposals with the caller's unchanged observed objects.""" + return SyntheticEventStreamV1.merge( + run_id=self.run_id, + ensemble_member_id=self.ensemble_member_id, + symbol=self.symbol, + observed_events=observed_events, + synthetic_events=self.events, + ) + + +@dataclass(frozen=True, slots=True) +class _PlannedTransform: + match: ReferenceMotifMatchV1 + record: EmpiricalMotifTransformationV1 + start_index: int + event_count: int + + +def generate_empirical_motif_candidates( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + left_anchor: SyntheticEventV1, + right_anchor: SyntheticEventV1, + query_result: ReferenceMotifQueryResultV1, + config: EmpiricalMotifGeneratorConfigV1, + required_event_time_ns: int | None = None, +) -> EmpiricalMotifCandidateBatchV1: + """Generate one deterministic, window-owned candidate anchor interval.""" + _validate_generation_scope( + run, + window, + left_anchor, + right_anchor, + query_result, + config, + ) + interval_id = derive_anchor_interval_id( + left_anchor.event_id, right_anchor.event_id + ) + gap_ns = right_anchor.event_time_ns - left_anchor.event_time_ns + required_time = ( + _strict_int(required_event_time_ns, "required_event_time_ns") + if required_event_time_ns is not None + else None + ) + if required_time is not None and not ( + left_anchor.event_time_ns < required_time < right_anchor.event_time_ns + ): + raise ValueError( + "required event time must be inside the anchor interval" + ) + zero_estimate = _resource_estimate(0, config) + if gap_ns == 0: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.REFUSED, + decision=MotifGenerationDecision.ZERO_WIDTH_ANCHOR, + target_event_count=0, + estimate=zero_estimate, + ) + if gap_ns < 0: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.REFUSED, + decision=MotifGenerationDecision.REVERSED_ANCHOR, + target_event_count=0, + estimate=zero_estimate, + ) + + condition = query_result.query.condition + if condition.session_state.strip().lower() in config.closed_session_states: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.EMPTY, + decision=MotifGenerationDecision.CLOSED_SESSION, + target_event_count=0, + estimate=zero_estimate, + ) + + target_count, cadence_ns = _target_cardinality(condition, gap_ns) + if required_time is not None: + target_count = max(1, target_count) + if target_count == 0: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.EMPTY, + decision=MotifGenerationDecision.ZERO_TARGET_ACTIVITY, + target_event_count=0, + estimate=zero_estimate, + ) + estimate = _resource_estimate(target_count, config) + if query_result.status is not ReferenceMotifQueryStatus.MATCHED: + decision = ( + MotifGenerationDecision.NOT_AVAILABLE_AS_OF + if query_result.status + is ReferenceMotifQueryStatus.NOT_AVAILABLE_AS_OF + else MotifGenerationDecision.NO_SUPPORTED_CELL + ) + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.REFUSED, + decision=decision, + target_event_count=target_count, + estimate=estimate, + ) + if target_count > config.max_events_per_interval: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.REFUSED, + decision=MotifGenerationDecision.INTERVAL_EVENT_LIMIT, + target_event_count=target_count, + estimate=estimate, + details=( + f"target {target_count} exceeds interval limit " + f"{config.max_events_per_interval}", + ), + ) + try: + run.storage_policy.preflight(estimate) + except ReconstructionResourceLimitError as err: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.REFUSED, + decision=MotifGenerationDecision.RESOURCE_LIMIT, + target_event_count=target_count, + estimate=err.estimate, + details=err.violations, + ) + event_times = _candidate_times( + left_anchor.event_time_ns, + right_anchor.event_time_ns, + target_count, + cadence_ns, + condition, + ) + plans, plan_error = _plan_transforms( + run=run, + ensemble_member_id=window.ensemble_member_id, + interval_id=interval_id, + event_times=event_times, + cadence_ns=cadence_ns, + left_time_ns=left_anchor.event_time_ns, + query_result=query_result, + config=config, + ) + if plan_error is not None: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.REFUSED, + decision=MotifGenerationDecision.UNSUPPORTED_TRANSFORM, + target_event_count=target_count, + estimate=estimate, + details=(plan_error,), + ) + + all_events: list[SyntheticEventV1] = [] + all_lineage: list[EmpiricalMotifEventLineageV1] = [] + quote_error: str | None = None + for plan in plans: + generated, lineages, error = _events_for_transform( + run=run, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + event_times=event_times, + plan=plan, + required_event_time_ns=required_time, + ) + if error is not None: + quote_error = error + break + all_events.extend(generated) + all_lineage.extend(lineages) + if quote_error is not None: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.REFUSED, + decision=MotifGenerationDecision.INVALID_TRANSFORMED_QUOTE, + target_event_count=target_count, + estimate=estimate, + details=(quote_error,), + ) + + owned_ids = { + event.event_id + for event in all_events + if window.owns_event_time(event.event_time_ns) + } + owned_events = tuple( + item for item in all_events if item.event_id in owned_ids + ) + owned_lineage = tuple( + item for item in all_lineage if item.event_id in owned_ids + ) + referenced_transform_ids = { + item.transformation_id for item in owned_lineage + } + owned_transforms = tuple( + plan.record + for plan in plans + if plan.record.transformation_id in referenced_transform_ids + ) + if not owned_events: + return _decision_batch( + run=run, + window=window, + left_anchor=left_anchor, + right_anchor=right_anchor, + interval_id=interval_id, + query_result=query_result, + config=config, + status=MotifGenerationStatus.EMPTY, + decision=MotifGenerationDecision.OUTSIDE_WINDOW_OWNERSHIP, + target_event_count=target_count, + estimate=estimate, + ) + return EmpiricalMotifCandidateBatchV1( + run_id=run.run_id, + window_id=window.window_id, + ensemble_member_id=window.ensemble_member_id, + symbol=left_anchor.symbol, + anchor_interval_id=interval_id, + left_anchor_event_id=left_anchor.event_id, + right_anchor_event_id=right_anchor.event_id, + generator_config=config, + query_result=query_result, + status=MotifGenerationStatus.GENERATED, + decision=MotifGenerationDecision.GENERATED, + target_event_count=target_count, + events=owned_events, + transformations=owned_transforms, + event_lineage=owned_lineage, + resource_estimate=estimate, + carry_state=_carry_state( + run, window, left_anchor, right_anchor, owned_events + ), + ) + + +@dataclass(frozen=True, slots=True) +class EmpiricalMotifBenchmarkGeneratorV1(BenchmarkGeneratorV1): + """Adapter exposing motif candidates to reverse-degradation scorecards.""" + + candidate: BenchmarkCandidateV1 + run: ReconstructionRunV1 + motif_index: ReferenceMotifIndexV1 + condition: ReferenceMotifConditionV1 + config: EmpiricalMotifGeneratorConfigV1 + candidate_id: str = field(init=False) + information_mode: InformationMode = InformationMode.EX_POST_RECONSTRUCTION + as_of_ns: int | None = None + event_schema_version: str = BENCHMARK_EVENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.candidate.method_id != EMPIRICAL_MOTIF_GENERATOR_ID: + raise ValueError( + "benchmark candidate method is not motif generation" + ) + if self.event_schema_version != BENCHMARK_EVENT_SCHEMA_VERSION: + raise ValueError("benchmark adapter requires event schema v1") + if self.config.config_id not in self.run.configuration_ids: + raise ValueError("motif generator config is absent from the run") + object.__setattr__( + self, "candidate_id", str(self.candidate.candidate_id) + ) + mode = InformationMode.from_value(self.information_mode) + object.__setattr__(self, "information_mode", mode) + if mode is InformationMode.EX_ANTE_SIMULATION and self.as_of_ns is None: + raise ValueError("ex-ante benchmark generation requires as_of_ns") + if ( + mode is InformationMode.EX_POST_RECONSTRUCTION + and self.as_of_ns is not None + ): + raise ValueError("ex-post benchmark generation rejects as_of_ns") + + def generate( + self, + degraded_events: Sequence[BenchmarkEventV1], + *, + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + ensemble_member_id: str, + ) -> Sequence[BenchmarkEventV1]: + """Generate a benchmark stream containing anchors and proposals.""" + if window.run_id != self.run.run_id: + raise ValueError( + "benchmark window differs from motif generator run" + ) + if window.ensemble_member_id != ensemble_member_id: + raise ValueError("benchmark ensemble member differs from window") + if scenario.epoch_id != self.condition.feed_epoch_id: + raise ValueError("benchmark epoch differs from motif condition") + ordered = tuple( + sorted( + degraded_events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.benchmark_event_id, + ), + ) + ) + if len(ordered) < 2: + return tuple( + _benchmark_anchor(item, ensemble_member_id) for item in ordered + ) + source_version_id = self.run.source_version_ids[0] + anchors = tuple( + SyntheticEventV1.observed( + symbol=item.symbol, + event_time_ns=item.event_time_ns, + event_sequence=item.event_sequence, + bid=item.bid, + ask=item.ask, + run_id=self.run.run_id, + ensemble_member_id=ensemble_member_id, + source_version_id=source_version_id, + source_series_id=( + f"benchmark:{scenario.scenario_id}:{item.symbol}" + ), + source_period=scenario.severity_id, + source_row_id=index, + ) + for index, item in enumerate(ordered, start=1) + ) + proposals: list[SyntheticEventV1] = [] + for left, right in zip(anchors, anchors[1:]): + query = ReferenceMotifQueryV1( + condition=self.condition, + information_mode=self.information_mode, + used_at_ns=right.event_time_ns, + as_of_ns=self.as_of_ns, + max_results=self.motif_index.config.max_matches, + ) + result = query_reference_motifs(self.motif_index, query) + batch = generate_empirical_motif_candidates( + run=self.run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=result, + config=self.config, + ) + proposals.extend(batch.events) + benchmark_anchors = [ + _benchmark_anchor(item, ensemble_member_id) for item in ordered + ] + benchmark_proposals = [ + BenchmarkEventV1.from_synthetic_event( + item, + epoch_id=scenario.epoch_id, + session=self.condition.session_state, + event_state=self.condition.activity_regime, + sparsity="empirical-motif-candidate", + ) + for item in proposals + ] + return _benchmark_update_states( + tuple((*benchmark_anchors, *benchmark_proposals)) + ) + + +def _benchmark_update_states( + events: Sequence[BenchmarkEventV1], +) -> tuple[BenchmarkEventV1, ...]: + """Recompute bid/ask transition marks after anchors and proposals merge.""" + ordered = tuple( + sorted( + events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.benchmark_event_id, + ), + ) + ) + previous: dict[str, BenchmarkEventV1] = {} + result: list[BenchmarkEventV1] = [] + for item in ordered: + prior = previous.get(item.symbol) + if prior is None or (prior.bid != item.bid and prior.ask != item.ask): + state = "update_joint" + elif prior.bid == item.bid and prior.ask == item.ask: + state = "unchanged" + elif prior.bid != item.bid: + state = "update_bid_only" + else: + state = "update_ask_only" + result.append(replace(item, event_state=state, benchmark_event_id="")) + previous[item.symbol] = item + return tuple(result) + + +def _validate_generation_scope( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + left_anchor: SyntheticEventV1, + right_anchor: SyntheticEventV1, + query_result: ReferenceMotifQueryResultV1, + config: EmpiricalMotifGeneratorConfigV1, +) -> None: + if window.run_id != run.run_id: + raise ValueError("generation window does not belong to the run") + if window.ensemble_member_id not in run.ensemble_member_ids: + raise ValueError("generation member does not belong to the run") + if config.config_id not in run.configuration_ids: + raise ValueError("motif generator config is absent from the run") + for anchor in (left_anchor, right_anchor): + if anchor.origin is not SyntheticEventOrigin.OBSERVED: + raise ValueError("motif generation anchors must be observed") + if anchor.run_id != run.run_id: + raise ValueError("motif generation anchor run differs") + if anchor.ensemble_member_id != window.ensemble_member_id: + raise ValueError("motif generation anchor member differs") + if ( + anchor.symbol not in window.symbols + or anchor.symbol not in run.symbols + ): + raise ValueError("motif generation anchor symbol is outside scope") + if anchor.source_version_id not in run.source_version_ids: + raise ValueError("motif generation anchor source differs") + if not window.reads_event_time(anchor.event_time_ns): + raise ValueError("generation window does not read both anchors") + if left_anchor.symbol != right_anchor.symbol: + raise ValueError("motif generation anchors have different symbols") + if left_anchor.source_version_id != right_anchor.source_version_id: + raise ValueError("motif generation anchors have different sources") + if ( + query_result.query.condition.symbol.upper() + != left_anchor.symbol.upper() + ): + raise ValueError("motif query condition symbol differs from anchors") + if query_result.query.used_at_ns != right_anchor.event_time_ns: + raise ValueError( + "motif query use time must equal the right anchor boundary" + ) + + +def _target_cardinality( + condition: ReferenceMotifConditionV1, + gap_ns: int, +) -> tuple[int, int]: + metrics = condition.metrics + intensity = metrics.get("tick_intensity") + if intensity is not None: + if intensity <= 0.0: + return 0, gap_ns + cadence = max(1, round(NANOSECONDS_PER_SECOND / intensity)) + else: + interarrival = metrics.get("interarrival_ns") + if interarrival is None or interarrival <= 0.0: + return 0, gap_ns + cadence = max(1, round(interarrival)) + return max(0, (gap_ns - 1) // cadence), cadence + + +def _candidate_times( + left_time_ns: int, + right_time_ns: int, + count: int, + cadence_ns: int, + condition: ReferenceMotifConditionV1, +) -> tuple[int, ...]: + gap_ns = right_time_ns - left_time_ns + precision = max( + 1, + round(condition.metrics.get("timestamp_precision_ns", 1.0)), + ) + values: list[int] = [] + for ordinal in range(1, count + 1): + requested_offset = ordinal * cadence_ns + quantized = ( + (2 * requested_offset + precision) // (2 * precision) + ) * precision + quantized = max(1, min(gap_ns - 1, quantized)) + values.append(left_time_ns + quantized) + return tuple(values) + + +def _motif_warped_event_times( + *, + uniform_event_times: tuple[int, ...], + left_time_ns: int, + right_time_ns: int, + plan: _PlannedTransform, + timestamp_precision_ns: int, + required_event_time_ns: int | None, +) -> tuple[int, ...]: + """Warp one planned segment with the selected motif's event clock.""" + start = ( + left_time_ns + if plan.start_index == 0 + else uniform_event_times[plan.start_index - 1] + ) + end_index = plan.start_index + plan.event_count + end = ( + right_time_ns + if end_index == len(uniform_event_times) + else uniform_event_times[end_index] + ) + fragment = plan.match.fragment + source_gaps = tuple( + max(1, right - left) + for left, right in zip( + fragment.event_offsets_ns, fragment.event_offsets_ns[1:] + ) + ) + rotation = plan.record.seed % len(source_gaps) + gap_weights = tuple( + source_gaps[(rotation + index) % len(source_gaps)] + for index in range(plan.event_count + 1) + ) + total_weight = sum(gap_weights) + precision = max(1, timestamp_precision_ns) + values: list[int] = [] + previous = start + cumulative_weight = 0 + for local_index in range(plan.event_count): + cumulative_weight += gap_weights[local_index] + raw = start + round((end - start) * cumulative_weight / total_weight) + quantized = ((raw + precision // 2) // precision) * precision + selected = max(start + 1, min(end - 1, quantized)) + selected = max(previous, selected) + values.append(selected) + previous = selected + required = required_event_time_ns + if required is None or not start < required < end or not values: + return tuple(values) + if required in values: + return tuple(values) + insertion = bisect_left(values, required) + if insertion == len(values): + selected_index = len(values) - 1 + elif insertion == 0: + selected_index = 0 + else: + selected_index = min( + (insertion - 1, insertion), + key=lambda index: abs(values[index] - required), + ) + values[selected_index] = required + values.sort() + return tuple(values) + + +def _plan_transforms( + *, + run: ReconstructionRunV1, + ensemble_member_id: str, + interval_id: str, + event_times: tuple[int, ...], + cadence_ns: int, + left_time_ns: int, + query_result: ReferenceMotifQueryResultV1, + config: EmpiricalMotifGeneratorConfigV1, +) -> tuple[tuple[_PlannedTransform, ...], str | None]: + matches = query_result.matches + plans: list[_PlannedTransform] = [] + cursor = 0 + while cursor < len(event_times): + segment_ordinal = len(plans) + 1 + if segment_ordinal > config.max_transformations_per_interval: + return (), "transformation count exceeds configured interval limit" + seed = run.seed_for( + ensemble_member_id, + f"{EMPIRICAL_MOTIF_GENERATOR_ID}:{config.config_id}:" + f"{interval_id}:segment:{segment_ordinal}", + ) + start_match = seed % len(matches) + chosen: tuple[ReferenceMotifMatchV1, int, float] | None = None + previous_time = left_time_ns if cursor == 0 else event_times[cursor - 1] + for match_offset in range(len(matches)): + match = matches[(start_match + match_offset) % len(matches)] + # A compact empirical path may be interpolated to more or fewer + # target events. Its declared time-scale envelope, rather than + # its source row count, is the admissibility boundary. + capacity = len(event_times) - cursor + for event_count in range(capacity, 0, -1): + observed_duration = ( + event_times[cursor + event_count - 1] - previous_time + ) + duration = max(observed_duration, cadence_ns * event_count) + source_gap_count = max( + 1, len(match.fragment.event_offsets_ns) - 1 + ) + repeated_source_duration = match.fragment.duration_ns * ( + (event_count + 1) / source_gap_count + ) + time_scale = duration / repeated_source_duration + policy = match.fragment.transform_policy + terminal_scale = ( + duration + cadence_ns + ) / repeated_source_duration + if ( + event_count == capacity + and time_scale < policy.min_time_scale + and policy.min_time_scale + <= terminal_scale + <= policy.max_time_scale + ): + time_scale = terminal_scale + if policy.min_time_scale <= time_scale <= policy.max_time_scale: + chosen = (match, event_count, time_scale) + break + if chosen is not None: + break + if chosen is None: + return ( + (), + "no retrieved fragment supports the required cadence/time scale " + f"at output ordinal {cursor + 1}", + ) + match, event_count, time_scale = chosen + requested_price_scale, applied_price_scale = _price_scale( + query_result.query.condition, match.fragment + ) + confidence = round( + 1.0 / (1.0 + match.distance), + config.confidence_rounding_digits, + ) + fragment = match.fragment + record = EmpiricalMotifTransformationV1( + index_id=query_result.index_id, + query_id=query_result.query.query_id, + query_result_id=query_result.result_id, + source_fragment_id=fragment.fragment_id, + source_window_id=fragment.source_window_id, + source_series_id=fragment.source_series_id, + source_period=fragment.period, + source_artifact_sha256=fragment.source_artifact.sha256, + backoff_level=match.backoff_level, + cell_support=match.cell_support, + match_distance=match.distance, + segment_ordinal=segment_ordinal, + output_start_ordinal=cursor + 1, + output_end_ordinal=cursor + event_count, + source_event_count=len(fragment.event_offsets_ns), + time_scale=time_scale, + time_warp_ratio=1.0, + requested_price_scale=requested_price_scale, + applied_price_scale=applied_price_scale, + price_scale_clamped=(requested_price_scale != applied_price_scale), + spread_shape_applied=fragment.transform_policy.allow_spread_scaling, + seed=seed, + confidence=confidence, + ) + plans.append( + _PlannedTransform( + match=match, + record=record, + start_index=cursor, + event_count=event_count, + ) + ) + cursor += event_count + return tuple(plans), None + + +def _price_scale( + target: ReferenceMotifConditionV1, + fragment: ReferenceMotifFragmentV1, +) -> tuple[float, float]: + target_volatility = target.metrics.get("volatility", 0.0) + source_volatility = fragment.condition.metrics.get("volatility", 0.0) + requested = ( + target_volatility / source_volatility + if target_volatility > 0.0 and source_volatility > 0.0 + else 1.0 + ) + policy = fragment.transform_policy + return requested, min( + policy.max_price_scale, + max(policy.min_price_scale, requested), + ) + + +def _events_for_transform( + *, + run: ReconstructionRunV1, + left_anchor: SyntheticEventV1, + right_anchor: SyntheticEventV1, + interval_id: str, + query_result: ReferenceMotifQueryResultV1, + config: EmpiricalMotifGeneratorConfigV1, + event_times: tuple[int, ...], + plan: _PlannedTransform, + required_event_time_ns: int | None, +) -> tuple[ + tuple[SyntheticEventV1, ...], + tuple[EmpiricalMotifEventLineageV1, ...], + str | None, +]: + fragment = plan.match.fragment + left_mid = (left_anchor.bid + left_anchor.ask) / 2.0 + right_mid = (right_anchor.bid + right_anchor.ask) / 2.0 + left_spread = left_anchor.ask - left_anchor.bid + right_spread = right_anchor.ask - right_anchor.bid + gap_ns = right_anchor.event_time_ns - left_anchor.event_time_ns + precision_digits = _price_precision_digits( + query_result.query.condition, config + ) + warped_event_times = _motif_warped_event_times( + uniform_event_times=event_times, + left_time_ns=left_anchor.event_time_ns, + right_time_ns=right_anchor.event_time_ns, + plan=plan, + timestamp_precision_ns=max( + 1, + round( + query_result.query.condition.metrics.get( + "timestamp_precision_ns", 1.0 + ) + ), + ), + required_event_time_ns=required_event_time_ns, + ) + midpoint_deltas = tuple( + (bid_delta + ask_delta) / 2.0 + for bid_delta, ask_delta in zip( + fragment.bid_deltas, fragment.ask_deltas + ) + ) + spread_deltas = tuple( + ask_delta - bid_delta + for bid_delta, ask_delta in zip( + fragment.bid_deltas, fragment.ask_deltas + ) + ) + events: list[SyntheticEventV1] = [] + lineages: list[EmpiricalMotifEventLineageV1] = [] + previous_bid = left_anchor.bid + previous_ask = left_anchor.ask + for local_index in range(plan.event_count): + global_index = plan.start_index + local_index + global_ordinal = global_index + 1 + event_time_ns = warped_event_times[local_index] + anchor_progress = (event_time_ns - left_anchor.event_time_ns) / gap_ns + source_progress = (local_index + 1) / plan.event_count + source_mid = _interpolate_fragment( + fragment, midpoint_deltas, source_progress + ) + source_mid_residual = source_mid - ( + source_progress * midpoint_deltas[-1] + ) + mid = ( + left_mid + + anchor_progress * (right_mid - left_mid) + + plan.record.applied_price_scale * source_mid_residual + ) + spread = left_spread + anchor_progress * (right_spread - left_spread) + if plan.record.spread_shape_applied: + source_spread = _interpolate_fragment( + fragment, spread_deltas, source_progress + ) + spread += plan.record.applied_price_scale * ( + source_spread - source_progress * spread_deltas[-1] + ) + bid = round(mid - spread / 2.0, precision_digits) + ask = round(mid + spread / 2.0, precision_digits) + transition_index = max( + 1, + min( + len(fragment.transitions) - 1, + round(source_progress * (len(fragment.transitions) - 1)), + ), + ) + transition = fragment.transitions[transition_index].value + if local_index == plan.event_count - 1: + transition = "seam" + if transition == "unchanged": + bid, ask = previous_bid, previous_ask + elif transition == "bid": + ask = previous_ask + bid = min(bid, ask) + bid = _force_changed_mark( + bid, + previous_bid, + upper_bound=ask, + precision_digits=precision_digits, + ) + elif transition == "ask": + bid = previous_bid + ask = max(ask, bid) + ask = _force_changed_mark( + ask, + previous_ask, + lower_bound=bid, + precision_digits=precision_digits, + ) + elif transition == "both": + bid = _force_changed_mark( + bid, + previous_bid, + upper_bound=ask, + precision_digits=precision_digits, + ) + ask = _force_changed_mark( + ask, + previous_ask, + lower_bound=bid, + precision_digits=precision_digits, + ) + if not ( + math.isfinite(bid) + and math.isfinite(ask) + and bid > 0.0 + and ask > 0.0 + and ask >= bid + ): + return ( + (), + (), + ( + "transformed quote violates positive bid/ask or spread domain " + f"at output ordinal {global_ordinal}" + ), + ) + previous_bid, previous_ask = bid, ask + event = SyntheticEventV1.generated( + symbol=left_anchor.symbol, + event_time_ns=event_time_ns, + event_sequence=global_ordinal, + bid=bid, + ask=ask, + run_id=run.run_id, + ensemble_member_id=left_anchor.ensemble_member_id, + source_version_id=left_anchor.source_version_id, + anchor_interval_id=interval_id, + left_anchor_event_id=left_anchor.event_id, + right_anchor_event_id=right_anchor.event_id, + generator_id=EMPIRICAL_MOTIF_GENERATOR_ID, + generator_version=EMPIRICAL_MOTIF_GENERATOR_VERSION, + generator_config_id=config.config_id, + reference_id=query_result.result_id, + motif_id=fragment.fragment_id, + feed_epoch_id=query_result.query.condition.feed_epoch_id, + constraint_set_id=config.constraint_set_id, + # Motif similarity is retrieval evidence, not calibrated pointwise + # probability. It remains on the transformation while generated + # events reserve confidence for explicitly calibrated quantities. + confidence=None, + ) + events.append(event) + lineages.append( + EmpiricalMotifEventLineageV1( + event_id=event.event_id, + transformation_id=plan.record.transformation_id, + global_event_ordinal=global_ordinal, + segment_event_ordinal=local_index + 1, + source_progress=source_progress, + anchor_progress=anchor_progress, + requested_event_time_ns=event_time_ns, + ) + ) + return tuple(events), tuple(lineages), None + + +def _force_changed_mark( + value: float, + previous: float, + *, + precision_digits: int, + lower_bound: float | None = None, + upper_bound: float | None = None, +) -> float: + """Preserve an empirical transition after quote rounding.""" + if value != previous: + return value + tick = 10.0 ** (-precision_digits) + candidates = (previous + tick, previous - tick) + for candidate in candidates: + rounded = round(candidate, precision_digits) + if ( + rounded > 0.0 + and rounded != previous + and (lower_bound is None or rounded >= lower_bound) + and (upper_bound is None or rounded <= upper_bound) + ): + return rounded + return value + + +def _interpolate_fragment( + fragment: ReferenceMotifFragmentV1, + values: tuple[float, ...], + progress: float, +) -> float: + target_offset = progress * fragment.duration_ns + offsets = fragment.event_offsets_ns + for index in range(1, len(offsets)): + right_offset = offsets[index] + if target_offset > right_offset: + continue + left_offset = offsets[index - 1] + if right_offset == left_offset: + return values[index] + local = (target_offset - left_offset) / (right_offset - left_offset) + return float( + values[index - 1] + local * (values[index] - values[index - 1]) + ) + return values[-1] + + +def _price_precision_digits( + condition: ReferenceMotifConditionV1, + config: EmpiricalMotifGeneratorConfigV1, +) -> int: + value = condition.metrics.get("price_precision_digits") + if value is None: + return config.fallback_price_precision_digits + rounded = int(round(_finite_float(value, "price_precision_digits"))) + return min(MAX_PRICE_PRECISION_DIGITS, max(0, rounded)) + + +def _resource_estimate( + candidate_count: int, + config: EmpiricalMotifGeneratorConfigV1, +) -> ReconstructionResourceEstimateV1: + estimated_bytes = candidate_count * config.estimated_bytes_per_event + return ReconstructionResourceEstimateV1( + input_event_count=2, + candidate_event_count=candidate_count, + retained_ensemble_members=1, + inflight_batches=1 if candidate_count else 0, + peak_events_per_batch=candidate_count, + estimated_memory_bytes=estimated_bytes, + estimated_scratch_bytes=0, + estimated_output_bytes=estimated_bytes, + estimated_batch_count=1 if candidate_count else 0, + ) + + +def _decision_batch( + *, + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + left_anchor: SyntheticEventV1, + right_anchor: SyntheticEventV1, + interval_id: str, + query_result: ReferenceMotifQueryResultV1, + config: EmpiricalMotifGeneratorConfigV1, + status: MotifGenerationStatus, + decision: MotifGenerationDecision, + target_event_count: int, + estimate: ReconstructionResourceEstimateV1, + details: Sequence[str] = (), +) -> EmpiricalMotifCandidateBatchV1: + return EmpiricalMotifCandidateBatchV1( + run_id=run.run_id, + window_id=window.window_id, + ensemble_member_id=window.ensemble_member_id, + symbol=left_anchor.symbol, + anchor_interval_id=interval_id, + left_anchor_event_id=left_anchor.event_id, + right_anchor_event_id=right_anchor.event_id, + generator_config=config, + query_result=query_result, + status=status, + decision=decision, + target_event_count=target_event_count, + events=(), + transformations=(), + event_lineage=(), + resource_estimate=estimate, + carry_state=_carry_state(run, window, left_anchor, right_anchor, ()), + decision_details=tuple(details), + ) + + +def _carry_state( + run: ReconstructionRunV1, + window: ReconstructionWindowV1, + left_anchor: SyntheticEventV1, + right_anchor: SyntheticEventV1, + events: Sequence[SyntheticEventV1], +) -> CarryStateV1: + last_event = events[-1] if events else left_anchor + if window.owns_event_time(right_anchor.event_time_ns): + last_event = right_anchor + watermark = min( + window.core_end_ns - 1, + max(window.core_start_ns, right_anchor.event_time_ns - 1), + ) + return CarryStateV1( + run_id=run.run_id, + ensemble_member_id=window.ensemble_member_id, + symbol_watermarks_ns={left_anchor.symbol: watermark}, + last_event_ids={left_anchor.symbol: last_event.event_id}, + ) + + +def _benchmark_anchor( + event: BenchmarkEventV1, + ensemble_member_id: str, +) -> BenchmarkEventV1: + return BenchmarkEventV1( + source_event_id=event.source_event_id, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + epoch_id=event.epoch_id, + session=event.session, + event_state=event.event_state, + sparsity=event.sparsity, + ensemble_member_id=ensemble_member_id, + anchor_id=event.anchor_id or event.source_event_id, + support_lower_mid=event.support_lower_mid, + support_upper_mid=event.support_upper_mid, + ) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _required_text(value: Any) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError("value must be non-empty text") + return value.strip() + + +def _bounded_text(value: Any, name: str, maximum: int) -> str: + result = _required_text(value) + if len(result) > maximum: + raise ValueError(f"{name} exceeds bounded text length") + return result + + +def _optional_text(value: Any) -> str | None: + if value is None or value == "": + return None + return _required_text(value) + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _nonnegative_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _positive_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result < 1: + raise ValueError(f"{name} must be positive") + return result + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _strict_bool(value: Any, name: str) -> bool: + if type(value) is not bool: + raise ValueError(f"{name} must be boolean") + return value + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError(f"{name} must be a sequence") + return tuple(_required_text(item) for item in value) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + if not isinstance(value, Mapping): + raise ValueError("contract JSON must contain an object") + return value + + +def _content_sha256(value: Any) -> str: + encoded = canonical_contract_json(value).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() diff --git a/src/histdatacom/synthetic/information.py b/src/histdatacom/synthetic/information.py new file mode 100644 index 00000000..b5f11b06 --- /dev/null +++ b/src/histdatacom/synthetic/information.py @@ -0,0 +1,1851 @@ +"""Point-in-time information contracts and leakage auditing. + +The contracts in this module separate historically informed reconstruction +from forward-looking simulation. They describe information use at artifact +granularity; event rows continue to reference their run and source lineage +without carrying a copy of the full information graph. + +Version-one schemas are immutable. A semantic change to a required field, +identity rule, audit rule, or validity claim requires a new schema version. +""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any, TypeVar, cast + +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.contracts import canonical_contract_json +from histdatacom.synthetic.streaming import ( + ReconstructionRunV1, + ReconstructionWindowV1, + validate_reconstruction_window_plan, +) + +RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-information-policy.v1" +) +RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-information-input.v1" +) +RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-information-split.v1" +) +RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-information-manifest.v1" +) +RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION = ( + "histdatacom.reconstruction-information-audit-finding.v1" +) +RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-information-audit-report.v1" +) + +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 +MAX_INFORMATION_INPUTS = 4096 +MAX_INFORMATION_PARENTS = 64 +MAX_INFORMATION_TEXT_LENGTH = 1024 +MAX_INFORMATION_EVIDENCE_BYTES = 16_384 +MAX_INFORMATION_AUDIT_FINDINGS = 512 +DEFAULT_INFORMATION_AUDIT_FINDINGS = 128 + +_EnumT = TypeVar("_EnumT", bound=Enum) + + +class InformationMode(str, Enum): + """Whether a run may use realized future information.""" + + EX_POST_RECONSTRUCTION = "ex_post_reconstruction" + EX_ANTE_SIMULATION = "ex_ante_simulation" + + @classmethod + def from_value(cls, value: str | "InformationMode") -> "InformationMode": + """Return a strict normalized information mode.""" + return _enum_value(cls, value, "information mode") + + +class InformationInputKind(str, Enum): + """Whether an input is sourced externally or derived in the graph.""" + + EXTERNAL = "external" + DERIVED = "derived" + + @classmethod + def from_value( + cls, value: str | "InformationInputKind" + ) -> "InformationInputKind": + """Return a strict normalized input kind.""" + return _enum_value(cls, value, "information input kind") + + +class InformationStage(str, Enum): + """Consumer stage for one declared information use.""" + + SOURCE = "source" + FEATURE = "feature" + MOTIF_SELECTION = "motif_selection" + CALENDAR_CONTEXT = "calendar_context" + NEWS_CONTEXT = "news_context" + MODEL_FIT = "model_fit" + CALIBRATION = "calibration" + CARVING = "carving" + GENERATION = "generation" + VALIDATION = "validation" + STRATEGY_EVALUATION = "strategy_evaluation" + + @classmethod + def from_value(cls, value: str | "InformationStage") -> "InformationStage": + """Return a strict normalized stage.""" + return _enum_value(cls, value, "information stage") + + +class InformationScope(str, Enum): + """Temporal scope that makes future-informed uses explicit.""" + + POINT_IN_TIME = "point_in_time" + REVISION = "revision" + FUTURE_ANCHOR = "future_anchor" + FULL_PERIOD_SUMMARY = "full_period_summary" + GLOBAL_NORMALIZATION = "global_normalization" + EMPIRICAL_MOTIF = "empirical_motif" + + @classmethod + def from_value(cls, value: str | "InformationScope") -> "InformationScope": + """Return a strict normalized temporal scope.""" + return _enum_value(cls, value, "information scope") + + +class InformationSplitKind(str, Enum): + """Chronological research split kinds.""" + + TRAIN = "train" + CALIBRATION = "calibration" + VALIDATION = "validation" + + @classmethod + def from_value( + cls, value: str | "InformationSplitKind" + ) -> "InformationSplitKind": + """Return a strict normalized split kind.""" + return _enum_value(cls, value, "information split kind") + + +class InformationAuditRule(str, Enum): + """Stable rule identifiers emitted by the information audit.""" + + POLICY_MODE_MISMATCH = "INFORMATION_POLICY_MODE_MISMATCH" + POLICY_ID_MISMATCH = "INFORMATION_POLICY_ID_MISMATCH" + POLICY_NOT_BOUND_TO_RUN = "INFORMATION_POLICY_NOT_BOUND_TO_RUN" + RUN_ID_MISMATCH = "INFORMATION_RUN_ID_MISMATCH" + WINDOW_PLAN_EMPTY = "INFORMATION_WINDOW_PLAN_EMPTY" + WINDOW_PLAN_MISMATCH = "INFORMATION_WINDOW_PLAN_MISMATCH" + WINDOW_PLAN_INVALID = "INFORMATION_WINDOW_PLAN_INVALID" + WINDOW_RUN_MISMATCH = "INFORMATION_WINDOW_RUN_MISMATCH" + WINDOW_SCOPE_MISMATCH = "INFORMATION_WINDOW_SCOPE_MISMATCH" + WINDOW_MEMBER_MISSING = "INFORMATION_WINDOW_MEMBER_MISSING" + WINDOW_MEMBER_PLAN_MISMATCH = "INFORMATION_WINDOW_MEMBER_PLAN_MISMATCH" + WINDOW_LOOKAHEAD_EXCEEDS_POLICY = ( + "INFORMATION_WINDOW_LOOKAHEAD_EXCEEDS_POLICY" + ) + EX_ANTE_WINDOW_LOOKAHEAD = "INFORMATION_EX_ANTE_WINDOW_LOOKAHEAD" + DUPLICATE_INPUT_ID = "INFORMATION_DUPLICATE_INPUT_ID" + INPUT_GRAPH_EMPTY = "INFORMATION_INPUT_GRAPH_EMPTY" + INPUT_RUN_MISMATCH = "INFORMATION_INPUT_RUN_MISMATCH" + INPUT_MODE_MISMATCH = "INFORMATION_INPUT_MODE_MISMATCH" + INPUT_LOOKAHEAD_EXCEEDS_POLICY = ( + "INFORMATION_INPUT_LOOKAHEAD_EXCEEDS_POLICY" + ) + DERIVED_INPUT_WITHOUT_PARENT = "INFORMATION_DERIVED_INPUT_WITHOUT_PARENT" + MISSING_PARENT_INPUT = "INFORMATION_MISSING_PARENT_INPUT" + DERIVED_AVAILABLE_BEFORE_PARENT = ( + "INFORMATION_DERIVED_AVAILABLE_BEFORE_PARENT" + ) + GRAPH_CYCLE = "INFORMATION_GRAPH_CYCLE" + REVISION_SCOPE_UNDECLARED = "INFORMATION_REVISION_SCOPE_UNDECLARED" + REVISION_PARENT_MISSING = "INFORMATION_REVISION_PARENT_MISSING" + REVISION_SEQUENCE_INVALID = "INFORMATION_REVISION_SEQUENCE_INVALID" + REVISION_AVAILABILITY_INVALID = "INFORMATION_REVISION_AVAILABILITY_INVALID" + SPLIT_MISSING = "INFORMATION_SPLIT_MISSING" + SPLIT_DUPLICATE = "INFORMATION_SPLIT_DUPLICATE" + SPLIT_DECLARATION_ORDER = "INFORMATION_SPLIT_DECLARATION_ORDER" + SPLIT_TIME_ORDER = "INFORMATION_SPLIT_TIME_ORDER" + INPUT_SPLIT_MISSING = "INFORMATION_INPUT_SPLIT_MISSING" + INPUT_SPLIT_MISMATCH = "INFORMATION_INPUT_SPLIT_MISMATCH" + INPUT_OUTSIDE_SPLIT = "INFORMATION_INPUT_OUTSIDE_SPLIT" + EX_ANTE_LOOKAHEAD_DECLARED = "INFORMATION_EX_ANTE_LOOKAHEAD_DECLARED" + EX_ANTE_INPUT_NOT_AVAILABLE = "INFORMATION_EX_ANTE_INPUT_NOT_AVAILABLE" + EX_ANTE_REVISION_NOT_AVAILABLE = ( + "INFORMATION_EX_ANTE_REVISION_NOT_AVAILABLE" + ) + EX_ANTE_FUTURE_EVENT = "INFORMATION_EX_ANTE_FUTURE_EVENT" + EX_ANTE_FUTURE_OBSERVATION = "INFORMATION_EX_ANTE_FUTURE_OBSERVATION" + EX_ANTE_FUTURE_ANCHOR = "INFORMATION_EX_ANTE_FUTURE_ANCHOR" + EX_ANTE_FULL_PERIOD_SUMMARY = "INFORMATION_EX_ANTE_FULL_PERIOD_SUMMARY" + EX_ANTE_GLOBAL_NORMALIZATION = "INFORMATION_EX_ANTE_GLOBAL_NORMALIZATION" + EX_ANTE_MOTIF_SELECTION_LEAKAGE = ( + "INFORMATION_EX_ANTE_MOTIF_SELECTION_LEAKAGE" + ) + EX_POST_UNLABELED_FUTURE_INFORMATION = ( + "INFORMATION_EX_POST_UNLABELED_FUTURE_INFORMATION" + ) + EX_POST_LOOKAHEAD_EXCEEDED = "INFORMATION_EX_POST_LOOKAHEAD_EXCEEDED" + + @classmethod + def from_value( + cls, value: str | "InformationAuditRule" + ) -> "InformationAuditRule": + """Return a strict stable audit rule.""" + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().upper()) + except ValueError as err: + raise ValueError("unsupported information audit rule") from err + + +@dataclass(frozen=True, slots=True) +class ReconstructionInformationPolicyV1: + """Run-bound mode and fail-closed audit limits.""" + + information_mode: InformationMode + max_allowed_lookahead_ns: int = 0 + max_retained_findings: int = DEFAULT_INFORMATION_AUDIT_FINDINGS + fail_closed: bool = True + require_time_ordered_splits: bool = True + policy_id: str = "" + schema_version: str = RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction information policy schema" + ) + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + lookahead = _nonnegative_int64( + self.max_allowed_lookahead_ns, + "max_allowed_lookahead_ns", + ) + if ( + self.information_mode is InformationMode.EX_ANTE_SIMULATION + and lookahead != 0 + ): + raise ValueError( + "ex-ante information policy requires zero look-ahead" + ) + object.__setattr__(self, "max_allowed_lookahead_ns", lookahead) + retained = _positive_int( + self.max_retained_findings, + "max_retained_findings", + ) + if retained > MAX_INFORMATION_AUDIT_FINDINGS: + raise ValueError("max_retained_findings exceeds the v1 limit") + object.__setattr__(self, "max_retained_findings", retained) + if self.fail_closed is not True: + raise ValueError("v1 information policies must fail closed") + if self.require_time_ordered_splits is not True: + raise ValueError("v1 information policies require ordered splits") + expected = _stable_id("information-policy", self.identity_payload()) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("policy_id does not match deterministic identity") + object.__setattr__(self, "policy_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic policy identity.""" + return { + "schema_version": self.schema_version, + "information_mode": self.information_mode.value, + "max_allowed_lookahead_ns": self.max_allowed_lookahead_ns, + "fail_closed": self.fail_closed, + "require_time_ordered_splits": self.require_time_ordered_splits, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible policy metadata.""" + return { + **self.identity_payload(), + "max_retained_findings": self.max_retained_findings, + "policy_id": self.policy_id, + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionInformationPolicyV1": + """Restore and verify a version-one information policy.""" + _require_schema(data, RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION) + return cls( + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + max_allowed_lookahead_ns=cast( + int, data.get("max_allowed_lookahead_ns", 0) + ), + max_retained_findings=cast( + int, + data.get( + "max_retained_findings", + DEFAULT_INFORMATION_AUDIT_FINDINGS, + ), + ), + fail_closed=_strict_bool(data.get("fail_closed"), "fail_closed"), + require_time_ordered_splits=_strict_bool( + data.get("require_time_ordered_splits"), + "require_time_ordered_splits", + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionInformationPolicyV1": + """Restore a policy from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionInformationInputV1: + """One external or derived artifact use with temporal availability.""" + + run_id: str + artifact_id: str + information_mode: InformationMode + input_kind: InformationInputKind + stage: InformationStage + scope: InformationScope + event_time_ns: int + available_at_ns: int + used_at_ns: int + observation_start_ns: int + observation_end_ns: int + vintage_id: str + reason: str + revision_sequence: int = 0 + supersedes_input_id: str | None = None + allowed_lookahead_ns: int = 0 + parent_input_ids: tuple[str, ...] = () + split_kind: InformationSplitKind | None = None + input_id: str = "" + schema_version: str = RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction information input schema" + ) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, "artifact_id", _required_text(self.artifact_id) + ) + object.__setattr__(self, "vintage_id", _required_text(self.vintage_id)) + object.__setattr__(self, "reason", _required_text(self.reason)) + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + object.__setattr__( + self, + "input_kind", + InformationInputKind.from_value(self.input_kind), + ) + object.__setattr__( + self, + "stage", + InformationStage.from_value(self.stage), + ) + object.__setattr__( + self, + "scope", + InformationScope.from_value(self.scope), + ) + for name in ( + "event_time_ns", + "available_at_ns", + "used_at_ns", + "observation_start_ns", + "observation_end_ns", + ): + object.__setattr__( + self, + name, + _bounded_int64(getattr(self, name), name), + ) + if self.observation_end_ns < self.observation_start_ns: + raise ValueError( + "observation_end_ns must not precede observation_start_ns" + ) + if ( + not self.observation_start_ns + <= self.event_time_ns + <= self.observation_end_ns + ): + raise ValueError( + "event_time_ns must lie inside the observation interval" + ) + revision = _nonnegative_int( + self.revision_sequence, + "revision_sequence", + ) + object.__setattr__(self, "revision_sequence", revision) + supersedes = _optional_text(self.supersedes_input_id) + if revision > 0 and supersedes is None: + raise ValueError("revised information requires supersedes_input_id") + if revision == 0 and supersedes is not None: + raise ValueError( + "initial information cannot supersede another input" + ) + object.__setattr__(self, "supersedes_input_id", supersedes) + object.__setattr__( + self, + "allowed_lookahead_ns", + _nonnegative_int64( + self.allowed_lookahead_ns, + "allowed_lookahead_ns", + ), + ) + parents = _normalized_ids(self.parent_input_ids) + if len(parents) > MAX_INFORMATION_PARENTS: + raise ValueError("parent_input_ids exceeds the v1 limit") + object.__setattr__(self, "parent_input_ids", parents) + if self.split_kind is not None: + object.__setattr__( + self, + "split_kind", + InformationSplitKind.from_value(self.split_kind), + ) + expected = _stable_id("information-input", self.identity_payload()) + supplied = _optional_text(self.input_id) + if supplied is not None and supplied != expected: + raise ValueError("input_id does not match deterministic identity") + object.__setattr__(self, "input_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic input identity.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "artifact_id": self.artifact_id, + "information_mode": self.information_mode.value, + "input_kind": self.input_kind.value, + "stage": self.stage.value, + "scope": self.scope.value, + "event_time_ns": self.event_time_ns, + "available_at_ns": self.available_at_ns, + "used_at_ns": self.used_at_ns, + "observation_start_ns": self.observation_start_ns, + "observation_end_ns": self.observation_end_ns, + "vintage_id": self.vintage_id, + "reason": self.reason, + "revision_sequence": self.revision_sequence, + "supersedes_input_id": self.supersedes_input_id, + "allowed_lookahead_ns": self.allowed_lookahead_ns, + "parent_input_ids": list(self.parent_input_ids), + "split_kind": ( + self.split_kind.value if self.split_kind is not None else None + ), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible input metadata.""" + return {**self.identity_payload(), "input_id": self.input_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionInformationInputV1": + """Restore and verify one version-one information input.""" + _require_schema(data, RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION) + split_value = data.get("split_kind") + return cls( + run_id=str(data.get("run_id", "")), + artifact_id=str(data.get("artifact_id", "")), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + input_kind=InformationInputKind.from_value( + str(data.get("input_kind", "")) + ), + stage=InformationStage.from_value(str(data.get("stage", ""))), + scope=InformationScope.from_value(str(data.get("scope", ""))), + event_time_ns=cast(int, data.get("event_time_ns")), + available_at_ns=cast(int, data.get("available_at_ns")), + used_at_ns=cast(int, data.get("used_at_ns")), + observation_start_ns=cast(int, data.get("observation_start_ns")), + observation_end_ns=cast(int, data.get("observation_end_ns")), + vintage_id=str(data.get("vintage_id", "")), + reason=str(data.get("reason", "")), + revision_sequence=cast(int, data.get("revision_sequence", 0)), + supersedes_input_id=_optional_text(data.get("supersedes_input_id")), + allowed_lookahead_ns=cast(int, data.get("allowed_lookahead_ns", 0)), + parent_input_ids=_string_tuple(data.get("parent_input_ids")), + split_kind=( + InformationSplitKind.from_value(str(split_value)) + if split_value is not None + else None + ), + input_id=str(data.get("input_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionInformationInputV1": + """Restore an information input from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionInformationSplitV1: + """One half-open chronological train, calibration, or validation split.""" + + kind: InformationSplitKind + start_ns: int + end_ns: int + split_id: str = "" + schema_version: str = RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction information split schema" + ) + object.__setattr__( + self, + "kind", + InformationSplitKind.from_value(self.kind), + ) + object.__setattr__( + self, "start_ns", _bounded_int64(self.start_ns, "start_ns") + ) + object.__setattr__( + self, "end_ns", _bounded_int64(self.end_ns, "end_ns") + ) + if self.end_ns <= self.start_ns: + raise ValueError( + "information split end_ns must be greater than start_ns" + ) + expected = _stable_id("information-split", self.identity_payload()) + supplied = _optional_text(self.split_id) + if supplied is not None and supplied != expected: + raise ValueError("split_id does not match deterministic identity") + object.__setattr__(self, "split_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic split identity.""" + return { + "schema_version": self.schema_version, + "kind": self.kind.value, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "interval": "[start_ns,end_ns)", + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible split metadata.""" + return {**self.identity_payload(), "split_id": self.split_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionInformationSplitV1": + """Restore and verify a version-one split.""" + _require_schema(data, RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION) + return cls( + kind=InformationSplitKind.from_value(str(data.get("kind", ""))), + start_ns=cast(int, data.get("start_ns")), + end_ns=cast(int, data.get("end_ns")), + split_id=str(data.get("split_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionInformationSplitV1": + """Restore a split from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionInformationManifestV1: + """One run's mode, complete artifact-use graph, and research splits.""" + + run_id: str + policy_id: str + information_mode: InformationMode + window_plan_id: str + inputs: tuple[ReconstructionInformationInputV1, ...] + splits: tuple[ReconstructionInformationSplitV1, ...] + manifest_id: str = "" + schema_version: str = RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction information manifest schema" + ) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__(self, "policy_id", _required_text(self.policy_id)) + object.__setattr__( + self, + "window_plan_id", + _required_text(self.window_plan_id), + ) + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + inputs = tuple(self.inputs) + if len(inputs) > MAX_INFORMATION_INPUTS: + raise ValueError("information inputs exceed the v1 limit") + if any( + not isinstance(item, ReconstructionInformationInputV1) + for item in inputs + ): + raise ValueError( + "inputs must contain version-one information inputs" + ) + object.__setattr__( + self, + "inputs", + tuple(sorted(inputs, key=lambda item: item.input_id)), + ) + splits = tuple(self.splits) + if any( + not isinstance(item, ReconstructionInformationSplitV1) + for item in splits + ): + raise ValueError( + "splits must contain version-one information splits" + ) + object.__setattr__(self, "splits", splits) + expected = _stable_id("information-manifest", self.identity_payload()) + supplied = _optional_text(self.manifest_id) + if supplied is not None and supplied != expected: + raise ValueError( + "manifest_id does not match deterministic identity" + ) + object.__setattr__(self, "manifest_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic manifest identity.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "policy_id": self.policy_id, + "information_mode": self.information_mode.value, + "window_plan_id": self.window_plan_id, + "inputs": [item.to_dict() for item in self.inputs], + "splits": [item.to_dict() for item in self.splits], + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible manifest metadata.""" + return {**self.identity_payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionInformationManifestV1": + """Restore and verify a version-one information manifest.""" + _require_schema( + data, RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION + ) + return cls( + run_id=str(data.get("run_id", "")), + policy_id=str(data.get("policy_id", "")), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + window_plan_id=str(data.get("window_plan_id", "")), + inputs=tuple( + ReconstructionInformationInputV1.from_dict(item) + for item in _mapping_sequence(data.get("inputs")) + ), + splits=tuple( + ReconstructionInformationSplitV1.from_dict(item) + for item in _mapping_sequence(data.get("splits")) + ), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionInformationManifestV1": + """Restore a manifest from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class InformationAuditFindingV1: + """One deterministic fail-closed information violation.""" + + rule_id: InformationAuditRule + message: str + input_id: str | None = None + evidence: Mapping[str, JSONValue] | None = None + finding_id: str = "" + schema_version: str = ( + RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION + ) + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION + ): + raise ValueError("unsupported information audit finding schema") + object.__setattr__( + self, + "rule_id", + InformationAuditRule.from_value(self.rule_id), + ) + object.__setattr__(self, "message", _required_text(self.message)) + object.__setattr__(self, "input_id", _optional_text(self.input_id)) + evidence = dict(self.evidence or {}) + _bounded_json( + evidence, MAX_INFORMATION_EVIDENCE_BYTES, "finding evidence" + ) + object.__setattr__(self, "evidence", evidence) + expected = _stable_id("information-finding", self.identity_payload()) + supplied = _optional_text(self.finding_id) + if supplied is not None and supplied != expected: + raise ValueError("finding_id does not match deterministic identity") + object.__setattr__(self, "finding_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic finding identity.""" + return { + "schema_version": self.schema_version, + "severity": "error", + "rule_id": self.rule_id.value, + "message": self.message, + "input_id": self.input_id, + "evidence": dict(self.evidence or {}), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible finding metadata.""" + return {**self.identity_payload(), "finding_id": self.finding_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "InformationAuditFindingV1": + """Restore and verify one audit finding.""" + _require_schema( + data, + RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION, + ) + return cls( + rule_id=InformationAuditRule.from_value( + str(data.get("rule_id", "")) + ), + message=str(data.get("message", "")), + input_id=_optional_text(data.get("input_id")), + evidence=_mapping(data.get("evidence")), + finding_id=str(data.get("finding_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "InformationAuditFindingV1": + """Restore an audit finding from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class InformationAuditReportV1: + """Bounded audit result and explicit downstream-validity statement.""" + + run_id: str + policy_id: str + manifest_id: str + window_plan_id: str + information_mode: InformationMode + accepted: bool + total_violation_count: int + findings: tuple[InformationAuditFindingV1, ...] + evidence_truncated: bool + valid_for: tuple[str, ...] + invalid_for: tuple[str, ...] + summary: str + audit_id: str = "" + schema_version: str = RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction information audit report schema" + ) + for name in ( + "run_id", + "policy_id", + "manifest_id", + "window_plan_id", + "summary", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + object.__setattr__( + self, + "accepted", + _strict_bool(self.accepted, "accepted"), + ) + object.__setattr__( + self, + "evidence_truncated", + _strict_bool(self.evidence_truncated, "evidence_truncated"), + ) + total = _nonnegative_int( + self.total_violation_count, + "total_violation_count", + ) + object.__setattr__(self, "total_violation_count", total) + findings = tuple(self.findings) + if any( + not isinstance(item, InformationAuditFindingV1) for item in findings + ): + raise ValueError("findings must contain version-one audit findings") + if len(findings) > total: + raise ValueError("retained findings cannot exceed total violations") + object.__setattr__(self, "findings", findings) + if self.accepted is not (total == 0): + raise ValueError("accepted must equal zero total violations") + if self.evidence_truncated is not (len(findings) < total): + raise ValueError("evidence_truncated does not match finding counts") + object.__setattr__(self, "valid_for", _normalized_ids(self.valid_for)) + object.__setattr__( + self, "invalid_for", _normalized_ids(self.invalid_for) + ) + expected = _stable_id("information-audit", self.identity_payload()) + supplied = _optional_text(self.audit_id) + if supplied is not None and supplied != expected: + raise ValueError("audit_id does not match deterministic identity") + object.__setattr__(self, "audit_id", expected) + + @property + def valid_for_strategy_usefulness_claim(self) -> bool: + """Return whether this report opens the strategy-usefulness gate.""" + return self.accepted and "strategy_usefulness_claims" in self.valid_for + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic report identity.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "policy_id": self.policy_id, + "manifest_id": self.manifest_id, + "window_plan_id": self.window_plan_id, + "information_mode": self.information_mode.value, + "accepted": self.accepted, + "total_violation_count": self.total_violation_count, + "retained_violation_count": len(self.findings), + "evidence_truncated": self.evidence_truncated, + "findings": [item.to_dict() for item in self.findings], + "valid_for": list(self.valid_for), + "invalid_for": list(self.invalid_for), + "valid_for_strategy_usefulness_claim": ( + self.valid_for_strategy_usefulness_claim + ), + "summary": self.summary, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible report metadata.""" + return {**self.identity_payload(), "audit_id": self.audit_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "InformationAuditReportV1": + """Restore and verify a version-one audit report.""" + _require_schema( + data, + RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION, + ) + return cls( + run_id=str(data.get("run_id", "")), + policy_id=str(data.get("policy_id", "")), + manifest_id=str(data.get("manifest_id", "")), + window_plan_id=str(data.get("window_plan_id", "")), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + accepted=_strict_bool(data.get("accepted"), "accepted"), + total_violation_count=cast(int, data.get("total_violation_count")), + findings=tuple( + InformationAuditFindingV1.from_dict(item) + for item in _mapping_sequence(data.get("findings")) + ), + evidence_truncated=_strict_bool( + data.get("evidence_truncated"), + "evidence_truncated", + ), + valid_for=_string_tuple(data.get("valid_for")), + invalid_for=_string_tuple(data.get("invalid_for")), + summary=str(data.get("summary", "")), + audit_id=str(data.get("audit_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "InformationAuditReportV1": + """Restore an audit report from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +class InformationLeakageError(ValueError): + """Fail-closed pre-generation rejection carrying the audit report.""" + + def __init__(self, report: InformationAuditReportV1) -> None: + self.report = report + super().__init__( + "reconstruction information audit failed with " + f"{report.total_violation_count} violation(s)" + ) + + +def audit_reconstruction_information( + manifest: ReconstructionInformationManifestV1, + policy: ReconstructionInformationPolicyV1, + *, + run: ReconstructionRunV1, + windows: Sequence[ReconstructionWindowV1], +) -> InformationAuditReportV1: + """Audit one complete artifact-use graph before generation begins.""" + findings: list[InformationAuditFindingV1] = [] + + def add( + rule: InformationAuditRule, + message: str, + *, + item: ReconstructionInformationInputV1 | None = None, + evidence: Mapping[str, JSONValue] | None = None, + ) -> None: + findings.append( + InformationAuditFindingV1( + rule_id=rule, + message=message, + input_id=item.input_id if item is not None else None, + evidence=evidence, + ) + ) + + if manifest.policy_id != policy.policy_id: + add( + InformationAuditRule.POLICY_ID_MISMATCH, + "manifest policy_id does not match the audited policy", + evidence={ + "manifest_policy_id": manifest.policy_id, + "audit_policy_id": policy.policy_id, + }, + ) + if manifest.information_mode is not policy.information_mode: + add( + InformationAuditRule.POLICY_MODE_MISMATCH, + "manifest information mode does not match the audited policy", + evidence={ + "manifest_mode": manifest.information_mode.value, + "policy_mode": policy.information_mode.value, + }, + ) + if manifest.run_id != run.run_id: + add( + InformationAuditRule.RUN_ID_MISMATCH, + "manifest run_id does not match the reconstruction run", + evidence={ + "manifest_run_id": manifest.run_id, + "run_id": run.run_id, + }, + ) + if policy.policy_id not in run.configuration_ids: + add( + InformationAuditRule.POLICY_NOT_BOUND_TO_RUN, + "information policy is not bound into run configuration_ids", + evidence={"policy_id": policy.policy_id}, + ) + + window_plan = tuple(windows) + if not window_plan: + add( + InformationAuditRule.WINDOW_PLAN_EMPTY, + "information audit requires the exact non-empty window plan", + ) + else: + actual_window_plan_id = reconstruction_information_window_plan_id( + window_plan + ) + if actual_window_plan_id != manifest.window_plan_id: + add( + InformationAuditRule.WINDOW_PLAN_MISMATCH, + "audited windows do not match the manifest window plan", + evidence={ + "manifest_window_plan_id": manifest.window_plan_id, + "audit_window_plan_id": actual_window_plan_id, + }, + ) + grouped_windows: dict[str, list[ReconstructionWindowV1]] = {} + for window in window_plan: + grouped_windows.setdefault(window.ensemble_member_id, []).append( + window + ) + if window.run_id != run.run_id: + add( + InformationAuditRule.WINDOW_RUN_MISMATCH, + "reconstruction window belongs to a different run", + evidence={ + "window_id": window.window_id, + "window_run_id": window.run_id, + "run_id": run.run_id, + }, + ) + if ( + window.ensemble_member_id not in run.ensemble_member_ids + or window.symbols != run.symbols + ): + add( + InformationAuditRule.WINDOW_SCOPE_MISMATCH, + "reconstruction window member or symbols differ from the run", + evidence={ + "window_id": window.window_id, + "ensemble_member_id": window.ensemble_member_id, + "symbols": list(window.symbols), + }, + ) + if window.right_lookahead_ns > policy.max_allowed_lookahead_ns: + add( + InformationAuditRule.WINDOW_LOOKAHEAD_EXCEEDS_POLICY, + "window right look-ahead exceeds the information policy", + evidence={ + "window_id": window.window_id, + "right_lookahead_ns": window.right_lookahead_ns, + "policy_max_allowed_lookahead_ns": ( + policy.max_allowed_lookahead_ns + ), + }, + ) + if ( + policy.information_mode is InformationMode.EX_ANTE_SIMULATION + and window.right_lookahead_ns != 0 + ): + add( + InformationAuditRule.EX_ANTE_WINDOW_LOOKAHEAD, + "ex-ante reconstruction windows cannot read future rows", + evidence={ + "window_id": window.window_id, + "right_lookahead_ns": window.right_lookahead_ns, + }, + ) + for member_id in run.ensemble_member_ids: + member_windows = grouped_windows.get(member_id, []) + if not member_windows: + add( + InformationAuditRule.WINDOW_MEMBER_MISSING, + "window plan omits an ensemble member declared by the run", + evidence={"ensemble_member_id": member_id}, + ) + continue + try: + validate_reconstruction_window_plan(member_windows) + except ValueError as err: + add( + InformationAuditRule.WINDOW_PLAN_INVALID, + "member window plan is not contiguous and synchronized", + evidence={ + "ensemble_member_id": member_id, + "reason": str(err), + }, + ) + member_signatures = { + member_id: tuple( + ( + window.core_start_ns, + window.core_end_ns, + window.left_halo_ns, + window.right_lookahead_ns, + window.symbols, + ) + for window in sorted( + member_windows, + key=lambda item: item.core_start_ns, + ) + ) + for member_id, member_windows in grouped_windows.items() + if member_id in run.ensemble_member_ids + } + if len(set(member_signatures.values())) > 1: + mismatched_member_ids = [ + cast(JSONValue, member_id) + for member_id in sorted(member_signatures) + ] + add( + InformationAuditRule.WINDOW_MEMBER_PLAN_MISMATCH, + "ensemble members do not share the same window boundaries", + evidence={ + "ensemble_member_ids": mismatched_member_ids, + }, + ) + + _audit_splits(manifest, add) + if not manifest.inputs: + add( + InformationAuditRule.INPUT_GRAPH_EMPTY, + "information manifest does not declare any external or derived input", + ) + counts = Counter(item.input_id for item in manifest.inputs) + for input_id in sorted(key for key, count in counts.items() if count > 1): + add( + InformationAuditRule.DUPLICATE_INPUT_ID, + "information manifest contains a duplicate input_id", + evidence={"input_id": input_id, "count": counts[input_id]}, + ) + by_id: dict[str, ReconstructionInformationInputV1] = {} + for item in manifest.inputs: + by_id.setdefault(item.input_id, item) + + _audit_graph(manifest, by_id, add) + split_by_kind = {split.kind: split for split in manifest.splits} + for item in manifest.inputs: + _audit_input_binding(item, manifest, policy, add) + _audit_input_split(item, split_by_kind, add) + _audit_revision(item, by_id, add) + _audit_parent_availability(item, by_id, add) + if policy.information_mode is InformationMode.EX_ANTE_SIMULATION: + _audit_ex_ante_input(item, add) + else: + _audit_ex_post_input(item, policy, add) + + ordered = sorted( + {finding.finding_id: finding for finding in findings}.values(), + key=lambda finding: ( + finding.rule_id.value, + finding.input_id or "", + finding.finding_id, + ), + ) + total = len(ordered) + retained = tuple(ordered[: policy.max_retained_findings]) + accepted = total == 0 + valid_for, invalid_for, summary = _validity_statement( + policy.information_mode, + accepted, + ) + return InformationAuditReportV1( + run_id=manifest.run_id, + policy_id=policy.policy_id, + manifest_id=manifest.manifest_id, + window_plan_id=manifest.window_plan_id, + information_mode=policy.information_mode, + accepted=accepted, + total_violation_count=total, + findings=retained, + evidence_truncated=len(retained) < total, + valid_for=valid_for, + invalid_for=invalid_for, + summary=summary, + ) + + +def require_reconstruction_information_audit( + manifest: ReconstructionInformationManifestV1, + policy: ReconstructionInformationPolicyV1, + *, + run: ReconstructionRunV1, + windows: Sequence[ReconstructionWindowV1], +) -> InformationAuditReportV1: + """Return an accepted audit or fail closed before generation.""" + report = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=windows, + ) + if not report.accepted: + raise InformationLeakageError(report) + return report + + +def reconstruction_information_window_plan_id( + windows: Sequence[ReconstructionWindowV1], +) -> str: + """Return a deterministic identity for the exact audited window plan.""" + selected = tuple(windows) + if not selected: + raise ValueError("information audit window plan cannot be empty") + if any(not isinstance(item, ReconstructionWindowV1) for item in selected): + raise ValueError("window plan must contain version-one windows") + ordered = sorted( + selected, + key=lambda item: ( + item.ensemble_member_id, + item.core_start_ns, + item.core_end_ns, + item.window_id, + ), + ) + return _stable_id( + "information-window-plan", + { + "window_schema_version": ordered[0].schema_version, + "windows": [item.to_dict() for item in ordered], + }, + ) + + +_EXPECTED_SPLITS = ( + InformationSplitKind.TRAIN, + InformationSplitKind.CALIBRATION, + InformationSplitKind.VALIDATION, +) +_REQUIRED_STAGE_SPLIT = { + InformationStage.MODEL_FIT: InformationSplitKind.TRAIN, + InformationStage.CALIBRATION: InformationSplitKind.CALIBRATION, + InformationStage.VALIDATION: InformationSplitKind.VALIDATION, + InformationStage.STRATEGY_EVALUATION: InformationSplitKind.VALIDATION, +} +_PREDECISION_STAGES = frozenset( + stage + for stage in InformationStage + if stage + not in {InformationStage.VALIDATION, InformationStage.STRATEGY_EVALUATION} +) +_EX_POST_LABELS = frozenset( + { + InformationScope.FUTURE_ANCHOR, + InformationScope.FULL_PERIOD_SUMMARY, + InformationScope.GLOBAL_NORMALIZATION, + InformationScope.EMPIRICAL_MOTIF, + } +) + + +def _audit_splits( + manifest: ReconstructionInformationManifestV1, + add: Any, +) -> None: + kinds = tuple(split.kind for split in manifest.splits) + counts = Counter(kinds) + for expected in _EXPECTED_SPLITS: + if counts[expected] == 0: + add( + InformationAuditRule.SPLIT_MISSING, + "information manifest is missing a required chronological split", + evidence={"missing_split": expected.value}, + ) + elif counts[expected] > 1: + add( + InformationAuditRule.SPLIT_DUPLICATE, + "information manifest contains a duplicate split kind", + evidence={ + "split_kind": expected.value, + "count": counts[expected], + }, + ) + if kinds != _EXPECTED_SPLITS: + add( + InformationAuditRule.SPLIT_DECLARATION_ORDER, + "splits must be declared in train, calibration, validation order", + evidence={"declared_order": [kind.value for kind in kinds]}, + ) + unique = {split.kind: split for split in manifest.splits} + if all(kind in unique for kind in _EXPECTED_SPLITS): + train = unique[InformationSplitKind.TRAIN] + calibration = unique[InformationSplitKind.CALIBRATION] + validation = unique[InformationSplitKind.VALIDATION] + if ( + not train.end_ns + <= calibration.start_ns + <= calibration.end_ns + <= validation.start_ns + ): + add( + InformationAuditRule.SPLIT_TIME_ORDER, + "train, calibration, and validation intervals overlap or regress", + evidence={ + "train_end_ns": train.end_ns, + "calibration_start_ns": calibration.start_ns, + "calibration_end_ns": calibration.end_ns, + "validation_start_ns": validation.start_ns, + }, + ) + + +def _audit_graph( + manifest: ReconstructionInformationManifestV1, + by_id: Mapping[str, ReconstructionInformationInputV1], + add: Any, +) -> None: + for item in manifest.inputs: + if ( + item.input_kind is InformationInputKind.DERIVED + and not item.parent_input_ids + ): + add( + InformationAuditRule.DERIVED_INPUT_WITHOUT_PARENT, + "derived information input does not declare a parent artifact", + item=item, + ) + for parent_id in item.parent_input_ids: + if parent_id not in by_id: + add( + InformationAuditRule.MISSING_PARENT_INPUT, + "information input references a parent absent from the manifest", + item=item, + evidence={"missing_parent_input_id": parent_id}, + ) + cycle = _first_graph_cycle(by_id) + if cycle: + add( + InformationAuditRule.GRAPH_CYCLE, + "information artifact graph contains a dependency cycle", + evidence={"cycle_input_ids": list(cycle)}, + ) + + +def _first_graph_cycle( + by_id: Mapping[str, ReconstructionInformationInputV1], +) -> tuple[str, ...]: + visiting: list[str] = [] + visiting_set: set[str] = set() + complete: set[str] = set() + + def visit(input_id: str) -> tuple[str, ...]: + if input_id in complete: + return () + if input_id in visiting_set: + start = visiting.index(input_id) + return tuple(visiting[start:] + [input_id]) + visiting.append(input_id) + visiting_set.add(input_id) + item = by_id[input_id] + for parent_id in item.parent_input_ids: + if parent_id not in by_id: + continue + cycle = visit(parent_id) + if cycle: + return cycle + visiting.pop() + visiting_set.remove(input_id) + complete.add(input_id) + return () + + for input_id in sorted(by_id): + cycle = visit(input_id) + if cycle: + return cycle + return () + + +def _audit_input_binding( + item: ReconstructionInformationInputV1, + manifest: ReconstructionInformationManifestV1, + policy: ReconstructionInformationPolicyV1, + add: Any, +) -> None: + if item.run_id != manifest.run_id: + add( + InformationAuditRule.INPUT_RUN_MISMATCH, + "information input belongs to a different reconstruction run", + item=item, + evidence={ + "input_run_id": item.run_id, + "manifest_run_id": manifest.run_id, + }, + ) + if item.information_mode is not manifest.information_mode: + add( + InformationAuditRule.INPUT_MODE_MISMATCH, + "information input mode differs from the run manifest mode", + item=item, + evidence={ + "input_mode": item.information_mode.value, + "manifest_mode": manifest.information_mode.value, + }, + ) + if item.allowed_lookahead_ns > policy.max_allowed_lookahead_ns: + add( + InformationAuditRule.INPUT_LOOKAHEAD_EXCEEDS_POLICY, + "input look-ahead exceeds the run information policy", + item=item, + evidence={ + "input_allowed_lookahead_ns": item.allowed_lookahead_ns, + "policy_max_allowed_lookahead_ns": policy.max_allowed_lookahead_ns, + }, + ) + + +def _audit_input_split( + item: ReconstructionInformationInputV1, + split_by_kind: Mapping[ + InformationSplitKind, ReconstructionInformationSplitV1 + ], + add: Any, +) -> None: + required = _REQUIRED_STAGE_SPLIT.get(item.stage) + if required is not None and item.split_kind is None: + add( + InformationAuditRule.INPUT_SPLIT_MISSING, + "information stage requires an explicit research split", + item=item, + evidence={ + "required_split": required.value, + "stage": item.stage.value, + }, + ) + return + if required is not None and item.split_kind is not required: + add( + InformationAuditRule.INPUT_SPLIT_MISMATCH, + "information stage is assigned to the wrong research split", + item=item, + evidence={ + "required_split": required.value, + "declared_split": ( + item.split_kind.value + if item.split_kind is not None + else None + ), + }, + ) + if item.split_kind is None: + return + split = split_by_kind.get(item.split_kind) + if split is None: + return + if not ( + split.start_ns <= item.observation_start_ns + and item.observation_end_ns <= split.end_ns + ): + add( + InformationAuditRule.INPUT_OUTSIDE_SPLIT, + "input observation interval falls outside its declared split", + item=item, + evidence={ + "split_start_ns": split.start_ns, + "split_end_ns": split.end_ns, + "observation_start_ns": item.observation_start_ns, + "observation_end_ns": item.observation_end_ns, + }, + ) + + +def _audit_revision( + item: ReconstructionInformationInputV1, + by_id: Mapping[str, ReconstructionInformationInputV1], + add: Any, +) -> None: + if item.revision_sequence == 0: + return + if item.scope is not InformationScope.REVISION: + add( + InformationAuditRule.REVISION_SCOPE_UNDECLARED, + "revised information is not labeled with revision scope", + item=item, + evidence={"revision_sequence": item.revision_sequence}, + ) + predecessor = by_id.get(item.supersedes_input_id or "") + if predecessor is None: + add( + InformationAuditRule.REVISION_PARENT_MISSING, + "revised information does not include its superseded vintage", + item=item, + evidence={"supersedes_input_id": item.supersedes_input_id}, + ) + return + if predecessor.revision_sequence >= item.revision_sequence: + add( + InformationAuditRule.REVISION_SEQUENCE_INVALID, + "revision sequence does not advance beyond the superseded input", + item=item, + evidence={ + "revision_sequence": item.revision_sequence, + "superseded_revision_sequence": predecessor.revision_sequence, + }, + ) + if predecessor.available_at_ns >= item.available_at_ns: + add( + InformationAuditRule.REVISION_AVAILABILITY_INVALID, + "revision availability must follow the superseded vintage", + item=item, + evidence={ + "available_at_ns": item.available_at_ns, + "superseded_available_at_ns": predecessor.available_at_ns, + }, + ) + + +def _audit_parent_availability( + item: ReconstructionInformationInputV1, + by_id: Mapping[str, ReconstructionInformationInputV1], + add: Any, +) -> None: + if item.input_kind is not InformationInputKind.DERIVED: + return + parent_times = [ + by_id[parent_id].available_at_ns + for parent_id in item.parent_input_ids + if parent_id in by_id + ] + if parent_times and item.available_at_ns < max(parent_times): + add( + InformationAuditRule.DERIVED_AVAILABLE_BEFORE_PARENT, + "derived information is available before one of its parent inputs", + item=item, + evidence={ + "available_at_ns": item.available_at_ns, + "latest_parent_available_at_ns": max(parent_times), + }, + ) + + +def _audit_ex_ante_input( + item: ReconstructionInformationInputV1, + add: Any, +) -> None: + if item.allowed_lookahead_ns != 0: + add( + InformationAuditRule.EX_ANTE_LOOKAHEAD_DECLARED, + "ex-ante inputs cannot declare realized look-ahead", + item=item, + evidence={"allowed_lookahead_ns": item.allowed_lookahead_ns}, + ) + if item.available_at_ns > item.used_at_ns: + rule = ( + InformationAuditRule.EX_ANTE_REVISION_NOT_AVAILABLE + if item.revision_sequence > 0 + else InformationAuditRule.EX_ANTE_INPUT_NOT_AVAILABLE + ) + add( + rule, + "input was not point-in-time available when consumed", + item=item, + evidence={ + "available_at_ns": item.available_at_ns, + "used_at_ns": item.used_at_ns, + "vintage_id": item.vintage_id, + "revision_sequence": item.revision_sequence, + }, + ) + if item.stage not in _PREDECISION_STAGES: + return + scope_rule = { + InformationScope.FUTURE_ANCHOR: InformationAuditRule.EX_ANTE_FUTURE_ANCHOR, + InformationScope.FULL_PERIOD_SUMMARY: ( + InformationAuditRule.EX_ANTE_FULL_PERIOD_SUMMARY + ), + InformationScope.GLOBAL_NORMALIZATION: ( + InformationAuditRule.EX_ANTE_GLOBAL_NORMALIZATION + ), + }.get(item.scope) + if scope_rule is not None: + add( + scope_rule, + "ex-post-only information scope cannot feed an ex-ante stage", + item=item, + evidence={"scope": item.scope.value, "stage": item.stage.value}, + ) + return + if ( + item.stage is InformationStage.MOTIF_SELECTION + and item.observation_end_ns > item.used_at_ns + ): + add( + InformationAuditRule.EX_ANTE_MOTIF_SELECTION_LEAKAGE, + "motif selection observes data after its point-in-time use", + item=item, + evidence={ + "observation_end_ns": item.observation_end_ns, + "used_at_ns": item.used_at_ns, + }, + ) + return + if item.event_time_ns > item.used_at_ns: + add( + InformationAuditRule.EX_ANTE_FUTURE_EVENT, + "future event information feeds an ex-ante stage", + item=item, + evidence={ + "event_time_ns": item.event_time_ns, + "used_at_ns": item.used_at_ns, + "stage": item.stage.value, + }, + ) + elif item.observation_end_ns > item.used_at_ns: + add( + InformationAuditRule.EX_ANTE_FUTURE_OBSERVATION, + "observation window extends beyond its ex-ante use time", + item=item, + evidence={ + "observation_end_ns": item.observation_end_ns, + "used_at_ns": item.used_at_ns, + }, + ) + + +def _audit_ex_post_input( + item: ReconstructionInformationInputV1, + policy: ReconstructionInformationPolicyV1, + add: Any, +) -> None: + future_delta = ( + max( + item.event_time_ns, + item.available_at_ns, + item.observation_end_ns, + ) + - item.used_at_ns + ) + if future_delta <= 0: + return + if item.scope not in _EX_POST_LABELS: + add( + InformationAuditRule.EX_POST_UNLABELED_FUTURE_INFORMATION, + "future-informed ex-post input lacks an explicit future scope label", + item=item, + evidence={ + "scope": item.scope.value, + "required_lookahead_ns": future_delta, + }, + ) + if ( + future_delta > item.allowed_lookahead_ns + or future_delta > policy.max_allowed_lookahead_ns + ): + add( + InformationAuditRule.EX_POST_LOOKAHEAD_EXCEEDED, + "future-informed input exceeds declared ex-post look-ahead", + item=item, + evidence={ + "required_lookahead_ns": future_delta, + "input_allowed_lookahead_ns": item.allowed_lookahead_ns, + "policy_max_allowed_lookahead_ns": policy.max_allowed_lookahead_ns, + }, + ) + + +def _validity_statement( + mode: InformationMode, + accepted: bool, +) -> tuple[tuple[str, ...], tuple[str, ...], str]: + if not accepted: + return ( + (), + ("generation", "validation", "strategy_usefulness_claims"), + "Rejected: invalid for generation and downstream claims until all " + "information violations are corrected.", + ) + if mode is InformationMode.EX_POST_RECONSTRUCTION: + return ( + ( + "diagnostic_counterfactuals", + "historically_informed_reconstruction", + ), + ("prospective_simulation", "strategy_usefulness_claims"), + "Accepted for historically informed reconstruction and diagnostic " + "counterfactuals; not valid for prospective strategy claims.", + ) + return ( + ( + "historically_grounded_reconstruction", + "point_in_time_simulation", + "strategy_usefulness_claims", + ), + (), + "Accepted for point-in-time simulation and downstream strategy claims " + "subject to the declared chronological splits.", + ) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _required_text(value: Any) -> str: + if not isinstance(value, str): + raise ValueError("required text value must be a string") + normalized = value.strip() + if not normalized: + raise ValueError("required text value cannot be empty") + if len(normalized) > MAX_INFORMATION_TEXT_LENGTH: + raise ValueError("text value exceeds the v1 length limit") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError("optional text value must be a string") + normalized = value.strip() + if not normalized: + return None + if len(normalized) > MAX_INFORMATION_TEXT_LENGTH: + raise ValueError("text value exceeds the v1 length limit") + return normalized + + +def _bounded_int64(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + if not INT64_MIN <= value <= INT64_MAX: + raise ValueError(f"{name} is outside signed 64-bit range") + return value + + +def _nonnegative_int64(value: Any, name: str) -> int: + normalized = _bounded_int64(value, name) + if normalized < 0: + raise ValueError(f"{name} must be non-negative") + return normalized + + +def _nonnegative_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{name} must be a non-negative integer") + return value + + +def _positive_int(value: Any, name: str) -> int: + normalized = _nonnegative_int(value, name) + if normalized < 1: + raise ValueError(f"{name} must be positive") + return normalized + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a boolean") + return value + + +def _normalized_ids(values: Sequence[str]) -> tuple[str, ...]: + return tuple(sorted({_required_text(value) for value in values})) + + +def _enum_value(enum_type: type[_EnumT], value: Any, label: str) -> _EnumT: + if isinstance(value, enum_type): + return value + try: + return enum_type(str(value).strip().lower()) + except ValueError as err: + raise ValueError(f"unsupported {label}") from err + + +def _bounded_json( + value: Mapping[str, JSONValue], limit: int, label: str +) -> None: + encoded = canonical_contract_json(value).encode("utf-8") + if len(encoded) > limit: + raise ValueError(f"{label} exceeds the v1 serialized-size limit") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError("unsupported schema version") + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return value + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError("expected a sequence of mappings") + return tuple(_mapping(item) for item in value) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError("expected a sequence of strings") + if any(not isinstance(item, str) for item in value): + raise ValueError("expected a sequence of strings") + return tuple(cast(Sequence[str], value)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + try: + value = json.loads(text) + except (TypeError, json.JSONDecodeError) as err: + raise ValueError("invalid contract JSON") from err + return _mapping(value) + + +__all__ = [ + "DEFAULT_INFORMATION_AUDIT_FINDINGS", + "MAX_INFORMATION_AUDIT_FINDINGS", + "MAX_INFORMATION_INPUTS", + "RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION", + "RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION", + "InformationAuditFindingV1", + "InformationAuditReportV1", + "InformationAuditRule", + "InformationInputKind", + "InformationLeakageError", + "InformationMode", + "InformationScope", + "InformationSplitKind", + "InformationStage", + "ReconstructionInformationInputV1", + "ReconstructionInformationManifestV1", + "ReconstructionInformationPolicyV1", + "ReconstructionInformationSplitV1", + "audit_reconstruction_information", + "reconstruction_information_window_plan_id", + "require_reconstruction_information_audit", +] diff --git a/src/histdatacom/synthetic/motif_library.py b/src/histdatacom/synthetic/motif_library.py new file mode 100644 index 00000000..c1a8f849 --- /dev/null +++ b/src/histdatacom/synthetic/motif_library.py @@ -0,0 +1,1332 @@ +"""Build and qualify the production modern empirical motif library. + +The builder consumes only immutable monthly Arrow caches whose digests are +declared by the stable v2 feed-epoch evidence. It keeps dense rows in memory, +publishes compact lineage and aggregate coverage only, removes train shapes +that occur in any later split, and qualifies the retained train index against +the frozen real reverse-degradation campaign. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import time +from typing import Any + +from histdatacom.data_analytics.feed_epochs_v2 import ( + read_active_time_feed_epoch_definition, +) +from histdatacom.resource_usage import peak_rss_bytes +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.benchmark_corpus import ( + PREDECLARED_GATE_COMMIT, + ReverseDegradationBenchmarkCampaignV1, + read_reverse_degradation_benchmark_corpus, + run_reverse_degradation_benchmark_campaign, +) +from histdatacom.synthetic.contracts import canonical_contract_json +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.motifs import ( + REFERENCE_MOTIF_FEATURE_SCHEMA_VERSION, + ReferenceMotifConditionV1, + ReferenceMotifIndexConfigV1, + ReferenceMotifIndexV1, + ReferenceMotifQueryStatus, + ReferenceMotifQueryV1, + ReferenceMotifSourceEventV1, + ReferenceMotifSourceWindowV1, + ReferenceMotifSplitKind, + ReferenceMotifSplitV1, + ReferenceMotifTransformPolicyV1, + build_reference_motif_index, + extract_reference_motif_fragment, + query_reference_motifs, + reference_motif_condition_from_quotes, +) + +MODERN_REFERENCE_MOTIF_PROFILE_SCHEMA_VERSION = ( + "histdatacom.modern-reference-motif-profile.v1" +) +MODERN_REFERENCE_MOTIF_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.modern-reference-motif-manifest.v1" +) +MODERN_REFERENCE_MOTIF_LEAKAGE_SCHEMA_VERSION = ( + "histdatacom.modern-reference-motif-leakage-audit.v1" +) +MODERN_REFERENCE_MOTIF_COVERAGE_SCHEMA_VERSION = ( + "histdatacom.modern-reference-motif-coverage.v1" +) +MODERN_REFERENCE_MOTIF_QUALIFICATION_SCHEMA_VERSION = ( + "histdatacom.modern-reference-motif-qualification.v1" +) +MODERN_REFERENCE_MOTIF_RESOURCE_SCHEMA_VERSION = ( + "histdatacom.modern-reference-motif-resource-audit.v1" +) + +NANOSECONDS_PER_SECOND = 1_000_000_000 +NANOSECONDS_PER_MILLISECOND = 1_000_000 +DEFAULT_MODERN_MOTIF_SPLIT_PERIODS = { + "train": ("201901", "202001", "202101", "202201", "202301"), + "calibration": ("202307",), + "validation": ("202401",), + "final_holdout": ("202510",), +} +_EXPECTED_SPLITS = ("train", "calibration", "validation", "final_holdout") +_PERIOD_RE = re.compile(r"^\d{6}$") +_CONTENT_ADDRESS_RE = re.compile(r"^([a-z0-9-]+)-([0-9a-f]{64})\.json$") + + +def _default_split_periods() -> dict[str, tuple[str, ...]]: + return dict(DEFAULT_MODERN_MOTIF_SPLIT_PERIODS) + + +@dataclass(frozen=True, slots=True) +class ModernReferenceMotifProfileV1: + """Bounded source selection and production-index policy.""" + + symbols: tuple[str, ...] = ("EURGBP", "EURUSD", "GBPUSD") + split_periods: Mapping[str, tuple[str, ...]] = field( + default_factory=_default_split_periods + ) + synchronized_windows_per_period: int = 6 + window_duration_seconds: int = 600 + minimum_events_per_symbol: int = 32 + max_events_per_symbol: int = 96 + fragment_event_count: int = 3 + max_fragments: int = 256 + max_matches: int = 64 + neighbor_guard_seconds: int = 1800 + max_source_bytes: int = 2 * 1024**3 + max_runtime_seconds: float = 900.0 + max_peak_memory_bytes: int = 2 * 1024**3 + max_artifact_bytes: int = 64 * 1024**2 + profile_id: str = "" + schema_version: str = MODERN_REFERENCE_MOTIF_PROFILE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != MODERN_REFERENCE_MOTIF_PROFILE_SCHEMA_VERSION: + raise ValueError("unsupported modern motif profile schema") + symbols = tuple(sorted({str(item).upper() for item in self.symbols})) + if not symbols or any( + not re.fullmatch(r"[A-Z]{6}", item) for item in symbols + ): + raise ValueError("modern motif symbols must be six-letter pairs") + object.__setattr__(self, "symbols", symbols) + periods = { + str(name): tuple(str(value) for value in values) + for name, values in self.split_periods.items() + } + if tuple(periods) != _EXPECTED_SPLITS: + raise ValueError( + "modern motif periods must declare ordered split keys" + ) + flattened: list[str] = [] + for name in _EXPECTED_SPLITS: + values = periods[name] + if not values or tuple(sorted(values)) != values: + raise ValueError(f"modern motif {name} periods must be sorted") + if any(_PERIOD_RE.fullmatch(value) is None for value in values): + raise ValueError("modern motif periods must use YYYYMM") + flattened.extend(values) + if len(set(flattened)) != len(flattened) or flattened != sorted( + flattened + ): + raise ValueError("modern motif split periods overlap or regress") + object.__setattr__(self, "split_periods", periods) + for name, minimum, maximum in ( + ("synchronized_windows_per_period", 1, 64), + ("window_duration_seconds", 1, 86_400), + ("minimum_events_per_symbol", 3, 4096), + ("max_events_per_symbol", 3, 4096), + ("fragment_event_count", 3, 64), + ("max_fragments", 1, 4096), + ("max_matches", 1, 128), + ("neighbor_guard_seconds", 0, 86_400), + ("max_source_bytes", 1, 2**63 - 1), + ("max_peak_memory_bytes", 1, 2**63 - 1), + ("max_artifact_bytes", 1024, 256 * 1024**2), + ): + selected = int(getattr(self, name)) + if not minimum <= selected <= maximum: + raise ValueError(f"modern motif {name} is outside limits") + object.__setattr__(self, name, selected) + if self.max_events_per_symbol < self.minimum_events_per_symbol: + raise ValueError("modern motif event maximum is below minimum") + runtime = float(self.max_runtime_seconds) + if not 0.0 < runtime < float("inf"): + raise ValueError("modern motif runtime bound must be positive") + object.__setattr__(self, "max_runtime_seconds", runtime) + expected = _stable_id( + "modern-reference-motif-profile", self.identity_payload() + ) + if self.profile_id and self.profile_id != expected: + raise ValueError("modern motif profile_id differs") + object.__setattr__(self, "profile_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbols": list(self.symbols), + "split_periods": { + name: list(self.split_periods[name]) + for name in _EXPECTED_SPLITS + }, + "synchronized_windows_per_period": self.synchronized_windows_per_period, + "window_duration_seconds": self.window_duration_seconds, + "minimum_events_per_symbol": self.minimum_events_per_symbol, + "max_events_per_symbol": self.max_events_per_symbol, + "fragment_event_count": self.fragment_event_count, + "max_fragments": self.max_fragments, + "max_matches": self.max_matches, + "neighbor_guard_seconds": self.neighbor_guard_seconds, + "max_source_bytes": self.max_source_bytes, + "max_runtime_seconds": self.max_runtime_seconds, + "max_peak_memory_bytes": self.max_peak_memory_bytes, + "max_artifact_bytes": self.max_artifact_bytes, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "profile_id": self.profile_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ModernReferenceMotifProfileV1": + split_periods = _mapping(data.get("split_periods")) + return cls( + symbols=tuple(str(item) for item in _sequence(data.get("symbols"))), + split_periods={ + name: tuple( + str(item) for item in _sequence(split_periods.get(name)) + ) + for name in _EXPECTED_SPLITS + }, + synchronized_windows_per_period=int( + data.get("synchronized_windows_per_period", 0) + ), + window_duration_seconds=int(data.get("window_duration_seconds", 0)), + minimum_events_per_symbol=int( + data.get("minimum_events_per_symbol", 0) + ), + max_events_per_symbol=int(data.get("max_events_per_symbol", 0)), + fragment_event_count=int(data.get("fragment_event_count", 0)), + max_fragments=int(data.get("max_fragments", 0)), + max_matches=int(data.get("max_matches", 0)), + neighbor_guard_seconds=int(data.get("neighbor_guard_seconds", 0)), + max_source_bytes=int(data.get("max_source_bytes", 0)), + max_runtime_seconds=float(data.get("max_runtime_seconds", 0.0)), + max_peak_memory_bytes=int(data.get("max_peak_memory_bytes", 0)), + max_artifact_bytes=int(data.get("max_artifact_bytes", 0)), + profile_id=str(data.get("profile_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ModernReferenceMotifBuildV1: + """In-memory build result before atomic content-addressed persistence.""" + + profile: ModernReferenceMotifProfileV1 + index: ReferenceMotifIndexV1 + library_id: str + manifest: Mapping[str, Any] + leakage_audit: Mapping[str, Any] + coverage: Mapping[str, Any] + qualification: Mapping[str, Any] + resource_audit: Mapping[str, Any] + + +@dataclass(frozen=True, slots=True) +class _TickRow: + row_id: int + timestamp_ms: int + bid: float + ask: float + + +def build_modern_reference_motif_library( + source_root: str | Path, + *, + feed_epoch_definition_path: str | Path, + market_context_corpus_path: str | Path, + cftc_positioning_corpus_path: str | Path, + benchmark_manifest_path: str | Path, + profile: ModernReferenceMotifProfileV1 | None = None, +) -> ModernReferenceMotifBuildV1: + """Build, leakage-audit, replay, and qualify the real modern library.""" + from histdatacom.market_context import ( # pylint: disable=import-outside-toplevel + CftcReportFamily, + CftcReportScope, + MarketContextView, + cftc_positioning_state_label, + market_context_benchmark_event_state, + query_cftc_positioning_corpus, + query_market_context_corpus, + read_cftc_positioning_corpus, + read_market_context_corpus, + ) + + selected = profile or ModernReferenceMotifProfileV1() + started = time.monotonic() + root = Path(source_root).expanduser().resolve() + definition_path = Path(feed_epoch_definition_path).expanduser().resolve() + context_path = Path(market_context_corpus_path).expanduser().resolve() + positioning_path = Path(cftc_positioning_corpus_path).expanduser().resolve() + benchmark_path = Path(benchmark_manifest_path).expanduser().resolve() + if not root.is_dir(): + raise ValueError("modern motif source root is not a directory") + definition = read_active_time_feed_epoch_definition(definition_path) + if not definition.valid_for_observation_models: + raise ValueError("modern motif feed epoch definition is not stable") + definition_payload = _read_json_mapping( + definition_path, selected.max_artifact_bytes + ) + if definition_payload.get("definition_id") != definition.definition_id: + raise ValueError("modern motif definition identity differs on restore") + modern_epochs = [ + item + for item in definition.epochs + if item.label == "technology_epoch_04" + ] + if len(modern_epochs) != 1: + raise ValueError("modern motif build requires technology_epoch_04") + modern_epoch = modern_epochs[0] + all_periods = tuple( + period + for name in _EXPECTED_SPLITS + for period in selected.split_periods[name] + ) + if any( + not modern_epoch.period_start <= period <= modern_epoch.period_end + for period in all_periods + ): + raise ValueError( + "modern motif source period falls outside stable epoch 04" + ) + + context_corpus = read_market_context_corpus(context_path) + positioning_corpus = read_cftc_positioning_corpus(positioning_path) + source_lineage = { + (str(item["period"]), str(item["symbol"])): item + for item in _sequence( + _mapping(definition_payload.get("lineage")).get("sources") + ) + if isinstance(item, Mapping) + } + split_by_period = { + period: ReferenceMotifSplitKind(name) + for name in _EXPECTED_SPLITS + for period in selected.split_periods[name] + } + transform_policy = ReferenceMotifTransformPolicyV1( + min_time_scale=0.5, + max_time_scale=2.0, + min_price_scale=0.05, + max_price_scale=1.0, + max_time_warp_ratio=1.25, + ) + index_config = ReferenceMotifIndexConfigV1( + min_events_per_fragment=selected.fragment_event_count, + max_events_per_fragment=selected.fragment_event_count, + max_source_windows=10_000, + max_fragments=selected.max_fragments, + min_cell_support=1, + max_matches=selected.max_matches, + source_overlap_guard_ns=( + selected.neighbor_guard_seconds * NANOSECONDS_PER_SECOND + ), + near_duplicate_rounding_digits=4, + max_artifact_bytes=min(selected.max_artifact_bytes, 256 * 1024**2), + ) + + source_partitions: list[dict[str, Any]] = [] + source_artifacts: dict[tuple[str, str], ArtifactRef] = {} + source_rows: dict[tuple[str, str, int], tuple[_TickRow, ...]] = {} + interval_evidence: dict[ + tuple[str, int], tuple[int, int, str, tuple[str, ...]] + ] = {} + source_bytes = 0 + for period in all_periods: + candidates = _candidate_intervals( + period, + duration_seconds=selected.window_duration_seconds, + context_event_times=tuple( + event.event_time_ns + for event in context_corpus.timeline.events + if _period_for_ns(event.event_time_ns) == period + ), + ) + accepted = 0 + for start_ns, end_ns, session in candidates: + rows_by_symbol: dict[str, tuple[_TickRow, ...]] = {} + for symbol in selected.symbols: + path = root / _relative_source_path(symbol, period) + rows_by_symbol[symbol] = _read_arrow_interval( + path, + start_ns=start_ns, + end_ns=end_ns, + maximum=selected.max_events_per_symbol, + ) + if any( + len(rows) < selected.minimum_events_per_symbol + for rows in rows_by_symbol.values() + ): + continue + context_query = query_market_context_corpus( + context_corpus, + start_ns=start_ns, + end_ns=end_ns, + view=MarketContextView.EX_ANTE, + as_of_ns=start_ns, + symbols=selected.symbols, + include_calendar=True, + max_events=64, + require_supported=False, + ) + positioning_query = query_cftc_positioning_corpus( + positioning_corpus, + start_ns=start_ns, + end_ns=end_ns, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + symbols=selected.symbols, + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + tags = ( + market_context_benchmark_event_state(context_query), + cftc_positioning_state_label(positioning_query), + ) + interval_evidence[(period, accepted)] = ( + start_ns, + end_ns, + session, + tags, + ) + for symbol, rows in rows_by_symbol.items(): + source_rows[(period, symbol, accepted)] = rows + accepted += 1 + if accepted == selected.synchronized_windows_per_period: + break + if accepted != selected.synchronized_windows_per_period: + raise ValueError( + f"only {accepted} synchronized windows qualify for {period}" + ) + _enforce_runtime(started, selected.max_runtime_seconds) + + for period in all_periods: + for symbol in selected.symbols: + relative = _relative_source_path(symbol, period) + path = root / relative + lineage = source_lineage.get((period, symbol)) + if lineage is None: + raise ValueError(f"feed epoch lineage omits {symbol} {period}") + size = path.stat().st_size + source_bytes += size + if source_bytes > selected.max_source_bytes: + raise ValueError("modern motif sources exceed byte bound") + digest = _file_sha256(path) + expected = str( + lineage.get("source_artifact_sha256", "") + ).removeprefix("sha256:") + if digest != expected: + raise ValueError( + f"modern motif source hash differs for {symbol} {period}" + ) + row_count = _arrow_row_count(path) + artifact = ArtifactRef( + kind="histdata_ascii_tick_arrow", + path=relative.as_posix(), + size_bytes=size, + sha256=digest, + metadata={ + "symbol": symbol, + "period": period, + "evidence_id": str(lineage.get("evidence_id", "")), + }, + ) + source_artifacts[(period, symbol)] = artifact + source_partitions.append( + { + "symbol": symbol, + "period": period, + "split_kind": split_by_period[period].value, + "relative_path": relative.as_posix(), + "size_bytes": size, + "row_count": row_count, + "sha256": digest, + "feed_epoch_evidence_id": str( + lineage.get("evidence_id", "") + ), + "selected_parent_window_count": selected.synchronized_windows_per_period, + } + ) + + windows: list[ReferenceMotifSourceWindowV1] = [] + for period in all_periods: + split_kind = split_by_period[period] + for interval_number in range(selected.synchronized_windows_per_period): + start_ns, end_ns, session, event_tags = interval_evidence[ + (period, interval_number) + ] + assignment = definition.assign( + symbol=selected.symbols[0], + timestamp_utc_ms=((start_ns + end_ns) // 2) + // NANOSECONDS_PER_MILLISECOND, + ) + if assignment.label != modern_epoch.label: + raise ValueError( + "selected modern motif window is outside epoch 04" + ) + for symbol in selected.symbols: + rows = source_rows[(period, symbol, interval_number)] + for chunk_number, offset in enumerate( + range( + 0, + len(rows) - selected.fragment_event_count + 1, + selected.fragment_event_count, + ) + ): + chunk = rows[ + offset : offset + selected.fragment_event_count + ] + if len(chunk) != selected.fragment_event_count: + continue + event_times = tuple( + item.timestamp_ms * NANOSECONDS_PER_MILLISECOND + for item in chunk + ) + condition = reference_motif_condition_from_quotes( + symbol=symbol, + feed_epoch_id=modern_epoch.label, + session_state=session, + event_times_ns=event_times, + bids=tuple(item.bid for item in chunk), + asks=tuple(item.ask for item in chunk), + event_tags=event_tags, + ) + events = tuple( + ReferenceMotifSourceEventV1( + event_time_ns=item.timestamp_ms + * NANOSECONDS_PER_MILLISECOND, + event_sequence=item.row_id, + bid=item.bid, + ask=item.ask, + source_row_id=item.row_id, + ) + for item in chunk + ) + windows.append( + ReferenceMotifSourceWindowV1( + source_series_id=( + f"ascii:T:{symbol}:histdata.com:{period}:" + f"window-{interval_number:02d}:chunk-{chunk_number:02d}" + ), + period=period, + source_artifact=source_artifacts[(period, symbol)], + split_kind=split_kind, + condition=condition, + events=events, + first_known_at_ns=event_times[-1], + available_at_ns=event_times[-1], + transform_policy=transform_policy, + ) + ) + if len(windows) > index_config.max_source_windows: + raise ValueError("modern motif source-window count exceeds index bound") + projected = { + item.source_window_id: extract_reference_motif_fragment( + item, config=index_config + ) + for item in windows + } + withheld_signatures = { + item.near_duplicate_signature + for item in projected.values() + if item.split_kind is not ReferenceMotifSplitKind.TRAIN + } + excluded_train = tuple( + sorted( + item.source_window_id + for item in windows + if item.split_kind is ReferenceMotifSplitKind.TRAIN + and projected[item.source_window_id].near_duplicate_signature + in withheld_signatures + ) + ) + excluded_ids = set(excluded_train) + filtered = tuple( + item for item in windows if item.source_window_id not in excluded_ids + ) + splits = _reference_splits(selected) + index = build_reference_motif_index( + filtered, splits=splits, config=index_config + ) + if not index.fragments: + raise ValueError("modern motif production index is empty") + + coverage = _coverage_payload(index, filtered, excluded_train) + unsupported_condition = reference_motif_condition_from_quotes( + symbol=index.fragments[0].condition.symbol, + feed_epoch_id="unsupported-technology-epoch", + session_state=index.fragments[0].condition.session_state, + event_times_ns=(1_700_000_000_000_000_000, 1_700_000_001_000_000_000), + bids=(1.0, 1.0001), + asks=(1.0002, 1.0003), + ) + refusal = query_reference_motifs( + index, + ReferenceMotifQueryV1( + condition=unsupported_condition, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + used_at_ns=1_700_000_002_000_000_000, + max_results=selected.max_matches, + max_distance=0.0, + ), + ) + if refusal.status is ReferenceMotifQueryStatus.MATCHED: + raise ValueError("unsupported modern motif condition did not refuse") + + benchmark_corpus = read_reverse_degradation_benchmark_corpus(benchmark_path) + campaign, returned_index = run_reverse_degradation_benchmark_campaign( + benchmark_corpus, + root, + motif_index=index, + motif_candidate_provisional=False, + ) + replay, replay_index = run_reverse_degradation_benchmark_campaign( + benchmark_corpus, + root, + motif_index=index, + motif_candidate_provisional=False, + ) + if ( + returned_index.index_id != index.index_id + or replay_index.index_id != index.index_id + ): + raise ValueError("benchmark returned a different modern motif index") + candidate = _candidate_report(campaign) + replay_candidate = _candidate_report(replay) + deterministic = candidate.to_dict() == replay_candidate.to_dict() + controls = { + item.method_name: { + "role": item.role, + "promotion_eligible": item.gate_decision.promotion_eligible, + "report_id": item.report_id, + } + for item in campaign.candidate_reports + if item.method_name != "empirical_motif" + } + real_contracts = { + "immutable_raw_anchors_pass": ( + candidate.metrics["immutable_anchor_violation_count"] == 0 + ), + "variable_cardinality_pass": ( + float(candidate.metrics["max_event_count_relative_error"]) <= 0.5 + and candidate.refusal_count == 0 + ), + "deterministic_seed_replay_pass": deterministic, + "boundary_continuity_pass": candidate.failure_count == 0, + "unsupported_context_refusal_pass": ( + refusal.status is ReferenceMotifQueryStatus.NO_SUPPORTED_CELL + and not refusal.matches + ), + } + dependencies = { + "feed_epoch_definition": _artifact_ref( + definition_path, "feed_epoch_definition_v2" + ), + "market_context_corpus": _artifact_ref( + context_path, "market_context_corpus_v1" + ), + "cftc_positioning_corpus": _artifact_ref( + positioning_path, "cftc_positioning_corpus_v1" + ), + "benchmark_manifest": _artifact_ref( + benchmark_path, "reverse_degradation_manifest_v1" + ), + } + library_id = _stable_id( + "modern-reference-motif-library", + { + "profile_id": selected.profile_id, + "index_id": index.index_id, + "dependency_sha256": { + name: item.sha256 for name, item in sorted(dependencies.items()) + }, + "candidate_report_id": candidate.report_id, + }, + ) + split_source_counts = Counter(item.split_kind.value for item in filtered) + manifest: dict[str, Any] = { + "schema_version": MODERN_REFERENCE_MOTIF_MANIFEST_SCHEMA_VERSION, + "library_id": library_id, + "profile": selected.to_dict(), + "stable_feed_epoch": modern_epoch.to_dict(), + "dependencies": { + name: item.to_dict() for name, item in sorted(dependencies.items()) + }, + "source_partitions": sorted( + source_partitions, key=lambda item: (item["period"], item["symbol"]) + ), + "feature_schema": _feature_schema(), + "split_assignment": { + name: list(selected.split_periods[name]) + for name in _EXPECTED_SPLITS + }, + "support": { + "source_window_count_before_exclusion": len(windows), + "source_window_count_after_exclusion": len(filtered), + "source_windows_by_split": dict( + sorted(split_source_counts.items()) + ), + "retained_fragment_count": len(index.fragments), + "selection_omitted_count": index.selection_omitted_count, + "near_duplicate_train_exclusion_count": len(excluded_train), + }, + "index": { + "index_id": index.index_id, + "config_id": index.config.config_id, + "payload_layout": "compact-offsets-and-deltas-v1", + "raw_rows_embedded": False, + }, + "artifact_contract": { + "content_addressed": True, + "dense_rows_embedded": False, + "holdout_rows_embedded": False, + "installed_reader_required": True, + }, + } + leakage_audit: dict[str, Any] = { + "schema_version": MODERN_REFERENCE_MOTIF_LEAKAGE_SCHEMA_VERSION, + "library_id": library_id, + "policy": "prefilter-train-near-duplicates-then-fail-closed-index-build", + "neighbor_guard_seconds": selected.neighbor_guard_seconds, + "train_near_duplicate_exclusion_count": len(excluded_train), + "train_near_duplicate_excluded_source_window_ids": list(excluded_train), + "retained_nontrain_fragment_count": 0, + "retained_holdout_fragment_count": 0, + "post_exclusion_cross_split_finding_count": 0, + "post_exclusion_comparison_count": index.leakage_comparison_count, + "indexed_splits": [ReferenceMotifSplitKind.TRAIN.value], + } + qualification: dict[str, Any] = { + "schema_version": MODERN_REFERENCE_MOTIF_QUALIFICATION_SCHEMA_VERSION, + "library_id": library_id, + "frozen_gate_policy_commit": PREDECLARED_GATE_COMMIT, + "campaign": campaign.to_dict(), + "candidate_report_id": candidate.report_id, + "candidate_promotion_eligible": candidate.gate_decision.promotion_eligible, + "candidate_provisional": candidate.provisional, + "candidate_replay_report_id": replay_candidate.report_id, + "transparent_controls": controls, + "real_window_contracts": real_contracts, + } + runtime = time.monotonic() - started + peak_memory = _peak_memory_bytes() + resource_audit: dict[str, Any] = { + "schema_version": MODERN_REFERENCE_MOTIF_RESOURCE_SCHEMA_VERSION, + "library_id": library_id, + "runtime_seconds": round(runtime, 6), + "peak_memory_bytes": peak_memory, + "source_partition_count": len(source_partitions), + "source_bytes": source_bytes, + "source_row_count": sum( + int(item["row_count"]) for item in source_partitions + ), + "scratch_bytes": 0, + "index_json_bytes": len(index.to_json().encode("utf-8") + b"\n"), + "compact_artifact_bytes": 0, + "profile_bounds": { + "max_runtime_seconds": selected.max_runtime_seconds, + "max_peak_memory_bytes": selected.max_peak_memory_bytes, + "max_source_bytes": selected.max_source_bytes, + "max_artifact_bytes": selected.max_artifact_bytes, + }, + } + if runtime > selected.max_runtime_seconds: + raise RuntimeError("modern motif campaign exceeded runtime bound") + if peak_memory > selected.max_peak_memory_bytes: + raise RuntimeError("modern motif campaign exceeded peak-memory bound") + return ModernReferenceMotifBuildV1( + profile=selected, + index=index, + library_id=library_id, + manifest=manifest, + leakage_audit=leakage_audit, + coverage=coverage, + qualification=qualification, + resource_audit=resource_audit, + ) + + +def write_modern_reference_motif_artifacts( + build: ModernReferenceMotifBuildV1, + artifact_directory: str | Path, +) -> Mapping[str, ArtifactRef]: + """Write the compact library and qualification evidence exactly once.""" + if not isinstance(build, ModernReferenceMotifBuildV1): + raise ValueError("modern motif writer requires a v1 build") + root = Path(artifact_directory).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + index_encoded = build.index.to_json().encode("utf-8") + b"\n" + index_digest = hashlib.sha256(index_encoded).hexdigest() + index_path = root / f"modern-reference-motif-index-{index_digest}.json" + index_ref = ArtifactRef( + kind="modern_reference_motif_index_v1", + path=str(index_path), + size_bytes=len(index_encoded), + sha256=index_digest, + metadata={ + "library_id": build.library_id, + "index_id": build.index.index_id, + }, + ) + manifest = {**build.manifest, "index_artifact": index_ref.to_dict()} + resource_audit = dict(build.resource_audit) + base_payloads: dict[str, tuple[str, Mapping[str, Any]]] = { + "index": ("modern-reference-motif-index", build.index.to_dict()), + "manifest": ("modern-reference-motif-manifest", manifest), + "leakage_audit": ( + "modern-reference-motif-leakage-audit", + build.leakage_audit, + ), + "coverage": ("modern-reference-motif-coverage", build.coverage), + "qualification": ( + "modern-reference-motif-qualification", + build.qualification, + ), + "resource_audit": ( + "modern-reference-motif-resource-audit", + resource_audit, + ), + } + total = 0 + for _ in range(8): + resource_audit["compact_artifact_bytes"] = total + total_next = sum( + len(canonical_contract_json(payload).encode("utf-8") + b"\n") + for _, payload in base_payloads.values() + ) + if total_next == total: + break + total = total_next + else: + raise RuntimeError( + "modern motif artifact byte measurement did not converge" + ) + if total > build.profile.max_artifact_bytes: + raise ValueError("modern motif artifact set exceeds configured bound") + artifacts: dict[str, ArtifactRef] = {} + measured = 0 + for name, (prefix, payload) in base_payloads.items(): + encoded = canonical_contract_json(payload).encode("utf-8") + b"\n" + digest = hashlib.sha256(encoded).hexdigest() + path = root / f"{prefix}-{digest}.json" + _write_once(path, encoded) + measured += len(encoded) + artifacts[name] = ArtifactRef( + kind=f"modern_reference_motif_{name}_v1", + path=str(path), + size_bytes=len(encoded), + sha256=digest, + metadata={"library_id": build.library_id}, + ) + if measured != total: + raise ValueError( + "modern motif artifact bytes differ from resource evidence" + ) + return artifacts + + +def read_modern_reference_motif_index( + path: str | Path, +) -> ReferenceMotifIndexV1: + """Hash-verify and restore a content-addressed production index.""" + payload = _read_content_addressed_json(path, "modern-reference-motif-index") + return ReferenceMotifIndexV1.from_dict(payload) + + +def read_modern_reference_motif_artifact( + path: str | Path, + *, + kind: str, +) -> Mapping[str, Any]: + """Hash-verify one compact manifest, audit, or qualification artifact.""" + schemas = { + "manifest": MODERN_REFERENCE_MOTIF_MANIFEST_SCHEMA_VERSION, + "leakage-audit": MODERN_REFERENCE_MOTIF_LEAKAGE_SCHEMA_VERSION, + "coverage": MODERN_REFERENCE_MOTIF_COVERAGE_SCHEMA_VERSION, + "qualification": MODERN_REFERENCE_MOTIF_QUALIFICATION_SCHEMA_VERSION, + "resource-audit": MODERN_REFERENCE_MOTIF_RESOURCE_SCHEMA_VERSION, + } + try: + schema = schemas[kind] + except KeyError as exc: + raise ValueError("unsupported modern motif artifact kind") from exc + payload = _read_content_addressed_json( + path, f"modern-reference-motif-{kind}" + ) + if payload.get("schema_version") != schema: + raise ValueError("unsupported modern motif artifact schema") + return payload + + +def _coverage_payload( + index: ReferenceMotifIndexV1, + windows: Sequence[ReferenceMotifSourceWindowV1], + excluded_train: Sequence[str], +) -> dict[str, Any]: + retained_axes: Counter[str] = Counter() + for fragment in index.fragments: + values = _coverage_axes(fragment.condition) + retained_axes.update( + f"{name}={value}" for name, value in values.items() + ) + backoff: Counter[str] = Counter() + status: Counter[str] = Counter() + axis_queries: dict[str, Counter[str]] = {} + scanned = 0 + for window in windows: + if window.split_kind is ReferenceMotifSplitKind.TRAIN: + continue + result = query_reference_motifs( + index, + ReferenceMotifQueryV1( + condition=window.condition, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + used_at_ns=window.end_ns + 1, + max_results=index.config.max_matches, + ), + ) + status[result.status.value] += 1 + scanned += result.scanned_fragment_count + query_axis_counts: list[Counter[str]] = [] + for name, value in _coverage_axes(window.condition).items(): + counts = axis_queries.setdefault(f"{name}={value}", Counter()) + counts["query_count"] += 1 + counts[f"status:{result.status.value}"] += 1 + query_axis_counts.append(counts) + if result.matches: + level = result.matches[0].backoff_level + backoff[level] += 1 + for counts in query_axis_counts: + counts[f"backoff:{level}"] += 1 + input_axes = Counter( + f"split={window.split_kind.value}" for window in windows + ) + event_windows = sum( + any( + tag.startswith("market_context:") + and not tag.startswith("market_context:none:") + for tag in window.condition.event_tags + ) + for window in windows + ) + return { + "schema_version": MODERN_REFERENCE_MOTIF_COVERAGE_SCHEMA_VERSION, + "index_id": index.index_id, + "retained_support_by_axis": dict(sorted(retained_axes.items())), + "source_windows_by_split": dict(sorted(input_axes.items())), + "withheld_query_count": sum(status.values()), + "withheld_query_status_counts": dict(sorted(status.items())), + "withheld_backoff_counts": dict(sorted(backoff.items())), + "withheld_backoff_by_axis": { + axis: _coverage_axis_rates(counts) + for axis, counts in sorted(axis_queries.items()) + }, + "withheld_exact_match_rate": ( + backoff["exact"] / max(1, sum(status.values())) + ), + "withheld_mean_scanned_fragment_count": ( + scanned / max(1, sum(status.values())) + ), + "event_conditioned_source_window_count": event_windows, + "near_duplicate_train_exclusion_count": len(excluded_train), + "cross_symbol_balance": { + symbol: sum( + item.condition.symbol == symbol for item in index.fragments + ) + for symbol in sorted( + {item.condition.symbol for item in index.fragments} + ) + }, + } + + +def _coverage_axes( + condition: ReferenceMotifConditionV1, +) -> dict[str, str]: + return { + "symbol": condition.symbol, + "session": condition.session_state, + "epoch": condition.feed_epoch_id, + "event_state": "+".join(condition.event_tags) or "none", + "volatility": condition.volatility_regime, + "activity": condition.activity_regime, + "spread": condition.spread_regime, + "weekday": next( + ( + item + for item in condition.special_tags + if item.startswith("weekday:") + ), + "weekday:unknown", + ), + } + + +def _coverage_axis_rates(counts: Mapping[str, int]) -> dict[str, Any]: + query_count = int(counts.get("query_count", 0)) + status_counts = { + key.removeprefix("status:"): int(value) + for key, value in sorted(counts.items()) + if key.startswith("status:") + } + backoff_counts = { + key.removeprefix("backoff:"): int(value) + for key, value in sorted(counts.items()) + if key.startswith("backoff:") + } + return { + "query_count": query_count, + "status_counts": status_counts, + "backoff_counts": backoff_counts, + "backoff_rates": { + level: count / max(1, query_count) + for level, count in backoff_counts.items() + }, + } + + +def _feature_schema() -> dict[str, Any]: + return { + "schema_version": REFERENCE_MOTIF_FEATURE_SCHEMA_VERSION, + "categorical_features": [ + "symbol", + "session_state", + "weekday", + "event/news state", + "feed_epoch_id", + "return_regime", + "range_regime", + "volatility_regime", + "spread_regime", + "activity_regime", + "interarrival_regime", + "timestamp_precision", + "price_precision", + "source_quality_state", + ], + "numeric_features": [ + "return_value", + "range_value", + "volatility", + "spread", + "tick_intensity", + "interarrival_ns", + "timestamp_precision_ns", + "price_precision_digits", + "source_quality_score", + ], + "fixed_thresholds": { + "return_abs": [1e-5, 5e-5], + "range": [2e-5, 1e-4], + "volatility": [5e-6, 2.5e-5], + "relative_spread": [5e-5, 2e-4], + "tick_intensity_per_second": [1.0, 5.0], + "interarrival_ns": [200_000_000, 2_000_000_000], + }, + "weekday_encoding": "special_tags:weekday:", + "fitted_on_holdout": False, + } + + +def _reference_splits( + profile: ModernReferenceMotifProfileV1, +) -> tuple[ReferenceMotifSplitV1, ...]: + return tuple( + ReferenceMotifSplitV1( + kind=ReferenceMotifSplitKind(name), + start_ns=_month_start_ns(profile.split_periods[name][0]), + end_ns=_month_start_ns( + _next_period(profile.split_periods[name][-1]) + ), + ) + for name in _EXPECTED_SPLITS + ) + + +def _candidate_intervals( + period: str, + *, + duration_seconds: int, + context_event_times: Sequence[int] = (), +) -> tuple[tuple[int, int, str], ...]: + year, month = int(period[:4]), int(period[4:]) + names = {0: "asia", 8: "london", 14: "new_york"} + ordinary: list[tuple[int, int, str]] = [] + for day in range(3, 27): + value = datetime(year, month, day, tzinfo=timezone.utc) + if value.weekday() >= 5: + continue + for hour in (0, 8, 14): + start = datetime(year, month, day, hour, tzinfo=timezone.utc) + start_ns = int(start.timestamp() * NANOSECONDS_PER_SECOND) + ordinary.append( + ( + start_ns, + start_ns + duration_seconds * NANOSECONDS_PER_SECOND, + names[hour], + ) + ) + baseline = tuple( + next(item for item in ordinary if item[2] == session) + for session in ("asia", "london", "new_york") + ) + contexts: list[tuple[int, int, str]] = [] + for start_ns in sorted(set(context_event_times)): + if _period_for_ns(start_ns) != period: + continue + end_ns = start_ns + duration_seconds * NANOSECONDS_PER_SECOND + candidate = (start_ns, end_ns, _session_for_ns(start_ns)) + if any( + left < end_ns and start_ns < right + for left, right, _ in (*baseline, *contexts) + ): + continue + contexts.append(candidate) + if len(contexts) == 3: + break + used = {(start, end) for start, end, _ in (*baseline, *contexts)} + remainder = tuple( + item for item in ordinary if (item[0], item[1]) not in used + ) + return (*baseline, *contexts, *remainder) + + +def _read_arrow_interval( + path: Path, + *, + start_ns: int, + end_ns: int, + maximum: int, +) -> tuple[_TickRow, ...]: + import pyarrow as pa # pylint: disable=import-outside-toplevel + import pyarrow.ipc as ipc # pylint: disable=import-outside-toplevel + + start_ms = start_ns // NANOSECONDS_PER_MILLISECOND + end_ms = (end_ns - 1) // NANOSECONDS_PER_MILLISECOND + 1 + rows: list[_TickRow] = [] + row_offset = 0 + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_file(source) + if not {"datetime", "bid", "ask"}.issubset(reader.schema.names): + raise ValueError("modern motif Arrow cache lacks quote columns") + for batch_index in range(reader.num_record_batches): + batch = reader.get_batch(batch_index) + count = batch.num_rows + if count == 0: + continue + timestamps = batch.column(batch.schema.get_field_index("datetime")) + first = int(timestamps[0].as_py()) + last = int(timestamps[count - 1].as_py()) + if last < start_ms: + row_offset += count + continue + if first >= end_ms: + break + bids = batch.column(batch.schema.get_field_index("bid")) + asks = batch.column(batch.schema.get_field_index("ask")) + for index in range(count): + timestamp = int(timestamps[index].as_py()) + if start_ms <= timestamp < end_ms: + rows.append( + _TickRow( + row_id=row_offset + index, + timestamp_ms=timestamp, + bid=float(bids[index].as_py()), + ask=float(asks[index].as_py()), + ) + ) + if len(rows) == maximum: + return tuple(rows) + row_offset += count + return tuple(rows) + + +def _arrow_row_count(path: Path) -> int: + import pyarrow as pa # pylint: disable=import-outside-toplevel + import pyarrow.ipc as ipc # pylint: disable=import-outside-toplevel + + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_file(source) + return sum( + reader.get_batch(index).num_rows + for index in range(reader.num_record_batches) + ) + + +def _candidate_report(campaign: ReverseDegradationBenchmarkCampaignV1) -> Any: + candidates = [ + item + for item in campaign.candidate_reports + if item.method_name == "empirical_motif" + ] + if len(candidates) != 1: + raise ValueError( + "modern motif qualification lacks one candidate report" + ) + return candidates[0] + + +def _relative_source_path(symbol: str, period: str) -> Path: + return ( + Path(symbol.lower()) + / str(int(period[:4])) + / str(int(period[4:])) + / ".data" + ) + + +def _month_start_ns(period: str) -> int: + return int( + datetime( + int(period[:4]), int(period[4:]), 1, tzinfo=timezone.utc + ).timestamp() + * NANOSECONDS_PER_SECOND + ) + + +def _next_period(period: str) -> str: + year, month = int(period[:4]), int(period[4:]) + if month == 12: + return f"{year + 1:04d}01" + return f"{year:04d}{month + 1:02d}" + + +def _period_for_ns(value: int) -> str: + return datetime.fromtimestamp( + value / NANOSECONDS_PER_SECOND, tz=timezone.utc + ).strftime("%Y%m") + + +def _session_for_ns(value: int) -> str: + hour = datetime.fromtimestamp( + value / NANOSECONDS_PER_SECOND, tz=timezone.utc + ).hour + if hour < 7: + return "asia" + if hour < 12: + return "london" + return "new_york" + + +def _artifact_ref(path: Path, kind: str) -> ArtifactRef: + return ArtifactRef( + kind=kind, + path=str(path), + size_bytes=path.stat().st_size, + sha256=_file_sha256(path), + ) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while block := handle.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def _stable_id(namespace: str, payload: Mapping[str, Any]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{namespace}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _peak_memory_bytes() -> int: + return peak_rss_bytes() + + +def _enforce_runtime(started: float, maximum: float) -> None: + if time.monotonic() - started > maximum: + raise RuntimeError("modern motif build exceeded runtime bound") + + +def _write_once(path: Path, content: bytes) -> None: + if path.exists(): + if path.read_bytes() != content: + raise ValueError("content-addressed modern motif artifact differs") + return + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + with temporary.open("xb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _read_content_addressed_json( + path: str | Path, prefix: str +) -> Mapping[str, Any]: + source = Path(path).expanduser().resolve() + match = _CONTENT_ADDRESS_RE.fullmatch(source.name) + if match is None or match.group(1) != prefix: + raise ValueError("modern motif artifact name is not content addressed") + content = source.read_bytes() + if len(content) > 256 * 1024**2: + raise ValueError("modern motif artifact exceeds size bound") + if hashlib.sha256(content).hexdigest() != match.group(2): + raise ValueError("modern motif artifact content hash differs from name") + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("modern motif artifact is invalid JSON") from exc + return _mapping(payload) + + +def _read_json_mapping(path: Path, maximum: int) -> Mapping[str, Any]: + if path.stat().st_size > maximum: + raise ValueError("modern motif dependency exceeds size bound") + return _mapping(json.loads(path.read_text(encoding="utf-8"))) + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("modern motif JSON value must be an object") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError("modern motif JSON value must be an array") + return value + + +__all__ = [ + "DEFAULT_MODERN_MOTIF_SPLIT_PERIODS", + "MODERN_REFERENCE_MOTIF_COVERAGE_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_LEAKAGE_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_MANIFEST_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_PROFILE_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_QUALIFICATION_SCHEMA_VERSION", + "MODERN_REFERENCE_MOTIF_RESOURCE_SCHEMA_VERSION", + "ModernReferenceMotifBuildV1", + "ModernReferenceMotifProfileV1", + "build_modern_reference_motif_library", + "read_modern_reference_motif_artifact", + "read_modern_reference_motif_index", + "write_modern_reference_motif_artifacts", +] diff --git a/src/histdatacom/synthetic/motifs.py b/src/histdatacom/synthetic/motifs.py new file mode 100644 index 00000000..b8eb6c1d --- /dev/null +++ b/src/histdatacom/synthetic/motifs.py @@ -0,0 +1,3084 @@ +"""Deterministic empirical reference-motif artifacts and retrieval. + +The augmented tick row is an evidence surface, not a storage format for motif +records. This module projects bounded source windows into compact event-time +offsets, bid/ask deltas, transition marks, conditioning coordinates, and +complete artifact/row lineage. It also owns chronological split exclusion, +near-duplicate leakage refusal, deterministic fallback retrieval, and the +point-in-time bridge to the reconstruction information audit. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +import hashlib +import json +import math +import os +from pathlib import Path +import re +import statistics +from typing import Any, cast + +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.information import ( + InformationInputKind, + InformationMode, + InformationScope, + InformationSplitKind, + InformationStage, + ReconstructionInformationInputV1, +) + +REFERENCE_MOTIF_SPLIT_SCHEMA_VERSION = "histdatacom.reference-motif-split.v1" +REFERENCE_MOTIF_CONDITION_SCHEMA_VERSION = ( + "histdatacom.reference-motif-condition.v1" +) +REFERENCE_MOTIF_TRANSFORM_SCHEMA_VERSION = ( + "histdatacom.reference-motif-transform-policy.v1" +) +REFERENCE_MOTIF_SOURCE_EVENT_SCHEMA_VERSION = ( + "histdatacom.reference-motif-source-event.v1" +) +REFERENCE_MOTIF_SOURCE_WINDOW_SCHEMA_VERSION = ( + "histdatacom.reference-motif-source-window.v1" +) +REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION = ( + "histdatacom.reference-motif-fragment.v1" +) +REFERENCE_MOTIF_INDEX_CONFIG_SCHEMA_VERSION = ( + "histdatacom.reference-motif-index-config.v1" +) +REFERENCE_MOTIF_INDEX_SCHEMA_VERSION = "histdatacom.reference-motif-index.v1" +REFERENCE_MOTIF_QUERY_SCHEMA_VERSION = "histdatacom.reference-motif-query.v1" +REFERENCE_MOTIF_BACKOFF_ATTEMPT_SCHEMA_VERSION = ( + "histdatacom.reference-motif-backoff-attempt.v1" +) +REFERENCE_MOTIF_MATCH_SCHEMA_VERSION = "histdatacom.reference-motif-match.v1" +REFERENCE_MOTIF_QUERY_RESULT_SCHEMA_VERSION = ( + "histdatacom.reference-motif-query-result.v1" +) +REFERENCE_MOTIF_ARTIFACT_KIND = "reference-motif-index" +REFERENCE_MOTIF_FEATURE_SCHEMA_VERSION = ( + "histdatacom.reference-motif-features.v1" +) + +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 +MAX_REFERENCE_MOTIF_TEXT = 1024 +MAX_REFERENCE_MOTIF_TAGS = 64 +MAX_REFERENCE_MOTIF_EVENTS = 4096 +MAX_REFERENCE_MOTIF_SOURCE_WINDOWS = 10_000 +MAX_REFERENCE_MOTIF_FRAGMENTS = 4096 +MAX_REFERENCE_MOTIF_MATCHES = 128 +MAX_REFERENCE_MOTIF_EXCLUSIONS = 4096 +MAX_REFERENCE_MOTIF_ARTIFACT_BYTES = 256 * 1024**2 +MAX_REFERENCE_MOTIF_LEAKAGE_FINDINGS = 128 + +REFERENCE_MOTIF_METRIC_NAMES = ( + "return_value", + "range_value", + "volatility", + "spread", + "tick_intensity", + "interarrival_ns", + "timestamp_precision_ns", + "price_precision_digits", + "source_quality_score", +) +_NONNEGATIVE_METRICS = frozenset(REFERENCE_MOTIF_METRIC_NAMES) - { + "return_value" +} +REFERENCE_MOTIF_BACKOFF_LEVELS = ( + "exact", + "symbol_epoch_session_event", + "symbol_epoch_session", + "symbol_epoch_state", + "symbol_epoch", + "currency_epoch", + "symbol", + "epoch", + "global", +) +_EXPECTED_SPLITS = ( + "train", + "calibration", + "validation", + "final_holdout", +) +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_PERIOD_RE = re.compile(r"^\d{6}$") + + +class ReferenceMotifSplitKind(str, Enum): + """Chronological source roles for motif leakage control.""" + + TRAIN = "train" + CALIBRATION = "calibration" + VALIDATION = "validation" + FINAL_HOLDOUT = "final_holdout" + + @classmethod + def from_value( + cls, value: str | "ReferenceMotifSplitKind" + ) -> "ReferenceMotifSplitKind": + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError("unsupported reference motif split kind") from err + + +class ReferenceMotifTransition(str, Enum): + """Which quote marks changed from the preceding source event.""" + + START = "start" + UNCHANGED = "unchanged" + BID = "bid" + ASK = "ask" + BOTH = "both" + + @classmethod + def from_value( + cls, value: str | "ReferenceMotifTransition" + ) -> "ReferenceMotifTransition": + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError("unsupported reference motif transition") from err + + +class ReferenceMotifQueryStatus(str, Enum): + """Whether deterministic retrieval found admissible evidence.""" + + MATCHED = "matched" + NO_SUPPORTED_CELL = "no_supported_cell" + NOT_AVAILABLE_AS_OF = "not_available_as_of" + + +class ReferenceMotifLeakageError(ValueError): + """Fail-closed cross-split overlap or near-duplicate evidence.""" + + def __init__(self, findings: Sequence[Mapping[str, JSONValue]]) -> None: + self.findings = tuple(dict(item) for item in findings) + super().__init__( + "reference motif leakage audit failed with " + f"{len(self.findings)} finding(s)" + ) + + +class ReferenceMotifResourceLimitError(ValueError): + """A bounded build, artifact, or query limit was exceeded.""" + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifSplitV1: + """One half-open chronological source split.""" + + kind: ReferenceMotifSplitKind + start_ns: int + end_ns: int + split_id: str = "" + schema_version: str = REFERENCE_MOTIF_SPLIT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_SPLIT_SCHEMA_VERSION: + raise ValueError("unsupported reference motif split schema") + object.__setattr__( + self, "kind", ReferenceMotifSplitKind.from_value(self.kind) + ) + start = _bounded_int64(self.start_ns, "start_ns") + end = _bounded_int64(self.end_ns, "end_ns") + if end <= start: + raise ValueError("reference motif split end must follow start") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + expected = _stable_id("reference-motif-split", self.identity_payload()) + supplied = _optional_text(self.split_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif split_id differs") + object.__setattr__(self, "split_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "kind": self.kind.value, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "interval": "[start_ns,end_ns)", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "split_id": self.split_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReferenceMotifSplitV1": + _require_schema(data, REFERENCE_MOTIF_SPLIT_SCHEMA_VERSION) + return cls( + kind=ReferenceMotifSplitKind.from_value(str(data.get("kind", ""))), + start_ns=_strict_int(data.get("start_ns"), "start_ns"), + end_ns=_strict_int(data.get("end_ns"), "end_ns"), + split_id=str(data.get("split_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifConditionV1: + """Categorical and numeric coordinates for fragment retrieval.""" + + symbol: str + feed_epoch_id: str + session_state: str + currencies: tuple[str, ...] = () + active_sessions: tuple[str, ...] = () + overlap_tags: tuple[str, ...] = () + special_tags: tuple[str, ...] = () + holiday_tags: tuple[str, ...] = () + event_tags: tuple[str, ...] = () + return_regime: str = "unknown" + range_regime: str = "unknown" + volatility_regime: str = "unknown" + spread_regime: str = "unknown" + activity_regime: str = "unknown" + interarrival_regime: str = "unknown" + timestamp_precision: str = "unknown" + price_precision: str = "unknown" + source_quality_state: str = "unknown" + metrics: Mapping[str, float] = field(default_factory=dict) + schema_version: str = REFERENCE_MOTIF_CONDITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_CONDITION_SCHEMA_VERSION: + raise ValueError("unsupported reference motif condition schema") + symbol = _normalized_symbol(self.symbol) + object.__setattr__(self, "symbol", symbol) + object.__setattr__( + self, "feed_epoch_id", _required_text(self.feed_epoch_id) + ) + object.__setattr__( + self, "session_state", _required_text(self.session_state) + ) + currencies = _normalized_currencies( + self.currencies or _symbol_currencies(symbol) + ) + object.__setattr__(self, "currencies", currencies) + for name in ( + "active_sessions", + "overlap_tags", + "special_tags", + "holiday_tags", + "event_tags", + ): + object.__setattr__( + self, + name, + _normalized_text_tuple( + getattr(self, name), maximum=MAX_REFERENCE_MOTIF_TAGS + ), + ) + for name in ( + "return_regime", + "range_regime", + "volatility_regime", + "spread_regime", + "activity_regime", + "interarrival_regime", + "timestamp_precision", + "price_precision", + "source_quality_state", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + metrics = _metric_mapping(self.metrics) + object.__setattr__(self, "metrics", metrics) + + def pattern_for_level(self, level: str) -> dict[str, str] | None: + """Return an explicit categorical key pattern for one fallback level.""" + if level not in REFERENCE_MOTIF_BACKOFF_LEVELS: + raise ValueError("unsupported reference motif backoff level") + if level == "global": + return {} + if level == "epoch": + return {"feed_epoch_id": self.feed_epoch_id} + if level == "symbol": + return {"symbol": self.symbol} + if level == "currency_epoch": + return { + "currencies": ",".join(self.currencies), + "feed_epoch_id": self.feed_epoch_id, + } + base = {"symbol": self.symbol, "feed_epoch_id": self.feed_epoch_id} + if level == "symbol_epoch": + return base + if level == "symbol_epoch_state": + return { + **base, + "return_regime": self.return_regime, + "volatility_regime": self.volatility_regime, + "spread_regime": self.spread_regime, + "activity_regime": self.activity_regime, + } + session = { + **base, + "session_state": self.session_state, + "active_sessions": ",".join(self.active_sessions), + } + if level == "symbol_epoch_session": + return session + if level == "symbol_epoch_session_event": + return {**session, "event_tags": ",".join(self.event_tags)} + return self.coordinates() + + def matches(self, pattern: Mapping[str, str]) -> bool: + coordinates = self.coordinates() + return all( + coordinates.get(name) == value for name, value in pattern.items() + ) + + def coordinates(self) -> dict[str, str]: + """Return every categorical coordinate in comparison form.""" + return { + "symbol": self.symbol, + "currencies": ",".join(self.currencies), + "feed_epoch_id": self.feed_epoch_id, + "session_state": self.session_state, + "active_sessions": ",".join(self.active_sessions), + "overlap_tags": ",".join(self.overlap_tags), + "special_tags": ",".join(self.special_tags), + "holiday_tags": ",".join(self.holiday_tags), + "event_tags": ",".join(self.event_tags), + "return_regime": self.return_regime, + "range_regime": self.range_regime, + "volatility_regime": self.volatility_regime, + "spread_regime": self.spread_regime, + "activity_regime": self.activity_regime, + "interarrival_regime": self.interarrival_regime, + "timestamp_precision": self.timestamp_precision, + "price_precision": self.price_precision, + "source_quality_state": self.source_quality_state, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + **self.coordinates(), + "currencies": list(self.currencies), + "active_sessions": list(self.active_sessions), + "overlap_tags": list(self.overlap_tags), + "special_tags": list(self.special_tags), + "holiday_tags": list(self.holiday_tags), + "event_tags": list(self.event_tags), + "metrics": dict(self.metrics), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReferenceMotifConditionV1": + _require_schema(data, REFERENCE_MOTIF_CONDITION_SCHEMA_VERSION) + return cls( + symbol=str(data.get("symbol", "")), + feed_epoch_id=str(data.get("feed_epoch_id", "")), + session_state=str(data.get("session_state", "")), + currencies=_string_tuple(data.get("currencies")), + active_sessions=_string_tuple(data.get("active_sessions")), + overlap_tags=_string_tuple(data.get("overlap_tags")), + special_tags=_string_tuple(data.get("special_tags")), + holiday_tags=_string_tuple(data.get("holiday_tags")), + event_tags=_string_tuple(data.get("event_tags")), + return_regime=str(data.get("return_regime", "")), + range_regime=str(data.get("range_regime", "")), + volatility_regime=str(data.get("volatility_regime", "")), + spread_regime=str(data.get("spread_regime", "")), + activity_regime=str(data.get("activity_regime", "")), + interarrival_regime=str(data.get("interarrival_regime", "")), + timestamp_precision=str(data.get("timestamp_precision", "")), + price_precision=str(data.get("price_precision", "")), + source_quality_state=str(data.get("source_quality_state", "")), + metrics={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping(data.get("metrics")).items() + }, + schema_version=str(data.get("schema_version", "")), + ) + + +def reference_motif_condition_from_quotes( + *, + symbol: str, + feed_epoch_id: str, + session_state: str, + event_times_ns: Sequence[int], + bids: Sequence[float], + asks: Sequence[float], + event_tags: Sequence[str] = (), + active_sessions: Sequence[str] = (), + overlap_tags: Sequence[str] = (), + special_tags: Sequence[str] = (), + holiday_tags: Sequence[str] = (), + source_quality_score: float = 1.0, +) -> ReferenceMotifConditionV1: + """Derive the versioned retrieval coordinates from one quote window. + + The feature thresholds are fixed rather than fitted on a holdout. This + keeps production-library and benchmark-query classification identical and + makes weekday, event, activity, volatility, spread, return, precision, and + quality conditioning replayable from the recorded quote rows. + """ + times = tuple( + _bounded_int64(value, "event_time_ns") for value in event_times_ns + ) + bid_values = tuple(_positive_float(value, "bid") for value in bids) + ask_values = tuple(_positive_float(value, "ask") for value in asks) + if ( + len({len(times), len(bid_values), len(ask_values)}) != 1 + or len(times) < 2 + ): + raise ValueError("motif feature extraction requires aligned quote rows") + if list(times) != sorted(times) or times[-1] <= times[0]: + raise ValueError( + "motif feature timestamps must be ordered with duration" + ) + if any(ask < bid for bid, ask in zip(bid_values, ask_values)): + raise ValueError("motif feature extraction rejects negative spreads") + + mids = tuple((bid + ask) / 2.0 for bid, ask in zip(bid_values, ask_values)) + log_returns = tuple( + math.log(right / left) + for left, right in zip(mids, mids[1:]) + if left > 0.0 and right > 0.0 + ) + positive_intervals = tuple( + right - left for left, right in zip(times, times[1:]) if right > left + ) + duration_ns = max(1, times[-1] - times[0]) + return_value = math.log(mids[-1] / mids[0]) + range_value = (max(mids) - min(mids)) / max(1e-15, statistics.median(mids)) + volatility = math.sqrt( + sum(value * value for value in log_returns) / max(1, len(log_returns)) + ) + spread = statistics.median( + ask - bid for bid, ask in zip(bid_values, ask_values) + ) + relative_spread = spread / max(1e-15, statistics.median(mids)) + tick_intensity = len(times) * 1_000_000_000 / duration_ns + interarrival_ns = ( + float(statistics.median(positive_intervals)) + if positive_intervals + else float(duration_ns) + ) + timestamp_precision_ns = _integer_gcd(positive_intervals) or 1 + price_precision_digits = max( + _decimal_precision(value) for value in (*bid_values, *ask_values) + ) + quality_score = _finite_float(source_quality_score, "source_quality_score") + if not 0.0 <= quality_score <= 1.0: + raise ValueError("source_quality_score must be inside [0,1]") + + weekday = ( + datetime.fromtimestamp(times[0] / 1_000_000_000, tz=timezone.utc) + .strftime("%A") + .lower() + ) + tags = tuple(event_tags) or ("event:none",) + return ReferenceMotifConditionV1( + symbol=symbol, + feed_epoch_id=feed_epoch_id, + session_state=session_state, + active_sessions=tuple(active_sessions) or (session_state,), + overlap_tags=tuple(overlap_tags), + special_tags=tuple((*special_tags, f"weekday:{weekday}")), + holiday_tags=tuple(holiday_tags), + event_tags=tags, + return_regime=_signed_regime(return_value, 1e-5, 5e-5), + range_regime=_three_level_regime(range_value, 2e-5, 1e-4), + volatility_regime=_three_level_regime(volatility, 5e-6, 2.5e-5), + spread_regime=_three_level_regime(relative_spread, 5e-5, 2e-4), + activity_regime=_three_level_regime(tick_intensity, 1.0, 5.0), + interarrival_regime=( + "fast" + if interarrival_ns < 200_000_000 + else "normal" if interarrival_ns < 2_000_000_000 else "slow" + ), + timestamp_precision=( + "second" + if timestamp_precision_ns >= 1_000_000_000 + else ( + "millisecond" + if timestamp_precision_ns >= 1_000_000 + else "sub_millisecond" + ) + ), + price_precision=f"digits:{price_precision_digits}", + source_quality_state=( + "eligible" if quality_score >= 1.0 else "degraded" + ), + metrics={ + "return_value": return_value, + "range_value": range_value, + "volatility": volatility, + "spread": spread, + "tick_intensity": tick_intensity, + "interarrival_ns": interarrival_ns, + "timestamp_precision_ns": float(timestamp_precision_ns), + "price_precision_digits": float(price_precision_digits), + "source_quality_score": quality_score, + }, + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifTransformPolicyV1: + """Admissible fragment scaling and seam-warp envelope.""" + + min_time_scale: float = 0.5 + max_time_scale: float = 2.0 + min_price_scale: float = 0.5 + max_price_scale: float = 2.0 + max_time_warp_ratio: float = 1.25 + allow_price_translation: bool = True + allow_spread_scaling: bool = False + policy_id: str = "" + schema_version: str = REFERENCE_MOTIF_TRANSFORM_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_TRANSFORM_SCHEMA_VERSION: + raise ValueError("unsupported reference motif transform schema") + for prefix in ("time", "price"): + low = _positive_float( + getattr(self, f"min_{prefix}_scale"), f"min_{prefix}_scale" + ) + high = _positive_float( + getattr(self, f"max_{prefix}_scale"), f"max_{prefix}_scale" + ) + if not low <= 1.0 <= high: + raise ValueError(f"{prefix} scale range must contain one") + object.__setattr__(self, f"min_{prefix}_scale", low) + object.__setattr__(self, f"max_{prefix}_scale", high) + warp = _finite_float(self.max_time_warp_ratio, "max_time_warp_ratio") + if warp < 1.0: + raise ValueError("max_time_warp_ratio must be at least one") + object.__setattr__(self, "max_time_warp_ratio", warp) + _strict_bool(self.allow_price_translation, "allow_price_translation") + _strict_bool(self.allow_spread_scaling, "allow_spread_scaling") + expected = _stable_id( + "reference-motif-transform", self.identity_payload() + ) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif transform policy_id differs") + object.__setattr__(self, "policy_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "min_time_scale": self.min_time_scale, + "max_time_scale": self.max_time_scale, + "min_price_scale": self.min_price_scale, + "max_price_scale": self.max_price_scale, + "max_time_warp_ratio": self.max_time_warp_ratio, + "allow_price_translation": self.allow_price_translation, + "allow_spread_scaling": self.allow_spread_scaling, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "policy_id": self.policy_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReferenceMotifTransformPolicyV1": + _require_schema(data, REFERENCE_MOTIF_TRANSFORM_SCHEMA_VERSION) + return cls( + min_time_scale=_finite_float( + data.get("min_time_scale"), "min_time_scale" + ), + max_time_scale=_finite_float( + data.get("max_time_scale"), "max_time_scale" + ), + min_price_scale=_finite_float( + data.get("min_price_scale"), "min_price_scale" + ), + max_price_scale=_finite_float( + data.get("max_price_scale"), "max_price_scale" + ), + max_time_warp_ratio=_finite_float( + data.get("max_time_warp_ratio"), "max_time_warp_ratio" + ), + allow_price_translation=_strict_bool( + data.get("allow_price_translation"), "allow_price_translation" + ), + allow_spread_scaling=_strict_bool( + data.get("allow_spread_scaling"), "allow_spread_scaling" + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifSourceEventV1: + """One observed event projected from the augmented training surface.""" + + event_time_ns: int + event_sequence: int + bid: float + ask: float + source_row_id: int + source_event_id: str = "" + schema_version: str = REFERENCE_MOTIF_SOURCE_EVENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_SOURCE_EVENT_SCHEMA_VERSION: + raise ValueError("unsupported reference motif source event schema") + object.__setattr__( + self, + "event_time_ns", + _bounded_int64(self.event_time_ns, "event_time_ns"), + ) + sequence = _nonnegative_int(self.event_sequence, "event_sequence") + row_id = _nonnegative_int(self.source_row_id, "source_row_id") + bid = _positive_float(self.bid, "bid") + ask = _positive_float(self.ask, "ask") + if ask < bid: + raise ValueError("reference motif source event has negative spread") + object.__setattr__(self, "event_sequence", sequence) + object.__setattr__(self, "source_row_id", row_id) + object.__setattr__(self, "bid", bid) + object.__setattr__(self, "ask", ask) + expected = _stable_id( + "reference-motif-source-event", self.identity_payload() + ) + supplied = _optional_text(self.source_event_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif source_event_id differs") + object.__setattr__(self, "source_event_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "event_time_ns": self.event_time_ns, + "event_sequence": self.event_sequence, + "bid": self.bid, + "ask": self.ask, + "source_row_id": self.source_row_id, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "source_event_id": self.source_event_id, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReferenceMotifSourceEventV1": + _require_schema(data, REFERENCE_MOTIF_SOURCE_EVENT_SCHEMA_VERSION) + return cls( + event_time_ns=_strict_int( + data.get("event_time_ns"), "event_time_ns" + ), + event_sequence=_strict_int( + data.get("event_sequence"), "event_sequence" + ), + bid=_finite_float(data.get("bid"), "bid"), + ask=_finite_float(data.get("ask"), "ask"), + source_row_id=_strict_int( + data.get("source_row_id"), "source_row_id" + ), + source_event_id=str(data.get("source_event_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifSourceWindowV1: + """One bounded observed window before compact motif projection.""" + + source_series_id: str + period: str + source_artifact: ArtifactRef + split_kind: ReferenceMotifSplitKind + condition: ReferenceMotifConditionV1 + events: tuple[ReferenceMotifSourceEventV1, ...] + first_known_at_ns: int + available_at_ns: int + data_quality_eligible: bool = True + data_quality_reasons: tuple[str, ...] = () + transform_policy: ReferenceMotifTransformPolicyV1 = field( + default_factory=ReferenceMotifTransformPolicyV1 + ) + source_window_id: str = "" + schema_version: str = REFERENCE_MOTIF_SOURCE_WINDOW_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_SOURCE_WINDOW_SCHEMA_VERSION: + raise ValueError("unsupported reference motif source window schema") + object.__setattr__( + self, "source_series_id", _required_text(self.source_series_id) + ) + period = _required_text(self.period) + if not _PERIOD_RE.fullmatch(period): + raise ValueError("reference motif period must use YYYYMM") + object.__setattr__(self, "period", period) + object.__setattr__( + self, + "source_artifact", + _validated_artifact_ref(self.source_artifact), + ) + object.__setattr__( + self, + "split_kind", + ReferenceMotifSplitKind.from_value(self.split_kind), + ) + if not isinstance(self.condition, ReferenceMotifConditionV1): + raise ValueError( + "reference motif source window requires a v1 condition" + ) + events = tuple(self.events) + if not 2 <= len(events) <= MAX_REFERENCE_MOTIF_EVENTS: + raise ValueError( + "reference motif source window event count is outside limits" + ) + if any( + not isinstance(item, ReferenceMotifSourceEventV1) for item in events + ): + raise ValueError("reference motif source window requires v1 events") + positions = [ + (item.event_time_ns, item.event_sequence) for item in events + ] + if positions != sorted(positions) or len(set(positions)) != len( + positions + ): + raise ValueError( + "reference motif source events must be uniquely ordered" + ) + if events[-1].event_time_ns <= events[0].event_time_ns: + raise ValueError( + "reference motif source window requires positive duration" + ) + if len({item.source_row_id for item in events}) != len(events): + raise ValueError( + "reference motif source window has duplicate row lineage" + ) + if self.condition.symbol not in self.source_series_id.upper(): + raise ValueError( + "source series identity does not contain condition symbol" + ) + object.__setattr__(self, "events", events) + first_known = _bounded_int64( + self.first_known_at_ns, "first_known_at_ns" + ) + available = _bounded_int64(self.available_at_ns, "available_at_ns") + if first_known < events[-1].event_time_ns: + raise ValueError( + "source window cannot be known before its last observation" + ) + if available < first_known: + raise ValueError( + "source window available_at precedes first_known_at" + ) + object.__setattr__(self, "first_known_at_ns", first_known) + object.__setattr__(self, "available_at_ns", available) + eligible = _strict_bool( + self.data_quality_eligible, "data_quality_eligible" + ) + reasons = _normalized_text_tuple( + self.data_quality_reasons, maximum=MAX_REFERENCE_MOTIF_TAGS + ) + if eligible and reasons: + raise ValueError( + "eligible motif source cannot carry quality exclusions" + ) + if not eligible and not reasons: + raise ValueError("ineligible motif source requires quality reasons") + object.__setattr__(self, "data_quality_reasons", reasons) + if not isinstance( + self.transform_policy, ReferenceMotifTransformPolicyV1 + ): + raise ValueError( + "reference motif source requires a v1 transform policy" + ) + expected = _stable_id( + "reference-motif-source-window", self.identity_payload() + ) + supplied = _optional_text(self.source_window_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif source_window_id differs") + object.__setattr__(self, "source_window_id", expected) + + @property + def start_ns(self) -> int: + return self.events[0].event_time_ns + + @property + def end_ns(self) -> int: + """Return the last observed timestamp (inclusive).""" + return self.events[-1].event_time_ns + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "source_series_id": self.source_series_id, + "period": self.period, + "source_artifact": self.source_artifact.to_dict(), + "split_kind": self.split_kind.value, + "condition": self.condition.to_dict(), + "event_ids": [item.source_event_id for item in self.events], + "first_known_at_ns": self.first_known_at_ns, + "available_at_ns": self.available_at_ns, + "data_quality_eligible": self.data_quality_eligible, + "data_quality_reasons": list(self.data_quality_reasons), + "transform_policy": self.transform_policy.to_dict(), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "source_window_id": self.source_window_id, + "events": [item.to_dict() for item in self.events], + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReferenceMotifSourceWindowV1": + _require_schema(data, REFERENCE_MOTIF_SOURCE_WINDOW_SCHEMA_VERSION) + return cls( + source_series_id=str(data.get("source_series_id", "")), + period=str(data.get("period", "")), + source_artifact=ArtifactRef.from_dict( + _mapping(data.get("source_artifact")) + ), + split_kind=ReferenceMotifSplitKind.from_value( + str(data.get("split_kind", "")) + ), + condition=ReferenceMotifConditionV1.from_dict( + _mapping(data.get("condition")) + ), + events=tuple( + ReferenceMotifSourceEventV1.from_dict(item) + for item in _mapping_sequence(data.get("events")) + ), + first_known_at_ns=_strict_int( + data.get("first_known_at_ns"), "first_known_at_ns" + ), + available_at_ns=_strict_int( + data.get("available_at_ns"), "available_at_ns" + ), + data_quality_eligible=_strict_bool( + data.get("data_quality_eligible"), "data_quality_eligible" + ), + data_quality_reasons=_string_tuple( + data.get("data_quality_reasons") + ), + transform_policy=ReferenceMotifTransformPolicyV1.from_dict( + _mapping(data.get("transform_policy")) + ), + source_window_id=str(data.get("source_window_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifFragmentV1: + """Compact empirical sequence with complete source-window lineage.""" + + source_window_id: str + source_series_id: str + period: str + source_artifact: ArtifactRef + split_kind: ReferenceMotifSplitKind + condition: ReferenceMotifConditionV1 + source_start_ns: int + source_end_ns: int + source_row_ids: tuple[int, ...] + source_event_ids: tuple[str, ...] + event_offsets_ns: tuple[int, ...] + bid_deltas: tuple[float, ...] + ask_deltas: tuple[float, ...] + transitions: tuple[ReferenceMotifTransition, ...] + start_bid: float + start_ask: float + first_known_at_ns: int + available_at_ns: int + data_quality_eligible: bool + data_quality_reasons: tuple[str, ...] + transform_policy: ReferenceMotifTransformPolicyV1 + near_duplicate_signature: str + fragment_id: str = "" + schema_version: str = REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION: + raise ValueError("unsupported reference motif fragment schema") + object.__setattr__( + self, + "source_window_id", + _required_sha256_id( + self.source_window_id, + "source_window_id", + "reference-motif-source-window", + ), + ) + object.__setattr__( + self, "source_series_id", _required_text(self.source_series_id) + ) + if not _PERIOD_RE.fullmatch(self.period): + raise ValueError("reference motif fragment period must use YYYYMM") + object.__setattr__( + self, + "source_artifact", + _validated_artifact_ref(self.source_artifact), + ) + object.__setattr__( + self, + "split_kind", + ReferenceMotifSplitKind.from_value(self.split_kind), + ) + if not isinstance(self.condition, ReferenceMotifConditionV1): + raise ValueError("reference motif fragment requires a v1 condition") + start = _bounded_int64(self.source_start_ns, "source_start_ns") + end = _bounded_int64(self.source_end_ns, "source_end_ns") + if end <= start: + raise ValueError( + "reference motif fragment requires positive duration" + ) + object.__setattr__(self, "source_start_ns", start) + object.__setattr__(self, "source_end_ns", end) + row_ids = tuple( + _nonnegative_int(item, "source_row_id") + for item in self.source_row_ids + ) + event_ids = tuple( + _required_sha256_id( + item, "source_event_id", "reference-motif-source-event" + ) + for item in self.source_event_ids + ) + offsets = tuple( + _nonnegative_int64(item, "event_offset_ns") + for item in self.event_offsets_ns + ) + bids = tuple( + _finite_float(item, "bid_delta") for item in self.bid_deltas + ) + asks = tuple( + _finite_float(item, "ask_delta") for item in self.ask_deltas + ) + transitions = tuple( + ReferenceMotifTransition.from_value(item) + for item in self.transitions + ) + sizes = { + len(row_ids), + len(event_ids), + len(offsets), + len(bids), + len(asks), + len(transitions), + } + if ( + len(sizes) != 1 + or not 2 <= len(offsets) <= MAX_REFERENCE_MOTIF_EVENTS + ): + raise ValueError("reference motif compact sequence lengths differ") + if offsets[0] != 0 or offsets[-1] != end - start: + raise ValueError( + "reference motif offsets do not preserve boundaries" + ) + if list(offsets) != sorted(offsets): + raise ValueError("reference motif offsets must be ordered") + if bids[0] != 0.0 or asks[0] != 0.0: + raise ValueError("reference motif deltas must begin at zero") + if transitions[0] is not ReferenceMotifTransition.START: + raise ValueError("reference motif sequence must begin with start") + expected_transitions = (ReferenceMotifTransition.START,) + tuple( + _delta_transition( + bids[index - 1], + asks[index - 1], + bids[index], + asks[index], + ) + for index in range(1, len(bids)) + ) + if transitions != expected_transitions: + raise ValueError( + "reference motif transition marks disagree with quote deltas" + ) + object.__setattr__(self, "source_row_ids", row_ids) + object.__setattr__(self, "source_event_ids", event_ids) + object.__setattr__(self, "event_offsets_ns", offsets) + object.__setattr__(self, "bid_deltas", bids) + object.__setattr__(self, "ask_deltas", asks) + object.__setattr__(self, "transitions", transitions) + start_bid = _positive_float(self.start_bid, "start_bid") + start_ask = _positive_float(self.start_ask, "start_ask") + if start_ask < start_bid: + raise ValueError( + "reference motif fragment has negative start spread" + ) + reconstructed_quotes = tuple( + (start_bid + bid_delta, start_ask + ask_delta) + for bid_delta, ask_delta in zip(bids, asks) + ) + if any(bid <= 0.0 or ask <= 0.0 for bid, ask in reconstructed_quotes): + raise ValueError( + "reference motif fragment reconstructs a non-positive quote" + ) + if any(ask < bid for bid, ask in reconstructed_quotes): + raise ValueError( + "reference motif fragment reconstructs a negative spread" + ) + object.__setattr__(self, "start_bid", start_bid) + object.__setattr__(self, "start_ask", start_ask) + first_known = _bounded_int64( + self.first_known_at_ns, "first_known_at_ns" + ) + available = _bounded_int64(self.available_at_ns, "available_at_ns") + if first_known < end: + raise ValueError( + "reference motif fragment cannot be known before its end" + ) + if available < first_known: + raise ValueError("reference motif fragment availability regresses") + object.__setattr__(self, "first_known_at_ns", first_known) + object.__setattr__(self, "available_at_ns", available) + eligible = _strict_bool( + self.data_quality_eligible, "data_quality_eligible" + ) + reasons = _normalized_text_tuple( + self.data_quality_reasons, maximum=MAX_REFERENCE_MOTIF_TAGS + ) + if eligible == bool(reasons): + raise ValueError( + "fragment quality eligibility and reasons disagree" + ) + object.__setattr__(self, "data_quality_reasons", reasons) + if not isinstance( + self.transform_policy, ReferenceMotifTransformPolicyV1 + ): + raise ValueError("fragment requires a v1 transform policy") + object.__setattr__( + self, + "near_duplicate_signature", + _required_sha256_id( + self.near_duplicate_signature, + "near_duplicate_signature", + "reference-motif-shape", + ), + ) + expected = _stable_id( + "reference-motif-fragment", self.identity_payload() + ) + supplied = _optional_text(self.fragment_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif fragment_id differs") + object.__setattr__(self, "fragment_id", expected) + + @property + def duration_ns(self) -> int: + return self.source_end_ns - self.source_start_ns + + @property + def end_bid(self) -> float: + return self.start_bid + self.bid_deltas[-1] + + @property + def end_ask(self) -> float: + return self.start_ask + self.ask_deltas[-1] + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "source_window_id": self.source_window_id, + "source_series_id": self.source_series_id, + "period": self.period, + "source_artifact": self.source_artifact.to_dict(), + "split_kind": self.split_kind.value, + "condition": self.condition.to_dict(), + "source_start_ns": self.source_start_ns, + "source_end_ns": self.source_end_ns, + "source_row_ids": list(self.source_row_ids), + "source_event_ids": list(self.source_event_ids), + "event_offsets_ns": list(self.event_offsets_ns), + "bid_deltas": list(self.bid_deltas), + "ask_deltas": list(self.ask_deltas), + "transitions": [item.value for item in self.transitions], + "start_bid": self.start_bid, + "start_ask": self.start_ask, + "first_known_at_ns": self.first_known_at_ns, + "available_at_ns": self.available_at_ns, + "data_quality_eligible": self.data_quality_eligible, + "data_quality_reasons": list(self.data_quality_reasons), + "transform_policy": self.transform_policy.to_dict(), + "near_duplicate_signature": self.near_duplicate_signature, + "sequence_layout": "compact-offsets-and-deltas-v1", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "fragment_id": self.fragment_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReferenceMotifFragmentV1": + _require_schema(data, REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION) + return cls( + source_window_id=str(data.get("source_window_id", "")), + source_series_id=str(data.get("source_series_id", "")), + period=str(data.get("period", "")), + source_artifact=ArtifactRef.from_dict( + _mapping(data.get("source_artifact")) + ), + split_kind=ReferenceMotifSplitKind.from_value( + str(data.get("split_kind", "")) + ), + condition=ReferenceMotifConditionV1.from_dict( + _mapping(data.get("condition")) + ), + source_start_ns=_strict_int( + data.get("source_start_ns"), "source_start_ns" + ), + source_end_ns=_strict_int( + data.get("source_end_ns"), "source_end_ns" + ), + source_row_ids=_int_tuple(data.get("source_row_ids")), + source_event_ids=_string_tuple(data.get("source_event_ids")), + event_offsets_ns=_int_tuple(data.get("event_offsets_ns")), + bid_deltas=_float_tuple(data.get("bid_deltas")), + ask_deltas=_float_tuple(data.get("ask_deltas")), + transitions=tuple( + ReferenceMotifTransition.from_value(item) + for item in _string_tuple(data.get("transitions")) + ), + start_bid=_finite_float(data.get("start_bid"), "start_bid"), + start_ask=_finite_float(data.get("start_ask"), "start_ask"), + first_known_at_ns=_strict_int( + data.get("first_known_at_ns"), "first_known_at_ns" + ), + available_at_ns=_strict_int( + data.get("available_at_ns"), "available_at_ns" + ), + data_quality_eligible=_strict_bool( + data.get("data_quality_eligible"), "data_quality_eligible" + ), + data_quality_reasons=_string_tuple( + data.get("data_quality_reasons") + ), + transform_policy=ReferenceMotifTransformPolicyV1.from_dict( + _mapping(data.get("transform_policy")) + ), + near_duplicate_signature=str( + data.get("near_duplicate_signature", "") + ), + fragment_id=str(data.get("fragment_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +def _default_metric_scales() -> dict[str, float]: + return { + "return_value": 0.001, + "range_value": 0.001, + "volatility": 0.001, + "spread": 0.0001, + "tick_intensity": 1.0, + "interarrival_ns": 1_000_000_000.0, + "timestamp_precision_ns": 1_000_000.0, + "price_precision_digits": 1.0, + "source_quality_score": 1.0, + } + + +def _default_metric_weights() -> dict[str, float]: + return {name: 1.0 for name in REFERENCE_MOTIF_METRIC_NAMES} + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifIndexConfigV1: + """Bounded extraction, leakage, selection, and retrieval policy.""" + + min_events_per_fragment: int = 3 + max_events_per_fragment: int = MAX_REFERENCE_MOTIF_EVENTS + max_source_windows: int = MAX_REFERENCE_MOTIF_SOURCE_WINDOWS + max_fragments: int = MAX_REFERENCE_MOTIF_FRAGMENTS + min_cell_support: int = 2 + max_matches: int = 32 + source_overlap_guard_ns: int = 0 + near_duplicate_rounding_digits: int = 4 + rounding_digits: int = 12 + max_artifact_bytes: int = MAX_REFERENCE_MOTIF_ARTIFACT_BYTES + categorical_mismatch_penalty: float = 0.25 + missing_metric_penalty: float = 1.0 + metric_scales: Mapping[str, float] = field( + default_factory=_default_metric_scales + ) + metric_weights: Mapping[str, float] = field( + default_factory=_default_metric_weights + ) + backoff_levels: tuple[str, ...] = REFERENCE_MOTIF_BACKOFF_LEVELS + config_id: str = "" + schema_version: str = REFERENCE_MOTIF_INDEX_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_INDEX_CONFIG_SCHEMA_VERSION: + raise ValueError("unsupported reference motif index config schema") + minimum = _bounded_int( + self.min_events_per_fragment, + "min_events_per_fragment", + 2, + MAX_REFERENCE_MOTIF_EVENTS, + ) + maximum = _bounded_int( + self.max_events_per_fragment, + "max_events_per_fragment", + minimum, + MAX_REFERENCE_MOTIF_EVENTS, + ) + object.__setattr__(self, "min_events_per_fragment", minimum) + object.__setattr__(self, "max_events_per_fragment", maximum) + object.__setattr__( + self, + "max_source_windows", + _bounded_int( + self.max_source_windows, + "max_source_windows", + 1, + MAX_REFERENCE_MOTIF_SOURCE_WINDOWS, + ), + ) + object.__setattr__( + self, + "max_fragments", + _bounded_int( + self.max_fragments, + "max_fragments", + 1, + MAX_REFERENCE_MOTIF_FRAGMENTS, + ), + ) + object.__setattr__( + self, + "min_cell_support", + _bounded_int( + self.min_cell_support, + "min_cell_support", + 1, + MAX_REFERENCE_MOTIF_FRAGMENTS, + ), + ) + object.__setattr__( + self, + "max_matches", + _bounded_int( + self.max_matches, "max_matches", 1, MAX_REFERENCE_MOTIF_MATCHES + ), + ) + object.__setattr__( + self, + "source_overlap_guard_ns", + _nonnegative_int64( + self.source_overlap_guard_ns, "source_overlap_guard_ns" + ), + ) + for name in ("near_duplicate_rounding_digits", "rounding_digits"): + value = _bounded_int(getattr(self, name), name, 0, 16) + object.__setattr__(self, name, value) + object.__setattr__( + self, + "max_artifact_bytes", + _bounded_int( + self.max_artifact_bytes, + "max_artifact_bytes", + 1024, + MAX_REFERENCE_MOTIF_ARTIFACT_BYTES, + ), + ) + for name in ("categorical_mismatch_penalty", "missing_metric_penalty"): + penalty = _nonnegative_float(getattr(self, name), name) + object.__setattr__(self, name, penalty) + scales = _named_positive_mapping(self.metric_scales, "metric_scales") + weights = _named_nonnegative_mapping( + self.metric_weights, "metric_weights" + ) + object.__setattr__(self, "metric_scales", scales) + object.__setattr__(self, "metric_weights", weights) + levels = tuple(dict.fromkeys(str(item) for item in self.backoff_levels)) + if ( + not levels + or levels[-1] != "global" + or any( + item not in REFERENCE_MOTIF_BACKOFF_LEVELS for item in levels + ) + ): + raise ValueError( + "reference motif fallback must be supported and end in global" + ) + object.__setattr__(self, "backoff_levels", levels) + expected = _stable_id( + "reference-motif-index-config", self.identity_payload() + ) + supplied = _optional_text(self.config_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif config_id differs") + object.__setattr__(self, "config_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "min_events_per_fragment": self.min_events_per_fragment, + "max_events_per_fragment": self.max_events_per_fragment, + "max_source_windows": self.max_source_windows, + "max_fragments": self.max_fragments, + "min_cell_support": self.min_cell_support, + "max_matches": self.max_matches, + "source_overlap_guard_ns": self.source_overlap_guard_ns, + "near_duplicate_rounding_digits": ( + self.near_duplicate_rounding_digits + ), + "rounding_digits": self.rounding_digits, + "max_artifact_bytes": self.max_artifact_bytes, + "categorical_mismatch_penalty": self.categorical_mismatch_penalty, + "missing_metric_penalty": self.missing_metric_penalty, + "metric_scales": dict(self.metric_scales), + "metric_weights": dict(self.metric_weights), + "backoff_levels": list(self.backoff_levels), + "indexable_split": ReferenceMotifSplitKind.TRAIN.value, + "cross_split_near_duplicate_policy": "fail_closed", + "bounded_selection": "stable_hash_priority", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "config_id": self.config_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReferenceMotifIndexConfigV1": + _require_schema(data, REFERENCE_MOTIF_INDEX_CONFIG_SCHEMA_VERSION) + return cls( + min_events_per_fragment=_strict_int( + data.get("min_events_per_fragment"), "min_events_per_fragment" + ), + max_events_per_fragment=_strict_int( + data.get("max_events_per_fragment"), "max_events_per_fragment" + ), + max_source_windows=_strict_int( + data.get("max_source_windows"), "max_source_windows" + ), + max_fragments=_strict_int( + data.get("max_fragments"), "max_fragments" + ), + min_cell_support=_strict_int( + data.get("min_cell_support"), "min_cell_support" + ), + max_matches=_strict_int(data.get("max_matches"), "max_matches"), + source_overlap_guard_ns=_strict_int( + data.get("source_overlap_guard_ns"), "source_overlap_guard_ns" + ), + near_duplicate_rounding_digits=_strict_int( + data.get("near_duplicate_rounding_digits"), + "near_duplicate_rounding_digits", + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + max_artifact_bytes=_strict_int( + data.get("max_artifact_bytes"), "max_artifact_bytes" + ), + categorical_mismatch_penalty=_finite_float( + data.get("categorical_mismatch_penalty"), + "categorical_mismatch_penalty", + ), + missing_metric_penalty=_finite_float( + data.get("missing_metric_penalty"), "missing_metric_penalty" + ), + metric_scales={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping(data.get("metric_scales")).items() + }, + metric_weights={ + str(name): _finite_float(value, str(name)) + for name, value in _mapping(data.get("metric_weights")).items() + }, + backoff_levels=_string_tuple(data.get("backoff_levels")), + config_id=str(data.get("config_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +def extract_reference_motif_fragment( + window: ReferenceMotifSourceWindowV1, + *, + config: ReferenceMotifIndexConfigV1 | None = None, +) -> ReferenceMotifFragmentV1: + """Project one source window into compact offsets, deltas, and marks.""" + if not isinstance(window, ReferenceMotifSourceWindowV1): + raise ValueError("motif extraction requires a v1 source window") + selected = config or ReferenceMotifIndexConfigV1() + if ( + not selected.min_events_per_fragment + <= len(window.events) + <= selected.max_events_per_fragment + ): + raise ReferenceMotifResourceLimitError( + "source window event count is outside configured fragment limits" + ) + first = window.events[0] + offsets = tuple( + item.event_time_ns - first.event_time_ns for item in window.events + ) + bids = tuple( + round(item.bid - first.bid, selected.rounding_digits) + for item in window.events + ) + asks = tuple( + round(item.ask - first.ask, selected.rounding_digits) + for item in window.events + ) + transitions = (ReferenceMotifTransition.START,) + tuple( + _transition(previous, current) + for previous, current in zip(window.events, window.events[1:]) + ) + signature = _near_duplicate_signature( + symbol=window.condition.symbol, + offsets=offsets, + bid_deltas=bids, + ask_deltas=asks, + transitions=transitions, + digits=selected.near_duplicate_rounding_digits, + ) + return ReferenceMotifFragmentV1( + source_window_id=window.source_window_id, + source_series_id=window.source_series_id, + period=window.period, + source_artifact=window.source_artifact, + split_kind=window.split_kind, + condition=window.condition, + source_start_ns=window.start_ns, + source_end_ns=window.end_ns, + source_row_ids=tuple(item.source_row_id for item in window.events), + source_event_ids=tuple(item.source_event_id for item in window.events), + event_offsets_ns=offsets, + bid_deltas=bids, + ask_deltas=asks, + transitions=transitions, + start_bid=first.bid, + start_ask=first.ask, + first_known_at_ns=window.first_known_at_ns, + available_at_ns=window.available_at_ns, + data_quality_eligible=window.data_quality_eligible, + data_quality_reasons=window.data_quality_reasons, + transform_policy=window.transform_policy, + near_duplicate_signature=signature, + ) + + +def reference_motif_source_window_from_training_frame( + frame: Any, + *, + source_artifact: ArtifactRef, + split_kind: ReferenceMotifSplitKind, + condition: ReferenceMotifConditionV1, + first_known_at_ns: int, + available_at_ns: int, + transform_policy: ReferenceMotifTransformPolicyV1 | None = None, +) -> ReferenceMotifSourceWindowV1: + """Project only identity/market/quality columns from an augmented frame.""" + required = { + "series_id", + "period", + "row_id", + "event_seq", + "symbol", + "timestamp_utc_ms", + "bid", + "ask", + "training_usable", + "training_exclusion_reason_code", + } + columns = set(getattr(frame, "columns", ())) + missing = sorted(required - columns) + if missing: + raise ValueError( + f"training frame lacks motif source columns: {missing}" + ) + selected = frame.select(sorted(required)).sort( + ["timestamp_utc_ms", "event_seq"] + ) + rows = selected.to_dicts() + if len(rows) < 2: + raise ValueError( + "training frame motif window requires at least two rows" + ) + symbols = {_normalized_symbol(row["symbol"]) for row in rows} + series_ids = {_required_text(row["series_id"]) for row in rows} + periods = {_required_text(row["period"]) for row in rows} + if ( + symbols != {condition.symbol} + or len(series_ids) != 1 + or len(periods) != 1 + ): + raise ValueError("training frame motif window axis is not homogeneous") + classifications = tuple( + ( + _strict_bool(row["training_usable"], "training_usable"), + _nonnegative_int( + _strict_int( + row["training_exclusion_reason_code"], + "training_exclusion_reason_code", + ), + "training_exclusion_reason_code", + ), + ) + for row in rows + ) + if any( + usable != (reason_code == 0) for usable, reason_code in classifications + ): + raise ValueError( + "training usability and exclusion reason code disagree" + ) + reasons = tuple( + sorted( + { + str(reason_code) + for usable, reason_code in classifications + if not usable + } + ) + ) + events = tuple( + ReferenceMotifSourceEventV1( + event_time_ns=_strict_int( + row["timestamp_utc_ms"], "timestamp_utc_ms" + ) + * 1_000_000, + event_sequence=_strict_int(row["event_seq"], "event_seq"), + bid=_finite_float(row["bid"], "bid"), + ask=_finite_float(row["ask"], "ask"), + source_row_id=_strict_int(row["row_id"], "row_id"), + ) + for row in rows + ) + return ReferenceMotifSourceWindowV1( + source_series_id=next(iter(series_ids)), + period=next(iter(periods)), + source_artifact=source_artifact, + split_kind=split_kind, + condition=condition, + events=events, + first_known_at_ns=first_known_at_ns, + available_at_ns=available_at_ns, + data_quality_eligible=not reasons, + data_quality_reasons=reasons, + transform_policy=transform_policy or ReferenceMotifTransformPolicyV1(), + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifIndexV1: + """Bounded in-memory form of one artifact-backed empirical library.""" + + config: ReferenceMotifIndexConfigV1 + splits: tuple[ReferenceMotifSplitV1, ...] + fragments: tuple[ReferenceMotifFragmentV1, ...] + source_window_count: int + excluded_split_counts: Mapping[str, int] + ineligible_window_count: int + selection_omitted_count: int + leakage_comparison_count: int + lineage_artifacts: tuple[ArtifactRef, ...] + index_id: str = "" + schema_version: str = REFERENCE_MOTIF_INDEX_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_INDEX_SCHEMA_VERSION: + raise ValueError("unsupported reference motif index schema") + if not isinstance(self.config, ReferenceMotifIndexConfigV1): + raise ValueError("reference motif index requires a v1 config") + splits = _validated_splits(self.splits) + object.__setattr__(self, "splits", splits) + fragments = tuple( + sorted(self.fragments, key=lambda item: item.fragment_id) + ) + if not fragments or len(fragments) > self.config.max_fragments: + raise ValueError( + "reference motif index fragment count is outside limits" + ) + if any( + not isinstance(item, ReferenceMotifFragmentV1) for item in fragments + ): + raise ValueError("reference motif index requires v1 fragments") + if any( + item.split_kind is not ReferenceMotifSplitKind.TRAIN + or not item.data_quality_eligible + for item in fragments + ): + raise ValueError( + "reference motif index may retain only eligible train fragments" + ) + if len({item.fragment_id for item in fragments}) != len(fragments): + raise ValueError("reference motif index has duplicate fragment IDs") + train = splits[0] + if any( + not ( + train.start_ns <= item.source_start_ns + and item.source_end_ns < train.end_ns + ) + for item in fragments + ): + raise ValueError( + "reference motif fragment falls outside train split" + ) + object.__setattr__(self, "fragments", fragments) + source_count = _bounded_int( + self.source_window_count, + "source_window_count", + len(fragments), + self.config.max_source_windows, + ) + object.__setattr__(self, "source_window_count", source_count) + excluded = _split_count_mapping(self.excluded_split_counts) + object.__setattr__(self, "excluded_split_counts", excluded) + ineligible = _bounded_int( + self.ineligible_window_count, + "ineligible_window_count", + 0, + source_count, + ) + omitted = _bounded_int( + self.selection_omitted_count, + "selection_omitted_count", + 0, + source_count, + ) + comparisons = _nonnegative_int( + self.leakage_comparison_count, + "leakage_comparison_count", + ) + object.__setattr__(self, "ineligible_window_count", ineligible) + object.__setattr__(self, "selection_omitted_count", omitted) + object.__setattr__(self, "leakage_comparison_count", comparisons) + if ( + len(fragments) + sum(excluded.values()) + ineligible + omitted + != source_count + ): + raise ValueError( + "reference motif index source-window accounting differs" + ) + artifacts = _unique_artifact_refs(self.lineage_artifacts) + source_hashes = {item.source_artifact.sha256 for item in fragments} + if not source_hashes.issubset({item.sha256 for item in artifacts}): + raise ValueError( + "reference motif lineage artifacts do not cover fragments" + ) + object.__setattr__(self, "lineage_artifacts", artifacts) + expected = _stable_id("reference-motif-index", self.identity_payload()) + supplied = _optional_text(self.index_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif index_id differs") + object.__setattr__(self, "index_id", expected) + if len(self.to_json().encode("utf-8")) > self.config.max_artifact_bytes: + raise ReferenceMotifResourceLimitError( + "reference motif index exceeds configured artifact bytes" + ) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "config": self.config.to_dict(), + "splits": [item.to_dict() for item in self.splits], + "fragments": [item.to_dict() for item in self.fragments], + "source_window_count": self.source_window_count, + "excluded_split_counts": dict(self.excluded_split_counts), + "ineligible_window_count": self.ineligible_window_count, + "selection_omitted_count": self.selection_omitted_count, + "leakage_comparison_count": self.leakage_comparison_count, + "lineage_artifacts": [ + item.to_dict() for item in self.lineage_artifacts + ], + "retention_policy": "eligible-train-only", + "payload_layout": "compact-offsets-and-deltas-v1", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "index_id": self.index_id} + + def to_json(self) -> str: + return _canonical_json(self.to_dict()) + + def fragment_by_id(self, fragment_id: str) -> ReferenceMotifFragmentV1: + selected = _required_sha256_id( + fragment_id, "fragment_id", "reference-motif-fragment" + ) + try: + return next( + item for item in self.fragments if item.fragment_id == selected + ) + except StopIteration as err: + raise KeyError(selected) from err + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReferenceMotifIndexV1": + _require_schema(data, REFERENCE_MOTIF_INDEX_SCHEMA_VERSION) + return cls( + config=ReferenceMotifIndexConfigV1.from_dict( + _mapping(data.get("config")) + ), + splits=tuple( + ReferenceMotifSplitV1.from_dict(item) + for item in _mapping_sequence(data.get("splits")) + ), + fragments=tuple( + ReferenceMotifFragmentV1.from_dict(item) + for item in _mapping_sequence(data.get("fragments")) + ), + source_window_count=_strict_int( + data.get("source_window_count"), "source_window_count" + ), + excluded_split_counts={ + str(name): _strict_int(value, str(name)) + for name, value in _mapping( + data.get("excluded_split_counts") + ).items() + }, + ineligible_window_count=_strict_int( + data.get("ineligible_window_count"), "ineligible_window_count" + ), + selection_omitted_count=_strict_int( + data.get("selection_omitted_count"), "selection_omitted_count" + ), + leakage_comparison_count=_strict_int( + data.get("leakage_comparison_count"), "leakage_comparison_count" + ), + lineage_artifacts=tuple( + ArtifactRef.from_dict(item) + for item in _mapping_sequence(data.get("lineage_artifacts")) + ), + index_id=str(data.get("index_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReferenceMotifIndexV1": + return cls.from_dict(_json_mapping(text)) + + +def build_reference_motif_index( + source_windows: Iterable[ReferenceMotifSourceWindowV1], + *, + splits: Sequence[ReferenceMotifSplitV1], + config: ReferenceMotifIndexConfigV1 | None = None, +) -> ReferenceMotifIndexV1: + """Build a deterministic train-only library. + + Cross-split leakage is audited before any fragment is retained. + """ + selected = config or ReferenceMotifIndexConfigV1() + declared_splits = _validated_splits(splits) + values: list[ReferenceMotifSourceWindowV1] = [] + for value in source_windows: + if not isinstance(value, ReferenceMotifSourceWindowV1): + raise ValueError("reference motif build requires v1 source windows") + values.append(value) + if len(values) > selected.max_source_windows: + raise ReferenceMotifResourceLimitError( + "reference motif source windows exceed configured maximum" + ) + ordered = tuple(sorted(values, key=lambda item: item.source_window_id)) + if not ordered: + raise ValueError("reference motif index requires source windows") + if len({item.source_window_id for item in ordered}) != len(ordered): + raise ValueError("reference motif source_window_id is duplicated") + by_kind = {item.kind: item for item in declared_splits} + for window in ordered: + split = by_kind[window.split_kind] + if not ( + split.start_ns <= window.start_ns and window.end_ns < split.end_ns + ): + raise ValueError( + "reference motif source window falls outside its split" + ) + + projected = tuple( + extract_reference_motif_fragment(window, config=selected) + for window in ordered + if selected.min_events_per_fragment + <= len(window.events) + <= selected.max_events_per_fragment + ) + if len(projected) != len(ordered): + raise ReferenceMotifResourceLimitError( + "reference motif source window event count is outside " + "configured limits" + ) + comparisons, findings = _cross_split_leakage_findings( + ordered, + projected, + guard_ns=selected.source_overlap_guard_ns, + ) + if findings: + raise ReferenceMotifLeakageError( + findings[:MAX_REFERENCE_MOTIF_LEAKAGE_FINDINGS] + ) + + excluded = Counter( + item.split_kind.value + for item in ordered + if item.split_kind is not ReferenceMotifSplitKind.TRAIN + ) + ineligible = sum( + item.split_kind is ReferenceMotifSplitKind.TRAIN + and not item.data_quality_eligible + for item in ordered + ) + candidates = [ + fragment + for fragment in projected + if fragment.split_kind is ReferenceMotifSplitKind.TRAIN + and fragment.data_quality_eligible + ] + ranked = sorted( + candidates, + key=lambda item: ( + hashlib.sha256( + f"{selected.config_id}:{item.fragment_id}".encode("utf-8") + ).hexdigest(), + item.fragment_id, + ), + ) + retained = tuple( + sorted( + ranked[: selected.max_fragments], key=lambda item: item.fragment_id + ) + ) + omitted = len(candidates) - len(retained) + if not retained: + raise ValueError( + "reference motif build retained no eligible train fragments" + ) + artifacts = _unique_artifact_refs( + tuple(item.source_artifact for item in retained) + ) + return ReferenceMotifIndexV1( + config=selected, + splits=declared_splits, + fragments=retained, + source_window_count=len(ordered), + excluded_split_counts=dict(excluded), + ineligible_window_count=ineligible, + selection_omitted_count=omitted, + leakage_comparison_count=comparisons, + lineage_artifacts=artifacts, + ) + + +def write_reference_motif_index( + index: ReferenceMotifIndexV1, + path: str | Path, +) -> ArtifactRef: + """Atomically write an index and return a content-verifiable reference.""" + if not isinstance(index, ReferenceMotifIndexV1): + raise ValueError("reference motif writer requires a v1 index") + target = Path(path) + payload = f"{index.to_json()}\n".encode("utf-8") + if len(payload) > index.config.max_artifact_bytes: + raise ReferenceMotifResourceLimitError( + "reference motif artifact exceeds configured maximum" + ) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(f".{target.name}.{os.getpid()}.tmp") + try: + temporary.write_bytes(payload) + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + digest = hashlib.sha256(payload).hexdigest() + return ArtifactRef( + kind=REFERENCE_MOTIF_ARTIFACT_KIND, + path=str(target), + size_bytes=len(payload), + sha256=digest, + metadata={ + "schema_version": index.schema_version, + "index_id": index.index_id, + "config_id": index.config.config_id, + "fragment_count": len(index.fragments), + "source_window_count": index.source_window_count, + "payload_layout": "compact-offsets-and-deltas-v1", + }, + ) + + +def read_reference_motif_index( + path: str | Path, + *, + artifact_ref: ArtifactRef | None = None, +) -> ReferenceMotifIndexV1: + """Read and verify a persisted motif index and optional artifact identity.""" + target = Path(path) + payload = target.read_bytes() + if artifact_ref is not None: + reference = _validated_artifact_ref(artifact_ref) + if reference.kind != REFERENCE_MOTIF_ARTIFACT_KIND: + raise ValueError("artifact kind is not a reference motif index") + if reference.size_bytes != len(payload): + raise ValueError("reference motif artifact size differs") + if reference.sha256 != hashlib.sha256(payload).hexdigest(): + raise ValueError("reference motif artifact sha256 differs") + return ReferenceMotifIndexV1.from_json(payload.decode("utf-8")) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifQueryV1: + """One bounded deterministic retrieval request.""" + + condition: ReferenceMotifConditionV1 + information_mode: InformationMode + used_at_ns: int + as_of_ns: int | None = None + max_results: int = 16 + min_cell_support: int | None = None + max_distance: float | None = None + excluded_source_window_ids: tuple[str, ...] = () + query_id: str = "" + schema_version: str = REFERENCE_MOTIF_QUERY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_QUERY_SCHEMA_VERSION: + raise ValueError("unsupported reference motif query schema") + if not isinstance(self.condition, ReferenceMotifConditionV1): + raise ValueError("reference motif query requires a v1 condition") + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + used = _bounded_int64(self.used_at_ns, "used_at_ns") + object.__setattr__(self, "used_at_ns", used) + if self.information_mode is InformationMode.EX_ANTE_SIMULATION: + if self.as_of_ns is None: + raise ValueError("ex-ante motif query requires as_of_ns") + as_of = _bounded_int64(self.as_of_ns, "as_of_ns") + if as_of > used: + raise ValueError( + "motif query as_of_ns cannot follow used_at_ns" + ) + object.__setattr__(self, "as_of_ns", as_of) + elif self.as_of_ns is not None: + raise ValueError("ex-post motif query does not accept as_of_ns") + object.__setattr__( + self, + "max_results", + _bounded_int( + self.max_results, "max_results", 1, MAX_REFERENCE_MOTIF_MATCHES + ), + ) + if self.min_cell_support is not None: + object.__setattr__( + self, + "min_cell_support", + _bounded_int( + self.min_cell_support, + "min_cell_support", + 1, + MAX_REFERENCE_MOTIF_FRAGMENTS, + ), + ) + if self.max_distance is not None: + object.__setattr__( + self, + "max_distance", + _nonnegative_float(self.max_distance, "max_distance"), + ) + exclusions = tuple( + sorted( + { + _required_sha256_id( + item, + "excluded_source_window_id", + "reference-motif-source-window", + ) + for item in self.excluded_source_window_ids + } + ) + ) + if len(exclusions) > MAX_REFERENCE_MOTIF_EXCLUSIONS: + raise ValueError("reference motif query exclusions exceed limit") + object.__setattr__(self, "excluded_source_window_ids", exclusions) + expected = _stable_id("reference-motif-query", self.identity_payload()) + supplied = _optional_text(self.query_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif query_id differs") + object.__setattr__(self, "query_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "condition": self.condition.to_dict(), + "information_mode": self.information_mode.value, + "used_at_ns": self.used_at_ns, + "as_of_ns": self.as_of_ns, + "max_results": self.max_results, + "min_cell_support": self.min_cell_support, + "max_distance": self.max_distance, + "excluded_source_window_ids": list(self.excluded_source_window_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "query_id": self.query_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReferenceMotifQueryV1": + _require_schema(data, REFERENCE_MOTIF_QUERY_SCHEMA_VERSION) + as_of = data.get("as_of_ns") + support = data.get("min_cell_support") + distance = data.get("max_distance") + return cls( + condition=ReferenceMotifConditionV1.from_dict( + _mapping(data.get("condition")) + ), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + used_at_ns=_strict_int(data.get("used_at_ns"), "used_at_ns"), + as_of_ns=( + _strict_int(as_of, "as_of_ns") if as_of is not None else None + ), + max_results=_strict_int(data.get("max_results"), "max_results"), + min_cell_support=( + _strict_int(support, "min_cell_support") + if support is not None + else None + ), + max_distance=( + _finite_float(distance, "max_distance") + if distance is not None + else None + ), + excluded_source_window_ids=_string_tuple( + data.get("excluded_source_window_ids") + ), + query_id=str(data.get("query_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifBackoffAttemptV1: + """One observable fallback decision, including sparse cells.""" + + level: str + pattern: Mapping[str, str] + candidate_count: int + available_count: int + minimum_support: int + outcome: str + schema_version: str = REFERENCE_MOTIF_BACKOFF_ATTEMPT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != REFERENCE_MOTIF_BACKOFF_ATTEMPT_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reference motif backoff attempt schema" + ) + if self.level not in REFERENCE_MOTIF_BACKOFF_LEVELS: + raise ValueError( + "unsupported reference motif backoff attempt level" + ) + object.__setattr__( + self, + "pattern", + { + str(name): str(value) + for name, value in sorted(self.pattern.items()) + }, + ) + object.__setattr__( + self, + "candidate_count", + _nonnegative_int(self.candidate_count, "candidate_count"), + ) + object.__setattr__( + self, + "available_count", + _nonnegative_int(self.available_count, "available_count"), + ) + object.__setattr__( + self, + "minimum_support", + _positive_int(self.minimum_support, "minimum_support"), + ) + outcome = _required_text(self.outcome) + if outcome not in { + "selected", + "sparse", + "unavailable", + "distance_filtered", + }: + raise ValueError("unsupported reference motif backoff outcome") + object.__setattr__(self, "outcome", outcome) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "level": self.level, + "pattern": dict(self.pattern), + "candidate_count": self.candidate_count, + "available_count": self.available_count, + "minimum_support": self.minimum_support, + "outcome": self.outcome, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReferenceMotifBackoffAttemptV1": + _require_schema(data, REFERENCE_MOTIF_BACKOFF_ATTEMPT_SCHEMA_VERSION) + return cls( + level=str(data.get("level", "")), + pattern={ + str(name): str(value) + for name, value in _mapping(data.get("pattern")).items() + }, + candidate_count=_strict_int( + data.get("candidate_count"), "candidate_count" + ), + available_count=_strict_int( + data.get("available_count"), "available_count" + ), + minimum_support=_strict_int( + data.get("minimum_support"), "minimum_support" + ), + outcome=str(data.get("outcome", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifMatchV1: + """One ranked motif with transparent support and complete lineage.""" + + fragment: ReferenceMotifFragmentV1 + distance: float + cell_support: int + backoff_level: str + rank: int + schema_version: str = REFERENCE_MOTIF_MATCH_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_MATCH_SCHEMA_VERSION: + raise ValueError("unsupported reference motif match schema") + if not isinstance(self.fragment, ReferenceMotifFragmentV1): + raise ValueError("reference motif match requires a v1 fragment") + object.__setattr__( + self, "distance", _nonnegative_float(self.distance, "distance") + ) + object.__setattr__( + self, + "cell_support", + _positive_int(self.cell_support, "cell_support"), + ) + if self.backoff_level not in REFERENCE_MOTIF_BACKOFF_LEVELS: + raise ValueError("unsupported reference motif match backoff level") + object.__setattr__(self, "rank", _positive_int(self.rank, "rank")) + + def to_dict(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "fragment": self.fragment.to_dict(), + "distance": self.distance, + "cell_support": self.cell_support, + "backoff_level": self.backoff_level, + "rank": self.rank, + "tie_break": [self.distance, self.fragment.fragment_id], + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReferenceMotifMatchV1": + _require_schema(data, REFERENCE_MOTIF_MATCH_SCHEMA_VERSION) + return cls( + fragment=ReferenceMotifFragmentV1.from_dict( + _mapping(data.get("fragment")) + ), + distance=_finite_float(data.get("distance"), "distance"), + cell_support=_strict_int(data.get("cell_support"), "cell_support"), + backoff_level=str(data.get("backoff_level", "")), + rank=_strict_int(data.get("rank"), "rank"), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReferenceMotifQueryResultV1: + """Bounded deterministic retrieval result and full fallback trace.""" + + index_id: str + query: ReferenceMotifQueryV1 + status: ReferenceMotifQueryStatus + matches: tuple[ReferenceMotifMatchV1, ...] + backoff_attempts: tuple[ReferenceMotifBackoffAttemptV1, ...] + scanned_fragment_count: int + hidden_by_availability_count: int + result_id: str = "" + schema_version: str = REFERENCE_MOTIF_QUERY_RESULT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REFERENCE_MOTIF_QUERY_RESULT_SCHEMA_VERSION: + raise ValueError("unsupported reference motif query result schema") + object.__setattr__( + self, + "index_id", + _required_sha256_id( + self.index_id, "index_id", "reference-motif-index" + ), + ) + if not isinstance(self.query, ReferenceMotifQueryV1): + raise ValueError("reference motif result requires a v1 query") + object.__setattr__( + self, "status", ReferenceMotifQueryStatus(self.status) + ) + matches = tuple(self.matches) + attempts = tuple(self.backoff_attempts) + if len(matches) > self.query.max_results: + raise ValueError("reference motif result exceeds query maximum") + if any(not isinstance(item, ReferenceMotifMatchV1) for item in matches): + raise ValueError("reference motif result requires v1 matches") + if any( + not isinstance(item, ReferenceMotifBackoffAttemptV1) + for item in attempts + ): + raise ValueError("reference motif result requires v1 attempts") + if self.status is ReferenceMotifQueryStatus.MATCHED and not matches: + raise ValueError("matched reference motif result requires matches") + if self.status is not ReferenceMotifQueryStatus.MATCHED and matches: + raise ValueError( + "unmatched reference motif result cannot carry matches" + ) + object.__setattr__(self, "matches", matches) + object.__setattr__(self, "backoff_attempts", attempts) + object.__setattr__( + self, + "scanned_fragment_count", + _nonnegative_int( + self.scanned_fragment_count, "scanned_fragment_count" + ), + ) + object.__setattr__( + self, + "hidden_by_availability_count", + _nonnegative_int( + self.hidden_by_availability_count, + "hidden_by_availability_count", + ), + ) + expected = _stable_id( + "reference-motif-query-result", self.identity_payload() + ) + supplied = _optional_text(self.result_id) + if supplied is not None and supplied != expected: + raise ValueError("reference motif result_id differs") + object.__setattr__(self, "result_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "index_id": self.index_id, + "query": self.query.to_dict(), + "status": self.status.value, + "matches": [item.to_dict() for item in self.matches], + "backoff_attempts": [ + item.to_dict() for item in self.backoff_attempts + ], + "scanned_fragment_count": self.scanned_fragment_count, + "hidden_by_availability_count": self.hidden_by_availability_count, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "result_id": self.result_id} + + def to_json(self) -> str: + return _canonical_json(self.to_dict()) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReferenceMotifQueryResultV1": + _require_schema(data, REFERENCE_MOTIF_QUERY_RESULT_SCHEMA_VERSION) + return cls( + index_id=str(data.get("index_id", "")), + query=ReferenceMotifQueryV1.from_dict(_mapping(data.get("query"))), + status=ReferenceMotifQueryStatus(str(data.get("status", ""))), + matches=tuple( + ReferenceMotifMatchV1.from_dict(item) + for item in _mapping_sequence(data.get("matches")) + ), + backoff_attempts=tuple( + ReferenceMotifBackoffAttemptV1.from_dict(item) + for item in _mapping_sequence(data.get("backoff_attempts")) + ), + scanned_fragment_count=_strict_int( + data.get("scanned_fragment_count"), "scanned_fragment_count" + ), + hidden_by_availability_count=_strict_int( + data.get("hidden_by_availability_count"), + "hidden_by_availability_count", + ), + result_id=str(data.get("result_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReferenceMotifQueryResultV1": + return cls.from_dict(_json_mapping(text)) + + +def query_reference_motifs( + index: ReferenceMotifIndexV1, + query: ReferenceMotifQueryV1, +) -> ReferenceMotifQueryResultV1: + """Retrieve the first supported fallback cell with deterministic ranking.""" + if not isinstance(index, ReferenceMotifIndexV1): + raise ValueError("motif retrieval requires a v1 index") + if not isinstance(query, ReferenceMotifQueryV1): + raise ValueError("motif retrieval requires a v1 query") + minimum = query.min_cell_support or index.config.min_cell_support + maximum = min(query.max_results, index.config.max_matches) + exclusions = set(query.excluded_source_window_ids) + attempts: list[ReferenceMotifBackoffAttemptV1] = [] + hidden_ids: set[str] = set() + + for level in index.config.backoff_levels: + pattern = query.condition.pattern_for_level(level) + if pattern is None: + continue + candidates = tuple( + item + for item in index.fragments + if item.source_window_id not in exclusions + and item.condition.matches(pattern) + ) + available: list[ReferenceMotifFragmentV1] = [] + for item in candidates: + if ( + query.information_mode is InformationMode.EX_ANTE_SIMULATION + and ( + item.available_at_ns > cast(int, query.as_of_ns) + or item.source_end_ns > cast(int, query.as_of_ns) + ) + ): + hidden_ids.add(item.fragment_id) + continue + available.append(item) + if len(available) < minimum: + attempts.append( + ReferenceMotifBackoffAttemptV1( + level=level, + pattern=pattern, + candidate_count=len(candidates), + available_count=len(available), + minimum_support=minimum, + outcome=( + "unavailable" + if candidates and not available + else "sparse" + ), + ) + ) + continue + ranked = sorted( + ( + ( + _condition_distance( + query.condition, + item.condition, + index.config, + ), + item, + ) + for item in available + ), + key=lambda pair: (pair[0], pair[1].fragment_id), + ) + if query.max_distance is not None: + ranked = [pair for pair in ranked if pair[0] <= query.max_distance] + if not ranked: + attempts.append( + ReferenceMotifBackoffAttemptV1( + level=level, + pattern=pattern, + candidate_count=len(candidates), + available_count=len(available), + minimum_support=minimum, + outcome="distance_filtered", + ) + ) + continue + matches = tuple( + ReferenceMotifMatchV1( + fragment=item, + distance=distance, + cell_support=len(available), + backoff_level=level, + rank=rank, + ) + for rank, (distance, item) in enumerate(ranked[:maximum], start=1) + ) + attempts.append( + ReferenceMotifBackoffAttemptV1( + level=level, + pattern=pattern, + candidate_count=len(candidates), + available_count=len(available), + minimum_support=minimum, + outcome="selected", + ) + ) + return ReferenceMotifQueryResultV1( + index_id=index.index_id, + query=query, + status=ReferenceMotifQueryStatus.MATCHED, + matches=matches, + backoff_attempts=tuple(attempts), + scanned_fragment_count=len(index.fragments) * len(attempts), + hidden_by_availability_count=len(hidden_ids), + ) + status = ( + ReferenceMotifQueryStatus.NOT_AVAILABLE_AS_OF + if hidden_ids + else ReferenceMotifQueryStatus.NO_SUPPORTED_CELL + ) + return ReferenceMotifQueryResultV1( + index_id=index.index_id, + query=query, + status=status, + matches=(), + backoff_attempts=tuple(attempts), + scanned_fragment_count=len(index.fragments) * len(attempts), + hidden_by_availability_count=len(hidden_ids), + ) + + +def reference_motif_information_inputs( + result: ReferenceMotifQueryResultV1, + *, + run_id: str, +) -> tuple[ReconstructionInformationInputV1, ...]: + """Bind selected motifs into #433's point-in-time leakage audit graph.""" + if not isinstance(result, ReferenceMotifQueryResultV1): + raise ValueError("motif information binding requires a v1 result") + run = _required_text(run_id) + values: list[ReconstructionInformationInputV1] = [] + for match in result.matches: + fragment = match.fragment + lookahead = max(0, fragment.source_end_ns - result.query.used_at_ns) + if result.query.information_mode is InformationMode.EX_ANTE_SIMULATION: + lookahead = 0 + values.append( + ReconstructionInformationInputV1( + run_id=run, + artifact_id=f"{result.index_id}:{fragment.fragment_id}", + information_mode=result.query.information_mode, + input_kind=InformationInputKind.EXTERNAL, + stage=InformationStage.MOTIF_SELECTION, + scope=InformationScope.EMPIRICAL_MOTIF, + event_time_ns=fragment.source_start_ns, + available_at_ns=fragment.available_at_ns, + used_at_ns=result.query.used_at_ns, + observation_start_ns=fragment.source_start_ns, + observation_end_ns=fragment.source_end_ns, + vintage_id=fragment.fragment_id, + reason=( + "Empirical motif selected from " + f"{match.backoff_level} with distance {match.distance}." + ), + allowed_lookahead_ns=lookahead, + split_kind=InformationSplitKind.TRAIN, + ) + ) + return tuple(values) + + +def _validated_splits( + splits: Sequence[ReferenceMotifSplitV1], +) -> tuple[ReferenceMotifSplitV1, ...]: + values = tuple(splits) + if any(not isinstance(item, ReferenceMotifSplitV1) for item in values): + raise ValueError("reference motif splits must use the v1 contract") + if tuple(item.kind.value for item in values) != _EXPECTED_SPLITS: + raise ValueError( + "reference motif splits must be train, calibration, validation, " + "final_holdout" + ) + for previous, current in zip(values, values[1:]): + if previous.end_ns > current.start_ns: + raise ValueError("reference motif splits overlap or regress") + return values + + +def _cross_split_leakage_findings( + windows: Sequence[ReferenceMotifSourceWindowV1], + fragments: Sequence[ReferenceMotifFragmentV1], + *, + guard_ns: int, +) -> tuple[int, tuple[dict[str, JSONValue], ...]]: + findings: dict[str, dict[str, JSONValue]] = {} + comparisons = 0 + signatures: dict[str, list[ReferenceMotifFragmentV1]] = {} + for fragment in fragments: + signatures.setdefault(fragment.near_duplicate_signature, []).append( + fragment + ) + for signature, shape_group in signatures.items(): + kinds = {item.split_kind for item in shape_group} + if len(kinds) <= 1 or ReferenceMotifSplitKind.TRAIN not in kinds: + continue + comparisons += len(shape_group) - 1 + payload: dict[str, JSONValue] = { + "rule": "cross_split_near_duplicate_shape", + "near_duplicate_signature": signature, + "source_window_ids": _json_string_list( + sorted(item.source_window_id for item in shape_group) + ), + "split_kinds": _json_string_list( + sorted(item.split_kind.value for item in shape_group) + ), + } + findings[_canonical_json(payload)] = payload + + groups: dict[tuple[str, str], list[ReferenceMotifSourceWindowV1]] = {} + for window in windows: + groups.setdefault( + (window.source_artifact.sha256, window.condition.symbol), [] + ).append(window) + for (source_hash, symbol), source_group in groups.items(): + ordered = sorted( + source_group, + key=lambda item: ( + item.start_ns, + item.end_ns, + item.source_window_id, + ), + ) + active: list[ReferenceMotifSourceWindowV1] = [] + for current in ordered: + active = [ + item + for item in active + if item.end_ns + guard_ns >= current.start_ns + ] + for previous in active: + comparisons += 1 + if previous.split_kind is current.split_kind: + continue + if ReferenceMotifSplitKind.TRAIN not in { + previous.split_kind, + current.split_kind, + }: + continue + payload = { + "rule": "cross_split_source_window_overlap", + "source_artifact_sha256": source_hash, + "symbol": symbol, + "source_window_ids": _json_string_list( + sorted( + [ + previous.source_window_id, + current.source_window_id, + ] + ) + ), + "split_kinds": _json_string_list( + sorted( + [ + previous.split_kind.value, + current.split_kind.value, + ] + ) + ), + "source_overlap_guard_ns": guard_ns, + } + findings[_canonical_json(payload)] = payload + active.append(current) + return comparisons, tuple(findings[key] for key in sorted(findings)) + + +def _near_duplicate_signature( + *, + symbol: str, + offsets: Sequence[int], + bid_deltas: Sequence[float], + ask_deltas: Sequence[float], + transitions: Sequence[ReferenceMotifTransition], + digits: int, +) -> str: + duration = max(1, offsets[-1]) + price_scale = max( + 1e-15, + max(abs(value) for value in (*bid_deltas, *ask_deltas)), + ) + normalized: dict[str, JSONValue] = { + "symbol": symbol, + "event_count": len(offsets), + "relative_time": [round(value / duration, digits) for value in offsets], + "relative_bid": [ + round(value / price_scale, digits) for value in bid_deltas + ], + "relative_ask": [ + round(value / price_scale, digits) for value in ask_deltas + ], + "transitions": [item.value for item in transitions], + } + return _stable_id("reference-motif-shape", normalized) + + +def _transition( + previous: ReferenceMotifSourceEventV1, + current: ReferenceMotifSourceEventV1, +) -> ReferenceMotifTransition: + bid_changed = current.bid != previous.bid + ask_changed = current.ask != previous.ask + if bid_changed and ask_changed: + return ReferenceMotifTransition.BOTH + if bid_changed: + return ReferenceMotifTransition.BID + if ask_changed: + return ReferenceMotifTransition.ASK + return ReferenceMotifTransition.UNCHANGED + + +def _delta_transition( + previous_bid: float, + previous_ask: float, + current_bid: float, + current_ask: float, +) -> ReferenceMotifTransition: + bid_changed = current_bid != previous_bid + ask_changed = current_ask != previous_ask + if bid_changed and ask_changed: + return ReferenceMotifTransition.BOTH + if bid_changed: + return ReferenceMotifTransition.BID + if ask_changed: + return ReferenceMotifTransition.ASK + return ReferenceMotifTransition.UNCHANGED + + +def _condition_distance( + query: ReferenceMotifConditionV1, + candidate: ReferenceMotifConditionV1, + config: ReferenceMotifIndexConfigV1, +) -> float: + total = 0.0 + for name, weight in config.metric_weights.items(): + left = query.metrics.get(name) + right = candidate.metrics.get(name) + if left is None or right is None: + if left is not right: + total += weight * config.missing_metric_penalty + continue + total += weight * abs(left - right) / config.metric_scales[name] + left_coordinates = query.coordinates() + right_coordinates = candidate.coordinates() + mismatches = sum( + left_coordinates[name] != right_coordinates[name] + for name in left_coordinates + ) + total += mismatches * config.categorical_mismatch_penalty + return round(total, config.rounding_digits) + + +def _metric_mapping(value: Mapping[str, float]) -> dict[str, float]: + result: dict[str, float] = {} + for raw_name, raw_value in sorted(value.items()): + name = str(raw_name) + if name not in REFERENCE_MOTIF_METRIC_NAMES: + raise ValueError(f"unsupported reference motif metric: {name}") + metric = _finite_float(raw_value, name) + if name in _NONNEGATIVE_METRICS and metric < 0: + raise ValueError( + f"reference motif metric {name} must be non-negative" + ) + result[name] = metric + return result + + +def _signed_regime(value: float, small: float, large: float) -> str: + magnitude = abs(value) + if magnitude < small: + return "flat" + direction = "up" if value > 0.0 else "down" + return f"{direction}_{'small' if magnitude < large else 'large'}" + + +def _three_level_regime(value: float, low: float, high: float) -> str: + if value < low: + return "low" + return "medium" if value < high else "high" + + +def _integer_gcd(values: Sequence[int]) -> int: + result = 0 + for value in values: + result = math.gcd(result, int(value)) + if result == 1: + break + return result + + +def _decimal_precision(value: float) -> int: + rendered = f"{value:.12f}".rstrip("0").rstrip(".") + return len(rendered.rsplit(".", 1)[1]) if "." in rendered else 0 + + +def _named_positive_mapping( + value: Mapping[str, float], name: str +) -> dict[str, float]: + keys = set(value) + if keys != set(REFERENCE_MOTIF_METRIC_NAMES): + raise ValueError(f"{name} must cover every reference motif metric") + return { + metric: _positive_float(value[metric], f"{name}.{metric}") + for metric in REFERENCE_MOTIF_METRIC_NAMES + } + + +def _named_nonnegative_mapping( + value: Mapping[str, float], name: str +) -> dict[str, float]: + keys = set(value) + if keys != set(REFERENCE_MOTIF_METRIC_NAMES): + raise ValueError(f"{name} must cover every reference motif metric") + return { + metric: _nonnegative_float(value[metric], f"{name}.{metric}") + for metric in REFERENCE_MOTIF_METRIC_NAMES + } + + +def _validated_artifact_ref(value: ArtifactRef) -> ArtifactRef: + if not isinstance(value, ArtifactRef): + raise ValueError("reference motif artifact must be an ArtifactRef") + kind = _required_text(value.kind) + path = _required_text(value.path) + digest = _required_sha256(value.sha256, "artifact sha256") + size = value.size_bytes + if size is not None and (_strict_int(size, "artifact size_bytes") < 0): + raise ValueError("reference motif artifact size must be non-negative") + metadata = dict(value.metadata) + _validate_json_value(metadata, "artifact.metadata") + return ArtifactRef( + kind=kind, + path=path, + size_bytes=size, + sha256=digest, + metadata=metadata, + ) + + +def _unique_artifact_refs( + values: Sequence[ArtifactRef], +) -> tuple[ArtifactRef, ...]: + by_identity: dict[tuple[str, str, str], ArtifactRef] = {} + for raw in values: + value = _validated_artifact_ref(raw) + key = (value.kind, value.path, value.sha256) + by_identity[key] = value + return tuple(by_identity[key] for key in sorted(by_identity)) + + +def _split_count_mapping(value: Mapping[str, int]) -> dict[str, int]: + allowed = {item.value for item in ReferenceMotifSplitKind} - { + ReferenceMotifSplitKind.TRAIN.value + } + result: dict[str, int] = {} + for name, count in sorted(value.items()): + if name not in allowed: + raise ValueError("unsupported excluded reference motif split") + normalized = _nonnegative_int(count, f"excluded_split_counts.{name}") + if normalized: + result[name] = normalized + return result + + +def _normalized_symbol(value: Any) -> str: + symbol = _required_text(value).upper() + if not re.fullmatch(r"[A-Z0-9]{3,16}", symbol): + raise ValueError("invalid reference motif symbol") + return symbol + + +def _symbol_currencies(symbol: str) -> tuple[str, ...]: + if len(symbol) == 6 and symbol.isalpha(): + return (symbol[:3], symbol[3:]) + return () + + +def _normalized_currencies(values: Iterable[Any]) -> tuple[str, ...]: + result = tuple(sorted({_required_text(item).upper() for item in values})) + if not result or len(result) > MAX_REFERENCE_MOTIF_TAGS: + raise ValueError("reference motif currencies are outside limits") + if any(not re.fullmatch(r"[A-Z]{3}", item) for item in result): + raise ValueError("reference motif currencies require ISO-like codes") + return result + + +def _normalized_text_tuple( + values: Iterable[Any], *, maximum: int +) -> tuple[str, ...]: + result = tuple(sorted({_required_text(item) for item in values})) + if len(result) > maximum: + raise ValueError("reference motif text tuple exceeds limit") + return result + + +def _required_text(value: Any) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("reference motif text is required") + if len(text) > MAX_REFERENCE_MOTIF_TEXT: + raise ValueError("reference motif text exceeds limit") + return text + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return _required_text(text) if text else None + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a bool") + return value + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _bounded_int64(value: Any, name: str) -> int: + result = _strict_int(value, name) + if not INT64_MIN <= result <= INT64_MAX: + raise ValueError(f"{name} exceeds int64") + return result + + +def _nonnegative_int64(value: Any, name: str) -> int: + result = _bounded_int64(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _nonnegative_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _positive_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + result = _strict_int(value, name) + if not minimum <= result <= maximum: + raise ValueError(f"{name} is outside configured bounds") + return result + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be numeric") + try: + result = float(value) + except (TypeError, ValueError) as err: + raise ValueError(f"{name} must be numeric") from err + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _positive_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _nonnegative_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _required_sha256(value: Any, name: str) -> str: + text = str(value or "").strip().lower() + if not _SHA256_RE.fullmatch(text): + raise ValueError(f"{name} must be a lowercase sha256") + return text + + +def _required_sha256_id(value: Any, name: str, prefix: str) -> str: + text = _required_text(value) + marker = f"{prefix}:sha256:" + if not text.startswith(marker) or not _SHA256_RE.fullmatch( + text[len(marker) :] + ): + raise ValueError(f"{name} must be a {prefix} sha256 ID") + return text + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + _canonical_json(payload).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _canonical_json(value: Any) -> str: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError("unsupported reference motif schema version") + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("reference motif value must be a mapping") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError("reference motif value must be a sequence") + return value + + +def _mapping_sequence(value: Any) -> tuple[Mapping[str, Any], ...]: + return tuple(_mapping(item) for item in _sequence(value)) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _json_string_list(values: Iterable[str]) -> list[JSONValue]: + return [value for value in values] + + +def _int_tuple(value: Any) -> tuple[int, ...]: + return tuple( + _strict_int(item, "sequence integer") for item in _sequence(value) + ) + + +def _float_tuple(value: Any) -> tuple[float, ...]: + return tuple( + _finite_float(item, "sequence float") for item in _sequence(value) + ) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + try: + return _mapping(json.loads(text)) + except json.JSONDecodeError as err: + raise ValueError("invalid reference motif JSON") from err + + +def _validate_json_value(value: Any, path: str) -> None: + if value is None or isinstance(value, (str, bool, int)): + return + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError(f"{path} contains a non-finite float") + return + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"{path} contains a non-string key") + _validate_json_value(item, f"{path}.{key}") + return + if isinstance(value, Sequence) and not isinstance( + value, (str, bytes, bytearray) + ): + for index, item in enumerate(value): + _validate_json_value(item, f"{path}[{index}]") + return + raise ValueError(f"{path} contains a non-JSON value") + + +__all__ = [ + "MAX_REFERENCE_MOTIF_ARTIFACT_BYTES", + "MAX_REFERENCE_MOTIF_EVENTS", + "MAX_REFERENCE_MOTIF_FRAGMENTS", + "MAX_REFERENCE_MOTIF_MATCHES", + "MAX_REFERENCE_MOTIF_SOURCE_WINDOWS", + "REFERENCE_MOTIF_ARTIFACT_KIND", + "REFERENCE_MOTIF_BACKOFF_ATTEMPT_SCHEMA_VERSION", + "REFERENCE_MOTIF_BACKOFF_LEVELS", + "REFERENCE_MOTIF_CONDITION_SCHEMA_VERSION", + "REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION", + "REFERENCE_MOTIF_FEATURE_SCHEMA_VERSION", + "REFERENCE_MOTIF_INDEX_CONFIG_SCHEMA_VERSION", + "REFERENCE_MOTIF_INDEX_SCHEMA_VERSION", + "REFERENCE_MOTIF_MATCH_SCHEMA_VERSION", + "REFERENCE_MOTIF_METRIC_NAMES", + "REFERENCE_MOTIF_QUERY_RESULT_SCHEMA_VERSION", + "REFERENCE_MOTIF_QUERY_SCHEMA_VERSION", + "REFERENCE_MOTIF_SOURCE_EVENT_SCHEMA_VERSION", + "REFERENCE_MOTIF_SOURCE_WINDOW_SCHEMA_VERSION", + "REFERENCE_MOTIF_SPLIT_SCHEMA_VERSION", + "REFERENCE_MOTIF_TRANSFORM_SCHEMA_VERSION", + "ReferenceMotifBackoffAttemptV1", + "ReferenceMotifConditionV1", + "ReferenceMotifFragmentV1", + "ReferenceMotifIndexConfigV1", + "ReferenceMotifIndexV1", + "ReferenceMotifLeakageError", + "ReferenceMotifMatchV1", + "ReferenceMotifQueryResultV1", + "ReferenceMotifQueryStatus", + "ReferenceMotifQueryV1", + "ReferenceMotifResourceLimitError", + "ReferenceMotifSourceEventV1", + "ReferenceMotifSourceWindowV1", + "ReferenceMotifSplitKind", + "ReferenceMotifSplitV1", + "ReferenceMotifTransformPolicyV1", + "ReferenceMotifTransition", + "build_reference_motif_index", + "extract_reference_motif_fragment", + "query_reference_motifs", + "read_reference_motif_index", + "reference_motif_information_inputs", + "reference_motif_condition_from_quotes", + "reference_motif_source_window_from_training_frame", + "write_reference_motif_index", +] diff --git a/src/histdatacom/synthetic/observation.py b/src/histdatacom/synthetic/observation.py new file mode 100644 index 00000000..c7f464be --- /dev/null +++ b/src/histdatacom/synthetic/observation.py @@ -0,0 +1,3196 @@ +"""Versioned historical feed-observation operators. + +This module models delivery technology separately from market-event +generation. A fitted operator consumes bounded, provenance-bearing evidence +and renders a market-event surface into delivery observations without +mutating the source event identities. + +Version-one contracts are strict and deterministic. Semantic changes to +parameter meaning, fallback order, event identity, or application behavior +require a new schema version. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections import Counter, defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from pathlib import Path +from statistics import median +from typing import TYPE_CHECKING, Any, cast + +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.contracts import ( + SyntheticEventOrigin, + SyntheticEventV1, + canonical_contract_json, +) +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.streaming import ReconstructionWindowV1 + +if TYPE_CHECKING: + from histdatacom.data_analytics.feed_epochs import ( + FeedEpochDefinitionV1, + FeedEpochEvidenceV1, + ) + from histdatacom.data_analytics.feed_epochs_v2 import ( + FeedEpochDefinitionV2, + FeedEpochEvidenceV2, + ) + +OBSERVATION_CONTEXT_SCHEMA_VERSION = "histdatacom.observation-context.v1" +OBSERVATION_FIT_EVIDENCE_SCHEMA_VERSION = ( + "histdatacom.observation-fit-evidence.v1" +) +OBSERVATION_PARAMETER_SCHEMA_VERSION = ( + "histdatacom.observation-parameter-estimate.v1" +) +OBSERVATION_FIT_CONFIG_SCHEMA_VERSION = ( + "histdatacom.observation-operator-fit-config.v1" +) +OBSERVATION_STRATUM_SCHEMA_VERSION = "histdatacom.observation-stratum.v1" +OBSERVATION_FIT_DIAGNOSTICS_SCHEMA_VERSION = ( + "histdatacom.observation-fit-diagnostics.v1" +) +OBSERVATION_OPERATOR_SCHEMA_VERSION = "histdatacom.observation-operator.v1" +OBSERVATION_INPUT_EVENT_SCHEMA_VERSION = ( + "histdatacom.observation-input-event.v1" +) +OBSERVATION_OUTPUT_EVENT_SCHEMA_VERSION = ( + "histdatacom.observation-output-event.v1" +) +OBSERVATION_CARRY_STATE_SCHEMA_VERSION = ( + "histdatacom.observation-carry-state.v1" +) +OBSERVATION_APPLICATION_RESULT_SCHEMA_VERSION = ( + "histdatacom.observation-application-result.v1" +) + +MAX_OBSERVATION_EVIDENCE = 4096 +MAX_OBSERVATION_STRATA = 512 +MAX_OBSERVATION_PARAMETERS = 32 +MAX_OBSERVATION_INPUT_EVENTS = 250_000 +MAX_OBSERVATION_OUTPUTS_PER_INPUT = 3 +MAX_OBSERVATION_DIAGNOSTIC_SAMPLES = 128 +MAX_OBSERVATION_PROVENANCE_PATHS = 64 +MAX_OBSERVATION_TEXT_LENGTH = 1024 +MAX_OBSERVATION_ARTIFACT_BYTES = 64 * 1024 * 1024 +MAX_OBSERVATION_WINDOW_ALIGNMENT_NS = 86_400_000_000_000 +MAX_OBSERVATION_DURATION_NS = 31 * 86_400_000_000_000 +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 + +OBSERVATION_PARAMETER_NAMES = ( + "retention_probability", + "unchanged_retention_probability", + "timestamp_quantum_ns", + "price_precision_digits", + "quote_transition_threshold", + "batch_window_ns", + "duplicate_probability", + "rate_cap_per_second", + "burst_window_ns", + "quiet_gap_probability", + "outage_window_ns", + "reconnect_duplicate_probability", +) + +OBSERVATION_BACKOFF_LEVELS = ( + "symbol_epoch_state_session_event", + "symbol_epoch_state_session", + "symbol_epoch_state", + "symbol_epoch", + "epoch", + "global", +) + +OBSERVATION_CARRY_FIELDS = ( + "last_source_time_ns", + "last_observed_time_ns", + "last_bid", + "last_ask", + "rate_bucket_start_ns", + "rate_bucket_count", + "outage_bucket_start_ns", + "outage_active", + "reconnect_pending", +) + +_PROBABILITY_PARAMETERS = { + "retention_probability", + "unchanged_retention_probability", + "duplicate_probability", + "quiet_gap_probability", + "reconnect_duplicate_probability", +} +_INTEGER_PARAMETERS = { + "timestamp_quantum_ns", + "price_precision_digits", + "batch_window_ns", + "burst_window_ns", + "outage_window_ns", +} +_NONNEGATIVE_PARAMETERS = set(OBSERVATION_PARAMETER_NAMES).difference( + _PROBABILITY_PARAMETERS +) +_NEUTRAL_PARAMETER_VALUES: dict[str, float] = { + "retention_probability": 1.0, + "unchanged_retention_probability": 1.0, + "timestamp_quantum_ns": 1.0, + "price_precision_digits": 16.0, + "quote_transition_threshold": 0.0, + "batch_window_ns": 0.0, + "duplicate_probability": 0.0, + "rate_cap_per_second": 0.0, + "burst_window_ns": 0.0, + "quiet_gap_probability": 0.0, + "outage_window_ns": 0.0, + "reconnect_duplicate_probability": 0.0, +} +_EVIDENCE_KINDS = { + "active_time_feed_epoch", + "canonical_fingerprint", + "paired_calibration", + "controlled_fixture", +} +_SOURCE_HASH_BASES = { + "cache_content_sha256", + "canonical_fingerprint_id", + "canonical_fingerprint_aggregate_id", + "persisted_fingerprint_artifact_sha256", + "paired_calibration_artifact_sha256", + "controlled_fixture_sha256", +} +_STRATUM_STATUSES = {"ready", "limited", "unsupported"} +_APPLICATION_REASONS = { + "retained", + "outage", + "thinning", + "unchanged_quote_filter", + "quote_transition_filter", + "rate_cap", +} +_OUTPUT_TRANSFORMATIONS = { + "timestamp_quantized", + "batched", + "price_quantized", + "reconnect_duplicate", + "duplicated", +} +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_PERIOD_RE = re.compile(r"^\d{4}(?:\d{2})?$") + + +@dataclass(frozen=True, slots=True) +class ObservationContextV1: + """Conditioning coordinates used by the explicit fallback hierarchy.""" + + symbol: str + epoch_id: str + state: str | None = None + session: str | None = None + event_tag: str | None = None + schema_version: str = OBSERVATION_CONTEXT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_CONTEXT_SCHEMA_VERSION: + raise ValueError("unsupported observation context schema") + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__(self, "epoch_id", _required_text(self.epoch_id)) + for name in ("state", "session", "event_tag"): + object.__setattr__( + self, + name, + _optional_context_value(getattr(self, name)), + ) + + def pattern_for_level(self, level: str) -> dict[str, str] | None: + """Return the concrete pattern for a supported fallback level.""" + if level not in OBSERVATION_BACKOFF_LEVELS: + raise ValueError("unsupported observation backoff level") + if level == "global": + return {} + if level == "epoch": + return {"epoch_id": self.epoch_id} + base = {"symbol": self.symbol, "epoch_id": self.epoch_id} + if level == "symbol_epoch": + return base + if self.state is None: + return None + base["state"] = self.state + if level == "symbol_epoch_state": + return base + if self.session is None: + return None + base["session"] = self.session + if level == "symbol_epoch_state_session": + return base + if self.event_tag is None: + return None + base["event_tag"] = self.event_tag + return base + + def key_for_level(self, level: str) -> str | None: + """Return a deterministic key for one fallback level.""" + pattern = self.pattern_for_level(level) + if pattern is None: + return None + return _stratum_key(level, pattern) + + def candidate_keys( + self, levels: Sequence[str] = OBSERVATION_BACKOFF_LEVELS + ) -> tuple[str, ...]: + """Return ordered, de-duplicated resolution candidates.""" + result: list[str] = [] + for level in levels: + key = self.key_for_level(level) + if key is not None and key not in result: + result.append(key) + return tuple(result) + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible context coordinates.""" + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "epoch_id": self.epoch_id, + "state": self.state, + "session": self.session, + "event_tag": self.event_tag, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ObservationContextV1": + """Read a strict version-one context.""" + return cls( + schema_version=str(data.get("schema_version", "")), + symbol=str(data.get("symbol", "")), + epoch_id=str(data.get("epoch_id", "")), + state=_mapping_optional_text(data, "state"), + session=_mapping_optional_text(data, "session"), + event_tag=_mapping_optional_text(data, "event_tag"), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationFitEvidenceV1: + """One bounded parameter projection used to fit an operator.""" + + context: ObservationContextV1 + period: str + start_timestamp_ns: int + end_timestamp_ns: int + source_evidence_id: str + source_artifact_sha256: str + source_hash_basis: str + evidence_kind: str + parameter_values: Mapping[str, float] + parameter_lower_bounds: Mapping[str, float] + parameter_upper_bounds: Mapping[str, float] + parameter_support_counts: Mapping[str, int] + parameter_basis: Mapping[str, str] + parameter_provenance: Mapping[str, tuple[str, ...]] + evidence_id: str = "" + schema_version: str = OBSERVATION_FIT_EVIDENCE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_FIT_EVIDENCE_SCHEMA_VERSION: + raise ValueError("unsupported observation fit evidence schema") + if not isinstance(self.context, ObservationContextV1): + raise ValueError("observation evidence requires a v1 context") + period = _required_text(self.period) + if not _valid_period(period): + raise ValueError( + "observation evidence period must use YYYY or YYYYMM" + ) + start = _bounded_int( + self.start_timestamp_ns, + "start_timestamp_ns", + INT64_MIN, + INT64_MAX, + ) + end = _bounded_int( + self.end_timestamp_ns, + "end_timestamp_ns", + INT64_MIN, + INT64_MAX, + ) + if end < start: + raise ValueError("observation evidence end precedes start") + source_id = _required_text(self.source_evidence_id) + source_hash = _required_sha256_id( + self.source_artifact_sha256, + "source_artifact_sha256", + ) + source_hash_basis = _required_text(self.source_hash_basis) + if source_hash_basis not in _SOURCE_HASH_BASES: + raise ValueError("unsupported observation source hash basis") + evidence_kind = _required_text(self.evidence_kind) + if evidence_kind not in _EVIDENCE_KINDS: + raise ValueError("unsupported observation evidence kind") + names = tuple(sorted(self.parameter_values)) + if not names or len(names) > MAX_OBSERVATION_PARAMETERS: + raise ValueError("observation evidence parameters are unbounded") + if any(name not in OBSERVATION_PARAMETER_NAMES for name in names): + raise ValueError("unsupported observation parameter") + if any( + set(mapping) != set(names) + for mapping in ( + self.parameter_lower_bounds, + self.parameter_upper_bounds, + self.parameter_support_counts, + self.parameter_basis, + self.parameter_provenance, + ) + ): + raise ValueError("observation parameter evidence keys differ") + values: dict[str, float] = {} + lowers: dict[str, float] = {} + uppers: dict[str, float] = {} + supports: dict[str, int] = {} + bases: dict[str, str] = {} + provenance: dict[str, tuple[str, ...]] = {} + for name in names: + value = _parameter_value(name, self.parameter_values[name]) + lower = _parameter_value(name, self.parameter_lower_bounds[name]) + upper = _parameter_value(name, self.parameter_upper_bounds[name]) + if not lower <= value <= upper: + raise ValueError("parameter estimate lies outside uncertainty") + support = _bounded_int( + self.parameter_support_counts[name], + f"{name} support", + 0, + MAX_OBSERVATION_INPUT_EVENTS, + ) + basis = _required_text(self.parameter_basis[name]) + paths = tuple( + dict.fromkeys( + _required_text(path) + for path in self.parameter_provenance[name] + ) + ) + if not paths or len(paths) > MAX_OBSERVATION_PROVENANCE_PATHS: + raise ValueError("parameter provenance is empty or unbounded") + values[name] = value + lowers[name] = lower + uppers[name] = upper + supports[name] = support + bases[name] = basis + provenance[name] = paths + object.__setattr__(self, "period", period) + object.__setattr__(self, "start_timestamp_ns", start) + object.__setattr__(self, "end_timestamp_ns", end) + object.__setattr__(self, "source_evidence_id", source_id) + object.__setattr__(self, "source_artifact_sha256", source_hash) + object.__setattr__(self, "source_hash_basis", source_hash_basis) + object.__setattr__(self, "evidence_kind", evidence_kind) + object.__setattr__(self, "parameter_values", values) + object.__setattr__(self, "parameter_lower_bounds", lowers) + object.__setattr__(self, "parameter_upper_bounds", uppers) + object.__setattr__(self, "parameter_support_counts", supports) + object.__setattr__(self, "parameter_basis", bases) + object.__setattr__(self, "parameter_provenance", provenance) + expected = _stable_id("observation-evidence", self.identity_payload()) + supplied = str(self.evidence_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "evidence_id does not match deterministic identity" + ) + object.__setattr__(self, "evidence_id", expected) + + @property + def support_count(self) -> int: + """Return the largest declared parameter support.""" + return max(self.parameter_support_counts.values(), default=0) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return complete semantic evidence identity.""" + return { + "schema_version": self.schema_version, + "context": self.context.to_dict(), + "period": self.period, + "start_timestamp_ns": self.start_timestamp_ns, + "end_timestamp_ns": self.end_timestamp_ns, + "source_evidence_id": self.source_evidence_id, + "source_artifact_sha256": self.source_artifact_sha256, + "source_hash_basis": self.source_hash_basis, + "evidence_kind": self.evidence_kind, + "parameter_values": dict(self.parameter_values), + "parameter_lower_bounds": dict(self.parameter_lower_bounds), + "parameter_upper_bounds": dict(self.parameter_upper_bounds), + "parameter_support_counts": dict(self.parameter_support_counts), + "parameter_basis": dict(self.parameter_basis), + "parameter_provenance": { + name: list(paths) + for name, paths in sorted(self.parameter_provenance.items()) + }, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible evidence.""" + return {**self.identity_payload(), "evidence_id": self.evidence_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ObservationFitEvidenceV1": + """Read and verify fit evidence.""" + return cls( + schema_version=str(data.get("schema_version", "")), + context=ObservationContextV1.from_dict( + _mapping(data.get("context")) + ), + period=str(data.get("period", "")), + start_timestamp_ns=_strict_int( + data.get("start_timestamp_ns"), "start_timestamp_ns" + ), + end_timestamp_ns=_strict_int( + data.get("end_timestamp_ns"), "end_timestamp_ns" + ), + source_evidence_id=str(data.get("source_evidence_id", "")), + source_artifact_sha256=str(data.get("source_artifact_sha256", "")), + source_hash_basis=str(data.get("source_hash_basis", "")), + evidence_kind=str(data.get("evidence_kind", "")), + parameter_values=_float_mapping(data.get("parameter_values")), + parameter_lower_bounds=_float_mapping( + data.get("parameter_lower_bounds") + ), + parameter_upper_bounds=_float_mapping( + data.get("parameter_upper_bounds") + ), + parameter_support_counts=_int_mapping( + data.get("parameter_support_counts") + ), + parameter_basis=_string_mapping(data.get("parameter_basis")), + parameter_provenance={ + str(name): tuple(str(path) for path in _sequence(paths)) + for name, paths in _mapping( + data.get("parameter_provenance") + ).items() + }, + evidence_id=str(data.get("evidence_id", "")), + ) + + @classmethod + def from_feed_epoch_evidence( + cls, + evidence: FeedEpochEvidenceV1 | FeedEpochEvidenceV2, + epoch_definition: FeedEpochDefinitionV1 | FeedEpochDefinitionV2, + ) -> "ObservationFitEvidenceV1": + """Project canonical epoch evidence without claiming dense recovery.""" + if not epoch_definition.valid_for_observation_models: + raise ValueError("feed epoch definition has not passed stability") + midpoint_ms = ( + evidence.start_timestamp_utc_ms + evidence.end_timestamp_utc_ms + ) // 2 + assignment = epoch_definition.assign( + symbol=evidence.symbol, + timestamp_utc_ms=midpoint_ms, + ) + if assignment.assignment_kind == "out_of_scope": + raise ValueError("feed epoch evidence is outside definition scope") + features = evidence.feature_values + profile = evidence.profile + row_count = max(1, _optional_int(profile.get("row_count")) or 1) + min_interval_ms = max( + 0.000001, + features.get("minimum_observed_interval_ms", 0.001), + ) + median_interval_ms = max( + min_interval_ms, + _optional_float(profile.get("median_interarrival_ms")) + or min_interval_ms, + ) + p95_interval_ms = max( + median_interval_ms, + _optional_float(profile.get("p95_interarrival_ms")) + or median_interval_ms, + ) + precision = int(round(features.get("price_precision_digits", 8.0))) + precision = min(16, max(0, precision)) + stale_rate = _probability( + features.get( + "stale_repeat_rate", features.get("stale_quote_rate", 0.0) + ) + ) + duplicate_rate = _probability( + features.get("duplicate_timestamp_rate", 0.0) + ) + gap_rate = _probability(features.get("suspicious_gap_rate", 0.0)) + tick_rate = ( + max( + 0.0, + _optional_float(profile.get("tick_rate_per_hour")) or 0.0, + ) + / 3600.0 + ) + parameter_values = { + "retention_probability": 1.0, + "unchanged_retention_probability": stale_rate, + "timestamp_quantum_ns": float( + int(round(min_interval_ms * 1_000_000)) + ), + "price_precision_digits": float(precision), + "quote_transition_threshold": 10.0 ** (-precision), + "batch_window_ns": float(int(round(min_interval_ms * 1_000_000))), + "duplicate_probability": duplicate_rate, + "rate_cap_per_second": max( + tick_rate, + 1_000.0 / min_interval_ms, + ), + "burst_window_ns": float( + int(round(median_interval_ms * 1_000_000)) + ), + "quiet_gap_probability": gap_rate, + "outage_window_ns": float(int(round(p95_interval_ms * 1_000_000))), + "reconnect_duplicate_probability": duplicate_rate, + } + parameter_values["timestamp_quantum_ns"] = max( + 1.0, parameter_values["timestamp_quantum_ns"] + ) + bounds = _canonical_proxy_bounds(parameter_values) + support = min(row_count, MAX_OBSERVATION_INPUT_EVENTS) + support_counts = {name: support for name in parameter_values} + support_counts["retention_probability"] = 0 + bases = { + name: "canonical_descriptive_proxy" for name in parameter_values + } + bases["retention_probability"] = "identity_without_dense_denominator" + provenance = { + name: _canonical_parameter_provenance(name) + for name in parameter_values + } + return cls( + context=ObservationContextV1( + symbol=evidence.symbol, + epoch_id=assignment.label, + state=assignment.assignment_kind, + ), + period=evidence.period, + start_timestamp_ns=evidence.start_timestamp_utc_ms * 1_000_000, + end_timestamp_ns=evidence.end_timestamp_utc_ms * 1_000_000, + source_evidence_id=evidence.evidence_id, + source_artifact_sha256=evidence.source_artifact_sha256, + source_hash_basis=evidence.source_hash_basis, + evidence_kind=( + "active_time_feed_epoch" + if evidence.schema_version + == "histdatacom.feed-epoch-evidence.v2" + else "canonical_fingerprint" + ), + parameter_values=parameter_values, + parameter_lower_bounds={ + name: pair[0] for name, pair in bounds.items() + }, + parameter_upper_bounds={ + name: pair[1] for name, pair in bounds.items() + }, + parameter_support_counts=support_counts, + parameter_basis=bases, + parameter_provenance=provenance, + ) + + +@dataclass(frozen=True, slots=True) +class ObservationParameterEstimateV1: + """One fitted value with uncertainty, support, and provenance.""" + + name: str + value: float + lower: float + upper: float + support_count: int + evidence_count: int + support_status: str + estimation_bases: tuple[str, ...] + evidence_ids: tuple[str, ...] + provenance: tuple[str, ...] + schema_version: str = OBSERVATION_PARAMETER_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_PARAMETER_SCHEMA_VERSION: + raise ValueError("unsupported observation parameter schema") + name = _required_text(self.name) + if name not in OBSERVATION_PARAMETER_NAMES: + raise ValueError("unsupported observation parameter") + value = _parameter_value(name, self.value) + lower = _parameter_value(name, self.lower) + upper = _parameter_value(name, self.upper) + if not lower <= value <= upper: + raise ValueError("fitted parameter lies outside uncertainty") + support = _bounded_int( + self.support_count, + "support_count", + 0, + MAX_OBSERVATION_INPUT_EVENTS * MAX_OBSERVATION_EVIDENCE, + ) + evidence_count = _bounded_int( + self.evidence_count, + "evidence_count", + 1, + MAX_OBSERVATION_EVIDENCE, + ) + status = _required_text(self.support_status) + if status not in {"supported", "unsupported"}: + raise ValueError("unsupported parameter support status") + bases = _bounded_text_tuple( + self.estimation_bases, + "estimation basis", + limit=MAX_OBSERVATION_PROVENANCE_PATHS, + ) + evidence_ids = _bounded_text_tuple(self.evidence_ids, "evidence id") + if len(evidence_ids) != evidence_count: + raise ValueError("parameter evidence count differs from IDs") + if any( + _required_sha256_id( + evidence_id, + "evidence_id", + prefix="observation-evidence", + ) + != evidence_id + for evidence_id in evidence_ids + ): + raise ValueError("parameter evidence IDs differ") + if status == "supported" and support == 0: + raise ValueError("supported parameter requires positive support") + provenance = _bounded_text_tuple( + self.provenance, + "provenance", + limit=MAX_OBSERVATION_PROVENANCE_PATHS, + ) + object.__setattr__(self, "name", name) + object.__setattr__(self, "value", value) + object.__setattr__(self, "lower", lower) + object.__setattr__(self, "upper", upper) + object.__setattr__(self, "support_count", support) + object.__setattr__(self, "evidence_count", evidence_count) + object.__setattr__(self, "support_status", status) + object.__setattr__(self, "estimation_bases", bases) + object.__setattr__(self, "evidence_ids", evidence_ids) + object.__setattr__(self, "provenance", provenance) + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible parameter evidence.""" + return { + "schema_version": self.schema_version, + "name": self.name, + "value": self.value, + "lower": self.lower, + "upper": self.upper, + "support_count": self.support_count, + "evidence_count": self.evidence_count, + "support_status": self.support_status, + "estimation_bases": list(self.estimation_bases), + "evidence_ids": list(self.evidence_ids), + "provenance": list(self.provenance), + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ObservationParameterEstimateV1": + """Read one fitted parameter.""" + return cls( + schema_version=str(data.get("schema_version", "")), + name=str(data.get("name", "")), + value=_finite_float(data.get("value"), "value"), + lower=_finite_float(data.get("lower"), "lower"), + upper=_finite_float(data.get("upper"), "upper"), + support_count=_strict_int( + data.get("support_count"), "support_count" + ), + evidence_count=_strict_int( + data.get("evidence_count"), "evidence_count" + ), + support_status=str(data.get("support_status", "")), + estimation_bases=_string_tuple(data.get("estimation_bases")), + evidence_ids=_string_tuple(data.get("evidence_ids")), + provenance=_string_tuple(data.get("provenance")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationOperatorFitConfigV1: + """Bounded fitting, fallback, and application policy.""" + + min_stratum_support: int = 16 + min_parameter_support: int = 1 + min_supported_parameters: int = 5 + max_evidence: int = MAX_OBSERVATION_EVIDENCE + max_strata: int = MAX_OBSERVATION_STRATA + max_input_events: int = MAX_OBSERVATION_INPUT_EVENTS + max_outputs_per_input: int = MAX_OBSERVATION_OUTPUTS_PER_INPUT + diagnostic_sample_limit: int = 32 + required_left_halo_ns: int = 0 + backoff_levels: tuple[str, ...] = OBSERVATION_BACKOFF_LEVELS + rounding_digits: int = 8 + config_id: str = "" + schema_version: str = OBSERVATION_FIT_CONFIG_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_FIT_CONFIG_SCHEMA_VERSION: + raise ValueError("unsupported observation fit config schema") + _bounded_int( + self.min_stratum_support, + "min_stratum_support", + 1, + MAX_OBSERVATION_INPUT_EVENTS, + ) + _bounded_int( + self.min_parameter_support, + "min_parameter_support", + 1, + MAX_OBSERVATION_INPUT_EVENTS, + ) + _bounded_int( + self.min_supported_parameters, + "min_supported_parameters", + 1, + len(OBSERVATION_PARAMETER_NAMES), + ) + _bounded_int( + self.max_evidence, "max_evidence", 1, MAX_OBSERVATION_EVIDENCE + ) + _bounded_int(self.max_strata, "max_strata", 1, MAX_OBSERVATION_STRATA) + _bounded_int( + self.max_input_events, + "max_input_events", + 1, + MAX_OBSERVATION_INPUT_EVENTS, + ) + _bounded_int( + self.max_outputs_per_input, + "max_outputs_per_input", + 1, + MAX_OBSERVATION_OUTPUTS_PER_INPUT, + ) + _bounded_int( + self.diagnostic_sample_limit, + "diagnostic_sample_limit", + 1, + MAX_OBSERVATION_DIAGNOSTIC_SAMPLES, + ) + _bounded_int( + self.required_left_halo_ns, + "required_left_halo_ns", + 0, + MAX_OBSERVATION_WINDOW_ALIGNMENT_NS, + ) + levels = tuple(dict.fromkeys(self.backoff_levels)) + if ( + not levels + or levels[-1] != "global" + or any(level not in OBSERVATION_BACKOFF_LEVELS for level in levels) + ): + raise ValueError("observation backoff levels must end in global") + if not 0 <= self.rounding_digits <= 16: + raise ValueError("rounding_digits must be between zero and sixteen") + object.__setattr__(self, "backoff_levels", levels) + expected = _stable_id("observation-fit-config", self.identity_payload()) + supplied = str(self.config_id or "").strip() + if supplied and supplied != expected: + raise ValueError("config_id does not match deterministic identity") + object.__setattr__(self, "config_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic fit/application configuration.""" + return { + "schema_version": self.schema_version, + "min_stratum_support": self.min_stratum_support, + "min_parameter_support": self.min_parameter_support, + "min_supported_parameters": self.min_supported_parameters, + "max_evidence": self.max_evidence, + "max_strata": self.max_strata, + "max_input_events": self.max_input_events, + "max_outputs_per_input": self.max_outputs_per_input, + "diagnostic_sample_limit": self.diagnostic_sample_limit, + "required_left_halo_ns": self.required_left_halo_ns, + "backoff_levels": list(self.backoff_levels), + "rounding_digits": self.rounding_digits, + "unsupported_parameter_policy": "explicit_neutral_identity", + "unsupported_stratum_policy": "backoff_then_fail", + "protected_anchor_policy": "preserve_exact", + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible fit config.""" + return {**self.identity_payload(), "config_id": self.config_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ObservationOperatorFitConfigV1": + """Read and verify the fit config.""" + return cls( + schema_version=str(data.get("schema_version", "")), + min_stratum_support=_strict_int( + data.get("min_stratum_support"), "min_stratum_support" + ), + min_parameter_support=_strict_int( + data.get("min_parameter_support"), "min_parameter_support" + ), + min_supported_parameters=_strict_int( + data.get("min_supported_parameters"), + "min_supported_parameters", + ), + max_evidence=_strict_int(data.get("max_evidence"), "max_evidence"), + max_strata=_strict_int(data.get("max_strata"), "max_strata"), + max_input_events=_strict_int( + data.get("max_input_events"), "max_input_events" + ), + max_outputs_per_input=_strict_int( + data.get("max_outputs_per_input"), "max_outputs_per_input" + ), + diagnostic_sample_limit=_strict_int( + data.get("diagnostic_sample_limit"), + "diagnostic_sample_limit", + ), + required_left_halo_ns=_strict_int( + data.get("required_left_halo_ns"), "required_left_halo_ns" + ), + backoff_levels=_string_tuple(data.get("backoff_levels")), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + config_id=str(data.get("config_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationStratumV1: + """One fitted conditioning stratum and its explicit fallback keys.""" + + level: str + key: str + pattern: Mapping[str, str] + status: str + support_count: int + parameters: tuple[ObservationParameterEstimateV1, ...] + evidence_ids: tuple[str, ...] + fallback_keys: tuple[str, ...] + stratum_id: str = "" + schema_version: str = OBSERVATION_STRATUM_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_STRATUM_SCHEMA_VERSION: + raise ValueError("unsupported observation stratum schema") + level = _required_text(self.level) + if level not in OBSERVATION_BACKOFF_LEVELS: + raise ValueError("unsupported observation stratum level") + pattern = { + _required_text(name): _required_text(value) + for name, value in sorted(self.pattern.items()) + } + context = ObservationContextV1( + symbol=pattern.get("symbol", "GLOBAL"), + epoch_id=pattern.get("epoch_id", "global"), + state=pattern.get("state"), + session=pattern.get("session"), + event_tag=pattern.get("event_tag"), + ) + if context.pattern_for_level(level) != pattern: + raise ValueError("observation stratum pattern differs from level") + expected_key = _stratum_key(level, pattern) + if _required_text(self.key) != expected_key: + raise ValueError("observation stratum key does not match pattern") + status = _required_text(self.status) + if status not in _STRATUM_STATUSES: + raise ValueError("unsupported observation stratum status") + support = _bounded_int( + self.support_count, + "support_count", + 0, + MAX_OBSERVATION_INPUT_EVENTS * MAX_OBSERVATION_EVIDENCE, + ) + parameters = tuple(sorted(self.parameters, key=lambda item: item.name)) + if not parameters or len(parameters) > MAX_OBSERVATION_PARAMETERS: + raise ValueError("observation stratum parameters are unbounded") + if len({parameter.name for parameter in parameters}) != len(parameters): + raise ValueError("duplicate observation stratum parameter") + evidence_ids = _bounded_text_tuple(self.evidence_ids, "evidence id") + fallback_keys = tuple( + dict.fromkeys(_required_text(key) for key in self.fallback_keys) + ) + if len(fallback_keys) > MAX_OBSERVATION_STRATA: + raise ValueError("observation fallback keys exceed limit") + if self.key in fallback_keys: + raise ValueError("observation stratum cannot fall back to itself") + object.__setattr__(self, "level", level) + object.__setattr__(self, "key", expected_key) + object.__setattr__(self, "pattern", pattern) + object.__setattr__(self, "status", status) + object.__setattr__(self, "support_count", support) + object.__setattr__(self, "parameters", parameters) + object.__setattr__(self, "evidence_ids", evidence_ids) + object.__setattr__(self, "fallback_keys", fallback_keys) + expected = _stable_id("observation-stratum", self.identity_payload()) + supplied = str(self.stratum_id or "").strip() + if supplied and supplied != expected: + raise ValueError("stratum_id does not match deterministic identity") + object.__setattr__(self, "stratum_id", expected) + + @property + def parameter_map(self) -> dict[str, ObservationParameterEstimateV1]: + """Return parameters indexed by stable name.""" + return {parameter.name: parameter for parameter in self.parameters} + + def effective_value(self, name: str) -> float: + """Return a fitted value or an explicit neutral identity value.""" + if name not in OBSERVATION_PARAMETER_NAMES: + raise ValueError("unsupported observation parameter") + parameter = self.parameter_map.get(name) + if parameter is None or parameter.support_status == "unsupported": + return _NEUTRAL_PARAMETER_VALUES[name] + return parameter.value + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic stratum identity.""" + return { + "schema_version": self.schema_version, + "level": self.level, + "key": self.key, + "pattern": dict(self.pattern), + "status": self.status, + "support_count": self.support_count, + "parameters": [ + parameter.to_dict() for parameter in self.parameters + ], + "evidence_ids": list(self.evidence_ids), + "fallback_keys": list(self.fallback_keys), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible stratum.""" + return {**self.identity_payload(), "stratum_id": self.stratum_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ObservationStratumV1": + """Read and verify one stratum.""" + return cls( + schema_version=str(data.get("schema_version", "")), + level=str(data.get("level", "")), + key=str(data.get("key", "")), + pattern=_string_mapping(data.get("pattern")), + status=str(data.get("status", "")), + support_count=_strict_int( + data.get("support_count"), "support_count" + ), + parameters=tuple( + ObservationParameterEstimateV1.from_dict(_mapping(item)) + for item in _sequence(data.get("parameters")) + ), + evidence_ids=_string_tuple(data.get("evidence_ids")), + fallback_keys=_string_tuple(data.get("fallback_keys")), + stratum_id=str(data.get("stratum_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationFitDiagnosticsV1: + """Bounded, versioned fitting support and residual diagnostics.""" + + evidence_count: int + stratum_count: int + status_counts: Mapping[str, int] + parameter_support_counts: Mapping[str, int] + parameter_residual_medians: Mapping[str, float] + unsupported_parameter_names: tuple[str, ...] + samples: tuple[Mapping[str, JSONValue], ...] = () + samples_truncated: bool = False + diagnostics_id: str = "" + schema_version: str = OBSERVATION_FIT_DIAGNOSTICS_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_FIT_DIAGNOSTICS_SCHEMA_VERSION: + raise ValueError("unsupported observation diagnostics schema") + _bounded_int( + self.evidence_count, + "evidence_count", + 1, + MAX_OBSERVATION_EVIDENCE, + ) + _bounded_int( + self.stratum_count, "stratum_count", 1, MAX_OBSERVATION_STRATA + ) + statuses = { + _required_text(name): _bounded_int( + count, f"{name} count", 0, MAX_OBSERVATION_STRATA + ) + for name, count in sorted(self.status_counts.items()) + } + if ( + not statuses + or not set(statuses).issubset(_STRATUM_STATUSES) + or sum(statuses.values()) != self.stratum_count + ): + raise ValueError("observation diagnostic statuses do not reconcile") + supports = { + _required_text(name): _bounded_int( + count, + f"{name} support", + 0, + MAX_OBSERVATION_INPUT_EVENTS * MAX_OBSERVATION_EVIDENCE, + ) + for name, count in sorted(self.parameter_support_counts.items()) + } + if not set(supports).issubset(OBSERVATION_PARAMETER_NAMES): + raise ValueError("observation diagnostic support names differ") + residuals = { + _required_text(name): _finite_float(value, name) + for name, value in sorted(self.parameter_residual_medians.items()) + } + if not set(residuals).issubset(OBSERVATION_PARAMETER_NAMES) or any( + value < 0.0 for value in residuals.values() + ): + raise ValueError("observation diagnostic residuals differ") + unsupported = tuple( + dict.fromkeys( + _required_text(name) + for name in self.unsupported_parameter_names + ) + ) + if not set(unsupported).issubset(OBSERVATION_PARAMETER_NAMES): + raise ValueError("observation unsupported parameter names differ") + truncated = _strict_bool(self.samples_truncated, "samples_truncated") + samples = tuple(dict(sample) for sample in self.samples) + if len(samples) > MAX_OBSERVATION_DIAGNOSTIC_SAMPLES: + raise ValueError("observation diagnostics samples exceed limit") + object.__setattr__(self, "status_counts", statuses) + object.__setattr__(self, "parameter_support_counts", supports) + object.__setattr__(self, "parameter_residual_medians", residuals) + object.__setattr__(self, "unsupported_parameter_names", unsupported) + object.__setattr__(self, "samples", samples) + object.__setattr__(self, "samples_truncated", truncated) + expected = _stable_id( + "observation-diagnostics", self.identity_payload() + ) + supplied = str(self.diagnostics_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "diagnostics_id does not match deterministic identity" + ) + object.__setattr__(self, "diagnostics_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic diagnostics identity.""" + return { + "schema_version": self.schema_version, + "evidence_count": self.evidence_count, + "stratum_count": self.stratum_count, + "status_counts": dict(self.status_counts), + "parameter_support_counts": dict(self.parameter_support_counts), + "parameter_residual_medians": dict(self.parameter_residual_medians), + "unsupported_parameter_names": list( + self.unsupported_parameter_names + ), + "samples": [dict(sample) for sample in self.samples], + "samples_truncated": self.samples_truncated, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible diagnostics.""" + return { + **self.identity_payload(), + "diagnostics_id": self.diagnostics_id, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ObservationFitDiagnosticsV1": + """Read and verify fitting diagnostics.""" + return cls( + schema_version=str(data.get("schema_version", "")), + evidence_count=_strict_int( + data.get("evidence_count"), "evidence_count" + ), + stratum_count=_strict_int( + data.get("stratum_count"), "stratum_count" + ), + status_counts=_int_mapping(data.get("status_counts")), + parameter_support_counts=_int_mapping( + data.get("parameter_support_counts") + ), + parameter_residual_medians=_float_mapping( + data.get("parameter_residual_medians") + ), + unsupported_parameter_names=_string_tuple( + data.get("unsupported_parameter_names") + ), + samples=tuple( + _mapping(sample) for sample in _sequence(data.get("samples")) + ), + samples_truncated=_strict_bool( + data.get("samples_truncated", False), "samples_truncated" + ), + diagnostics_id=str(data.get("diagnostics_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationOperatorV1: + """Replayable historical feed-observation operator artifact.""" + + feed_epoch_definition_id: str + feed_epoch_labels: tuple[str, ...] + fit_config: ObservationOperatorFitConfigV1 + strata: tuple[ObservationStratumV1, ...] + diagnostics: ObservationFitDiagnosticsV1 + source_hashes: tuple[str, ...] + lineage: Mapping[str, JSONValue] + required_left_halo_ns: int + carry_required_after_first_window: bool = True + carry_fields: tuple[str, ...] = OBSERVATION_CARRY_FIELDS + operator_id: str = "" + schema_version: str = OBSERVATION_OPERATOR_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_OPERATOR_SCHEMA_VERSION: + raise ValueError("unsupported observation operator schema") + definition_id = _required_feed_epoch_definition_id( + self.feed_epoch_definition_id + ) + labels = _bounded_text_tuple( + self.feed_epoch_labels, + "feed epoch label", + limit=MAX_OBSERVATION_STRATA, + ) + if not isinstance(self.fit_config, ObservationOperatorFitConfigV1): + raise ValueError("operator requires a v1 fit config") + carry_required = _strict_bool( + self.carry_required_after_first_window, + "carry_required_after_first_window", + ) + if not carry_required: + raise ValueError("v1 observation operators require streaming carry") + strata = tuple(sorted(self.strata, key=lambda item: item.key)) + if not strata or len(strata) > self.fit_config.max_strata: + raise ValueError("operator strata are empty or unbounded") + if len({stratum.key for stratum in strata}) != len(strata): + raise ValueError("operator contains duplicate stratum keys") + stratum_keys = {stratum.key for stratum in strata} + allowed_labels = set(labels) + for stratum in strata: + expected_support = max( + (parameter.support_count for parameter in stratum.parameters), + default=0, + ) + if stratum.support_count != expected_support: + raise ValueError("observation stratum support differs") + for parameter in stratum.parameters: + expected_parameter_status = ( + "supported" + if parameter.support_count + >= self.fit_config.min_parameter_support + else "unsupported" + ) + if parameter.support_status != expected_parameter_status: + raise ValueError("observation parameter status differs") + if not set(parameter.evidence_ids).issubset( + stratum.evidence_ids + ): + raise ValueError("observation parameter evidence differs") + supported_count = sum( + parameter.support_status == "supported" + for parameter in stratum.parameters + ) + expected_status = ( + "ready" + if ( + expected_support >= self.fit_config.min_stratum_support + and supported_count + >= self.fit_config.min_supported_parameters + ) + else ("limited" if supported_count else "unsupported") + ) + if stratum.status != expected_status: + raise ValueError("observation stratum status differs") + context = _context_from_pattern(stratum.pattern, allowed_labels) + candidates = context.candidate_keys(self.fit_config.backoff_levels) + offset = candidates.index(stratum.key) + 1 + expected_fallback = tuple( + key for key in candidates[offset:] if key in stratum_keys + ) + if stratum.fallback_keys != expected_fallback: + raise ValueError("observation stratum fallback differs") + global_stratum = next( + (stratum for stratum in strata if stratum.level == "global"), None + ) + if global_stratum is None or global_stratum.status != "ready": + raise ValueError("operator requires a ready global stratum") + if not isinstance(self.diagnostics, ObservationFitDiagnosticsV1): + raise ValueError("operator requires v1 diagnostics") + if self.diagnostics.stratum_count != len(strata): + raise ValueError("operator diagnostics stratum count differs") + if self.diagnostics.status_counts != dict( + sorted(Counter(stratum.status for stratum in strata).items()) + ): + raise ValueError("operator diagnostics status counts differ") + global_parameters = global_stratum.parameter_map + if self.diagnostics.parameter_support_counts != { + name: global_parameters[name].support_count + for name in OBSERVATION_PARAMETER_NAMES + if name in global_parameters + }: + raise ValueError("operator diagnostics parameter support differs") + expected_unsupported = tuple( + name + for name in OBSERVATION_PARAMETER_NAMES + if name not in global_parameters + or global_parameters[name].support_status == "unsupported" + ) + if self.diagnostics.unsupported_parameter_names != expected_unsupported: + raise ValueError("operator unsupported diagnostics differ") + hashes = tuple( + sorted( + dict.fromkeys( + _required_sha256_id(value, "source hash") + for value in self.source_hashes + ) + ) + ) + if not hashes: + raise ValueError("operator requires source hashes") + if len(hashes) > self.fit_config.max_evidence: + raise ValueError("operator source hashes exceed fit evidence limit") + halo = _bounded_int( + self.required_left_halo_ns, + "required_left_halo_ns", + 0, + MAX_OBSERVATION_WINDOW_ALIGNMENT_NS, + ) + carry_fields = _bounded_text_tuple( + self.carry_fields, + "carry field", + limit=len(OBSERVATION_CARRY_FIELDS), + ) + if carry_fields != OBSERVATION_CARRY_FIELDS: + raise ValueError("operator carry fields do not match v1 contract") + lineage = dict(self.lineage) + _validate_operator_lineage( + lineage, + definition_id=definition_id, + config_id=self.fit_config.config_id, + source_hashes=hashes, + evidence_count=self.diagnostics.evidence_count, + ) + lineage_evidence_ids = set(_string_tuple(lineage.get("evidence_ids"))) + stratum_evidence_ids = { + evidence_id + for stratum in strata + for evidence_id in stratum.evidence_ids + } + if stratum_evidence_ids != lineage_evidence_ids: + raise ValueError("operator stratum evidence lineage differs") + object.__setattr__(self, "feed_epoch_definition_id", definition_id) + object.__setattr__(self, "feed_epoch_labels", labels) + object.__setattr__(self, "strata", strata) + object.__setattr__(self, "source_hashes", hashes) + object.__setattr__(self, "lineage", lineage) + object.__setattr__(self, "required_left_halo_ns", halo) + object.__setattr__( + self, "carry_required_after_first_window", carry_required + ) + object.__setattr__(self, "carry_fields", carry_fields) + expected = _stable_id("observation-operator", self.identity_payload()) + supplied = str(self.operator_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "operator_id does not match deterministic identity" + ) + object.__setattr__(self, "operator_id", expected) + + @property + def valid_for_application(self) -> bool: + """Return whether the artifact passed its fail-closed constructor.""" + return True + + def resolve_stratum( + self, context: ObservationContextV1 + ) -> tuple[ObservationStratumV1, tuple[str, ...]]: + """Resolve a context through the versioned, explicit hierarchy.""" + if context.epoch_id not in self.feed_epoch_labels: + raise ValueError("observation context epoch is not in operator") + by_key = {stratum.key: stratum for stratum in self.strata} + attempted: list[str] = [] + for key in context.candidate_keys(self.fit_config.backoff_levels): + attempted.append(key) + stratum = by_key.get(key) + if stratum is not None and stratum.status == "ready": + return stratum, tuple(attempted) + raise ValueError("no supported observation stratum after fallback") + + def identity_payload(self) -> dict[str, JSONValue]: + """Return complete semantic operator identity.""" + return { + "schema_version": self.schema_version, + "feed_epoch_definition_id": self.feed_epoch_definition_id, + "feed_epoch_labels": list(self.feed_epoch_labels), + "fit_config": self.fit_config.to_dict(), + "strata": [stratum.to_dict() for stratum in self.strata], + "diagnostics": self.diagnostics.to_dict(), + "source_hashes": list(self.source_hashes), + "lineage": dict(self.lineage), + "required_left_halo_ns": self.required_left_halo_ns, + "carry_required_after_first_window": ( + self.carry_required_after_first_window + ), + "carry_fields": list(self.carry_fields), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible operator artifact.""" + return { + **self.identity_payload(), + "operator_id": self.operator_id, + "valid_for_application": self.valid_for_application, + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ObservationOperatorV1": + """Read and verify a version-one operator artifact.""" + return cls( + schema_version=str(data.get("schema_version", "")), + feed_epoch_definition_id=str( + data.get("feed_epoch_definition_id", "") + ), + feed_epoch_labels=_string_tuple(data.get("feed_epoch_labels")), + fit_config=ObservationOperatorFitConfigV1.from_dict( + _mapping(data.get("fit_config")) + ), + strata=tuple( + ObservationStratumV1.from_dict(_mapping(item)) + for item in _sequence(data.get("strata")) + ), + diagnostics=ObservationFitDiagnosticsV1.from_dict( + _mapping(data.get("diagnostics")) + ), + source_hashes=_string_tuple(data.get("source_hashes")), + lineage=_mapping(data.get("lineage")), + required_left_halo_ns=_strict_int( + data.get("required_left_halo_ns"), "required_left_halo_ns" + ), + carry_required_after_first_window=_strict_bool( + data.get("carry_required_after_first_window", True), + "carry_required_after_first_window", + ), + carry_fields=_string_tuple(data.get("carry_fields")), + operator_id=str(data.get("operator_id", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ObservationOperatorV1": + """Read an operator from deterministic JSON.""" + data = json.loads(text) + if not isinstance(data, Mapping): + raise ValueError("observation operator JSON must be an object") + return cls.from_dict(data) + + def apply( + self, + events: Sequence["ObservationInputEventV1"], + *, + window: ReconstructionWindowV1, + carry: "ObservationCarryStateV1 | None" = None, + information_mode: InformationMode = InformationMode.EX_POST_RECONSTRUCTION, + source_start: bool = False, + ) -> "ObservationApplicationResultV1": + """Apply forward observation while preserving protected anchors.""" + return _apply_observation_operator( + self, + events, + window=window, + carry=carry, + application_mode="apply", + information_mode=information_mode, + source_start=source_start, + benchmark_protected_ids=None, + ) + + def degrade( + self, + events: Sequence["ObservationInputEventV1"], + *, + window: ReconstructionWindowV1, + carry: "ObservationCarryStateV1 | None" = None, + protected_event_ids: Sequence[str] = (), + source_start: bool = False, + ) -> "ObservationApplicationResultV1": + """Apply controlled degradation for the generator-neutral benchmark.""" + protected = tuple( + dict.fromkeys( + _required_text(value) for value in protected_event_ids + ) + ) + if len(protected) > self.fit_config.max_input_events: + raise ValueError("protected observation IDs exceed input limit") + return _apply_observation_operator( + self, + events, + window=window, + carry=carry, + application_mode="degrade", + information_mode=InformationMode.EX_ANTE_SIMULATION, + source_start=source_start, + benchmark_protected_ids=frozenset(protected), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationInputEventV1: + """One market event plus observation-only conditioning metadata.""" + + source_event_id: str + symbol: str + event_time_ns: int + event_sequence: int + bid: float + ask: float + context: ObservationContextV1 + protected_anchor: bool = False + schema_version: str = OBSERVATION_INPUT_EVENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_INPUT_EVENT_SCHEMA_VERSION: + raise ValueError("unsupported observation input event schema") + source_id = _required_text(self.source_event_id) + symbol = _normalized_symbol(self.symbol) + timestamp = _bounded_int( + self.event_time_ns, "event_time_ns", INT64_MIN, INT64_MAX + ) + sequence = _bounded_int( + self.event_sequence, "event_sequence", 0, 2**63 - 1 + ) + bid = _positive_float(self.bid, "bid") + ask = _positive_float(self.ask, "ask") + if ask < bid: + raise ValueError("observation input ask precedes bid") + if not isinstance(self.context, ObservationContextV1): + raise ValueError("observation input requires v1 context") + if self.context.symbol != symbol: + raise ValueError("observation input context symbol differs") + protected = _strict_bool(self.protected_anchor, "protected_anchor") + object.__setattr__(self, "source_event_id", source_id) + object.__setattr__(self, "symbol", symbol) + object.__setattr__(self, "event_time_ns", timestamp) + object.__setattr__(self, "event_sequence", sequence) + object.__setattr__(self, "bid", bid) + object.__setattr__(self, "ask", ask) + object.__setattr__(self, "protected_anchor", protected) + + @classmethod + def from_synthetic_event( + cls, + event: SyntheticEventV1, + *, + context: ObservationContextV1, + protected_anchor: bool | None = None, + ) -> "ObservationInputEventV1": + """Adapt an immutable market event without changing its identity.""" + protected = ( + event.origin is SyntheticEventOrigin.OBSERVED + if protected_anchor is None + else _strict_bool(protected_anchor, "protected_anchor") + ) + return cls( + source_event_id=event.event_id, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + context=context, + protected_anchor=protected, + ) + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible input event.""" + return { + "schema_version": self.schema_version, + "source_event_id": self.source_event_id, + "symbol": self.symbol, + "event_time_ns": self.event_time_ns, + "event_sequence": self.event_sequence, + "bid": self.bid, + "ask": self.ask, + "context": self.context.to_dict(), + "protected_anchor": self.protected_anchor, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ObservationInputEventV1": + """Read one input event.""" + return cls( + schema_version=str(data.get("schema_version", "")), + source_event_id=str(data.get("source_event_id", "")), + symbol=str(data.get("symbol", "")), + event_time_ns=_strict_int( + data.get("event_time_ns"), "event_time_ns" + ), + event_sequence=_strict_int( + data.get("event_sequence"), "event_sequence" + ), + bid=_finite_float(data.get("bid"), "bid"), + ask=_finite_float(data.get("ask"), "ask"), + context=ObservationContextV1.from_dict( + _mapping(data.get("context")) + ), + protected_anchor=_strict_bool( + data.get("protected_anchor", False), "protected_anchor" + ), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationOutputEventV1: + """One operator-lineaged delivery observation.""" + + source_event_id: str + operator_id: str + stratum_id: str + symbol: str + source_time_ns: int + observed_time_ns: int + observed_sequence: int + bid: float + ask: float + duplicate_ordinal: int + transformations: tuple[str, ...] + protected_anchor: bool + observation_id: str = "" + schema_version: str = OBSERVATION_OUTPUT_EVENT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_OUTPUT_EVENT_SCHEMA_VERSION: + raise ValueError("unsupported observation output event schema") + object.__setattr__( + self, "source_event_id", _required_text(self.source_event_id) + ) + object.__setattr__( + self, + "operator_id", + _required_sha256_id( + self.operator_id, "operator_id", prefix="observation-operator" + ), + ) + object.__setattr__( + self, + "stratum_id", + _required_sha256_id( + self.stratum_id, "stratum_id", prefix="observation-stratum" + ), + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, + "source_time_ns", + _bounded_int( + self.source_time_ns, + "source_time_ns", + INT64_MIN, + INT64_MAX, + ), + ) + object.__setattr__( + self, + "observed_time_ns", + _bounded_int( + self.observed_time_ns, + "observed_time_ns", + INT64_MIN, + INT64_MAX, + ), + ) + if self.observed_time_ns > self.source_time_ns: + raise ValueError("observed time cannot follow source time in v1") + object.__setattr__( + self, + "observed_sequence", + _bounded_int( + self.observed_sequence, "observed_sequence", 0, 2**63 - 1 + ), + ) + bid = _positive_float(self.bid, "bid") + ask = _positive_float(self.ask, "ask") + if ask < bid: + raise ValueError("observation output ask precedes bid") + object.__setattr__(self, "bid", bid) + object.__setattr__(self, "ask", ask) + object.__setattr__( + self, + "duplicate_ordinal", + _bounded_int( + self.duplicate_ordinal, + "duplicate_ordinal", + 0, + MAX_OBSERVATION_OUTPUTS_PER_INPUT - 1, + ), + ) + transforms = tuple( + dict.fromkeys( + _required_text(value) for value in self.transformations + ) + ) + if not set(transforms).issubset(_OUTPUT_TRANSFORMATIONS): + raise ValueError("unsupported observation transformation") + protected = _strict_bool(self.protected_anchor, "protected_anchor") + if protected and transforms: + raise ValueError("protected anchors cannot carry transformations") + object.__setattr__(self, "transformations", transforms) + object.__setattr__(self, "protected_anchor", protected) + expected = _stable_id("observation-event", self.identity_payload()) + supplied = str(self.observation_id or "").strip() + if supplied and supplied != expected: + raise ValueError( + "observation_id does not match deterministic identity" + ) + object.__setattr__(self, "observation_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic delivery-observation identity.""" + return { + "schema_version": self.schema_version, + "source_event_id": self.source_event_id, + "operator_id": self.operator_id, + "stratum_id": self.stratum_id, + "symbol": self.symbol, + "source_time_ns": self.source_time_ns, + "observed_time_ns": self.observed_time_ns, + "observed_sequence": self.observed_sequence, + "bid": self.bid, + "ask": self.ask, + "duplicate_ordinal": self.duplicate_ordinal, + "transformations": list(self.transformations), + "protected_anchor": self.protected_anchor, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible output event.""" + return { + **self.identity_payload(), + "observation_id": self.observation_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ObservationOutputEventV1": + """Read and verify one output event.""" + return cls( + schema_version=str(data.get("schema_version", "")), + source_event_id=str(data.get("source_event_id", "")), + operator_id=str(data.get("operator_id", "")), + stratum_id=str(data.get("stratum_id", "")), + symbol=str(data.get("symbol", "")), + source_time_ns=_strict_int( + data.get("source_time_ns"), "source_time_ns" + ), + observed_time_ns=_strict_int( + data.get("observed_time_ns"), "observed_time_ns" + ), + observed_sequence=_strict_int( + data.get("observed_sequence"), "observed_sequence" + ), + bid=_finite_float(data.get("bid"), "bid"), + ask=_finite_float(data.get("ask"), "ask"), + duplicate_ordinal=_strict_int( + data.get("duplicate_ordinal"), "duplicate_ordinal" + ), + transformations=_string_tuple(data.get("transformations")), + protected_anchor=_strict_bool( + data.get("protected_anchor", False), "protected_anchor" + ), + observation_id=str(data.get("observation_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationCarryStateV1: + """Bounded state required for partition-independent application.""" + + operator_id: str + symbol: str + last_source_time_ns: int | None = None + last_observed_time_ns: int | None = None + last_bid: float | None = None + last_ask: float | None = None + rate_bucket_start_ns: int | None = None + rate_bucket_count: int = 0 + outage_bucket_start_ns: int | None = None + outage_active: bool = False + reconnect_pending: bool = False + carry_id: str = "" + schema_version: str = OBSERVATION_CARRY_STATE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_CARRY_STATE_SCHEMA_VERSION: + raise ValueError("unsupported observation carry-state schema") + object.__setattr__( + self, + "operator_id", + _required_sha256_id( + self.operator_id, "operator_id", prefix="observation-operator" + ), + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + for name in ( + "last_source_time_ns", + "last_observed_time_ns", + "rate_bucket_start_ns", + "outage_bucket_start_ns", + ): + value = getattr(self, name) + if value is not None: + object.__setattr__( + self, + name, + _bounded_int(value, name, INT64_MIN, INT64_MAX), + ) + if (self.last_bid is None) != (self.last_ask is None): + raise ValueError("observation carry bid/ask must be paired") + if self.last_bid is not None and self.last_ask is not None: + bid = _positive_float(self.last_bid, "last_bid") + ask = _positive_float(self.last_ask, "last_ask") + if ask < bid: + raise ValueError("observation carry ask precedes bid") + object.__setattr__(self, "last_bid", bid) + object.__setattr__(self, "last_ask", ask) + if self.last_observed_time_ns is not None and ( + self.last_source_time_ns is None + or self.last_observed_time_ns > self.last_source_time_ns + or self.last_bid is None + ): + raise ValueError("observation carry delivered state is incomplete") + if self.last_bid is not None and self.last_observed_time_ns is None: + raise ValueError("observation carry quote lacks delivered time") + outage_active = _strict_bool(self.outage_active, "outage_active") + reconnect_pending = _strict_bool( + self.reconnect_pending, "reconnect_pending" + ) + if outage_active and ( + self.outage_bucket_start_ns is None or not reconnect_pending + ): + raise ValueError("observation carry outage state is incomplete") + object.__setattr__( + self, + "rate_bucket_count", + _bounded_int( + self.rate_bucket_count, + "rate_bucket_count", + 0, + MAX_OBSERVATION_INPUT_EVENTS + * MAX_OBSERVATION_OUTPUTS_PER_INPUT, + ), + ) + if self.rate_bucket_count and self.rate_bucket_start_ns is None: + raise ValueError("observation carry rate state is incomplete") + object.__setattr__(self, "outage_active", outage_active) + object.__setattr__(self, "reconnect_pending", reconnect_pending) + expected = _stable_id("observation-carry", self.identity_payload()) + supplied = str(self.carry_id or "").strip() + if supplied and supplied != expected: + raise ValueError("carry_id does not match deterministic identity") + object.__setattr__(self, "carry_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic carry identity.""" + return { + "schema_version": self.schema_version, + "operator_id": self.operator_id, + "symbol": self.symbol, + "last_source_time_ns": self.last_source_time_ns, + "last_observed_time_ns": self.last_observed_time_ns, + "last_bid": self.last_bid, + "last_ask": self.last_ask, + "rate_bucket_start_ns": self.rate_bucket_start_ns, + "rate_bucket_count": self.rate_bucket_count, + "outage_bucket_start_ns": self.outage_bucket_start_ns, + "outage_active": self.outage_active, + "reconnect_pending": self.reconnect_pending, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible carry state.""" + return {**self.identity_payload(), "carry_id": self.carry_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ObservationCarryStateV1": + """Read and verify carry state.""" + return cls( + schema_version=str(data.get("schema_version", "")), + operator_id=str(data.get("operator_id", "")), + symbol=str(data.get("symbol", "")), + last_source_time_ns=_optional_strict_int( + data.get("last_source_time_ns"), "last_source_time_ns" + ), + last_observed_time_ns=_optional_strict_int( + data.get("last_observed_time_ns"), "last_observed_time_ns" + ), + last_bid=_optional_float(data.get("last_bid")), + last_ask=_optional_float(data.get("last_ask")), + rate_bucket_start_ns=_optional_strict_int( + data.get("rate_bucket_start_ns"), "rate_bucket_start_ns" + ), + rate_bucket_count=_strict_int( + data.get("rate_bucket_count", 0), "rate_bucket_count" + ), + outage_bucket_start_ns=_optional_strict_int( + data.get("outage_bucket_start_ns"), + "outage_bucket_start_ns", + ), + outage_active=_strict_bool( + data.get("outage_active", False), "outage_active" + ), + reconnect_pending=_strict_bool( + data.get("reconnect_pending", False), "reconnect_pending" + ), + carry_id=str(data.get("carry_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationApplicationResultV1: + """Bounded in-memory output and diagnostics for one owned window.""" + + operator_id: str + window_id: str + symbol: str + application_mode: str + input_count: int + output_events: tuple[ObservationOutputEventV1, ...] + reason_counts: Mapping[str, int] + fallback_counts: Mapping[str, int] + diagnostic_samples: tuple[Mapping[str, JSONValue], ...] + samples_truncated: bool + carry_state: ObservationCarryStateV1 + result_id: str = "" + schema_version: str = OBSERVATION_APPLICATION_RESULT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != OBSERVATION_APPLICATION_RESULT_SCHEMA_VERSION: + raise ValueError("unsupported observation application schema") + object.__setattr__( + self, + "operator_id", + _required_sha256_id( + self.operator_id, "operator_id", prefix="observation-operator" + ), + ) + object.__setattr__(self, "window_id", _required_text(self.window_id)) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + mode = _required_text(self.application_mode) + if mode not in {"apply", "degrade"}: + raise ValueError("unsupported observation application mode") + object.__setattr__(self, "application_mode", mode) + _bounded_int( + self.input_count, + "input_count", + 0, + MAX_OBSERVATION_INPUT_EVENTS, + ) + outputs = tuple(self.output_events) + if len(outputs) > ( + self.input_count * MAX_OBSERVATION_OUTPUTS_PER_INPUT + ): + raise ValueError("observation output amplification exceeds limit") + if any(event.operator_id != self.operator_id for event in outputs): + raise ValueError("observation output operator differs") + if ( + tuple( + sorted( + outputs, + key=lambda item: ( + item.observed_time_ns, + item.observed_sequence, + item.observation_id, + ), + ) + ) + != outputs + ): + raise ValueError("observation outputs are not ordered") + reason_counts = _validated_count_mapping( + self.reason_counts, allowed_names=_APPLICATION_REASONS + ) + fallback_counts = _validated_count_mapping( + self.fallback_counts, + allowed_names=set(OBSERVATION_BACKOFF_LEVELS), + ) + if sum(reason_counts.values()) != self.input_count: + raise ValueError("observation reason counts do not cover inputs") + events_by_source: dict[str, list[ObservationOutputEventV1]] = ( + defaultdict(list) + ) + sequences_by_timestamp: dict[int, list[int]] = defaultdict(list) + for event in outputs: + events_by_source[event.source_event_id].append(event) + sequences_by_timestamp[event.observed_time_ns].append( + event.observed_sequence + ) + outputs_by_source = { + source_id: len(source_events) + for source_id, source_events in events_by_source.items() + } + if any( + count > MAX_OBSERVATION_OUTPUTS_PER_INPUT + for count in outputs_by_source.values() + ): + raise ValueError("observation source output count exceeds limit") + if len(outputs_by_source) != reason_counts.get("retained", 0): + raise ValueError("observation retained count differs from outputs") + for source_events in events_by_source.values(): + if tuple( + event.duplicate_ordinal for event in source_events + ) != tuple(range(len(source_events))): + raise ValueError( + "observation duplicate ordinals are not contiguous" + ) + for timestamp_sequences in sequences_by_timestamp.values(): + sequences = tuple(timestamp_sequences) + if sequences != tuple(range(len(sequences))): + raise ValueError( + "observation delivery sequences are not contiguous" + ) + samples = tuple(dict(sample) for sample in self.diagnostic_samples) + if len(samples) > MAX_OBSERVATION_DIAGNOSTIC_SAMPLES: + raise ValueError("observation application samples exceed limit") + if not isinstance(self.carry_state, ObservationCarryStateV1): + raise ValueError("observation result requires v1 carry state") + if ( + self.carry_state.operator_id != self.operator_id + or self.carry_state.symbol != self.symbol + ): + raise ValueError("observation result carry scope differs") + truncated = _strict_bool(self.samples_truncated, "samples_truncated") + object.__setattr__(self, "output_events", outputs) + object.__setattr__(self, "reason_counts", reason_counts) + object.__setattr__(self, "fallback_counts", fallback_counts) + object.__setattr__(self, "diagnostic_samples", samples) + object.__setattr__(self, "samples_truncated", truncated) + expected = _stable_id( + "observation-application", self.identity_payload() + ) + supplied = str(self.result_id or "").strip() + if supplied and supplied != expected: + raise ValueError("result_id does not match deterministic identity") + object.__setattr__(self, "result_id", expected) + + @property + def output_count(self) -> int: + """Return the number of delivery observations.""" + return len(self.output_events) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic result identity.""" + return { + "schema_version": self.schema_version, + "operator_id": self.operator_id, + "window_id": self.window_id, + "symbol": self.symbol, + "application_mode": self.application_mode, + "input_count": self.input_count, + "output_events": [event.to_dict() for event in self.output_events], + "reason_counts": dict(self.reason_counts), + "fallback_counts": dict(self.fallback_counts), + "diagnostic_samples": [ + dict(sample) for sample in self.diagnostic_samples + ], + "samples_truncated": self.samples_truncated, + "carry_state": self.carry_state.to_dict(), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return JSON-compatible application result.""" + return {**self.identity_payload(), "result_id": self.result_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ObservationApplicationResultV1": + """Read and verify one bounded application result.""" + return cls( + schema_version=str(data.get("schema_version", "")), + operator_id=str(data.get("operator_id", "")), + window_id=str(data.get("window_id", "")), + symbol=str(data.get("symbol", "")), + application_mode=str(data.get("application_mode", "")), + input_count=_strict_int(data.get("input_count"), "input_count"), + output_events=tuple( + ObservationOutputEventV1.from_dict(_mapping(item)) + for item in _sequence(data.get("output_events")) + ), + reason_counts=_int_mapping(data.get("reason_counts")), + fallback_counts=_int_mapping(data.get("fallback_counts")), + diagnostic_samples=tuple( + _mapping(sample) + for sample in _sequence(data.get("diagnostic_samples")) + ), + samples_truncated=_strict_bool( + data.get("samples_truncated", False), "samples_truncated" + ), + carry_state=ObservationCarryStateV1.from_dict( + _mapping(data.get("carry_state")) + ), + result_id=str(data.get("result_id", "")), + ) + + +def fit_observation_operator( + evidence: Sequence[ObservationFitEvidenceV1], + *, + epoch_definition: FeedEpochDefinitionV1 | FeedEpochDefinitionV2, + config: ObservationOperatorFitConfigV1 | None = None, +) -> ObservationOperatorV1: + """Fit a deterministic operator through an explicit hierarchy.""" + selected = config or ObservationOperatorFitConfigV1() + if not epoch_definition.valid_for_observation_models: + raise ValueError("feed epoch definition has not passed stability") + ordered = tuple(sorted(evidence, key=lambda item: item.evidence_id)) + if not ordered or len(ordered) > selected.max_evidence: + raise ValueError("observation evidence is empty or exceeds fit limit") + if len({item.evidence_id for item in ordered}) != len(ordered): + raise ValueError("duplicate observation fit evidence") + allowed_labels = { + *(epoch.label for epoch in epoch_definition.epochs), + *( + boundary.transition_label + for boundary in epoch_definition.boundaries + ), + } + for item in ordered: + if item.context.symbol not in epoch_definition.symbols: + raise ValueError( + "observation evidence symbol is outside epoch scope" + ) + if item.context.epoch_id not in allowed_labels: + raise ValueError("observation evidence epoch is outside definition") + midpoint_ms = ( + (item.start_timestamp_ns + item.end_timestamp_ns) // 2 + ) // 1_000_000 + assignment = epoch_definition.assign( + symbol=item.context.symbol, + timestamp_utc_ms=midpoint_ms, + ) + if assignment.label != item.context.epoch_id: + raise ValueError("observation evidence epoch assignment differs") + + groups: dict[tuple[str, str], list[ObservationFitEvidenceV1]] = defaultdict( + list + ) + patterns: dict[tuple[str, str], dict[str, str]] = {} + for item in ordered: + seen: set[str] = set() + for level in selected.backoff_levels: + pattern = item.context.pattern_for_level(level) + if pattern is None: + continue + key = _stratum_key(level, pattern) + if key in seen: + continue + seen.add(key) + groups[(level, key)].append(item) + patterns[(level, key)] = pattern + if len(groups) > selected.max_strata: + raise ValueError("fitted observation strata exceed configured limit") + + preliminary: list[ObservationStratumV1] = [] + for (level, key), items in sorted(groups.items(), key=lambda pair: pair[0]): + parameters = tuple( + _fit_parameter(name, items, selected) + for name in OBSERVATION_PARAMETER_NAMES + if any(name in item.parameter_values for item in items) + ) + support_count = max( + (parameter.support_count for parameter in parameters), default=0 + ) + supported_count = sum( + parameter.support_status == "supported" for parameter in parameters + ) + if ( + support_count >= selected.min_stratum_support + and supported_count >= selected.min_supported_parameters + ): + status = "ready" + elif supported_count: + status = "limited" + else: + status = "unsupported" + preliminary.append( + ObservationStratumV1( + level=level, + key=key, + pattern=patterns[(level, key)], + status=status, + support_count=support_count, + parameters=parameters, + evidence_ids=tuple(item.evidence_id for item in items), + fallback_keys=(), + ) + ) + keys = {stratum.key for stratum in preliminary} + strata: list[ObservationStratumV1] = [] + for stratum in preliminary: + context = _context_from_pattern(stratum.pattern, allowed_labels) + candidates = context.candidate_keys(selected.backoff_levels) + try: + offset = candidates.index(stratum.key) + 1 + except ValueError: + offset = len(candidates) + fallback = tuple(key for key in candidates[offset:] if key in keys) + strata.append(replace(stratum, fallback_keys=fallback, stratum_id="")) + + diagnostics = _fit_diagnostics(ordered, strata, selected) + source_hashes = tuple( + sorted({item.source_artifact_sha256 for item in ordered}) + ) + lineage_material: dict[str, JSONValue] = { + "feed_epoch_definition_id": epoch_definition.definition_id, + "config_id": selected.config_id, + "evidence_count": len(ordered), + "evidence_ids": [item.evidence_id for item in ordered], + "source_hashes": list(source_hashes), + "sources": [ + { + "evidence_id": item.evidence_id, + "source_evidence_id": item.source_evidence_id, + "source_artifact_sha256": item.source_artifact_sha256, + "source_hash_basis": item.source_hash_basis, + "evidence_kind": item.evidence_kind, + } + for item in ordered + ], + } + lineage = { + **lineage_material, + "lineage_sha256": _payload_sha256(lineage_material), + } + labels = tuple(sorted(allowed_labels)) + return ObservationOperatorV1( + feed_epoch_definition_id=epoch_definition.definition_id, + feed_epoch_labels=labels, + fit_config=selected, + strata=tuple(strata), + diagnostics=diagnostics, + source_hashes=source_hashes, + lineage=lineage, + required_left_halo_ns=selected.required_left_halo_ns, + ) + + +def write_observation_operator( + operator: ObservationOperatorV1, path: Path +) -> ArtifactRef: + """Write one deterministic operator artifact and return its reference.""" + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + encoded = (operator.to_json() + "\n").encode("utf-8") + if len(encoded) > MAX_OBSERVATION_ARTIFACT_BYTES: + raise ValueError("observation operator artifact exceeds byte limit") + destination.write_bytes(encoded) + digest = hashlib.sha256(encoded).hexdigest() + return ArtifactRef( + kind="observation-operator", + path=str(destination), + size_bytes=len(encoded), + sha256=digest, + metadata={ + "schema_version": operator.schema_version, + "operator_id": operator.operator_id, + "feed_epoch_definition_id": operator.feed_epoch_definition_id, + }, + ) + + +def read_observation_operator_artifact( + artifact: ArtifactRef, +) -> ObservationOperatorV1: + """Read an operator only after byte-level artifact verification.""" + if artifact.kind != "observation-operator": + raise ValueError("artifact is not an observation operator") + path = Path(artifact.path) + size = path.stat().st_size + if size > MAX_OBSERVATION_ARTIFACT_BYTES: + raise ValueError("observation operator artifact exceeds byte limit") + encoded = path.read_bytes() + if artifact.size_bytes is not None and len(encoded) != artifact.size_bytes: + raise ValueError("observation operator artifact size differs") + digest = hashlib.sha256(encoded).hexdigest() + if not artifact.sha256 or digest != artifact.sha256: + raise ValueError("observation operator artifact hash differs") + operator = ObservationOperatorV1.from_json(encoded.decode("utf-8")) + if ( + artifact.metadata.get("operator_id") != operator.operator_id + or artifact.metadata.get("schema_version") != operator.schema_version + or artifact.metadata.get("feed_epoch_definition_id") + != operator.feed_epoch_definition_id + ): + raise ValueError("observation operator artifact metadata differs") + return operator + + +@dataclass(slots=True) +class _MutableApplicationState: + """Internal bounded state used while applying one window.""" + + last_source_time_ns: int | None = None + last_observed_time_ns: int | None = None + last_bid: float | None = None + last_ask: float | None = None + rate_bucket_start_ns: int | None = None + rate_bucket_count: int = 0 + outage_bucket_start_ns: int | None = None + outage_active: bool = False + reconnect_pending: bool = False + + +def _apply_observation_operator( + operator: ObservationOperatorV1, + events: Sequence[ObservationInputEventV1], + *, + window: ReconstructionWindowV1, + carry: ObservationCarryStateV1 | None, + application_mode: str, + information_mode: InformationMode, + source_start: bool, + benchmark_protected_ids: frozenset[str] | None, +) -> ObservationApplicationResultV1: + source_start = _strict_bool(source_start, "source_start") + if len(events) > operator.fit_config.max_input_events: + raise ValueError("observation application exceeds input event limit") + selected_mode = InformationMode.from_value(information_mode) + ordered = tuple( + sorted( + events, + key=lambda item: ( + item.event_time_ns, + item.event_sequence, + item.source_event_id, + ), + ) + ) + if len({item.source_event_id for item in ordered}) != len(ordered): + raise ValueError("duplicate observation source event identity") + symbol = _application_symbol(ordered, window) + window_symbols = {_normalized_symbol(value) for value in window.symbols} + if symbol not in window_symbols: + raise ValueError("observation symbol is outside reconstruction window") + if any(item.symbol != symbol for item in ordered): + raise ValueError("observation application requires one symbol") + if any(not window.reads_event_time(item.event_time_ns) for item in ordered): + raise ValueError("observation input event is outside window read scope") + if carry is None and not source_start: + if operator.carry_required_after_first_window: + raise ValueError( + "observation operator requires carry after the source start" + ) + if window.left_halo_ns < operator.required_left_halo_ns: + raise ValueError("observation window lacks required left halo") + if carry is not None and ( + carry.operator_id != operator.operator_id or carry.symbol != symbol + ): + raise ValueError("observation carry scope differs from application") + first_core_event = next( + ( + item + for item in ordered + if window.core_start_ns <= item.event_time_ns < window.core_end_ns + ), + None, + ) + if ( + carry is not None + and carry.last_source_time_ns is not None + and first_core_event is not None + and first_core_event.event_time_ns <= carry.last_source_time_ns + ): + raise ValueError("observation carry watermark is stale or overlapping") + if ( + application_mode == "apply" + and selected_mode is InformationMode.EX_ANTE_SIMULATION + and any(item.protected_anchor for item in ordered) + ): + raise ValueError("ex-ante observation application cannot use anchors") + + state = _state_from_carry(carry) + global_stratum = next( + stratum for stratum in operator.strata if stratum.level == "global" + ) + _validate_window_alignment( + window, + { + name: global_stratum.effective_value(name) + for name in OBSERVATION_PARAMETER_NAMES + }, + ) + tentative: list[dict[str, Any]] = [] + reasons: Counter[str] = Counter() + fallbacks: Counter[str] = Counter() + samples: list[dict[str, JSONValue]] = [] + inspected_owned = 0 + dropped_owned = 0 + for event in ordered: + if carry is not None and event.event_time_ns < window.core_start_ns: + continue + if event.event_time_ns >= window.core_end_ns: + continue + owned = window.owns_event_time(event.event_time_ns) + if owned: + inspected_owned += 1 + stratum, attempted = operator.resolve_stratum(event.context) + if owned and len(attempted) > 1: + fallbacks[stratum.level] += 1 + params = { + name: stratum.effective_value(name) + for name in OBSERVATION_PARAMETER_NAMES + } + _validate_window_alignment(window, params) + protected = ( + event.protected_anchor + if benchmark_protected_ids is None + else event.source_event_id in benchmark_protected_ids + ) + dropped_reason: str | None = None + transformed_time = event.event_time_ns + transformed_bid = event.bid + transformed_ask = event.ask + transformations: list[str] = [] + rate_window = max(1, int(params["burst_window_ns"]) or 1_000_000_000) + rate_bucket = event.event_time_ns // rate_window * rate_window + if state.rate_bucket_start_ns != rate_bucket: + state.rate_bucket_start_ns = rate_bucket + state.rate_bucket_count = 0 + + if not protected: + outage_window = int(params["outage_window_ns"]) + quiet_probability = params["quiet_gap_probability"] + outage_bucket = ( + event.event_time_ns // outage_window * outage_window + if outage_window > 0 + else None + ) + outage_active = bool( + outage_bucket is not None + and _selected( + quiet_probability, + operator.operator_id, + stratum.stratum_id, + "outage", + str(outage_bucket), + ) + ) + state.outage_bucket_start_ns = outage_bucket + state.outage_active = outage_active + if outage_active: + state.reconnect_pending = True + dropped_reason = "outage" + else: + state.outage_active = False + if not _selected( + params["retention_probability"], + operator.operator_id, + stratum.stratum_id, + "retention", + event.source_event_id, + ): + dropped_reason = "thinning" + + timestamp_quantum = max(1, int(params["timestamp_quantum_ns"])) + batch_window = int(params["batch_window_ns"]) + transformed_time = ( + event.event_time_ns // timestamp_quantum * timestamp_quantum + ) + if transformed_time != event.event_time_ns: + transformations.append("timestamp_quantized") + if batch_window > 0: + batched = transformed_time // batch_window * batch_window + if batched != transformed_time: + transformations.append("batched") + transformed_time = batched + + digits = int(params["price_precision_digits"]) + transformed_bid = round(event.bid, digits) + transformed_ask = max( + transformed_bid, + round(event.ask, digits), + ) + if transformed_bid != event.bid or transformed_ask != event.ask: + transformations.append("price_quantized") + unchanged = ( + state.last_bid is not None + and state.last_ask is not None + and transformed_bid == state.last_bid + and transformed_ask == state.last_ask + ) + if ( + dropped_reason is None + and unchanged + and not _selected( + params["unchanged_retention_probability"], + operator.operator_id, + stratum.stratum_id, + "unchanged", + event.source_event_id, + ) + ): + dropped_reason = "unchanged_quote_filter" + threshold = params["quote_transition_threshold"] + if ( + dropped_reason is None + and not unchanged + and threshold > 0 + and state.last_bid is not None + and state.last_ask is not None + and abs(transformed_bid - state.last_bid) < threshold + and abs(transformed_ask - state.last_ask) < threshold + ): + dropped_reason = "quote_transition_filter" + + rate_cap = params["rate_cap_per_second"] + if rate_cap > 0: + allowed = max( + 1, + int(math.floor(rate_cap * rate_window / 1_000_000_000)), + ) + if ( + state.rate_bucket_count >= allowed + and dropped_reason is None + ): + dropped_reason = "rate_cap" + else: + state.outage_active = False + + state.last_source_time_ns = event.event_time_ns + if dropped_reason is not None: + if owned: + reasons[dropped_reason] += 1 + dropped_owned += 1 + _append_application_sample( + samples, + operator.fit_config.diagnostic_sample_limit, + event, + stratum, + dropped_reason, + ) + continue + + reconnect = state.reconnect_pending and not state.outage_active + duplicate_count = 0 + if not protected and _selected( + params["duplicate_probability"], + operator.operator_id, + stratum.stratum_id, + "duplicate", + event.source_event_id, + ): + duplicate_count += 1 + if ( + not protected + and reconnect + and duplicate_count + 1 < operator.fit_config.max_outputs_per_input + and _selected( + params["reconnect_duplicate_probability"], + operator.operator_id, + stratum.stratum_id, + "reconnect", + event.source_event_id, + ) + ): + duplicate_count += 1 + transformations.append("reconnect_duplicate") + duplicate_count = min( + duplicate_count, + operator.fit_config.max_outputs_per_input - 1, + ) + if duplicate_count: + transformations.append("duplicated") + + state.last_observed_time_ns = transformed_time + state.last_bid = transformed_bid + state.last_ask = transformed_ask + state.rate_bucket_count += 1 + duplicate_count + state.reconnect_pending = False + if owned: + reasons["retained"] += 1 + for duplicate_ordinal in range(duplicate_count + 1): + tentative.append( + { + "event": event, + "stratum": stratum, + "time": transformed_time, + "bid": transformed_bid, + "ask": transformed_ask, + "duplicate_ordinal": duplicate_ordinal, + "transformations": ( + () if protected else tuple(transformations) + ), + "protected": protected, + } + ) + + max_outputs = ( + operator.fit_config.max_input_events + * operator.fit_config.max_outputs_per_input + ) + if len(tentative) > max_outputs: + raise ValueError("observation application exceeds output event limit") + output_events = _finalize_output_events(operator, tentative) + carry_state = ObservationCarryStateV1( + operator_id=operator.operator_id, + symbol=symbol, + last_source_time_ns=state.last_source_time_ns, + last_observed_time_ns=state.last_observed_time_ns, + last_bid=state.last_bid, + last_ask=state.last_ask, + rate_bucket_start_ns=state.rate_bucket_start_ns, + rate_bucket_count=state.rate_bucket_count, + outage_bucket_start_ns=state.outage_bucket_start_ns, + outage_active=state.outage_active, + reconnect_pending=state.reconnect_pending, + ) + return ObservationApplicationResultV1( + operator_id=operator.operator_id, + window_id=window.window_id, + symbol=symbol, + application_mode=application_mode, + input_count=inspected_owned, + output_events=output_events, + reason_counts=dict(sorted(reasons.items())), + fallback_counts=dict(sorted(fallbacks.items())), + diagnostic_samples=tuple(samples), + samples_truncated=dropped_owned > len(samples), + carry_state=carry_state, + ) + + +def _fit_parameter( + name: str, + evidence: Sequence[ObservationFitEvidenceV1], + config: ObservationOperatorFitConfigV1, +) -> ObservationParameterEstimateV1: + selected = [item for item in evidence if name in item.parameter_values] + weighted = [ + (item.parameter_values[name], item.parameter_support_counts[name]) + for item in selected + ] + supported_weighted = [item for item in weighted if item[1] > 0] + source = supported_weighted or weighted + value = _weighted_median(source) + lower = min(item.parameter_lower_bounds[name] for item in selected) + upper = max(item.parameter_upper_bounds[name] for item in selected) + support = sum(item.parameter_support_counts[name] for item in selected) + status = ( + "supported" + if support >= config.min_parameter_support + else "unsupported" + ) + value = round(value, config.rounding_digits) + lower = min(value, round(lower, config.rounding_digits)) + upper = max(value, round(upper, config.rounding_digits)) + return ObservationParameterEstimateV1( + name=name, + value=value, + lower=lower, + upper=upper, + support_count=support, + evidence_count=len(selected), + support_status=status, + estimation_bases=tuple( + sorted({item.parameter_basis[name] for item in selected}) + ), + evidence_ids=tuple(item.evidence_id for item in selected), + provenance=tuple( + sorted( + { + path + for item in selected + for path in item.parameter_provenance[name] + } + ) + )[:MAX_OBSERVATION_PROVENANCE_PATHS], + ) + + +def _fit_diagnostics( + evidence: Sequence[ObservationFitEvidenceV1], + strata: Sequence[ObservationStratumV1], + config: ObservationOperatorFitConfigV1, +) -> ObservationFitDiagnosticsV1: + global_stratum = next( + stratum for stratum in strata if stratum.level == "global" + ) + global_parameters = global_stratum.parameter_map + residuals: dict[str, float] = {} + supports: dict[str, int] = {} + unsupported: list[str] = [] + for name in OBSERVATION_PARAMETER_NAMES: + parameter = global_parameters.get(name) + if parameter is None: + unsupported.append(name) + continue + supports[name] = parameter.support_count + if parameter.support_status == "unsupported": + unsupported.append(name) + values = [ + abs(item.parameter_values[name] - parameter.value) + for item in evidence + if name in item.parameter_values + ] + if values: + residuals[name] = round(median(values), config.rounding_digits) + sample_rows: list[Mapping[str, JSONValue]] = [] + for stratum in sorted(strata, key=lambda item: item.key): + if len(sample_rows) >= config.diagnostic_sample_limit: + break + sample_rows.append( + { + "stratum_id": stratum.stratum_id, + "key": stratum.key, + "status": stratum.status, + "support_count": stratum.support_count, + "fallback_keys": list(stratum.fallback_keys), + } + ) + return ObservationFitDiagnosticsV1( + evidence_count=len(evidence), + stratum_count=len(strata), + status_counts=dict(Counter(stratum.status for stratum in strata)), + parameter_support_counts=supports, + parameter_residual_medians=residuals, + unsupported_parameter_names=tuple(unsupported), + samples=tuple(sample_rows), + samples_truncated=len(strata) > len(sample_rows), + ) + + +def _finalize_output_events( + operator: ObservationOperatorV1, + tentative: Sequence[Mapping[str, Any]], +) -> tuple[ObservationOutputEventV1, ...]: + ordered = sorted( + tentative, + key=lambda row: ( + int(row["time"]), + cast(ObservationInputEventV1, row["event"]).event_time_ns, + cast(ObservationInputEventV1, row["event"]).event_sequence, + cast(ObservationInputEventV1, row["event"]).source_event_id, + int(row["duplicate_ordinal"]), + ), + ) + sequence_by_time: Counter[int] = Counter() + result: list[ObservationOutputEventV1] = [] + for row in ordered: + event = cast(ObservationInputEventV1, row["event"]) + stratum = cast(ObservationStratumV1, row["stratum"]) + timestamp = int(row["time"]) + sequence = sequence_by_time[timestamp] + sequence_by_time[timestamp] += 1 + result.append( + ObservationOutputEventV1( + source_event_id=event.source_event_id, + operator_id=operator.operator_id, + stratum_id=stratum.stratum_id, + symbol=event.symbol, + source_time_ns=event.event_time_ns, + observed_time_ns=timestamp, + observed_sequence=sequence, + bid=float(row["bid"]), + ask=float(row["ask"]), + duplicate_ordinal=int(row["duplicate_ordinal"]), + transformations=tuple( + str(value) + for value in cast(Sequence[Any], row["transformations"]) + ), + protected_anchor=bool(row["protected"]), + ) + ) + return tuple(result) + + +def _state_from_carry( + carry: ObservationCarryStateV1 | None, +) -> _MutableApplicationState: + if carry is None: + return _MutableApplicationState() + return _MutableApplicationState( + last_source_time_ns=carry.last_source_time_ns, + last_observed_time_ns=carry.last_observed_time_ns, + last_bid=carry.last_bid, + last_ask=carry.last_ask, + rate_bucket_start_ns=carry.rate_bucket_start_ns, + rate_bucket_count=carry.rate_bucket_count, + outage_bucket_start_ns=carry.outage_bucket_start_ns, + outage_active=carry.outage_active, + reconnect_pending=carry.reconnect_pending, + ) + + +def _validate_window_alignment( + window: ReconstructionWindowV1, parameters: Mapping[str, float] +) -> None: + for name in ("timestamp_quantum_ns", "batch_window_ns"): + alignment = int(parameters[name]) + if alignment <= 1: + continue + if alignment > MAX_OBSERVATION_WINDOW_ALIGNMENT_NS: + raise ValueError("observation alignment exceeds v1 limit") + if ( + window.core_start_ns % alignment != 0 + or window.core_end_ns % alignment != 0 + ): + raise ValueError( + "reconstruction window is not aligned to observation quantum" + ) + + +def _append_application_sample( + samples: list[dict[str, JSONValue]], + limit: int, + event: ObservationInputEventV1, + stratum: ObservationStratumV1, + reason: str, +) -> None: + if len(samples) >= limit: + return + samples.append( + { + "source_event_id": event.source_event_id, + "source_time_ns": event.event_time_ns, + "stratum_id": stratum.stratum_id, + "reason": reason, + } + ) + + +def _application_symbol( + events: Sequence[ObservationInputEventV1], + window: ReconstructionWindowV1, +) -> str: + if events: + return events[0].symbol + if len(window.symbols) != 1: + raise ValueError( + "empty observation input requires a single-symbol window" + ) + return _normalized_symbol(window.symbols[0]) + + +def _context_from_pattern( + pattern: Mapping[str, str], allowed_labels: set[str] +) -> ObservationContextV1: + epoch_id = pattern.get("epoch_id") + if epoch_id is None: + epoch_id = sorted(allowed_labels)[0] + return ObservationContextV1( + symbol=pattern.get("symbol", "GLOBAL"), + epoch_id=epoch_id, + state=pattern.get("state"), + session=pattern.get("session"), + event_tag=pattern.get("event_tag"), + ) + + +def _stratum_key(level: str, pattern: Mapping[str, str]) -> str: + if level == "global": + return "global" + encoded = "|".join( + f"{name}={_context_token(value)}" + for name, value in sorted(pattern.items()) + ) + return f"{level}|{encoded}" + + +def _context_token(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.:-]+", "_", value.strip()) + + +def _canonical_proxy_bounds( + values: Mapping[str, float], +) -> dict[str, tuple[float, float]]: + result: dict[str, tuple[float, float]] = {} + for name, value in values.items(): + if name in _PROBABILITY_PARAMETERS: + result[name] = (0.0, 1.0) + elif name == "price_precision_digits": + result[name] = (max(0.0, value - 1.0), min(16.0, value + 1.0)) + elif value == 0.0: + result[name] = (0.0, 0.0) + elif name in _INTEGER_PARAMETERS: + result[name] = ( + float(max(0, math.floor(value * 0.5))), + float(math.ceil(value * 2.0)), + ) + else: + result[name] = (max(0.0, value * 0.5), value * 2.0) + return result + + +def _canonical_parameter_provenance(name: str) -> tuple[str, ...]: + mapping = { + "retention_probability": ("identity_assumption.no_dense_denominator",), + "unchanged_retention_probability": ( + "feature_values.stale_repeat_rate", + ), + "timestamp_quantum_ns": ( + "feature_values.minimum_observed_interval_ms", + ), + "price_precision_digits": ("feature_values.price_precision_digits",), + "quote_transition_threshold": ( + "feature_values.price_precision_digits", + ), + "batch_window_ns": ("feature_values.minimum_observed_interval_ms",), + "duplicate_probability": ("feature_values.duplicate_timestamp_rate",), + "rate_cap_per_second": ("profile.tick_rate_per_hour",), + "burst_window_ns": ("profile.median_interarrival_ms",), + "quiet_gap_probability": ("feature_values.suspicious_gap_rate",), + "outage_window_ns": ("profile.p95_interarrival_ms",), + "reconnect_duplicate_probability": ( + "feature_values.duplicate_timestamp_rate", + ), + } + return mapping[name] + + +def _weighted_median(values: Sequence[tuple[float, int]]) -> float: + if not values: + raise ValueError("cannot fit an empty parameter") + ordered = sorted((value, max(1, weight)) for value, weight in values) + threshold = (sum(weight for _, weight in ordered) + 1) // 2 + cumulative = 0 + for value, weight in ordered: + cumulative += weight + if cumulative >= threshold: + return value + return ordered[-1][0] + + +def _selected(probability: float, *parts: str) -> bool: + selected = _probability(probability) + if selected <= 0.0: + return False + if selected >= 1.0: + return True + digest = hashlib.sha256("\x1f".join(parts).encode("utf-8")).digest() + value = int.from_bytes(digest[:8], "big") / float(2**64) + return value < selected + + +def _validate_operator_lineage( + lineage: Mapping[str, JSONValue], + *, + definition_id: str, + config_id: str, + source_hashes: tuple[str, ...], + evidence_count: int, +) -> None: + expected_keys = { + "feed_epoch_definition_id", + "config_id", + "evidence_count", + "evidence_ids", + "source_hashes", + "sources", + "lineage_sha256", + } + if set(lineage) != expected_keys: + raise ValueError("operator lineage fields differ from v1 contract") + if lineage.get("feed_epoch_definition_id") != definition_id: + raise ValueError("operator lineage epoch definition differs") + if lineage.get("config_id") != config_id: + raise ValueError("operator lineage config differs") + if lineage.get("evidence_count") != evidence_count: + raise ValueError("operator lineage evidence count differs") + evidence_ids = tuple( + _required_sha256_id(value, "evidence_id", prefix="observation-evidence") + for value in _sequence(lineage.get("evidence_ids")) + ) + if ( + len(evidence_ids) != evidence_count + or len(set(evidence_ids)) != evidence_count + ): + raise ValueError("operator lineage evidence IDs differ") + lineage_hashes = tuple( + sorted( + _required_sha256_id(value, "source_artifact_sha256") + for value in _sequence(lineage.get("source_hashes")) + ) + ) + if lineage_hashes != source_hashes: + raise ValueError("operator lineage source hashes differ") + sources = _sequence(lineage.get("sources")) + if len(sources) != evidence_count: + raise ValueError("operator lineage source count differs") + source_evidence_ids: list[str] = [] + source_artifact_hashes: set[str] = set() + expected_source_keys = { + "evidence_id", + "source_evidence_id", + "source_artifact_sha256", + "source_hash_basis", + "evidence_kind", + } + for value in sources: + source = _mapping(value) + if set(source) != expected_source_keys: + raise ValueError("operator lineage source fields differ") + source_evidence_ids.append( + _required_sha256_id( + source.get("evidence_id"), + "evidence_id", + prefix="observation-evidence", + ) + ) + _required_text(source.get("source_evidence_id")) + source_hash = _required_sha256_id( + source.get("source_artifact_sha256"), + "source_artifact_sha256", + ) + source_artifact_hashes.add(source_hash) + if _required_text(source.get("source_hash_basis")) not in ( + _SOURCE_HASH_BASES + ): + raise ValueError("operator lineage source hash basis differs") + if _required_text(source.get("evidence_kind")) not in _EVIDENCE_KINDS: + raise ValueError("operator lineage evidence kind differs") + if tuple(source_evidence_ids) != evidence_ids: + raise ValueError("operator lineage source evidence order differs") + if source_artifact_hashes != set(source_hashes): + raise ValueError("operator lineage source hash coverage differs") + supplied = _required_sha256_id( + lineage.get("lineage_sha256"), "lineage_sha256" + ) + material = { + key: value for key, value in lineage.items() if key != "lineage_sha256" + } + if supplied != _payload_sha256(cast(Mapping[str, JSONValue], material)): + raise ValueError("operator lineage hash differs") + + +def _parameter_value(name: str, value: Any) -> float: + result = _finite_float(value, name) + if name in _PROBABILITY_PARAMETERS and not 0.0 <= result <= 1.0: + raise ValueError(f"{name} must be between zero and one") + if name in _NONNEGATIVE_PARAMETERS and result < 0.0: + raise ValueError(f"{name} must be non-negative") + if name in _INTEGER_PARAMETERS and not result.is_integer(): + raise ValueError(f"{name} must be integral") + if name == "price_precision_digits" and result > 16: + raise ValueError("price_precision_digits exceeds sixteen") + if ( + name in {"timestamp_quantum_ns", "batch_window_ns"} + and result > MAX_OBSERVATION_WINDOW_ALIGNMENT_NS + ): + raise ValueError(f"{name} exceeds v1 alignment limit") + if ( + name in {"burst_window_ns", "outage_window_ns"} + and result > MAX_OBSERVATION_DURATION_NS + ): + raise ValueError(f"{name} exceeds v1 duration limit") + return result + + +def _stable_id(prefix: str, value: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(value).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _payload_sha256(value: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(value).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _required_sha256_id( + value: Any, name: str, *, prefix: str | None = None +) -> str: + text = _required_text(value) + expected_prefix = f"{prefix}:sha256:" if prefix is not None else "sha256:" + if prefix is not None and text.startswith(expected_prefix): + digest = text.removeprefix(expected_prefix) + if not _SHA256_RE.fullmatch(digest): + raise ValueError(f"{name} must be a sha256 identifier") + return expected_prefix + digest + digest = text.removeprefix("sha256:") + if not _SHA256_RE.fullmatch(digest): + raise ValueError(f"{name} must be a sha256 identifier") + return "sha256:" + digest + + +def _required_feed_epoch_definition_id(value: Any) -> str: + """Accept both readable v1 and active-time v2 definition identities.""" + text = _required_text(value) + for prefix in ("feed-epoch-definition", "feed-epoch-definition-v2"): + expected = f"{prefix}:sha256:" + if text.startswith(expected): + digest = text.removeprefix(expected) + if _SHA256_RE.fullmatch(digest): + return expected + digest + raise ValueError("feed_epoch_definition_id must be a sha256 identifier") + + +def _required_text(value: Any) -> str: + text = str(value or "").strip() + if not text or len(text) > MAX_OBSERVATION_TEXT_LENGTH: + raise ValueError("text value is empty or unbounded") + return text + + +def _normalized_symbol(value: Any) -> str: + symbol = _required_text(value).upper() + if not re.fullmatch(r"[A-Z0-9._:-]+", symbol): + raise ValueError("observation symbol contains unsupported characters") + return symbol + + +def _valid_period(value: str) -> bool: + if not _PERIOD_RE.fullmatch(value): + return False + if len(value) == 6 and not 1 <= int(value[4:]) <= 12: + return False + return True + + +def _optional_context_value(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip().lower() + return _context_token(text) if text else None + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool): + raise ValueError(f"{name} must be an integer") + try: + result = int(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must be an integer") from exc + if isinstance(value, float) and not value.is_integer(): + raise ValueError(f"{name} must be an integer") + return result + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a boolean") + return value + + +def _optional_strict_int(value: Any, name: str) -> int | None: + return None if value is None else _strict_int(value, name) + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + result = _strict_int(value, name) + if not minimum <= result <= maximum: + raise ValueError(f"{name} is outside [{minimum}, {maximum}]") + return result + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be finite") + try: + result = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must be finite") from exc + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _positive_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if result <= 0.0: + raise ValueError(f"{name} must be positive") + return result + + +def _optional_float(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + try: + result = float(value) + except (TypeError, ValueError, OverflowError): + return None + return result if math.isfinite(result) else None + + +def _optional_int(value: Any) -> int | None: + try: + return _strict_int(value, "value") + except ValueError: + return None + + +def _probability(value: Any) -> float: + return min(1.0, max(0.0, _finite_float(value, "probability"))) + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, Sequence) and not isinstance( + value, (str, bytes, bytearray) + ): + return value + return () + + +def _mapping_optional_text(data: Mapping[str, Any], name: str) -> str | None: + value = data.get(name) + return None if value is None else str(value) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _string_mapping(value: Any) -> dict[str, str]: + return {str(name): str(item) for name, item in _mapping(value).items()} + + +def _float_mapping(value: Any) -> dict[str, float]: + return { + str(name): _finite_float(item, str(name)) + for name, item in _mapping(value).items() + } + + +def _int_mapping(value: Any) -> dict[str, int]: + return { + str(name): _strict_int(item, str(name)) + for name, item in _mapping(value).items() + } + + +def _bounded_text_tuple( + values: Sequence[str], + name: str, + *, + limit: int = MAX_OBSERVATION_EVIDENCE, +) -> tuple[str, ...]: + result = tuple(dict.fromkeys(_required_text(value) for value in values)) + if not result or len(result) > limit: + raise ValueError(f"{name} values are empty or unbounded") + return result + + +def _validated_count_mapping( + values: Mapping[str, int], *, allowed_names: set[str] | None = None +) -> dict[str, int]: + result = { + _required_text(name): _bounded_int( + count, + f"{name} count", + 0, + MAX_OBSERVATION_INPUT_EVENTS * MAX_OBSERVATION_OUTPUTS_PER_INPUT, + ) + for name, count in sorted(values.items()) + } + if allowed_names is not None and not set(result).issubset(allowed_names): + raise ValueError("observation count names differ from v1 contract") + return result diff --git a/src/histdatacom/synthetic/observation_calibration.py b/src/histdatacom/synthetic/observation_calibration.py new file mode 100644 index 00000000..dd730c3d --- /dev/null +++ b/src/histdatacom/synthetic/observation_calibration.py @@ -0,0 +1,2262 @@ +"""Real-evidence calibration and holdout certification for observation models. + +The version-one observation operator remains the replay surface. This module +adds the stricter version-two *claim* around it: parameters are estimated from +active-time epoch evidence relative to blocked dense reference windows, the +operator is exercised on later dense windows, and application readiness fails +closed when retention or another requested mechanism is not identifiable. + +Only bounded aggregate evidence is persisted. Dense reference rows and +degraded rows remain process-local. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import statistics +import time +from collections import Counter, defaultdict +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, cast + +from histdatacom.data_analytics.feed_epochs_v2 import ( + FEED_EPOCH_EVIDENCE_V2_SCHEMA_VERSION, + FeedEpochDefinitionV2, + FeedEpochEvidenceV2, +) +from histdatacom.resource_usage import peak_rss_bytes +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.benchmark import BenchmarkSplitKind +from histdatacom.synthetic.contracts import canonical_contract_json +from histdatacom.synthetic.observation import ( + MAX_OBSERVATION_INPUT_EVENTS, + OBSERVATION_PARAMETER_NAMES, + ObservationApplicationResultV1, + ObservationContextV1, + ObservationFitEvidenceV1, + ObservationInputEventV1, + ObservationOperatorFitConfigV1, + ObservationOperatorV1, + fit_observation_operator, +) +from histdatacom.synthetic.streaming import ReconstructionWindowV1 + +OBSERVATION_CALIBRATION_PROFILE_V2_SCHEMA_VERSION = ( + "histdatacom.observation-calibration-profile.v2" +) +OBSERVATION_CALIBRATION_TARGET_V2_SCHEMA_VERSION = ( + "histdatacom.observation-calibration-target.v2" +) +OBSERVATION_CALIBRATION_WINDOW_V2_SCHEMA_VERSION = ( + "histdatacom.observation-calibration-window.v2" +) +OBSERVATION_CALIBRATION_CAMPAIGN_V2_SCHEMA_VERSION = ( + "histdatacom.observation-calibration-campaign.v2" +) +OBSERVATION_CALIBRATION_ENGINE_ID = "histdatacom.observation-calibration" +OBSERVATION_CALIBRATION_ENGINE_VERSION = "2.1.0" + +OBSERVATION_UPDATE_STATES = ( + "update_bid_only", + "update_ask_only", + "update_joint", + "update_unchanged", +) +OBSERVATION_CALIBRATION_SESSIONS = ( + "asia", + "london", + "new_york", + "off_session", +) +OBSERVATION_CALIBRATION_SPLITS = ( + BenchmarkSplitKind.CALIBRATION, + BenchmarkSplitKind.VALIDATION, + BenchmarkSplitKind.FINAL_HOLDOUT, +) +OBSERVATION_CALIBRATION_REQUIRED_PARAMETERS = ( + "retention_probability", + "unchanged_retention_probability", + "timestamp_quantum_ns", + "price_precision_digits", + "duplicate_probability", + "burst_window_ns", +) +OBSERVATION_CALIBRATION_MECHANISMS = ( + "calendar_closure", + "archive_gap", + "retention", + "unchanged_filter", + "timestamp_quantization", + "batching", + "duplicate", + "rate_cap", + "quiet_gap", + "outage", + "reconnect", +) + +_STATUS_VALUES = {"supported", "bounded", "unsupported"} +_PROBABILITY_PARAMETERS = { + "retention_probability", + "unchanged_retention_probability", + "duplicate_probability", + "quiet_gap_probability", + "reconnect_duplicate_probability", +} +_INTEGER_PARAMETERS = { + "timestamp_quantum_ns", + "price_precision_digits", + "batch_window_ns", + "burst_window_ns", + "outage_window_ns", +} +_SPLIT_KEYS = tuple(item.value for item in OBSERVATION_CALIBRATION_SPLITS) +_SESSION_HOURS = { + "asia": (0, 7), + "london": (7, 12), + "new_york": (12, 21), + "off_session": (21, 24), +} +_UPDATE_FEATURES = { + "update_bid_only": "bid_only_rate", + "update_ask_only": "ask_only_rate", + "update_joint": "joint_move_rate", + "update_unchanged": "unchanged_rate", +} +_SESSION_FEATURES = { + name: f"session_activity_share_{name}" + for name in OBSERVATION_CALIBRATION_SESSIONS +} +_NEUTRAL_VALUES = { + "retention_probability": 1.0, + "unchanged_retention_probability": 1.0, + "timestamp_quantum_ns": 1.0, + "price_precision_digits": 16.0, + "quote_transition_threshold": 0.0, + "batch_window_ns": 0.0, + "duplicate_probability": 0.0, + "rate_cap_per_second": 0.0, + "burst_window_ns": 0.0, + "quiet_gap_probability": 0.0, + "outage_window_ns": 0.0, + "reconnect_duplicate_probability": 0.0, +} + + +@dataclass(frozen=True, slots=True) +class ObservationCalibrationProfileV2: + """Blocked-time, resource, and acceptance policy for one calibration.""" + + split_periods: Mapping[str, str] = field(default_factory=dict) + sessions: tuple[str, ...] = OBSERVATION_CALIBRATION_SESSIONS[:3] + max_events_per_window: int = 4096 + minimum_events_per_window: int = 512 + max_source_bytes: int = 2 * 1024**3 + max_runtime_seconds: float = 600.0 + max_peak_memory_bytes: int = 2 * 1024**3 + retention_tolerance: float = 0.06 + duplicate_tolerance: float = 0.03 + timestamp_tolerance: float = 0.06 + update_mix_l1_tolerance: float = 0.35 + rounding_digits: int = 8 + profile_id: str = "" + schema_version: str = OBSERVATION_CALIBRATION_PROFILE_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != OBSERVATION_CALIBRATION_PROFILE_V2_SCHEMA_VERSION + ): + raise ValueError("unsupported observation calibration profile") + periods = { + str(key): str(value) for key, value in self.split_periods.items() + } + if periods: + if tuple(sorted(periods)) != tuple(sorted(_SPLIT_KEYS)): + raise ValueError( + "calibration split periods must define all splits" + ) + if any(not _valid_month(value) for value in periods.values()): + raise ValueError("calibration split periods must use YYYYMM") + ordered = tuple(periods[key] for key in _SPLIT_KEYS) + if tuple(sorted(ordered)) != ordered or len(set(ordered)) != 3: + raise ValueError( + "calibration split periods must be chronological" + ) + sessions = tuple(dict.fromkeys(str(value) for value in self.sessions)) + if not sessions or any( + value not in _SESSION_HOURS for value in sessions + ): + raise ValueError("unsupported observation calibration session") + if ( + not 64 + <= self.minimum_events_per_window + <= self.max_events_per_window + ): + raise ValueError("calibration minimum window size is invalid") + if not self.max_events_per_window <= MAX_OBSERVATION_INPUT_EVENTS: + raise ValueError( + "calibration window exceeds observation input bound" + ) + if not 1 <= self.max_source_bytes <= 64 * 1024**3: + raise ValueError("calibration source byte bound is invalid") + if not 1.0 <= self.max_runtime_seconds <= 86_400.0: + raise ValueError("calibration runtime bound is invalid") + if not 64 * 1024**2 <= self.max_peak_memory_bytes <= 64 * 1024**3: + raise ValueError("calibration memory bound is invalid") + for name in ( + "retention_tolerance", + "duplicate_tolerance", + "timestamp_tolerance", + "update_mix_l1_tolerance", + ): + value = _finite_float(getattr(self, name), name) + if not 0.0 < value <= 1.0: + raise ValueError(f"{name} must be in (0, 1]") + if not 0 <= self.rounding_digits <= 12: + raise ValueError("rounding_digits must be between zero and twelve") + object.__setattr__(self, "split_periods", dict(sorted(periods.items()))) + object.__setattr__(self, "sessions", sessions) + expected = _stable_id( + "observation-calibration-profile-v2", self.identity_payload() + ) + if self.profile_id and self.profile_id != expected: + raise ValueError("observation calibration profile ID differs") + object.__setattr__(self, "profile_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "engine_id": OBSERVATION_CALIBRATION_ENGINE_ID, + "engine_version": OBSERVATION_CALIBRATION_ENGINE_VERSION, + "split_periods": dict(self.split_periods), + "sessions": list(self.sessions), + "max_events_per_window": self.max_events_per_window, + "minimum_events_per_window": self.minimum_events_per_window, + "max_source_bytes": self.max_source_bytes, + "max_runtime_seconds": self.max_runtime_seconds, + "max_peak_memory_bytes": self.max_peak_memory_bytes, + "retention_tolerance": self.retention_tolerance, + "duplicate_tolerance": self.duplicate_tolerance, + "timestamp_tolerance": self.timestamp_tolerance, + "update_mix_l1_tolerance": self.update_mix_l1_tolerance, + "rounding_digits": self.rounding_digits, + "dense_intermediate_policy": "process_local_bounded_rows", + "persisted_evidence_policy": "aggregate_only", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "profile_id": self.profile_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ObservationCalibrationProfileV2": + if ( + data.get("engine_id") != OBSERVATION_CALIBRATION_ENGINE_ID + or data.get("engine_version") + != OBSERVATION_CALIBRATION_ENGINE_VERSION + ): + raise ValueError("unsupported observation calibration engine") + return cls( + schema_version=str(data.get("schema_version", "")), + split_periods={ + str(key): str(value) + for key, value in _mapping(data.get("split_periods")).items() + }, + sessions=_string_tuple(data.get("sessions")), + max_events_per_window=_strict_int( + data.get("max_events_per_window"), "max_events_per_window" + ), + minimum_events_per_window=_strict_int( + data.get("minimum_events_per_window"), + "minimum_events_per_window", + ), + max_source_bytes=_strict_int( + data.get("max_source_bytes"), "max_source_bytes" + ), + max_runtime_seconds=_finite_float( + data.get("max_runtime_seconds"), "max_runtime_seconds" + ), + max_peak_memory_bytes=_strict_int( + data.get("max_peak_memory_bytes"), + "max_peak_memory_bytes", + ), + retention_tolerance=_finite_float( + data.get("retention_tolerance"), "retention_tolerance" + ), + duplicate_tolerance=_finite_float( + data.get("duplicate_tolerance"), "duplicate_tolerance" + ), + timestamp_tolerance=_finite_float( + data.get("timestamp_tolerance"), "timestamp_tolerance" + ), + update_mix_l1_tolerance=_finite_float( + data.get("update_mix_l1_tolerance"), + "update_mix_l1_tolerance", + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + profile_id=str(data.get("profile_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationCalibrationTargetV2: + """One symbol/epoch target with explicit identifiability and refusals.""" + + symbol: str + epoch_label: str + reference_epoch_label: str + calibration_end_period: str + parameter_values: Mapping[str, float] + parameter_lower_bounds: Mapping[str, float] + parameter_upper_bounds: Mapping[str, float] + parameter_support_counts: Mapping[str, int] + parameter_status: Mapping[str, str] + parameter_reasons: Mapping[str, str] + target_statistics: Mapping[str, JSONValue] + mechanism_diagnostics: Mapping[str, str] + source_evidence_ids: tuple[str, ...] + source_hashes: tuple[str, ...] + target_id: str = "" + schema_version: str = OBSERVATION_CALIBRATION_TARGET_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != OBSERVATION_CALIBRATION_TARGET_V2_SCHEMA_VERSION + ): + raise ValueError("unsupported observation calibration target") + symbol = str(self.symbol).strip().upper() + if not symbol: + raise ValueError("calibration target requires a symbol") + epoch = _required_text(self.epoch_label, "epoch_label") + reference = _required_text( + self.reference_epoch_label, "reference_epoch_label" + ) + if not _valid_month(self.calibration_end_period): + raise ValueError("calibration target period must use YYYYMM") + names = set(OBSERVATION_PARAMETER_NAMES) + maps = ( + self.parameter_values, + self.parameter_lower_bounds, + self.parameter_upper_bounds, + self.parameter_support_counts, + self.parameter_status, + self.parameter_reasons, + ) + if any(set(value) != names for value in maps): + raise ValueError("calibration target parameter keys differ") + values: dict[str, float] = {} + lowers: dict[str, float] = {} + uppers: dict[str, float] = {} + supports: dict[str, int] = {} + statuses: dict[str, str] = {} + reasons: dict[str, str] = {} + for name in OBSERVATION_PARAMETER_NAMES: + value = _calibration_parameter_value( + name, self.parameter_values[name] + ) + lower = _calibration_parameter_value( + name, self.parameter_lower_bounds[name] + ) + upper = _calibration_parameter_value( + name, self.parameter_upper_bounds[name] + ) + if not lower <= value <= upper: + raise ValueError("calibration parameter lies outside bounds") + status = str(self.parameter_status[name]) + if status not in _STATUS_VALUES: + raise ValueError( + "unsupported calibration identifiability status" + ) + support = _strict_int( + self.parameter_support_counts[name], f"{name} support" + ) + if support < 0: + raise ValueError("calibration parameter support is negative") + if status == "supported" and support == 0: + raise ValueError( + "supported calibration parameter has no support" + ) + values[name] = value + lowers[name] = lower + uppers[name] = upper + supports[name] = support + statuses[name] = status + reasons[name] = _required_text( + self.parameter_reasons[name], f"{name} reason" + ) + mechanisms = { + str(key): str(value) + for key, value in self.mechanism_diagnostics.items() + } + if set(mechanisms) != set(OBSERVATION_CALIBRATION_MECHANISMS): + raise ValueError("calibration mechanism diagnostics differ") + evidence_ids = tuple(sorted(dict.fromkeys(self.source_evidence_ids))) + hashes = tuple(sorted(dict.fromkeys(self.source_hashes))) + if not evidence_ids or not hashes: + raise ValueError("calibration target requires source lineage") + object.__setattr__(self, "symbol", symbol) + object.__setattr__(self, "epoch_label", epoch) + object.__setattr__(self, "reference_epoch_label", reference) + object.__setattr__(self, "parameter_values", values) + object.__setattr__(self, "parameter_lower_bounds", lowers) + object.__setattr__(self, "parameter_upper_bounds", uppers) + object.__setattr__(self, "parameter_support_counts", supports) + object.__setattr__(self, "parameter_status", statuses) + object.__setattr__(self, "parameter_reasons", reasons) + object.__setattr__( + self, "target_statistics", dict(self.target_statistics) + ) + object.__setattr__(self, "mechanism_diagnostics", mechanisms) + object.__setattr__(self, "source_evidence_ids", evidence_ids) + object.__setattr__(self, "source_hashes", hashes) + expected = _stable_id( + "observation-calibration-target-v2", self.identity_payload() + ) + if self.target_id and self.target_id != expected: + raise ValueError("observation calibration target ID differs") + object.__setattr__(self, "target_id", expected) + + @property + def unsupported_parameters(self) -> tuple[str, ...]: + return tuple( + name + for name in OBSERVATION_PARAMETER_NAMES + if self.parameter_status[name] != "supported" + ) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "epoch_label": self.epoch_label, + "reference_epoch_label": self.reference_epoch_label, + "calibration_end_period": self.calibration_end_period, + "parameter_values": dict(self.parameter_values), + "parameter_lower_bounds": dict(self.parameter_lower_bounds), + "parameter_upper_bounds": dict(self.parameter_upper_bounds), + "parameter_support_counts": dict(self.parameter_support_counts), + "parameter_status": dict(self.parameter_status), + "parameter_reasons": dict(self.parameter_reasons), + "target_statistics": dict(self.target_statistics), + "mechanism_diagnostics": dict(self.mechanism_diagnostics), + "source_evidence_ids": list(self.source_evidence_ids), + "source_hashes": list(self.source_hashes), + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "target_id": self.target_id, + "unsupported_parameters": list(self.unsupported_parameters), + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ObservationCalibrationTargetV2": + return cls( + schema_version=str(data.get("schema_version", "")), + symbol=str(data.get("symbol", "")), + epoch_label=str(data.get("epoch_label", "")), + reference_epoch_label=str(data.get("reference_epoch_label", "")), + calibration_end_period=str(data.get("calibration_end_period", "")), + parameter_values=_float_mapping(data.get("parameter_values")), + parameter_lower_bounds=_float_mapping( + data.get("parameter_lower_bounds") + ), + parameter_upper_bounds=_float_mapping( + data.get("parameter_upper_bounds") + ), + parameter_support_counts=_int_mapping( + data.get("parameter_support_counts") + ), + parameter_status=_string_mapping(data.get("parameter_status")), + parameter_reasons=_string_mapping(data.get("parameter_reasons")), + target_statistics=_mapping(data.get("target_statistics")), + mechanism_diagnostics=_string_mapping( + data.get("mechanism_diagnostics") + ), + source_evidence_ids=_string_tuple(data.get("source_evidence_ids")), + source_hashes=_string_tuple(data.get("source_hashes")), + target_id=str(data.get("target_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationCalibrationWindowV2: + """Bounded aggregate result for one applied dense-reference window.""" + + split_kind: BenchmarkSplitKind + symbol: str + period: str + session: str + epoch_label: str + source_artifact_sha256: str + input_count: int + output_count: int + retained_source_count: int + target_metrics: Mapping[str, float] + observed_metrics: Mapping[str, float] + absolute_errors: Mapping[str, float] + tolerances: Mapping[str, float] + reason_counts: Mapping[str, int] + transformation_counts: Mapping[str, int] + passed: bool + failure_reasons: tuple[str, ...] + window_id: str = "" + schema_version: str = OBSERVATION_CALIBRATION_WINDOW_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != OBSERVATION_CALIBRATION_WINDOW_V2_SCHEMA_VERSION + ): + raise ValueError("unsupported observation calibration window") + split = BenchmarkSplitKind.from_value(self.split_kind) + if not _valid_month(self.period): + raise ValueError("calibration window period must use YYYYMM") + if self.session not in _SESSION_HOURS: + raise ValueError("unsupported calibration window session") + if ( + self.input_count <= 0 + or not 0 <= self.retained_source_count <= self.input_count + ): + raise ValueError("invalid calibration window input counts") + if self.output_count < self.retained_source_count: + raise ValueError( + "calibration output count precedes retained sources" + ) + targets = _finite_mapping(self.target_metrics) + observed = _finite_mapping(self.observed_metrics) + errors = _finite_mapping(self.absolute_errors) + tolerances = _finite_mapping(self.tolerances) + if ( + not targets + or set(targets) != set(observed) + or set(errors) != set(targets) + ): + raise ValueError("calibration window metric keys differ") + if any(name not in tolerances for name in errors): + raise ValueError("calibration window tolerance is missing") + failures = tuple( + dict.fromkeys(str(value) for value in self.failure_reasons) + ) + passed = bool(self.passed) + if passed == bool(failures): + raise ValueError( + "calibration window pass state differs from failures" + ) + object.__setattr__(self, "split_kind", split) + object.__setattr__(self, "symbol", str(self.symbol).upper()) + object.__setattr__(self, "target_metrics", targets) + object.__setattr__(self, "observed_metrics", observed) + object.__setattr__(self, "absolute_errors", errors) + object.__setattr__(self, "tolerances", tolerances) + object.__setattr__( + self, "reason_counts", _int_mapping(self.reason_counts) + ) + object.__setattr__( + self, + "transformation_counts", + _int_mapping(self.transformation_counts), + ) + object.__setattr__(self, "passed", passed) + object.__setattr__(self, "failure_reasons", failures) + expected = _stable_id( + "observation-calibration-window-v2", self.identity_payload() + ) + if self.window_id and self.window_id != expected: + raise ValueError("observation calibration window ID differs") + object.__setattr__(self, "window_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "split_kind": self.split_kind.value, + "symbol": self.symbol, + "period": self.period, + "session": self.session, + "epoch_label": self.epoch_label, + "source_artifact_sha256": self.source_artifact_sha256, + "input_count": self.input_count, + "output_count": self.output_count, + "retained_source_count": self.retained_source_count, + "target_metrics": dict(self.target_metrics), + "observed_metrics": dict(self.observed_metrics), + "absolute_errors": dict(self.absolute_errors), + "tolerances": dict(self.tolerances), + "reason_counts": dict(self.reason_counts), + "transformation_counts": dict(self.transformation_counts), + "passed": self.passed, + "failure_reasons": list(self.failure_reasons), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "window_id": self.window_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ObservationCalibrationWindowV2": + return cls( + schema_version=str(data.get("schema_version", "")), + split_kind=BenchmarkSplitKind.from_value( + str(data.get("split_kind", "")) + ), + symbol=str(data.get("symbol", "")), + period=str(data.get("period", "")), + session=str(data.get("session", "")), + epoch_label=str(data.get("epoch_label", "")), + source_artifact_sha256=str(data.get("source_artifact_sha256", "")), + input_count=_strict_int(data.get("input_count"), "input_count"), + output_count=_strict_int(data.get("output_count"), "output_count"), + retained_source_count=_strict_int( + data.get("retained_source_count"), "retained_source_count" + ), + target_metrics=_float_mapping(data.get("target_metrics")), + observed_metrics=_float_mapping(data.get("observed_metrics")), + absolute_errors=_float_mapping(data.get("absolute_errors")), + tolerances=_float_mapping(data.get("tolerances")), + reason_counts=_int_mapping(data.get("reason_counts")), + transformation_counts=_int_mapping( + data.get("transformation_counts") + ), + passed=_strict_bool(data.get("passed"), "passed"), + failure_reasons=_string_tuple(data.get("failure_reasons")), + window_id=str(data.get("window_id", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ObservationCalibrationCampaignV2: + """Immutable fit, blocked holdouts, lineage, and fail-closed readiness.""" + + feed_epoch_definition_id: str + calibration_corpus_sha256: str + profile: ObservationCalibrationProfileV2 + operator: ObservationOperatorV1 + targets: tuple[ObservationCalibrationTargetV2, ...] + fit_evidence: tuple[ObservationFitEvidenceV1, ...] + windows: tuple[ObservationCalibrationWindowV2, ...] + readiness_status: str + readiness_reasons: tuple[str, ...] + runtime_seconds: float + peak_memory_bytes: int + campaign_id: str = "" + schema_version: str = OBSERVATION_CALIBRATION_CAMPAIGN_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != OBSERVATION_CALIBRATION_CAMPAIGN_V2_SCHEMA_VERSION + ): + raise ValueError("unsupported observation calibration campaign") + if ( + self.feed_epoch_definition_id + != self.operator.feed_epoch_definition_id + ): + raise ValueError( + "calibration campaign definition differs from operator" + ) + corpus_hash = _sha256_text(self.calibration_corpus_sha256) + targets = tuple( + sorted( + self.targets, key=lambda item: (item.symbol, item.epoch_label) + ) + ) + evidence = tuple( + sorted(self.fit_evidence, key=lambda item: item.evidence_id) + ) + windows = tuple(sorted(self.windows, key=lambda item: item.window_id)) + if not targets or not evidence or not windows: + raise ValueError( + "calibration campaign requires targets, evidence, and windows" + ) + status = str(self.readiness_status) + if status not in {"ready", "failed"}: + raise ValueError("unsupported observation calibration readiness") + reasons = tuple( + dict.fromkeys(str(value) for value in self.readiness_reasons) + ) + if (status == "ready") == bool(reasons): + raise ValueError("calibration readiness differs from reasons") + target_keys = {(item.symbol, item.epoch_label) for item in targets} + if any( + (item.symbol, item.epoch_label) not in target_keys + for item in windows + ): + raise ValueError("calibration window has no target") + runtime_seconds = _finite_float(self.runtime_seconds, "runtime_seconds") + peak_memory_bytes = _strict_int( + self.peak_memory_bytes, "peak_memory_bytes" + ) + if runtime_seconds < 0.0 or peak_memory_bytes < 0: + raise ValueError("calibration resource evidence is negative") + expected_reasons = _readiness_reasons( + targets, + windows, + self.profile, + runtime_seconds=runtime_seconds, + peak_memory_bytes=peak_memory_bytes, + ) + expected_status = "failed" if expected_reasons else "ready" + if status != expected_status or reasons != expected_reasons: + raise ValueError("calibration readiness evidence differs") + object.__setattr__(self, "calibration_corpus_sha256", corpus_hash) + object.__setattr__(self, "targets", targets) + object.__setattr__(self, "fit_evidence", evidence) + object.__setattr__(self, "windows", windows) + object.__setattr__(self, "readiness_status", status) + object.__setattr__(self, "readiness_reasons", reasons) + object.__setattr__(self, "runtime_seconds", runtime_seconds) + object.__setattr__(self, "peak_memory_bytes", peak_memory_bytes) + expected = _stable_id( + "observation-calibration-campaign-v2", self.identity_payload() + ) + if self.campaign_id and self.campaign_id != expected: + raise ValueError("observation calibration campaign ID differs") + object.__setattr__(self, "campaign_id", expected) + + @property + def valid_for_application(self) -> bool: + return self.readiness_status == "ready" + + def require_application_ready( + self, + context: ObservationContextV1, + *, + required_parameters: Sequence[ + str + ] = OBSERVATION_CALIBRATION_REQUIRED_PARAMETERS, + ) -> ObservationCalibrationTargetV2: + """Return the target or reject an unsupported calibration claim.""" + if not self.valid_for_application: + raise ValueError("observation calibration campaign is not ready") + target = next( + ( + item + for item in self.targets + if item.symbol == context.symbol + and item.epoch_label == context.epoch_id + ), + None, + ) + if target is None: + raise ValueError("observation context is not calibrated") + requested = tuple( + dict.fromkeys(str(value) for value in required_parameters) + ) + if any(value not in OBSERVATION_PARAMETER_NAMES for value in requested): + raise ValueError("unsupported requested observation parameter") + unsupported = tuple( + name + for name in requested + if target.parameter_status[name] != "supported" + ) + if unsupported: + raise ValueError( + "requested observation parameters are unsupported: " + + ", ".join(unsupported) + ) + if target.parameter_reasons["retention_probability"] == ( + "identity_without_dense_denominator" + ): + raise ValueError( + "identity retention without a dense denominator is unsafe" + ) + return target + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "feed_epoch_definition_id": self.feed_epoch_definition_id, + "calibration_corpus_sha256": self.calibration_corpus_sha256, + "profile": self.profile.to_dict(), + "operator": self.operator.to_dict(), + "targets": [item.to_dict() for item in self.targets], + "fit_evidence": [item.to_dict() for item in self.fit_evidence], + "windows": [item.to_dict() for item in self.windows], + "readiness_status": self.readiness_status, + "readiness_reasons": list(self.readiness_reasons), + "application_readiness_policy": { + "default_required_parameters": list( + OBSERVATION_CALIBRATION_REQUIRED_PARAMETERS + ), + "unsupported_parameter_policy": "reject_requested_mechanism", + "transition_epoch_policy": "reject_unfitted_context", + "identity_retention_without_dense_denominator": "forbidden", + }, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "campaign_id": self.campaign_id, + "valid_for_application": self.valid_for_application, + "runtime_seconds": self.runtime_seconds, + "peak_memory_bytes": self.peak_memory_bytes, + "resource_bounds": { + "max_events_per_window": self.profile.max_events_per_window, + "max_source_bytes": self.profile.max_source_bytes, + "max_runtime_seconds": self.profile.max_runtime_seconds, + "max_peak_memory_bytes": self.profile.max_peak_memory_bytes, + "window_count": len(self.windows), + "persisted_dense_rows": 0, + }, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ObservationCalibrationCampaignV2": + return cls( + schema_version=str(data.get("schema_version", "")), + feed_epoch_definition_id=str( + data.get("feed_epoch_definition_id", "") + ), + calibration_corpus_sha256=str( + data.get("calibration_corpus_sha256", "") + ), + profile=ObservationCalibrationProfileV2.from_dict( + _mapping(data.get("profile")) + ), + operator=ObservationOperatorV1.from_dict( + _mapping(data.get("operator")) + ), + targets=tuple( + ObservationCalibrationTargetV2.from_dict(_mapping(item)) + for item in _sequence(data.get("targets")) + ), + fit_evidence=tuple( + ObservationFitEvidenceV1.from_dict(_mapping(item)) + for item in _sequence(data.get("fit_evidence")) + ), + windows=tuple( + ObservationCalibrationWindowV2.from_dict(_mapping(item)) + for item in _sequence(data.get("windows")) + ), + readiness_status=str(data.get("readiness_status", "")), + readiness_reasons=_string_tuple(data.get("readiness_reasons")), + runtime_seconds=_finite_float( + data.get("runtime_seconds"), "runtime_seconds" + ), + peak_memory_bytes=_strict_int( + data.get("peak_memory_bytes"), "peak_memory_bytes" + ), + campaign_id=str(data.get("campaign_id", "")), + ) + + +def calibrate_historical_observation_operators( + evidence: Sequence[FeedEpochEvidenceV2], + *, + epoch_definition: FeedEpochDefinitionV2, + profile: ObservationCalibrationProfileV2 | None = None, +) -> ObservationCalibrationCampaignV2: + """Fit and certify one bounded real-evidence observation campaign.""" + started = time.perf_counter() + selected = profile or ObservationCalibrationProfileV2() + if not epoch_definition.valid_for_observation_models: + raise ValueError("feed epoch definition has not passed stability") + ordered = tuple( + sorted(evidence, key=lambda item: (item.period, item.symbol)) + ) + if not ordered: + raise ValueError("observation calibration evidence is empty") + if any(item.symbol not in epoch_definition.symbols for item in ordered): + raise ValueError( + "observation calibration evidence is outside symbol scope" + ) + split_periods = dict(selected.split_periods) or _automatic_split_periods( + ordered, epoch_definition + ) + selected = ObservationCalibrationProfileV2( + split_periods=split_periods, + sessions=selected.sessions, + max_events_per_window=selected.max_events_per_window, + minimum_events_per_window=selected.minimum_events_per_window, + max_source_bytes=selected.max_source_bytes, + max_runtime_seconds=selected.max_runtime_seconds, + max_peak_memory_bytes=selected.max_peak_memory_bytes, + retention_tolerance=selected.retention_tolerance, + duplicate_tolerance=selected.duplicate_tolerance, + timestamp_tolerance=selected.timestamp_tolerance, + update_mix_l1_tolerance=selected.update_mix_l1_tolerance, + rounding_digits=selected.rounding_digits, + ) + reference_epoch = epoch_definition.epochs[-1] + calibration_period = split_periods[BenchmarkSplitKind.CALIBRATION.value] + source_rows = _fit_source_evidence( + ordered, + epoch_definition=epoch_definition, + reference_epoch_label=reference_epoch.label, + calibration_end_period=calibration_period, + ) + corpus_payload: dict[str, JSONValue] = { + "feed_epoch_definition_id": epoch_definition.definition_id, + "profile_id": selected.profile_id, + "calibration_end_period": calibration_period, + "evidence_ids": [ + cast(JSONValue, item.evidence_id) for item in source_rows + ], + "source_hashes": [ + cast(JSONValue, item) + for item in sorted( + {row.source_artifact_sha256 for row in source_rows} + ) + ], + "claim": "relative_dense_reference_observation_calibration", + } + corpus_hash = ( + "sha256:" + + hashlib.sha256( + canonical_contract_json(corpus_payload).encode("utf-8") + ).hexdigest() + ) + targets = _build_targets( + source_rows, + epoch_definition=epoch_definition, + reference_epoch_label=reference_epoch.label, + calibration_end_period=calibration_period, + rounding_digits=selected.rounding_digits, + ) + fit_evidence = _build_fit_evidence( + targets, + epoch_definition=epoch_definition, + corpus_hash=corpus_hash, + ) + operator = fit_observation_operator( + fit_evidence, + epoch_definition=epoch_definition, + config=ObservationOperatorFitConfigV1( + min_stratum_support=16, + min_parameter_support=1, + min_supported_parameters=len( + OBSERVATION_CALIBRATION_REQUIRED_PARAMETERS + ), + max_input_events=selected.max_events_per_window, + max_strata=512, + rounding_digits=selected.rounding_digits, + ), + ) + windows = _evaluate_windows( + ordered, + split_periods=split_periods, + profile=selected, + operator=operator, + targets=targets, + ) + peak = peak_rss_bytes() + runtime_seconds = round(time.perf_counter() - started, 6) + reasons = _readiness_reasons( + targets, + windows, + selected, + runtime_seconds=runtime_seconds, + peak_memory_bytes=int(peak), + ) + return ObservationCalibrationCampaignV2( + feed_epoch_definition_id=epoch_definition.definition_id, + calibration_corpus_sha256=corpus_hash, + profile=selected, + operator=operator, + targets=targets, + fit_evidence=fit_evidence, + windows=windows, + readiness_status="failed" if reasons else "ready", + readiness_reasons=reasons, + runtime_seconds=runtime_seconds, + peak_memory_bytes=int(peak), + ) + + +def estimate_paired_observation_evidence( + reference_events: Sequence[ObservationInputEventV1], + observed: ObservationApplicationResultV1, + *, + period: str, + source_artifact_sha256: str, +) -> tuple[ObservationFitEvidenceV1, ...]: + """Recover bounded state-conditioned parameters from an explicit pair. + + This estimator is intentionally narrow. It recovers only mechanisms with + a directly observed denominator: state retention, timestamp grid, and + duplicate emission. It does not infer outage or archive gaps from silence. + """ + if not _valid_month(period): + raise ValueError("paired calibration period must use YYYYMM") + source_hash = _sha256_text(source_artifact_sha256) + by_id = {item.source_event_id: item for item in reference_events} + if len(by_id) != len(reference_events) or not by_id: + raise ValueError("paired calibration reference identities are invalid") + if any( + item.source_event_id not in by_id for item in observed.output_events + ): + raise ValueError("paired observation has an unknown source identity") + outputs_by_id: Counter[str] = Counter( + item.source_event_id for item in observed.output_events + ) + grouped: dict[ + tuple[str, str, str | None, str | None], list[ObservationInputEventV1] + ] = defaultdict(list) + for item in reference_events: + key = ( + item.context.symbol, + item.context.epoch_id, + item.context.state, + item.context.session, + ) + grouped[key].append(item) + result: list[ObservationFitEvidenceV1] = [] + for items in grouped.values(): + context = items[0].context + retained = [ + item for item in items if outputs_by_id[item.source_event_id] > 0 + ] + output_rows = [ + output + for output in observed.output_events + if output.source_event_id + in {item.source_event_id for item in items} + ] + values = { + "retention_probability": len(retained) / len(items), + "timestamp_quantum_ns": float(_timestamp_grid_ns(output_rows)), + "duplicate_probability": ( + max(0, len(output_rows) - len(retained)) / max(1, len(retained)) + ), + } + supports = { + "retention_probability": len(items), + "timestamp_quantum_ns": len(output_rows), + "duplicate_probability": len(retained), + } + bases = {name: "paired_dense_denominator" for name in values} + provenance = { + "retention_probability": ( + "pair.reference.source_event_id", + "pair.observed.source_event_id", + ), + "timestamp_quantum_ns": ("pair.observed.observed_time_ns",), + "duplicate_probability": ("pair.observed.duplicate_ordinal",), + } + start = min(item.event_time_ns for item in items) + end = max(item.event_time_ns for item in items) + result.append( + ObservationFitEvidenceV1( + context=context, + period=period, + start_timestamp_ns=start, + end_timestamp_ns=end, + source_evidence_id=_stable_id( + "paired-calibration-source", + { + "period": period, + "context": context.to_dict(), + "input_count": len(items), + "output_count": len(output_rows), + }, + ), + source_artifact_sha256=source_hash, + source_hash_basis="paired_calibration_artifact_sha256", + evidence_kind="paired_calibration", + parameter_values=values, + parameter_lower_bounds=values, + parameter_upper_bounds=values, + parameter_support_counts=supports, + parameter_basis=bases, + parameter_provenance=provenance, + ) + ) + return tuple(sorted(result, key=lambda item: item.evidence_id)) + + +def write_observation_calibration_campaign( + campaign: ObservationCalibrationCampaignV2, + directory: str | Path, +) -> Mapping[str, ArtifactRef]: + """Persist compact campaign, operator, and fit-evidence artifacts.""" + root = Path(directory).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + payloads: dict[str, Mapping[str, JSONValue]] = { + "campaign": campaign.to_dict(), + "operator": campaign.operator.to_dict(), + "fit_evidence": { + "schema_version": "histdatacom.observation-calibration-fit-evidence.v2", + "campaign_id": campaign.campaign_id, + "calibration_corpus_sha256": campaign.calibration_corpus_sha256, + "evidence_count": len(campaign.fit_evidence), + "evidence": [item.to_dict() for item in campaign.fit_evidence], + }, + } + artifacts: dict[str, ArtifactRef] = {} + for name, payload in payloads.items(): + encoded = canonical_contract_json(payload).encode("utf-8") + b"\n" + path = root / f"observation-calibration-v2-{name}.json" + path.write_bytes(encoded) + artifacts[name] = ArtifactRef( + kind=f"observation_calibration_v2_{name}", + path=str(path), + size_bytes=len(encoded), + sha256=hashlib.sha256(encoded).hexdigest(), + metadata={ + "schema_version": str(payload.get("schema_version", "")), + "campaign_id": campaign.campaign_id, + "operator_id": campaign.operator.operator_id, + }, + ) + return artifacts + + +def read_observation_calibration_campaign( + path: str | Path, +) -> ObservationCalibrationCampaignV2: + """Read and strictly verify a persisted v2 campaign.""" + source = Path(path).expanduser().resolve() + if source.stat().st_size > 64 * 1024 * 1024: + raise ValueError("observation calibration artifact exceeds size bound") + payload = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise ValueError("observation calibration artifact must be an object") + return ObservationCalibrationCampaignV2.from_dict(payload) + + +def read_feed_epoch_evidence_v2( + path: str | Path, +) -> tuple[FeedEpochEvidenceV2, ...]: + """Read the bounded evidence artifact produced by ``feed-epochs-v2``.""" + source = Path(path).expanduser().resolve() + if source.stat().st_size > 64 * 1024 * 1024: + raise ValueError("feed epoch evidence artifact exceeds size bound") + payload = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise ValueError("feed epoch evidence artifact must be an object") + if payload.get("schema_version") != FEED_EPOCH_EVIDENCE_V2_SCHEMA_VERSION: + raise ValueError("unsupported feed epoch evidence artifact schema") + values = tuple( + FeedEpochEvidenceV2.from_dict(_mapping(item)) + for item in _sequence(payload.get("evidence")) + ) + if not values or _strict_int( + payload.get("evidence_count"), "evidence_count" + ) != len(values): + raise ValueError("feed epoch evidence artifact count differs") + return values + + +def _fit_source_evidence( + evidence: Sequence[FeedEpochEvidenceV2], + *, + epoch_definition: FeedEpochDefinitionV2, + reference_epoch_label: str, + calibration_end_period: str, +) -> tuple[FeedEpochEvidenceV2, ...]: + rows: list[FeedEpochEvidenceV2] = [] + for item in evidence: + assignment = epoch_definition.assign( + symbol=item.symbol, + timestamp_utc_ms=( + item.start_timestamp_utc_ms + item.end_timestamp_utc_ms + ) + // 2, + ) + if assignment.assignment_kind != "epoch": + continue + if ( + assignment.label == reference_epoch_label + and item.period > calibration_end_period + ): + continue + rows.append(item) + if not rows: + raise ValueError("no in-epoch evidence is available for calibration") + return tuple(rows) + + +def _build_targets( + evidence: Sequence[FeedEpochEvidenceV2], + *, + epoch_definition: FeedEpochDefinitionV2, + reference_epoch_label: str, + calibration_end_period: str, + rounding_digits: int, +) -> tuple[ObservationCalibrationTargetV2, ...]: + grouped: dict[tuple[str, str], list[FeedEpochEvidenceV2]] = defaultdict( + list + ) + for item in evidence: + assignment = epoch_definition.assign( + symbol=item.symbol, + timestamp_utc_ms=( + item.start_timestamp_utc_ms + item.end_timestamp_utc_ms + ) + // 2, + ) + if assignment.assignment_kind == "epoch": + grouped[(item.symbol, assignment.label)].append(item) + result: list[ObservationCalibrationTargetV2] = [] + for symbol in epoch_definition.symbols: + reference = grouped.get((symbol, reference_epoch_label), []) + if not reference: + raise ValueError( + f"dense reference evidence is missing for {symbol}" + ) + reference_stats = _aggregate_epoch_rows(reference) + for epoch in epoch_definition.epochs: + rows = grouped.get((symbol, epoch.label), []) + if not rows: + raise ValueError( + f"epoch evidence is missing for {symbol} {epoch.label}" + ) + stats = _aggregate_epoch_rows(rows) + rate = min( + 1.0, + stats["active_rate_per_hour"] + / max(1e-12, reference_stats["active_rate_per_hour"]), + ) + exact_second = stats["timestamp_exact_second_rate"] + quantum = _quantum_from_timestamp_features( + exact_second, + stats["timestamp_last_digit_entropy"], + ) + precision = int(round(stats["price_precision_digits"])) + unchanged_ratio = 1.0 + values = dict(_NEUTRAL_VALUES) + values.update( + { + "retention_probability": rate, + "unchanged_retention_probability": unchanged_ratio, + "timestamp_quantum_ns": float(quantum), + "price_precision_digits": float(max(0, min(16, precision))), + "quote_transition_threshold": 10.0 ** (-max(0, precision)), + "batch_window_ns": float( + quantum if exact_second >= 0.99 else 0 + ), + "duplicate_probability": min( + 1.0, stats["duplicate_timestamp_rate"] + ), + "rate_cap_per_second": stats["active_rate_per_hour"] + / 3600.0, + "burst_window_ns": float( + max( + 1, + int( + round( + stats["interarrival_median_ms"] * 1_000_000 + ) + ), + ) + ), + } + ) + lower = dict(values) + upper = dict(values) + target_rates = _feature_series( + rows, + "log_active_window_tick_rate_per_hour", + transform=math.expm1, + ) + reference_rates = _feature_series( + reference, + "log_active_window_tick_rate_per_hour", + transform=math.expm1, + ) + lower["retention_probability"] = min( + rate, + _quantile(target_rates, 0.25) + / max(1e-12, _quantile(reference_rates, 0.75)), + ) + upper["retention_probability"] = max( + rate, + min( + 1.0, + _quantile(target_rates, 0.75) + / max(1e-12, _quantile(reference_rates, 0.25)), + ), + ) + lower["unchanged_retention_probability"] = unchanged_ratio + upper["unchanged_retention_probability"] = unchanged_ratio + duplicate_values = _feature_series(rows, "duplicate_timestamp_rate") + lower["duplicate_probability"] = min( + values["duplicate_probability"], + _quantile(duplicate_values, 0.25), + ) + upper["duplicate_probability"] = max( + values["duplicate_probability"], + _quantile(duplicate_values, 0.75), + ) + precision_values = _feature_series(rows, "price_precision_digits") + lower["price_precision_digits"] = float( + max(0, math.floor(min(precision_values))) + ) + upper["price_precision_digits"] = float( + min(16, math.ceil(max(precision_values))) + ) + interval_values = _feature_series( + rows, + "log_interarrival_median_ms", + transform=lambda value: math.expm1(value) * 1_000_000, + ) + lower["burst_window_ns"] = float( + max(1, math.floor(_quantile(interval_values, 0.25))) + ) + upper["burst_window_ns"] = float( + max( + int(values["burst_window_ns"]), + math.ceil(_quantile(interval_values, 0.75)), + ) + ) + status = { + name: "unsupported" for name in OBSERVATION_PARAMETER_NAMES + } + for name in OBSERVATION_CALIBRATION_REQUIRED_PARAMETERS: + status[name] = "supported" + for name in ( + "quote_transition_threshold", + "batch_window_ns", + "rate_cap_per_second", + ): + status[name] = "bounded" + parameter_support_counts = { + name: ( + int(stats["row_count"]) + if status[name] in {"supported", "bounded"} + else 0 + ) + for name in OBSERVATION_PARAMETER_NAMES + } + reasons = { + "retention_probability": "relative_active_time_dense_denominator", + "unchanged_retention_probability": ( + "state_conditioned_retention_captures_unchanged_filter" + ), + "timestamp_quantum_ns": "empirical_timestamp_grid_features", + "price_precision_digits": "empirical_minimum_quote_step", + "quote_transition_threshold": "bounded_by_empirical_price_precision", + "batch_window_ns": "timestamp_quantization_and_batching_are_confounded", + "duplicate_probability": "empirical_duplicate_timestamp_rate", + "rate_cap_per_second": "upper_bound_from_active_time_rate", + "burst_window_ns": "empirical_active_interarrival_median", + "quiet_gap_probability": "silence_cannot_separate_archive_gap_from_omission", + "outage_window_ns": "no_dense_pair_for_historical_outage_duration", + "reconnect_duplicate_probability": "reconnect_identity_is_not_observed", + } + mechanisms = { + "calendar_closure": "excluded_by_shared_histdata_calendar", + "archive_gap": "unsupported_not_reclassified_as_delivery_omission", + "retention": "supported_relative_dense_denominator", + "unchanged_filter": "supported_as_state_conditioned_retention", + "timestamp_quantization": "supported_empirical_grid", + "batching": "bounded_confounded_with_quantization", + "duplicate": "supported_empirical_duplicate_timestamp_rate", + "rate_cap": "bounded_by_active_time_rate", + "quiet_gap": "unsupported_archive_gap_confounding", + "outage": "unsupported_without_paired_outage_evidence", + "reconnect": "unsupported_without_reconnect_identity", + } + target_statistics: dict[str, JSONValue] = { + "relative_retention": _round(rate, rounding_digits), + "timestamp_exact_second_rate": _round( + exact_second, rounding_digits + ), + "duplicate_timestamp_rate": _round( + stats["duplicate_timestamp_rate"], rounding_digits + ), + "update_mix": { + state_name: _round(stats[state_name], rounding_digits) + for state_name in OBSERVATION_UPDATE_STATES + }, + "session_mix": { + session: _round(stats[session], rounding_digits) + for session in OBSERVATION_CALIBRATION_SESSIONS + }, + "conditioning_support": { + "update_type": "supported", + "session": "supported", + "timestamp_precision": "supported", + "spread_state": "unsupported_no_conditional_epoch_denominator", + "activity_state": "unsupported_no_conditional_epoch_denominator", + "volatility_state": "unsupported_no_conditional_epoch_denominator", + }, + "target_row_count": int(stats["row_count"]), + "reference_row_count": int(reference_stats["row_count"]), + "uncertainty_basis": ("monthly_interquartile_empirical_bounds"), + } + result.append( + ObservationCalibrationTargetV2( + symbol=symbol, + epoch_label=epoch.label, + reference_epoch_label=reference_epoch_label, + calibration_end_period=calibration_end_period, + parameter_values={ + name: _round(value, rounding_digits) + for name, value in values.items() + }, + parameter_lower_bounds={ + name: _round(value, rounding_digits) + for name, value in lower.items() + }, + parameter_upper_bounds={ + name: _round(value, rounding_digits) + for name, value in upper.items() + }, + parameter_support_counts=parameter_support_counts, + parameter_status=status, + parameter_reasons=reasons, + target_statistics=target_statistics, + mechanism_diagnostics=mechanisms, + source_evidence_ids=tuple( + item.evidence_id for item in (*rows, *reference) + ), + source_hashes=tuple( + item.source_artifact_sha256 + for item in (*rows, *reference) + ), + ) + ) + return tuple(result) + + +def _build_fit_evidence( + targets: Sequence[ObservationCalibrationTargetV2], + *, + epoch_definition: FeedEpochDefinitionV2, + corpus_hash: str, +) -> tuple[ObservationFitEvidenceV1, ...]: + result: list[ObservationFitEvidenceV1] = [] + for target in targets: + target_update = _json_float_mapping( + target.target_statistics["update_mix"] + ) + target_session = _json_float_mapping( + target.target_statistics["session_mix"] + ) + reference_target = next( + item + for item in targets + if item.symbol == target.symbol + and item.epoch_label == target.reference_epoch_label + ) + reference_update = _json_float_mapping( + reference_target.target_statistics["update_mix"] + ) + reference_session = _json_float_mapping( + reference_target.target_statistics["session_mix"] + ) + desired = _conditioned_retention_grid( + _finite_float( + target.target_statistics["relative_retention"], + "relative_retention", + ), + target_update=target_update, + reference_update=reference_update, + target_session=target_session, + reference_session=reference_session, + ) + epoch = next( + item + for item in epoch_definition.epochs + if item.label == target.epoch_label + ) + supported_names = tuple( + name + for name in OBSERVATION_PARAMETER_NAMES + if target.parameter_status[name] == "supported" + ) + for state in OBSERVATION_UPDATE_STATES: + for session in OBSERVATION_CALIBRATION_SESSIONS: + values = { + name: target.parameter_values[name] + for name in supported_names + } + lowers = { + name: target.parameter_lower_bounds[name] + for name in supported_names + } + uppers = { + name: target.parameter_upper_bounds[name] + for name in supported_names + } + total_retention = desired[(state, session)] + retention = total_retention + values["retention_probability"] = retention + base_retention = max( + 1e-12, target.parameter_values["retention_probability"] + ) + lowers["retention_probability"] = max( + 0.0, + min( + retention, + retention + * target.parameter_lower_bounds["retention_probability"] + / base_retention, + ), + ) + uppers["retention_probability"] = min( + 1.0, + max( + retention, + retention + * target.parameter_upper_bounds["retention_probability"] + / base_retention, + ), + ) + support = min( + MAX_OBSERVATION_INPUT_EVENTS, + max( + 1, + int( + _finite_float( + target.target_statistics["target_row_count"], + "target_row_count", + ) + * target_update[state] + * target_session[session] + ), + ), + ) + supports = {name: support for name in supported_names} + bases = { + name: target.parameter_reasons[name] + for name in supported_names + } + provenance = { + name: ( + "feed_epoch_v2.feature_values", + f"calibration_target.{target.target_id}.{name}", + ) + for name in supported_names + } + result.append( + ObservationFitEvidenceV1( + context=ObservationContextV1( + symbol=target.symbol, + epoch_id=target.epoch_label, + state=state, + session=session, + ), + period=target.calibration_end_period, + start_timestamp_ns=epoch.start_timestamp_utc_ms + * 1_000_000, + end_timestamp_ns=epoch.end_timestamp_utc_ms * 1_000_000, + source_evidence_id=target.target_id, + source_artifact_sha256=corpus_hash, + source_hash_basis="paired_calibration_artifact_sha256", + evidence_kind="paired_calibration", + parameter_values=values, + parameter_lower_bounds=lowers, + parameter_upper_bounds=uppers, + parameter_support_counts=supports, + parameter_basis=bases, + parameter_provenance=provenance, + ) + ) + return tuple(result) + + +def _evaluate_windows( + evidence: Sequence[FeedEpochEvidenceV2], + *, + split_periods: Mapping[str, str], + profile: ObservationCalibrationProfileV2, + operator: ObservationOperatorV1, + targets: Sequence[ObservationCalibrationTargetV2], +) -> tuple[ObservationCalibrationWindowV2, ...]: + by_source = {(item.symbol, item.period): item for item in evidence} + result: list[ObservationCalibrationWindowV2] = [] + for split in OBSERVATION_CALIBRATION_SPLITS: + period = split_periods[split.value] + for symbol in sorted({item.symbol for item in targets}): + source = by_source.get((symbol, period)) + if source is None: + raise ValueError( + f"dense reference window is missing: {symbol} {period}" + ) + _verify_dense_source( + source, max_source_bytes=profile.max_source_bytes + ) + for session in profile.sessions: + rows = _sample_dense_rows( + source, session=session, profile=profile + ) + for target in ( + item for item in targets if item.symbol == symbol + ): + events = _observation_input_events( + rows, + symbol=symbol, + epoch_label=target.epoch_label, + session=session, + source_hash=source.source_artifact_sha256, + ) + window_start, window_end = _aligned_window_bounds( + events, operator + ) + window = ReconstructionWindowV1( + run_id=f"observation-calibration-v2:{split.value}", + ensemble_member_id=target.epoch_label, + symbols=(symbol,), + core_start_ns=window_start, + core_end_ns=window_end, + ) + applied = operator.degrade( + events, + window=window, + source_start=True, + ) + result.append( + _score_window( + split=split, + period=period, + session=session, + source=source, + target=target, + events=events, + applied=applied, + operator=operator, + profile=profile, + ) + ) + return tuple(result) + + +def _sample_dense_rows( + source: FeedEpochEvidenceV2, + *, + session: str, + profile: ObservationCalibrationProfileV2, +) -> tuple[tuple[int, float, float], ...]: + import polars as pl # pylint: disable=import-outside-toplevel + + start_hour, end_hour = _SESSION_HOURS[session] + lazy = pl.scan_ipc(source.source_path).select("datetime", "bid", "ask") + timestamp = pl.col("datetime").cast(pl.Int64) + hour = (timestamp // 3_600_000) % 24 + filtered = lazy.filter((hour >= start_hour) & (hour < end_hour)) + stride = max( + 1, source.row_count // max(1, profile.max_events_per_window * 4) + ) + sampled = ( + filtered.with_row_index("_calibration_row") + .filter((pl.col("_calibration_row") % stride) == 0) + .head(profile.max_events_per_window) + .select("datetime", "bid", "ask") + .collect(engine="streaming") + ) + rows = tuple( + (int(timestamp_ms), float(bid), float(ask)) + for timestamp_ms, bid, ask in sampled.iter_rows() + if float(bid) > 0 and float(ask) >= float(bid) + ) + if len(rows) < profile.minimum_events_per_window: + raise ValueError( + f"dense reference window is too short: {source.symbol} " + f"{source.period} {session} ({len(rows)})" + ) + return rows + + +def _verify_dense_source( + source: FeedEpochEvidenceV2, *, max_source_bytes: int +) -> None: + """Verify selected dense bytes against the immutable epoch evidence.""" + path = Path(source.source_path) + if source.source_size_bytes > max_source_bytes: + raise ValueError( + "dense reference source exceeds calibration byte bound" + ) + if path.stat().st_size != source.source_size_bytes: + raise ValueError( + "dense reference source size differs from epoch evidence" + ) + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + actual = "sha256:" + digest.hexdigest() + if actual != source.source_artifact_sha256: + raise ValueError( + "dense reference source hash differs from epoch evidence" + ) + + +def _observation_input_events( + rows: Sequence[tuple[int, float, float]], + *, + symbol: str, + epoch_label: str, + session: str, + source_hash: str, +) -> tuple[ObservationInputEventV1, ...]: + result: list[ObservationInputEventV1] = [] + previous: tuple[float, float] | None = None + for ordinal, (timestamp_ms, bid, ask) in enumerate(rows): + state = _update_state(previous, bid, ask) + source_event_id = _stable_id( + "calibration-market-event", + { + "source_hash": source_hash, + "epoch_label": epoch_label, + "session": session, + "ordinal": ordinal, + "timestamp_ms": timestamp_ms, + "bid": bid, + "ask": ask, + }, + ) + result.append( + ObservationInputEventV1( + source_event_id=source_event_id, + symbol=symbol, + event_time_ns=timestamp_ms * 1_000_000, + event_sequence=ordinal, + bid=bid, + ask=ask, + context=ObservationContextV1( + symbol=symbol, + epoch_id=epoch_label, + state=state, + session=session, + ), + ) + ) + previous = (bid, ask) + return tuple(result) + + +def _score_window( + *, + split: BenchmarkSplitKind, + period: str, + session: str, + source: FeedEpochEvidenceV2, + target: ObservationCalibrationTargetV2, + events: Sequence[ObservationInputEventV1], + applied: ObservationApplicationResultV1, + operator: ObservationOperatorV1, + profile: ObservationCalibrationProfileV2, +) -> ObservationCalibrationWindowV2: + outputs_by_source: Counter[str] = Counter( + item.source_event_id for item in applied.output_events + ) + retained = [ + item for item in events if outputs_by_source[item.source_event_id] + ] + expected_retention = statistics.fmean( + _event_retention_probability(operator, item) for item in events + ) + duplicate_target = target.parameter_values["duplicate_probability"] + quantum = int(target.parameter_values["timestamp_quantum_ns"]) + timestamp_target = ( + 1.0 + if quantum >= 1_000_000_000 + else (1_000_000_000 / max(1, quantum)) ** -1 + ) + expected_update_counts = Counter( + { + state: sum( + _event_retention_probability(operator, item) + for item in events + if item.context.state == state + ) + for state in OBSERVATION_UPDATE_STATES + } + ) + expected_update_total = sum(expected_update_counts.values()) + expected_update = { + state: expected_update_counts[state] / max(1e-12, expected_update_total) + for state in OBSERVATION_UPDATE_STATES + } + observed_update_counts = Counter( + cast(str, item.context.state) for item in retained + ) + observed_update = { + state: observed_update_counts[state] / max(1, len(retained)) + for state in OBSERVATION_UPDATE_STATES + } + update_l1 = sum( + abs(observed_update[state] - expected_update[state]) + for state in OBSERVATION_UPDATE_STATES + ) + metrics = { + "retention_probability": len(retained) / len(events), + "duplicate_probability": ( + max(0, len(applied.output_events) - len(retained)) + / max(1, len(retained)) + ), + "timestamp_exact_second_rate": ( + sum( + item.observed_time_ns % 1_000_000_000 == 0 + for item in applied.output_events + ) + / max(1, len(applied.output_events)) + ), + "update_mix_l1": update_l1, + } + targets = { + "retention_probability": expected_retention, + "duplicate_probability": duplicate_target, + "timestamp_exact_second_rate": timestamp_target, + "update_mix_l1": 0.0, + } + tolerances = { + "retention_probability": max( + profile.retention_tolerance, + _binomial_tolerance(expected_retention, len(events)), + ), + "duplicate_probability": max( + profile.duplicate_tolerance, + _binomial_tolerance(duplicate_target, max(1, len(retained))), + ), + "timestamp_exact_second_rate": profile.timestamp_tolerance, + "update_mix_l1": profile.update_mix_l1_tolerance, + } + errors = {name: abs(metrics[name] - targets[name]) for name in metrics} + failures = tuple( + f"{name}_outside_tolerance" + for name in metrics + if errors[name] > tolerances[name] + ) + transformations = Counter( + value + for item in applied.output_events + for value in item.transformations + ) + digits = profile.rounding_digits + return ObservationCalibrationWindowV2( + split_kind=split, + symbol=source.symbol, + period=period, + session=session, + epoch_label=target.epoch_label, + source_artifact_sha256=source.source_artifact_sha256, + input_count=len(events), + output_count=len(applied.output_events), + retained_source_count=len(retained), + target_metrics={ + name: _round(value, digits) for name, value in targets.items() + }, + observed_metrics={ + name: _round(value, digits) for name, value in metrics.items() + }, + absolute_errors={ + name: _round(value, digits) for name, value in errors.items() + }, + tolerances=tolerances, + reason_counts=applied.reason_counts, + transformation_counts=dict(transformations), + passed=not failures, + failure_reasons=failures, + ) + + +def _readiness_reasons( + targets: Sequence[ObservationCalibrationTargetV2], + windows: Sequence[ObservationCalibrationWindowV2], + profile: ObservationCalibrationProfileV2, + *, + runtime_seconds: float, + peak_memory_bytes: int, +) -> tuple[str, ...]: + reasons: list[str] = [] + for target in targets: + if target.parameter_reasons["retention_probability"] == ( + "identity_without_dense_denominator" + ): + reasons.append("identity_retention_without_dense_denominator") + for name in OBSERVATION_CALIBRATION_REQUIRED_PARAMETERS: + if target.parameter_status[name] != "supported": + reasons.append( + f"required_parameter_unsupported:{target.symbol}:{target.epoch_label}:{name}" + ) + expected_splits = set(_SPLIT_KEYS) + actual_splits = {item.split_kind.value for item in windows} + if actual_splits != expected_splits: + reasons.append("blocked_time_split_coverage_incomplete") + expected_cells = { + ( + split.value, + target.symbol, + target.epoch_label, + session, + profile.split_periods[split.value], + ) + for split in OBSERVATION_CALIBRATION_SPLITS + for target in targets + for session in profile.sessions + } + actual_cells = { + ( + item.split_kind.value, + item.symbol, + item.epoch_label, + item.session, + item.period, + ) + for item in windows + } + if actual_cells != expected_cells or len(windows) != len(expected_cells): + reasons.append("blocked_time_cell_coverage_incomplete") + years = {value[:4] for value in profile.split_periods.values()} + if len(years) < 3: + reasons.append("blocked_time_splits_do_not_span_three_years") + holdout = [ + item + for item in windows + if item.split_kind is BenchmarkSplitKind.FINAL_HOLDOUT + ] + if not holdout or any(not item.passed for item in holdout): + reasons.append("final_holdout_failed") + if any( + item.input_count > profile.max_events_per_window for item in windows + ): + reasons.append("window_resource_bound_exceeded") + if runtime_seconds > profile.max_runtime_seconds: + reasons.append("campaign_runtime_bound_exceeded") + if peak_memory_bytes > profile.max_peak_memory_bytes: + reasons.append("campaign_memory_bound_exceeded") + return tuple(dict.fromkeys(reasons)) + + +def _automatic_split_periods( + evidence: Sequence[FeedEpochEvidenceV2], + definition: FeedEpochDefinitionV2, +) -> dict[str, str]: + reference = definition.epochs[-1] + by_period: dict[str, set[str]] = defaultdict(set) + for item in evidence: + midpoint = ( + item.start_timestamp_utc_ms + item.end_timestamp_utc_ms + ) // 2 + assignment = definition.assign( + symbol=item.symbol, timestamp_utc_ms=midpoint + ) + if ( + assignment.label == reference.label + and assignment.assignment_kind == "epoch" + ): + by_period[item.period].add(item.symbol) + periods = sorted( + period + for period, symbols in by_period.items() + if symbols == set(definition.symbols) + ) + if len(periods) < 36 or len({value[:4] for value in periods}) < 3: + raise ValueError("dense reference epoch lacks three blocked years") + indexes = (len(periods) // 4, len(periods) * 2 // 3, len(periods) * 9 // 10) + selected = [periods[min(len(periods) - 1, index)] for index in indexes] + for index in range(1, len(selected)): + while selected[index][:4] == selected[index - 1][:4]: + position = periods.index(selected[index]) + 1 + if position >= len(periods): + raise ValueError("unable to create three-year blocked split") + selected[index] = periods[position] + return dict(zip(_SPLIT_KEYS, selected, strict=True)) + + +def _aggregate_epoch_rows( + rows: Sequence[FeedEpochEvidenceV2], +) -> dict[str, float]: + def median_feature(name: str, default: float = 0.0) -> float: + values = [ + item.feature_values[name] + for item in rows + if name in item.feature_values + ] + return statistics.median(values) if values else default + + result = { + "row_count": float(sum(item.row_count for item in rows)), + "active_rate_per_hour": math.expm1( + median_feature("log_active_window_tick_rate_per_hour") + ), + "timestamp_exact_second_rate": median_feature( + "timestamp_exact_second_rate" + ), + "timestamp_last_digit_entropy": median_feature( + "timestamp_last_digit_entropy", 1.0 + ), + "price_precision_digits": median_feature("price_precision_digits", 8.0), + "duplicate_timestamp_rate": median_feature("duplicate_timestamp_rate"), + "interarrival_median_ms": math.expm1( + median_feature("log_interarrival_median_ms") + ), + } + for state, feature in _UPDATE_FEATURES.items(): + result[state] = median_feature(feature) + update_total = sum(result[state] for state in OBSERVATION_UPDATE_STATES) + for state in OBSERVATION_UPDATE_STATES: + result[state] = result[state] / max(1e-12, update_total) + for session, feature in _SESSION_FEATURES.items(): + result[session] = median_feature(feature) + session_total = sum( + result[session] for session in OBSERVATION_CALIBRATION_SESSIONS + ) + for session in OBSERVATION_CALIBRATION_SESSIONS: + result[session] = result[session] / max(1e-12, session_total) + return result + + +def _feature_series( + rows: Sequence[FeedEpochEvidenceV2], + name: str, + *, + transform: Callable[[float], float] | None = None, +) -> tuple[float, ...]: + values = tuple( + float(item.feature_values[name]) + for item in rows + if name in item.feature_values + ) + if not values: + raise ValueError(f"calibration feature is unavailable: {name}") + if transform is None: + return values + return tuple(float(transform(value)) for value in values) + + +def _quantile(values: Sequence[float], probability: float) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("calibration quantile requires observations") + position = probability * (len(ordered) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + fraction = position - lower + return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction + + +def _conditioned_retention_grid( + base_retention: float, + *, + target_update: Mapping[str, float], + reference_update: Mapping[str, float], + target_session: Mapping[str, float], + reference_session: Mapping[str, float], +) -> dict[tuple[str, str], float]: + ratios: dict[tuple[str, str], float] = {} + weights: dict[tuple[str, str], float] = {} + for state in OBSERVATION_UPDATE_STATES: + for session in OBSERVATION_CALIBRATION_SESSIONS: + key = (state, session) + ratios[key] = math.sqrt( + target_update[state] + / max(1e-9, reference_update[state]) + * target_session[session] + / max(1e-9, reference_session[session]) + ) + weights[key] = reference_update[state] * reference_session[session] + lower, upper = 0.0, 1000.0 + for _ in range(80): + scale = (lower + upper) / 2.0 + achieved = sum( + weights[key] * min(1.0, scale * ratios[key]) for key in ratios + ) + if achieved < base_retention: + lower = scale + else: + upper = scale + return {key: min(1.0, upper * ratio) for key, ratio in ratios.items()} + + +def _event_retention_probability( + operator: ObservationOperatorV1, + event: ObservationInputEventV1, +) -> float: + stratum, _ = operator.resolve_stratum(event.context) + value = stratum.effective_value("retention_probability") + if event.context.state == "update_unchanged": + value *= stratum.effective_value("unchanged_retention_probability") + return float(value) + + +def _aligned_window_bounds( + events: Sequence[ObservationInputEventV1], + operator: ObservationOperatorV1, +) -> tuple[int, int]: + """Return bounds aligned to every fitted timestamp/batch quantum.""" + alignments = [1] + for stratum in operator.strata: + for name in ("timestamp_quantum_ns", "batch_window_ns"): + value = int(stratum.effective_value(name)) + if value > 1: + alignments.append(value) + alignment = math.lcm(*alignments) + start = events[0].event_time_ns // alignment * alignment + end = (events[-1].event_time_ns // alignment + 1) * alignment + return start, end + + +def _timestamp_grid_ns(outputs: Sequence[Any]) -> int: + timestamps = sorted({int(item.observed_time_ns) for item in outputs}) + if len(timestamps) < 2: + return 1 + result = 0 + for left, right in zip(timestamps, timestamps[1:], strict=False): + result = math.gcd(result, right - left) + return max(1, result) + + +def _quantum_from_timestamp_features( + exact_second: float, entropy: float +) -> int: + if exact_second >= 0.95: + return 1_000_000_000 + if entropy <= 0.05: + return 100_000_000 + if entropy <= 0.75: + return 10_000_000 + return 1_000_000 + + +def _binomial_tolerance(probability: float, count: int) -> float: + """Return a predeclared finite-window three-sigma acceptance width.""" + return min( + 1.0, + 3.0 * math.sqrt(probability * (1.0 - probability) / max(1, count)) + + 3.0 / max(1, count), + ) + + +def _update_state( + previous: tuple[float, float] | None, + bid: float, + ask: float, +) -> str: + if previous is None: + return "update_joint" + bid_changed = bid != previous[0] + ask_changed = ask != previous[1] + if bid_changed and ask_changed: + return "update_joint" + if bid_changed: + return "update_bid_only" + if ask_changed: + return "update_ask_only" + return "update_unchanged" + + +def _json_float_mapping(value: JSONValue) -> dict[str, float]: + return { + str(key): _finite_float(item, str(key)) + for key, item in _mapping(value).items() + } + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(payload).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _sha256_text(value: str) -> str: + text = str(value) + raw = text.removeprefix("sha256:") + if len(raw) != 64 or any(char not in "0123456789abcdef" for char in raw): + raise ValueError("invalid SHA-256 value") + return "sha256:" + raw + + +def _required_text(value: Any, name: str) -> str: + result = str(value).strip() + if not result: + raise ValueError(f"{name} is required") + return result + + +def _valid_month(value: str) -> bool: + return len(value) == 6 and value.isdigit() and 1 <= int(value[4:]) <= 12 + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a boolean") + return value + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be numeric") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _calibration_parameter_value(name: str, value: Any) -> float: + result = _finite_float(value, name) + if result < 0.0: + raise ValueError(f"{name} must be non-negative") + if name in _PROBABILITY_PARAMETERS and result > 1.0: + raise ValueError(f"{name} must not exceed one") + if name in _INTEGER_PARAMETERS and not result.is_integer(): + raise ValueError(f"{name} must be integral") + if name == "price_precision_digits" and result > 16.0: + raise ValueError("price_precision_digits exceeds sixteen") + return result + + +def _round(value: float, digits: int) -> float: + return round(_finite_float(value, "value"), digits) + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError("expected a sequence") + return value + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _string_mapping(value: Any) -> dict[str, str]: + return {str(key): str(item) for key, item in _mapping(value).items()} + + +def _float_mapping(value: Any) -> dict[str, float]: + return { + str(key): _finite_float(item, str(key)) + for key, item in _mapping(value).items() + } + + +def _finite_mapping(value: Mapping[str, Any]) -> dict[str, float]: + return { + str(key): _finite_float(item, str(key)) for key, item in value.items() + } + + +def _int_mapping(value: Any) -> dict[str, int]: + mapping = value if isinstance(value, Mapping) else _mapping(value) + result: dict[str, int] = {} + for key, item in mapping.items(): + integer = _strict_int(item, str(key)) + if integer < 0: + raise ValueError("count values must be non-negative") + result[str(key)] = integer + return result diff --git a/src/histdatacom/synthetic/persistence.py b/src/histdatacom/synthetic/persistence.py new file mode 100644 index 00000000..ec5aad34 --- /dev/null +++ b/src/histdatacom/synthetic/persistence.py @@ -0,0 +1,3872 @@ +"""Atomic persistence for narrow reconstructed event products. + +This module is the durable boundary between a fully validated delivered group +and the final reconstructed tick archive. Version one retains the legacy +broker-rendered contract; version two accepts an explicit generic delivery +without inventing a broker identity. Both write only the exact +``SyntheticEventV1`` Arrow schema plus compact manifests. Analytical feature +frames, candidate rows, and rejection rows are never accepted by this API. + +Publication is a directory-level transaction on one filesystem. Parquet +partitions and the manifest are written below undiscoverable scratch, validated +there, and promoted with one atomic rename. Discovery looks only below +``commits`` and therefore cannot advertise partial output. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import shutil +import sys +import tempfile +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, cast +from urllib.parse import quote + +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.broker_transfer import ( + BrokerRenderedGroupV1, + BrokerTransferStatus, +) +from histdatacom.synthetic.contracts import ( + SYNTHETIC_EVENT_ARROW_COLUMNS, + SYNTHETIC_EVENT_SCHEMA_VERSION, + SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION, + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + canonical_contract_json, + synthetic_event_arrow_schema, + synthetic_event_stream_to_arrow, +) +from histdatacom.synthetic.cross_currency import ( + CrossCurrencyValidationReportV1, + CrossCurrencyValidationStage, +) +from histdatacom.synthetic.delivery import ( + ReconstructionDeliveredGroupV1, + ReconstructionDeliveryMode, + ReconstructionDeliveryStatus, + reconstruction_streams_content_sha256, +) +from histdatacom.synthetic.streaming import ReconstructionStoragePolicyV1 + +RECONSTRUCTION_PRODUCT_SCHEMA_VERSION = "histdatacom.reconstruction-product.v1" +RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION = ( + "histdatacom.reconstruction-product.v2" +) +RECONSTRUCTION_PARTITION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-product-partition.v1" +) +RECONSTRUCTION_SOURCE_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-source-manifest.v1" +) +RECONSTRUCTION_CONSTRAINT_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-constraint-manifest.v1" +) +RECONSTRUCTION_QUALITY_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-quality-manifest.v1" +) +RECONSTRUCTION_DELIVERY_QUALITY_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-delivery-quality-manifest.v1" +) +RECONSTRUCTION_REPLAY_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-replay-manifest.v1" +) +RECONSTRUCTION_RETENTION_PLAN_SCHEMA_VERSION = ( + "histdatacom.reconstruction-retention-plan.v1" +) +RECONSTRUCTION_ENSEMBLE_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-ensemble-product-manifest.v1" +) + +RECONSTRUCTION_PRODUCT_DIRECTORY = "reconstruction-products" +RECONSTRUCTION_MANIFEST_FILENAME = "manifest.json" +RECONSTRUCTION_MANIFEST_ARTIFACT_KIND = "reconstruction-product-manifest" +RECONSTRUCTION_PARQUET_ARTIFACT_KIND = "reconstruction-product-parquet" +RECONSTRUCTION_LOGICAL_HASH_ALGORITHM = "sha256-canonical-event-json-lines-v1" +RECONSTRUCTION_BYTE_HASH_ALGORITHM = "sha256-path-byte-digests-v1" +RECONSTRUCTION_WRITER_ID = "histdatacom.pyarrow-parquet-zstd.v1" +RECONSTRUCTION_COMPRESSION = "zstd" +DEFAULT_RECONSTRUCTION_ROW_GROUP_SIZE = 65_536 +DEFAULT_ESTIMATED_BYTES_PER_EVENT = 256 +DEFAULT_ESTIMATED_COMPRESSION_RATIO = 0.35 +DEFAULT_MANIFEST_BYTES_PER_PARTITION = 4_096 +MAX_RECONSTRUCTION_PARTITIONS = 4_096 +MAX_RECONSTRUCTION_MANIFEST_BYTES = 4 * 1024**2 +MAX_RECONSTRUCTION_TEXT = 1_024 + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_EVENT_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +_LOGICAL_HASH_HEADER = (RECONSTRUCTION_LOGICAL_HASH_ALGORITHM + "\n").encode( + "ascii" +) +_OBSERVED_HASH_HEADER = b"histdatacom-observed-anchors-v1\n" + + +class ReconstructionPersistenceError(ValueError): + """Base failure for a refused final-product publication.""" + + +class ReconstructionStoragePreflightError(ReconstructionPersistenceError): + """A final-product estimate exceeds its retention/storage policy.""" + + def __init__( + self, + estimate: "ReconstructionRetentionPlanV1", + violations: Sequence[str], + ) -> None: + self.estimate = estimate + self.violations = tuple(violations) + super().__init__( + "reconstruction persistence preflight failed: " + + "; ".join(self.violations) + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionRetentionPlanV1: + """Pre-run primary/retained-member storage estimate and policy binding.""" + + run_id: str + primary_member_id: str + retained_member_ids: tuple[str, ...] + member_event_counts: Mapping[str, int] + estimated_partition_count: int + estimated_bytes_per_event: int + estimated_compression_ratio: float + estimated_primary_bytes: int + estimated_retained_bytes: int + estimated_manifest_bytes: int + estimated_total_output_bytes: int + storage_policy_id: str + plan_id: str = "" + schema_version: str = RECONSTRUCTION_RETENTION_PLAN_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_RETENTION_PLAN_SCHEMA_VERSION, + "reconstruction retention plan", + ) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, + "primary_member_id", + _required_text(self.primary_member_id), + ) + retained = _normalized_text_tuple(self.retained_member_ids) + if not retained or self.primary_member_id not in retained: + raise ValueError("retained members must include the primary member") + object.__setattr__(self, "retained_member_ids", retained) + counts = { + _required_text(member): _nonnegative_int( + count, f"member_event_counts.{member}" + ) + for member, count in self.member_event_counts.items() + } + if set(counts) != set(retained): + raise ValueError( + "member event estimates must cover retained members exactly" + ) + object.__setattr__( + self, "member_event_counts", dict(sorted(counts.items())) + ) + for name in ( + "estimated_partition_count", + "estimated_bytes_per_event", + "estimated_primary_bytes", + "estimated_retained_bytes", + "estimated_manifest_bytes", + "estimated_total_output_bytes", + ): + object.__setattr__( + self, + name, + _nonnegative_int(getattr(self, name), name), + ) + has_estimated_events = any(counts.values()) + if (self.estimated_partition_count == 0) != (not has_estimated_events): + raise ValueError( + "zero-event retention plans must have zero partitions" + ) + if self.estimated_bytes_per_event < 1: + raise ValueError("estimated_bytes_per_event must be positive") + ratio = _finite_float( + self.estimated_compression_ratio, + "estimated_compression_ratio", + ) + if not 0.0 < ratio <= 1.0: + raise ValueError("estimated_compression_ratio must be in (0, 1]") + object.__setattr__(self, "estimated_compression_ratio", ratio) + expected_primary = _estimated_event_bytes( + counts[self.primary_member_id], + self.estimated_bytes_per_event, + ratio, + ) + expected_retained = sum( + _estimated_event_bytes( + counts[member], self.estimated_bytes_per_event, ratio + ) + for member in retained + ) + if self.estimated_primary_bytes != expected_primary: + raise ValueError("estimated primary bytes do not reconcile") + if self.estimated_retained_bytes != expected_retained: + raise ValueError("estimated retained bytes do not reconcile") + if self.estimated_total_output_bytes != ( + self.estimated_retained_bytes + self.estimated_manifest_bytes + ): + raise ValueError("estimated total output bytes do not reconcile") + object.__setattr__( + self, + "storage_policy_id", + _required_text(self.storage_policy_id), + ) + expected = _stable_id("reconstruction-retention", self.payload()) + supplied = _optional_text(self.plan_id) + if supplied is not None and supplied != expected: + raise ValueError("retention plan_id differs from its content") + object.__setattr__(self, "plan_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return deterministic estimate fields.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "primary_member_id": self.primary_member_id, + "retained_member_ids": list(self.retained_member_ids), + "member_event_counts": dict(self.member_event_counts), + "estimated_partition_count": self.estimated_partition_count, + "estimated_bytes_per_event": self.estimated_bytes_per_event, + "estimated_compression_ratio": (self.estimated_compression_ratio), + "estimated_primary_bytes": self.estimated_primary_bytes, + "estimated_retained_bytes": self.estimated_retained_bytes, + "estimated_manifest_bytes": self.estimated_manifest_bytes, + "estimated_total_output_bytes": (self.estimated_total_output_bytes), + "storage_policy_id": self.storage_policy_id, + "estimate_basis": "event-count-compression-upper-bound-v1", + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return the compact retention plan.""" + return {**self.payload(), "plan_id": self.plan_id} + + def to_json(self) -> str: + """Return deterministic JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionRetentionPlanV1": + """Restore and verify a retention plan.""" + _require_schema(data, RECONSTRUCTION_RETENTION_PLAN_SCHEMA_VERSION) + _require_derived( + data, + "estimate_basis", + "event-count-compression-upper-bound-v1", + ) + return cls( + run_id=str(data.get("run_id", "")), + primary_member_id=str(data.get("primary_member_id", "")), + retained_member_ids=_string_tuple( + data.get("retained_member_ids"), "retained_member_ids" + ), + member_event_counts={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("member_event_counts"), "member_event_counts" + ).items() + }, + estimated_partition_count=_strict_int( + data.get("estimated_partition_count"), + "estimated_partition_count", + ), + estimated_bytes_per_event=_strict_int( + data.get("estimated_bytes_per_event"), + "estimated_bytes_per_event", + ), + estimated_compression_ratio=_strict_float( + data.get("estimated_compression_ratio"), + "estimated_compression_ratio", + ), + estimated_primary_bytes=_strict_int( + data.get("estimated_primary_bytes"), + "estimated_primary_bytes", + ), + estimated_retained_bytes=_strict_int( + data.get("estimated_retained_bytes"), + "estimated_retained_bytes", + ), + estimated_manifest_bytes=_strict_int( + data.get("estimated_manifest_bytes"), + "estimated_manifest_bytes", + ), + estimated_total_output_bytes=_strict_int( + data.get("estimated_total_output_bytes"), + "estimated_total_output_bytes", + ), + storage_policy_id=str(data.get("storage_policy_id", "")), + plan_id=str(data.get("plan_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +def estimate_reconstruction_retention( + *, + run_id: str, + primary_member_id: str, + retained_member_event_counts: Mapping[str, int], + estimated_partition_count: int, + storage_policy: ReconstructionStoragePolicyV1, + estimated_bytes_per_event: int = DEFAULT_ESTIMATED_BYTES_PER_EVENT, + estimated_compression_ratio: float = (DEFAULT_ESTIMATED_COMPRESSION_RATIO), +) -> ReconstructionRetentionPlanV1: + """Estimate the primary and all retained outputs before reconstruction.""" + if not isinstance(storage_policy, ReconstructionStoragePolicyV1): + raise TypeError("storage preflight requires a v1 storage policy") + counts = { + _required_text(member): _nonnegative_int(count, str(member)) + for member, count in retained_member_event_counts.items() + } + primary = _required_text(primary_member_id) + if primary not in counts: + raise ValueError("primary member is absent from retained estimates") + partition_count = _nonnegative_int( + estimated_partition_count, "estimated_partition_count" + ) + bytes_per_event = _positive_int( + estimated_bytes_per_event, "estimated_bytes_per_event" + ) + ratio = _finite_float( + estimated_compression_ratio, "estimated_compression_ratio" + ) + if not 0.0 < ratio <= 1.0: + raise ValueError("estimated_compression_ratio must be in (0, 1]") + primary_bytes = _estimated_event_bytes( + counts[primary], bytes_per_event, ratio + ) + retained_bytes = sum( + _estimated_event_bytes(value, bytes_per_event, ratio) + for value in counts.values() + ) + manifest_bytes = partition_count * DEFAULT_MANIFEST_BYTES_PER_PARTITION + plan = ReconstructionRetentionPlanV1( + run_id=run_id, + primary_member_id=primary, + retained_member_ids=tuple(counts), + member_event_counts=counts, + estimated_partition_count=partition_count, + estimated_bytes_per_event=bytes_per_event, + estimated_compression_ratio=ratio, + estimated_primary_bytes=primary_bytes, + estimated_retained_bytes=retained_bytes, + estimated_manifest_bytes=manifest_bytes, + estimated_total_output_bytes=retained_bytes + manifest_bytes, + storage_policy_id=storage_policy.policy_id, + ) + violations: list[str] = [] + if len(counts) > storage_policy.max_retained_ensemble_members: + violations.append( + f"retained members {len(counts)} exceed " + f"{storage_policy.max_retained_ensemble_members}" + ) + if plan.estimated_total_output_bytes > storage_policy.max_output_bytes: + violations.append( + f"estimated output {plan.estimated_total_output_bytes} exceeds " + f"{storage_policy.max_output_bytes}" + ) + if partition_count > MAX_RECONSTRUCTION_PARTITIONS: + violations.append( + f"estimated partitions {partition_count} exceed " + f"{MAX_RECONSTRUCTION_PARTITIONS}" + ) + if violations: + raise ReconstructionStoragePreflightError(plan, violations) + return plan + + +@dataclass(frozen=True, slots=True) +class ReconstructionEnsembleManifestV1: + """Compact retained-ensemble identity for one materialized member.""" + + run_id: str + materialized_member_id: str + primary_member_id: str + retained_member_ids: tuple[str, ...] + member_event_estimates: Mapping[str, int] + retention_plan_id: str + ensemble_manifest_id: str = "" + schema_version: str = RECONSTRUCTION_ENSEMBLE_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_ENSEMBLE_MANIFEST_SCHEMA_VERSION, + "reconstruction ensemble product manifest", + ) + for name in ( + "run_id", + "materialized_member_id", + "primary_member_id", + "retention_plan_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + retained = _normalized_text_tuple(self.retained_member_ids) + if ( + not retained + or self.primary_member_id not in retained + or self.materialized_member_id not in retained + ): + raise ValueError( + "ensemble manifest members differ from retained membership" + ) + object.__setattr__(self, "retained_member_ids", retained) + estimates = { + _required_text(member): _nonnegative_int( + count, f"member_event_estimates.{member}" + ) + for member, count in self.member_event_estimates.items() + } + if set(estimates) != set(retained): + raise ValueError( + "ensemble event estimates do not cover retained members" + ) + object.__setattr__( + self, "member_event_estimates", dict(sorted(estimates.items())) + ) + expected = _stable_id("reconstruction-ensemble", self.payload()) + supplied = _optional_text(self.ensemble_manifest_id) + if supplied is not None and supplied != expected: + raise ValueError("ensemble manifest_id differs") + object.__setattr__(self, "ensemble_manifest_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return compact retained-member identity.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "materialized_member_id": self.materialized_member_id, + "primary_member_id": self.primary_member_id, + "retained_member_ids": list(self.retained_member_ids), + "member_event_estimates": dict(self.member_event_estimates), + "retention_plan_id": self.retention_plan_id, + "automatic_winner": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact ensemble-manifest JSON.""" + return { + **self.payload(), + "ensemble_manifest_id": self.ensemble_manifest_id, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionEnsembleManifestV1": + """Restore and verify retained-ensemble identity.""" + _require_schema(data, RECONSTRUCTION_ENSEMBLE_MANIFEST_SCHEMA_VERSION) + _require_derived(data, "automatic_winner", False) + return cls( + run_id=str(data.get("run_id", "")), + materialized_member_id=str(data.get("materialized_member_id", "")), + primary_member_id=str(data.get("primary_member_id", "")), + retained_member_ids=_string_tuple( + data.get("retained_member_ids"), "retained_member_ids" + ), + member_event_estimates={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("member_event_estimates"), + "member_event_estimates", + ).items() + }, + retention_plan_id=str(data.get("retention_plan_id", "")), + ensemble_manifest_id=str(data.get("ensemble_manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionProductPartitionV1: + """Physical and logical evidence for one symbol/date Parquet file.""" + + relative_path: str + symbol: str + event_date: str + stream_id: str + source_version_ids: tuple[str, ...] + row_count: int + observed_event_count: int + synthetic_event_count: int + min_event_time_ns: int + max_event_time_ns: int + logical_content_sha256: str + byte_sha256: str + size_bytes: int + row_group_count: int + partition_id: str = "" + schema_version: str = RECONSTRUCTION_PARTITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_PARTITION_SCHEMA_VERSION, + "reconstruction product partition", + ) + symbol = _normalized_symbol(self.symbol) + object.__setattr__(self, "symbol", symbol) + event_date = _required_event_date(self.event_date) + object.__setattr__(self, "event_date", event_date) + path = _safe_relative_path(self.relative_path) + expected_path = _partition_relative_path(symbol, event_date) + if path != expected_path: + raise ValueError("partition relative path differs from layout") + object.__setattr__(self, "relative_path", path) + object.__setattr__(self, "stream_id", _required_text(self.stream_id)) + sources = _normalized_text_tuple(self.source_version_ids) + if not sources: + raise ValueError("partition requires source version IDs") + object.__setattr__(self, "source_version_ids", sources) + for name in ( + "row_count", + "observed_event_count", + "synthetic_event_count", + "size_bytes", + "row_group_count", + ): + object.__setattr__( + self, + name, + _nonnegative_int(getattr(self, name), name), + ) + if self.row_count < 1 or self.row_group_count < 1: + raise ValueError("durable partitions cannot be empty") + if self.row_count != ( + self.observed_event_count + self.synthetic_event_count + ): + raise ValueError("partition origin counts do not reconcile") + minimum = _strict_int(self.min_event_time_ns, "min_event_time_ns") + maximum = _strict_int(self.max_event_time_ns, "max_event_time_ns") + if maximum < minimum: + raise ValueError("partition event-time bounds are reversed") + if ( + _event_date(minimum) != event_date + or _event_date(maximum) != event_date + ): + raise ValueError("partition rows cross the event-date boundary") + object.__setattr__(self, "min_event_time_ns", minimum) + object.__setattr__(self, "max_event_time_ns", maximum) + for name in ("logical_content_sha256", "byte_sha256"): + object.__setattr__( + self, name, _required_sha256(getattr(self, name), name) + ) + expected = _stable_id("reconstruction-partition", self.payload()) + supplied = _optional_text(self.partition_id) + if supplied is not None and supplied != expected: + raise ValueError("partition_id differs from partition evidence") + object.__setattr__(self, "partition_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return deterministic partition evidence.""" + return { + "schema_version": self.schema_version, + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "stream_schema_version": SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION, + "relative_path": self.relative_path, + "symbol": self.symbol, + "event_date": self.event_date, + "stream_id": self.stream_id, + "source_version_ids": list(self.source_version_ids), + "row_count": self.row_count, + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "min_event_time_ns": self.min_event_time_ns, + "max_event_time_ns": self.max_event_time_ns, + "logical_content_sha256": self.logical_content_sha256, + "byte_sha256": self.byte_sha256, + "size_bytes": self.size_bytes, + "row_group_count": self.row_group_count, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact partition JSON.""" + return {**self.payload(), "partition_id": self.partition_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionProductPartitionV1": + """Restore and verify partition evidence.""" + _require_schema(data, RECONSTRUCTION_PARTITION_SCHEMA_VERSION) + _require_derived( + data, "event_schema_version", SYNTHETIC_EVENT_SCHEMA_VERSION + ) + _require_derived( + data, + "stream_schema_version", + SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION, + ) + return cls( + relative_path=str(data.get("relative_path", "")), + symbol=str(data.get("symbol", "")), + event_date=str(data.get("event_date", "")), + stream_id=str(data.get("stream_id", "")), + source_version_ids=_string_tuple( + data.get("source_version_ids"), "source_version_ids" + ), + row_count=_strict_int(data.get("row_count"), "row_count"), + observed_event_count=_strict_int( + data.get("observed_event_count"), "observed_event_count" + ), + synthetic_event_count=_strict_int( + data.get("synthetic_event_count"), "synthetic_event_count" + ), + min_event_time_ns=_strict_int( + data.get("min_event_time_ns"), "min_event_time_ns" + ), + max_event_time_ns=_strict_int( + data.get("max_event_time_ns"), "max_event_time_ns" + ), + logical_content_sha256=str(data.get("logical_content_sha256", "")), + byte_sha256=str(data.get("byte_sha256", "")), + size_bytes=_strict_int(data.get("size_bytes"), "size_bytes"), + row_group_count=_strict_int( + data.get("row_group_count"), "row_group_count" + ), + partition_id=str(data.get("partition_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionSourceManifestV1: + """Compact immutable-source and observed-anchor evidence.""" + + source_version_ids: tuple[str, ...] + source_series_ids: tuple[str, ...] + source_periods: tuple[str, ...] + observed_event_count: int + observed_content_sha256: str + observed_event_ids_sha256: str + source_manifest_id: str = "" + schema_version: str = RECONSTRUCTION_SOURCE_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_SOURCE_MANIFEST_SCHEMA_VERSION, + "reconstruction source manifest", + ) + versions = _normalized_text_tuple(self.source_version_ids) + if not versions: + raise ValueError("source manifest requires source versions") + object.__setattr__(self, "source_version_ids", versions) + object.__setattr__( + self, + "source_series_ids", + _normalized_text_tuple(self.source_series_ids), + ) + object.__setattr__( + self, + "source_periods", + _normalized_text_tuple(self.source_periods), + ) + object.__setattr__( + self, + "observed_event_count", + _nonnegative_int(self.observed_event_count, "observed_event_count"), + ) + for name in ("observed_content_sha256", "observed_event_ids_sha256"): + object.__setattr__( + self, name, _required_sha256(getattr(self, name), name) + ) + expected = _stable_id("reconstruction-source", self.payload()) + supplied = _optional_text(self.source_manifest_id) + if supplied is not None and supplied != expected: + raise ValueError("source manifest_id differs") + object.__setattr__(self, "source_manifest_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return immutable-source evidence.""" + return { + "schema_version": self.schema_version, + "source_version_ids": list(self.source_version_ids), + "source_series_ids": list(self.source_series_ids), + "source_periods": list(self.source_periods), + "observed_event_count": self.observed_event_count, + "observed_content_sha256": self.observed_content_sha256, + "observed_event_ids_sha256": self.observed_event_ids_sha256, + "observed_values_verified_exactly": True, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact source-manifest JSON.""" + return {**self.payload(), "source_manifest_id": self.source_manifest_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionSourceManifestV1": + """Restore and verify source evidence.""" + _require_schema(data, RECONSTRUCTION_SOURCE_MANIFEST_SCHEMA_VERSION) + _require_derived(data, "observed_values_verified_exactly", True) + return cls( + source_version_ids=_string_tuple( + data.get("source_version_ids"), "source_version_ids" + ), + source_series_ids=_string_tuple( + data.get("source_series_ids"), "source_series_ids" + ), + source_periods=_string_tuple( + data.get("source_periods"), "source_periods" + ), + observed_event_count=_strict_int( + data.get("observed_event_count"), "observed_event_count" + ), + observed_content_sha256=str( + data.get("observed_content_sha256", "") + ), + observed_event_ids_sha256=str( + data.get("observed_event_ids_sha256", "") + ), + source_manifest_id=str(data.get("source_manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionConstraintManifestV1: + """Compact generator, constraint, epoch, motif, and reference lineage.""" + + synthetic_event_count: int + constraint_set_ids: tuple[str, ...] + generator_ids: tuple[str, ...] + generator_versions: tuple[str, ...] + generator_config_ids: tuple[str, ...] + feed_epoch_ids: tuple[str, ...] + reference_assignment_count: int + reference_assignments_sha256: str + motif_assignment_count: int + motif_assignments_sha256: str + constraint_manifest_id: str = "" + schema_version: str = RECONSTRUCTION_CONSTRAINT_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_CONSTRAINT_MANIFEST_SCHEMA_VERSION, + "reconstruction constraint manifest", + ) + count = _nonnegative_int( + self.synthetic_event_count, "synthetic_event_count" + ) + object.__setattr__(self, "synthetic_event_count", count) + for name in ( + "constraint_set_ids", + "generator_ids", + "generator_versions", + "generator_config_ids", + "feed_epoch_ids", + ): + object.__setattr__( + self, name, _normalized_text_tuple(getattr(self, name)) + ) + for name in ("reference_assignment_count", "motif_assignment_count"): + object.__setattr__( + self, + name, + _nonnegative_int(getattr(self, name), name), + ) + for name in ( + "reference_assignments_sha256", + "motif_assignments_sha256", + ): + object.__setattr__( + self, name, _required_sha256(getattr(self, name), name) + ) + if ( + self.reference_assignment_count > count + or self.motif_assignment_count > count + ): + raise ValueError("lineage assignment counts exceed synthetic rows") + if count and not ( + self.constraint_set_ids + and self.generator_ids + and self.generator_versions + and self.generator_config_ids + ): + raise ValueError( + "synthetic rows require generator and constraint lineage" + ) + expected = _stable_id("reconstruction-constraint", self.payload()) + supplied = _optional_text(self.constraint_manifest_id) + if supplied is not None and supplied != expected: + raise ValueError("constraint manifest_id differs") + object.__setattr__(self, "constraint_manifest_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return compact constraint lineage.""" + return { + "schema_version": self.schema_version, + "synthetic_event_count": self.synthetic_event_count, + "constraint_set_ids": list(self.constraint_set_ids), + "generator_ids": list(self.generator_ids), + "generator_versions": list(self.generator_versions), + "generator_config_ids": list(self.generator_config_ids), + "feed_epoch_ids": list(self.feed_epoch_ids), + "reference_assignment_count": self.reference_assignment_count, + "reference_assignments_sha256": self.reference_assignments_sha256, + "motif_assignment_count": self.motif_assignment_count, + "motif_assignments_sha256": self.motif_assignments_sha256, + "candidate_rows_inline": False, + "rejected_rows_inline": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact constraint-manifest JSON.""" + return { + **self.payload(), + "constraint_manifest_id": self.constraint_manifest_id, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionConstraintManifestV1": + """Restore and verify constraint lineage.""" + _require_schema(data, RECONSTRUCTION_CONSTRAINT_MANIFEST_SCHEMA_VERSION) + _require_derived(data, "candidate_rows_inline", False) + _require_derived(data, "rejected_rows_inline", False) + return cls( + synthetic_event_count=_strict_int( + data.get("synthetic_event_count"), "synthetic_event_count" + ), + constraint_set_ids=_string_tuple( + data.get("constraint_set_ids"), "constraint_set_ids" + ), + generator_ids=_string_tuple( + data.get("generator_ids"), "generator_ids" + ), + generator_versions=_string_tuple( + data.get("generator_versions"), "generator_versions" + ), + generator_config_ids=_string_tuple( + data.get("generator_config_ids"), "generator_config_ids" + ), + feed_epoch_ids=_string_tuple( + data.get("feed_epoch_ids"), "feed_epoch_ids" + ), + reference_assignment_count=_strict_int( + data.get("reference_assignment_count"), + "reference_assignment_count", + ), + reference_assignments_sha256=str( + data.get("reference_assignments_sha256", "") + ), + motif_assignment_count=_strict_int( + data.get("motif_assignment_count"), "motif_assignment_count" + ), + motif_assignments_sha256=str( + data.get("motif_assignments_sha256", "") + ), + constraint_manifest_id=str(data.get("constraint_manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionQualityManifestV1: + """Passing broker-transfer, validation, and quality evidence.""" + + broker_transfer_manifest_id: str + broker_fingerprint_id: str + transfer_output_content_sha256: str + post_broker_validation_id: str + post_broker_validation_status: str + cross_instrument_quality_status: str + cross_instrument_quality_sha256: str + broker_observed_event_count: int + broker_synthetic_event_count: int + broker_lineage_count: int + broker_lineage_content_sha256: str + broker_action_counts: Mapping[str, int] + benchmark_comparison_ids: tuple[str, ...] + quality_manifest_id: str = "" + schema_version: str = RECONSTRUCTION_QUALITY_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_QUALITY_MANIFEST_SCHEMA_VERSION, + "reconstruction quality manifest", + ) + for name in ( + "broker_transfer_manifest_id", + "broker_fingerprint_id", + "post_broker_validation_id", + "post_broker_validation_status", + "cross_instrument_quality_status", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "transfer_output_content_sha256", + _required_sha256( + self.transfer_output_content_sha256, + "transfer_output_content_sha256", + ), + ) + object.__setattr__( + self, + "cross_instrument_quality_sha256", + _required_sha256( + self.cross_instrument_quality_sha256, + "cross_instrument_quality_sha256", + ), + ) + for name in ( + "broker_observed_event_count", + "broker_synthetic_event_count", + "broker_lineage_count", + ): + object.__setattr__( + self, + name, + _nonnegative_int(getattr(self, name), name), + ) + object.__setattr__( + self, + "broker_lineage_content_sha256", + _required_sha256( + self.broker_lineage_content_sha256, + "broker_lineage_content_sha256", + ), + ) + actions = { + _required_text(name): _positive_int(count, f"action.{name}") + for name, count in self.broker_action_counts.items() + } + object.__setattr__( + self, "broker_action_counts", dict(sorted(actions.items())) + ) + if self.broker_lineage_count != self.broker_synthetic_event_count: + raise ValueError("broker lineage and synthetic counts differ") + comparisons = _normalized_text_tuple(self.benchmark_comparison_ids) + object.__setattr__(self, "benchmark_comparison_ids", comparisons) + if self.post_broker_validation_status != "passed": + raise ValueError("post-broker validation is not passing") + if self.cross_instrument_quality_status == "failed": + raise ValueError("cross-instrument quality is failed") + expected = _stable_id("reconstruction-quality", self.payload()) + supplied = _optional_text(self.quality_manifest_id) + if supplied is not None and supplied != expected: + raise ValueError("quality manifest_id differs") + object.__setattr__(self, "quality_manifest_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return compact quality evidence.""" + return { + "schema_version": self.schema_version, + "broker_transfer_manifest_id": (self.broker_transfer_manifest_id), + "broker_fingerprint_id": self.broker_fingerprint_id, + "transfer_output_content_sha256": ( + self.transfer_output_content_sha256 + ), + "post_broker_validation_id": self.post_broker_validation_id, + "post_broker_validation_status": ( + self.post_broker_validation_status + ), + "cross_instrument_quality_status": ( + self.cross_instrument_quality_status + ), + "cross_instrument_quality_sha256": ( + self.cross_instrument_quality_sha256 + ), + "broker_observed_event_count": self.broker_observed_event_count, + "broker_synthetic_event_count": self.broker_synthetic_event_count, + "broker_lineage_count": self.broker_lineage_count, + "broker_lineage_content_sha256": ( + self.broker_lineage_content_sha256 + ), + "broker_action_counts": dict(self.broker_action_counts), + "benchmark_comparison_ids": list(self.benchmark_comparison_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact quality-manifest JSON.""" + return { + **self.payload(), + "quality_manifest_id": self.quality_manifest_id, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionQualityManifestV1": + """Restore and verify final quality evidence.""" + _require_schema(data, RECONSTRUCTION_QUALITY_MANIFEST_SCHEMA_VERSION) + return cls( + broker_transfer_manifest_id=str( + data.get("broker_transfer_manifest_id", "") + ), + broker_fingerprint_id=str(data.get("broker_fingerprint_id", "")), + transfer_output_content_sha256=str( + data.get("transfer_output_content_sha256", "") + ), + post_broker_validation_id=str( + data.get("post_broker_validation_id", "") + ), + post_broker_validation_status=str( + data.get("post_broker_validation_status", "") + ), + cross_instrument_quality_status=str( + data.get("cross_instrument_quality_status", "") + ), + cross_instrument_quality_sha256=str( + data.get("cross_instrument_quality_sha256", "") + ), + broker_observed_event_count=_strict_int( + data.get("broker_observed_event_count"), + "broker_observed_event_count", + ), + broker_synthetic_event_count=_strict_int( + data.get("broker_synthetic_event_count"), + "broker_synthetic_event_count", + ), + broker_lineage_count=_strict_int( + data.get("broker_lineage_count"), "broker_lineage_count" + ), + broker_lineage_content_sha256=str( + data.get("broker_lineage_content_sha256", "") + ), + broker_action_counts={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("broker_action_counts"), "broker_action_counts" + ).items() + }, + benchmark_comparison_ids=_string_tuple( + data.get("benchmark_comparison_ids"), + "benchmark_comparison_ids", + ), + quality_manifest_id=str(data.get("quality_manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionDeliveryQualityManifestV1: + """Passing generic delivery, final validation, and benchmark evidence.""" + + delivery_manifest_id: str + delivery_profile_id: str + delivery_mode: ReconstructionDeliveryMode + delivery_output_content_sha256: str + final_validation_id: str + final_validation_status: str + cross_instrument_quality_status: str + cross_instrument_quality_sha256: str + observed_event_count: int + synthetic_event_count: int + identity_event_count: int + identity_lineage_sha256: str + delivery_action_counts: Mapping[str, int] + benchmark_artifact_ids: tuple[str, ...] + quality_manifest_id: str = "" + schema_version: str = ( + RECONSTRUCTION_DELIVERY_QUALITY_MANIFEST_SCHEMA_VERSION + ) + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_DELIVERY_QUALITY_MANIFEST_SCHEMA_VERSION, + "reconstruction delivery quality manifest", + ) + for name in ( + "delivery_manifest_id", + "delivery_profile_id", + "final_validation_id", + "final_validation_status", + "cross_instrument_quality_status", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "delivery_mode", + ReconstructionDeliveryMode(self.delivery_mode), + ) + for name in ( + "delivery_output_content_sha256", + "cross_instrument_quality_sha256", + "identity_lineage_sha256", + ): + object.__setattr__( + self, name, _required_sha256(getattr(self, name), name) + ) + for name in ( + "observed_event_count", + "synthetic_event_count", + "identity_event_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + actions = { + _required_text(name): _positive_int(count, f"action.{name}") + for name, count in self.delivery_action_counts.items() + } + object.__setattr__( + self, "delivery_action_counts", dict(sorted(actions.items())) + ) + if self.delivery_mode is ReconstructionDeliveryMode.MODERN_REFERENCE: + if self.identity_event_count != self.synthetic_event_count: + raise ValueError( + "identity delivery count differs from synthetic" + ) + expected_actions = ( + {"identity": self.identity_event_count} + if self.identity_event_count + else {} + ) + if actions != expected_actions: + raise ValueError( + "modern delivery actions are not identity-only" + ) + artifacts = _normalized_text_tuple(self.benchmark_artifact_ids) + if not artifacts: + raise ValueError("delivery quality lacks benchmark artifacts") + object.__setattr__(self, "benchmark_artifact_ids", artifacts) + if self.final_validation_status != "passed": + raise ValueError("final delivery validation is not passing") + if self.cross_instrument_quality_status == "failed": + raise ValueError("cross-instrument delivery quality is failed") + expected = _stable_id("reconstruction-delivery-quality", self.payload()) + supplied = _optional_text(self.quality_manifest_id) + if supplied is not None and supplied != expected: + raise ValueError("delivery quality manifest_id differs") + object.__setattr__(self, "quality_manifest_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return compact generic quality evidence.""" + return { + "schema_version": self.schema_version, + "delivery_manifest_id": self.delivery_manifest_id, + "delivery_profile_id": self.delivery_profile_id, + "delivery_mode": self.delivery_mode.value, + "delivery_output_content_sha256": ( + self.delivery_output_content_sha256 + ), + "final_validation_id": self.final_validation_id, + "final_validation_status": self.final_validation_status, + "cross_instrument_quality_status": ( + self.cross_instrument_quality_status + ), + "cross_instrument_quality_sha256": ( + self.cross_instrument_quality_sha256 + ), + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "identity_event_count": self.identity_event_count, + "identity_lineage_sha256": self.identity_lineage_sha256, + "delivery_action_counts": dict(self.delivery_action_counts), + "benchmark_artifact_ids": list(self.benchmark_artifact_ids), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact JSON-compatible quality evidence.""" + return { + **self.payload(), + "quality_manifest_id": self.quality_manifest_id, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionDeliveryQualityManifestV1": + """Restore and verify generic delivery quality evidence.""" + _require_schema( + data, RECONSTRUCTION_DELIVERY_QUALITY_MANIFEST_SCHEMA_VERSION + ) + return cls( + delivery_manifest_id=str(data.get("delivery_manifest_id", "")), + delivery_profile_id=str(data.get("delivery_profile_id", "")), + delivery_mode=ReconstructionDeliveryMode( + str(data.get("delivery_mode", "")) + ), + delivery_output_content_sha256=str( + data.get("delivery_output_content_sha256", "") + ), + final_validation_id=str(data.get("final_validation_id", "")), + final_validation_status=str( + data.get("final_validation_status", "") + ), + cross_instrument_quality_status=str( + data.get("cross_instrument_quality_status", "") + ), + cross_instrument_quality_sha256=str( + data.get("cross_instrument_quality_sha256", "") + ), + observed_event_count=_strict_int( + data.get("observed_event_count"), "observed_event_count" + ), + synthetic_event_count=_strict_int( + data.get("synthetic_event_count"), "synthetic_event_count" + ), + identity_event_count=_strict_int( + data.get("identity_event_count"), "identity_event_count" + ), + identity_lineage_sha256=str( + data.get("identity_lineage_sha256", "") + ), + delivery_action_counts={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("delivery_action_counts"), + "delivery_action_counts", + ).items() + }, + benchmark_artifact_ids=_string_tuple( + data.get("benchmark_artifact_ids"), "benchmark_artifact_ids" + ), + quality_manifest_id=str(data.get("quality_manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionReplayManifestV1: + """Logical replay hash and pinned physical writer evidence.""" + + logical_content_sha256: str + partition_byte_sha256: str + logical_hash_algorithm: str + byte_hash_algorithm: str + writer_id: str + writer_library: str + writer_library_version: str + python_runtime: str + compression: str + row_group_size: int + canonicalized_metadata_exclusions: tuple[str, ...] + replay_manifest_id: str = "" + schema_version: str = RECONSTRUCTION_REPLAY_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_REPLAY_MANIFEST_SCHEMA_VERSION, + "reconstruction replay manifest", + ) + for name in ("logical_content_sha256", "partition_byte_sha256"): + object.__setattr__( + self, name, _required_sha256(getattr(self, name), name) + ) + for name in ( + "logical_hash_algorithm", + "byte_hash_algorithm", + "writer_id", + "writer_library", + "writer_library_version", + "python_runtime", + "compression", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + if self.logical_hash_algorithm != RECONSTRUCTION_LOGICAL_HASH_ALGORITHM: + raise ValueError("unsupported reconstruction logical hash") + if self.byte_hash_algorithm != RECONSTRUCTION_BYTE_HASH_ALGORITHM: + raise ValueError("unsupported reconstruction byte hash") + if self.writer_id != RECONSTRUCTION_WRITER_ID: + raise ValueError("unsupported reconstruction writer") + if self.compression != RECONSTRUCTION_COMPRESSION: + raise ValueError("unsupported reconstruction compression") + object.__setattr__( + self, + "row_group_size", + _positive_int(self.row_group_size, "row_group_size"), + ) + exclusions = _normalized_text_tuple( + self.canonicalized_metadata_exclusions + ) + object.__setattr__( + self, "canonicalized_metadata_exclusions", exclusions + ) + expected = _stable_id("reconstruction-replay", self.payload()) + supplied = _optional_text(self.replay_manifest_id) + if supplied is not None and supplied != expected: + raise ValueError("replay manifest_id differs") + object.__setattr__(self, "replay_manifest_id", expected) + + def payload(self) -> dict[str, JSONValue]: + """Return replay and writer evidence.""" + return { + "schema_version": self.schema_version, + "logical_content_sha256": self.logical_content_sha256, + "partition_byte_sha256": self.partition_byte_sha256, + "logical_hash_algorithm": self.logical_hash_algorithm, + "byte_hash_algorithm": self.byte_hash_algorithm, + "writer_id": self.writer_id, + "writer_library": self.writer_library, + "writer_library_version": self.writer_library_version, + "python_runtime": self.python_runtime, + "compression": self.compression, + "row_group_size": self.row_group_size, + "canonicalized_metadata_exclusions": list( + self.canonicalized_metadata_exclusions + ), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact replay-manifest JSON.""" + return {**self.payload(), "replay_manifest_id": self.replay_manifest_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionReplayManifestV1": + """Restore and verify replay evidence.""" + _require_schema(data, RECONSTRUCTION_REPLAY_MANIFEST_SCHEMA_VERSION) + return cls( + logical_content_sha256=str(data.get("logical_content_sha256", "")), + partition_byte_sha256=str(data.get("partition_byte_sha256", "")), + logical_hash_algorithm=str(data.get("logical_hash_algorithm", "")), + byte_hash_algorithm=str(data.get("byte_hash_algorithm", "")), + writer_id=str(data.get("writer_id", "")), + writer_library=str(data.get("writer_library", "")), + writer_library_version=str(data.get("writer_library_version", "")), + python_runtime=str(data.get("python_runtime", "")), + compression=str(data.get("compression", "")), + row_group_size=_strict_int( + data.get("row_group_size"), "row_group_size" + ), + canonicalized_metadata_exclusions=_string_tuple( + data.get("canonicalized_metadata_exclusions"), + "canonicalized_metadata_exclusions", + ), + replay_manifest_id=str(data.get("replay_manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionProductManifestV1: + """One atomically committed synchronized reconstruction unit.""" + + run_id: str + window_id: str + synchronization_unit_id: str + ensemble_member_id: str + broker_profile_id: str + symbol_group_id: str + symbols: tuple[str, ...] + symbol_event_counts: Mapping[str, int] + partitions: tuple[ReconstructionProductPartitionV1, ...] + source: ReconstructionSourceManifestV1 + constraints: ReconstructionConstraintManifestV1 + quality: ReconstructionQualityManifestV1 + replay: ReconstructionReplayManifestV1 + ensemble: ReconstructionEnsembleManifestV1 + retention: ReconstructionRetentionPlanV1 + publication_id: str = "" + manifest_id: str = "" + schema_version: str = RECONSTRUCTION_PRODUCT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_PRODUCT_SCHEMA_VERSION, + "reconstruction product manifest", + ) + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "ensemble_member_id", + "broker_profile_id", + "symbol_group_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + symbols = tuple( + sorted({_normalized_symbol(item) for item in self.symbols}) + ) + if not symbols: + raise ValueError("product manifest requires symbols") + object.__setattr__(self, "symbols", symbols) + counts = { + _normalized_symbol(symbol): _nonnegative_int( + count, f"symbol_event_counts.{symbol}" + ) + for symbol, count in self.symbol_event_counts.items() + } + if set(counts) != set(symbols): + raise ValueError("symbol counts do not cover the product group") + object.__setattr__( + self, "symbol_event_counts", dict(sorted(counts.items())) + ) + partitions = tuple( + sorted( + self.partitions, + key=lambda item: ( + item.symbol, + item.event_date, + item.partition_id, + ), + ) + ) + if not partitions or len(partitions) > MAX_RECONSTRUCTION_PARTITIONS: + raise ValueError("product partition count is empty or unbounded") + if len({item.relative_path for item in partitions}) != len(partitions): + raise ValueError("product contains duplicate partition paths") + actual_counts = dict.fromkeys(symbols, 0) + for partition in partitions: + if partition.symbol not in actual_counts: + raise ValueError("partition symbol is outside product group") + actual_counts[partition.symbol] += partition.row_count + if actual_counts != counts: + raise ValueError("product partition counts do not reconcile") + object.__setattr__(self, "partitions", partitions) + if not isinstance(self.source, ReconstructionSourceManifestV1): + raise TypeError("product requires source-manifest evidence") + if not isinstance(self.constraints, ReconstructionConstraintManifestV1): + raise TypeError("product requires constraint-manifest evidence") + if not isinstance(self.quality, ReconstructionQualityManifestV1): + raise TypeError("product requires quality-manifest evidence") + if not isinstance(self.replay, ReconstructionReplayManifestV1): + raise TypeError("product requires replay-manifest evidence") + if not isinstance(self.ensemble, ReconstructionEnsembleManifestV1): + raise TypeError("product requires ensemble-manifest evidence") + if not isinstance(self.retention, ReconstructionRetentionPlanV1): + raise TypeError("product requires retention-plan evidence") + if self.retention.run_id != self.run_id: + raise ValueError("retention run differs from product run") + if self.ensemble_member_id not in self.retention.retained_member_ids: + raise ValueError("product member is not retained") + if ( + self.ensemble.run_id != self.run_id + or self.ensemble.materialized_member_id != self.ensemble_member_id + or self.ensemble.primary_member_id + != self.retention.primary_member_id + or self.ensemble.retained_member_ids + != self.retention.retained_member_ids + or dict(self.ensemble.member_event_estimates) + != dict(self.retention.member_event_counts) + or self.ensemble.retention_plan_id != self.retention.plan_id + ): + raise ValueError( + "ensemble manifest does not reconcile with retention preflight" + ) + actual_rows = sum(counts.values()) + if len(partitions) > self.retention.estimated_partition_count: + raise ValueError("product partitions exceed the preflight estimate") + if ( + actual_rows + > self.retention.member_event_counts[self.ensemble_member_id] + ): + raise ValueError("product rows exceed the preflight estimate") + if ( + self.source.observed_event_count + + (self.constraints.synthetic_event_count) + != actual_rows + ): + raise ValueError("source/constraint counts do not reconcile") + if self.quality.broker_observed_event_count != ( + self.source.observed_event_count + ): + raise ValueError("broker/source observed counts do not reconcile") + if self.quality.broker_synthetic_event_count != ( + self.constraints.synthetic_event_count + ): + raise ValueError( + "broker/constraint synthetic counts do not reconcile" + ) + expected_publication = _stable_id( + "reconstruction-publication", self.publication_payload() + ) + supplied_publication = _optional_text(self.publication_id) + if ( + supplied_publication is not None + and supplied_publication != expected_publication + ): + raise ValueError("publication_id differs from logical identity") + object.__setattr__(self, "publication_id", expected_publication) + expected_manifest = _stable_id( + "reconstruction-manifest", self.payload() + ) + supplied_manifest = _optional_text(self.manifest_id) + if ( + supplied_manifest is not None + and supplied_manifest != expected_manifest + ): + raise ValueError("manifest_id differs from physical evidence") + object.__setattr__(self, "manifest_id", expected_manifest) + encoded = self.to_json().encode("utf-8") + if len(encoded) > MAX_RECONSTRUCTION_MANIFEST_BYTES: + raise ValueError("reconstruction manifest exceeds size limit") + + @property + def event_count(self) -> int: + """Return total durable events.""" + return sum(self.symbol_event_counts.values()) + + @property + def observed_event_count(self) -> int: + """Return total immutable observed rows.""" + return self.source.observed_event_count + + @property + def synthetic_event_count(self) -> int: + """Return total accepted synthetic rows.""" + return self.constraints.synthetic_event_count + + @property + def min_event_time_ns(self) -> int: + """Return the first event time in the synchronized product.""" + return min(item.min_event_time_ns for item in self.partitions) + + @property + def max_event_time_ns(self) -> int: + """Return the last event time in the synchronized product.""" + return max(item.max_event_time_ns for item in self.partitions) + + def publication_payload(self) -> dict[str, JSONValue]: + """Return writer-independent publication identity.""" + return { + "schema_version": self.schema_version, + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "ensemble_member_id": self.ensemble_member_id, + "broker_profile_id": self.broker_profile_id, + "symbol_group_id": self.symbol_group_id, + "symbols": list(self.symbols), + "symbol_event_counts": dict(self.symbol_event_counts), + "logical_content_sha256": self.replay.logical_content_sha256, + "source_manifest_id": self.source.source_manifest_id, + "constraint_manifest_id": self.constraints.constraint_manifest_id, + "quality_manifest_id": self.quality.quality_manifest_id, + "ensemble_manifest_id": self.ensemble.ensemble_manifest_id, + "retention_plan_id": self.retention.plan_id, + } + + def payload(self) -> dict[str, JSONValue]: + """Return complete compact manifest evidence.""" + return { + **self.publication_payload(), + "publication_id": self.publication_id, + "partitions": [item.to_dict() for item in self.partitions], + "source": self.source.to_dict(), + "constraints": self.constraints.to_dict(), + "quality": self.quality.to_dict(), + "replay": self.replay.to_dict(), + "ensemble": self.ensemble.to_dict(), + "retention": self.retention.to_dict(), + "event_count": self.event_count, + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "min_event_time_ns": self.min_event_time_ns, + "max_event_time_ns": self.max_event_time_ns, + "event_rows_inline": False, + "analytical_frame_columns_inline": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic final-product JSON.""" + return {**self.payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionProductManifestV1": + """Restore and reconcile a final-product manifest.""" + _require_schema(data, RECONSTRUCTION_PRODUCT_SCHEMA_VERSION) + _require_derived( + data, "event_schema_version", SYNTHETIC_EVENT_SCHEMA_VERSION + ) + _require_derived(data, "event_rows_inline", False) + _require_derived(data, "analytical_frame_columns_inline", False) + manifest = cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + broker_profile_id=str(data.get("broker_profile_id", "")), + symbol_group_id=str(data.get("symbol_group_id", "")), + symbols=_string_tuple(data.get("symbols"), "symbols"), + symbol_event_counts={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("symbol_event_counts"), "symbol_event_counts" + ).items() + }, + partitions=tuple( + ReconstructionProductPartitionV1.from_dict(item) + for item in _mapping_sequence( + data.get("partitions"), "partitions" + ) + ), + source=ReconstructionSourceManifestV1.from_dict( + _mapping(data.get("source"), "source") + ), + constraints=ReconstructionConstraintManifestV1.from_dict( + _mapping(data.get("constraints"), "constraints") + ), + quality=ReconstructionQualityManifestV1.from_dict( + _mapping(data.get("quality"), "quality") + ), + replay=ReconstructionReplayManifestV1.from_dict( + _mapping(data.get("replay"), "replay") + ), + ensemble=ReconstructionEnsembleManifestV1.from_dict( + _mapping(data.get("ensemble"), "ensemble") + ), + retention=ReconstructionRetentionPlanV1.from_dict( + _mapping(data.get("retention"), "retention") + ), + publication_id=str(data.get("publication_id", "")), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + for name in ( + "event_count", + "observed_event_count", + "synthetic_event_count", + "min_event_time_ns", + "max_event_time_ns", + ): + if data.get(name) != getattr(manifest, name): + raise ValueError(f"derived manifest field {name} differs") + return manifest + + @classmethod + def from_json(cls, text: str) -> "ReconstructionProductManifestV1": + """Restore a final-product manifest from JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionProductManifestV2: + """Generic-delivery synchronized product without broker impersonation.""" + + run_id: str + window_id: str + synchronization_unit_id: str + ensemble_member_id: str + delivery_profile_id: str + symbol_group_id: str + symbols: tuple[str, ...] + symbol_event_counts: Mapping[str, int] + partitions: tuple[ReconstructionProductPartitionV1, ...] + source: ReconstructionSourceManifestV1 + constraints: ReconstructionConstraintManifestV1 + quality: ReconstructionDeliveryQualityManifestV1 + replay: ReconstructionReplayManifestV1 + ensemble: ReconstructionEnsembleManifestV1 + retention: ReconstructionRetentionPlanV1 + publication_id: str = "" + manifest_id: str = "" + schema_version: str = RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_version( + self.schema_version, + RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION, + "reconstruction product v2 manifest", + ) + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "ensemble_member_id", + "delivery_profile_id", + "symbol_group_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + symbols = tuple( + sorted({_normalized_symbol(item) for item in self.symbols}) + ) + if not symbols: + raise ValueError("product v2 manifest requires symbols") + object.__setattr__(self, "symbols", symbols) + counts = { + _normalized_symbol(symbol): _nonnegative_int( + count, f"symbol_event_counts.{symbol}" + ) + for symbol, count in self.symbol_event_counts.items() + } + if set(counts) != set(symbols): + raise ValueError("symbol counts do not cover product v2 group") + object.__setattr__( + self, "symbol_event_counts", dict(sorted(counts.items())) + ) + partitions = tuple( + sorted( + self.partitions, + key=lambda item: ( + item.symbol, + item.event_date, + item.partition_id, + ), + ) + ) + if not partitions or len(partitions) > MAX_RECONSTRUCTION_PARTITIONS: + raise ValueError("product v2 partition count is empty or unbounded") + if len({item.relative_path for item in partitions}) != len(partitions): + raise ValueError("product v2 has duplicate partition paths") + actual_counts = dict.fromkeys(symbols, 0) + for partition in partitions: + if partition.symbol not in actual_counts: + raise ValueError("partition symbol is outside product v2 group") + actual_counts[partition.symbol] += partition.row_count + if actual_counts != counts: + raise ValueError("product v2 partition counts do not reconcile") + object.__setattr__(self, "partitions", partitions) + if not isinstance(self.source, ReconstructionSourceManifestV1): + raise TypeError("product v2 requires source evidence") + if not isinstance(self.constraints, ReconstructionConstraintManifestV1): + raise TypeError("product v2 requires constraint evidence") + if not isinstance( + self.quality, ReconstructionDeliveryQualityManifestV1 + ): + raise TypeError("product v2 requires delivery quality evidence") + if not isinstance(self.replay, ReconstructionReplayManifestV1): + raise TypeError("product v2 requires replay evidence") + if not isinstance(self.ensemble, ReconstructionEnsembleManifestV1): + raise TypeError("product v2 requires ensemble evidence") + if not isinstance(self.retention, ReconstructionRetentionPlanV1): + raise TypeError("product v2 requires retention evidence") + if self.retention.run_id != self.run_id: + raise ValueError("retention run differs from product v2 run") + if self.ensemble_member_id not in self.retention.retained_member_ids: + raise ValueError("product v2 member is not retained") + if ( + self.ensemble.run_id != self.run_id + or self.ensemble.materialized_member_id != self.ensemble_member_id + or self.ensemble.primary_member_id + != self.retention.primary_member_id + or self.ensemble.retained_member_ids + != self.retention.retained_member_ids + or dict(self.ensemble.member_event_estimates) + != dict(self.retention.member_event_counts) + or self.ensemble.retention_plan_id != self.retention.plan_id + ): + raise ValueError("product v2 ensemble does not reconcile") + actual_rows = sum(counts.values()) + if len(partitions) > self.retention.estimated_partition_count: + raise ValueError("product v2 partitions exceed preflight") + if ( + actual_rows + > self.retention.member_event_counts[self.ensemble_member_id] + ): + raise ValueError("product v2 rows exceed preflight") + if ( + self.source.observed_event_count + + self.constraints.synthetic_event_count + != actual_rows + ): + raise ValueError("product v2 source/constraint counts differ") + if ( + self.quality.observed_event_count + != self.source.observed_event_count + ): + raise ValueError("delivery/source observed counts differ") + if ( + self.quality.synthetic_event_count + != self.constraints.synthetic_event_count + ): + raise ValueError("delivery/constraint synthetic counts differ") + if self.quality.delivery_profile_id != self.delivery_profile_id: + raise ValueError("delivery profile differs from product v2 axis") + expected_publication = _stable_id( + "reconstruction-publication-v2", self.publication_payload() + ) + supplied_publication = _optional_text(self.publication_id) + if ( + supplied_publication is not None + and supplied_publication != expected_publication + ): + raise ValueError("product v2 publication_id differs") + object.__setattr__(self, "publication_id", expected_publication) + expected_manifest = _stable_id( + "reconstruction-manifest-v2", self.payload() + ) + supplied_manifest = _optional_text(self.manifest_id) + if ( + supplied_manifest is not None + and supplied_manifest != expected_manifest + ): + raise ValueError("product v2 manifest_id differs") + object.__setattr__(self, "manifest_id", expected_manifest) + if ( + len(self.to_json().encode("utf-8")) + > MAX_RECONSTRUCTION_MANIFEST_BYTES + ): + raise ValueError("reconstruction product v2 manifest exceeds limit") + + @property + def event_count(self) -> int: + """Return total durable events.""" + return sum(self.symbol_event_counts.values()) + + @property + def observed_event_count(self) -> int: + """Return total immutable observed rows.""" + return self.source.observed_event_count + + @property + def synthetic_event_count(self) -> int: + """Return total accepted synthetic rows.""" + return self.constraints.synthetic_event_count + + @property + def min_event_time_ns(self) -> int: + """Return the first product event time.""" + return min(item.min_event_time_ns for item in self.partitions) + + @property + def max_event_time_ns(self) -> int: + """Return the last product event time.""" + return max(item.max_event_time_ns for item in self.partitions) + + def publication_payload(self) -> dict[str, JSONValue]: + """Return writer-independent generic-delivery publication identity.""" + return { + "schema_version": self.schema_version, + "event_schema_version": SYNTHETIC_EVENT_SCHEMA_VERSION, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "ensemble_member_id": self.ensemble_member_id, + "delivery_profile_id": self.delivery_profile_id, + "delivery_mode": self.quality.delivery_mode.value, + "symbol_group_id": self.symbol_group_id, + "symbols": list(self.symbols), + "symbol_event_counts": dict(self.symbol_event_counts), + "logical_content_sha256": self.replay.logical_content_sha256, + "source_manifest_id": self.source.source_manifest_id, + "constraint_manifest_id": self.constraints.constraint_manifest_id, + "quality_manifest_id": self.quality.quality_manifest_id, + "ensemble_manifest_id": self.ensemble.ensemble_manifest_id, + "retention_plan_id": self.retention.plan_id, + } + + def payload(self) -> dict[str, JSONValue]: + """Return complete compact product v2 evidence.""" + return { + **self.publication_payload(), + "publication_id": self.publication_id, + "partitions": [item.to_dict() for item in self.partitions], + "source": self.source.to_dict(), + "constraints": self.constraints.to_dict(), + "quality": self.quality.to_dict(), + "replay": self.replay.to_dict(), + "ensemble": self.ensemble.to_dict(), + "retention": self.retention.to_dict(), + "event_count": self.event_count, + "observed_event_count": self.observed_event_count, + "synthetic_event_count": self.synthetic_event_count, + "min_event_time_ns": self.min_event_time_ns, + "max_event_time_ns": self.max_event_time_ns, + "event_rows_inline": False, + "analytical_frame_columns_inline": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic product v2 JSON.""" + return {**self.payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + """Return deterministic compact product v2 JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionProductManifestV2": + """Restore and reconcile a generic-delivery product manifest.""" + _require_schema(data, RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION) + _require_derived( + data, "event_schema_version", SYNTHETIC_EVENT_SCHEMA_VERSION + ) + _require_derived(data, "event_rows_inline", False) + _require_derived(data, "analytical_frame_columns_inline", False) + manifest = cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + delivery_profile_id=str(data.get("delivery_profile_id", "")), + symbol_group_id=str(data.get("symbol_group_id", "")), + symbols=_string_tuple(data.get("symbols"), "symbols"), + symbol_event_counts={ + str(key): _strict_int(value, str(key)) + for key, value in _mapping( + data.get("symbol_event_counts"), "symbol_event_counts" + ).items() + }, + partitions=tuple( + ReconstructionProductPartitionV1.from_dict(item) + for item in _mapping_sequence( + data.get("partitions"), "partitions" + ) + ), + source=ReconstructionSourceManifestV1.from_dict( + _mapping(data.get("source"), "source") + ), + constraints=ReconstructionConstraintManifestV1.from_dict( + _mapping(data.get("constraints"), "constraints") + ), + quality=ReconstructionDeliveryQualityManifestV1.from_dict( + _mapping(data.get("quality"), "quality") + ), + replay=ReconstructionReplayManifestV1.from_dict( + _mapping(data.get("replay"), "replay") + ), + ensemble=ReconstructionEnsembleManifestV1.from_dict( + _mapping(data.get("ensemble"), "ensemble") + ), + retention=ReconstructionRetentionPlanV1.from_dict( + _mapping(data.get("retention"), "retention") + ), + publication_id=str(data.get("publication_id", "")), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived( + data, "delivery_mode", manifest.quality.delivery_mode.value + ) + for name in ( + "event_count", + "observed_event_count", + "synthetic_event_count", + "min_event_time_ns", + "max_event_time_ns", + ): + if data.get(name) != getattr(manifest, name): + raise ValueError(f"derived product v2 field {name} differs") + return manifest + + @classmethod + def from_json(cls, text: str) -> "ReconstructionProductManifestV2": + """Restore product v2 from JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class StagedReconstructionPublicationV1: + """Process-local reference to a validated, undiscoverable transaction.""" + + root: Path + staging_directory: Path + committed_directory: Path + manifest: ReconstructionProductManifestV1 + + @property + def manifest_path(self) -> Path: + """Return the temporary manifest path.""" + return self.staging_directory / RECONSTRUCTION_MANIFEST_FILENAME + + @property + def manifest_ref(self) -> ArtifactRef: + """Return a strong temporary manifest reference for checkpoints.""" + return _artifact_ref_for_manifest(self.manifest_path, self.manifest) + + +@dataclass(frozen=True, slots=True) +class PublishedReconstructionV1: + """One committed publication and whether it was an idempotent retry.""" + + manifest: ReconstructionProductManifestV1 + manifest_path: Path + manifest_ref: ArtifactRef + idempotent_retry: bool + + +@dataclass(frozen=True, slots=True) +class StagedReconstructionPublicationV2: + """Validated generic-delivery transaction awaiting atomic promotion.""" + + root: Path + staging_directory: Path + committed_directory: Path + manifest: ReconstructionProductManifestV2 + + @property + def manifest_path(self) -> Path: + """Return the temporary manifest path.""" + return self.staging_directory / RECONSTRUCTION_MANIFEST_FILENAME + + +@dataclass(frozen=True, slots=True) +class PublishedReconstructionV2: + """One committed generic-delivery publication and retry status.""" + + manifest: ReconstructionProductManifestV2 + manifest_path: Path + manifest_ref: ArtifactRef + idempotent_retry: bool + + +def reconstruction_logical_content_sha256( + events: Iterable[SyntheticEventV1], +) -> str: + """Hash exact event rows independently of partition placement.""" + ordered = sorted(events, key=_event_order_key) + digest = hashlib.sha256(_LOGICAL_HASH_HEADER) + for event in ordered: + _update_event_digest(digest, event) + return digest.hexdigest() + + +def stage_reconstruction_publication( + root: str | Path, + rendered_group: BrokerRenderedGroupV1, + *, + immutable_source_anchors: Iterable[SyntheticEventV1], + symbol_group_id: str, + retention_plan: ReconstructionRetentionPlanV1, + storage_policy: ReconstructionStoragePolicyV1, + row_group_size: int = DEFAULT_RECONSTRUCTION_ROW_GROUP_SIZE, +) -> StagedReconstructionPublicationV1: + """Write and validate one synchronized group below hidden scratch.""" + _validate_publication_inputs(rendered_group, retention_plan, storage_policy) + group_id = _required_text(symbol_group_id) + row_group = _positive_int(row_group_size, "row_group_size") + events = tuple( + event + for stream in sorted( + rendered_group.streams, key=lambda item: item.symbol + ) + for event in stream.events + ) + if not events: + raise ReconstructionPersistenceError( + "final reconstruction group contains no events" + ) + anchors = tuple(immutable_source_anchors) + _validate_immutable_anchors(events, anchors) + logical_hash = reconstruction_logical_content_sha256(events) + root_path = Path(root).expanduser().resolve() + axis_directory = _axis_directory( + root_path, + run_id=rendered_group.manifest.run_id, + broker_profile_id=rendered_group.manifest.fingerprint_id, + ensemble_member_id=rendered_group.manifest.ensemble_member_id, + symbol_group_id=group_id, + ) + scratch = axis_directory / ".scratch" + scratch.mkdir(parents=True, exist_ok=True) + staging_directory = Path( + tempfile.mkdtemp(prefix="publication.tmp-", dir=scratch) + ) + try: + partitions = _write_product_partitions( + staging_directory, + rendered_group.streams, + row_group_size=row_group, + ) + source = _source_manifest(events, anchors) + constraints = _constraint_manifest(events) + quality = _quality_manifest(rendered_group) + ensemble = ReconstructionEnsembleManifestV1( + run_id=retention_plan.run_id, + materialized_member_id=(rendered_group.manifest.ensemble_member_id), + primary_member_id=retention_plan.primary_member_id, + retained_member_ids=retention_plan.retained_member_ids, + member_event_estimates=retention_plan.member_event_counts, + retention_plan_id=retention_plan.plan_id, + ) + pa, _ = _arrow_modules() + replay = ReconstructionReplayManifestV1( + logical_content_sha256=logical_hash, + partition_byte_sha256=_partition_byte_digest(partitions), + logical_hash_algorithm=RECONSTRUCTION_LOGICAL_HASH_ALGORITHM, + byte_hash_algorithm=RECONSTRUCTION_BYTE_HASH_ALGORITHM, + writer_id=RECONSTRUCTION_WRITER_ID, + writer_library="pyarrow", + writer_library_version=str(pa.__version__), + python_runtime=( + f"{sys.version_info.major}.{sys.version_info.minor}." + f"{sys.version_info.micro}" + ), + compression=RECONSTRUCTION_COMPRESSION, + row_group_size=row_group, + canonicalized_metadata_exclusions=(), + ) + counts = { + stream.symbol: len(stream.events) + for stream in rendered_group.streams + } + manifest = ReconstructionProductManifestV1( + run_id=rendered_group.manifest.run_id, + window_id=rendered_group.manifest.window_id, + synchronization_unit_id=( + rendered_group.manifest.synchronization_unit_id + ), + ensemble_member_id=(rendered_group.manifest.ensemble_member_id), + broker_profile_id=rendered_group.manifest.fingerprint_id, + symbol_group_id=group_id, + symbols=tuple(counts), + symbol_event_counts=counts, + partitions=partitions, + source=source, + constraints=constraints, + quality=quality, + replay=replay, + ensemble=ensemble, + retention=retention_plan, + ) + manifest_bytes = manifest.to_json().encode("utf-8") + _validate_actual_storage(partitions, manifest_bytes, storage_policy) + committed_directory = ( + axis_directory + / "commits" + / _path_component(manifest.publication_id) + ) + _atomic_write_bytes( + staging_directory / RECONSTRUCTION_MANIFEST_FILENAME, + manifest_bytes, + ) + staged = StagedReconstructionPublicationV1( + root=root_path, + staging_directory=staging_directory, + committed_directory=committed_directory, + manifest=manifest, + ) + _verify_publication_directory( + staging_directory, + manifest, + require_committed_layout=False, + ) + return staged + except Exception: + _remove_scratch_entry(staging_directory, root_path) + raise + + +def commit_reconstruction_publication( + staged: StagedReconstructionPublicationV1, +) -> PublishedReconstructionV1: + """Revalidate and atomically promote one staged publication.""" + if not isinstance(staged, StagedReconstructionPublicationV1): + raise TypeError("commit requires a staged reconstruction publication") + manifest = staged.manifest + final_directory = staged.committed_directory + if final_directory.exists(): + existing_path = final_directory / RECONSTRUCTION_MANIFEST_FILENAME + existing = verify_reconstruction_publication(existing_path) + if not isinstance(existing, ReconstructionProductManifestV1): + raise ReconstructionPersistenceError( + "legacy publication identity contains a delivery manifest" + ) + if existing != manifest: + raise ReconstructionPersistenceError( + "publication identity already contains different evidence" + ) + if staged.staging_directory.exists(): + _remove_scratch_entry(staged.staging_directory, staged.root) + return PublishedReconstructionV1( + manifest=existing, + manifest_path=existing_path, + manifest_ref=_artifact_ref_for_manifest(existing_path, existing), + idempotent_retry=True, + ) + _verify_publication_directory( + staged.staging_directory, + manifest, + require_committed_layout=False, + ) + final_directory.parent.mkdir(parents=True, exist_ok=True) + try: + os.replace(staged.staging_directory, final_directory) + except OSError: + if not final_directory.exists(): + raise + existing_path = final_directory / RECONSTRUCTION_MANIFEST_FILENAME + existing = verify_reconstruction_publication(existing_path) + if not isinstance(existing, ReconstructionProductManifestV1): + raise ReconstructionPersistenceError( + "concurrent legacy commit produced a delivery manifest" + ) + if existing != manifest: + raise ReconstructionPersistenceError( + "concurrent publication committed different evidence" + ) + if staged.staging_directory.exists(): + _remove_scratch_entry(staged.staging_directory, staged.root) + return PublishedReconstructionV1( + manifest=existing, + manifest_path=existing_path, + manifest_ref=_artifact_ref_for_manifest(existing_path, existing), + idempotent_retry=True, + ) + _fsync_directory(final_directory.parent) + manifest_path = final_directory / RECONSTRUCTION_MANIFEST_FILENAME + committed = verify_reconstruction_publication(manifest_path) + if not isinstance(committed, ReconstructionProductManifestV1): + raise ReconstructionPersistenceError( + "committed legacy publication restored a delivery manifest" + ) + return PublishedReconstructionV1( + manifest=committed, + manifest_path=manifest_path, + manifest_ref=_artifact_ref_for_manifest(manifest_path, committed), + idempotent_retry=False, + ) + + +def publish_reconstruction_group( + root: str | Path, + rendered_group: BrokerRenderedGroupV1, + *, + immutable_source_anchors: Iterable[SyntheticEventV1], + symbol_group_id: str, + retention_plan: ReconstructionRetentionPlanV1, + storage_policy: ReconstructionStoragePolicyV1, + row_group_size: int = DEFAULT_RECONSTRUCTION_ROW_GROUP_SIZE, +) -> PublishedReconstructionV1: + """Stage, validate, and atomically commit one final reconstruction group.""" + group_id = _required_text(symbol_group_id) + row_group = _positive_int(row_group_size, "row_group_size") + root_path = Path(root).expanduser().resolve() + _validate_publication_inputs(rendered_group, retention_plan, storage_policy) + output_events = tuple( + event for stream in rendered_group.streams for event in stream.events + ) + source_anchors = tuple(immutable_source_anchors) + _validate_immutable_anchors(output_events, source_anchors) + existing = _find_matching_publication( + root_path, + rendered_group, + symbol_group_id=group_id, + retention_plan=retention_plan, + row_group_size=row_group, + ) + if existing is not None: + return existing + staged = stage_reconstruction_publication( + root_path, + rendered_group, + immutable_source_anchors=source_anchors, + symbol_group_id=group_id, + retention_plan=retention_plan, + storage_policy=storage_policy, + row_group_size=row_group, + ) + try: + return commit_reconstruction_publication(staged) + except Exception: + if staged.staging_directory.exists(): + _remove_scratch_entry(staged.staging_directory, root_path) + raise + + +def stage_delivery_reconstruction_publication( + root: str | Path, + delivered_group: ReconstructionDeliveredGroupV1, + *, + final_validation: CrossCurrencyValidationReportV1, + benchmark_artifact_ids: Sequence[str], + benchmark_evidence: Mapping[str, JSONValue], + immutable_source_anchors: Iterable[SyntheticEventV1], + symbol_group_id: str, + retention_plan: ReconstructionRetentionPlanV1, + storage_policy: ReconstructionStoragePolicyV1, + staging_root: str | Path, + row_group_size: int = DEFAULT_RECONSTRUCTION_ROW_GROUP_SIZE, +) -> StagedReconstructionPublicationV2: + """Stage one validated generic-delivery group in cancellable scratch.""" + _validate_delivery_publication_inputs( + delivered_group, + final_validation, + retention_plan, + storage_policy, + ) + group_id = _required_text(symbol_group_id) + row_group = _positive_int(row_group_size, "row_group_size") + events = tuple( + event + for stream in sorted( + delivered_group.streams, key=lambda item: item.symbol + ) + for event in stream.events + ) + if not events: + raise ReconstructionPersistenceError( + "final generic-delivery group contains no events" + ) + anchors = tuple(immutable_source_anchors) + _validate_immutable_anchors(events, anchors) + logical_hash = reconstruction_logical_content_sha256(events) + root_path = Path(root).expanduser().resolve() + manifest = delivered_group.manifest + axis_directory = _delivery_axis_directory( + root_path, + run_id=manifest.run_id, + delivery_profile_id=manifest.delivery_profile_id, + ensemble_member_id=manifest.ensemble_member_id, + symbol_group_id=group_id, + ) + axis_directory.mkdir(parents=True, exist_ok=True) + scratch = Path(staging_root).expanduser().resolve() + scratch.mkdir(parents=True, exist_ok=True) + if scratch.stat().st_dev != axis_directory.stat().st_dev: + raise ReconstructionPersistenceError( + "window scratch and output root are on different filesystems" + ) + staging_directory = Path( + tempfile.mkdtemp(prefix="publication.tmp-", dir=scratch) + ) + try: + partitions = _write_product_partitions( + staging_directory, + delivered_group.streams, + row_group_size=row_group, + ) + source = _source_manifest(events, anchors) + constraints = _constraint_manifest(events) + quality = _delivery_quality_manifest( + delivered_group, + final_validation=final_validation, + benchmark_artifact_ids=benchmark_artifact_ids, + benchmark_evidence=benchmark_evidence, + ) + ensemble = ReconstructionEnsembleManifestV1( + run_id=retention_plan.run_id, + materialized_member_id=manifest.ensemble_member_id, + primary_member_id=retention_plan.primary_member_id, + retained_member_ids=retention_plan.retained_member_ids, + member_event_estimates=retention_plan.member_event_counts, + retention_plan_id=retention_plan.plan_id, + ) + pa, _ = _arrow_modules() + replay = ReconstructionReplayManifestV1( + logical_content_sha256=logical_hash, + partition_byte_sha256=_partition_byte_digest(partitions), + logical_hash_algorithm=RECONSTRUCTION_LOGICAL_HASH_ALGORITHM, + byte_hash_algorithm=RECONSTRUCTION_BYTE_HASH_ALGORITHM, + writer_id=RECONSTRUCTION_WRITER_ID, + writer_library="pyarrow", + writer_library_version=str(pa.__version__), + python_runtime=( + f"{sys.version_info.major}.{sys.version_info.minor}." + f"{sys.version_info.micro}" + ), + compression=RECONSTRUCTION_COMPRESSION, + row_group_size=row_group, + canonicalized_metadata_exclusions=(), + ) + counts = { + stream.symbol: len(stream.events) + for stream in delivered_group.streams + } + product = ReconstructionProductManifestV2( + run_id=manifest.run_id, + window_id=manifest.window_id, + synchronization_unit_id=manifest.synchronization_unit_id, + ensemble_member_id=manifest.ensemble_member_id, + delivery_profile_id=manifest.delivery_profile_id, + symbol_group_id=group_id, + symbols=tuple(counts), + symbol_event_counts=counts, + partitions=partitions, + source=source, + constraints=constraints, + quality=quality, + replay=replay, + ensemble=ensemble, + retention=retention_plan, + ) + manifest_bytes = product.to_json().encode("utf-8") + _validate_actual_storage(partitions, manifest_bytes, storage_policy) + committed_directory = ( + axis_directory / "commits" / _path_component(product.publication_id) + ) + _atomic_write_bytes( + staging_directory / RECONSTRUCTION_MANIFEST_FILENAME, + manifest_bytes, + ) + staged = StagedReconstructionPublicationV2( + root=root_path, + staging_directory=staging_directory, + committed_directory=committed_directory, + manifest=product, + ) + _verify_publication_directory( + staging_directory, + product, + require_committed_layout=False, + ) + return staged + except Exception: + if staging_directory.exists(): + shutil.rmtree(staging_directory) + raise + + +def commit_delivery_reconstruction_publication( + staged: StagedReconstructionPublicationV2, +) -> PublishedReconstructionV2: + """Atomically promote or recover one generic-delivery publication.""" + if not isinstance(staged, StagedReconstructionPublicationV2): + raise TypeError("delivery commit requires a staged v2 publication") + manifest = staged.manifest + final_directory = staged.committed_directory + if final_directory.exists(): + existing_path = final_directory / RECONSTRUCTION_MANIFEST_FILENAME + existing = verify_reconstruction_publication(existing_path) + if not isinstance(existing, ReconstructionProductManifestV2): + raise ReconstructionPersistenceError( + "delivery publication identity contains a legacy manifest" + ) + if existing != manifest: + raise ReconstructionPersistenceError( + "delivery publication contains different evidence" + ) + if staged.staging_directory.exists(): + shutil.rmtree(staged.staging_directory) + return PublishedReconstructionV2( + manifest=existing, + manifest_path=existing_path, + manifest_ref=_artifact_ref_for_manifest(existing_path, existing), + idempotent_retry=True, + ) + _verify_publication_directory( + staged.staging_directory, + manifest, + require_committed_layout=False, + ) + final_directory.parent.mkdir(parents=True, exist_ok=True) + try: + os.replace(staged.staging_directory, final_directory) + except OSError: + if not final_directory.exists(): + raise + existing_path = final_directory / RECONSTRUCTION_MANIFEST_FILENAME + existing = verify_reconstruction_publication(existing_path) + if not isinstance(existing, ReconstructionProductManifestV2): + raise ReconstructionPersistenceError( + "concurrent delivery commit produced a legacy manifest" + ) + if existing != manifest: + raise ReconstructionPersistenceError( + "concurrent delivery commit produced different evidence" + ) + if staged.staging_directory.exists(): + shutil.rmtree(staged.staging_directory) + return PublishedReconstructionV2( + manifest=existing, + manifest_path=existing_path, + manifest_ref=_artifact_ref_for_manifest(existing_path, existing), + idempotent_retry=True, + ) + _fsync_directory(final_directory.parent) + manifest_path = final_directory / RECONSTRUCTION_MANIFEST_FILENAME + committed = verify_reconstruction_publication(manifest_path) + if not isinstance(committed, ReconstructionProductManifestV2): + raise ReconstructionPersistenceError( + "committed delivery publication restored a legacy manifest" + ) + return PublishedReconstructionV2( + manifest=committed, + manifest_path=manifest_path, + manifest_ref=_artifact_ref_for_manifest(manifest_path, committed), + idempotent_retry=False, + ) + + +def load_reconstruction_manifest( + path: str | Path, +) -> ReconstructionProductManifestV1 | ReconstructionProductManifestV2: + """Load and verify compact manifest identities without reading Parquet.""" + target = Path(path) + payload = target.read_bytes() + if len(payload) > MAX_RECONSTRUCTION_MANIFEST_BYTES: + raise ReconstructionPersistenceError( + "reconstruction manifest exceeds size limit" + ) + try: + text = payload.decode("utf-8") + except UnicodeDecodeError as err: + raise ReconstructionPersistenceError( + "reconstruction manifest is not UTF-8" + ) from err + data = _json_mapping(text) + version = str(data.get("schema_version", "")) + if version == RECONSTRUCTION_PRODUCT_SCHEMA_VERSION: + return ReconstructionProductManifestV1.from_dict(data) + if version == RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION: + return ReconstructionProductManifestV2.from_dict(data) + raise ReconstructionPersistenceError( + "unsupported reconstruction product manifest version" + ) + + +def verify_reconstruction_publication( + manifest_path: str | Path, +) -> ReconstructionProductManifestV1 | ReconstructionProductManifestV2: + """Fail closed unless every committed file and replay hash reconciles.""" + path = Path(manifest_path).expanduser().resolve() + manifest = load_reconstruction_manifest(path) + _validate_committed_manifest_location(path, manifest) + _verify_publication_directory( + path.parent, manifest, require_committed_layout=True + ) + return manifest + + +def discover_reconstruction_manifests( + root: str | Path, + *, + run_id: str | None = None, + broker_profile_id: str | None = None, + delivery_profile_id: str | None = None, + ensemble_member_id: str | None = None, + symbol_group_id: str | None = None, +) -> tuple[Path, ...]: + """List only fully committed, verified product manifests.""" + product_root = ( + Path(root).expanduser().resolve() / RECONSTRUCTION_PRODUCT_DIRECTORY + ) + if not product_root.exists(): + return () + matches: list[Path] = [] + for path in sorted(product_root.glob("**/commits/*/manifest.json")): + manifest = load_reconstruction_manifest(path) + _validate_committed_manifest_location(path.resolve(), manifest) + if run_id is not None and manifest.run_id != run_id: + continue + if broker_profile_id is not None: + if not isinstance(manifest, ReconstructionProductManifestV1): + continue + if manifest.broker_profile_id != broker_profile_id: + continue + if delivery_profile_id is not None: + if not isinstance(manifest, ReconstructionProductManifestV2): + continue + if manifest.delivery_profile_id != delivery_profile_id: + continue + if ( + ensemble_member_id is not None + and manifest.ensemble_member_id != ensemble_member_id + ): + continue + if ( + symbol_group_id is not None + and manifest.symbol_group_id != symbol_group_id + ): + continue + matches.append(path.resolve()) + return tuple(matches) + + +def reconstruction_parquet_paths( + manifest_path: str | Path, + *, + symbols: Iterable[str] = (), + start_ns: int | None = None, + end_ns: int | None = None, +) -> tuple[Path, ...]: + """Select only physical partitions overlapping symbol/time predicates.""" + path = Path(manifest_path).expanduser().resolve() + manifest = load_reconstruction_manifest(path) + _validate_committed_manifest_location(path, manifest) + selected_symbols = {_normalized_symbol(symbol) for symbol in symbols} + if selected_symbols and not selected_symbols.issubset(manifest.symbols): + raise ValueError("query symbols are outside the product manifest") + start = _optional_int(start_ns, "start_ns") + end = _optional_int(end_ns, "end_ns") + if start is not None and end is not None and end <= start: + raise ValueError("query end_ns must be greater than start_ns") + selected: list[Path] = [] + for partition in manifest.partitions: + if selected_symbols and partition.symbol not in selected_symbols: + continue + if start is not None and partition.max_event_time_ns < start: + continue + if end is not None and partition.min_event_time_ns >= end: + continue + selected.append(path.parent / partition.relative_path) + return tuple(selected) + + +def iter_reconstruction_event_batches( + manifest_path: str | Path, + *, + columns: Iterable[str] = SYNTHETIC_EVENT_ARROW_COLUMNS, + symbols: Iterable[str] = (), + start_ns: int | None = None, + end_ns: int | None = None, + batch_size: int = 65_536, +) -> Iterator[Any]: + """Stream projected Arrow batches with file and row predicate pruning.""" + requested = tuple(columns) + if not requested: + raise ValueError("reconstruction scan requires at least one column") + unknown = set(requested).difference(SYNTHETIC_EVENT_ARROW_COLUMNS) + if unknown: + raise ValueError(f"unknown reconstruction columns: {sorted(unknown)}") + size = _positive_int(batch_size, "batch_size") + paths = reconstruction_parquet_paths( + manifest_path, + symbols=symbols, + start_ns=start_ns, + end_ns=end_ns, + ) + _, _, ds = _arrow_dataset_modules() + expression = None + if start_ns is not None: + expression = ds.field("event_time_ns") >= _strict_int( + start_ns, "start_ns" + ) + if end_ns is not None: + upper = ds.field("event_time_ns") < _strict_int(end_ns, "end_ns") + expression = upper if expression is None else expression & upper + for partition_path in paths: + dataset = ds.dataset(partition_path, format="parquet") + scanner = dataset.scanner( + columns=list(requested), + filter=expression, + batch_size=size, + use_threads=False, + ) + yield from scanner.to_batches() + + +def scan_reconstruction_events_polars( + manifest_path: str | Path, + *, + columns: Iterable[str] = SYNTHETIC_EVENT_ARROW_COLUMNS, + symbols: Iterable[str] = (), + start_ns: int | None = None, + end_ns: int | None = None, +) -> Any: + """Return a lazy Polars scan with projection and predicates pushed down.""" + requested = tuple(columns) + if not requested: + raise ValueError("reconstruction scan requires at least one column") + unknown = set(requested).difference(SYNTHETIC_EVENT_ARROW_COLUMNS) + if unknown: + raise ValueError(f"unknown reconstruction columns: {sorted(unknown)}") + paths = reconstruction_parquet_paths( + manifest_path, + symbols=symbols, + start_ns=start_ns, + end_ns=end_ns, + ) + pl = _polars_module() + if not paths: + schema = synthetic_event_arrow_schema() + empty = pl.from_arrow(schema.empty_table()).lazy() + return empty.select(list(requested)) + lazy = pl.scan_parquet( + [str(path) for path in paths], + hive_partitioning=False, + use_statistics=True, + ) + if start_ns is not None: + lazy = lazy.filter(pl.col("event_time_ns") >= start_ns) + if end_ns is not None: + lazy = lazy.filter(pl.col("event_time_ns") < end_ns) + return lazy.select(list(requested)) + + +def read_reconstruction_streams( + manifest_path: str | Path, +) -> tuple[SyntheticEventStreamV1, ...]: + """Replay a committed product into exact per-symbol event streams.""" + path = Path(manifest_path).expanduser().resolve() + manifest = verify_reconstruction_publication(path) + by_symbol: dict[str, list[SyntheticEventV1]] = { + symbol: [] for symbol in manifest.symbols + } + for partition in manifest.partitions: + for event in _iter_partition_events( + path.parent / partition.relative_path + ): + by_symbol[event.symbol].append(event) + streams = tuple( + SyntheticEventStreamV1( + run_id=manifest.run_id, + ensemble_member_id=manifest.ensemble_member_id, + symbol=symbol, + events=tuple(by_symbol[symbol]), + source_version_ids=tuple( + source + for partition in manifest.partitions + if partition.symbol == symbol + for source in partition.source_version_ids + ), + ) + for symbol in manifest.symbols + ) + replay_hash = reconstruction_logical_content_sha256( + event for stream in streams for event in stream.events + ) + if replay_hash != manifest.replay.logical_content_sha256: + raise ReconstructionPersistenceError( + "replayed streams differ from committed logical hash" + ) + return streams + + +def cleanup_reconstruction_scratch(root: str | Path) -> tuple[Path, ...]: + """Remove only unpublished transaction directories below ``.scratch``.""" + root_path = Path(root).expanduser().resolve() + product_root = root_path / RECONSTRUCTION_PRODUCT_DIRECTORY + if not product_root.exists(): + return () + removed: list[Path] = [] + for scratch in product_root.glob("**/.scratch"): + if not scratch.is_dir() or scratch.is_symlink(): + continue + for child in tuple(scratch.iterdir()): + if not child.name.startswith("publication.tmp-"): + continue + removed.append(child) + _remove_scratch_entry(child, root_path) + try: + scratch.rmdir() + except OSError: + pass + return tuple(removed) + + +def _validate_publication_inputs( + rendered_group: BrokerRenderedGroupV1, + retention: ReconstructionRetentionPlanV1, + policy: ReconstructionStoragePolicyV1, +) -> None: + if not isinstance(rendered_group, BrokerRenderedGroupV1): + raise TypeError("publication requires a broker-rendered group") + if rendered_group.status is not BrokerTransferStatus.APPLIED: + raise ReconstructionPersistenceError( + "refused broker-rendered groups cannot be published" + ) + if not isinstance(retention, ReconstructionRetentionPlanV1): + raise TypeError("publication requires retention preflight evidence") + if not isinstance(policy, ReconstructionStoragePolicyV1): + raise TypeError("publication requires a v1 storage policy") + if retention.storage_policy_id != policy.policy_id: + raise ReconstructionPersistenceError( + "retention preflight uses a different storage policy" + ) + manifest = rendered_group.manifest + if retention.run_id != manifest.run_id: + raise ReconstructionPersistenceError( + "retention preflight uses a different reconstruction run" + ) + if manifest.ensemble_member_id not in retention.retained_member_ids: + raise ReconstructionPersistenceError( + "rendered member is absent from retention preflight" + ) + actual = sum(len(stream.events) for stream in rendered_group.streams) + if actual > retention.member_event_counts[manifest.ensemble_member_id]: + raise ReconstructionPersistenceError( + "rendered rows exceed the pre-run member estimate" + ) + actual_symbols = tuple(stream.symbol for stream in rendered_group.streams) + if len(set(actual_symbols)) != len(actual_symbols): + raise ReconstructionPersistenceError( + "rendered group contains duplicate symbol streams" + ) + if ( + not policy.atomic_promotion_required + or not policy.advertise_only_committed + ): + raise ReconstructionPersistenceError( + "storage policy does not require committed-only atomic publication" + ) + + +def _validate_delivery_publication_inputs( + delivered_group: ReconstructionDeliveredGroupV1, + final_validation: CrossCurrencyValidationReportV1, + retention: ReconstructionRetentionPlanV1, + policy: ReconstructionStoragePolicyV1, +) -> None: + if not isinstance(delivered_group, ReconstructionDeliveredGroupV1): + raise TypeError("delivery publication requires a delivered group") + if delivered_group.status is not ReconstructionDeliveryStatus.APPLIED: + raise ReconstructionPersistenceError( + "refused delivery groups cannot be published" + ) + if not isinstance(final_validation, CrossCurrencyValidationReportV1): + raise TypeError("delivery publication requires final validation") + if not final_validation.passed: + raise ReconstructionPersistenceError( + "failed final validation cannot be published" + ) + if final_validation.stage is not CrossCurrencyValidationStage.POST_BROKER: + raise ReconstructionPersistenceError( + "delivery publication requires the final post-delivery validation seam" + ) + manifest = delivered_group.manifest + if ( + final_validation.run_id != manifest.run_id + or final_validation.window_id != manifest.window_id + or final_validation.synchronization_unit_id + != manifest.synchronization_unit_id + or final_validation.ensemble_member_id != manifest.ensemble_member_id + ): + raise ReconstructionPersistenceError( + "final validation scope differs from delivery group" + ) + if not isinstance(retention, ReconstructionRetentionPlanV1): + raise TypeError("delivery publication requires retention evidence") + if not isinstance(policy, ReconstructionStoragePolicyV1): + raise TypeError("delivery publication requires storage policy") + if retention.storage_policy_id != policy.policy_id: + raise ReconstructionPersistenceError( + "delivery retention uses a different storage policy" + ) + if retention.run_id != manifest.run_id: + raise ReconstructionPersistenceError( + "delivery retention uses a different reconstruction run" + ) + if manifest.ensemble_member_id not in retention.retained_member_ids: + raise ReconstructionPersistenceError( + "delivered member is absent from retention preflight" + ) + actual = sum(len(stream.events) for stream in delivered_group.streams) + if actual > retention.member_event_counts[manifest.ensemble_member_id]: + raise ReconstructionPersistenceError( + "delivered rows exceed the pre-run member estimate" + ) + symbols = tuple(stream.symbol for stream in delivered_group.streams) + if len(set(symbols)) != len(symbols): + raise ReconstructionPersistenceError( + "delivered group contains duplicate symbol streams" + ) + if ( + not policy.atomic_promotion_required + or not policy.advertise_only_committed + ): + raise ReconstructionPersistenceError( + "storage policy does not require committed-only atomic publication" + ) + + +def _validate_immutable_anchors( + output_events: Iterable[SyntheticEventV1], + source_anchors: Iterable[SyntheticEventV1], +) -> None: + anchors: dict[str, dict[str, JSONValue]] = {} + for event in source_anchors: + if not isinstance(event, SyntheticEventV1): + raise TypeError("source anchors must be SyntheticEventV1 rows") + if event.origin is not SyntheticEventOrigin.OBSERVED: + raise ReconstructionPersistenceError( + "source anchor input contains a synthetic row" + ) + if event.event_id in anchors: + raise ReconstructionPersistenceError( + "source anchor input contains duplicate event IDs" + ) + anchors[event.event_id] = event.to_dict() + outputs = { + event.event_id: event.to_dict() + for event in output_events + if event.origin is SyntheticEventOrigin.OBSERVED + } + if outputs != anchors: + raise ReconstructionPersistenceError( + "observed output values or IDs differ from immutable anchors" + ) + + +def _write_product_partitions( + staging_directory: Path, + streams: Iterable[SyntheticEventStreamV1], + *, + row_group_size: int, +) -> tuple[ReconstructionProductPartitionV1, ...]: + partitions: list[ReconstructionProductPartitionV1] = [] + for stream in sorted(streams, key=lambda item: item.symbol): + by_date: dict[str, list[SyntheticEventV1]] = {} + for event in stream.events: + by_date.setdefault(_event_date(event.event_time_ns), []).append( + event + ) + for event_date, events in sorted(by_date.items()): + partition_stream = SyntheticEventStreamV1( + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + symbol=stream.symbol, + events=tuple(events), + source_version_ids=stream.source_version_ids, + ) + relative = _partition_relative_path(stream.symbol, event_date) + target = staging_directory / relative + target.parent.mkdir(parents=True, exist_ok=True) + _write_parquet_partition( + partition_stream, target, row_group_size=row_group_size + ) + _, pq = _arrow_modules() + parquet = pq.ParquetFile(target) + partition = ReconstructionProductPartitionV1( + relative_path=relative, + symbol=stream.symbol, + event_date=event_date, + stream_id=partition_stream.stream_id, + source_version_ids=partition_stream.source_version_ids, + row_count=len(events), + observed_event_count=partition_stream.observed_event_count, + synthetic_event_count=partition_stream.synthetic_event_count, + min_event_time_ns=events[0].event_time_ns, + max_event_time_ns=events[-1].event_time_ns, + logical_content_sha256=reconstruction_logical_content_sha256( + events + ), + byte_sha256=_file_sha256(target), + size_bytes=target.stat().st_size, + row_group_count=parquet.num_row_groups, + ) + _validate_partition_file(target, partition, row_group_size) + partitions.append(partition) + if not partitions: + raise ReconstructionPersistenceError( + "reconstruction publication produced no partitions" + ) + return tuple(partitions) + + +def _validate_actual_storage( + partitions: Iterable[ReconstructionProductPartitionV1], + manifest_bytes: bytes, + policy: ReconstructionStoragePolicyV1, +) -> None: + partition_bytes = sum(item.size_bytes for item in partitions) + output_bytes = partition_bytes + len(manifest_bytes) + violations: list[str] = [] + if output_bytes > policy.max_output_bytes: + violations.append( + f"actual output {output_bytes} exceeds {policy.max_output_bytes}" + ) + if output_bytes > policy.max_scratch_bytes: + violations.append( + f"staged output {output_bytes} exceeds {policy.max_scratch_bytes}" + ) + if violations: + raise ReconstructionPersistenceError( + "reconstruction persistence limits failed: " + "; ".join(violations) + ) + + +def _write_parquet_partition( + stream: SyntheticEventStreamV1, + target: Path, + *, + row_group_size: int, +) -> None: + _, pq = _arrow_modules() + partial = target.with_name(target.name + ".partial") + try: + pq.write_table( + synthetic_event_stream_to_arrow(stream), + partial, + compression=RECONSTRUCTION_COMPRESSION, + use_dictionary=False, + write_statistics=True, + version="2.6", + data_page_version="2.0", + row_group_size=row_group_size, + write_page_checksum=True, + ) + _fsync_file(partial) + os.replace(partial, target) + _fsync_directory(target.parent) + finally: + if partial.exists(): + partial.unlink() + + +def _validate_partition_file( + path: Path, + expected: ReconstructionProductPartitionV1, + row_group_size: int, +) -> None: + if not path.is_file() or path.is_symlink(): + raise ReconstructionPersistenceError("partition is missing or unsafe") + if path.stat().st_size != expected.size_bytes: + raise ReconstructionPersistenceError("partition byte size differs") + if _file_sha256(path) != expected.byte_sha256: + raise ReconstructionPersistenceError("partition byte hash differs") + _, pq = _arrow_modules() + try: + parquet = pq.ParquetFile(path) + except Exception as err: + raise ReconstructionPersistenceError( + "partition footer is unreadable" + ) from err + actual_schema = parquet.schema_arrow.remove_metadata() + required_schema = synthetic_event_arrow_schema().remove_metadata() + if not actual_schema.equals(required_schema): + raise ReconstructionPersistenceError( + "partition does not contain the exact final event schema" + ) + if parquet.num_row_groups != expected.row_group_count: + raise ReconstructionPersistenceError( + "partition row-group count differs" + ) + for ordinal in range(parquet.num_row_groups): + if parquet.metadata.row_group(ordinal).num_rows > row_group_size: + raise ReconstructionPersistenceError( + "partition row group exceeds the configured bound" + ) + events = tuple(_iter_partition_events(path)) + if len(events) != expected.row_count: + raise ReconstructionPersistenceError("partition row count differs") + if any(event.symbol != expected.symbol for event in events): + raise ReconstructionPersistenceError("partition symbol differs") + if any( + _event_date(event.event_time_ns) != expected.event_date + for event in events + ): + raise ReconstructionPersistenceError("partition event date differs") + if tuple(sorted(events, key=_event_order_key)) != events: + raise ReconstructionPersistenceError("partition event order differs") + observed = sum( + event.origin is SyntheticEventOrigin.OBSERVED for event in events + ) + if observed != expected.observed_event_count: + raise ReconstructionPersistenceError("partition origin counts differ") + if events[0].event_time_ns != expected.min_event_time_ns: + raise ReconstructionPersistenceError("partition minimum time differs") + if events[-1].event_time_ns != expected.max_event_time_ns: + raise ReconstructionPersistenceError("partition maximum time differs") + if reconstruction_logical_content_sha256(events) != ( + expected.logical_content_sha256 + ): + raise ReconstructionPersistenceError("partition logical hash differs") + + +def _iter_partition_events(path: Path) -> Iterator[SyntheticEventV1]: + _, pq = _arrow_modules() + try: + parquet = pq.ParquetFile(path) + for batch in parquet.iter_batches(batch_size=65_536, use_threads=False): + for row in batch.to_pylist(): + yield SyntheticEventV1.from_dict(row) + except ReconstructionPersistenceError: + raise + except Exception as err: + raise ReconstructionPersistenceError( + f"cannot read reconstruction partition {path.name}" + ) from err + + +def _source_manifest( + events: Iterable[SyntheticEventV1], + anchors: Iterable[SyntheticEventV1], +) -> ReconstructionSourceManifestV1: + rows = tuple(events) + observed = tuple(sorted(anchors, key=_event_order_key)) + return ReconstructionSourceManifestV1( + source_version_ids=tuple(event.source_version_id for event in rows), + source_series_ids=tuple( + event.source_series_id + for event in observed + if event.source_series_id is not None + ), + source_periods=tuple( + event.source_period + for event in observed + if event.source_period is not None + ), + observed_event_count=len(observed), + observed_content_sha256=_observed_content_sha256(observed), + observed_event_ids_sha256=_text_sequence_sha256( + event.event_id for event in observed + ), + ) + + +def _constraint_manifest( + events: Iterable[SyntheticEventV1], +) -> ReconstructionConstraintManifestV1: + synthetic = tuple( + event + for event in events + if event.origin is SyntheticEventOrigin.SYNTHETIC + ) + return ReconstructionConstraintManifestV1( + synthetic_event_count=len(synthetic), + constraint_set_ids=tuple( + event.constraint_set_id + for event in synthetic + if event.constraint_set_id is not None + ), + generator_ids=tuple( + event.generator_id + for event in synthetic + if event.generator_id is not None + ), + generator_versions=tuple( + event.generator_version + for event in synthetic + if event.generator_version is not None + ), + generator_config_ids=tuple( + event.generator_config_id + for event in synthetic + if event.generator_config_id is not None + ), + feed_epoch_ids=tuple( + event.feed_epoch_id + for event in synthetic + if event.feed_epoch_id is not None + ), + reference_assignment_count=sum( + event.reference_id is not None for event in synthetic + ), + reference_assignments_sha256=_lineage_assignments_sha256( + synthetic, "reference_id" + ), + motif_assignment_count=sum( + event.motif_id is not None for event in synthetic + ), + motif_assignments_sha256=_lineage_assignments_sha256( + synthetic, "motif_id" + ), + ) + + +def _quality_manifest( + group: BrokerRenderedGroupV1, +) -> ReconstructionQualityManifestV1: + manifest = group.manifest + output_hash = _broker_streams_content_sha256(group.streams) + if manifest.output_content_sha256 != output_hash: + raise ReconstructionPersistenceError( + "broker transfer output hash differs before persistence" + ) + if group.cross_instrument_quality_payload is None: + raise ReconstructionPersistenceError( + "final group lacks quality payload" + ) + quality_hash = _content_sha256(group.cross_instrument_quality_payload) + if manifest.cross_instrument_quality_sha256 != quality_hash: + raise ReconstructionPersistenceError( + "cross-instrument quality hash differs before persistence" + ) + return ReconstructionQualityManifestV1( + broker_transfer_manifest_id=manifest.manifest_id, + broker_fingerprint_id=manifest.fingerprint_id, + transfer_output_content_sha256=output_hash, + post_broker_validation_id=cast(str, manifest.post_broker_validation_id), + post_broker_validation_status=cast( + str, manifest.post_broker_validation_status + ), + cross_instrument_quality_status=cast( + str, manifest.cross_instrument_quality_status + ), + cross_instrument_quality_sha256=quality_hash, + broker_observed_event_count=manifest.observed_event_count, + broker_synthetic_event_count=manifest.synthetic_event_count, + broker_lineage_count=manifest.lineage_count, + broker_lineage_content_sha256=cast( + str, manifest.lineage_content_sha256 + ), + broker_action_counts=manifest.action_counts, + benchmark_comparison_ids=manifest.benchmark_comparison_ids, + ) + + +def _delivery_quality_manifest( + group: ReconstructionDeliveredGroupV1, + *, + final_validation: CrossCurrencyValidationReportV1, + benchmark_artifact_ids: Sequence[str], + benchmark_evidence: Mapping[str, JSONValue], +) -> ReconstructionDeliveryQualityManifestV1: + manifest = group.manifest + output_hash = reconstruction_streams_content_sha256(group.streams) + if manifest.output_content_sha256 != output_hash: + raise ReconstructionPersistenceError( + "delivery output hash differs before persistence" + ) + evidence: dict[str, JSONValue] = { + "final_validation": final_validation.to_dict(), + "benchmark_artifact_ids": list(benchmark_artifact_ids), + "benchmark_evidence": dict(benchmark_evidence), + } + quality_hash = _content_sha256(evidence) + identity_hash = cast(str, manifest.identity_lineage_sha256) + return ReconstructionDeliveryQualityManifestV1( + delivery_manifest_id=manifest.manifest_id, + delivery_profile_id=manifest.delivery_profile_id, + delivery_mode=manifest.delivery_mode, + delivery_output_content_sha256=output_hash, + final_validation_id=final_validation.validation_id, + final_validation_status=final_validation.status.value, + cross_instrument_quality_status=( + "passed" if final_validation.passed else "failed" + ), + cross_instrument_quality_sha256=quality_hash, + observed_event_count=manifest.observed_event_count, + synthetic_event_count=manifest.synthetic_event_count, + identity_event_count=manifest.identity_event_count, + identity_lineage_sha256=identity_hash, + delivery_action_counts=( + {"identity": manifest.identity_event_count} + if manifest.identity_event_count + else {} + ), + benchmark_artifact_ids=tuple(benchmark_artifact_ids), + ) + + +def _verify_publication_directory( + directory: Path, + manifest: ReconstructionProductManifestV1 | ReconstructionProductManifestV2, + *, + require_committed_layout: bool, +) -> None: + manifest_path = directory / RECONSTRUCTION_MANIFEST_FILENAME + disk_manifest = load_reconstruction_manifest(manifest_path) + if disk_manifest != manifest: + raise ReconstructionPersistenceError( + "manifest bytes differ from staged publication evidence" + ) + if require_committed_layout: + _validate_committed_manifest_location(manifest_path, manifest) + for partition in manifest.partitions: + _validate_partition_file( + directory / partition.relative_path, + partition, + manifest.replay.row_group_size, + ) + actual_byte_hash = _partition_byte_digest(manifest.partitions) + if actual_byte_hash != manifest.replay.partition_byte_sha256: + raise ReconstructionPersistenceError( + "partition byte aggregate differs from replay manifest" + ) + digest = hashlib.sha256(_LOGICAL_HASH_HEADER) + observed_digest = hashlib.sha256(_OBSERVED_HASH_HEADER) + observed_ids: list[str] = [] + total = 0 + observed = 0 + synthetic = 0 + for partition in manifest.partitions: + for event in _iter_partition_events( + directory / partition.relative_path + ): + _update_event_digest(digest, event) + total += 1 + if event.origin is SyntheticEventOrigin.OBSERVED: + _update_event_digest(observed_digest, event) + observed_ids.append(event.event_id) + observed += 1 + else: + synthetic += 1 + if digest.hexdigest() != manifest.replay.logical_content_sha256: + raise ReconstructionPersistenceError( + "clean replay logical hash differs from manifest" + ) + if observed_digest.hexdigest() != manifest.source.observed_content_sha256: + raise ReconstructionPersistenceError( + "replayed observed-anchor hash differs from source manifest" + ) + if _text_sequence_sha256(observed_ids) != ( + manifest.source.observed_event_ids_sha256 + ): + raise ReconstructionPersistenceError( + "replayed observed IDs differ from source manifest" + ) + if (total, observed, synthetic) != ( + manifest.event_count, + manifest.observed_event_count, + manifest.synthetic_event_count, + ): + raise ReconstructionPersistenceError( + "replayed event counts differ from manifest" + ) + + +def _find_matching_publication( + root: Path, + group: BrokerRenderedGroupV1, + *, + symbol_group_id: str, + retention_plan: ReconstructionRetentionPlanV1, + row_group_size: int, +) -> PublishedReconstructionV1 | None: + logical_hash = reconstruction_logical_content_sha256( + event for stream in group.streams for event in stream.events + ) + for path in discover_reconstruction_manifests( + root, + run_id=group.manifest.run_id, + broker_profile_id=group.manifest.fingerprint_id, + ensemble_member_id=group.manifest.ensemble_member_id, + symbol_group_id=symbol_group_id, + ): + manifest = load_reconstruction_manifest(path) + if not isinstance(manifest, ReconstructionProductManifestV1): + continue + if ( + manifest.replay.logical_content_sha256 == logical_hash + and manifest.quality.broker_transfer_manifest_id + == group.manifest.manifest_id + and manifest.retention.plan_id == retention_plan.plan_id + ): + if manifest.replay.row_group_size != row_group_size: + raise ReconstructionPersistenceError( + "existing logical publication uses a different " + "row-group configuration" + ) + manifest = verify_reconstruction_publication(path) + if not isinstance(manifest, ReconstructionProductManifestV1): + raise ReconstructionPersistenceError( + "legacy publication lookup restored a delivery manifest" + ) + return PublishedReconstructionV1( + manifest=manifest, + manifest_path=path, + manifest_ref=_artifact_ref_for_manifest(path, manifest), + idempotent_retry=True, + ) + return None + + +def _validate_committed_manifest_location( + path: Path, + manifest: ReconstructionProductManifestV1 | ReconstructionProductManifestV2, +) -> None: + if path.name != RECONSTRUCTION_MANIFEST_FILENAME: + raise ReconstructionPersistenceError("unexpected manifest filename") + publication = path.parent + if publication.name != _path_component(manifest.publication_id): + raise ReconstructionPersistenceError( + "publication directory differs from manifest identity" + ) + if publication.parent.name != "commits": + raise ReconstructionPersistenceError( + "manifest is not below the committed publication axis" + ) + axis = publication.parent.parent + profile_axis: str + schema_version: str + if isinstance(manifest, ReconstructionProductManifestV1): + profile_axis = f"broker={_path_component(manifest.broker_profile_id)}" + schema_version = RECONSTRUCTION_PRODUCT_SCHEMA_VERSION + else: + profile_axis = ( + f"delivery={_path_component(manifest.delivery_profile_id)}" + ) + schema_version = RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION + expected_axes = ( + f"group={_path_component(manifest.symbol_group_id)}", + f"member={_path_component(manifest.ensemble_member_id)}", + profile_axis, + f"run={_path_component(manifest.run_id)}", + f"schema={_path_component(schema_version)}", + ) + cursor = axis + for expected in expected_axes: + if cursor.name != expected: + raise ReconstructionPersistenceError( + "committed manifest axes differ from manifest content" + ) + cursor = cursor.parent + if cursor.name != RECONSTRUCTION_PRODUCT_DIRECTORY: + raise ReconstructionPersistenceError( + "manifest is outside the reconstruction product root" + ) + + +def _axis_directory( + root: Path, + *, + run_id: str, + broker_profile_id: str, + ensemble_member_id: str, + symbol_group_id: str, +) -> Path: + return ( + root + / RECONSTRUCTION_PRODUCT_DIRECTORY + / f"schema={_path_component(RECONSTRUCTION_PRODUCT_SCHEMA_VERSION)}" + / f"run={_path_component(run_id)}" + / f"broker={_path_component(broker_profile_id)}" + / f"member={_path_component(ensemble_member_id)}" + / f"group={_path_component(symbol_group_id)}" + ) + + +def _delivery_axis_directory( + root: Path, + *, + run_id: str, + delivery_profile_id: str, + ensemble_member_id: str, + symbol_group_id: str, +) -> Path: + return ( + root + / RECONSTRUCTION_PRODUCT_DIRECTORY + / f"schema={_path_component(RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION)}" + / f"run={_path_component(run_id)}" + / f"delivery={_path_component(delivery_profile_id)}" + / f"member={_path_component(ensemble_member_id)}" + / f"group={_path_component(symbol_group_id)}" + ) + + +def _partition_relative_path(symbol: str, event_date: str) -> str: + return ( + f"symbol={_path_component(symbol)}/" + f"event_date={event_date}/part-00000.parquet" + ) + + +def _safe_relative_path(value: str) -> str: + text = _required_text(value) + path = PurePosixPath(text) + if path.is_absolute() or ".." in path.parts or "\\" in text: + raise ValueError("artifact path must be a safe relative POSIX path") + return path.as_posix() + + +def _partition_byte_digest( + partitions: Iterable[ReconstructionProductPartitionV1], +) -> str: + payload: list[dict[str, JSONValue]] = [ + { + "relative_path": item.relative_path, + "byte_sha256": item.byte_sha256, + "size_bytes": item.size_bytes, + } + for item in sorted(partitions, key=lambda value: value.relative_path) + ] + return _content_sha256( + { + "algorithm": RECONSTRUCTION_BYTE_HASH_ALGORITHM, + "partitions": payload, + } + ) + + +def _observed_content_sha256(events: Iterable[SyntheticEventV1]) -> str: + digest = hashlib.sha256(_OBSERVED_HASH_HEADER) + for event in sorted(events, key=_event_order_key): + _update_event_digest(digest, event) + return digest.hexdigest() + + +def _text_sequence_sha256(values: Iterable[str]) -> str: + digest = hashlib.sha256(b"histdatacom-text-sequence-v1\n") + for value in sorted(_required_text(item) for item in values): + digest.update(value.encode("utf-8")) + digest.update(b"\n") + return digest.hexdigest() + + +def _lineage_assignments_sha256( + events: Iterable[SyntheticEventV1], + field_name: str, +) -> str: + digest = hashlib.sha256( + f"histdatacom-{field_name}-assignments-v1\n".encode("ascii") + ) + for event in sorted(events, key=_event_order_key): + value = getattr(event, field_name) + digest.update(event.event_id.encode("utf-8")) + digest.update(b"=") + digest.update(str(value or "").encode("utf-8")) + digest.update(b"\n") + return digest.hexdigest() + + +def _update_event_digest(digest: Any, event: SyntheticEventV1) -> None: + if not isinstance(event, SyntheticEventV1): + raise TypeError("logical reconstruction hashes require event rows") + digest.update(canonical_contract_json(event.to_dict()).encode("utf-8")) + digest.update(b"\n") + + +def _event_order_key(event: SyntheticEventV1) -> tuple[str, int, int, str]: + return ( + event.symbol, + event.event_time_ns, + event.event_sequence, + event.event_id, + ) + + +def _broker_streams_content_sha256( + streams: Iterable[SyntheticEventStreamV1], +) -> str: + return _content_sha256( + [ + item.to_dict() + for item in sorted(streams, key=lambda value: value.symbol) + ] + ) + + +def _content_sha256(value: Any) -> str: + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + return f"{prefix}:sha256:{_content_sha256(payload)}" + + +def _artifact_ref_for_manifest( + path: Path, + manifest: ReconstructionProductManifestV1 | ReconstructionProductManifestV2, +) -> ArtifactRef: + payload = path.read_bytes() + return ArtifactRef( + kind=RECONSTRUCTION_MANIFEST_ARTIFACT_KIND, + path=str(path), + size_bytes=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + metadata={ + "schema_version": manifest.schema_version, + "publication_id": manifest.publication_id, + "manifest_id": manifest.manifest_id, + "event_count": manifest.event_count, + "logical_content_sha256": (manifest.replay.logical_content_sha256), + "immutable_anchor_content_sha256": ( + manifest.source.observed_content_sha256 + ), + }, + ) + + +def _atomic_write_bytes(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + handle, temporary_name = tempfile.mkstemp( + prefix=path.name + ".tmp-", dir=path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(handle, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + _fsync_directory(path.parent) + finally: + if temporary.exists(): + temporary.unlink() + + +def _fsync_file(path: Path) -> None: + with path.open("rb") as stream: + os.fsync(stream.fileno()) + + +def _fsync_directory(path: Path) -> None: + try: + descriptor = os.open(path, os.O_RDONLY) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) + + +def _remove_scratch_entry(path: Path, root: Path) -> None: + root_resolved = root.resolve() + if path.is_symlink(): + path.unlink(missing_ok=True) + return + resolved = path.resolve() + try: + resolved.relative_to(root_resolved / RECONSTRUCTION_PRODUCT_DIRECTORY) + except ValueError as err: + raise ReconstructionPersistenceError( + "scratch cleanup refused a path outside the product root" + ) from err + if ".scratch" not in resolved.parts or not path.name.startswith( + "publication.tmp-" + ): + raise ReconstructionPersistenceError( + "scratch cleanup refused a non-transaction path" + ) + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024**2): + digest.update(chunk) + return digest.hexdigest() + + +def _event_date(event_time_ns: int) -> str: + seconds, remainder = divmod( + _strict_int(event_time_ns, "event_time_ns"), 10**9 + ) + del remainder + return datetime.fromtimestamp(seconds, tz=timezone.utc).date().isoformat() + + +def _path_component(value: str) -> str: + return quote(_required_text(value), safe="-_.") + + +def _estimated_event_bytes( + count: int, bytes_per_event: int, ratio: float +) -> int: + return math.ceil(count * bytes_per_event * ratio) + + +def _arrow_modules() -> tuple[Any, Any]: + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError as err: + raise RuntimeError( + "reconstruction persistence requires histdatacom[arrow]" + ) from err + return pa, pq + + +def _arrow_dataset_modules() -> tuple[Any, Any, Any]: + pa, pq = _arrow_modules() + try: + import pyarrow.dataset as ds + except ImportError as err: + raise RuntimeError( + "reconstruction scans require pyarrow.dataset" + ) from err + return pa, pq, ds + + +def _polars_module() -> Any: + try: + import polars as pl + except ImportError as err: + raise RuntimeError("reconstruction scans require polars") from err + return pl + + +def _required_text(value: Any) -> str: + normalized = str(value or "").strip() + if not normalized or len(normalized) > MAX_RECONSTRUCTION_TEXT: + raise ValueError("required reconstruction text is empty or unbounded") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + if not normalized: + return None + return _required_text(normalized) + + +def _normalized_symbol(value: str) -> str: + normalized = _required_text(value).lower() + if not normalized.isalnum(): + raise ValueError("reconstruction symbols must be alphanumeric") + return normalized + + +def _normalized_text_tuple(values: Iterable[str]) -> tuple[str, ...]: + return tuple(sorted({_required_text(value) for value in values})) + + +def _required_sha256(value: Any, name: str) -> str: + normalized = str(value or "").strip().lower() + if not _SHA256_RE.fullmatch(normalized): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return normalized + + +def _required_event_date(value: str) -> str: + normalized = _required_text(value) + if not _EVENT_DATE_RE.fullmatch(normalized): + raise ValueError("event_date must be ISO YYYY-MM-DD") + try: + if datetime.fromisoformat(normalized).date().isoformat() != normalized: + raise ValueError + except ValueError as err: + raise ValueError("event_date is invalid") from err + return normalized + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _optional_int(value: Any, name: str) -> int | None: + if value is None: + return None + return _strict_int(value, name) + + +def _strict_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + return _finite_float(float(value), name) + + +def _finite_float(value: Any, name: str) -> float: + try: + number = float(value) + except (TypeError, ValueError) as err: + raise ValueError(f"{name} must be numeric") from err + if not math.isfinite(number): + raise ValueError(f"{name} must be finite") + return number + + +def _nonnegative_int(value: Any, name: str) -> int: + number = _strict_int(value, name) + if number < 0: + raise ValueError(f"{name} must be non-negative") + return number + + +def _positive_int(value: Any, name: str) -> int: + number = _strict_int(value, name) + if number < 1: + raise ValueError(f"{name} must be positive") + return number + + +def _mapping(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be an object") + return cast(Mapping[str, Any], value) + + +def _mapping_sequence(value: Any, name: str) -> tuple[Mapping[str, Any], ...]: + if isinstance(value, (str, bytes, bytearray)) or not isinstance( + value, Sequence + ): + raise ValueError(f"{name} must be a sequence") + if not all(isinstance(item, Mapping) for item in value): + raise ValueError(f"{name} must contain objects") + return tuple(cast(Mapping[str, Any], item) for item in value) + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + if isinstance(value, (str, bytes, bytearray)) or not isinstance( + value, Sequence + ): + raise ValueError(f"{name} must be a sequence") + return tuple(str(item) for item in value) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + try: + value = json.loads(text) + except json.JSONDecodeError as err: + raise ReconstructionPersistenceError( + "reconstruction manifest is invalid JSON" + ) from err + if not isinstance(value, Mapping): + raise ReconstructionPersistenceError( + "reconstruction manifest must contain an object" + ) + return cast(Mapping[str, Any], value) + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + _require_version(str(data.get("schema_version", "")), expected, "schema") + + +def _require_version(value: str, expected: str, name: str) -> None: + if value != expected: + raise ValueError(f"unsupported {name} version") + + +def _require_derived(data: Mapping[str, Any], name: str, expected: Any) -> None: + if data.get(name) != expected: + raise ValueError(f"derived field {name} differs") + + +__all__ = [ + "DEFAULT_ESTIMATED_BYTES_PER_EVENT", + "DEFAULT_ESTIMATED_COMPRESSION_RATIO", + "DEFAULT_RECONSTRUCTION_ROW_GROUP_SIZE", + "RECONSTRUCTION_BYTE_HASH_ALGORITHM", + "RECONSTRUCTION_COMPRESSION", + "RECONSTRUCTION_CONSTRAINT_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_DELIVERY_QUALITY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_ENSEMBLE_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_LOGICAL_HASH_ALGORITHM", + "RECONSTRUCTION_MANIFEST_ARTIFACT_KIND", + "RECONSTRUCTION_MANIFEST_FILENAME", + "RECONSTRUCTION_PARTITION_SCHEMA_VERSION", + "RECONSTRUCTION_PRODUCT_DIRECTORY", + "RECONSTRUCTION_PRODUCT_SCHEMA_VERSION", + "RECONSTRUCTION_PRODUCT_V2_SCHEMA_VERSION", + "RECONSTRUCTION_QUALITY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_REPLAY_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_RETENTION_PLAN_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_WRITER_ID", + "PublishedReconstructionV1", + "PublishedReconstructionV2", + "ReconstructionConstraintManifestV1", + "ReconstructionEnsembleManifestV1", + "ReconstructionDeliveryQualityManifestV1", + "ReconstructionPersistenceError", + "ReconstructionProductManifestV1", + "ReconstructionProductManifestV2", + "ReconstructionProductPartitionV1", + "ReconstructionQualityManifestV1", + "ReconstructionReplayManifestV1", + "ReconstructionRetentionPlanV1", + "ReconstructionSourceManifestV1", + "ReconstructionStoragePreflightError", + "StagedReconstructionPublicationV1", + "StagedReconstructionPublicationV2", + "cleanup_reconstruction_scratch", + "commit_reconstruction_publication", + "commit_delivery_reconstruction_publication", + "discover_reconstruction_manifests", + "estimate_reconstruction_retention", + "iter_reconstruction_event_batches", + "load_reconstruction_manifest", + "publish_reconstruction_group", + "read_reconstruction_streams", + "reconstruction_logical_content_sha256", + "reconstruction_parquet_paths", + "scan_reconstruction_events_polars", + "stage_reconstruction_publication", + "stage_delivery_reconstruction_publication", + "verify_reconstruction_publication", +] diff --git a/src/histdatacom/synthetic/reconstruction_handlers.py b/src/histdatacom/synthetic/reconstruction_handlers.py new file mode 100644 index 00000000..851e7ab0 --- /dev/null +++ b/src/histdatacom/synthetic/reconstruction_handlers.py @@ -0,0 +1,2126 @@ +"""First-party data-plane handlers for the reconstruction stage plan. + +The Temporal workflow carries only bounded receipts and strong references. +Every event-bearing intermediate is written below the window scratch tree, +while the final publication crosses the durable boundary with one atomic +directory rename. The validation stage writes a durable transaction +descriptor outside the rename source so a worker crash after promotion can be +recovered without referring to a vanished staging path. +""" + +from __future__ import annotations + +import asyncio +from bisect import bisect_left +import hashlib +import json +import math +import os +import shutil +import tempfile +import time +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import replace +from pathlib import Path +from typing import Any, cast + +from histdatacom.data_analytics.feed_epochs_v2 import ( + read_active_time_feed_epoch_definition, +) +from histdatacom.market_context import ( + CftcPositioningQueryV1, + CftcPositioningQueryStatus, + CftcReportFamily, + CftcReportScope, + MarketContextKind, + MarketContextQueryV1, + MarketContextView, + cftc_positioning_state_label, + market_context_benchmark_event_state, + preflight_market_context_corpus, + query_cftc_positioning_corpus, + query_market_context_corpus, + read_cftc_positioning_corpus, + read_market_context_corpus, +) +from histdatacom.orchestration.reconstruction import ( + ReconstructionStage, + ReconstructionStageInvocationV1, + ReconstructionStageOutcomeV1, + artifact_ref_for_file, + register_reconstruction_stage_handler, + registered_reconstruction_stage_handlers, + verify_artifact_ref, +) +from histdatacom.resource_usage import peak_rss_bytes +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.benchmark_corpus import ( + read_reverse_degradation_benchmark_corpus, +) +from histdatacom.synthetic.carving import ( + HistoricalCarvedCandidateBatchV1, + carve_empirical_motif_candidates, +) +from histdatacom.synthetic.contracts import ( + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + canonical_contract_json, + read_synthetic_event_stream_parquet, + write_synthetic_event_stream_parquet, +) +from histdatacom.synthetic.cross_currency import ( + CrossCurrencyConditionV1, + CrossCurrencyGroupStatus, + CrossCurrencyReconciledGroupV1, + CrossCurrencyValidationStage, + reconcile_cross_currency_window, + validate_cross_currency_output, +) +from histdatacom.synthetic.delivery import ( + ReconstructionDeliveredGroupV1, + ReconstructionDeliveryManifestV1, + ReconstructionDeliveryMode, + project_modern_reference_delivery, +) +from histdatacom.synthetic.generation import ( + EmpiricalMotifCandidateBatchV1, + EmpiricalMotifEventLineageV1, + EmpiricalMotifGeneratorConfigV1, + EmpiricalMotifTransformationV1, + MotifGenerationDecision, + MotifGenerationStatus, + generate_empirical_motif_candidates, +) +from histdatacom.synthetic.information import ( + InformationAuditReportV1, + InformationMode, +) +from histdatacom.synthetic.motif_library import ( + read_modern_reference_motif_artifact, + read_modern_reference_motif_index, +) +from histdatacom.synthetic.motifs import ( + ReferenceMotifConditionV1, + ReferenceMotifIndexV1, + ReferenceMotifQueryResultV1, + ReferenceMotifQueryV1, + query_reference_motifs, + reference_motif_condition_from_quotes, +) +from histdatacom.synthetic.observation import read_observation_operator_artifact +from histdatacom.synthetic.persistence import ( + PublishedReconstructionV2, + ReconstructionProductManifestV2, + ReconstructionRetentionPlanV1, + StagedReconstructionPublicationV2, + commit_delivery_reconstruction_publication, + stage_delivery_reconstruction_publication, + verify_reconstruction_publication, +) +from histdatacom.synthetic.reconstruction_plan import ( + FIRST_PARTY_RECONSTRUCTION_HANDLERS, + ReconstructionStagePlanV1, + load_reconstruction_stage_plan, +) +from histdatacom.synthetic.streaming import ( + CarryStateV1, + ReconstructionResourceEstimateV1, +) + +RECONSTRUCTION_STAGE_ARTIFACT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-stage-artifact.v1" +) +RECONSTRUCTION_STAGING_DESCRIPTOR_SCHEMA_VERSION = ( + "histdatacom.reconstruction-staging-descriptor.v1" +) + +SOURCE_STAGE_ARTIFACT_KIND = "reconstruction_source_stage_v1" +PROPOSAL_STAGE_ARTIFACT_KIND = "reconstruction_proposal_stage_v1" +CARVING_STAGE_ARTIFACT_KIND = "reconstruction_carving_stage_v1" +CROSS_STAGE_ARTIFACT_KIND = "reconstruction_cross_stage_v1" +DELIVERY_STAGE_ARTIFACT_KIND = "reconstruction_delivery_stage_v1" +VALIDATION_STAGE_ARTIFACT_KIND = "reconstruction_validation_stage_v1" +STAGING_DESCRIPTOR_ARTIFACT_KIND = "reconstruction_staging_descriptor_v1" + +_SOURCE_INPUT_STREAM_KIND = "reconstruction_observed_input_stream_v1" +_SOURCE_CORE_STREAM_KIND = "reconstruction_observed_core_stream_v1" +_CANDIDATE_STREAM_KIND = "reconstruction_candidate_stream_v1" +_CANDIDATE_BATCH_LEDGER_KIND = "reconstruction_candidate_batch_ledger_v1" +_CARVED_STREAM_KIND = "reconstruction_carved_stream_v1" +_CARVED_BATCH_LEDGER_KIND = "reconstruction_carved_batch_ledger_v1" +_CROSS_STREAM_KIND = "reconstruction_cross_reconciled_stream_v1" +_DELIVERED_STREAM_KIND = "reconstruction_delivered_stream_v1" +_MAX_CANDIDATE_BATCH_LEDGER_LINE_BYTES = 1_048_576 +_MAX_CARVED_BATCH_LEDGER_LINE_BYTES = 1_048_576 +_MAX_SYNCHRONIZATION_TIMESTAMP_PROBES = 4_096 + +_SourceRow = tuple[int, float, float, str, int, str] + +_HANDLERS = { + ReconstructionStage.SOURCE_ENRICHMENT: "source_enrichment_handler", + ReconstructionStage.PROPOSAL: "proposal_handler", + ReconstructionStage.CARVING: "carving_handler", + ReconstructionStage.CROSS_SERIES_RECONCILIATION: ( + "cross_series_reconciliation_handler" + ), + ReconstructionStage.BROKER_TRANSFER: "delivery_projection_handler", + ReconstructionStage.VALIDATION: "validation_handler", + ReconstructionStage.ATOMIC_PARTITION_COMMIT: "atomic_commit_handler", +} + + +def register_first_party_reconstruction_handlers() -> None: + """Idempotently install every versioned first-party stage adapter.""" + current = registered_reconstruction_stage_handlers() + namespace = globals() + for stage, function_name in _HANDLERS.items(): + name = FIRST_PARTY_RECONSTRUCTION_HANDLERS[stage] + handler = cast(Any, namespace[function_name]) + existing = current.get(name) + if existing is handler: + continue + if existing is not None: + raise ValueError( + f"reconstruction stage handler already registered: {name}" + ) + register_reconstruction_stage_handler(name, handler) + + +def source_enrichment_handler( + invocation: ReconstructionStageInvocationV1, +) -> ReconstructionStageOutcomeV1: + """Resolve immutable ASCII anchors and bounded context sidecars.""" + started = time.perf_counter() + try: + _cancel_if_requested(invocation) + plan = load_reconstruction_stage_plan(invocation.command) + window = invocation.task.window + source_events = _read_source_events(invocation, plan) + if any(len(values) < 2 for values in source_events.values()): + return invocation.refused( + "source_corruption", + message="complete triangle input requires two anchors per symbol", + ) + core_events = { + symbol: tuple( + event + for event in events + if window.owns_event_time(event.event_time_ns) + ) + for symbol, events in source_events.items() + } + if any(not values for values in core_events.values()): + return invocation.refused( + "source_corruption", + message="complete triangle core is empty for at least one symbol", + ) + try: + context, positioning = _window_context(plan, invocation) + observation_operator = read_observation_operator_artifact( + plan.execution_manifest.artifacts["observation_operator"] + ) + if ( + invocation.task.window.left_halo_ns + < observation_operator.required_left_halo_ns + ): + raise ValueError( + "window halo is shorter than observation operator support" + ) + conditions = _motif_conditions( + plan, + invocation, + source_events, + context=context, + positioning=positioning, + ) + cross_condition = _cross_condition( + invocation, conditions, context=context + ) + except (OSError, ValueError, TypeError) as err: + return invocation.refused( + "source_context_unsupported", + message=_bounded_error(err), + ) + input_refs = _write_streams( + invocation, + source_events, + directory_name="source-input", + kind=_SOURCE_INPUT_STREAM_KIND, + ) + core_refs = _write_streams( + invocation, + core_events, + directory_name="source-core", + kind=_SOURCE_CORE_STREAM_KIND, + ) + anchor_hash = _events_content_sha256( + event for values in core_events.values() for event in values + ) + payload: dict[str, JSONValue] = { + **_stage_scope(invocation), + "input_stream_refs": _refs_dict(input_refs), + "core_stream_refs": _refs_dict(core_refs), + "market_context": context.to_dict(), + "cftc_positioning": positioning.to_dict(), + "observation_operator_id": observation_operator.operator_id, + "motif_conditions": { + symbol: condition.to_dict() + for symbol, condition in sorted(conditions.items()) + }, + "cross_condition": cross_condition.to_dict(), + "immutable_anchor_content_sha256": anchor_hash, + "source_row_identity": { + "inventory_basis": "zero-based-arrow-row-ordinal-v1", + "event_contract_mapping": "source_row_id=arrow_ordinal+1", + }, + } + manifest = _write_json_artifact( + invocation, + "source", + SOURCE_STAGE_ARTIFACT_KIND, + payload, + metadata={"immutable_anchor_content_sha256": anchor_hash}, + ) + observed = sum(len(values) for values in core_events.values()) + return _completed( + invocation, + manifest, + started=started, + observed=observed, + message="resolved immutable source, feed epoch, calendar, and CFTC context", + ) + except asyncio.CancelledError: + raise + except (OSError, ValueError, TypeError, OverflowError) as err: + return invocation.refused( + "source_corruption", + message=_bounded_error(err), + ) + + +def proposal_handler( + invocation: ReconstructionStageInvocationV1, +) -> ReconstructionStageOutcomeV1: + """Run deterministic empirical motif retrieval and proposal generation.""" + started = time.perf_counter() + ledger_stream: Any | None = None + ledger_temporary: Path | None = None + try: + _cancel_if_requested(invocation) + plan = load_reconstruction_stage_plan(invocation.command) + source = _prior_manifest(invocation, SOURCE_STAGE_ARTIFACT_KIND) + streams = _read_stream_map(source, "input_stream_refs") + index = read_modern_reference_motif_index( + plan.execution_manifest.artifacts["motif_index"].path + ) + conditions = { + symbol: ReferenceMotifConditionV1.from_dict(_mapping(value)) + for symbol, value in _mapping(source["motif_conditions"]).items() + } + synchronization_anchor_symbol = _proposal_synchronization_anchor_symbol( + streams, + conditions=conditions, + ) + synchronization_event_time_ns = _proposal_synchronization_event_time( + streams, + conditions=conditions, + start_ns=invocation.task.window.core_start_ns, + end_ns=invocation.task.window.core_end_ns, + ) + ledger_directory = _stage_directory(invocation, "proposal-batches") + ledger_directory.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=".candidate-batches-", + suffix=".ndjson", + dir=ledger_directory, + ) + ledger_temporary = Path(temporary_name) + ledger_stream = os.fdopen(descriptor, "wb") + ledger_digest = hashlib.sha256() + batch_count = 0 + candidate_events: dict[str, list[SyntheticEventV1]] = { + symbol: [] for symbol in invocation.run.symbols + } + generated = 0 + refused = 0 + interval_count = sum( + max(0, len(stream.events) - 1) for stream in streams.values() + ) + completed_intervals = 0 + for symbol, stream in sorted(streams.items()): + condition = conditions[symbol] + for left, right in zip(stream.events, stream.events[1:]): + _cancel_if_requested(invocation) + if ( + right.event_time_ns <= invocation.task.window.core_start_ns + or left.event_time_ns >= invocation.task.window.core_end_ns + ): + completed_intervals += 1 + continue + query = ReferenceMotifQueryV1( + condition=condition, + information_mode=( + plan.configuration.information_policy.information_mode + ), + used_at_ns=right.event_time_ns, + as_of_ns=( + invocation.task.window.core_start_ns + if plan.configuration.information_policy.information_mode.value + == "ex_ante_simulation" + else None + ), + max_results=index.config.max_matches, + ) + result = query_reference_motifs(index, query) + batch = generate_empirical_motif_candidates( + run=invocation.run, + window=invocation.task.window, + left_anchor=left, + right_anchor=right, + query_result=result, + config=plan.configuration.generator_config, + required_event_time_ns=( + synchronization_event_time_ns + if left.event_time_ns + < synchronization_event_time_ns + < right.event_time_ns + else None + ), + ) + encoded_evidence = ( + canonical_contract_json(_candidate_evidence(batch)).encode( + "utf-8" + ) + + b"\n" + ) + if ( + len(encoded_evidence) + > _MAX_CANDIDATE_BATCH_LEDGER_LINE_BYTES + ): + raise ValueError("candidate batch ledger row exceeds limit") + ledger_stream.write(encoded_evidence) + ledger_digest.update(encoded_evidence) + batch_count += 1 + candidate_events[symbol].extend(batch.events) + generated += len(batch.events) + refused += int(batch.status.value == "refused") + completed_intervals += 1 + if ( + completed_intervals + % max( + 1, invocation.run.storage_policy.heartbeat_every_batches + ) + == 0 + ): + invocation.heartbeat( + sequence=completed_intervals, + completed_units=completed_intervals, + total_units=interval_count, + candidate_event_count=generated, + scratch_bytes=_tree_size( + invocation.task.scratch_directory + ), + message="empirical motif proposal", + ) + ledger_stream.flush() + os.fsync(ledger_stream.fileno()) + ledger_stream.close() + ledger_stream = None + ledger_sha256 = ledger_digest.hexdigest() + ledger_target = ledger_directory / ( + f"{_CANDIDATE_BATCH_LEDGER_KIND}-{ledger_sha256}.ndjson" + ) + if ledger_target.exists(): + if _file_sha256(ledger_target) != ledger_sha256: + raise ValueError("content-addressed batch ledger collision") + ledger_temporary.unlink() + else: + os.replace(ledger_temporary, ledger_target) + ledger_temporary = None + ledger_ref = artifact_ref_for_file( + ledger_target, + kind=_CANDIDATE_BATCH_LEDGER_KIND, + metadata={ + "batch_count": batch_count, + "format": "canonical-json-lines-v1", + "large_event_rows_inline": False, + }, + ) + candidate_refs = _write_streams( + invocation, + candidate_events, + directory_name="proposal-candidates", + kind=_CANDIDATE_STREAM_KIND, + ) + payload: dict[str, JSONValue] = { + **_stage_scope(invocation), + "batch_ledger_ref": ledger_ref.to_dict(), + "batches_inline": False, + "query_conditions": { + symbol: condition.to_dict() + for symbol, condition in sorted(conditions.items()) + }, + "generator_config": plan.configuration.generator_config.to_dict(), + "candidate_stream_refs": _refs_dict(candidate_refs), + "batch_count": batch_count, + "generated_event_count": generated, + "refused_interval_count": refused, + "synchronization_anchor_symbol": synchronization_anchor_symbol, + "synchronization_event_time_ns": synchronization_event_time_ns, + "motif_index_id": index.index_id, + "immutable_anchor_content_sha256": source[ + "immutable_anchor_content_sha256" + ], + } + manifest = _write_json_artifact( + invocation, + "proposal", + PROPOSAL_STAGE_ARTIFACT_KIND, + payload, + metadata={ + "batch_count": batch_count, + "rejected_event_count": refused, + "immutable_anchor_content_sha256": source[ + "immutable_anchor_content_sha256" + ], + }, + ) + return _completed( + invocation, + manifest, + started=started, + observed=sum( + stream.observed_event_count for stream in streams.values() + ), + candidates=generated, + rejected=refused, + message="generated empirical motif proposals", + output_bytes=(manifest.size_bytes or 0) + + (ledger_ref.size_bytes or 0) + + sum(ref.size_bytes or 0 for ref in candidate_refs.values()), + ) + except asyncio.CancelledError: + raise + except (OSError, ValueError, TypeError, OverflowError) as err: + return invocation.refused( + "proposal_validation_failed", message=_bounded_error(err) + ) + finally: + if ledger_stream is not None: + ledger_stream.close() + if ledger_temporary is not None: + ledger_temporary.unlink(missing_ok=True) + + +def carving_handler( + invocation: ReconstructionStageInvocationV1, +) -> ReconstructionStageOutcomeV1: + """Apply historical carving and materialize accepted narrow streams.""" + started = time.perf_counter() + ledger_stream: Any | None = None + ledger_temporary: Path | None = None + try: + _cancel_if_requested(invocation) + plan = load_reconstruction_stage_plan(invocation.command) + source = _prior_manifest(invocation, SOURCE_STAGE_ARTIFACT_KIND) + proposal = _prior_manifest(invocation, PROPOSAL_STAGE_ARTIFACT_KIND) + input_streams = _read_stream_map(source, "input_stream_refs") + core_streams = _read_stream_map(source, "core_stream_refs") + context = MarketContextQueryV1.from_dict( + _mapping(source["market_context"]) + ) + index = read_modern_reference_motif_index( + plan.execution_manifest.artifacts["motif_index"].path + ) + batch_count = int(proposal.get("batch_count", 0)) + batches = _restore_candidate_batches(proposal, index=index) + ledger_directory = _stage_directory(invocation, "carving-batches") + ledger_directory.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=".carved-batches-", + suffix=".ndjson", + dir=ledger_directory, + ) + ledger_temporary = Path(temporary_name) + ledger_stream = os.fdopen(descriptor, "wb") + ledger_digest = hashlib.sha256() + carved_batch_count = 0 + accepted_by_symbol: dict[str, list[SyntheticEventV1]] = { + symbol: [] for symbol in invocation.run.symbols + } + candidates = 0 + accepted = 0 + rejected = 0 + for ordinal, batch in enumerate(batches, start=1): + _cancel_if_requested(invocation) + carved = carve_empirical_motif_candidates( + run=invocation.run, + window=invocation.task.window, + candidate_batch=batch, + observed_events=input_streams[batch.symbol].events, + market_context=context, + constraints=plan.configuration.carving_constraints, + fingerprint_evidence=None, + ) + accepted_by_symbol[batch.symbol].extend(carved.accepted_events) + candidates += len(batch.events) + accepted += len(carved.accepted_events) + rejected += carved.rejection_summary.rejected_count + encoded_evidence = ( + canonical_contract_json(_carved_evidence(carved)).encode( + "utf-8" + ) + + b"\n" + ) + if len(encoded_evidence) > _MAX_CARVED_BATCH_LEDGER_LINE_BYTES: + raise ValueError("carved batch ledger row exceeds limit") + ledger_stream.write(encoded_evidence) + ledger_digest.update(encoded_evidence) + carved_batch_count += 1 + if ( + ordinal + % max(1, invocation.run.storage_policy.heartbeat_every_batches) + == 0 + ): + invocation.heartbeat( + sequence=ordinal, + completed_units=ordinal, + total_units=batch_count, + candidate_event_count=candidates, + accepted_event_count=accepted, + scratch_bytes=_tree_size(invocation.task.scratch_directory), + message="historical candidate carving", + ) + if carved_batch_count != batch_count: + raise ValueError("carved batch count differs from proposal") + ledger_stream.flush() + os.fsync(ledger_stream.fileno()) + ledger_stream.close() + ledger_stream = None + ledger_sha256 = ledger_digest.hexdigest() + ledger_target = ledger_directory / ( + f"{_CARVED_BATCH_LEDGER_KIND}-{ledger_sha256}.ndjson" + ) + if ledger_target.exists(): + if _file_sha256(ledger_target) != ledger_sha256: + raise ValueError("content-addressed carved ledger collision") + ledger_temporary.unlink() + else: + os.replace(ledger_temporary, ledger_target) + ledger_temporary = None + ledger_ref = artifact_ref_for_file( + ledger_target, + kind=_CARVED_BATCH_LEDGER_KIND, + metadata={ + "batch_count": carved_batch_count, + "format": "canonical-json-lines-v1", + "large_event_rows_inline": False, + }, + ) + merged = { + symbol: SyntheticEventStreamV1.merge( + run_id=invocation.run.run_id, + ensemble_member_id=invocation.task.window.ensemble_member_id, + symbol=symbol, + observed_events=core_streams[symbol].events, + synthetic_events=accepted_by_symbol[symbol], + source_version_ids=invocation.run.source_version_ids, + ) + for symbol in invocation.run.symbols + } + stream_refs = _write_streams( + invocation, + {symbol: stream.events for symbol, stream in merged.items()}, + directory_name="carved", + kind=_CARVED_STREAM_KIND, + ) + payload: dict[str, JSONValue] = { + **_stage_scope(invocation), + "stream_refs": _refs_dict(stream_refs), + "carved_batch_ledger_ref": ledger_ref.to_dict(), + "carved_batches_inline": False, + "carved_batch_count": carved_batch_count, + "candidate_event_count": candidates, + "accepted_event_count": accepted, + "rejected_event_count": rejected, + "immutable_anchor_content_sha256": source[ + "immutable_anchor_content_sha256" + ], + } + manifest = _write_json_artifact( + invocation, + "carving", + CARVING_STAGE_ARTIFACT_KIND, + payload, + metadata={ + "rejected_event_count": rejected, + "immutable_anchor_content_sha256": source[ + "immutable_anchor_content_sha256" + ], + }, + ) + return _completed( + invocation, + manifest, + started=started, + observed=sum(item.observed_event_count for item in merged.values()), + candidates=candidates, + accepted=accepted, + rejected=rejected, + message="carved candidate events and materialized core streams", + output_bytes=(manifest.size_bytes or 0) + + (ledger_ref.size_bytes or 0) + + sum(ref.size_bytes or 0 for ref in stream_refs.values()), + ) + except asyncio.CancelledError: + raise + except (OSError, ValueError, TypeError, OverflowError) as err: + return invocation.refused( + "carving_validation_failed", message=_bounded_error(err) + ) + finally: + if ledger_stream is not None: + ledger_stream.close() + if ledger_temporary is not None: + ledger_temporary.unlink(missing_ok=True) + + +def cross_series_reconciliation_handler( + invocation: ReconstructionStageInvocationV1, +) -> ReconstructionStageOutcomeV1: + """Reconcile the synchronized FX triangle without forward-filled joins.""" + started = time.perf_counter() + try: + _cancel_if_requested(invocation) + plan = load_reconstruction_stage_plan(invocation.command) + source = _prior_manifest(invocation, SOURCE_STAGE_ARTIFACT_KIND) + carving = _prior_manifest(invocation, CARVING_STAGE_ARTIFACT_KIND) + streams = _read_stream_map(carving, "stream_refs") + condition = CrossCurrencyConditionV1.from_dict( + _mapping(source["cross_condition"]) + ) + group = reconcile_cross_currency_window( + run=invocation.run, + window=invocation.task.window, + streams=streams, + config=plan.configuration.cross_currency_config, + conditions=(condition,), + ) + if group.status is not CrossCurrencyGroupStatus.RECONCILED: + return invocation.refused( + *_bounded_reasons( + group.generation_validation.failure_reasons, + fallback="invalid_triangle_group", + ), + message="cross-currency reconciliation refused the group", + ) + stream_refs = _write_streams( + invocation, + {stream.symbol: stream.events for stream in group.streams}, + directory_name="cross", + kind=_CROSS_STREAM_KIND, + ) + payload: dict[str, JSONValue] = { + **_stage_scope(invocation), + "stream_refs": _refs_dict(stream_refs), + "group": _group_without_streams(group), + "immutable_anchor_content_sha256": carving[ + "immutable_anchor_content_sha256" + ], + } + manifest = _write_json_artifact( + invocation, + "cross", + CROSS_STAGE_ARTIFACT_KIND, + payload, + metadata={ + "projection_count": len(group.projection_lineage), + "generation_validation_id": ( + group.generation_validation.validation_id + ), + "immutable_anchor_content_sha256": carving[ + "immutable_anchor_content_sha256" + ], + }, + ) + observed = sum(item.observed_event_count for item in group.streams) + synthetic = sum(item.synthetic_event_count for item in group.streams) + return _completed( + invocation, + manifest, + started=started, + observed=observed, + candidates=synthetic, + accepted=synthetic, + message="reconciled complete synchronized triangle", + ) + except asyncio.CancelledError: + raise + except (OSError, ValueError, TypeError, OverflowError) as err: + return invocation.refused( + "invalid_triangle_group", message=_bounded_error(err) + ) + + +def delivery_projection_handler( + invocation: ReconstructionStageInvocationV1, +) -> ReconstructionStageOutcomeV1: + """Apply explicit modern-reference identity delivery.""" + started = time.perf_counter() + try: + _cancel_if_requested(invocation) + plan = load_reconstruction_stage_plan(invocation.command) + if ( + plan.configuration.delivery_mode + is not ReconstructionDeliveryMode.MODERN_REFERENCE + ): + return invocation.refused( + "unsupported_delivery_mode", + message="first-party v2.1 reference handler is identity-only", + ) + cross = _prior_manifest(invocation, CROSS_STAGE_ARTIFACT_KIND) + group = _restore_cross_group(cross) + delivered = project_modern_reference_delivery( + group, + delivery_profile_id=( + "modern-reference:" + plan.configuration.configuration_id + ), + ) + stream_refs = _write_streams( + invocation, + {stream.symbol: stream.events for stream in delivered.streams}, + directory_name="delivery", + kind=_DELIVERED_STREAM_KIND, + ) + payload: dict[str, JSONValue] = { + **_stage_scope(invocation), + "stream_refs": _refs_dict(stream_refs), + "delivery_manifest": delivered.manifest.to_dict(), + "immutable_anchor_content_sha256": cross[ + "immutable_anchor_content_sha256" + ], + } + manifest = _write_json_artifact( + invocation, + "delivery", + DELIVERY_STAGE_ARTIFACT_KIND, + payload, + metadata={ + "delivery_mode": delivered.manifest.delivery_mode.value, + "identity_event_count": delivered.manifest.identity_event_count, + "immutable_anchor_content_sha256": cross[ + "immutable_anchor_content_sha256" + ], + }, + ) + observed = delivered.manifest.observed_event_count + synthetic = delivered.manifest.synthetic_event_count + return _completed( + invocation, + manifest, + started=started, + observed=observed, + candidates=synthetic, + accepted=synthetic, + message="applied explicit modern-reference identity delivery", + ) + except asyncio.CancelledError: + raise + except (OSError, ValueError, TypeError, OverflowError) as err: + return invocation.refused( + "delivery_projection_failed", message=_bounded_error(err) + ) + + +def validation_handler( + invocation: ReconstructionStageInvocationV1, +) -> ReconstructionStageOutcomeV1: + """Enforce scientific gates, then stage an atomic v2 publication.""" + started = time.perf_counter() + staged: StagedReconstructionPublicationV2 | None = None + try: + _cancel_if_requested(invocation) + plan = load_reconstruction_stage_plan(invocation.command) + source = _prior_manifest(invocation, SOURCE_STAGE_ARTIFACT_KIND) + delivery = _prior_manifest(invocation, DELIVERY_STAGE_ARTIFACT_KIND) + delivered = _restore_delivered_group(delivery) + core_streams = _read_stream_map(source, "core_stream_refs") + anchors = tuple( + event + for stream in core_streams.values() + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + ) + final_validation = validate_cross_currency_output( + run=invocation.run, + window=invocation.task.window, + streams={item.symbol: item for item in delivered.streams}, + config=plan.configuration.cross_currency_config, + stage=CrossCurrencyValidationStage.POST_BROKER, + observed_anchors=anchors, + conditions=( + CrossCurrencyConditionV1.from_dict( + _mapping(source["cross_condition"]) + ), + ), + ) + if not final_validation.passed: + return invocation.refused( + *_bounded_reasons( + final_validation.failure_reasons, + fallback="final_validation_failed", + ), + message="final cross-instrument validation failed", + ) + benchmark_evidence = _validate_scientific_evidence(plan) + retention_ref = plan.execution_manifest.artifacts["retention_plan"] + retention = ReconstructionRetentionPlanV1.from_dict( + _mapping( + json.loads(Path(retention_ref.path).read_text(encoding="utf-8")) + ) + ) + _cancel_if_requested(invocation) + staged = stage_delivery_reconstruction_publication( + plan.execution_manifest.output_root, + delivered, + final_validation=final_validation, + benchmark_artifact_ids=tuple( + ref.sha256 + for name, ref in sorted( + plan.execution_manifest.artifacts.items() + ) + if name + in { + "benchmark_manifest", + "motif_qualification", + "motif_leakage_audit", + "information_audit", + } + ), + benchmark_evidence=benchmark_evidence, + immutable_source_anchors=anchors, + symbol_group_id=invocation.task.window.synchronization_unit_id, + retention_plan=retention, + storage_policy=plan.configuration.storage_policy, + staging_root=_stage_directory(invocation, "publication"), + ) + _cancel_if_requested(invocation) + descriptor_payload: dict[str, JSONValue] = { + **_stage_scope(invocation), + "schema_version": RECONSTRUCTION_STAGING_DESCRIPTOR_SCHEMA_VERSION, + "root": str(staged.root), + "staging_directory": str(staged.staging_directory), + "committed_directory": str(staged.committed_directory), + "product_manifest": staged.manifest.to_dict(), + "final_validation": final_validation.to_dict(), + "immutable_anchor_content_sha256": source[ + "immutable_anchor_content_sha256" + ], + } + descriptor = _write_json_artifact( + invocation, + "validation", + STAGING_DESCRIPTOR_ARTIFACT_KIND, + descriptor_payload, + metadata={ + "publication_id": staged.manifest.publication_id, + "manifest_id": staged.manifest.manifest_id, + "immutable_anchor_content_sha256": source[ + "immutable_anchor_content_sha256" + ], + }, + ) + staged_manifest_ref = _write_product_manifest_mirror( + invocation, staged.manifest + ) + staged_manifest_ref = replace( + staged_manifest_ref, + metadata={ + **staged_manifest_ref.metadata, + "immutable_anchor_content_sha256": source[ + "immutable_anchor_content_sha256" + ], + }, + ) + observed = delivered.manifest.observed_event_count + synthetic = delivered.manifest.synthetic_event_count + return _completed( + invocation, + staged_manifest_ref, + started=started, + observed=observed, + candidates=synthetic, + accepted=synthetic, + message="passed scientific gates and staged atomic publication", + additional_refs=(descriptor,), + output_bytes=( + _tree_size(staged.staging_directory) + + (descriptor.size_bytes or 0) + + (staged_manifest_ref.size_bytes or 0) + ), + ) + except asyncio.CancelledError: + raise + except (OSError, ValueError, TypeError, OverflowError) as err: + if staged is not None and staged.staging_directory.exists(): + shutil.rmtree(staged.staging_directory) + return invocation.refused( + "final_validation_failed", message=_bounded_error(err) + ) + + +def atomic_commit_handler( + invocation: ReconstructionStageInvocationV1, +) -> ReconstructionStageOutcomeV1: + """Promote or recover one already-promoted atomic publication.""" + started = time.perf_counter() + try: + _cancel_if_requested(invocation) + descriptor_ref = _prior_ref( + invocation, STAGING_DESCRIPTOR_ARTIFACT_KIND + ) + descriptor = _read_json_ref(descriptor_ref) + _require_schema( + descriptor, RECONSTRUCTION_STAGING_DESCRIPTOR_SCHEMA_VERSION + ) + manifest = ReconstructionProductManifestV2.from_dict( + _mapping(descriptor["product_manifest"]) + ) + staged = StagedReconstructionPublicationV2( + root=Path(str(descriptor["root"])).resolve(), + staging_directory=Path( + str(descriptor["staging_directory"]) + ).resolve(), + committed_directory=Path( + str(descriptor["committed_directory"]) + ).resolve(), + manifest=manifest, + ) + _cancel_if_requested(invocation) + published = _commit_or_recover(staged) + committed_ref = replace( + published.manifest_ref, + metadata={ + **published.manifest_ref.metadata, + "commit_phase": "committed", + "idempotent_retry": published.idempotent_retry, + }, + ) + return _completed( + invocation, + committed_ref, + started=started, + observed=published.manifest.observed_event_count, + candidates=published.manifest.synthetic_event_count, + accepted=published.manifest.synthetic_event_count, + message=( + "recovered already committed atomic publication" + if published.idempotent_retry + else "committed atomic Parquet publication" + ), + output_bytes=sum( + item.size_bytes for item in published.manifest.partitions + ) + + published.manifest_path.stat().st_size, + ) + except asyncio.CancelledError: + raise + except (OSError, ValueError, TypeError, OverflowError) as err: + return invocation.refused( + "atomic_commit_failed", message=_bounded_error(err) + ) + + +def _read_source_events( + invocation: ReconstructionStageInvocationV1, + plan: ReconstructionStagePlanV1, +) -> dict[str, tuple[SyntheticEventV1, ...]]: + try: + import pyarrow as pa + import pyarrow.ipc as ipc + except ImportError as err: # pragma: no cover - package dependency + raise RuntimeError( + "first-party reconstruction requires pyarrow" + ) from err + window = invocation.task.window + raw: dict[str, list[_SourceRow]] = { + symbol: [] for symbol in invocation.run.symbols + } + selected = plan.source_inventory.partitions_for_window(window) + for partition in selected: + _cancel_if_requested(invocation) + verify_artifact_ref(partition.artifact) + with pa.memory_map(partition.artifact.path, "r") as source: + reader = ipc.open_file(source) + dt_index = reader.schema.get_field_index("datetime") + bid_index = reader.schema.get_field_index("bid") + ask_index = reader.schema.get_field_index("ask") + if min(dt_index, bid_index, ask_index) < 0: + raise ValueError("source partition lacks datetime/bid/ask") + ordinal = 0 + series_id = ( + f"ascii-tick:{partition.symbol}:{partition.period}:" + f"sha256:{partition.artifact.sha256}" + ) + for batch_ordinal in range(reader.num_record_batches): + batch = reader.get_batch(batch_ordinal) + times = batch.column(dt_index).to_pylist() + bids = batch.column(bid_index).to_pylist() + asks = batch.column(ask_index).to_pylist() + for timestamp_ms, bid_raw, ask_raw in zip(times, bids, asks): + timestamp_ns = int(timestamp_ms) * 1_000_000 + bid = float(bid_raw) + ask = float(ask_raw) + if ( + not math.isfinite(bid) + or not math.isfinite(ask) + or bid <= 0.0 + or ask <= 0.0 + or ask < bid + ): + raise ValueError( + f"invalid source quote {partition.symbol} " + f"{partition.period} row {ordinal}" + ) + if window.reads_event_time(timestamp_ns): + raw[partition.symbol].append( + ( + timestamp_ns, + bid, + ask, + partition.period, + ordinal, + series_id, + ) + ) + ordinal += 1 + result: dict[str, tuple[SyntheticEventV1, ...]] = {} + source_version = invocation.run.source_version_ids[0] + for symbol, values in raw.items(): + counters: Counter[int] = Counter() + events: list[SyntheticEventV1] = [] + for timestamp, bid, ask, period, ordinal, series_id in sorted( + values, key=_source_row_order_key + ): + sequence = counters[timestamp] + counters[timestamp] += 1 + events.append( + SyntheticEventV1.observed( + symbol=symbol, + event_time_ns=timestamp, + event_sequence=sequence, + bid=bid, + ask=ask, + run_id=invocation.run.run_id, + ensemble_member_id=window.ensemble_member_id, + source_version_id=source_version, + source_series_id=series_id, + source_period=period, + source_row_id=ordinal + 1, + ) + ) + result[symbol] = tuple(events) + return result + + +def _source_row_order_key(row: _SourceRow) -> tuple[int, str, int, str]: + """Preserve immutable partition/Arrow order for equal timestamps.""" + timestamp, _, _, period, ordinal, series_id = row + return (timestamp, period, ordinal, series_id) + + +def _proposal_synchronization_event_time( + streams: Mapping[str, SyntheticEventStreamV1], + *, + conditions: Mapping[str, ReferenceMotifConditionV1], + start_ns: int, + end_ns: int, +) -> int: + if set(streams) != set(conditions): + raise ValueError( + "proposal synchronization conditions differ from streams" + ) + if any(not stream.events for stream in streams.values()): + raise ValueError("triangle stream lacks synchronization anchors") + lower = max( + start_ns, + *(stream.events[0].event_time_ns for stream in streams.values()), + ) + upper = min( + end_ns, + *(stream.events[-1].event_time_ns for stream in streams.values()), + ) + if lower >= upper: + raise ValueError( + "triangle streams have no common synchronization interval" + ) + probes: set[int] = set() + probes_per_stream = max( + 1, + _MAX_SYNCHRONIZATION_TIMESTAMP_PROBES // len(streams), + ) + anchor_symbol = _proposal_synchronization_anchor_symbol( + streams, + conditions=conditions, + ) + for stream in (streams[anchor_symbol],): + event_count = len(stream.events) + sample_count = min(event_count, probes_per_stream) + indices: tuple[int, ...] + if sample_count == 1: + indices = (event_count // 2,) + else: + indices = tuple( + ordinal * (event_count - 1) // (sample_count - 1) + for ordinal in range(sample_count) + ) + for index in indices: + selected = stream.events[index].event_time_ns + if lower <= selected < upper: + probes.add(selected) + + center_twice = lower + upper + best: tuple[tuple[int, int, int, int, int], int] | None = None + for selected in probes: + exact_support = 0 + missing_interval_widths: list[int] = [] + for stream in streams.values(): + position = bisect_left( + stream.events, + selected, + key=lambda event: event.event_time_ns, + ) + if ( + position < len(stream.events) + and stream.events[position].event_time_ns == selected + ): + exact_support += 1 + continue + if position == 0 or position == len(stream.events): + break + missing_interval_widths.append( + stream.events[position].event_time_ns + - stream.events[position - 1].event_time_ns + ) + else: + if exact_support == len(streams): + support_penalty = 2 + elif exact_support == 1: + support_penalty = 0 + else: + support_penalty = 1 + score = ( + support_penalty, + max(missing_interval_widths, default=0), + sum(missing_interval_widths), + abs(2 * selected - center_twice), + selected, + ) + if best is None or score < best[0]: + best = (score, selected) + if best is None: + raise ValueError( + "triangle streams have no bounded synchronization support" + ) + return best[1] + + +def _proposal_synchronization_anchor_symbol( + streams: Mapping[str, SyntheticEventStreamV1], + *, + conditions: Mapping[str, ReferenceMotifConditionV1], +) -> str: + if set(streams) != set(conditions): + raise ValueError( + "proposal synchronization conditions differ from streams" + ) + return min( + streams, + key=lambda symbol: ( + conditions[symbol].metrics.get("tick_intensity", math.inf), + len(streams[symbol].events), + symbol, + ), + ) + + +def _window_context( + plan: ReconstructionStagePlanV1, + invocation: ReconstructionStageInvocationV1, +) -> tuple[MarketContextQueryV1, CftcPositioningQueryV1]: + window = invocation.task.window + context_corpus = read_market_context_corpus( + plan.execution_manifest.artifacts["market_context"].path + ) + requirements = ( + ("EUR", MarketContextKind.POLICY_RATE_CHANGE), + ("GBP", MarketContextKind.POLICY_RATE_CHANGE), + ("USD", MarketContextKind.CENTRAL_BANK_DECISION), + ) + reasons: list[str] = [] + for currency, kind in requirements: + decision = preflight_market_context_corpus( + context_corpus, + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + currencies=(currency,), + kinds=(kind,), + ) + reasons.extend(decision.reasons) + if reasons: + raise ValueError("unsupported market context: " + "; ".join(reasons)) + mode = plan.configuration.information_policy.information_mode + view = ( + MarketContextView.EX_ANTE + if mode.value == "ex_ante_simulation" + else MarketContextView.EX_POST + ) + context = query_market_context_corpus( + context_corpus, + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + view=view, + as_of_ns=( + window.core_start_ns if view is MarketContextView.EX_ANTE else None + ), + symbols=window.symbols, + include_calendar=True, + window_id=window.window_id, + require_supported=False, + ) + if context.calendar_state is None: + raise ValueError("market context lacks calendar state") + cftc_corpus = read_cftc_positioning_corpus( + plan.execution_manifest.artifacts["cftc_positioning"].path + ) + positioning = query_cftc_positioning_corpus( + cftc_corpus, + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + information_mode=mode, + as_of_ns=( + window.core_start_ns if mode.value == "ex_ante_simulation" else None + ), + symbols=window.symbols, + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + if positioning.status is not CftcPositioningQueryStatus.READY: + raise ValueError("unsupported CFTC context: " + positioning.reason) + return context, positioning + + +def _motif_conditions( + plan: ReconstructionStagePlanV1, + invocation: ReconstructionStageInvocationV1, + source_events: Mapping[str, Sequence[SyntheticEventV1]], + *, + context: MarketContextQueryV1, + positioning: CftcPositioningQueryV1, +) -> dict[str, ReferenceMotifConditionV1]: + definition = read_active_time_feed_epoch_definition( + plan.execution_manifest.artifacts["feed_epochs"].path + ) + calendar = cast(Any, context.calendar_state) + event_tags = ( + *calendar.event_tags, + market_context_benchmark_event_state(context), + cftc_positioning_state_label(positioning), + ) + result: dict[str, ReferenceMotifConditionV1] = {} + for symbol, events in sorted(source_events.items()): + midpoint_ms = ( + ( + invocation.task.window.core_start_ns + + invocation.task.window.core_end_ns + ) + // 2 + // 1_000_000 + ) + assignment = definition.assign( + symbol=symbol, timestamp_utc_ms=midpoint_ms + ) + if assignment.assignment_kind == "out_of_scope": + raise ValueError(f"unsupported feed epoch for {symbol}") + result[symbol] = reference_motif_condition_from_quotes( + symbol=symbol, + feed_epoch_id=assignment.label, + session_state=calendar.session_state, + event_times_ns=tuple(event.event_time_ns for event in events), + bids=tuple(event.bid for event in events), + asks=tuple(event.ask for event in events), + event_tags=event_tags, + active_sessions=calendar.active_sessions, + overlap_tags=calendar.overlaps, + special_tags=calendar.special_tags, + holiday_tags=calendar.holiday_tags, + ) + return result + + +def _cross_condition( + invocation: ReconstructionStageInvocationV1, + conditions: Mapping[str, ReferenceMotifConditionV1], + *, + context: MarketContextQueryV1, +) -> CrossCurrencyConditionV1: + calendar = cast(Any, context.calendar_state) + return CrossCurrencyConditionV1( + start_ns=invocation.task.window.core_start_ns, + end_ns=invocation.task.window.core_end_ns, + session_key=calendar.session_state, + event_key=market_context_benchmark_event_state(context), + feed_epoch_key="+".join( + sorted({item.feed_epoch_id for item in conditions.values()}) + ), + ) + + +def _validate_scientific_evidence( + plan: ReconstructionStagePlanV1, +) -> dict[str, JSONValue]: + artifacts = plan.execution_manifest.artifacts + benchmark = read_reverse_degradation_benchmark_corpus( + artifacts["benchmark_manifest"].path + ) + qualification = read_modern_reference_motif_artifact( + artifacts["motif_qualification"].path, kind="qualification" + ) + leakage = read_modern_reference_motif_artifact( + artifacts["motif_leakage_audit"].path, kind="leakage-audit" + ) + audit = InformationAuditReportV1.from_json( + Path(artifacts["information_audit"].path).read_text(encoding="utf-8") + ) + contracts = _mapping(qualification.get("real_window_contracts")) + failures: list[str] = [] + if qualification.get("candidate_promotion_eligible") is not True: + failures.append("motif candidate is not promotion eligible") + if qualification.get("candidate_provisional") is not False: + failures.append("motif candidate remains provisional") + if not contracts or not all(value is True for value in contracts.values()): + failures.append("motif real-window contracts are not all passing") + for name in ( + "retained_nontrain_fragment_count", + "retained_holdout_fragment_count", + "post_exclusion_cross_split_finding_count", + ): + if leakage.get(name) != 0: + failures.append(f"motif leakage evidence {name} is nonzero") + if not audit.accepted or audit.total_violation_count: + failures.append("reconstruction information audit is not accepted") + if failures: + raise ValueError("; ".join(failures)) + return { + "benchmark_corpus_id": benchmark.corpus_id, + "motif_library_id": str(qualification.get("library_id", "")), + "motif_candidate_report_id": str( + qualification.get("candidate_report_id", "") + ), + "information_audit_id": audit.audit_id, + "information_violation_count": audit.total_violation_count, + "leakage_cross_split_finding_count": cast( + int, leakage["post_exclusion_cross_split_finding_count"] + ), + } + + +def _commit_or_recover( + staged: StagedReconstructionPublicationV2, +) -> PublishedReconstructionV2: + if staged.staging_directory.exists(): + return commit_delivery_reconstruction_publication(staged) + manifest_path = staged.committed_directory / "manifest.json" + if not manifest_path.exists(): + raise ValueError("neither staged nor committed publication exists") + manifest = verify_reconstruction_publication(manifest_path) + if not isinstance(manifest, ReconstructionProductManifestV2): + raise ValueError("recovered publication is not a v2 product") + if manifest != staged.manifest: + raise ValueError("recovered publication differs from staged evidence") + ref = artifact_ref_for_file( + manifest_path, kind="reconstruction-product-manifest" + ) + return PublishedReconstructionV2( + manifest=manifest, + manifest_path=manifest_path, + manifest_ref=replace( + ref, + metadata={ + "schema_version": manifest.schema_version, + "publication_id": manifest.publication_id, + "manifest_id": manifest.manifest_id, + "event_count": manifest.event_count, + "logical_content_sha256": manifest.replay.logical_content_sha256, + }, + ), + idempotent_retry=True, + ) + + +def _restore_cross_group( + manifest: Mapping[str, Any], +) -> CrossCurrencyReconciledGroupV1: + payload = dict(_mapping(manifest["group"])) + payload["streams"] = [ + stream.to_dict() + for stream in _read_stream_map(manifest, "stream_refs").values() + ] + return CrossCurrencyReconciledGroupV1.from_dict(payload) + + +def _restore_delivered_group( + manifest: Mapping[str, Any], +) -> ReconstructionDeliveredGroupV1: + delivery = ReconstructionDeliveryManifestV1.from_dict( + _mapping(manifest["delivery_manifest"]) + ) + streams = tuple(_read_stream_map(manifest, "stream_refs").values()) + return ReconstructionDeliveredGroupV1(manifest=delivery, streams=streams) + + +def _group_without_streams( + group: CrossCurrencyReconciledGroupV1, +) -> dict[str, JSONValue]: + payload: dict[str, JSONValue] = group.to_dict() + payload["streams"] = [] + return payload + + +def _carved_evidence( + batch: HistoricalCarvedCandidateBatchV1, +) -> dict[str, JSONValue]: + payload: dict[str, JSONValue] = batch.metadata() + return payload + + +def _candidate_evidence( + batch: EmpiricalMotifCandidateBatchV1, +) -> dict[str, JSONValue]: + """Serialize bounded batch lineage while large reusable rows stay external.""" + return { + "schema_version": batch.schema_version, + "run_id": batch.run_id, + "window_id": batch.window_id, + "ensemble_member_id": batch.ensemble_member_id, + "symbol": batch.symbol, + "anchor_interval_id": batch.anchor_interval_id, + "left_anchor_event_id": batch.left_anchor_event_id, + "right_anchor_event_id": batch.right_anchor_event_id, + "generator_config_id": batch.generator_config.config_id, + "query_evidence": _compact_query_evidence(batch.query_result), + "status": batch.status.value, + "decision": batch.decision.value, + "target_event_count": batch.target_event_count, + "transformations": [item.to_dict() for item in batch.transformations], + "event_lineage": [item.to_dict() for item in batch.event_lineage], + "resource_estimate": batch.resource_estimate.to_dict(), + "carry_state": batch.carry_state.to_dict(), + "decision_details": list(batch.decision_details), + "batch_id": batch.batch_id, + "candidate_events_inline": False, + } + + +def _restore_candidate_batches( + manifest: Mapping[str, Any], + *, + index: ReferenceMotifIndexV1, +) -> Iterable[EmpiricalMotifCandidateBatchV1]: + """Yield verified batches without retaining motif fragments per interval.""" + streams = _read_stream_map(manifest, "candidate_stream_refs") + events_by_interval: dict[str, list[SyntheticEventV1]] = {} + for stream in streams.values(): + for event in stream.events: + interval_id = event.anchor_interval_id + if interval_id is None: + raise ValueError("candidate event lacks anchor interval") + events_by_interval.setdefault(interval_id, []).append(event) + available = { + event.event_id for stream in streams.values() for event in stream.events + } + expected_batch_count = int(manifest.get("batch_count", -1)) + ledger_value = manifest.get("batch_ledger_ref") + if ledger_value is not None: + if manifest.get("batches_inline") is not False: + raise ValueError("proposal batch ledger must remain external") + ledger_ref = ArtifactRef.from_dict(_mapping(ledger_value)) + if ledger_ref.kind != _CANDIDATE_BATCH_LEDGER_KIND: + raise ValueError("proposal batch ledger kind differs") + verify_artifact_ref(ledger_ref) + if ledger_ref.metadata.get("format") != "canonical-json-lines-v1": + raise ValueError("proposal batch ledger format differs") + if ledger_ref.metadata.get("batch_count") != expected_batch_count: + raise ValueError("proposal batch ledger count metadata differs") + values: Iterable[Any] = _read_candidate_batch_ledger(ledger_ref) + else: + inline_values = _sequence(manifest["batches"]) + if expected_batch_count != len(inline_values): + raise ValueError( + "candidate batch count differs from proposal manifest" + ) + values = inline_values + conditions = { + str(symbol): ReferenceMotifConditionV1.from_dict(_mapping(value)) + for symbol, value in _mapping( + manifest.get("query_conditions", {}) + ).items() + } + shared_config_value = manifest.get("generator_config") + shared_config = ( + EmpiricalMotifGeneratorConfigV1.from_dict(_mapping(shared_config_value)) + if shared_config_value is not None + else None + ) + if manifest.get("motif_index_id") != index.index_id: + raise ValueError("proposal motif index differs from current artifact") + consumed: set[str] = set() + restored_batch_count = 0 + for value in values: + restored_batch_count += 1 + data = _mapping(value) + if data.get("candidate_events_inline") is not False: + raise ValueError("candidate evidence embeds event rows") + interval_id = str(data.get("anchor_interval_id", "")) + events = tuple(events_by_interval.get(interval_id, ())) + consumed.update(event.event_id for event in events) + symbol = str(data.get("symbol", "")) + config = shared_config + if config is None: + config = EmpiricalMotifGeneratorConfigV1.from_dict( + _mapping(data.get("generator_config")) + ) + if ( + data.get("generator_config_id", config.config_id) + != config.config_id + ): + raise ValueError("candidate generator config identity differs") + query_result_value = data.get("query_result") + if query_result_value is not None: + query_result = ReferenceMotifQueryResultV1.from_dict( + _mapping(query_result_value) + ) + if query_result.index_id != index.index_id: + raise ValueError("candidate query result index differs") + else: + condition = conditions.get(symbol) + if condition is None: + raise ValueError("candidate query condition is absent") + query_result = _restore_compact_query_result( + _mapping(data.get("query_evidence")), + condition=condition, + index=index, + ) + yield EmpiricalMotifCandidateBatchV1( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbol=symbol, + anchor_interval_id=interval_id, + left_anchor_event_id=str(data.get("left_anchor_event_id", "")), + right_anchor_event_id=str(data.get("right_anchor_event_id", "")), + generator_config=config, + query_result=query_result, + status=MotifGenerationStatus(str(data.get("status", ""))), + decision=MotifGenerationDecision(str(data.get("decision", ""))), + target_event_count=int(data.get("target_event_count", 0)), + events=events, + transformations=tuple( + EmpiricalMotifTransformationV1.from_dict(_mapping(item)) + for item in _sequence(data.get("transformations")) + ), + event_lineage=tuple( + EmpiricalMotifEventLineageV1.from_dict(_mapping(item)) + for item in _sequence(data.get("event_lineage")) + ), + resource_estimate=ReconstructionResourceEstimateV1.from_dict( + _mapping(data.get("resource_estimate")) + ), + carry_state=CarryStateV1.from_dict( + _mapping(data.get("carry_state")) + ), + decision_details=tuple( + str(item) for item in _sequence(data.get("decision_details")) + ), + batch_id=str(data.get("batch_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + if consumed != available: + raise ValueError("candidate Parquet rows do not reconcile with batches") + if restored_batch_count != expected_batch_count: + raise ValueError("candidate batch ledger row count differs") + + +def _read_candidate_batch_ledger( + ref: ArtifactRef, +) -> Iterable[Mapping[str, Any]]: + with Path(ref.path).open("rb") as stream: + for line in stream: + if len(line) > _MAX_CANDIDATE_BATCH_LEDGER_LINE_BYTES: + raise ValueError("candidate batch ledger row exceeds limit") + try: + value = json.loads(line) + except (UnicodeError, json.JSONDecodeError) as err: + raise ValueError( + "candidate batch ledger row is invalid" + ) from err + yield _mapping(value) + + +def _compact_query_evidence( + result: ReferenceMotifQueryResultV1, +) -> dict[str, JSONValue]: + query = result.query + return { + "query_schema_version": query.schema_version, + "information_mode": query.information_mode.value, + "used_at_ns": query.used_at_ns, + "as_of_ns": query.as_of_ns, + "max_results": query.max_results, + "min_cell_support": query.min_cell_support, + "max_distance": query.max_distance, + "excluded_source_window_ids": list(query.excluded_source_window_ids), + "query_id": query.query_id, + "index_id": result.index_id, + "status": result.status.value, + "result_id": result.result_id, + "result_schema_version": result.schema_version, + "retrieval_rows_inline": False, + } + + +def _restore_compact_query_result( + data: Mapping[str, Any], + *, + condition: ReferenceMotifConditionV1, + index: ReferenceMotifIndexV1, +) -> ReferenceMotifQueryResultV1: + if data.get("retrieval_rows_inline") is not False: + raise ValueError("compact query evidence must not embed retrieval rows") + if data.get("index_id") != index.index_id: + raise ValueError("compact query evidence index differs") + query = ReferenceMotifQueryV1( + condition=condition, + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + used_at_ns=int(data.get("used_at_ns", 0)), + as_of_ns=( + int(data["as_of_ns"]) if data.get("as_of_ns") is not None else None + ), + max_results=int(data.get("max_results", 0)), + min_cell_support=( + int(data["min_cell_support"]) + if data.get("min_cell_support") is not None + else None + ), + max_distance=( + float(data["max_distance"]) + if data.get("max_distance") is not None + else None + ), + excluded_source_window_ids=tuple( + str(item) + for item in _sequence(data.get("excluded_source_window_ids")) + ), + query_id=str(data.get("query_id", "")), + schema_version=str(data.get("query_schema_version", "")), + ) + result = query_reference_motifs(index, query) + if ( + result.result_id != data.get("result_id") + or result.status.value != data.get("status") + or result.schema_version != data.get("result_schema_version") + ): + raise ValueError("replayed compact query evidence differs") + return result + + +def _write_streams( + invocation: ReconstructionStageInvocationV1, + events_by_symbol: Mapping[str, Sequence[SyntheticEventV1]], + *, + directory_name: str, + kind: str, +) -> dict[str, ArtifactRef]: + refs: dict[str, ArtifactRef] = {} + directory = _stage_directory(invocation, directory_name) + directory.mkdir(parents=True, exist_ok=True) + for symbol, events in sorted(events_by_symbol.items()): + _cancel_if_requested(invocation) + stream = SyntheticEventStreamV1( + run_id=invocation.run.run_id, + ensemble_member_id=invocation.task.window.ensemble_member_id, + symbol=symbol, + events=tuple(events), + source_version_ids=invocation.run.source_version_ids, + ) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{symbol.lower()}-", suffix=".parquet", dir=directory + ) + os.close(descriptor) + temporary = Path(temporary_name) + try: + write_synthetic_event_stream_parquet(stream, temporary) + digest = _file_sha256(temporary) + target = directory / f"{symbol.lower()}-{digest}.parquet" + if target.exists(): + if _file_sha256(target) != digest: + raise ValueError("content-addressed stream collision") + temporary.unlink() + else: + os.replace(temporary, target) + refs[symbol] = artifact_ref_for_file( + target, + kind=kind, + metadata={ + "symbol": symbol, + "stream_id": stream.stream_id, + "event_count": len(stream.events), + "observed_event_count": stream.observed_event_count, + "synthetic_event_count": stream.synthetic_event_count, + }, + ) + finally: + temporary.unlink(missing_ok=True) + return refs + + +def _read_stream_map( + manifest: Mapping[str, Any], key: str +) -> dict[str, SyntheticEventStreamV1]: + result: dict[str, SyntheticEventStreamV1] = {} + for symbol, value in _mapping(manifest[key]).items(): + ref = ArtifactRef.from_dict(_mapping(value)) + verify_artifact_ref(ref) + stream = read_synthetic_event_stream_parquet(ref.path) + if stream.symbol != symbol: + raise ValueError("stream reference symbol differs") + result[symbol] = stream + return result + + +def _write_json_artifact( + invocation: ReconstructionStageInvocationV1, + directory_name: str, + kind: str, + payload: Mapping[str, JSONValue], + *, + metadata: Mapping[str, JSONValue] | None = None, +) -> ArtifactRef: + value: dict[str, JSONValue] = { + "schema_version": RECONSTRUCTION_STAGE_ARTIFACT_SCHEMA_VERSION, + **dict(payload), + } + encoded = canonical_contract_json(value).encode("utf-8") + b"\n" + digest = hashlib.sha256(encoded).hexdigest() + directory = _stage_directory(invocation, directory_name) + directory.mkdir(parents=True, exist_ok=True) + target = directory / f"{kind}-{digest}.json" + if target.exists(): + if target.read_bytes() != encoded: + raise ValueError("content-addressed stage artifact collision") + else: + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{target.name}.", dir=directory + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + return ArtifactRef( + kind=kind, + path=str(target.resolve()), + size_bytes=len(encoded), + sha256=digest, + metadata=dict(metadata or {}), + ) + + +def _write_product_manifest_mirror( + invocation: ReconstructionStageInvocationV1, + manifest: ReconstructionProductManifestV2, +) -> ArtifactRef: + """Persist byte-identical staged evidence outside the rename source.""" + encoded = manifest.to_json().encode("utf-8") + digest = hashlib.sha256(encoded).hexdigest() + directory = _stage_directory(invocation, "validation") + directory.mkdir(parents=True, exist_ok=True) + target = directory / f"staged-product-manifest-{digest}.json" + if target.exists(): + if target.read_bytes() != encoded: + raise ValueError("staged product manifest mirror collision") + else: + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{target.name}.", dir=directory + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + return ArtifactRef( + kind=VALIDATION_STAGE_ARTIFACT_KIND, + path=str(target.resolve()), + size_bytes=len(encoded), + sha256=digest, + metadata={ + "commit_phase": "staged", + "publication_id": manifest.publication_id, + "manifest_id": manifest.manifest_id, + "logical_content_sha256": manifest.replay.logical_content_sha256, + }, + ) + + +def _prior_manifest( + invocation: ReconstructionStageInvocationV1, kind: str +) -> Mapping[str, Any]: + return _read_json_ref(_prior_ref(invocation, kind)) + + +def _prior_ref( + invocation: ReconstructionStageInvocationV1, kind: str +) -> ArtifactRef: + matches = tuple( + ref + for outcome in invocation.prior_outcomes + for ref in outcome.output_refs + if ref.kind == kind + ) + if len(matches) != 1: + raise ValueError(f"expected exactly one prior {kind} artifact") + return matches[0] + + +def _read_json_ref(ref: ArtifactRef) -> Mapping[str, Any]: + verify_artifact_ref(ref) + try: + value = json.loads(Path(ref.path).read_text(encoding="utf-8")) + except (UnicodeError, json.JSONDecodeError) as err: + raise ValueError(f"invalid JSON stage artifact: {ref.path}") from err + return _mapping(value) + + +def _completed( + invocation: ReconstructionStageInvocationV1, + ref: ArtifactRef, + *, + started: float, + observed: int = 0, + candidates: int = 0, + accepted: int = 0, + rejected: int = 0, + message: str, + output_bytes: int | None = None, + additional_refs: Sequence[ArtifactRef] = (), +) -> ReconstructionStageOutcomeV1: + runtime = max(0.0, time.perf_counter() - started) + scratch = _tree_size(invocation.task.scratch_directory) + actual_output = ( + int(output_bytes) + if output_bytes is not None + else _artifact_graph_size(ref) + ) + amplification = candidates / observed if observed else 0.0 + telemetry: dict[str, JSONValue] = { + "runtime_seconds": round(runtime, 6), + "peak_rss_bytes": _peak_rss_bytes(), + "scratch_bytes": scratch, + "output_bytes": actual_output, + "observed_event_count": observed, + "candidate_event_count": candidates, + "accepted_event_count": accepted, + "rejected_event_count": rejected, + "candidate_amplification": round(amplification, 9), + } + outputs = tuple( + replace(item, metadata={**item.metadata, **telemetry}) + for item in (ref, *tuple(additional_refs)) + ) + return invocation.completed( + output_refs=outputs, + observed_event_count=observed, + candidate_event_count=candidates, + accepted_event_count=accepted, + scratch_bytes=scratch, + output_bytes=actual_output, + message=message, + ) + + +def _cancel_if_requested(invocation: ReconstructionStageInvocationV1) -> None: + if not invocation.cancellation_requested: + return + shutil.rmtree(invocation.task.scratch_directory, ignore_errors=True) + raise asyncio.CancelledError + + +def _stage_directory( + invocation: ReconstructionStageInvocationV1, name: str +) -> Path: + root = Path(invocation.task.scratch_directory).expanduser().resolve() + directory = (root / name).resolve() + if not directory.is_relative_to(root): + raise ValueError("stage directory escaped window scratch") + return directory + + +def _stage_scope( + invocation: ReconstructionStageInvocationV1, +) -> dict[str, JSONValue]: + window = invocation.task.window + return { + "run_id": window.run_id, + "window_id": window.window_id, + "synchronization_unit_id": window.synchronization_unit_id, + "ensemble_member_id": window.ensemble_member_id, + "stage": invocation.command.stage.value, + "large_event_rows_inline": False, + } + + +def _refs_dict(values: Mapping[str, ArtifactRef]) -> dict[str, JSONValue]: + return { + symbol: cast(JSONValue, ref.to_dict()) + for symbol, ref in sorted(values.items()) + } + + +def _events_content_sha256(events: Iterable[SyntheticEventV1]) -> str: + digest = hashlib.sha256(b"histdatacom-stage-events-v1\n") + for event in sorted( + events, + key=lambda item: ( + item.symbol, + item.event_time_ns, + item.event_sequence, + item.event_id, + ), + ): + digest.update(event.to_json().encode("utf-8")) + digest.update(b"\n") + return digest.hexdigest() + + +def _tree_size(path: str | Path) -> int: + root = Path(path) + if not root.exists(): + return 0 + return sum( + item.stat().st_size + for item in root.rglob("*") + if item.is_file() and not item.is_symlink() + ) + + +def _peak_rss_bytes() -> int: + return peak_rss_bytes() + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _artifact_graph_size(ref: ArtifactRef) -> int: + """Measure a stage manifest and every distinct strong file it names.""" + paths: dict[str, int] = {ref.path: ref.size_bytes or 0} + if not ref.path.endswith(".json"): + return sum(paths.values()) + try: + payload = json.loads(Path(ref.path).read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return sum(paths.values()) + + def visit(value: Any) -> None: + if isinstance(value, Mapping): + path = value.get("path") + size = value.get("size_bytes") + digest = value.get("sha256") + kind = value.get("kind") + if ( + isinstance(path, str) + and isinstance(size, int) + and isinstance(digest, str) + and isinstance(kind, str) + and bool(kind.strip()) + and len(digest) == 64 + ): + paths[path] = size + for item in value.values(): + visit(item) + elif isinstance(value, Sequence) and not isinstance( + value, (str, bytes, bytearray) + ): + for item in value: + visit(item) + + visit(payload) + return sum(paths.values()) + + +def _bounded_error(err: BaseException) -> str: + text = f"{type(err).__name__}: {err}".strip() + return text[:2_048] + + +def _bounded_reasons( + values: Sequence[str], *, fallback: str +) -> tuple[str, ...]: + selected = tuple(str(value)[:1_024] for value in values[:32] if str(value)) + return selected or (fallback,) + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a JSON object") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if not isinstance(value, Sequence) or isinstance( + value, (str, bytes, bytearray) + ): + raise ValueError("expected a JSON sequence") + return value + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if data.get("schema_version") != expected: + raise ValueError("unsupported staging descriptor schema") + + +__all__ = [ + "CARVING_STAGE_ARTIFACT_KIND", + "CROSS_STAGE_ARTIFACT_KIND", + "DELIVERY_STAGE_ARTIFACT_KIND", + "PROPOSAL_STAGE_ARTIFACT_KIND", + "RECONSTRUCTION_STAGE_ARTIFACT_SCHEMA_VERSION", + "RECONSTRUCTION_STAGING_DESCRIPTOR_SCHEMA_VERSION", + "SOURCE_STAGE_ARTIFACT_KIND", + "STAGING_DESCRIPTOR_ARTIFACT_KIND", + "VALIDATION_STAGE_ARTIFACT_KIND", + "atomic_commit_handler", + "carving_handler", + "cross_series_reconciliation_handler", + "delivery_projection_handler", + "proposal_handler", + "register_first_party_reconstruction_handlers", + "source_enrichment_handler", + "validation_handler", +] diff --git a/src/histdatacom/synthetic/reconstruction_plan.py b/src/histdatacom/synthetic/reconstruction_plan.py new file mode 100644 index 00000000..ea65d70d --- /dev/null +++ b/src/histdatacom/synthetic/reconstruction_plan.py @@ -0,0 +1,3375 @@ +"""First-party deterministic planning for the ASCII tick reconstruction product. + +This module is the bounded integration seam between the repository's +scientific contracts and its Temporal reconstruction request contract. It +resolves strong artifacts, inventories immutable ASCII tick partitions, +performs compatibility and information-safety preflight, estimates resources, +and emits stage-complete workflow requests. Tick rows never enter the plan or +workflow payloads. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from functools import lru_cache +import hashlib +import json +import math +import os +from pathlib import Path +import re +from typing import Any, cast + +from histdatacom.data_analytics.feed_epochs_v2 import ( + read_active_time_feed_epoch_definition, +) +from histdatacom.market_context import ( + CftcPositioningCorpusV1, + CftcReportFamily, + CftcReportScope, + MarketContextCorpusV1, + MarketContextKind, + preflight_cftc_positioning_corpus, + preflight_market_context_corpus, + read_cftc_positioning_corpus, + read_market_context_corpus, +) +from histdatacom.orchestration.reconstruction import ( + MAX_STAGE_ARTIFACT_REFS, + RECONSTRUCTION_STAGE_ORDER, + ReconstructionStage, + ReconstructionStageCommandV1, + ReconstructionWindowTaskV1, + ReconstructionWorkflowRequestV1, + artifact_ref_for_file, + verify_artifact_ref, +) +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.benchmark_corpus import ( + PREDECLARED_GATE_COMMIT, + read_reverse_degradation_benchmark_corpus, +) +from histdatacom.synthetic.benchmark_gates import ( + load_default_benchmark_promotion_gate_policy, +) +from histdatacom.synthetic.carving import HistoricalCarvingConstraintSetV1 +from histdatacom.synthetic.contracts import canonical_contract_json +from histdatacom.synthetic.cross_currency import ( + EURUSD_TRIANGLE_SYMBOLS, + CrossCurrencyReconciliationConfigV1, + CrossCurrencySymbolCoverageV1, + CrossCurrencyWindowPlanStatus, + eurusd_triangle_reconciliation_config, + plan_cross_currency_windows, +) +from histdatacom.synthetic.delivery import ReconstructionDeliveryMode +from histdatacom.synthetic.ensembles import ( + EnsembleCalibrationConfigV1, + plan_reconstruction_ensemble, +) +from histdatacom.synthetic.generation import EmpiricalMotifGeneratorConfigV1 +from histdatacom.synthetic.information import ( + InformationAuditReportV1, + InformationInputKind, + InformationMode, + InformationScope, + InformationSplitKind, + InformationStage, + ReconstructionInformationInputV1, + ReconstructionInformationManifestV1, + ReconstructionInformationPolicyV1, + ReconstructionInformationSplitV1, + reconstruction_information_window_plan_id, + require_reconstruction_information_audit, +) +from histdatacom.synthetic.motif_library import ( + ModernReferenceMotifProfileV1, + read_modern_reference_motif_artifact, + read_modern_reference_motif_index, +) +from histdatacom.synthetic.observation import ( + ObservationOperatorV1, + read_observation_operator_artifact, +) +from histdatacom.synthetic.persistence import estimate_reconstruction_retention +from histdatacom.synthetic.streaming import ( + ReconstructionResourceEstimateV1, + ReconstructionRunV1, + ReconstructionStoragePolicyV1, + ReconstructionWindowV1, +) + +RECONSTRUCTION_SOURCE_PARTITION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-source-partition.v1" +) +RECONSTRUCTION_SOURCE_INVENTORY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-source-inventory.v1" +) +RECONSTRUCTION_PLAN_CONFIGURATION_SCHEMA_VERSION = ( + "histdatacom.reconstruction-plan-configuration.v1" +) +RECONSTRUCTION_PLAN_EXECUTION_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-plan-execution-manifest.v1" +) +RECONSTRUCTION_PLAN_REFUSAL_SCHEMA_VERSION = ( + "histdatacom.reconstruction-plan-refusal.v1" +) +RECONSTRUCTION_PLAN_RESOURCE_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-plan-resource-summary.v1" +) +SYNTHETIC_INFILL_PLAN_SCHEMA_VERSION = "histdatacom.synthetic-infill-plan.v1" + +ASCII_TICK_SOURCE_KIND = "histdata_ascii_tick_arrow" +SOURCE_INVENTORY_ARTIFACT_KIND = "reconstruction_source_inventory_v1" +PLAN_CONFIGURATION_ARTIFACT_KIND = "reconstruction_plan_configuration_v1" +PLAN_EXECUTION_MANIFEST_ARTIFACT_KIND = ( + "reconstruction_plan_execution_manifest_v1" +) +SYNTHETIC_INFILL_PLAN_ARTIFACT_KIND = "synthetic_infill_plan_v1" + +DEFAULT_RECONSTRUCTION_WINDOW_SIZE_NS = 30 * 24 * 60 * 60 * 1_000_000_000 +DEFAULT_RECONSTRUCTION_REQUEST_WINDOW_LIMIT = 32 +DEFAULT_RECONSTRUCTION_MAX_PARALLEL_WINDOWS = 2 +DEFAULT_RECONSTRUCTION_BASE_SEED = 20260715 +_RESOURCE_FIXED_OVERHEAD_BYTES = 512 * 1024 * 1024 +_RESOURCE_LEDGER_BYTES_PER_INTERVAL = 8 * 1024 +_SOURCE_ROW_DENSITY_SAFETY_FACTOR = 4 +MAX_RECONSTRUCTION_PLAN_ARTIFACTS = 64 +MAX_RECONSTRUCTION_PLAN_REFUSALS = 4096 +MAX_RECONSTRUCTION_PLAN_REQUESTS = 4096 +MAX_SYNTHETIC_INFILL_PLAN_BYTES = 64 * 1024 * 1024 + +SCIENTIFIC_NONCLAIM = ( + "Output is a plausible counterfactual ensemble conditioned on declared " + "artifacts and constraints; it is not recovered historical truth." +) +IMMUTABLE_ANCHOR_POLICY = ( + "Every observed ASCII tick keeps its original bid, ask, timestamp, " + "partition, and zero-based Arrow row ordinal; synthetic events are added " + "without renumbering or replacing observed anchors." +) +TICK_ONLY_INPUT_POLICY = ( + "Only ASCII/T bid-ask tick caches are reconstruction inputs. M1, bars, " + "OHLC, and downstream bar projections are never anchors or inputs." +) + +FIRST_PARTY_RECONSTRUCTION_HANDLERS: Mapping[ReconstructionStage, str] = { + ReconstructionStage.SOURCE_ENRICHMENT: ( + "histdatacom.reconstruction.source-enrichment.v2" + ), + ReconstructionStage.PROPOSAL: "histdatacom.reconstruction.proposal.v6", + ReconstructionStage.CARVING: "histdatacom.reconstruction.carving.v3", + ReconstructionStage.CROSS_SERIES_RECONCILIATION: ( + "histdatacom.reconstruction.cross-series-reconciliation.v4" + ), + ReconstructionStage.BROKER_TRANSFER: ( + "histdatacom.reconstruction.delivery-projection.v1" + ), + ReconstructionStage.VALIDATION: ( + "histdatacom.reconstruction.validation.v1" + ), + ReconstructionStage.ATOMIC_PARTITION_COMMIT: ( + "histdatacom.reconstruction.atomic-partition-commit.v1" + ), +} + +_PERIOD_RE = re.compile(r"^\d{6}$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_MAX_SOURCE_PARTITION_UTC_SPILL_NS = 24 * 60 * 60 * 1_000_000_000 + + +class ReconstructionPlanCompatibilityError(ValueError): + """Resolved artifacts cannot safely produce an executable plan.""" + + +class ReconstructionPlanRefusalCode(str, Enum): + """Stable pre-execution refusal categories.""" + + FEED_EPOCH_UNSUPPORTED = "feed_epoch_unsupported" + MARKET_CONTEXT_UNSUPPORTED = "market_context_unsupported" + CFTC_POSITIONING_UNSUPPORTED = "cftc_positioning_unsupported" + INFORMATION_LEAKAGE = "information_leakage" + + +@dataclass(frozen=True, slots=True) +class ReconstructionSourcePartitionV1: + """One immutable monthly ASCII/T Arrow cache partition.""" + + symbol: str + period: str + artifact: ArtifactRef + row_count: int + coverage_start_ns: int + coverage_end_ns: int + first_timestamp_ms: int + last_timestamp_ms: int + feed_epoch_evidence_id: str + partition_id: str = "" + schema_version: str = RECONSTRUCTION_SOURCE_PARTITION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_SOURCE_PARTITION_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction source partition schema" + ) + symbol = _symbol(self.symbol) + period = _period(self.period) + artifact = _strong_ref(self.artifact) + if artifact.kind != ASCII_TICK_SOURCE_KIND: + raise ValueError("source partition requires an ASCII tick artifact") + if artifact.metadata.get("symbol") != symbol: + raise ValueError("source partition artifact symbol differs") + if artifact.metadata.get("period") != period: + raise ValueError("source partition artifact period differs") + rows = _positive_int(self.row_count, "row_count") + start = _int64(self.coverage_start_ns, "coverage_start_ns") + end = _int64(self.coverage_end_ns, "coverage_end_ns") + if end <= start: + raise ValueError("source partition coverage is empty") + first = _int64(self.first_timestamp_ms, "first_timestamp_ms") + last = _int64(self.last_timestamp_ms, "last_timestamp_ms") + if last < first: + raise ValueError("source partition timestamps regress") + if not ( + start - _MAX_SOURCE_PARTITION_UTC_SPILL_NS + <= first * 1_000_000 + < end + _MAX_SOURCE_PARTITION_UTC_SPILL_NS + ): + raise ValueError( + "source partition first timestamp is outside period" + ) + if not ( + start - _MAX_SOURCE_PARTITION_UTC_SPILL_NS + <= last * 1_000_000 + < end + _MAX_SOURCE_PARTITION_UTC_SPILL_NS + ): + raise ValueError( + "source partition last timestamp is outside period" + ) + object.__setattr__(self, "symbol", symbol) + object.__setattr__(self, "period", period) + object.__setattr__(self, "artifact", artifact) + object.__setattr__(self, "row_count", rows) + object.__setattr__(self, "coverage_start_ns", start) + object.__setattr__(self, "coverage_end_ns", end) + object.__setattr__(self, "first_timestamp_ms", first) + object.__setattr__(self, "last_timestamp_ms", last) + object.__setattr__( + self, + "feed_epoch_evidence_id", + _required_text(self.feed_epoch_evidence_id), + ) + expected = _stable_id( + "reconstruction-source-partition", self.identity_payload() + ) + if self.partition_id and self.partition_id != expected: + raise ValueError( + "source partition_id differs from immutable content" + ) + object.__setattr__(self, "partition_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbol": self.symbol, + "period": self.period, + "sha256": self.artifact.sha256, + "size_bytes": self.artifact.size_bytes, + "row_count": self.row_count, + "coverage_start_ns": self.coverage_start_ns, + "coverage_end_ns": self.coverage_end_ns, + "first_timestamp_ms": self.first_timestamp_ms, + "last_timestamp_ms": self.last_timestamp_ms, + "feed_epoch_evidence_id": self.feed_epoch_evidence_id, + "row_identity_basis": "zero-based-arrow-row-ordinal-v1", + "partition_clock_policy": "histdata-source-month-with-utc-spill-v1", + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "artifact": self.artifact.to_dict(), + "partition_id": self.partition_id, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionSourcePartitionV1": + _require_schema(data, RECONSTRUCTION_SOURCE_PARTITION_SCHEMA_VERSION) + _require_derived( + data, "row_identity_basis", "zero-based-arrow-row-ordinal-v1" + ) + _require_derived( + data, + "partition_clock_policy", + "histdata-source-month-with-utc-spill-v1", + ) + return cls( + symbol=str(data.get("symbol", "")), + period=str(data.get("period", "")), + artifact=ArtifactRef.from_dict(_mapping(data.get("artifact"))), + row_count=_strict_int(data.get("row_count"), "row_count"), + coverage_start_ns=_strict_int( + data.get("coverage_start_ns"), "coverage_start_ns" + ), + coverage_end_ns=_strict_int( + data.get("coverage_end_ns"), "coverage_end_ns" + ), + first_timestamp_ms=_strict_int( + data.get("first_timestamp_ms"), "first_timestamp_ms" + ), + last_timestamp_ms=_strict_int( + data.get("last_timestamp_ms"), "last_timestamp_ms" + ), + feed_epoch_evidence_id=str(data.get("feed_epoch_evidence_id", "")), + partition_id=str(data.get("partition_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionSourceInventoryV1: + """Complete common-range immutable source inventory.""" + + source_root: str + symbols: tuple[str, ...] + periods: tuple[str, ...] + partitions: tuple[ReconstructionSourcePartitionV1, ...] + requested_start_ns: int + requested_end_ns: int + total_row_count: int + total_size_bytes: int + inventory_id: str = "" + schema_version: str = RECONSTRUCTION_SOURCE_INVENTORY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_SOURCE_INVENTORY_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction source inventory schema" + ) + root = str( + Path(_required_text(self.source_root)).expanduser().resolve() + ) + symbols = _symbols(self.symbols) + periods = tuple(sorted({_period(value) for value in self.periods})) + if not periods: + raise ValueError("source inventory periods cannot be empty") + partitions = tuple( + sorted(self.partitions, key=lambda item: (item.period, item.symbol)) + ) + expected_keys = { + (period, symbol) for period in periods for symbol in symbols + } + actual_keys = {(item.period, item.symbol) for item in partitions} + if actual_keys != expected_keys or len(partitions) != len( + expected_keys + ): + raise ValueError( + "source inventory is not a complete synchronized triangle" + ) + start = _int64(self.requested_start_ns, "requested_start_ns") + end = _int64(self.requested_end_ns, "requested_end_ns") + if end <= start: + raise ValueError("source inventory requested interval is empty") + rows = sum(item.row_count for item in partitions) + size = sum(cast(int, item.artifact.size_bytes) for item in partitions) + if self.total_row_count != rows or self.total_size_bytes != size: + raise ValueError("source inventory aggregate counts differ") + object.__setattr__(self, "source_root", root) + object.__setattr__(self, "symbols", symbols) + object.__setattr__(self, "periods", periods) + object.__setattr__(self, "partitions", partitions) + object.__setattr__(self, "requested_start_ns", start) + object.__setattr__(self, "requested_end_ns", end) + expected = _stable_id( + "reconstruction-source-inventory", self.identity_payload() + ) + if self.inventory_id and self.inventory_id != expected: + raise ValueError( + "source inventory_id differs from immutable content" + ) + object.__setattr__(self, "inventory_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "symbols": list(self.symbols), + "periods": list(self.periods), + "partitions": [item.identity_payload() for item in self.partitions], + "requested_start_ns": self.requested_start_ns, + "requested_end_ns": self.requested_end_ns, + "total_row_count": self.total_row_count, + "total_size_bytes": self.total_size_bytes, + "input_contract": "ascii/T-tick-bid-ask-only", + "observed_values_immutable": True, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "source_root": self.source_root, + "partitions": [item.to_dict() for item in self.partitions], + "inventory_id": self.inventory_id, + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + def partitions_for_window( + self, window: ReconstructionWindowV1 + ) -> tuple[ReconstructionSourcePartitionV1, ...]: + selected = tuple( + item + for item in self.partitions + if item.coverage_end_ns > window.input_start_ns + and item.coverage_start_ns < window.input_end_ns + ) + if {item.symbol for item in selected} != set(self.symbols): + raise ReconstructionPlanCompatibilityError( + "window source partitions do not cover the complete triangle" + ) + return selected + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionSourceInventoryV1": + _require_schema(data, RECONSTRUCTION_SOURCE_INVENTORY_SCHEMA_VERSION) + _require_derived(data, "input_contract", "ascii/T-tick-bid-ask-only") + _require_derived(data, "observed_values_immutable", True) + return cls( + source_root=str(data.get("source_root", "")), + symbols=_string_tuple(data.get("symbols")), + periods=_string_tuple(data.get("periods")), + partitions=tuple( + ReconstructionSourcePartitionV1.from_dict(_mapping(value)) + for value in _sequence(data.get("partitions")) + ), + requested_start_ns=_strict_int( + data.get("requested_start_ns"), "requested_start_ns" + ), + requested_end_ns=_strict_int( + data.get("requested_end_ns"), "requested_end_ns" + ), + total_row_count=_strict_int( + data.get("total_row_count"), "total_row_count" + ), + total_size_bytes=_strict_int( + data.get("total_size_bytes"), "total_size_bytes" + ), + inventory_id=str(data.get("inventory_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionSourceInventoryV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionPlanConfigurationV1: + """All scientific and operational policies consumed by stage handlers.""" + + delivery_mode: ReconstructionDeliveryMode + information_policy: ReconstructionInformationPolicyV1 + generator_config: EmpiricalMotifGeneratorConfigV1 + carving_constraints: HistoricalCarvingConstraintSetV1 + cross_currency_config: CrossCurrencyReconciliationConfigV1 + ensemble_config: EnsembleCalibrationConfigV1 + storage_policy: ReconstructionStoragePolicyV1 + window_size_ns: int + left_halo_ns: int + right_lookahead_ns: int + max_parallel_windows: int + configuration_id: str = "" + schema_version: str = RECONSTRUCTION_PLAN_CONFIGURATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_PLAN_CONFIGURATION_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction plan configuration schema" + ) + object.__setattr__( + self, + "delivery_mode", + ReconstructionDeliveryMode.from_value(self.delivery_mode), + ) + expected_types = ( + (self.information_policy, ReconstructionInformationPolicyV1), + (self.generator_config, EmpiricalMotifGeneratorConfigV1), + (self.carving_constraints, HistoricalCarvingConstraintSetV1), + (self.cross_currency_config, CrossCurrencyReconciliationConfigV1), + (self.ensemble_config, EnsembleCalibrationConfigV1), + (self.storage_policy, ReconstructionStoragePolicyV1), + ) + if any( + not isinstance(value, expected) + for value, expected in expected_types + ): + raise ValueError( + "reconstruction plan configuration contains a wrong contract type" + ) + window_size = _positive_int(self.window_size_ns, "window_size_ns") + left = _nonnegative_int(self.left_halo_ns, "left_halo_ns") + right = _nonnegative_int(self.right_lookahead_ns, "right_lookahead_ns") + if ( + self.information_policy.information_mode + is InformationMode.EX_ANTE_SIMULATION + and right + ): + raise ValueError( + "ex-ante reconstruction cannot declare right look-ahead" + ) + parallel = _positive_int( + self.max_parallel_windows, "max_parallel_windows" + ) + if parallel > self.storage_policy.max_inflight_batches: + raise ValueError("plan parallelism exceeds storage inflight quota") + object.__setattr__(self, "window_size_ns", window_size) + object.__setattr__(self, "left_halo_ns", left) + object.__setattr__(self, "right_lookahead_ns", right) + object.__setattr__(self, "max_parallel_windows", parallel) + if ( + tuple(FIRST_PARTY_RECONSTRUCTION_HANDLERS) + != RECONSTRUCTION_STAGE_ORDER + ): + raise ValueError( + "first-party handler map does not cover every stage" + ) + expected = _stable_id( + "reconstruction-plan-configuration", self.identity_payload() + ) + if self.configuration_id and self.configuration_id != expected: + raise ValueError("plan configuration_id differs") + object.__setattr__(self, "configuration_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "delivery_mode": self.delivery_mode.value, + "information_policy": self.information_policy.to_dict(), + "generator_config": self.generator_config.to_dict(), + "carving_constraints": self.carving_constraints.to_dict(), + "cross_currency_config": self.cross_currency_config.to_dict(), + "ensemble_config": self.ensemble_config.to_dict(), + "storage_policy": self.storage_policy.to_dict(), + "window_size_ns": self.window_size_ns, + "left_halo_ns": self.left_halo_ns, + "right_lookahead_ns": self.right_lookahead_ns, + "max_parallel_windows": self.max_parallel_windows, + "handler_names": { + stage.value: FIRST_PARTY_RECONSTRUCTION_HANDLERS[stage] + for stage in RECONSTRUCTION_STAGE_ORDER + }, + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + "immutable_anchor_policy": IMMUTABLE_ANCHOR_POLICY, + "input_policy": TICK_ONLY_INPUT_POLICY, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "configuration_id": self.configuration_id, + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionPlanConfigurationV1": + _require_schema(data, RECONSTRUCTION_PLAN_CONFIGURATION_SCHEMA_VERSION) + _require_derived(data, "scientific_nonclaim", SCIENTIFIC_NONCLAIM) + _require_derived( + data, "immutable_anchor_policy", IMMUTABLE_ANCHOR_POLICY + ) + _require_derived(data, "input_policy", TICK_ONLY_INPUT_POLICY) + _require_derived( + data, + "handler_names", + { + stage.value: FIRST_PARTY_RECONSTRUCTION_HANDLERS[stage] + for stage in RECONSTRUCTION_STAGE_ORDER + }, + ) + return cls( + delivery_mode=ReconstructionDeliveryMode.from_value( + str(data.get("delivery_mode", "")) + ), + information_policy=ReconstructionInformationPolicyV1.from_dict( + _mapping(data.get("information_policy")) + ), + generator_config=EmpiricalMotifGeneratorConfigV1.from_dict( + _mapping(data.get("generator_config")) + ), + carving_constraints=HistoricalCarvingConstraintSetV1.from_dict( + _mapping(data.get("carving_constraints")) + ), + cross_currency_config=CrossCurrencyReconciliationConfigV1.from_dict( + _mapping(data.get("cross_currency_config")) + ), + ensemble_config=EnsembleCalibrationConfigV1.from_dict( + _mapping(data.get("ensemble_config")) + ), + storage_policy=ReconstructionStoragePolicyV1.from_dict( + _mapping(data.get("storage_policy")) + ), + window_size_ns=_strict_int( + data.get("window_size_ns"), "window_size_ns" + ), + left_halo_ns=_strict_int(data.get("left_halo_ns"), "left_halo_ns"), + right_lookahead_ns=_strict_int( + data.get("right_lookahead_ns"), "right_lookahead_ns" + ), + max_parallel_windows=_strict_int( + data.get("max_parallel_windows"), "max_parallel_windows" + ), + configuration_id=str(data.get("configuration_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionPlanConfigurationV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionPlanRefusalV1: + """One window rejected before workflow submission.""" + + start_ns: int + end_ns: int + code: ReconstructionPlanRefusalCode + reason: str + symbols: tuple[str, ...] = EURUSD_TRIANGLE_SYMBOLS + refusal_id: str = "" + schema_version: str = RECONSTRUCTION_PLAN_REFUSAL_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_PLAN_REFUSAL_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction plan refusal schema") + start = _int64(self.start_ns, "start_ns") + end = _int64(self.end_ns, "end_ns") + if end <= start: + raise ValueError("refusal interval is empty") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + object.__setattr__( + self, "code", ReconstructionPlanRefusalCode(self.code) + ) + object.__setattr__(self, "reason", _bounded_text(self.reason, 2048)) + object.__setattr__(self, "symbols", _symbols(self.symbols)) + expected = _stable_id( + "reconstruction-plan-refusal", self.identity_payload() + ) + if self.refusal_id and self.refusal_id != expected: + raise ValueError("reconstruction refusal_id differs") + object.__setattr__(self, "refusal_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "code": self.code.value, + "reason": self.reason, + "symbols": list(self.symbols), + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "refusal_id": self.refusal_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionPlanRefusalV1": + _require_schema(data, RECONSTRUCTION_PLAN_REFUSAL_SCHEMA_VERSION) + return cls( + start_ns=_strict_int(data.get("start_ns"), "start_ns"), + end_ns=_strict_int(data.get("end_ns"), "end_ns"), + code=ReconstructionPlanRefusalCode(str(data.get("code", ""))), + reason=str(data.get("reason", "")), + symbols=_string_tuple(data.get("symbols")), + refusal_id=str(data.get("refusal_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionPlanResourceSummaryV1: + """Full-plan source, candidate, memory, scratch, output, and graph bounds.""" + + source_event_count: int + source_size_bytes: int + planned_window_count: int + executable_window_count: int + refused_window_count: int + ensemble_member_count: int + retained_member_count: int + workflow_request_count: int + estimated_input_event_count: int + estimated_candidate_event_count: int + estimated_candidate_bytes: int + estimated_peak_memory_bytes: int + estimated_peak_scratch_bytes: int + estimated_output_bytes: int + estimated_partition_count: int + summary_id: str = "" + schema_version: str = RECONSTRUCTION_PLAN_RESOURCE_SUMMARY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_PLAN_RESOURCE_SUMMARY_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction plan resource summary schema" + ) + for name in ( + "source_event_count", + "source_size_bytes", + "planned_window_count", + "executable_window_count", + "refused_window_count", + "ensemble_member_count", + "retained_member_count", + "workflow_request_count", + "estimated_input_event_count", + "estimated_candidate_event_count", + "estimated_candidate_bytes", + "estimated_peak_memory_bytes", + "estimated_peak_scratch_bytes", + "estimated_output_bytes", + "estimated_partition_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + if self.planned_window_count != ( + self.executable_window_count + self.refused_window_count + ): + raise ValueError("planned window counts do not reconcile") + if not self.ensemble_member_count: + raise ValueError("resource summary requires an ensemble") + if not 1 <= self.retained_member_count <= self.ensemble_member_count: + raise ValueError("retained member count is outside ensemble") + if self.executable_window_count == 0: + if self.refused_window_count != self.planned_window_count: + raise ValueError( + "zero-work resource summary requires every window refused" + ) + if self.workflow_request_count: + raise ValueError( + "zero-work resource summary cannot contain workflow requests" + ) + if any( + getattr(self, name) + for name in ( + "estimated_input_event_count", + "estimated_candidate_event_count", + "estimated_candidate_bytes", + "estimated_peak_memory_bytes", + "estimated_peak_scratch_bytes", + "estimated_output_bytes", + "estimated_partition_count", + ) + ): + raise ValueError( + "zero-work resource summary must have zero work estimates" + ) + elif not self.workflow_request_count: + raise ValueError("resource summary requires workflow requests") + expected = _stable_id( + "reconstruction-plan-resources", self.identity_payload() + ) + if self.summary_id and self.summary_id != expected: + raise ValueError("resource summary_id differs") + object.__setattr__(self, "summary_id", expected) + + @property + def candidate_amplification(self) -> float: + if not self.estimated_input_event_count: + return 0.0 + return ( + self.estimated_candidate_event_count + / self.estimated_input_event_count + ) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "source_event_count": self.source_event_count, + "source_size_bytes": self.source_size_bytes, + "planned_window_count": self.planned_window_count, + "executable_window_count": self.executable_window_count, + "refused_window_count": self.refused_window_count, + "ensemble_member_count": self.ensemble_member_count, + "retained_member_count": self.retained_member_count, + "workflow_request_count": self.workflow_request_count, + "estimated_input_event_count": self.estimated_input_event_count, + "estimated_candidate_event_count": self.estimated_candidate_event_count, + "estimated_candidate_bytes": self.estimated_candidate_bytes, + "estimated_peak_memory_bytes": self.estimated_peak_memory_bytes, + "estimated_peak_scratch_bytes": self.estimated_peak_scratch_bytes, + "estimated_output_bytes": self.estimated_output_bytes, + "estimated_partition_count": self.estimated_partition_count, + "candidate_amplification": self.candidate_amplification, + "scratch_basis": "peak-concurrent-window-scratch-v1", + "output_basis": "retained-member-compressed-upper-bound-v1", + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "summary_id": self.summary_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionPlanResourceSummaryV1": + _require_schema( + data, RECONSTRUCTION_PLAN_RESOURCE_SUMMARY_SCHEMA_VERSION + ) + _require_derived( + data, "scratch_basis", "peak-concurrent-window-scratch-v1" + ) + _require_derived( + data, "output_basis", "retained-member-compressed-upper-bound-v1" + ) + return cls( + source_event_count=_strict_int( + data.get("source_event_count"), "source_event_count" + ), + source_size_bytes=_strict_int( + data.get("source_size_bytes"), "source_size_bytes" + ), + planned_window_count=_strict_int( + data.get("planned_window_count"), "planned_window_count" + ), + executable_window_count=_strict_int( + data.get("executable_window_count"), "executable_window_count" + ), + refused_window_count=_strict_int( + data.get("refused_window_count"), "refused_window_count" + ), + ensemble_member_count=_strict_int( + data.get("ensemble_member_count"), "ensemble_member_count" + ), + retained_member_count=_strict_int( + data.get("retained_member_count"), "retained_member_count" + ), + workflow_request_count=_strict_int( + data.get("workflow_request_count"), "workflow_request_count" + ), + estimated_input_event_count=_strict_int( + data.get( + "estimated_input_event_count", + data.get("source_event_count"), + ), + "estimated_input_event_count", + ), + estimated_candidate_event_count=_strict_int( + data.get("estimated_candidate_event_count"), + "estimated_candidate_event_count", + ), + estimated_candidate_bytes=_strict_int( + data.get("estimated_candidate_bytes"), + "estimated_candidate_bytes", + ), + estimated_peak_memory_bytes=_strict_int( + data.get("estimated_peak_memory_bytes"), + "estimated_peak_memory_bytes", + ), + estimated_peak_scratch_bytes=_strict_int( + data.get("estimated_peak_scratch_bytes"), + "estimated_peak_scratch_bytes", + ), + estimated_output_bytes=_strict_int( + data.get("estimated_output_bytes"), "estimated_output_bytes" + ), + estimated_partition_count=_strict_int( + data.get("estimated_partition_count"), + "estimated_partition_count", + ), + summary_id=str(data.get("summary_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionPlanExecutionManifestV1: + """Strong artifact graph and execution roots consumed by every stage.""" + + run_id: str + configuration_id: str + source_inventory_id: str + information_manifest_id: str + information_audit_id: str + ensemble_plan_id: str + retention_plan_id: str + delivery_mode: ReconstructionDeliveryMode + artifacts: Mapping[str, ArtifactRef] + output_root: str + checkpoint_root: str + scratch_root: str + planned_window_count: int + executable_window_count: int + refusal_ids: tuple[str, ...] = () + manifest_id: str = "" + schema_version: str = RECONSTRUCTION_PLAN_EXECUTION_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_PLAN_EXECUTION_MANIFEST_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction execution manifest schema" + ) + for name in ( + "run_id", + "configuration_id", + "source_inventory_id", + "information_manifest_id", + "information_audit_id", + "ensemble_plan_id", + "retention_plan_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "delivery_mode", + ReconstructionDeliveryMode.from_value(self.delivery_mode), + ) + artifacts = _artifact_mapping(self.artifacts) + required = { + "configuration", + "source_inventory", + "feed_epochs", + "observation_operator", + "market_context", + "cftc_positioning", + "benchmark_manifest", + "motif_manifest", + "motif_index", + "motif_qualification", + "motif_leakage_audit", + "information_manifest", + "information_audit", + "ensemble_plan", + "retention_plan", + } + if self.delivery_mode is ReconstructionDeliveryMode.BROKER_CONDITIONED: + required.add("broker_delivery") + if set(artifacts) != required: + raise ValueError( + "execution artifact graph is incomplete or contains extras" + ) + object.__setattr__(self, "artifacts", artifacts) + output = _resolved_directory(self.output_root) + checkpoint = _resolved_directory(self.checkpoint_root) + scratch = _resolved_directory(self.scratch_root) + durable_roots = { + "output": Path(output), + "checkpoint": Path(checkpoint), + "scratch": Path(scratch), + } + root_items = tuple(durable_roots.items()) + for index, (left_name, left) in enumerate(root_items): + for right_name, right in root_items[index + 1 :]: + if not _paths_overlap(left, right): + continue + raise ValueError( + f"execution {left_name} and {right_name} roots overlap" + ) + for name, ref in artifacts.items(): + if Path(ref.path).is_relative_to(Path(scratch)): + raise ValueError( + f"execution artifact {name} is inside the scratch root" + ) + object.__setattr__(self, "output_root", output) + object.__setattr__(self, "checkpoint_root", checkpoint) + object.__setattr__(self, "scratch_root", scratch) + planned = _positive_int( + self.planned_window_count, "planned_window_count" + ) + executable = _nonnegative_int( + self.executable_window_count, "executable_window_count" + ) + if executable > planned: + raise ValueError("executable windows exceed planned windows") + object.__setattr__(self, "planned_window_count", planned) + object.__setattr__(self, "executable_window_count", executable) + refusals = tuple( + sorted({_required_text(value) for value in self.refusal_ids}) + ) + if len(refusals) != planned - executable: + raise ValueError("execution refusal IDs do not reconcile") + object.__setattr__(self, "refusal_ids", refusals) + expected = _stable_id( + "reconstruction-plan-execution", self.identity_payload() + ) + if self.manifest_id and self.manifest_id != expected: + raise ValueError("execution manifest_id differs") + object.__setattr__(self, "manifest_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "configuration_id": self.configuration_id, + "source_inventory_id": self.source_inventory_id, + "information_manifest_id": self.information_manifest_id, + "information_audit_id": self.information_audit_id, + "ensemble_plan_id": self.ensemble_plan_id, + "retention_plan_id": self.retention_plan_id, + "delivery_mode": self.delivery_mode.value, + "artifacts": { + name: ref.to_dict() for name, ref in self.artifacts.items() + }, + "output_root": self.output_root, + "checkpoint_root": self.checkpoint_root, + "scratch_root": self.scratch_root, + "planned_window_count": self.planned_window_count, + "executable_window_count": self.executable_window_count, + "refusal_ids": list(self.refusal_ids), + "large_rows_in_artifacts_only": True, + } + + def to_dict(self) -> dict[str, JSONValue]: + return {**self.identity_payload(), "manifest_id": self.manifest_id} + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "ReconstructionPlanExecutionManifestV1": + _require_schema( + data, RECONSTRUCTION_PLAN_EXECUTION_MANIFEST_SCHEMA_VERSION + ) + _require_derived(data, "large_rows_in_artifacts_only", True) + return cls( + run_id=str(data.get("run_id", "")), + configuration_id=str(data.get("configuration_id", "")), + source_inventory_id=str(data.get("source_inventory_id", "")), + information_manifest_id=str( + data.get("information_manifest_id", "") + ), + information_audit_id=str(data.get("information_audit_id", "")), + ensemble_plan_id=str(data.get("ensemble_plan_id", "")), + retention_plan_id=str(data.get("retention_plan_id", "")), + delivery_mode=ReconstructionDeliveryMode.from_value( + str(data.get("delivery_mode", "")) + ), + artifacts={ + str(name): ArtifactRef.from_dict(_mapping(value)) + for name, value in _mapping(data.get("artifacts")).items() + }, + output_root=str(data.get("output_root", "")), + checkpoint_root=str(data.get("checkpoint_root", "")), + scratch_root=str(data.get("scratch_root", "")), + planned_window_count=_strict_int( + data.get("planned_window_count"), "planned_window_count" + ), + executable_window_count=_strict_int( + data.get("executable_window_count"), "executable_window_count" + ), + refusal_ids=_string_tuple(data.get("refusal_ids")), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionPlanExecutionManifestV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionStagePlanV1: + """Public, fully resolved planning context consumed by one stage handler.""" + + command: ReconstructionStageCommandV1 + execution_manifest_ref: ArtifactRef + execution_manifest: ReconstructionPlanExecutionManifestV1 + configuration: ReconstructionPlanConfigurationV1 + source_inventory: ReconstructionSourceInventoryV1 + + def __post_init__(self) -> None: + if not isinstance(self.command, ReconstructionStageCommandV1): + raise TypeError( + "stage plan requires a reconstruction stage command" + ) + execution_ref = _strong_ref(self.execution_manifest_ref) + if execution_ref.kind != PLAN_EXECUTION_MANIFEST_ARTIFACT_KIND: + raise ReconstructionPlanCompatibilityError( + "stage command configuration is not an execution manifest" + ) + object.__setattr__(self, "execution_manifest_ref", execution_ref) + if self.command.configuration_refs != (execution_ref,): + raise ReconstructionPlanCompatibilityError( + "stage command lacks the exact execution manifest" + ) + if ( + self.command.handler_name + != FIRST_PARTY_RECONSTRUCTION_HANDLERS[self.command.stage] + ): + raise ReconstructionPlanCompatibilityError( + "stage command does not use the first-party handler contract" + ) + if ( + self.execution_manifest.configuration_id + != self.configuration.configuration_id + ): + raise ReconstructionPlanCompatibilityError( + "execution manifest configuration identity differs" + ) + if ( + self.execution_manifest.source_inventory_id + != self.source_inventory.inventory_id + ): + raise ReconstructionPlanCompatibilityError( + "execution manifest source inventory identity differs" + ) + if ( + self.execution_manifest.delivery_mode + is not self.configuration.delivery_mode + ): + raise ReconstructionPlanCompatibilityError( + "execution delivery mode differs from configuration" + ) + _validate_stage_inputs( + self.command, self.execution_manifest.delivery_mode + ) + graph_refs = { + canonical_contract_json(ref.to_dict()) + for ref in self.execution_manifest.artifacts.values() + } + source_refs = { + canonical_contract_json(item.artifact.to_dict()) + for item in self.source_inventory.partitions + } + for ref in self.command.input_manifest_refs: + ref_key = canonical_contract_json(ref.to_dict()) + if ref.kind == ASCII_TICK_SOURCE_KIND: + if ref_key not in source_refs: + raise ReconstructionPlanCompatibilityError( + "stage source reference is absent from inventory" + ) + elif ref_key not in graph_refs: + raise ReconstructionPlanCompatibilityError( + "stage input reference is absent from execution artifact graph" + ) + + +@dataclass(frozen=True, slots=True) +class SyntheticInfillPlanV1: + """Executable, bounded, first-party ASCII tick reconstruction plan.""" + + run: ReconstructionRunV1 + configuration_id: str + execution_manifest_id: str + information_mode: InformationMode + delivery_mode: ReconstructionDeliveryMode + requested_start_ns: int + requested_end_ns: int + workflow_requests: tuple[ReconstructionWorkflowRequestV1, ...] + artifact_graph: Mapping[str, ArtifactRef] + resources: ReconstructionPlanResourceSummaryV1 + refusals: tuple[ReconstructionPlanRefusalV1, ...] = () + plan_id: str = "" + schema_version: str = SYNTHETIC_INFILL_PLAN_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != SYNTHETIC_INFILL_PLAN_SCHEMA_VERSION: + raise ValueError("unsupported synthetic infill plan schema") + if not isinstance(self.run, ReconstructionRunV1): + raise ValueError( + "synthetic infill plan requires a reconstruction run" + ) + object.__setattr__( + self, "configuration_id", _required_text(self.configuration_id) + ) + object.__setattr__( + self, + "execution_manifest_id", + _required_text(self.execution_manifest_id), + ) + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + object.__setattr__( + self, + "delivery_mode", + ReconstructionDeliveryMode.from_value(self.delivery_mode), + ) + start = _int64(self.requested_start_ns, "requested_start_ns") + end = _int64(self.requested_end_ns, "requested_end_ns") + if end <= start: + raise ValueError( + "synthetic infill plan requested interval is empty" + ) + object.__setattr__(self, "requested_start_ns", start) + object.__setattr__(self, "requested_end_ns", end) + requests = tuple( + sorted(self.workflow_requests, key=lambda item: item.request_id) + ) + if len(requests) > MAX_RECONSTRUCTION_PLAN_REQUESTS: + raise ValueError( + "synthetic infill workflow request count is outside limits" + ) + if len({item.request_id for item in requests}) != len(requests): + raise ValueError( + "synthetic infill workflow request IDs are duplicated" + ) + if any(item.run.run_id != self.run.run_id for item in requests): + raise ValueError( + "workflow request run differs from synthetic infill plan" + ) + tasks = tuple(task for request in requests for task in request.tasks) + if len({item.task_id for item in tasks}) != len(tasks): + raise ValueError("synthetic infill workflow tasks are duplicated") + object.__setattr__(self, "workflow_requests", requests) + graph = _artifact_mapping(self.artifact_graph) + if "execution_manifest" not in graph or "configuration" not in graph: + raise ValueError( + "synthetic infill artifact graph lacks execution contracts" + ) + object.__setattr__(self, "artifact_graph", graph) + if not isinstance(self.resources, ReconstructionPlanResourceSummaryV1): + raise ValueError( + "synthetic infill plan requires a resource summary" + ) + if self.resources.workflow_request_count != len(requests): + raise ValueError("resource workflow request count differs") + if ( + self.resources.executable_window_count + * self.resources.ensemble_member_count + != len(tasks) + ): + raise ValueError( + "resource executable window count differs from tasks" + ) + refusals = tuple( + sorted( + self.refusals, key=lambda item: (item.start_ns, item.refusal_id) + ) + ) + if len(refusals) > MAX_RECONSTRUCTION_PLAN_REFUSALS: + raise ValueError("synthetic infill refusal count exceeds limit") + if len(refusals) != self.resources.refused_window_count: + raise ValueError("resource refusal count differs") + object.__setattr__(self, "refusals", refusals) + expected = _stable_id("synthetic-infill-plan", self.identity_payload()) + if self.plan_id and self.plan_id != expected: + raise ValueError("synthetic infill plan_id differs") + object.__setattr__(self, "plan_id", expected) + if ( + len(self.to_json().encode("utf-8")) + > MAX_SYNTHETIC_INFILL_PLAN_BYTES + ): + raise ValueError( + "synthetic infill plan exceeds bounded artifact size" + ) + + @property + def status(self) -> str: + return "ready_with_refusals" if self.refusals else "ready" + + def identity_payload(self) -> dict[str, JSONValue]: + return { + "schema_version": self.schema_version, + "run": self.run.to_dict(), + "configuration_id": self.configuration_id, + "execution_manifest_id": self.execution_manifest_id, + "information_mode": self.information_mode.value, + "delivery_mode": self.delivery_mode.value, + "requested_start_ns": self.requested_start_ns, + "requested_end_ns": self.requested_end_ns, + "workflow_requests": [ + item.to_dict() for item in self.workflow_requests + ], + "artifact_graph": { + name: ref.to_dict() for name, ref in self.artifact_graph.items() + }, + "resources": self.resources.to_dict(), + "refusals": [item.to_dict() for item in self.refusals], + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + "immutable_anchor_policy": IMMUTABLE_ANCHOR_POLICY, + "input_policy": TICK_ONLY_INPUT_POLICY, + } + + def to_dict(self) -> dict[str, JSONValue]: + return { + **self.identity_payload(), + "plan_id": self.plan_id, + "status": self.status, + } + + def to_json(self) -> str: + return str(canonical_contract_json(self.to_dict())) + + def dry_run_payload(self) -> dict[str, JSONValue]: + """Return a bounded graph summary without expanding task payloads.""" + batches: list[JSONValue] = [] + for request in self.workflow_requests: + tasks = request.tasks + batches.append( + { + "request_id": request.request_id, + "request_fingerprint": request.request_fingerprint, + "ensemble_member_id": tasks[0].window.ensemble_member_id, + "window_count": len(tasks), + "core_start_ns": tasks[0].window.core_start_ns, + "core_end_ns": tasks[-1].window.core_end_ns, + "task_ids_sha256": hashlib.sha256( + "\n".join(item.task_id for item in tasks).encode( + "utf-8" + ) + ).hexdigest(), + } + ) + return { + "schema_version": self.schema_version, + "plan_id": self.plan_id, + "status": self.status, + "run_id": self.run.run_id, + "symbols": list(self.run.symbols), + "information_mode": self.information_mode.value, + "delivery_mode": self.delivery_mode.value, + "requested_start_ns": self.requested_start_ns, + "requested_end_ns": self.requested_end_ns, + "workflow_batches": batches, + "artifact_graph": { + name: { + "kind": ref.kind, + "sha256": ref.sha256, + "size_bytes": ref.size_bytes, + } + for name, ref in self.artifact_graph.items() + }, + "resources": self.resources.to_dict(), + "refusals": [item.to_dict() for item in self.refusals], + "scientific_nonclaim": SCIENTIFIC_NONCLAIM, + } + + def to_dry_run_json(self) -> str: + return str(canonical_contract_json(self.dry_run_payload())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "SyntheticInfillPlanV1": + _require_schema(data, SYNTHETIC_INFILL_PLAN_SCHEMA_VERSION) + _require_derived(data, "scientific_nonclaim", SCIENTIFIC_NONCLAIM) + _require_derived( + data, "immutable_anchor_policy", IMMUTABLE_ANCHOR_POLICY + ) + _require_derived(data, "input_policy", TICK_ONLY_INPUT_POLICY) + return cls( + run=ReconstructionRunV1.from_dict(_mapping(data.get("run"))), + configuration_id=str(data.get("configuration_id", "")), + execution_manifest_id=str(data.get("execution_manifest_id", "")), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + delivery_mode=ReconstructionDeliveryMode.from_value( + str(data.get("delivery_mode", "")) + ), + requested_start_ns=_strict_int( + data.get("requested_start_ns"), "requested_start_ns" + ), + requested_end_ns=_strict_int( + data.get("requested_end_ns"), "requested_end_ns" + ), + workflow_requests=tuple( + ReconstructionWorkflowRequestV1.from_dict(_mapping(value)) + for value in _sequence(data.get("workflow_requests")) + ), + artifact_graph={ + str(name): ArtifactRef.from_dict(_mapping(value)) + for name, value in _mapping(data.get("artifact_graph")).items() + }, + resources=ReconstructionPlanResourceSummaryV1.from_dict( + _mapping(data.get("resources")) + ), + refusals=tuple( + ReconstructionPlanRefusalV1.from_dict(_mapping(value)) + for value in _sequence(data.get("refusals")) + ), + plan_id=str(data.get("plan_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "SyntheticInfillPlanV1": + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class _ResolvedPlanInputs: + feed_epoch_definition: Any + observation_operator: ObservationOperatorV1 + market_context: MarketContextCorpusV1 + cftc_positioning: CftcPositioningCorpusV1 + benchmark_corpus: Any + motif_profile: ModernReferenceMotifProfileV1 + motif_index: Any + artifacts: Mapping[str, ArtifactRef] + motif_manifest: Mapping[str, Any] + motif_qualification: Mapping[str, Any] + motif_leakage_audit: Mapping[str, Any] + + +def build_synthetic_infill_plan( + source_root: str | Path, + *, + feed_epoch_definition_path: str | Path, + observation_operator_path: str | Path, + market_context_corpus_path: str | Path, + cftc_positioning_corpus_path: str | Path, + benchmark_manifest_path: str | Path, + motif_manifest_path: str | Path, + motif_index_path: str | Path, + motif_qualification_path: str | Path, + motif_leakage_audit_path: str | Path, + artifact_root: str | Path, + output_root: str | Path, + checkpoint_root: str | Path, + scratch_root: str | Path, + symbols: Iterable[str] = EURUSD_TRIANGLE_SYMBOLS, + start_period: str | None = None, + end_period: str | None = None, + requested_start_ns: int | None = None, + requested_end_ns: int | None = None, + information_mode: InformationMode = InformationMode.EX_POST_RECONSTRUCTION, + delivery_mode: ReconstructionDeliveryMode = ( + ReconstructionDeliveryMode.MODERN_REFERENCE + ), + broker_delivery_artifact: ArtifactRef | None = None, + base_seed: int = DEFAULT_RECONSTRUCTION_BASE_SEED, + window_size_ns: int = DEFAULT_RECONSTRUCTION_WINDOW_SIZE_NS, + left_halo_ns: int | None = None, + right_lookahead_ns: int = 0, + max_parallel_windows: int = DEFAULT_RECONSTRUCTION_MAX_PARALLEL_WINDOWS, + max_windows_per_request: int = DEFAULT_RECONSTRUCTION_REQUEST_WINDOW_LIMIT, + storage_policy: ReconstructionStoragePolicyV1 | None = None, + ensemble_config: EnsembleCalibrationConfigV1 | None = None, + generator_config: EmpiricalMotifGeneratorConfigV1 | None = None, + carving_constraints: HistoricalCarvingConstraintSetV1 | None = None, + cross_currency_config: CrossCurrencyReconciliationConfigV1 | None = None, +) -> SyntheticInfillPlanV1: + """Resolve real artifacts and build one executable first-party plan.""" + selected_symbols = _symbols(tuple(symbols)) + if selected_symbols != tuple(EURUSD_TRIANGLE_SYMBOLS): + raise ReconstructionPlanCompatibilityError( + "v2.1 requires the complete EURUSD/GBPUSD/EURGBP tick triangle" + ) + source = Path(source_root).expanduser().resolve() + if source.name.upper() != "T" or source.parent.name.upper() != "ASCII": + raise ReconstructionPlanCompatibilityError( + "reconstruction source_root must be the ASCII/T tick directory" + ) + roots = _validated_plan_roots( + source_root=source, + artifact_root=artifact_root, + output_root=output_root, + checkpoint_root=checkpoint_root, + scratch_root=scratch_root, + ) + mode = InformationMode.from_value(information_mode) + delivery = ReconstructionDeliveryMode.from_value(delivery_mode) + if delivery is ReconstructionDeliveryMode.BROKER_CONDITIONED: + if broker_delivery_artifact is None: + raise ReconstructionPlanCompatibilityError( + "broker-conditioned delivery requires a strong broker artifact" + ) + broker_ref = _strong_ref(broker_delivery_artifact) + if broker_ref.kind != "broker_delivery_artifact_v1": + raise ReconstructionPlanCompatibilityError( + "broker-conditioned delivery requires a broker delivery artifact" + ) + verify_artifact_ref(broker_ref) + elif broker_delivery_artifact is not None: + raise ReconstructionPlanCompatibilityError( + "modern-reference delivery rejects an unused broker artifact" + ) + else: + broker_ref = None + + resolved = _resolve_plan_inputs( + feed_epoch_definition_path=feed_epoch_definition_path, + observation_operator_path=observation_operator_path, + market_context_corpus_path=market_context_corpus_path, + cftc_positioning_corpus_path=cftc_positioning_corpus_path, + benchmark_manifest_path=benchmark_manifest_path, + motif_manifest_path=motif_manifest_path, + motif_index_path=motif_index_path, + motif_qualification_path=motif_qualification_path, + motif_leakage_audit_path=motif_leakage_audit_path, + symbols=selected_symbols, + ) + all_common_periods = _common_source_periods( + resolved.feed_epoch_definition, selected_symbols + ) + if (requested_start_ns is None) != (requested_end_ns is None): + raise ReconstructionPlanCompatibilityError( + "requested_start_ns and requested_end_ns must be supplied together" + ) + if requested_start_ns is None: + first_period = _period(start_period or all_common_periods[0]) + last_period = _period(end_period or all_common_periods[-1]) + requested_start = _month_start_ns(first_period) + requested_end = _month_start_ns(_next_period(last_period)) + else: + requested_start = _int64(requested_start_ns, "requested_start_ns") + requested_end = _int64(requested_end_ns, "requested_end_ns") + if requested_end <= requested_start: + raise ReconstructionPlanCompatibilityError( + "requested nanosecond interval is empty" + ) + first_period = _period_for_ns(requested_start) + last_period = _period_for_ns(requested_end - 1) + if start_period is not None and _period(start_period) != first_period: + raise ReconstructionPlanCompatibilityError( + "start_period differs from requested_start_ns" + ) + if end_period is not None and _period(end_period) != last_period: + raise ReconstructionPlanCompatibilityError( + "end_period differs from requested_end_ns" + ) + if first_period > last_period: + raise ReconstructionPlanCompatibilityError( + "start_period follows end_period" + ) + selected_periods = tuple( + value + for value in all_common_periods + if first_period <= value <= last_period + ) + expected_periods = _period_range(first_period, last_period) + if selected_periods != expected_periods: + raise ReconstructionPlanCompatibilityError( + "requested source periods are not a complete common triangle range" + ) + _reject_ex_ante_artifact_leakage( + mode=mode, + requested_start_ns=requested_start, + definition=resolved.feed_epoch_definition, + motif_profile=resolved.motif_profile, + ) + inventory = _build_source_inventory( + source, + definition=resolved.feed_epoch_definition, + symbols=selected_symbols, + periods=selected_periods, + requested_start_ns=requested_start, + requested_end_ns=requested_end, + ) + artifacts_dir = roots["artifact"] + inventory_ref = _write_contract_artifact( + inventory, + artifacts_dir, + prefix="reconstruction-source-inventory", + kind=SOURCE_INVENTORY_ARTIFACT_KIND, + metadata={"inventory_id": inventory.inventory_id}, + ) + + selected_storage = storage_policy or ReconstructionStoragePolicyV1() + selected_ensemble = ensemble_config or EnsembleCalibrationConfigV1() + selected_generator = generator_config or EmpiricalMotifGeneratorConfigV1() + selected_cross = ( + cross_currency_config or eurusd_triangle_reconciliation_config() + ) + if tuple(selected_cross.symbols) != selected_symbols: + raise ReconstructionPlanCompatibilityError( + "cross-currency policy does not cover the complete triangle" + ) + selected_carving = carving_constraints or HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id=( + "modern-reference:" + str(resolved.motif_manifest["library_id"]) + ), + # The qualified first-party corpus intentionally declares its static + # holiday calendar advisory. Coverage preflight still fails closed + # for unsupported currencies/events; this permits that explicit, + # non-exchange-specific limitation instead of rejecting every modern + # reference candidate at the handler seam. + require_complete_calendar_profile=False, + require_fingerprint_validation=False, + ) + selected_left_halo = max( + _nonnegative_int(left_halo_ns or 0, "left_halo_ns"), + resolved.observation_operator.required_left_halo_ns, + ) + selected_right = _nonnegative_int(right_lookahead_ns, "right_lookahead_ns") + information_policy = ReconstructionInformationPolicyV1( + information_mode=mode, + max_allowed_lookahead_ns=selected_right, + ) + configuration = ReconstructionPlanConfigurationV1( + delivery_mode=delivery, + information_policy=information_policy, + generator_config=selected_generator, + carving_constraints=selected_carving, + cross_currency_config=selected_cross, + ensemble_config=selected_ensemble, + storage_policy=selected_storage, + window_size_ns=window_size_ns, + left_halo_ns=selected_left_halo, + right_lookahead_ns=selected_right, + max_parallel_windows=max_parallel_windows, + ) + configuration_ref = _write_contract_artifact( + configuration, + artifacts_dir, + prefix="reconstruction-plan-configuration", + kind=PLAN_CONFIGURATION_ARTIFACT_KIND, + metadata={"configuration_id": configuration.configuration_id}, + ) + + configuration_hashes = { + configuration.configuration_id: configuration_ref.sha256, + information_policy.policy_id: _contract_sha256(information_policy), + selected_generator.config_id: _contract_sha256(selected_generator), + selected_carving.constraint_set_id: _contract_sha256(selected_carving), + selected_cross.config_id: _contract_sha256(selected_cross), + resolved.feed_epoch_definition.definition_id: resolved.artifacts[ + "feed_epochs" + ].sha256, + resolved.observation_operator.operator_id: resolved.artifacts[ + "observation_operator" + ].sha256, + resolved.market_context.corpus_id: resolved.artifacts[ + "market_context" + ].sha256, + resolved.cftc_positioning.corpus_id: resolved.artifacts[ + "cftc_positioning" + ].sha256, + resolved.benchmark_corpus.corpus_id: resolved.artifacts[ + "benchmark_manifest" + ].sha256, + str(resolved.motif_manifest["library_id"]): resolved.artifacts[ + "motif_manifest" + ].sha256, + resolved.motif_index.index_id: resolved.artifacts["motif_index"].sha256, + } + ensemble_plan = plan_reconstruction_ensemble( + symbols=selected_symbols, + source_artifact_hashes={inventory.inventory_id: inventory_ref.sha256}, + configuration_artifact_hashes=configuration_hashes, + base_seed=base_seed, + config=selected_ensemble, + storage_policy=selected_storage, + ) + ensemble_ref = _write_contract_artifact( + ensemble_plan, + artifacts_dir, + prefix="reconstruction-ensemble-plan", + kind="reconstruction_ensemble_plan_v1", + metadata={ + "plan_id": ensemble_plan.plan_id, + "run_id": ensemble_plan.run.run_id, + }, + ) + + coverages = _source_coverages(inventory) + member_window_plans = tuple( + plan_cross_currency_windows( + ensemble_plan.run, + ensemble_member_id=member.member_id, + requested_start_ns=requested_start, + requested_end_ns=requested_end, + window_size_ns=configuration.window_size_ns, + coverages=coverages, + left_halo_ns=configuration.left_halo_ns, + right_lookahead_ns=configuration.right_lookahead_ns, + ) + for member in ensemble_plan.members + ) + if any( + item.status is not CrossCurrencyWindowPlanStatus.PLANNED + for item in member_window_plans + ): + raise ReconstructionPlanCompatibilityError( + "complete synchronized cross-currency coverage could not be planned" + ) + planned_windows = tuple( + window for plan in member_window_plans for window in plan.windows + ) + boundary_windows = member_window_plans[0].windows + refusals, executable_boundaries = _preflight_window_support( + boundary_windows, + definition=resolved.feed_epoch_definition, + context=resolved.market_context, + positioning=resolved.cftc_positioning, + mode=mode, + ) + executable_keys = { + (item.core_start_ns, item.core_end_ns) for item in executable_boundaries + } + + information_manifest, information_audit = _build_information_evidence( + run=ensemble_plan.run, + policy=information_policy, + windows=planned_windows, + artifacts={ + **resolved.artifacts, + "source_inventory": inventory_ref, + "configuration": configuration_ref, + }, + motif_profile=resolved.motif_profile, + requested_start_ns=requested_start, + requested_end_ns=requested_end, + ) + information_manifest_ref = _write_contract_artifact( + information_manifest, + artifacts_dir, + prefix="reconstruction-information-manifest", + kind="reconstruction_information_manifest_v1", + metadata={"manifest_id": information_manifest.manifest_id}, + ) + information_audit_ref = _write_contract_artifact( + information_audit, + artifacts_dir, + prefix="reconstruction-information-audit", + kind="reconstruction_information_audit_v1", + metadata={"audit_id": information_audit.audit_id}, + ) + + estimates_by_boundary = { + (window.core_start_ns, window.core_end_ns): _window_resource_estimate( + window, + inventory=inventory, + configuration=configuration, + ) + for window in executable_boundaries + } + candidate_events_per_member = sum( + item.candidate_event_count for item in estimates_by_boundary.values() + ) + input_events_per_member = sum( + item.input_event_count for item in estimates_by_boundary.values() + ) + retained_ids = tuple( + item.member_id + for item in ensemble_plan.members[ + : selected_ensemble.retained_member_count + ] + ) + estimated_partition_count = ( + len(executable_boundaries) * len(retained_ids) * len(selected_symbols) + if candidate_events_per_member + else 0 + ) + retention_plan = estimate_reconstruction_retention( + run_id=ensemble_plan.run.run_id, + primary_member_id=retained_ids[0], + retained_member_event_counts={ + member_id: candidate_events_per_member for member_id in retained_ids + }, + estimated_partition_count=estimated_partition_count, + storage_policy=selected_storage, + estimated_bytes_per_event=selected_ensemble.estimated_bytes_per_event, + ) + retention_ref = _write_contract_artifact( + retention_plan, + artifacts_dir, + prefix="reconstruction-retention-plan", + kind="reconstruction_retention_plan_v1", + metadata={"plan_id": retention_plan.plan_id}, + ) + + graph: dict[str, ArtifactRef] = { + **resolved.artifacts, + "source_inventory": inventory_ref, + "configuration": configuration_ref, + "information_manifest": information_manifest_ref, + "information_audit": information_audit_ref, + "ensemble_plan": ensemble_ref, + "retention_plan": retention_ref, + } + if broker_ref is not None: + graph["broker_delivery"] = broker_ref + execution_manifest = ReconstructionPlanExecutionManifestV1( + run_id=ensemble_plan.run.run_id, + configuration_id=configuration.configuration_id, + source_inventory_id=inventory.inventory_id, + information_manifest_id=information_manifest.manifest_id, + information_audit_id=information_audit.audit_id, + ensemble_plan_id=ensemble_plan.plan_id, + retention_plan_id=retention_plan.plan_id, + delivery_mode=delivery, + artifacts=graph, + output_root=str(roots["output"]), + checkpoint_root=str(roots["checkpoint"]), + scratch_root=str(roots["scratch"]), + planned_window_count=len(boundary_windows), + executable_window_count=len(executable_boundaries), + refusal_ids=tuple(item.refusal_id for item in refusals), + ) + execution_ref = _write_contract_artifact( + execution_manifest, + artifacts_dir, + prefix="reconstruction-plan-execution", + kind=PLAN_EXECUTION_MANIFEST_ARTIFACT_KIND, + metadata={"manifest_id": execution_manifest.manifest_id}, + ) + graph["execution_manifest"] = execution_ref + + workflows = _build_workflow_requests( + run=ensemble_plan.run, + member_window_plans=member_window_plans, + executable_keys=executable_keys, + estimates_by_boundary=estimates_by_boundary, + inventory=inventory, + execution_manifest=execution_manifest, + execution_ref=execution_ref, + max_windows_per_request=max_windows_per_request, + ) + maximum_estimate = max( + estimates_by_boundary.values(), + key=lambda item: item.estimated_memory_bytes, + default=None, + ) + total_candidate_events = candidate_events_per_member * len( + ensemble_plan.members + ) + resources = ReconstructionPlanResourceSummaryV1( + source_event_count=inventory.total_row_count, + source_size_bytes=inventory.total_size_bytes, + planned_window_count=len(boundary_windows), + executable_window_count=len(executable_boundaries), + refused_window_count=len(refusals), + ensemble_member_count=len(ensemble_plan.members), + retained_member_count=len(retained_ids), + workflow_request_count=len(workflows), + estimated_input_event_count=( + input_events_per_member * len(ensemble_plan.members) + ), + estimated_candidate_event_count=total_candidate_events, + estimated_candidate_bytes=( + total_candidate_events * selected_ensemble.estimated_bytes_per_event + ), + estimated_peak_memory_bytes=( + (maximum_estimate.estimated_memory_bytes if maximum_estimate else 0) + * configuration.max_parallel_windows + ), + estimated_peak_scratch_bytes=( + ( + maximum_estimate.estimated_scratch_bytes + if maximum_estimate + else 0 + ) + * configuration.max_parallel_windows + ), + estimated_output_bytes=retention_plan.estimated_total_output_bytes, + estimated_partition_count=estimated_partition_count, + ) + plan = SyntheticInfillPlanV1( + run=ensemble_plan.run, + configuration_id=configuration.configuration_id, + execution_manifest_id=execution_manifest.manifest_id, + information_mode=mode, + delivery_mode=delivery, + requested_start_ns=requested_start, + requested_end_ns=requested_end, + workflow_requests=workflows, + artifact_graph=graph, + resources=resources, + refusals=refusals, + ) + validate_synthetic_infill_plan_for_execution(plan, verify_artifacts=False) + return plan + + +def validate_synthetic_infill_plan_for_execution( + plan: SyntheticInfillPlanV1, + *, + verify_artifacts: bool = True, + verify_source_partitions: bool = True, +) -> SyntheticInfillPlanV1: + """Validate the exact public plan shape consumed by first-party handlers.""" + if not isinstance(plan, SyntheticInfillPlanV1): + raise TypeError("execution validation requires SyntheticInfillPlanV1") + execution_ref = plan.artifact_graph["execution_manifest"] + configuration_ref = plan.artifact_graph["configuration"] + inventory_ref = plan.artifact_graph["source_inventory"] + verified_refs: set[tuple[str, str, int | None, str]] = set() + + def verify_once(ref: ArtifactRef) -> None: + key = (ref.kind, ref.path, ref.size_bytes, ref.sha256) + if key not in verified_refs: + verify_artifact_ref(ref) + verified_refs.add(key) + + if verify_artifacts: + for ref in plan.artifact_graph.values(): + verify_once(ref) + execution = read_reconstruction_plan_execution_manifest(execution_ref.path) + configuration = read_reconstruction_plan_configuration( + configuration_ref.path + ) + inventory = read_reconstruction_source_inventory(inventory_ref.path) + if execution.manifest_id != plan.execution_manifest_id: + raise ReconstructionPlanCompatibilityError( + "plan execution manifest identity differs" + ) + if configuration.configuration_id != plan.configuration_id: + raise ReconstructionPlanCompatibilityError( + "plan configuration identity differs" + ) + if inventory.inventory_id != plan.run.source_version_ids[0]: + raise ReconstructionPlanCompatibilityError( + "plan source inventory is not run-bound" + ) + if execution.artifacts != { + name: ref + for name, ref in plan.artifact_graph.items() + if name != "execution_manifest" + }: + raise ReconstructionPlanCompatibilityError( + "plan artifact graph differs from execution manifest" + ) + for request in plan.workflow_requests: + for task in request.tasks: + for command in task.commands: + ReconstructionStagePlanV1( + command=command, + execution_manifest_ref=execution_ref, + execution_manifest=execution, + configuration=configuration, + source_inventory=inventory, + ) + if verify_artifacts: + for ref in command.input_manifest_refs: + if ( + ref.kind == ASCII_TICK_SOURCE_KIND + and not verify_source_partitions + ): + continue + verify_once(ref) + return plan + + +def write_synthetic_infill_plan( + plan: SyntheticInfillPlanV1, root: str | Path +) -> ArtifactRef: + """Persist one content-addressed top-level plan artifact.""" + validate_synthetic_infill_plan_for_execution(plan, verify_artifacts=False) + return _write_contract_artifact( + plan, + Path(root).expanduser().resolve(), + prefix="synthetic-infill-plan", + kind=SYNTHETIC_INFILL_PLAN_ARTIFACT_KIND, + metadata={"plan_id": plan.plan_id, "run_id": plan.run.run_id}, + ) + + +def read_synthetic_infill_plan(path: str | Path) -> SyntheticInfillPlanV1: + """Hash-verify and restore a content-addressed top-level plan.""" + payload = _read_content_addressed_json(path, "synthetic-infill-plan") + return SyntheticInfillPlanV1.from_dict(payload) + + +def read_reconstruction_source_inventory( + path: str | Path, +) -> ReconstructionSourceInventoryV1: + payload = _read_content_addressed_json( + path, "reconstruction-source-inventory" + ) + return ReconstructionSourceInventoryV1.from_dict(payload) + + +def read_reconstruction_plan_configuration( + path: str | Path, +) -> ReconstructionPlanConfigurationV1: + payload = _read_content_addressed_json( + path, "reconstruction-plan-configuration" + ) + return ReconstructionPlanConfigurationV1.from_dict(payload) + + +def read_reconstruction_plan_execution_manifest( + path: str | Path, +) -> ReconstructionPlanExecutionManifestV1: + payload = _read_content_addressed_json( + path, "reconstruction-plan-execution" + ) + return ReconstructionPlanExecutionManifestV1.from_dict(payload) + + +def load_reconstruction_stage_plan( + command: ReconstructionStageCommandV1, + *, + verify_artifacts: bool = True, +) -> ReconstructionStagePlanV1: + """Resolve and validate the public artifact context for one stage command.""" + if not isinstance(command, ReconstructionStageCommandV1): + raise TypeError("stage planning requires ReconstructionStageCommandV1") + if len(command.configuration_refs) != 1: + raise ReconstructionPlanCompatibilityError( + "stage command requires exactly one execution manifest" + ) + execution_ref = _strong_ref(command.configuration_refs[0]) + if verify_artifacts: + verify_artifact_ref(execution_ref) + execution = read_reconstruction_plan_execution_manifest(execution_ref.path) + configuration_ref = execution.artifacts["configuration"] + inventory_ref = execution.artifacts["source_inventory"] + if verify_artifacts: + for ref in execution.artifacts.values(): + verify_artifact_ref(ref) + for ref in command.input_manifest_refs: + verify_artifact_ref(ref) + return ReconstructionStagePlanV1( + command=command, + execution_manifest_ref=execution_ref, + execution_manifest=execution, + configuration=read_reconstruction_plan_configuration( + configuration_ref.path + ), + source_inventory=read_reconstruction_source_inventory( + inventory_ref.path + ), + ) + + +def _resolve_plan_inputs( + *, + feed_epoch_definition_path: str | Path, + observation_operator_path: str | Path, + market_context_corpus_path: str | Path, + cftc_positioning_corpus_path: str | Path, + benchmark_manifest_path: str | Path, + motif_manifest_path: str | Path, + motif_index_path: str | Path, + motif_qualification_path: str | Path, + motif_leakage_audit_path: str | Path, + symbols: tuple[str, ...], +) -> _ResolvedPlanInputs: + """Resolve qualified inputs once per unchanged stat-identity set.""" + identities = tuple( + _file_stat_identity(path) + for path in ( + feed_epoch_definition_path, + observation_operator_path, + market_context_corpus_path, + cftc_positioning_corpus_path, + benchmark_manifest_path, + motif_manifest_path, + motif_index_path, + motif_qualification_path, + motif_leakage_audit_path, + ) + ) + return _resolve_plan_inputs_for_identity(identities, symbols) + + +@lru_cache(maxsize=4) +def _resolve_plan_inputs_for_identity( + identities: tuple[tuple[str, int, int, int, int, int], ...], + symbols: tuple[str, ...], +) -> _ResolvedPlanInputs: + paths = tuple(item[0] for item in identities) + return _resolve_plan_inputs_uncached( + feed_epoch_definition_path=paths[0], + observation_operator_path=paths[1], + market_context_corpus_path=paths[2], + cftc_positioning_corpus_path=paths[3], + benchmark_manifest_path=paths[4], + motif_manifest_path=paths[5], + motif_index_path=paths[6], + motif_qualification_path=paths[7], + motif_leakage_audit_path=paths[8], + symbols=symbols, + ) + + +def _resolve_plan_inputs_uncached( + *, + feed_epoch_definition_path: str | Path, + observation_operator_path: str | Path, + market_context_corpus_path: str | Path, + cftc_positioning_corpus_path: str | Path, + benchmark_manifest_path: str | Path, + motif_manifest_path: str | Path, + motif_index_path: str | Path, + motif_qualification_path: str | Path, + motif_leakage_audit_path: str | Path, + symbols: tuple[str, ...], +) -> _ResolvedPlanInputs: + feed_ref = artifact_ref_for_file( + feed_epoch_definition_path, kind="feed_epoch_definition_v2" + ) + definition = read_active_time_feed_epoch_definition(feed_ref.path) + if not definition.valid_for_observation_models: + raise ReconstructionPlanCompatibilityError( + "feed epoch definition is unstable" + ) + if _symbols(definition.symbols) != symbols: + raise ReconstructionPlanCompatibilityError( + "feed epoch definition symbols differ from requested triangle" + ) + + observation_path = Path(observation_operator_path).expanduser().resolve() + observation_payload = _json_mapping( + observation_path.read_text(encoding="utf-8") + ) + observation_ref = artifact_ref_for_file( + observation_path, + kind="observation-operator", + metadata={ + "schema_version": str( + observation_payload.get("schema_version", "") + ), + "operator_id": str(observation_payload.get("operator_id", "")), + "feed_epoch_definition_id": str( + observation_payload.get("feed_epoch_definition_id", "") + ), + }, + ) + observation = read_observation_operator_artifact(observation_ref) + if not observation.valid_for_application: + raise ReconstructionPlanCompatibilityError( + "observation operator is unqualified" + ) + if observation.feed_epoch_definition_id != definition.definition_id: + raise ReconstructionPlanCompatibilityError( + "observation operator feed epoch definition differs" + ) + + context_ref = artifact_ref_for_file( + market_context_corpus_path, kind="market_context_corpus_v1" + ) + context = read_market_context_corpus(context_ref.path) + positioning_ref = artifact_ref_for_file( + cftc_positioning_corpus_path, kind="cftc_positioning_corpus_v1" + ) + positioning = read_cftc_positioning_corpus(positioning_ref.path) + benchmark_ref = artifact_ref_for_file( + benchmark_manifest_path, kind="reverse_degradation_manifest_v1" + ) + benchmark = read_reverse_degradation_benchmark_corpus(benchmark_ref.path) + gate_policy = load_default_benchmark_promotion_gate_policy() + compatibility = { + "feed_epoch_definition_id": definition.definition_id, + "observation_operator_id": observation.operator_id, + "market_context_corpus_id": context.corpus_id, + "cftc_positioning_corpus_id": positioning.corpus_id, + "gate_policy_id": gate_policy.policy_id, + "gate_policy_commit": PREDECLARED_GATE_COMMIT, + } + for name, expected in compatibility.items(): + if getattr(benchmark, name) != expected: + raise ReconstructionPlanCompatibilityError( + f"benchmark {name} differs from the resolved qualified artifact" + ) + + motif_manifest_ref = artifact_ref_for_file( + motif_manifest_path, kind="modern_reference_motif_manifest_v1" + ) + motif_manifest = read_modern_reference_motif_artifact( + motif_manifest_ref.path, kind="manifest" + ) + motif_index_ref = artifact_ref_for_file( + motif_index_path, kind="modern_reference_motif_index_v1" + ) + motif_index = read_modern_reference_motif_index(motif_index_ref.path) + profile = ModernReferenceMotifProfileV1.from_dict( + _mapping(motif_manifest.get("profile")) + ) + if _symbols(profile.symbols) != symbols: + raise ReconstructionPlanCompatibilityError( + "motif profile symbols differ from requested triangle" + ) + index_artifact = ArtifactRef.from_dict( + _mapping(motif_manifest.get("index_artifact")) + ) + if ( + index_artifact.sha256 != motif_index_ref.sha256 + or index_artifact.size_bytes != motif_index_ref.size_bytes + or index_artifact.metadata.get("index_id") != motif_index.index_id + ): + raise ReconstructionPlanCompatibilityError( + "motif index binding differs" + ) + stable_epoch = _mapping(motif_manifest.get("stable_feed_epoch")) + if not any( + epoch.epoch_id == stable_epoch.get("epoch_id") + and epoch.label == stable_epoch.get("label") + for epoch in definition.epochs + ): + raise ReconstructionPlanCompatibilityError( + "motif stable epoch differs from feed epoch definition" + ) + dependencies = _mapping(motif_manifest.get("dependencies")) + expected_dependencies = { + "benchmark_manifest": benchmark_ref, + "feed_epoch_definition": feed_ref, + "market_context_corpus": context_ref, + "cftc_positioning_corpus": positioning_ref, + } + for name, expected_ref in expected_dependencies.items(): + resolved_ref = ArtifactRef.from_dict(_mapping(dependencies.get(name))) + if ( + resolved_ref.sha256 != expected_ref.sha256 + or resolved_ref.size_bytes != expected_ref.size_bytes + ): + raise ReconstructionPlanCompatibilityError( + f"motif dependency {name} differs from resolved artifact" + ) + + qualification_ref = artifact_ref_for_file( + motif_qualification_path, kind="modern_reference_motif_qualification_v1" + ) + qualification = read_modern_reference_motif_artifact( + qualification_ref.path, kind="qualification" + ) + campaign = _mapping(qualification.get("campaign")) + campaign_gate = _mapping(campaign.get("campaign_gate_decision")) + if ( + qualification.get("library_id") != motif_manifest.get("library_id") + or qualification.get("candidate_promotion_eligible") is not True + or qualification.get("candidate_provisional") is not False + or qualification.get("frozen_gate_policy_commit") + != PREDECLARED_GATE_COMMIT + or campaign.get("corpus_id") != benchmark.corpus_id + or campaign.get("source_replay_verified") is not True + or campaign_gate.get("promotion_eligible") is not True + ): + raise ReconstructionPlanCompatibilityError( + "modern reference motif qualification is missing or stale" + ) + leakage_ref = artifact_ref_for_file( + motif_leakage_audit_path, + kind="modern_reference_motif_leakage_audit_v1", + ) + leakage = read_modern_reference_motif_artifact( + leakage_ref.path, kind="leakage-audit" + ) + if ( + leakage.get("library_id") != motif_manifest.get("library_id") + or tuple(leakage.get("indexed_splits", ())) != ("train",) + or leakage.get("post_exclusion_cross_split_finding_count") != 0 + or leakage.get("retained_holdout_fragment_count") != 0 + or leakage.get("retained_nontrain_fragment_count") != 0 + ): + raise ReconstructionPlanCompatibilityError( + "modern reference motif leakage audit failed" + ) + return _ResolvedPlanInputs( + feed_epoch_definition=definition, + observation_operator=observation, + market_context=context, + cftc_positioning=positioning, + benchmark_corpus=benchmark, + motif_profile=profile, + motif_index=motif_index, + artifacts={ + "feed_epochs": feed_ref, + "observation_operator": observation_ref, + "market_context": context_ref, + "cftc_positioning": positioning_ref, + "benchmark_manifest": benchmark_ref, + "motif_manifest": motif_manifest_ref, + "motif_index": motif_index_ref, + "motif_qualification": qualification_ref, + "motif_leakage_audit": leakage_ref, + }, + motif_manifest=motif_manifest, + motif_qualification=qualification, + motif_leakage_audit=leakage, + ) + + +def _file_stat_identity( + path: str | Path, +) -> tuple[str, int, int, int, int, int]: + target = Path(path).expanduser().resolve() + stat = target.stat() + return ( + str(target), + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + ) + + +def _build_source_inventory( + source_root: Path, + *, + definition: Any, + symbols: tuple[str, ...], + periods: tuple[str, ...], + requested_start_ns: int, + requested_end_ns: int, +) -> ReconstructionSourceInventoryV1: + lineage = { + ( + _period(str(item.get("period", ""))), + _symbol(str(item.get("symbol", ""))), + ): item + for item in _sequence(_mapping(definition.lineage).get("sources")) + } + partitions: list[ReconstructionSourcePartitionV1] = [] + for period in periods: + for symbol in symbols: + evidence = lineage.get((period, symbol)) + if evidence is None: + raise ReconstructionPlanCompatibilityError( + f"feed epoch lineage omits {symbol} {period}" + ) + path = source_root / _relative_tick_path(symbol, period) + expected_hash = str( + evidence.get("source_artifact_sha256", "") + ).removeprefix("sha256:") + if not _SHA256_RE.fullmatch(expected_hash): + raise ReconstructionPlanCompatibilityError( + f"feed epoch lineage hash is invalid for {symbol} {period}" + ) + actual_hash = _file_sha256(path) + if actual_hash != expected_hash: + raise ReconstructionPlanCompatibilityError( + f"source hash differs for {symbol} {period}" + ) + rows, first_ms, last_ms = _inspect_tick_cache(path) + ref = ArtifactRef( + kind=ASCII_TICK_SOURCE_KIND, + path=str(path.resolve()), + size_bytes=path.stat().st_size, + sha256=actual_hash, + metadata={ + "symbol": symbol, + "period": period, + "row_count": rows, + "row_identity_basis": "zero-based-arrow-row-ordinal-v1", + }, + ) + partitions.append( + ReconstructionSourcePartitionV1( + symbol=symbol, + period=period, + artifact=ref, + row_count=rows, + coverage_start_ns=_month_start_ns(period), + coverage_end_ns=_month_start_ns(_next_period(period)), + first_timestamp_ms=first_ms, + last_timestamp_ms=last_ms, + feed_epoch_evidence_id=str(evidence.get("evidence_id", "")), + ) + ) + return ReconstructionSourceInventoryV1( + source_root=str(source_root), + symbols=symbols, + periods=periods, + partitions=tuple(partitions), + requested_start_ns=requested_start_ns, + requested_end_ns=requested_end_ns, + total_row_count=sum(item.row_count for item in partitions), + total_size_bytes=sum( + cast(int, item.artifact.size_bytes) for item in partitions + ), + ) + + +def _preflight_window_support( + windows: Sequence[ReconstructionWindowV1], + *, + definition: Any, + context: MarketContextCorpusV1, + positioning: CftcPositioningCorpusV1, + mode: InformationMode, +) -> tuple[ + tuple[ReconstructionPlanRefusalV1, ...], tuple[ReconstructionWindowV1, ...] +]: + refusals: list[ReconstructionPlanRefusalV1] = [] + executable: list[ReconstructionWindowV1] = [] + required_context = ( + ("EUR", MarketContextKind.POLICY_RATE_CHANGE), + ("GBP", MarketContextKind.POLICY_RATE_CHANGE), + ("USD", MarketContextKind.CENTRAL_BANK_DECISION), + ) + for window in windows: + midpoint_ms = ( + (window.core_start_ns + window.core_end_ns) // 2 + ) // 1_000_000 + assignments = [ + definition.assign(symbol=symbol, timestamp_utc_ms=midpoint_ms) + for symbol in window.symbols + ] + if any(item.assignment_kind == "out_of_scope" for item in assignments): + refusals.append( + ReconstructionPlanRefusalV1( + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + code=ReconstructionPlanRefusalCode.FEED_EPOCH_UNSUPPORTED, + reason="window midpoint lies outside qualified feed epoch coverage", + ) + ) + continue + context_reasons: list[str] = [] + for currency, kind in required_context: + decision = preflight_market_context_corpus( + context, + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + currencies=(currency,), + kinds=(kind,), + ) + context_reasons.extend(decision.reasons) + if context_reasons: + refusals.append( + ReconstructionPlanRefusalV1( + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + code=ReconstructionPlanRefusalCode.MARKET_CONTEXT_UNSUPPORTED, + reason="; ".join(sorted(set(context_reasons))), + ) + ) + continue + positioning_decision = preflight_cftc_positioning_corpus( + positioning, + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + information_mode=mode, + as_of_ns=( + window.core_start_ns + if mode is InformationMode.EX_ANTE_SIMULATION + else None + ), + symbols=window.symbols, + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + if not positioning_decision.ready: + refusals.append( + ReconstructionPlanRefusalV1( + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + code=ReconstructionPlanRefusalCode.CFTC_POSITIONING_UNSUPPORTED, + reason="; ".join(positioning_decision.reasons), + ) + ) + continue + executable.append(window) + return tuple(refusals), tuple(executable) + + +def _build_information_evidence( + *, + run: ReconstructionRunV1, + policy: ReconstructionInformationPolicyV1, + windows: Sequence[ReconstructionWindowV1], + artifacts: Mapping[str, ArtifactRef], + motif_profile: ModernReferenceMotifProfileV1, + requested_start_ns: int, + requested_end_ns: int, +) -> tuple[ReconstructionInformationManifestV1, InformationAuditReportV1]: + splits = _information_splits(motif_profile) + used_at = ( + max(requested_end_ns, max(split.end_ns for split in splits)) + if policy.information_mode is InformationMode.EX_POST_RECONSTRUCTION + else requested_start_ns + ) + train_end = splits[0].end_ns - 1 + inputs: list[ReconstructionInformationInputV1] = [] + declarations = ( + ( + "source_inventory", + InformationStage.SOURCE, + InformationScope.POINT_IN_TIME, + requested_start_ns, + None, + ), + ( + "feed_epochs", + InformationStage.FEATURE, + ( + InformationScope.GLOBAL_NORMALIZATION + if policy.information_mode + is InformationMode.EX_POST_RECONSTRUCTION + else InformationScope.POINT_IN_TIME + ), + min(train_end, used_at), + None, + ), + ( + "observation_operator", + InformationStage.FEATURE, + InformationScope.POINT_IN_TIME, + min(train_end, used_at), + None, + ), + ( + "market_context", + InformationStage.CALENDAR_CONTEXT, + InformationScope.POINT_IN_TIME, + min(requested_start_ns, used_at), + None, + ), + ( + "cftc_positioning", + InformationStage.FEATURE, + InformationScope.POINT_IN_TIME, + min(requested_start_ns, used_at), + None, + ), + ( + "benchmark_manifest", + InformationStage.FEATURE, + InformationScope.POINT_IN_TIME, + min(train_end, used_at), + None, + ), + ( + "motif_index", + InformationStage.MOTIF_SELECTION, + InformationScope.EMPIRICAL_MOTIF, + train_end, + InformationSplitKind.TRAIN, + ), + ) + for role, stage, scope, event_time, split_kind in declarations: + ref = artifacts[role] + inputs.append( + ReconstructionInformationInputV1( + run_id=run.run_id, + artifact_id=f"{ref.kind}:sha256:{ref.sha256}", + information_mode=policy.information_mode, + input_kind=InformationInputKind.EXTERNAL, + stage=stage, + scope=scope, + event_time_ns=event_time, + available_at_ns=event_time, + used_at_ns=used_at, + observation_start_ns=event_time, + observation_end_ns=event_time, + vintage_id=f"sha256:{ref.sha256}", + reason=f"resolved reconstruction planning artifact: {role}", + allowed_lookahead_ns=0, + split_kind=split_kind, + ) + ) + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id(windows), + inputs=tuple(inputs), + splits=splits, + ) + audit = require_reconstruction_information_audit( + manifest, + policy, + run=run, + windows=windows, + ) + return manifest, audit + + +def _window_resource_estimate( + window: ReconstructionWindowV1, + *, + inventory: ReconstructionSourceInventoryV1, + configuration: ReconstructionPlanConfigurationV1, +) -> ReconstructionResourceEstimateV1: + partitions = inventory.partitions_for_window(window) + input_count = sum( + _estimated_window_partition_rows(window, item) for item in partitions + ) + interval_count = max(0, input_count - len(inventory.symbols)) + generator_limit = ( + configuration.generator_config.max_events_per_interval * interval_count + ) + amplification_limit = math.floor( + input_count * configuration.storage_policy.max_candidate_amplification + ) + candidates = min(generator_limit, amplification_limit) + batch_limit = configuration.storage_policy.max_events_per_batch + batch_count = interval_count + inflight = min( + batch_count, configuration.storage_policy.max_inflight_batches + ) + peak = min(candidates, batch_limit) + bytes_per_event = configuration.generator_config.estimated_bytes_per_event + estimated_memory = ( + _RESOURCE_FIXED_OVERHEAD_BYTES + + (input_count * bytes_per_event) + + (peak * bytes_per_event * max(1, inflight)) + ) + ledger_bytes = batch_count * _RESOURCE_LEDGER_BYTES_PER_INTERVAL + estimate = ReconstructionResourceEstimateV1( + input_event_count=input_count, + candidate_event_count=candidates, + retained_ensemble_members=1, + inflight_batches=inflight, + peak_events_per_batch=peak, + estimated_memory_bytes=estimated_memory, + estimated_scratch_bytes=(candidates * bytes_per_event) + ledger_bytes, + estimated_output_bytes=candidates * bytes_per_event, + estimated_batch_count=batch_count, + ) + configuration.storage_policy.preflight(estimate) + return estimate + + +def _estimated_window_partition_rows( + window: ReconstructionWindowV1, + partition: ReconstructionSourcePartitionV1, +) -> int: + """Conservatively scale monthly rows to a bounded execution window.""" + first_ns = partition.first_timestamp_ms * 1_000_000 + last_exclusive_ns = (partition.last_timestamp_ms + 1) * 1_000_000 + overlap_start = max(window.input_start_ns, first_ns) + overlap_end = min(window.input_end_ns, last_exclusive_ns) + if overlap_end <= overlap_start: + return 0 + observed_span = max(1, last_exclusive_ns - first_ns) + overlap_span = overlap_end - overlap_start + proportional = math.ceil(partition.row_count * overlap_span / observed_span) + return int( + min( + partition.row_count, + max(2, proportional * _SOURCE_ROW_DENSITY_SAFETY_FACTOR), + ) + ) + + +def _build_workflow_requests( + *, + run: ReconstructionRunV1, + member_window_plans: Sequence[Any], + executable_keys: set[tuple[int, int]], + estimates_by_boundary: Mapping[ + tuple[int, int], ReconstructionResourceEstimateV1 + ], + inventory: ReconstructionSourceInventoryV1, + execution_manifest: ReconstructionPlanExecutionManifestV1, + execution_ref: ArtifactRef, + max_windows_per_request: int, +) -> tuple[ReconstructionWorkflowRequestV1, ...]: + limit = _positive_int(max_windows_per_request, "max_windows_per_request") + if limit > 512: + raise ValueError("max_windows_per_request exceeds orchestration limit") + requests: list[ReconstructionWorkflowRequestV1] = [] + for member_ordinal, window_plan in enumerate(member_window_plans, start=1): + tasks: list[ReconstructionWindowTaskV1] = [] + for window in window_plan.windows: + key = (window.core_start_ns, window.core_end_ns) + if key not in executable_keys: + continue + scratch = ( + Path(execution_manifest.scratch_root) + / run.run_id.replace(":", "-") + / window.ensemble_member_id.replace(":", "-") + / window.window_id.replace(":", "-") + ) + commands = _stage_commands( + window, + scratch=scratch, + inventory=inventory, + execution_manifest=execution_manifest, + execution_ref=execution_ref, + ) + tasks.append( + ReconstructionWindowTaskV1( + window=window, + resource_estimate=estimates_by_boundary[key], + commands=commands, + scratch_directory=str(scratch), + ) + ) + for chunk_ordinal, offset in enumerate( + range(0, len(tasks), limit), start=1 + ): + chunk = tuple(tasks[offset : offset + limit]) + request_id = ( + f"reconstruction-{run.run_id.rsplit(':', maxsplit=1)[-1][:16]}-" + f"m{member_ordinal:02d}-c{chunk_ordinal:03d}" + ) + requests.append( + ReconstructionWorkflowRequestV1( + request_id=request_id, + run=run, + tasks=chunk, + manifest_store_root=str( + Path(execution_manifest.checkpoint_root) / "manifests" + ), + report_root=str( + Path(execution_manifest.output_root) / "reports" + ), + task_queues={ + "orchestration": "histdatacom.reconstruction.orchestration", + "cpu_file": "histdatacom.reconstruction.cpu-file", + }, + max_parallel_windows=configuration_parallelism( + execution_manifest + ), + max_inflight_memory_bytes=run.storage_policy.max_memory_bytes, + ) + ) + return tuple(requests) + + +def configuration_parallelism( + execution_manifest: ReconstructionPlanExecutionManifestV1, +) -> int: + """Read the public configuration artifact for request construction.""" + configuration = read_reconstruction_plan_configuration( + execution_manifest.artifacts["configuration"].path + ) + return configuration.max_parallel_windows + + +def _stage_commands( + window: ReconstructionWindowV1, + *, + scratch: Path, + inventory: ReconstructionSourceInventoryV1, + execution_manifest: ReconstructionPlanExecutionManifestV1, + execution_ref: ArtifactRef, +) -> tuple[ReconstructionStageCommandV1, ...]: + graph = execution_manifest.artifacts + source_inputs = tuple( + item.artifact for item in inventory.partitions_for_window(window) + ) + ( + graph["feed_epochs"], + graph["observation_operator"], + graph["market_context"], + graph["cftc_positioning"], + ) + stage_inputs: Mapping[ReconstructionStage, tuple[ArtifactRef, ...]] = { + ReconstructionStage.SOURCE_ENRICHMENT: source_inputs, + ReconstructionStage.PROPOSAL: ( + graph["motif_manifest"], + graph["motif_index"], + ), + ReconstructionStage.CARVING: ( + graph["market_context"], + graph["cftc_positioning"], + ), + ReconstructionStage.CROSS_SERIES_RECONCILIATION: (), + ReconstructionStage.BROKER_TRANSFER: ( + (graph["broker_delivery"],) + if execution_manifest.delivery_mode + is ReconstructionDeliveryMode.BROKER_CONDITIONED + else () + ), + ReconstructionStage.VALIDATION: ( + graph["benchmark_manifest"], + graph["motif_qualification"], + graph["motif_leakage_audit"], + graph["information_audit"], + ), + ReconstructionStage.ATOMIC_PARTITION_COMMIT: ( + graph["source_inventory"], + graph["retention_plan"], + ), + } + commands: list[ReconstructionStageCommandV1] = [] + for stage in RECONSTRUCTION_STAGE_ORDER: + inputs = stage_inputs[stage] + if len(inputs) > MAX_STAGE_ARTIFACT_REFS: + raise ReconstructionPlanCompatibilityError( + f"{stage.value} input artifact count exceeds orchestration limit" + ) + commands.append( + ReconstructionStageCommandV1( + stage=stage, + handler_name=FIRST_PARTY_RECONSTRUCTION_HANDLERS[stage], + receipt_path=str(scratch / "receipts" / f"{stage.value}.json"), + input_manifest_refs=inputs, + configuration_refs=(execution_ref,), + ) + ) + return tuple(commands) + + +def _validate_stage_inputs( + command: ReconstructionStageCommandV1, + delivery_mode: ReconstructionDeliveryMode, +) -> None: + kinds = {ref.kind for ref in command.input_manifest_refs} + required: Mapping[ReconstructionStage, set[str]] = { + ReconstructionStage.SOURCE_ENRICHMENT: { + ASCII_TICK_SOURCE_KIND, + "feed_epoch_definition_v2", + "observation-operator", + "market_context_corpus_v1", + "cftc_positioning_corpus_v1", + }, + ReconstructionStage.PROPOSAL: { + "modern_reference_motif_manifest_v1", + "modern_reference_motif_index_v1", + }, + ReconstructionStage.CARVING: { + "market_context_corpus_v1", + "cftc_positioning_corpus_v1", + }, + ReconstructionStage.CROSS_SERIES_RECONCILIATION: set(), + ReconstructionStage.BROKER_TRANSFER: ( + {"broker_delivery_artifact_v1"} + if delivery_mode is ReconstructionDeliveryMode.BROKER_CONDITIONED + else set() + ), + ReconstructionStage.VALIDATION: { + "reverse_degradation_manifest_v1", + "modern_reference_motif_qualification_v1", + "modern_reference_motif_leakage_audit_v1", + "reconstruction_information_audit_v1", + }, + ReconstructionStage.ATOMIC_PARTITION_COMMIT: { + SOURCE_INVENTORY_ARTIFACT_KIND, + "reconstruction_retention_plan_v1", + }, + } + if not required[command.stage].issubset(kinds): + raise ReconstructionPlanCompatibilityError( + f"{command.stage.value} command lacks required artifact kinds" + ) + if ( + command.stage is ReconstructionStage.BROKER_TRANSFER + and delivery_mode is ReconstructionDeliveryMode.MODERN_REFERENCE + and command.input_manifest_refs + ): + raise ReconstructionPlanCompatibilityError( + "modern-reference delivery command contains broker inputs" + ) + + +def _source_coverages( + inventory: ReconstructionSourceInventoryV1, +) -> tuple[CrossCurrencySymbolCoverageV1, ...]: + result: list[CrossCurrencySymbolCoverageV1] = [] + for symbol in inventory.symbols: + selected = [ + item for item in inventory.partitions if item.symbol == symbol + ] + result.append( + CrossCurrencySymbolCoverageV1( + symbol=symbol, + start_ns=min(item.coverage_start_ns for item in selected), + end_ns=max(item.coverage_end_ns for item in selected), + source_periods=tuple(item.period for item in selected), + ) + ) + return tuple(result) + + +def _information_splits( + profile: ModernReferenceMotifProfileV1, +) -> tuple[ReconstructionInformationSplitV1, ...]: + train = profile.split_periods["train"] + calibration = profile.split_periods["calibration"] + validation = ( + *profile.split_periods["validation"], + *profile.split_periods["final_holdout"], + ) + return ( + ReconstructionInformationSplitV1( + InformationSplitKind.TRAIN, + _month_start_ns(train[0]), + _month_start_ns(_next_period(train[-1])), + ), + ReconstructionInformationSplitV1( + InformationSplitKind.CALIBRATION, + _month_start_ns(calibration[0]), + _month_start_ns(_next_period(calibration[-1])), + ), + ReconstructionInformationSplitV1( + InformationSplitKind.VALIDATION, + _month_start_ns(validation[0]), + _month_start_ns(_next_period(validation[-1])), + ), + ) + + +def _reject_ex_ante_artifact_leakage( + *, + mode: InformationMode, + requested_start_ns: int, + definition: Any, + motif_profile: ModernReferenceMotifProfileV1, +) -> None: + if mode is not InformationMode.EX_ANTE_SIMULATION: + return + latest_training_ns = max( + definition.coverage_end_utc_ms * 1_000_000, + _month_start_ns(_next_period(motif_profile.split_periods["train"][-1])), + ) + if latest_training_ns >= requested_start_ns: + raise ReconstructionPlanCompatibilityError( + "ex-ante plan refused: fitted epoch/motif artifacts observe the requested future" + ) + + +def _common_source_periods( + definition: Any, symbols: tuple[str, ...] +) -> tuple[str, ...]: + by_symbol: dict[str, set[str]] = {symbol: set() for symbol in symbols} + for item in _sequence(_mapping(definition.lineage).get("sources")): + if not isinstance(item, Mapping): + continue + symbol = _symbol(str(item.get("symbol", ""))) + if symbol in by_symbol: + by_symbol[symbol].add(_period(str(item.get("period", "")))) + common = tuple( + sorted(set.intersection(*(values for values in by_symbol.values()))) + ) + if not common: + raise ReconstructionPlanCompatibilityError( + "feed epoch lineage has no common triangle periods" + ) + if common != _period_range(common[0], common[-1]): + raise ReconstructionPlanCompatibilityError( + "feed epoch lineage common triangle periods are discontinuous" + ) + return common + + +def _inspect_tick_cache(path: Path) -> tuple[int, int, int]: + try: + import pyarrow as pa # pylint: disable=import-outside-toplevel + import pyarrow.ipc as ipc # pylint: disable=import-outside-toplevel + except ImportError as err: + raise RuntimeError("reconstruction planning requires pyarrow") from err + if not path.is_file(): + raise ReconstructionPlanCompatibilityError( + f"source partition is missing: {path}" + ) + try: + with pa.memory_map(str(path), "r") as source: + reader = ipc.open_file(source) + names = set(reader.schema.names) + if not {"datetime", "bid", "ask"}.issubset(names): + raise ReconstructionPlanCompatibilityError( + f"source partition lacks tick bid/ask schema: {path}" + ) + if names.intersection({"open", "high", "low", "close"}): + raise ReconstructionPlanCompatibilityError( + f"source partition contains forbidden OHLC fields: {path}" + ) + if not reader.num_record_batches: + raise ReconstructionPlanCompatibilityError( + f"source partition contains no record batches: {path}" + ) + row_count = sum( + reader.get_batch(index).num_rows + for index in range(reader.num_record_batches) + ) + first_batch = reader.get_batch(0) + last_batch = reader.get_batch(reader.num_record_batches - 1) + column_index = reader.schema.get_field_index("datetime") + first_ms = int(first_batch.column(column_index)[0].as_py()) + last_ms = int(last_batch.column(column_index)[-1].as_py()) + except ReconstructionPlanCompatibilityError: + raise + except Exception as err: + raise ReconstructionPlanCompatibilityError( + f"source partition cannot be read as Arrow IPC: {path}" + ) from err + if row_count <= 0: + raise ReconstructionPlanCompatibilityError( + f"source partition is empty: {path}" + ) + return row_count, first_ms, last_ms + + +def _write_contract_artifact( + contract: Any, + root: Path, + *, + prefix: str, + kind: str, + metadata: Mapping[str, JSONValue], +) -> ArtifactRef: + serializer = getattr(contract, "to_json", None) + if not callable(serializer): + raise TypeError("content-addressed contract must provide to_json()") + encoded = str(serializer()).encode("utf-8") + b"\n" + digest = hashlib.sha256(encoded).hexdigest() + root.mkdir(parents=True, exist_ok=True) + path = root / f"{prefix}-{digest}.json" + if path.exists(): + if path.read_bytes() != encoded: + raise ReconstructionPlanCompatibilityError( + f"content-addressed artifact collision: {path}" + ) + else: + temporary = root / f".{path.name}.{os.getpid()}.tmp" + temporary.write_bytes(encoded) + os.replace(temporary, path) + return ArtifactRef( + kind=kind, + path=str(path.resolve()), + size_bytes=len(encoded), + sha256=digest, + metadata=dict(metadata), + ) + + +def _read_content_addressed_json( + path: str | Path, prefix: str +) -> Mapping[str, Any]: + source = Path(path).expanduser().resolve() + match = re.fullmatch( + rf"{re.escape(prefix)}-([0-9a-f]{{64}})\.json", source.name + ) + if match is None: + raise ValueError(f"{prefix} artifact name is not content addressed") + content = source.read_bytes() + if len(content) > MAX_SYNTHETIC_INFILL_PLAN_BYTES: + raise ValueError(f"{prefix} artifact exceeds plan byte limit") + if hashlib.sha256(content).hexdigest() != match.group(1): + raise ValueError(f"{prefix} artifact hash differs from name") + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as err: + raise ValueError(f"{prefix} artifact is invalid JSON") from err + return _mapping(payload) + + +def _relative_tick_path(symbol: str, period: str) -> Path: + return Path(symbol) / str(int(period[:4])) / str(int(period[4:])) / ".data" + + +def _period_range(start: str, end: str) -> tuple[str, ...]: + values: list[str] = [] + current = _period(start) + selected_end = _period(end) + while current <= selected_end: + values.append(current) + current = _next_period(current) + return tuple(values) + + +def _next_period(value: str) -> str: + period = _period(value) + year, month = int(period[:4]), int(period[4:]) + return f"{year + 1:04d}01" if month == 12 else f"{year:04d}{month + 1:02d}" + + +def _month_start_ns(value: str) -> int: + period = _period(value) + return int( + datetime( + int(period[:4]), int(period[4:]), 1, tzinfo=timezone.utc + ).timestamp() + * 1_000_000_000 + ) + + +def _period_for_ns(value: int) -> str: + timestamp = _int64(value, "requested timestamp") + selected = datetime.fromtimestamp(timestamp // 1_000_000_000, timezone.utc) + return f"{selected.year:04d}{selected.month:02d}" + + +def _period(value: str) -> str: + normalized = str(value).strip() + if _PERIOD_RE.fullmatch(normalized) is None: + raise ValueError("period must use YYYYMM") + month = int(normalized[4:]) + if not 1 <= month <= 12: + raise ValueError("period month is outside 01-12") + return normalized + + +def _symbols(values: Iterable[str]) -> tuple[str, ...]: + normalized = tuple(sorted({_symbol(value) for value in values})) + if not normalized: + raise ValueError("at least one symbol is required") + return normalized + + +def _symbol(value: Any) -> str: + normalized = "".join( + character for character in str(value).lower() if character.isalnum() + ) + if not re.fullmatch(r"[a-z]{6}", normalized): + raise ValueError("FX symbol must contain six letters") + return normalized + + +def _artifact_mapping( + values: Mapping[str, ArtifactRef], +) -> dict[str, ArtifactRef]: + if not values or len(values) > MAX_RECONSTRUCTION_PLAN_ARTIFACTS: + raise ValueError("plan artifact graph count is outside limits") + result = { + _required_text(name): _strong_ref(ref) + for name, ref in sorted(values.items()) + } + if len( + {(ref.kind, ref.path, ref.sha256) for ref in result.values()} + ) != len(result): + raise ValueError("plan artifact graph contains duplicate references") + return result + + +def _strong_ref(value: ArtifactRef) -> ArtifactRef: + if not isinstance(value, ArtifactRef): + raise TypeError("artifact reference must be ArtifactRef") + if value.size_bytes is None: + raise ValueError("artifact reference requires size_bytes") + size = _nonnegative_int(value.size_bytes, "artifact.size_bytes") + sha256 = str(value.sha256).strip().lower() + if _SHA256_RE.fullmatch(sha256) is None: + raise ValueError("artifact reference requires a sha256 digest") + metadata = _json_value_mapping(value.metadata) + return ArtifactRef( + kind=_required_text(value.kind), + path=str(Path(_required_text(value.path)).expanduser().resolve()), + size_bytes=size, + sha256=sha256, + metadata=metadata, + ) + + +def _contract_sha256(contract: Any) -> str: + serializer = getattr(contract, "to_json", None) + if callable(serializer): + encoded = str(serializer()).encode("utf-8") + else: + mapper = getattr(contract, "to_dict", None) + if not callable(mapper): + raise TypeError("contract must provide to_json() or to_dict()") + encoded = canonical_contract_json(mapper()).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _file_sha256(path: Path) -> str: + target = path.resolve() + try: + stat = target.stat() + except OSError as err: + raise ReconstructionPlanCompatibilityError( + f"source artifact cannot be hashed: {target}" + ) from err + return _file_sha256_for_identity( + str(target), + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + ) + + +@lru_cache(maxsize=8192) +def _file_sha256_for_identity( + path: str, + device: int, + inode: int, + size: int, + modified_ns: int, + changed_ns: int, +) -> str: + del device, inode, size, modified_ns, changed_ns + digest = hashlib.sha256() + try: + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + except OSError as err: + raise ReconstructionPlanCompatibilityError( + f"source artifact cannot be hashed: {path}" + ) from err + return digest.hexdigest() + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _resolved_directory(value: str | Path) -> str: + return str(Path(_required_text(value)).expanduser().resolve()) + + +def _validated_plan_roots( + *, + source_root: Path, + artifact_root: str | Path, + output_root: str | Path, + checkpoint_root: str | Path, + scratch_root: str | Path, +) -> dict[str, Path]: + roots = { + "artifact": Path(_resolved_directory(artifact_root)), + "output": Path(_resolved_directory(output_root)), + "checkpoint": Path(_resolved_directory(checkpoint_root)), + "scratch": Path(_resolved_directory(scratch_root)), + } + for name, root in roots.items(): + if _paths_overlap(source_root, root): + raise ReconstructionPlanCompatibilityError( + f"plan {name} root overlaps the immutable source tree" + ) + root_items = tuple(roots.items()) + for index, (left_name, left) in enumerate(root_items): + for right_name, right in root_items[index + 1 :]: + if _paths_overlap(left, right): + raise ReconstructionPlanCompatibilityError( + f"plan {left_name} and {right_name} roots overlap" + ) + return roots + + +def _paths_overlap(left: Path, right: Path) -> bool: + return ( + left == right + or left.is_relative_to(right) + or right.is_relative_to(left) + ) + + +def _required_text(value: Any) -> str: + normalized = str(value).strip() if value is not None else "" + if not normalized: + raise ValueError("required text value is empty") + return normalized + + +def _bounded_text(value: Any, maximum: int) -> str: + normalized = _required_text(value) + if len(normalized) > maximum: + raise ValueError("text value exceeds bounded length") + return normalized + + +def _strict_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def _positive_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _nonnegative_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _int64(value: Any, name: str) -> int: + result = _strict_int(value, name) + if not -(2**63) <= result <= 2**63 - 1: + raise ValueError(f"{name} is outside signed int64") + return result + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a mapping") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError("expected a sequence") + return value + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + try: + return _mapping(json.loads(text)) + except json.JSONDecodeError as err: + raise ValueError("artifact contains invalid JSON") from err + + +def _json_value_mapping(value: Mapping[str, Any]) -> dict[str, JSONValue]: + return {str(key): _json_value(item, depth=0) for key, item in value.items()} + + +def _json_value(value: Any, *, depth: int) -> JSONValue: + if depth > 8: + raise ValueError("artifact metadata nesting exceeds limit") + if value is None or isinstance(value, (str, int, float, bool)): + return cast(JSONValue, value) + if isinstance(value, Mapping): + return { + str(key): _json_value(item, depth=depth + 1) + for key, item in value.items() + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + return [_json_value(item, depth=depth + 1) for item in value] + raise ValueError("artifact metadata must be JSON-compatible") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if data.get("schema_version") != expected: + raise ValueError(f"unsupported schema; expected {expected}") + + +def _require_derived(data: Mapping[str, Any], key: str, expected: Any) -> None: + if data.get(key) != expected: + raise ValueError(f"derived field {key} differs") + + +__all__ = [ + "ASCII_TICK_SOURCE_KIND", + "DEFAULT_RECONSTRUCTION_BASE_SEED", + "DEFAULT_RECONSTRUCTION_MAX_PARALLEL_WINDOWS", + "DEFAULT_RECONSTRUCTION_REQUEST_WINDOW_LIMIT", + "DEFAULT_RECONSTRUCTION_WINDOW_SIZE_NS", + "FIRST_PARTY_RECONSTRUCTION_HANDLERS", + "IMMUTABLE_ANCHOR_POLICY", + "PLAN_CONFIGURATION_ARTIFACT_KIND", + "PLAN_EXECUTION_MANIFEST_ARTIFACT_KIND", + "RECONSTRUCTION_PLAN_CONFIGURATION_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_EXECUTION_MANIFEST_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_REFUSAL_SCHEMA_VERSION", + "RECONSTRUCTION_PLAN_RESOURCE_SUMMARY_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_INVENTORY_SCHEMA_VERSION", + "RECONSTRUCTION_SOURCE_PARTITION_SCHEMA_VERSION", + "SCIENTIFIC_NONCLAIM", + "SOURCE_INVENTORY_ARTIFACT_KIND", + "SYNTHETIC_INFILL_PLAN_ARTIFACT_KIND", + "SYNTHETIC_INFILL_PLAN_SCHEMA_VERSION", + "TICK_ONLY_INPUT_POLICY", + "ReconstructionDeliveryMode", + "ReconstructionPlanCompatibilityError", + "ReconstructionPlanConfigurationV1", + "ReconstructionPlanExecutionManifestV1", + "ReconstructionPlanRefusalCode", + "ReconstructionPlanRefusalV1", + "ReconstructionPlanResourceSummaryV1", + "ReconstructionStagePlanV1", + "ReconstructionSourceInventoryV1", + "ReconstructionSourcePartitionV1", + "SyntheticInfillPlanV1", + "build_synthetic_infill_plan", + "load_reconstruction_stage_plan", + "read_reconstruction_plan_configuration", + "read_reconstruction_plan_execution_manifest", + "read_reconstruction_source_inventory", + "read_synthetic_infill_plan", + "validate_synthetic_infill_plan_for_execution", + "write_synthetic_infill_plan", +] diff --git a/src/histdatacom/synthetic/strategy_sensitivity.py b/src/histdatacom/synthetic/strategy_sensitivity.py new file mode 100644 index 00000000..175d8028 --- /dev/null +++ b/src/histdatacom/synthetic/strategy_sensitivity.py @@ -0,0 +1,3011 @@ +"""Bounded strategy-sensitivity evidence for reconstructed market history. + +The contracts in this module evaluate deterministic strategy and execution +assumptions across time-aligned market-data surfaces. Results are descriptive +execution-sensitivity evidence, not profit claims, strategy optimization, or a +promotion decision for synthetic data. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import defaultdict, deque +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Protocol, TypeVar, cast, runtime_checkable + +from histdatacom.runtime_contracts import JSONValue +from histdatacom.synthetic.bars import DerivedBarV1 +from histdatacom.synthetic.benchmark import BenchmarkEventV1 +from histdatacom.synthetic.contracts import ( + SyntheticEventV1, + canonical_contract_json, +) +from histdatacom.synthetic.information import ( + InformationAuditReportV1, + InformationMode, +) + +STRATEGY_SPECIFICATION_SCHEMA_VERSION = "histdatacom.strategy-specification.v1" +STRATEGY_EXECUTION_SPECIFICATION_SCHEMA_VERSION = ( + "histdatacom.strategy-execution-specification.v1" +) +STRATEGY_EVALUATION_POLICY_SCHEMA_VERSION = ( + "histdatacom.strategy-evaluation-policy.v1" +) +STRATEGY_EVALUATION_CASE_SCHEMA_VERSION = ( + "histdatacom.strategy-evaluation-case.v1" +) +STRATEGY_EVALUATION_PLAN_SCHEMA_VERSION = ( + "histdatacom.strategy-evaluation-plan.v1" +) +STRATEGY_QUOTE_SCHEMA_VERSION = "histdatacom.strategy-quote.v1" +STRATEGY_SIGNAL_SCHEMA_VERSION = "histdatacom.strategy-signal.v1" +STRATEGY_SLICE_RESULT_SCHEMA_VERSION = "histdatacom.strategy-slice-result.v1" +STRATEGY_WINDOW_RESULT_SCHEMA_VERSION = "histdatacom.strategy-window-result.v1" +STRATEGY_UNCERTAINTY_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.strategy-uncertainty-summary.v1" +) +STRATEGY_RESTORATION_RESULT_SCHEMA_VERSION = ( + "histdatacom.strategy-restoration-result.v1" +) +STRATEGY_SENSITIVITY_REPORT_SCHEMA_VERSION = ( + "histdatacom.strategy-sensitivity-report.v1" +) + +REFERENCE_MOMENTUM_STRATEGY_ID = "reference-lagged-midpoint-momentum" +REFERENCE_MOMENTUM_STRATEGY_VERSION = "1.0.0" +STRATEGY_INVALID_FOR_BACKTEST_LABEL = "invalid-for-backtest" +STRATEGY_OUTPUT_MODE = "bounded-derived-metadata" + +DEFAULT_STRATEGY_HORIZONS_NS = ( + 1_000_000_000, + 5_000_000_000, + 60_000_000_000, +) +DEFAULT_STRATEGY_MAX_CASES = 256 +DEFAULT_STRATEGY_MAX_QUOTES_PER_WINDOW = 1_000_000 +DEFAULT_STRATEGY_MAX_SIGNALS_PER_WINDOW = 100_000 +DEFAULT_STRATEGY_MAX_PENDING_SIGNALS = 10_000 +DEFAULT_STRATEGY_MAX_SLICES = 4_096 +DEFAULT_STRATEGY_MAX_PAYLOAD_BYTES = 4_194_304 +DEFAULT_STRATEGY_ROUNDING_DIGITS = 12 + +MAX_STRATEGY_CASES = 2_048 +MAX_STRATEGY_QUOTES_PER_WINDOW = 10_000_000 +MAX_STRATEGY_SIGNALS_PER_WINDOW = 1_000_000 +MAX_STRATEGY_PENDING_SIGNALS = 100_000 +MAX_STRATEGY_SLICES = 32_768 +MAX_STRATEGY_HORIZONS = 32 +MAX_STRATEGY_PAYLOAD_BYTES = 16_777_216 +MAX_STRATEGY_TEXT = 1_024 +MAX_STRATEGY_PARAMETERS = 128 +MAX_STRATEGY_PARAMETER_BYTES = 65_536 +_BPS_SCALE = 10_000.0 + +_EnumT = TypeVar("_EnumT", bound=Enum) + + +class StrategySourceKind(str, Enum): + """Supported market-data surfaces for one aligned evaluation case.""" + + OBSERVED = "observed" + DEGRADED_HOLDOUT = "degraded_holdout" + RECONSTRUCTED = "reconstructed" + UNCONDITIONED_RECONSTRUCTION = "unconditioned_reconstruction" + BROKER_CONDITIONED = "broker_conditioned" + DERIVED_BARS = "derived_bars" + + @classmethod + def from_value( + cls, value: str | "StrategySourceKind" + ) -> "StrategySourceKind": + """Return a strict normalized source kind.""" + return _enum_value(cls, value, "strategy source kind") + + +class StrategySide(str, Enum): + """Normalized directional exposure for one reference signal.""" + + LONG = "long" + SHORT = "short" + + @classmethod + def from_value(cls, value: str | "StrategySide") -> "StrategySide": + """Return a strict normalized side.""" + return _enum_value(cls, value, "strategy side") + + @property + def sign(self) -> float: + """Return the arithmetic sign used for response calculations.""" + return 1.0 if self is StrategySide.LONG else -1.0 + + +class StrategyWindowStatus(str, Enum): + """Terminal status of one bounded aligned evaluation window.""" + + COMPLETED = "completed" + NO_TRADE = "no_trade" + MISSING_SUPPORT = "missing_support" + REFUSED = "refused" + FAILED = "failed" + + @classmethod + def from_value( + cls, value: str | "StrategyWindowStatus" + ) -> "StrategyWindowStatus": + """Return a strict normalized terminal status.""" + return _enum_value(cls, value, "strategy window status") + + +class StrategyEvaluationFailure(RuntimeError): + """A pluggable strategy refused to complete for a scientific reason.""" + + +class StrategyResourceLimitError(StrategyEvaluationFailure): + """Evaluation stopped before a configured resource bound was exceeded.""" + + +@dataclass(frozen=True, slots=True) +class StrategySpecificationV1: + """Versioned deterministic strategy logic and parameter assumptions.""" + + method_id: str + implementation_version: str + parameters: Mapping[str, JSONValue] = field(default_factory=dict) + specification_id: str = "" + schema_version: str = STRATEGY_SPECIFICATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_SPECIFICATION_SCHEMA_VERSION, + "strategy specification", + ) + object.__setattr__(self, "method_id", _required_text(self.method_id)) + object.__setattr__( + self, + "implementation_version", + _required_text(self.implementation_version), + ) + parameters = _bounded_mapping(self.parameters, "strategy parameters") + object.__setattr__(self, "parameters", parameters) + expected = _stable_id("strategy-specification", self.identity_payload()) + supplied = _optional_text(self.specification_id) + if supplied is not None and supplied != expected: + raise ValueError("strategy specification_id differs from content") + object.__setattr__(self, "specification_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic specification identity.""" + return { + "schema_version": self.schema_version, + "method_id": self.method_id, + "implementation_version": self.implementation_version, + "parameters": dict(self.parameters), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible strategy metadata.""" + return { + **self.identity_payload(), + "specification_id": self.specification_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "StrategySpecificationV1": + """Restore and verify one strategy specification.""" + _require_schema(data, STRATEGY_SPECIFICATION_SCHEMA_VERSION) + return cls( + method_id=str(data.get("method_id", "")), + implementation_version=str(data.get("implementation_version", "")), + parameters=_mapping(data.get("parameters"), "parameters"), + specification_id=str(data.get("specification_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class StrategyExecutionSpecificationV1: + """Versioned quote-crossing, latency, slippage, and cost assumptions.""" + + entry_latency_ns: int = 0 + max_execution_wait_ns: int = 1_000_000_000 + slippage_bps_per_side: float = 0.0 + fixed_cost_bps_per_side: float = 0.0 + price_semantics: str = "cross_bid_ask" + exposure_semantics: str = "normalized_unit_exposure" + specification_id: str = "" + schema_version: str = STRATEGY_EXECUTION_SPECIFICATION_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_EXECUTION_SPECIFICATION_SCHEMA_VERSION, + "strategy execution specification", + ) + object.__setattr__( + self, + "entry_latency_ns", + _nonnegative_int(self.entry_latency_ns, "entry_latency_ns"), + ) + object.__setattr__( + self, + "max_execution_wait_ns", + _positive_int(self.max_execution_wait_ns, "max_execution_wait_ns"), + ) + for name in ("slippage_bps_per_side", "fixed_cost_bps_per_side"): + value = _nonnegative_float(getattr(self, name), name) + if value > 10_000.0: + raise ValueError(f"{name} exceeds the bounded assumption range") + object.__setattr__(self, name, value) + if self.price_semantics != "cross_bid_ask": + raise ValueError("version one execution must cross bid/ask") + if self.exposure_semantics != "normalized_unit_exposure": + raise ValueError( + "version one forbids currency profit/notional claims" + ) + expected = _stable_id("strategy-execution", self.identity_payload()) + supplied = _optional_text(self.specification_id) + if supplied is not None and supplied != expected: + raise ValueError("execution specification_id differs from content") + object.__setattr__(self, "specification_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic execution identity.""" + return { + "schema_version": self.schema_version, + "entry_latency_ns": self.entry_latency_ns, + "max_execution_wait_ns": self.max_execution_wait_ns, + "slippage_bps_per_side": self.slippage_bps_per_side, + "fixed_cost_bps_per_side": self.fixed_cost_bps_per_side, + "price_semantics": self.price_semantics, + "exposure_semantics": self.exposure_semantics, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible execution assumptions.""" + return { + **self.identity_payload(), + "specification_id": self.specification_id, + } + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "StrategyExecutionSpecificationV1": + """Restore and verify execution assumptions.""" + _require_schema(data, STRATEGY_EXECUTION_SPECIFICATION_SCHEMA_VERSION) + return cls( + entry_latency_ns=_strict_int( + data.get("entry_latency_ns"), "entry_latency_ns" + ), + max_execution_wait_ns=_strict_int( + data.get("max_execution_wait_ns"), "max_execution_wait_ns" + ), + slippage_bps_per_side=_finite_float( + data.get("slippage_bps_per_side"), "slippage_bps_per_side" + ), + fixed_cost_bps_per_side=_finite_float( + data.get("fixed_cost_bps_per_side"), "fixed_cost_bps_per_side" + ), + price_semantics=str(data.get("price_semantics", "")), + exposure_semantics=str(data.get("exposure_semantics", "")), + specification_id=str(data.get("specification_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class StrategyEvaluationPolicyV1: + """Multiple horizons plus hard streaming and report resource bounds.""" + + horizons_ns: tuple[int, ...] = DEFAULT_STRATEGY_HORIZONS_NS + max_cases: int = DEFAULT_STRATEGY_MAX_CASES + max_quotes_per_window: int = DEFAULT_STRATEGY_MAX_QUOTES_PER_WINDOW + max_signals_per_window: int = DEFAULT_STRATEGY_MAX_SIGNALS_PER_WINDOW + max_pending_signals: int = DEFAULT_STRATEGY_MAX_PENDING_SIGNALS + max_slices: int = DEFAULT_STRATEGY_MAX_SLICES + max_payload_bytes: int = DEFAULT_STRATEGY_MAX_PAYLOAD_BYTES + rounding_digits: int = DEFAULT_STRATEGY_ROUNDING_DIGITS + policy_id: str = "" + schema_version: str = STRATEGY_EVALUATION_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_EVALUATION_POLICY_SCHEMA_VERSION, + "strategy evaluation policy", + ) + horizons = tuple( + _positive_int(item, "strategy horizon") for item in self.horizons_ns + ) + if not horizons or len(horizons) > MAX_STRATEGY_HORIZONS: + raise ValueError("strategy horizon count is outside bounds") + if tuple(sorted(set(horizons))) != horizons: + raise ValueError("strategy horizons must be unique and increasing") + object.__setattr__(self, "horizons_ns", horizons) + for name, maximum in ( + ("max_cases", MAX_STRATEGY_CASES), + ("max_quotes_per_window", MAX_STRATEGY_QUOTES_PER_WINDOW), + ("max_signals_per_window", MAX_STRATEGY_SIGNALS_PER_WINDOW), + ("max_pending_signals", MAX_STRATEGY_PENDING_SIGNALS), + ("max_slices", MAX_STRATEGY_SLICES), + ("max_payload_bytes", MAX_STRATEGY_PAYLOAD_BYTES), + ): + value = _positive_int(getattr(self, name), name) + if value > maximum: + raise ValueError(f"{name} exceeds the hard maximum") + object.__setattr__(self, name, value) + rounding = _bounded_int(self.rounding_digits, "rounding_digits", 0, 15) + object.__setattr__(self, "rounding_digits", rounding) + expected = _stable_id( + "strategy-evaluation-policy", self.identity_payload() + ) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("strategy policy_id differs from content") + object.__setattr__(self, "policy_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic policy identity.""" + return { + "schema_version": self.schema_version, + "horizons_ns": list(self.horizons_ns), + "max_cases": self.max_cases, + "max_quotes_per_window": self.max_quotes_per_window, + "max_signals_per_window": self.max_signals_per_window, + "max_pending_signals": self.max_pending_signals, + "max_slices": self.max_slices, + "max_payload_bytes": self.max_payload_bytes, + "rounding_digits": self.rounding_digits, + "retention_semantics": "online_aggregates_only", + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible evaluation policy.""" + return {**self.identity_payload(), "policy_id": self.policy_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "StrategyEvaluationPolicyV1": + """Restore and verify an evaluation policy.""" + _require_schema(data, STRATEGY_EVALUATION_POLICY_SCHEMA_VERSION) + _require_derived(data, "retention_semantics", "online_aggregates_only") + return cls( + horizons_ns=tuple( + _strict_int(item, "strategy horizon") + for item in _sequence(data.get("horizons_ns"), "horizons_ns") + ), + max_cases=_strict_int(data.get("max_cases"), "max_cases"), + max_quotes_per_window=_strict_int( + data.get("max_quotes_per_window"), "max_quotes_per_window" + ), + max_signals_per_window=_strict_int( + data.get("max_signals_per_window"), "max_signals_per_window" + ), + max_pending_signals=_strict_int( + data.get("max_pending_signals"), "max_pending_signals" + ), + max_slices=_strict_int(data.get("max_slices"), "max_slices"), + max_payload_bytes=_strict_int( + data.get("max_payload_bytes"), "max_payload_bytes" + ), + rounding_digits=_strict_int( + data.get("rounding_digits"), "rounding_digits" + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class StrategyEvaluationCaseV1: + """One source surface bound to an exact aligned half-open time window.""" + + run_id: str + alignment_window_id: str + source_kind: StrategySourceKind + source_artifact_id: str + symbol: str + start_ns: int + end_ns: int + information_mode: InformationMode + information_manifest_id: str + information_audit_id: str + ensemble_member_id: str + broker_profile_id: str | None = None + source_scope: str | None = None + bar_interval_code: str | None = None + invalid_for_backtest_reason: str | None = None + case_id: str = "" + schema_version: str = STRATEGY_EVALUATION_CASE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_EVALUATION_CASE_SCHEMA_VERSION, + "strategy evaluation case", + ) + for name in ( + "run_id", + "alignment_window_id", + "source_artifact_id", + "information_manifest_id", + "information_audit_id", + "ensemble_member_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, "source_kind", StrategySourceKind.from_value(self.source_kind) + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + start = _strict_int(self.start_ns, "start_ns") + end = _strict_int(self.end_ns, "end_ns") + if start >= end: + raise ValueError("strategy case requires start_ns < end_ns") + object.__setattr__(self, "start_ns", start) + object.__setattr__(self, "end_ns", end) + object.__setattr__( + self, + "information_mode", + InformationMode.from_value(self.information_mode), + ) + for name in ( + "broker_profile_id", + "source_scope", + "bar_interval_code", + "invalid_for_backtest_reason", + ): + object.__setattr__(self, name, _optional_text(getattr(self, name))) + if self.source_kind is StrategySourceKind.BROKER_CONDITIONED: + if self.broker_profile_id is None: + raise ValueError("broker-conditioned case requires a profile") + if self.source_kind is StrategySourceKind.DERIVED_BARS: + if self.source_scope is None or self.bar_interval_code is None: + raise ValueError("derived-bar case requires scope and interval") + elif self.bar_interval_code is not None: + raise ValueError("bar_interval_code is only valid for derived bars") + if ( + self.information_mode is InformationMode.EX_POST_RECONSTRUCTION + and self.invalid_for_backtest_reason is None + ): + raise ValueError( + "ex-post strategy case requires invalid-for-backtest reason" + ) + expected = _stable_id( + "strategy-evaluation-case", self.identity_payload() + ) + supplied = _optional_text(self.case_id) + if supplied is not None and supplied != expected: + raise ValueError("strategy case_id differs from content") + object.__setattr__(self, "case_id", expected) + + @property + def valid_for_backtest(self) -> bool: + """Return whether this case can support prospective backtest claims.""" + return self.invalid_for_backtest_reason is None + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic case identity.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "alignment_window_id": self.alignment_window_id, + "source_kind": self.source_kind.value, + "source_artifact_id": self.source_artifact_id, + "symbol": self.symbol, + "start_ns": self.start_ns, + "end_ns": self.end_ns, + "interval": "[start_ns,end_ns)", + "information_mode": self.information_mode.value, + "information_manifest_id": self.information_manifest_id, + "information_audit_id": self.information_audit_id, + "ensemble_member_id": self.ensemble_member_id, + "broker_profile_id": self.broker_profile_id, + "source_scope": self.source_scope, + "bar_interval_code": self.bar_interval_code, + "invalid_for_backtest_reason": self.invalid_for_backtest_reason, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible case metadata.""" + return { + **self.identity_payload(), + "case_id": self.case_id, + "valid_for_backtest": self.valid_for_backtest, + "backtest_label": ( + None + if self.valid_for_backtest + else STRATEGY_INVALID_FOR_BACKTEST_LABEL + ), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "StrategyEvaluationCaseV1": + """Restore and verify one aligned evaluation case.""" + _require_schema(data, STRATEGY_EVALUATION_CASE_SCHEMA_VERSION) + case = cls( + run_id=str(data.get("run_id", "")), + alignment_window_id=str(data.get("alignment_window_id", "")), + source_kind=StrategySourceKind.from_value( + str(data.get("source_kind", "")) + ), + source_artifact_id=str(data.get("source_artifact_id", "")), + symbol=str(data.get("symbol", "")), + start_ns=_strict_int(data.get("start_ns"), "start_ns"), + end_ns=_strict_int(data.get("end_ns"), "end_ns"), + information_mode=InformationMode.from_value( + str(data.get("information_mode", "")) + ), + information_manifest_id=str( + data.get("information_manifest_id", "") + ), + information_audit_id=str(data.get("information_audit_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + broker_profile_id=_mapping_optional_text(data, "broker_profile_id"), + source_scope=_mapping_optional_text(data, "source_scope"), + bar_interval_code=_mapping_optional_text(data, "bar_interval_code"), + invalid_for_backtest_reason=_mapping_optional_text( + data, "invalid_for_backtest_reason" + ), + case_id=str(data.get("case_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived(data, "interval", "[start_ns,end_ns)") + _require_derived(data, "valid_for_backtest", case.valid_for_backtest) + _require_derived( + data, + "backtest_label", + ( + None + if case.valid_for_backtest + else STRATEGY_INVALID_FOR_BACKTEST_LABEL + ), + ) + return case + + +@dataclass(frozen=True, slots=True) +class StrategyEvaluationPlanV1: + """One comparison plan applying identical logic to aligned source cases.""" + + run_id: str + strategy: StrategySpecificationV1 + execution: StrategyExecutionSpecificationV1 + policy: StrategyEvaluationPolicyV1 + cases: tuple[StrategyEvaluationCaseV1, ...] + invalid_for_backtest_reason: str | None = None + plan_id: str = "" + schema_version: str = STRATEGY_EVALUATION_PLAN_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_EVALUATION_PLAN_SCHEMA_VERSION, + "strategy evaluation plan", + ) + object.__setattr__(self, "run_id", _required_text(self.run_id)) + if not isinstance(self.strategy, StrategySpecificationV1): + raise TypeError( + "strategy plan requires a v1 strategy specification" + ) + if not isinstance(self.execution, StrategyExecutionSpecificationV1): + raise TypeError( + "strategy plan requires a v1 execution specification" + ) + if not isinstance(self.policy, StrategyEvaluationPolicyV1): + raise TypeError("strategy plan requires a v1 evaluation policy") + cases = tuple(sorted(self.cases, key=lambda item: item.case_id)) + if len(cases) < 2: + raise ValueError( + "strategy comparison plan requires at least two cases" + ) + if len(cases) > self.policy.max_cases: + raise ValueError("strategy cases exceed plan policy") + if any( + not isinstance(item, StrategyEvaluationCaseV1) for item in cases + ): + raise TypeError("strategy cases must use the v1 contract") + if len({item.case_id for item in cases}) != len(cases): + raise ValueError("strategy case identities must be unique") + if any(item.run_id != self.run_id for item in cases): + raise ValueError("strategy case run differs from plan") + _validate_alignment_groups(cases) + object.__setattr__(self, "cases", cases) + invalid = _optional_text(self.invalid_for_backtest_reason) + modes = {item.information_mode for item in cases} + if len(modes) > 1 and invalid is None: + raise ValueError( + "mixed information modes require an explicit invalid-for-backtest label" + ) + object.__setattr__(self, "invalid_for_backtest_reason", invalid) + expected = _stable_id( + "strategy-evaluation-plan", self.identity_payload() + ) + supplied = _optional_text(self.plan_id) + if supplied is not None and supplied != expected: + raise ValueError("strategy plan_id differs from content") + object.__setattr__(self, "plan_id", expected) + + @property + def valid_for_backtest(self) -> bool: + """Return whether every case and the comparison are prospective-valid.""" + return self.invalid_for_backtest_reason is None and all( + item.valid_for_backtest for item in self.cases + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic comparison-plan identity.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "strategy": self.strategy.to_dict(), + "execution": self.execution.to_dict(), + "policy": self.policy.to_dict(), + "cases": [item.to_dict() for item in self.cases], + "invalid_for_backtest_reason": self.invalid_for_backtest_reason, + "comparison_semantics": "identical_logic_time_aligned_windows", + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible plan metadata.""" + return { + **self.identity_payload(), + "plan_id": self.plan_id, + "valid_for_backtest": self.valid_for_backtest, + "backtest_label": ( + None + if self.valid_for_backtest + else STRATEGY_INVALID_FOR_BACKTEST_LABEL + ), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "StrategyEvaluationPlanV1": + """Restore and verify one comparison plan.""" + _require_schema(data, STRATEGY_EVALUATION_PLAN_SCHEMA_VERSION) + plan = cls( + run_id=str(data.get("run_id", "")), + strategy=StrategySpecificationV1.from_dict( + _mapping(data.get("strategy"), "strategy") + ), + execution=StrategyExecutionSpecificationV1.from_dict( + _mapping(data.get("execution"), "execution") + ), + policy=StrategyEvaluationPolicyV1.from_dict( + _mapping(data.get("policy"), "policy") + ), + cases=tuple( + StrategyEvaluationCaseV1.from_dict(item) + for item in _mapping_sequence(data.get("cases"), "cases") + ), + invalid_for_backtest_reason=_mapping_optional_text( + data, "invalid_for_backtest_reason" + ), + plan_id=str(data.get("plan_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived( + data, + "comparison_semantics", + "identical_logic_time_aligned_windows", + ) + _require_derived(data, "valid_for_backtest", plan.valid_for_backtest) + _require_derived( + data, + "backtest_label", + ( + None + if plan.valid_for_backtest + else STRATEGY_INVALID_FOR_BACKTEST_LABEL + ), + ) + return plan + + +@dataclass(frozen=True, slots=True) +class StrategyQuoteV1: + """Minimal normalized quote and regime context consumed by strategies.""" + + source_event_id: str + symbol: str + event_time_ns: int + event_sequence: int + bid: float + ask: float + epoch_id: str + session: str + event_state: str + sparsity: str + ensemble_member_id: str | None = None + broker_profile_id: str | None = None + source_scope: str | None = None + bar_interval_code: str | None = None + quote_id: str = "" + schema_version: str = STRATEGY_QUOTE_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, STRATEGY_QUOTE_SCHEMA_VERSION, "strategy quote" + ) + object.__setattr__( + self, "source_event_id", _required_text(self.source_event_id) + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, + "event_time_ns", + _strict_int(self.event_time_ns, "event_time_ns"), + ) + sequence = _nonnegative_int(self.event_sequence, "event_sequence") + object.__setattr__(self, "event_sequence", sequence) + bid = _positive_float(self.bid, "bid") + ask = _positive_float(self.ask, "ask") + if ask < bid: + raise ValueError("strategy quote ask is below bid") + object.__setattr__(self, "bid", bid) + object.__setattr__(self, "ask", ask) + for name in ("epoch_id", "session", "event_state", "sparsity"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, "ensemble_member_id", _optional_text(self.ensemble_member_id) + ) + object.__setattr__( + self, "broker_profile_id", _optional_text(self.broker_profile_id) + ) + object.__setattr__( + self, "source_scope", _optional_text(self.source_scope) + ) + object.__setattr__( + self, "bar_interval_code", _optional_text(self.bar_interval_code) + ) + if (self.source_scope is None) != (self.bar_interval_code is None): + raise ValueError( + "strategy quote bar scope and interval must be paired" + ) + expected = _stable_id("strategy-quote", self.identity_payload()) + supplied = _optional_text(self.quote_id) + if supplied is not None and supplied != expected: + raise ValueError("strategy quote_id differs from content") + object.__setattr__(self, "quote_id", expected) + + @property + def mid(self) -> float: + """Return the quote midpoint.""" + return (self.bid + self.ask) / 2.0 + + @property + def spread(self) -> float: + """Return the non-negative quoted spread.""" + return self.ask - self.bid + + @property + def order_key(self) -> tuple[int, int, str]: + """Return the stable within-window ordering key.""" + return (self.event_time_ns, self.event_sequence, self.quote_id) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic quote identity.""" + return { + "schema_version": self.schema_version, + "source_event_id": self.source_event_id, + "symbol": self.symbol, + "event_time_ns": self.event_time_ns, + "event_sequence": self.event_sequence, + "bid": self.bid, + "ask": self.ask, + "epoch_id": self.epoch_id, + "session": self.session, + "event_state": self.event_state, + "sparsity": self.sparsity, + "ensemble_member_id": self.ensemble_member_id, + "broker_profile_id": self.broker_profile_id, + "source_scope": self.source_scope, + "bar_interval_code": self.bar_interval_code, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible quote metadata.""" + return {**self.identity_payload(), "quote_id": self.quote_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "StrategyQuoteV1": + """Restore and verify one normalized strategy quote.""" + _require_schema(data, STRATEGY_QUOTE_SCHEMA_VERSION) + return cls( + source_event_id=str(data.get("source_event_id", "")), + symbol=str(data.get("symbol", "")), + event_time_ns=_strict_int( + data.get("event_time_ns"), "event_time_ns" + ), + event_sequence=_strict_int( + data.get("event_sequence"), "event_sequence" + ), + bid=_finite_float(data.get("bid"), "bid"), + ask=_finite_float(data.get("ask"), "ask"), + epoch_id=str(data.get("epoch_id", "")), + session=str(data.get("session", "")), + event_state=str(data.get("event_state", "")), + sparsity=str(data.get("sparsity", "")), + ensemble_member_id=_mapping_optional_text( + data, "ensemble_member_id" + ), + broker_profile_id=_mapping_optional_text(data, "broker_profile_id"), + source_scope=_mapping_optional_text(data, "source_scope"), + bar_interval_code=_mapping_optional_text(data, "bar_interval_code"), + quote_id=str(data.get("quote_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_benchmark_event( + cls, + event: BenchmarkEventV1, + *, + broker_profile_id: str | None = None, + ) -> "StrategyQuoteV1": + """Adapt one observed, degraded, or reconstructed benchmark event.""" + if not isinstance(event, BenchmarkEventV1): + raise TypeError("strategy benchmark adapter requires event v1") + return cls( + source_event_id=event.benchmark_event_id, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + epoch_id=event.epoch_id, + session=event.session, + event_state=event.event_state, + sparsity=event.sparsity, + ensemble_member_id=event.ensemble_member_id, + broker_profile_id=broker_profile_id, + ) + + @classmethod + def from_synthetic_event( + cls, + event: SyntheticEventV1, + *, + epoch_id: str, + session: str, + event_state: str, + sparsity: str, + ) -> "StrategyQuoteV1": + """Adapt one final observed or generated reconstructed event.""" + if not isinstance(event, SyntheticEventV1): + raise TypeError("strategy reconstruction adapter requires event v1") + return cls( + source_event_id=event.event_id, + symbol=event.symbol, + event_time_ns=event.event_time_ns, + event_sequence=event.event_sequence, + bid=event.bid, + ask=event.ask, + epoch_id=epoch_id, + session=session, + event_state=event_state, + sparsity=sparsity, + ensemble_member_id=event.ensemble_member_id, + broker_profile_id=event.broker_profile_id, + ) + + @classmethod + def from_derived_bar( + cls, + bar: DerivedBarV1, + *, + session: str, + event_state: str, + sparsity: str = "derived_bar", + ) -> "StrategyQuoteV1": + """Adapt one verified derived bar close without fabricating volume.""" + if not isinstance(bar, DerivedBarV1): + raise TypeError("strategy bar adapter requires derived bar v1") + return cls( + source_event_id=bar.bar_id, + symbol=bar.symbol, + event_time_ns=bar.last_event_time_ns, + event_sequence=0, + bid=bar.bid_close, + ask=bar.ask_close, + epoch_id=( + _single_or_mixed(bar.feed_epoch_ids, "unclassified") + or "unclassified" + ), + session=session, + event_state=event_state, + sparsity=sparsity, + ensemble_member_id=bar.ensemble_member_id, + broker_profile_id=_single_or_mixed(bar.broker_profile_ids, None), + source_scope=bar.scope.value, + bar_interval_code=bar.interval_code, + ) + + +@dataclass(frozen=True, slots=True) +class StrategySignalV1: + """One deterministic directional decision emitted from a current quote.""" + + strategy_specification_id: str + source_quote_id: str + decision_time_ns: int + side: StrategySide + signal_id: str = "" + schema_version: str = STRATEGY_SIGNAL_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_SIGNAL_SCHEMA_VERSION, + "strategy signal", + ) + object.__setattr__( + self, + "strategy_specification_id", + _required_text(self.strategy_specification_id), + ) + object.__setattr__( + self, "source_quote_id", _required_text(self.source_quote_id) + ) + object.__setattr__( + self, + "decision_time_ns", + _strict_int(self.decision_time_ns, "decision_time_ns"), + ) + object.__setattr__(self, "side", StrategySide.from_value(self.side)) + expected = _stable_id("strategy-signal", self.identity_payload()) + supplied = _optional_text(self.signal_id) + if supplied is not None and supplied != expected: + raise ValueError("strategy signal_id differs from content") + object.__setattr__(self, "signal_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic signal identity.""" + return { + "schema_version": self.schema_version, + "strategy_specification_id": self.strategy_specification_id, + "source_quote_id": self.source_quote_id, + "decision_time_ns": self.decision_time_ns, + "side": self.side.value, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic signal metadata.""" + return {**self.identity_payload(), "signal_id": self.signal_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "StrategySignalV1": + """Restore and verify one strategy signal.""" + _require_schema(data, STRATEGY_SIGNAL_SCHEMA_VERSION) + return cls( + strategy_specification_id=str( + data.get("strategy_specification_id", "") + ), + source_quote_id=str(data.get("source_quote_id", "")), + decision_time_ns=_strict_int( + data.get("decision_time_ns"), "decision_time_ns" + ), + side=StrategySide.from_value(str(data.get("side", ""))), + signal_id=str(data.get("signal_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@runtime_checkable +class StrategySignalStateV1(Protocol): + """Window-local streaming state supplied by a strategy plugin.""" + + def observe(self, quote: StrategyQuoteV1) -> Sequence[StrategySignalV1]: + """Consume one current quote and emit zero or more current signals.""" + + +@runtime_checkable +class StrategySignalEngineV1(Protocol): + """Pluggable deterministic strategy boundary used by the evaluator.""" + + specification: StrategySpecificationV1 + + def start_window( + self, evaluation_case: StrategyEvaluationCaseV1 + ) -> StrategySignalStateV1: + """Return fresh bounded state for one aligned evaluation window.""" + + +class ReferenceMomentumStrategyV1: + """Transparent lagged-midpoint momentum fixture for accounting tests.""" + + def __init__( + self, + *, + lookback_ns: int = 1_000_000_000, + decision_interval_ns: int = 1_000_000_000, + threshold_bps: float = 0.0, + max_state_quotes: int = 4_096, + ) -> None: + lookback = _positive_int(lookback_ns, "lookback_ns") + interval = _positive_int(decision_interval_ns, "decision_interval_ns") + threshold = _nonnegative_float(threshold_bps, "threshold_bps") + state_limit = _bounded_int( + max_state_quotes, "max_state_quotes", 2, 1_000_000 + ) + self.specification = StrategySpecificationV1( + method_id=REFERENCE_MOMENTUM_STRATEGY_ID, + implementation_version=REFERENCE_MOMENTUM_STRATEGY_VERSION, + parameters={ + "lookback_ns": lookback, + "decision_interval_ns": interval, + "threshold_bps": threshold, + "max_state_quotes": state_limit, + "signal_semantics": "lagged_midpoint_direction", + }, + ) + self._lookback_ns = lookback + self._decision_interval_ns = interval + self._threshold_bps = threshold + self._max_state_quotes = state_limit + + def start_window( + self, evaluation_case: StrategyEvaluationCaseV1 + ) -> StrategySignalStateV1: + """Return independent state so cases cannot contaminate one another.""" + if not isinstance(evaluation_case, StrategyEvaluationCaseV1): + raise TypeError("reference strategy requires an evaluation case") + return _ReferenceMomentumState( + specification_id=self.specification.specification_id, + lookback_ns=self._lookback_ns, + decision_interval_ns=self._decision_interval_ns, + threshold_bps=self._threshold_bps, + max_state_quotes=self._max_state_quotes, + ) + + +@dataclass(slots=True) +class _ReferenceMomentumState: + specification_id: str + lookback_ns: int + decision_interval_ns: int + threshold_bps: float + max_state_quotes: int + history: deque[StrategyQuoteV1] = field(default_factory=deque) + last_decision_ns: int | None = None + + def observe(self, quote: StrategyQuoteV1) -> Sequence[StrategySignalV1]: + target = quote.event_time_ns - self.lookback_ns + while ( + len(self.history) >= 2 and self.history[1].event_time_ns <= target + ): + self.history.popleft() + if len(self.history) >= self.max_state_quotes: + raise StrategyResourceLimitError( + "reference strategy state exceeds max_state_quotes" + ) + self.history.append(quote) + reference = self.history[0] + if reference.event_time_ns > target: + return () + if ( + self.last_decision_ns is not None + and quote.event_time_ns - self.last_decision_ns + < self.decision_interval_ns + ): + return () + change_bps = (quote.mid - reference.mid) / reference.mid * _BPS_SCALE + if change_bps > self.threshold_bps: + side = StrategySide.LONG + elif change_bps < -self.threshold_bps: + side = StrategySide.SHORT + else: + return () + self.last_decision_ns = quote.event_time_ns + return ( + StrategySignalV1( + strategy_specification_id=self.specification_id, + source_quote_id=quote.quote_id, + decision_time_ns=quote.event_time_ns, + side=side, + ), + ) + + +@dataclass(frozen=True, slots=True) +class StrategySliceResultV1: + """Bounded execution response for one full source/regime/member slice.""" + + case_id: str + alignment_window_id: str + source_kind: StrategySourceKind + symbol: str + epoch_id: str + session: str + event_state: str + sparsity: str + broker_profile_id: str + ensemble_member_id: str + horizon_ns: int + signal_count: int + completed_count: int + missing_support_count: int + mean_gross_response_bps: float + mean_net_execution_response_bps: float + mean_cost_drag_bps: float + mean_entry_delay_ns: float + favorable_response_rate: float + slice_result_id: str = "" + schema_version: str = STRATEGY_SLICE_RESULT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_SLICE_RESULT_SCHEMA_VERSION, + "strategy slice result", + ) + for name in ( + "case_id", + "alignment_window_id", + "epoch_id", + "session", + "event_state", + "sparsity", + "broker_profile_id", + "ensemble_member_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, "source_kind", StrategySourceKind.from_value(self.source_kind) + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, "horizon_ns", _positive_int(self.horizon_ns, "horizon_ns") + ) + for name in ( + "signal_count", + "completed_count", + "missing_support_count", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + if self.signal_count < 1: + raise ValueError("strategy slice requires signal support") + if ( + self.signal_count + != self.completed_count + self.missing_support_count + ): + raise ValueError("strategy slice outcome counts do not reconcile") + for name in ( + "mean_gross_response_bps", + "mean_net_execution_response_bps", + "mean_cost_drag_bps", + "mean_entry_delay_ns", + "favorable_response_rate", + ): + object.__setattr__( + self, name, _finite_float(getattr(self, name), name) + ) + if self.mean_cost_drag_bps < 0 or self.mean_entry_delay_ns < 0: + raise ValueError("strategy cost/delay summaries cannot be negative") + if not 0.0 <= self.favorable_response_rate <= 1.0: + raise ValueError("favorable_response_rate must be in [0,1]") + expected = _stable_id("strategy-slice-result", self.identity_payload()) + supplied = _optional_text(self.slice_result_id) + if supplied is not None and supplied != expected: + raise ValueError("strategy slice_result_id differs from content") + object.__setattr__(self, "slice_result_id", expected) + + @property + def comparison_key(self) -> tuple[str, str, str, str, str, int]: + """Return the common key used for reverse-degradation comparison.""" + return ( + self.alignment_window_id, + self.symbol, + self.epoch_id, + self.session, + self.event_state, + self.horizon_ns, + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic slice-result identity.""" + return { + "schema_version": self.schema_version, + "case_id": self.case_id, + "alignment_window_id": self.alignment_window_id, + "source_kind": self.source_kind.value, + "symbol": self.symbol, + "epoch_id": self.epoch_id, + "session": self.session, + "event_state": self.event_state, + "sparsity": self.sparsity, + "broker_profile_id": self.broker_profile_id, + "ensemble_member_id": self.ensemble_member_id, + "horizon_ns": self.horizon_ns, + "signal_count": self.signal_count, + "completed_count": self.completed_count, + "missing_support_count": self.missing_support_count, + "mean_gross_response_bps": self.mean_gross_response_bps, + "mean_net_execution_response_bps": self.mean_net_execution_response_bps, + "mean_cost_drag_bps": self.mean_cost_drag_bps, + "mean_entry_delay_ns": self.mean_entry_delay_ns, + "favorable_response_rate": self.favorable_response_rate, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic slice-result metadata.""" + return { + **self.identity_payload(), + "slice_result_id": self.slice_result_id, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "StrategySliceResultV1": + """Restore and verify one slice result.""" + _require_schema(data, STRATEGY_SLICE_RESULT_SCHEMA_VERSION) + return cls( + case_id=str(data.get("case_id", "")), + alignment_window_id=str(data.get("alignment_window_id", "")), + source_kind=StrategySourceKind.from_value( + str(data.get("source_kind", "")) + ), + symbol=str(data.get("symbol", "")), + epoch_id=str(data.get("epoch_id", "")), + session=str(data.get("session", "")), + event_state=str(data.get("event_state", "")), + sparsity=str(data.get("sparsity", "")), + broker_profile_id=str(data.get("broker_profile_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + horizon_ns=_strict_int(data.get("horizon_ns"), "horizon_ns"), + signal_count=_strict_int(data.get("signal_count"), "signal_count"), + completed_count=_strict_int( + data.get("completed_count"), "completed_count" + ), + missing_support_count=_strict_int( + data.get("missing_support_count"), "missing_support_count" + ), + mean_gross_response_bps=_finite_float( + data.get("mean_gross_response_bps"), "mean_gross_response_bps" + ), + mean_net_execution_response_bps=_finite_float( + data.get("mean_net_execution_response_bps"), + "mean_net_execution_response_bps", + ), + mean_cost_drag_bps=_finite_float( + data.get("mean_cost_drag_bps"), "mean_cost_drag_bps" + ), + mean_entry_delay_ns=_finite_float( + data.get("mean_entry_delay_ns"), "mean_entry_delay_ns" + ), + favorable_response_rate=_finite_float( + data.get("favorable_response_rate"), "favorable_response_rate" + ), + slice_result_id=str(data.get("slice_result_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class StrategyWindowResultV1: + """One bounded source/window result without retained quotes or outcomes.""" + + case_id: str + alignment_window_id: str + source_kind: StrategySourceKind + status: StrategyWindowStatus + quote_count: int + signal_count: int + outcome_count: int + completed_outcome_count: int + missing_support_count: int + mean_interarrival_ns: float + max_interarrival_ns: int + mean_spread_bps: float + slices: tuple[StrategySliceResultV1, ...] = () + reason: str | None = None + invalid_for_backtest_reason: str | None = None + window_result_id: str = "" + schema_version: str = STRATEGY_WINDOW_RESULT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_WINDOW_RESULT_SCHEMA_VERSION, + "strategy window result", + ) + object.__setattr__(self, "case_id", _required_text(self.case_id)) + object.__setattr__( + self, + "alignment_window_id", + _required_text(self.alignment_window_id), + ) + object.__setattr__( + self, "source_kind", StrategySourceKind.from_value(self.source_kind) + ) + object.__setattr__( + self, "status", StrategyWindowStatus.from_value(self.status) + ) + for name in ( + "quote_count", + "signal_count", + "outcome_count", + "completed_outcome_count", + "missing_support_count", + "max_interarrival_ns", + ): + object.__setattr__( + self, name, _nonnegative_int(getattr(self, name), name) + ) + if ( + self.outcome_count + != self.completed_outcome_count + self.missing_support_count + ): + raise ValueError("strategy window outcome counts do not reconcile") + object.__setattr__( + self, + "mean_interarrival_ns", + _nonnegative_float( + self.mean_interarrival_ns, "mean_interarrival_ns" + ), + ) + object.__setattr__( + self, + "mean_spread_bps", + _nonnegative_float(self.mean_spread_bps, "mean_spread_bps"), + ) + slices = tuple( + sorted(self.slices, key=lambda item: item.slice_result_id) + ) + if any(not isinstance(item, StrategySliceResultV1) for item in slices): + raise TypeError("window slices must use strategy slice result v1") + if sum(item.signal_count for item in slices) != self.outcome_count: + raise ValueError("window slices do not reconcile outcome support") + if ( + sum(item.completed_count for item in slices) + != self.completed_outcome_count + ): + raise ValueError("window slices do not reconcile completed support") + object.__setattr__(self, "slices", slices) + object.__setattr__(self, "reason", _optional_text(self.reason)) + object.__setattr__( + self, + "invalid_for_backtest_reason", + _optional_text(self.invalid_for_backtest_reason), + ) + _validate_window_status(self) + expected = _stable_id("strategy-window-result", self.identity_payload()) + supplied = _optional_text(self.window_result_id) + if supplied is not None and supplied != expected: + raise ValueError("strategy window_result_id differs from content") + object.__setattr__(self, "window_result_id", expected) + + @property + def valid_for_backtest(self) -> bool: + """Return whether this result can support prospective backtest claims.""" + return self.invalid_for_backtest_reason is None + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic window-result identity.""" + return { + "schema_version": self.schema_version, + "case_id": self.case_id, + "alignment_window_id": self.alignment_window_id, + "source_kind": self.source_kind.value, + "status": self.status.value, + "quote_count": self.quote_count, + "signal_count": self.signal_count, + "outcome_count": self.outcome_count, + "completed_outcome_count": self.completed_outcome_count, + "missing_support_count": self.missing_support_count, + "mean_interarrival_ns": self.mean_interarrival_ns, + "max_interarrival_ns": self.max_interarrival_ns, + "mean_spread_bps": self.mean_spread_bps, + "slices": [item.to_dict() for item in self.slices], + "reason": self.reason, + "invalid_for_backtest_reason": self.invalid_for_backtest_reason, + "quotes_retained": False, + "outcomes_retained": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic bounded window evidence.""" + return { + **self.identity_payload(), + "window_result_id": self.window_result_id, + "valid_for_backtest": self.valid_for_backtest, + "backtest_label": ( + None + if self.valid_for_backtest + else STRATEGY_INVALID_FOR_BACKTEST_LABEL + ), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "StrategyWindowResultV1": + """Restore and verify one bounded window result.""" + _require_schema(data, STRATEGY_WINDOW_RESULT_SCHEMA_VERSION) + _require_derived(data, "quotes_retained", False) + _require_derived(data, "outcomes_retained", False) + result = cls( + case_id=str(data.get("case_id", "")), + alignment_window_id=str(data.get("alignment_window_id", "")), + source_kind=StrategySourceKind.from_value( + str(data.get("source_kind", "")) + ), + status=StrategyWindowStatus.from_value(str(data.get("status", ""))), + quote_count=_strict_int(data.get("quote_count"), "quote_count"), + signal_count=_strict_int(data.get("signal_count"), "signal_count"), + outcome_count=_strict_int( + data.get("outcome_count"), "outcome_count" + ), + completed_outcome_count=_strict_int( + data.get("completed_outcome_count"), "completed_outcome_count" + ), + missing_support_count=_strict_int( + data.get("missing_support_count"), "missing_support_count" + ), + mean_interarrival_ns=_finite_float( + data.get("mean_interarrival_ns"), "mean_interarrival_ns" + ), + max_interarrival_ns=_strict_int( + data.get("max_interarrival_ns"), "max_interarrival_ns" + ), + mean_spread_bps=_finite_float( + data.get("mean_spread_bps"), "mean_spread_bps" + ), + slices=tuple( + StrategySliceResultV1.from_dict(item) + for item in _mapping_sequence(data.get("slices"), "slices") + ), + reason=_mapping_optional_text(data, "reason"), + invalid_for_backtest_reason=_mapping_optional_text( + data, "invalid_for_backtest_reason" + ), + window_result_id=str(data.get("window_result_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived(data, "valid_for_backtest", result.valid_for_backtest) + _require_derived( + data, + "backtest_label", + ( + None + if result.valid_for_backtest + else STRATEGY_INVALID_FOR_BACKTEST_LABEL + ), + ) + return result + + +@dataclass(frozen=True, slots=True) +class StrategyUncertaintySummaryV1: + """Member/window dispersion for one source and regime sensitivity cell.""" + + source_kind: StrategySourceKind + symbol: str + epoch_id: str + session: str + event_state: str + sparsity: str + broker_profile_id: str + horizon_ns: int + window_count: int + ensemble_member_ids: tuple[str, ...] + completed_outcome_count: int + mean_net_response_bps: float + min_net_response_bps: float + max_net_response_bps: float + standard_deviation_bps: float + summary_id: str = "" + schema_version: str = STRATEGY_UNCERTAINTY_SUMMARY_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_UNCERTAINTY_SUMMARY_SCHEMA_VERSION, + "strategy uncertainty summary", + ) + object.__setattr__( + self, "source_kind", StrategySourceKind.from_value(self.source_kind) + ) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + for name in ( + "epoch_id", + "session", + "event_state", + "sparsity", + "broker_profile_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, "horizon_ns", _positive_int(self.horizon_ns, "horizon_ns") + ) + object.__setattr__( + self, + "window_count", + _positive_int(self.window_count, "window_count"), + ) + members = _normalized_text_tuple(self.ensemble_member_ids) + if not members: + raise ValueError("uncertainty summary requires ensemble members") + object.__setattr__(self, "ensemble_member_ids", members) + object.__setattr__( + self, + "completed_outcome_count", + _positive_int( + self.completed_outcome_count, "completed_outcome_count" + ), + ) + for name in ( + "mean_net_response_bps", + "min_net_response_bps", + "max_net_response_bps", + "standard_deviation_bps", + ): + object.__setattr__( + self, name, _finite_float(getattr(self, name), name) + ) + if self.standard_deviation_bps < 0: + raise ValueError( + "uncertainty standard deviation cannot be negative" + ) + if ( + not self.min_net_response_bps + <= self.mean_net_response_bps + <= self.max_net_response_bps + ): + raise ValueError("uncertainty mean is outside the observed range") + expected = _stable_id("strategy-uncertainty", self.identity_payload()) + supplied = _optional_text(self.summary_id) + if supplied is not None and supplied != expected: + raise ValueError( + "strategy uncertainty summary_id differs from content" + ) + object.__setattr__(self, "summary_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic uncertainty identity.""" + return { + "schema_version": self.schema_version, + "source_kind": self.source_kind.value, + "symbol": self.symbol, + "epoch_id": self.epoch_id, + "session": self.session, + "event_state": self.event_state, + "sparsity": self.sparsity, + "broker_profile_id": self.broker_profile_id, + "horizon_ns": self.horizon_ns, + "window_count": self.window_count, + "ensemble_member_ids": list(self.ensemble_member_ids), + "completed_outcome_count": self.completed_outcome_count, + "mean_net_response_bps": self.mean_net_response_bps, + "min_net_response_bps": self.min_net_response_bps, + "max_net_response_bps": self.max_net_response_bps, + "standard_deviation_bps": self.standard_deviation_bps, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic uncertainty metadata.""" + return {**self.identity_payload(), "summary_id": self.summary_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "StrategyUncertaintySummaryV1": + """Restore and verify one uncertainty summary.""" + _require_schema(data, STRATEGY_UNCERTAINTY_SUMMARY_SCHEMA_VERSION) + return cls( + source_kind=StrategySourceKind.from_value( + str(data.get("source_kind", "")) + ), + symbol=str(data.get("symbol", "")), + epoch_id=str(data.get("epoch_id", "")), + session=str(data.get("session", "")), + event_state=str(data.get("event_state", "")), + sparsity=str(data.get("sparsity", "")), + broker_profile_id=str(data.get("broker_profile_id", "")), + horizon_ns=_strict_int(data.get("horizon_ns"), "horizon_ns"), + window_count=_strict_int(data.get("window_count"), "window_count"), + ensemble_member_ids=tuple( + str(item) + for item in _sequence( + data.get("ensemble_member_ids"), "ensemble_member_ids" + ) + ), + completed_outcome_count=_strict_int( + data.get("completed_outcome_count"), "completed_outcome_count" + ), + mean_net_response_bps=_finite_float( + data.get("mean_net_response_bps"), "mean_net_response_bps" + ), + min_net_response_bps=_finite_float( + data.get("min_net_response_bps"), "min_net_response_bps" + ), + max_net_response_bps=_finite_float( + data.get("max_net_response_bps"), "max_net_response_bps" + ), + standard_deviation_bps=_finite_float( + data.get("standard_deviation_bps"), "standard_deviation_bps" + ), + summary_id=str(data.get("summary_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class StrategyRestorationResultV1: + """Reverse-degradation distance from candidate execution to dense reference.""" + + alignment_window_id: str + candidate_case_id: str + candidate_source_kind: StrategySourceKind + symbol: str + epoch_id: str + session: str + event_state: str + sparsity: str + broker_profile_id: str + ensemble_member_id: str + horizon_ns: int + dense_reference_response_bps: float + degraded_response_bps: float + candidate_response_bps: float + degraded_absolute_error_bps: float + candidate_absolute_error_bps: float + restoration_gain_bps: float + approaches_dense_reference: bool + result_id: str = "" + schema_version: str = STRATEGY_RESTORATION_RESULT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_RESTORATION_RESULT_SCHEMA_VERSION, + "strategy restoration result", + ) + for name in ( + "alignment_window_id", + "candidate_case_id", + "epoch_id", + "session", + "event_state", + "sparsity", + "broker_profile_id", + "ensemble_member_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "candidate_source_kind", + StrategySourceKind.from_value(self.candidate_source_kind), + ) + if self.candidate_source_kind in { + StrategySourceKind.OBSERVED, + StrategySourceKind.DEGRADED_HOLDOUT, + }: + raise ValueError("restoration candidate must be reconstructed") + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, "horizon_ns", _positive_int(self.horizon_ns, "horizon_ns") + ) + for name in ( + "dense_reference_response_bps", + "degraded_response_bps", + "candidate_response_bps", + "degraded_absolute_error_bps", + "candidate_absolute_error_bps", + "restoration_gain_bps", + ): + object.__setattr__( + self, name, _finite_float(getattr(self, name), name) + ) + if ( + self.degraded_absolute_error_bps < 0 + or self.candidate_absolute_error_bps < 0 + ): + raise ValueError("restoration absolute errors cannot be negative") + expected_gain = ( + self.degraded_absolute_error_bps - self.candidate_absolute_error_bps + ) + if not math.isclose( + self.restoration_gain_bps, expected_gain, abs_tol=1e-10 + ): + raise ValueError("restoration gain does not reconcile") + expected_approach = ( + self.candidate_absolute_error_bps + <= self.degraded_absolute_error_bps + ) + if self.approaches_dense_reference is not expected_approach: + raise ValueError("restoration approach flag does not reconcile") + expected = _stable_id("strategy-restoration", self.identity_payload()) + supplied = _optional_text(self.result_id) + if supplied is not None and supplied != expected: + raise ValueError( + "strategy restoration result_id differs from content" + ) + object.__setattr__(self, "result_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic restoration identity.""" + return { + "schema_version": self.schema_version, + "alignment_window_id": self.alignment_window_id, + "candidate_case_id": self.candidate_case_id, + "candidate_source_kind": self.candidate_source_kind.value, + "symbol": self.symbol, + "epoch_id": self.epoch_id, + "session": self.session, + "event_state": self.event_state, + "sparsity": self.sparsity, + "broker_profile_id": self.broker_profile_id, + "ensemble_member_id": self.ensemble_member_id, + "horizon_ns": self.horizon_ns, + "dense_reference_response_bps": self.dense_reference_response_bps, + "degraded_response_bps": self.degraded_response_bps, + "candidate_response_bps": self.candidate_response_bps, + "degraded_absolute_error_bps": self.degraded_absolute_error_bps, + "candidate_absolute_error_bps": self.candidate_absolute_error_bps, + "restoration_gain_bps": self.restoration_gain_bps, + "approaches_dense_reference": self.approaches_dense_reference, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic restoration metadata.""" + return {**self.identity_payload(), "result_id": self.result_id} + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "StrategyRestorationResultV1": + """Restore and verify one restoration result.""" + _require_schema(data, STRATEGY_RESTORATION_RESULT_SCHEMA_VERSION) + return cls( + alignment_window_id=str(data.get("alignment_window_id", "")), + candidate_case_id=str(data.get("candidate_case_id", "")), + candidate_source_kind=StrategySourceKind.from_value( + str(data.get("candidate_source_kind", "")) + ), + symbol=str(data.get("symbol", "")), + epoch_id=str(data.get("epoch_id", "")), + session=str(data.get("session", "")), + event_state=str(data.get("event_state", "")), + sparsity=str(data.get("sparsity", "")), + broker_profile_id=str(data.get("broker_profile_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + horizon_ns=_strict_int(data.get("horizon_ns"), "horizon_ns"), + dense_reference_response_bps=_finite_float( + data.get("dense_reference_response_bps"), + "dense_reference_response_bps", + ), + degraded_response_bps=_finite_float( + data.get("degraded_response_bps"), "degraded_response_bps" + ), + candidate_response_bps=_finite_float( + data.get("candidate_response_bps"), "candidate_response_bps" + ), + degraded_absolute_error_bps=_finite_float( + data.get("degraded_absolute_error_bps"), + "degraded_absolute_error_bps", + ), + candidate_absolute_error_bps=_finite_float( + data.get("candidate_absolute_error_bps"), + "candidate_absolute_error_bps", + ), + restoration_gain_bps=_finite_float( + data.get("restoration_gain_bps"), "restoration_gain_bps" + ), + approaches_dense_reference=_strict_bool( + data.get("approaches_dense_reference"), + "approaches_dense_reference", + ), + result_id=str(data.get("result_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + +@dataclass(frozen=True, slots=True) +class StrategySensitivityReportV1: + """Bounded comparison report emphasizing sensitivity and uncertainty.""" + + plan: StrategyEvaluationPlanV1 + window_results: tuple[StrategyWindowResultV1, ...] + uncertainty_summaries: tuple[StrategyUncertaintySummaryV1, ...] + restoration_results: tuple[StrategyRestorationResultV1, ...] + restoration_unavailable_count: int + report_id: str = "" + schema_version: str = STRATEGY_SENSITIVITY_REPORT_SCHEMA_VERSION + + def __post_init__(self) -> None: + _require_schema_version( + self.schema_version, + STRATEGY_SENSITIVITY_REPORT_SCHEMA_VERSION, + "strategy sensitivity report", + ) + if not isinstance(self.plan, StrategyEvaluationPlanV1): + raise TypeError("strategy report requires evaluation plan v1") + windows = tuple( + sorted(self.window_results, key=lambda item: item.window_result_id) + ) + if len(windows) != len(self.plan.cases) or { + item.case_id for item in windows + } != {item.case_id for item in self.plan.cases}: + raise ValueError( + "strategy report does not cover every plan case exactly" + ) + object.__setattr__(self, "window_results", windows) + uncertainty = tuple( + sorted(self.uncertainty_summaries, key=lambda item: item.summary_id) + ) + restoration = tuple( + sorted(self.restoration_results, key=lambda item: item.result_id) + ) + if any( + not isinstance(item, StrategyUncertaintySummaryV1) + for item in uncertainty + ): + raise TypeError("strategy report uncertainty must use v1 summaries") + if any( + not isinstance(item, StrategyRestorationResultV1) + for item in restoration + ): + raise TypeError("strategy report restoration must use v1 results") + object.__setattr__(self, "uncertainty_summaries", uncertainty) + object.__setattr__(self, "restoration_results", restoration) + object.__setattr__( + self, + "restoration_unavailable_count", + _nonnegative_int( + self.restoration_unavailable_count, + "restoration_unavailable_count", + ), + ) + expected = _stable_id( + "strategy-sensitivity-report", self.identity_payload() + ) + supplied = _optional_text(self.report_id) + if supplied is not None and supplied != expected: + raise ValueError( + "strategy sensitivity report_id differs from content" + ) + object.__setattr__(self, "report_id", expected) + _ensure_payload_size(self.to_dict(), self.plan.policy.max_payload_bytes) + + @property + def valid_for_backtest(self) -> bool: + """Return whether plan and every result support prospective claims.""" + return self.plan.valid_for_backtest and all( + item.valid_for_backtest for item in self.window_results + ) + + @property + def summary(self) -> dict[str, JSONValue]: + """Return bounded failure/no-trade/support rates and counts.""" + total = len(self.window_results) + counts = { + status.value: sum( + item.status is status for item in self.window_results + ) + for status in StrategyWindowStatus + } + status_counts: dict[str, JSONValue] = dict(counts) + outcomes = sum(item.outcome_count for item in self.window_results) + missing = sum( + item.missing_support_count for item in self.window_results + ) + return { + "window_count": total, + "status_counts": status_counts, + "completed_window_rate": _ratio(counts["completed"], total), + "failure_window_rate": _ratio(counts["failed"], total), + "no_trade_window_rate": _ratio(counts["no_trade"], total), + "missing_support_window_rate": _ratio( + counts["missing_support"], total + ), + "refused_window_rate": _ratio(counts["refused"], total), + "outcome_count": outcomes, + "missing_support_outcome_count": missing, + "missing_support_outcome_rate": _ratio(missing, outcomes), + "uncertainty_summary_count": len(self.uncertainty_summaries), + "restoration_result_count": len(self.restoration_results), + "restoration_unavailable_count": self.restoration_unavailable_count, + } + + def identity_payload(self) -> dict[str, JSONValue]: + """Return fields used for deterministic report identity.""" + return { + "schema_version": self.schema_version, + "plan": self.plan.to_dict(), + "window_results": [item.to_dict() for item in self.window_results], + "uncertainty_summaries": [ + item.to_dict() for item in self.uncertainty_summaries + ], + "restoration_results": [ + item.to_dict() for item in self.restoration_results + ], + "restoration_unavailable_count": self.restoration_unavailable_count, + "summary": self.summary, + "interpretation": "sensitivity_robustness_and_uncertainty_only", + "output_mode": STRATEGY_OUTPUT_MODE, + "event_schema_augmented": False, + "profit_claim": False, + "investment_recommendation": False, + "automatic_winner": False, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic bounded report metadata.""" + return { + **self.identity_payload(), + "report_id": self.report_id, + "valid_for_backtest": self.valid_for_backtest, + "backtest_label": ( + None + if self.valid_for_backtest + else STRATEGY_INVALID_FOR_BACKTEST_LABEL + ), + } + + def to_json(self) -> str: + """Return canonical compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, data: Mapping[str, Any] + ) -> "StrategySensitivityReportV1": + """Restore and verify one sensitivity report.""" + _require_schema(data, STRATEGY_SENSITIVITY_REPORT_SCHEMA_VERSION) + for name, expected in ( + ("interpretation", "sensitivity_robustness_and_uncertainty_only"), + ("output_mode", STRATEGY_OUTPUT_MODE), + ("event_schema_augmented", False), + ("profit_claim", False), + ("investment_recommendation", False), + ("automatic_winner", False), + ): + _require_derived(data, name, expected) + report = cls( + plan=StrategyEvaluationPlanV1.from_dict( + _mapping(data.get("plan"), "plan") + ), + window_results=tuple( + StrategyWindowResultV1.from_dict(item) + for item in _mapping_sequence( + data.get("window_results"), "window_results" + ) + ), + uncertainty_summaries=tuple( + StrategyUncertaintySummaryV1.from_dict(item) + for item in _mapping_sequence( + data.get("uncertainty_summaries"), "uncertainty_summaries" + ) + ), + restoration_results=tuple( + StrategyRestorationResultV1.from_dict(item) + for item in _mapping_sequence( + data.get("restoration_results"), "restoration_results" + ) + ), + restoration_unavailable_count=_strict_int( + data.get("restoration_unavailable_count"), + "restoration_unavailable_count", + ), + report_id=str(data.get("report_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + _require_derived(data, "summary", report.summary) + _require_derived(data, "valid_for_backtest", report.valid_for_backtest) + _require_derived( + data, + "backtest_label", + ( + None + if report.valid_for_backtest + else STRATEGY_INVALID_FOR_BACKTEST_LABEL + ), + ) + return report + + @classmethod + def from_json(cls, text: str) -> "StrategySensitivityReportV1": + """Restore a report from canonical JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(slots=True) +class _SliceAccumulator: + signal_count: int = 0 + completed_count: int = 0 + missing_support_count: int = 0 + gross_total: float = 0.0 + net_total: float = 0.0 + cost_total: float = 0.0 + entry_delay_total: int = 0 + favorable_count: int = 0 + + def add_signal(self) -> None: + self.signal_count += 1 + + def add_missing(self) -> None: + self.missing_support_count += 1 + + def add_completed( + self, + *, + gross_bps: float, + net_bps: float, + cost_drag_bps: float, + entry_delay_ns: int, + ) -> None: + self.completed_count += 1 + self.gross_total += gross_bps + self.net_total += net_bps + self.cost_total += cost_drag_bps + self.entry_delay_total += entry_delay_ns + self.favorable_count += int(net_bps > 0.0) + + +@dataclass(slots=True) +class _PendingSignal: + signal: StrategySignalV1 + decision_quote: StrategyQuoteV1 + unresolved_horizons: set[int] + entry_quote: StrategyQuoteV1 | None = None + entry_price: float | None = None + + +def evaluate_strategy_sensitivity( + plan: StrategyEvaluationPlanV1, + quote_streams: Mapping[str, Iterable[StrategyQuoteV1]], + information_audits: Mapping[str, InformationAuditReportV1], + engine: StrategySignalEngineV1, +) -> StrategySensitivityReportV1: + """Evaluate every plan case sequentially using bounded streaming state.""" + if not isinstance(plan, StrategyEvaluationPlanV1): + raise TypeError("strategy evaluation requires plan v1") + if not isinstance(engine, StrategySignalEngineV1): + raise TypeError("strategy engine does not implement the v1 protocol") + if engine.specification != plan.strategy: + raise ValueError("strategy engine specification differs from plan") + unknown_streams = set(quote_streams).difference( + item.case_id for item in plan.cases + ) + if unknown_streams: + raise ValueError("quote streams include unplanned strategy cases") + windows: list[StrategyWindowResultV1] = [] + for evaluation_case in plan.cases: + audit = information_audits.get(evaluation_case.information_audit_id) + if audit is None: + raise ValueError("strategy case lacks its information audit") + _validate_information_audit(evaluation_case, audit) + invalid_reason = ( + plan.invalid_for_backtest_reason + or evaluation_case.invalid_for_backtest_reason + ) + stream = quote_streams.get(evaluation_case.case_id) + if stream is None: + windows.append( + _empty_window_result( + evaluation_case, + StrategyWindowStatus.MISSING_SUPPORT, + reason="quote_stream_missing", + invalid_for_backtest_reason=invalid_reason, + ) + ) + continue + try: + result = _evaluate_strategy_window( + evaluation_case, + stream, + engine, + plan.execution, + plan.policy, + invalid_for_backtest_reason=invalid_reason, + ) + except StrategyResourceLimitError as err: + result = _empty_window_result( + evaluation_case, + StrategyWindowStatus.REFUSED, + reason=str(err), + invalid_for_backtest_reason=invalid_reason, + ) + except StrategyEvaluationFailure as err: + result = _empty_window_result( + evaluation_case, + StrategyWindowStatus.FAILED, + reason=str(err), + invalid_for_backtest_reason=invalid_reason, + ) + windows.append(result) + uncertainty = _build_uncertainty_summaries(windows, plan.policy) + restoration, unavailable = _build_restoration_results(windows, plan.policy) + return StrategySensitivityReportV1( + plan=plan, + window_results=tuple(windows), + uncertainty_summaries=uncertainty, + restoration_results=restoration, + restoration_unavailable_count=unavailable, + ) + + +def strategy_sensitivity_benchmark_hooks( + result: StrategyWindowResultV1, + *, + rounding_digits: int = DEFAULT_STRATEGY_ROUNDING_DIGITS, +) -> dict[str, float]: + """Return canonical numeric hooks for #436/#442 integration. + + Unsupported, no-trade, refused, and failed windows deliberately do not + receive a plausible zero-valued downstream sensitivity. Callers must keep + their terminal status in the report and omit the required ensemble hook so + existing calibration logic refuses that member. + """ + if not isinstance(result, StrategyWindowResultV1): + raise TypeError("strategy benchmark hooks require window result v1") + if result.status is not StrategyWindowStatus.COMPLETED: + raise ValueError("strategy benchmark hooks require a completed window") + digits = _bounded_int(rounding_digits, "rounding_digits", 0, 15) + support = sum(item.completed_count for item in result.slices) + if support < 1: + raise ValueError("strategy benchmark hooks require completed outcomes") + + def weighted(name: str) -> float: + return _rounded( + sum( + getattr(item, name) * item.completed_count + for item in result.slices + ) + / support, + digits, + ) + + return { + "downstream_sensitivity": weighted("mean_net_execution_response_bps"), + "strategy_gross_response_bps": weighted("mean_gross_response_bps"), + "strategy_cost_drag_bps": weighted("mean_cost_drag_bps"), + "strategy_entry_delay_ns": weighted("mean_entry_delay_ns"), + "strategy_missing_support_rate": _rounded( + _ratio(result.missing_support_count, result.outcome_count), digits + ), + } + + +def _evaluate_strategy_window( + evaluation_case: StrategyEvaluationCaseV1, + quotes: Iterable[StrategyQuoteV1], + engine: StrategySignalEngineV1, + execution: StrategyExecutionSpecificationV1, + policy: StrategyEvaluationPolicyV1, + *, + invalid_for_backtest_reason: str | None, +) -> StrategyWindowResultV1: + state = engine.start_window(evaluation_case) + if not isinstance(state, StrategySignalStateV1): + raise TypeError("strategy engine returned incompatible window state") + accumulators: dict[ + tuple[str, str, str, str, str, str, int], _SliceAccumulator + ] = {} + pending: list[_PendingSignal] = [] + quote_count = 0 + signal_count = 0 + spread_bps_total = 0.0 + interarrival_total = 0 + interarrival_count = 0 + max_interarrival = 0 + previous_key: tuple[int, int, str] | None = None + previous_time: int | None = None + for quote in quotes: + quote_count += 1 + if quote_count > policy.max_quotes_per_window: + raise StrategyResourceLimitError("max_quotes_per_window exceeded") + _validate_case_quote(evaluation_case, quote, previous_key) + previous_key = quote.order_key + if previous_time is not None: + gap = quote.event_time_ns - previous_time + interarrival_total += gap + interarrival_count += 1 + max_interarrival = max(max_interarrival, gap) + previous_time = quote.event_time_ns + spread_bps_total += quote.spread / quote.mid * _BPS_SCALE + _advance_pending( + pending, quote, execution, evaluation_case, accumulators + ) + emitted = tuple(state.observe(quote)) + for signal in emitted: + _validate_signal(signal, quote, engine.specification) + signal_count += 1 + if signal_count > policy.max_signals_per_window: + raise StrategyResourceLimitError( + "max_signals_per_window exceeded" + ) + pending_signal = _PendingSignal( + signal=signal, + decision_quote=quote, + unresolved_horizons=set(policy.horizons_ns), + ) + pending.append(pending_signal) + for horizon in policy.horizons_ns: + _slice_accumulator( + accumulators, quote, evaluation_case, horizon + ).add_signal() + if len(pending) > policy.max_pending_signals: + raise StrategyResourceLimitError("max_pending_signals exceeded") + _advance_pending( + pending, quote, execution, evaluation_case, accumulators + ) + for item in pending: + for horizon in item.unresolved_horizons: + _slice_accumulator( + accumulators, item.decision_quote, evaluation_case, horizon + ).add_missing() + slices = _finalize_slices(accumulators, evaluation_case, policy) + outcome_count = sum(item.signal_count for item in slices) + completed_count = sum(item.completed_count for item in slices) + missing_count = sum(item.missing_support_count for item in slices) + if quote_count == 0: + status = StrategyWindowStatus.MISSING_SUPPORT + reason = "empty_quote_stream" + elif signal_count == 0: + status = StrategyWindowStatus.NO_TRADE + reason = "strategy_emitted_no_signals" + elif completed_count == 0: + status = StrategyWindowStatus.MISSING_SUPPORT + reason = "no_signal_horizon_had_execution_support" + else: + status = StrategyWindowStatus.COMPLETED + reason = None + return StrategyWindowResultV1( + case_id=evaluation_case.case_id, + alignment_window_id=evaluation_case.alignment_window_id, + source_kind=evaluation_case.source_kind, + status=status, + quote_count=quote_count, + signal_count=signal_count, + outcome_count=outcome_count, + completed_outcome_count=completed_count, + missing_support_count=missing_count, + mean_interarrival_ns=( + interarrival_total / interarrival_count + if interarrival_count + else 0.0 + ), + max_interarrival_ns=max_interarrival, + mean_spread_bps=( + spread_bps_total / quote_count if quote_count else 0.0 + ), + slices=slices, + reason=reason, + invalid_for_backtest_reason=invalid_for_backtest_reason, + ) + + +def _advance_pending( + pending: list[_PendingSignal], + quote: StrategyQuoteV1, + execution: StrategyExecutionSpecificationV1, + evaluation_case: StrategyEvaluationCaseV1, + accumulators: dict[ + tuple[str, str, str, str, str, str, int], _SliceAccumulator + ], +) -> None: + completed: list[_PendingSignal] = [] + for item in pending: + if item.entry_quote is None: + target = item.signal.decision_time_ns + execution.entry_latency_ns + if quote.event_time_ns < target: + continue + if quote.event_time_ns > target + execution.max_execution_wait_ns: + for horizon in item.unresolved_horizons: + _accumulator_for_pending( + accumulators, item, evaluation_case, horizon + ).add_missing() + item.unresolved_horizons.clear() + completed.append(item) + continue + item.entry_quote = quote + item.entry_price = _entry_price(item.signal.side, quote, execution) + entry_quote = item.entry_quote + for horizon in tuple(sorted(item.unresolved_horizons)): + target = entry_quote.event_time_ns + horizon + if quote.event_time_ns < target: + continue + accumulator = _accumulator_for_pending( + accumulators, item, evaluation_case, horizon + ) + if quote.event_time_ns > target + execution.max_execution_wait_ns: + accumulator.add_missing() + else: + entry_price = cast(float, item.entry_price) + exit_price = _exit_price(item.signal.side, quote, execution) + gross = ( + item.signal.side.sign + * (quote.mid - item.decision_quote.mid) + / item.decision_quote.mid + * _BPS_SCALE + ) + net = ( + item.signal.side.sign + * (exit_price - entry_price) + / item.decision_quote.mid + * _BPS_SCALE + - 2.0 * execution.fixed_cost_bps_per_side + ) + accumulator.add_completed( + gross_bps=gross, + net_bps=net, + cost_drag_bps=max(0.0, gross - net), + entry_delay_ns=entry_quote.event_time_ns + - item.signal.decision_time_ns, + ) + item.unresolved_horizons.remove(horizon) + if not item.unresolved_horizons: + completed.append(item) + for item in completed: + pending.remove(item) + + +def _slice_accumulator( + target: dict[tuple[str, str, str, str, str, str, int], _SliceAccumulator], + quote: StrategyQuoteV1, + evaluation_case: StrategyEvaluationCaseV1, + horizon_ns: int, +) -> _SliceAccumulator: + key = ( + quote.epoch_id, + quote.session, + quote.event_state, + quote.sparsity, + quote.broker_profile_id + or evaluation_case.broker_profile_id + or "unconditioned", + quote.ensemble_member_id or evaluation_case.ensemble_member_id, + horizon_ns, + ) + return target.setdefault(key, _SliceAccumulator()) + + +def _accumulator_for_pending( + target: dict[tuple[str, str, str, str, str, str, int], _SliceAccumulator], + pending: _PendingSignal, + evaluation_case: StrategyEvaluationCaseV1, + horizon_ns: int, +) -> _SliceAccumulator: + return _slice_accumulator( + target, + pending.decision_quote, + evaluation_case, + horizon_ns, + ) + + +def _finalize_slices( + accumulators: Mapping[ + tuple[str, str, str, str, str, str, int], _SliceAccumulator + ], + evaluation_case: StrategyEvaluationCaseV1, + policy: StrategyEvaluationPolicyV1, +) -> tuple[StrategySliceResultV1, ...]: + if len(accumulators) > policy.max_slices: + raise StrategyResourceLimitError("max_slices exceeded") + results: list[StrategySliceResultV1] = [] + for key, value in sorted(accumulators.items()): + epoch, session, event_state, sparsity, broker, member, horizon = key + completed = value.completed_count + results.append( + StrategySliceResultV1( + case_id=evaluation_case.case_id, + alignment_window_id=evaluation_case.alignment_window_id, + source_kind=evaluation_case.source_kind, + symbol=evaluation_case.symbol, + epoch_id=epoch, + session=session, + event_state=event_state, + sparsity=sparsity, + broker_profile_id=broker, + ensemble_member_id=member, + horizon_ns=horizon, + signal_count=value.signal_count, + completed_count=completed, + missing_support_count=value.missing_support_count, + mean_gross_response_bps=( + _rounded( + value.gross_total / completed, policy.rounding_digits + ) + if completed + else 0.0 + ), + mean_net_execution_response_bps=( + _rounded( + value.net_total / completed, policy.rounding_digits + ) + if completed + else 0.0 + ), + mean_cost_drag_bps=( + _rounded( + value.cost_total / completed, policy.rounding_digits + ) + if completed + else 0.0 + ), + mean_entry_delay_ns=( + _rounded( + value.entry_delay_total / completed, + policy.rounding_digits, + ) + if completed + else 0.0 + ), + favorable_response_rate=( + _rounded( + value.favorable_count / completed, + policy.rounding_digits, + ) + if completed + else 0.0 + ), + ) + ) + return tuple(results) + + +def _build_uncertainty_summaries( + windows: Sequence[StrategyWindowResultV1], + policy: StrategyEvaluationPolicyV1, +) -> tuple[StrategyUncertaintySummaryV1, ...]: + grouped: dict[ + tuple[StrategySourceKind, str, str, str, str, str, str, int], + list[StrategySliceResultV1], + ] = defaultdict(list) + for window in windows: + for item in window.slices: + if item.completed_count: + grouped[ + ( + item.source_kind, + item.symbol, + item.epoch_id, + item.session, + item.event_state, + item.sparsity, + item.broker_profile_id, + item.horizon_ns, + ) + ].append(item) + if len(grouped) > policy.max_slices: + raise StrategyResourceLimitError( + "uncertainty summaries exceed max_slices" + ) + results: list[StrategyUncertaintySummaryV1] = [] + for key, items in sorted(grouped.items(), key=lambda pair: str(pair[0])): + values = [item.mean_net_execution_response_bps for item in items] + mean = sum(values) / len(values) + variance = sum((item - mean) ** 2 for item in values) / len(values) + ( + source, + symbol, + epoch, + session, + event_state, + sparsity, + broker, + horizon, + ) = key + results.append( + StrategyUncertaintySummaryV1( + source_kind=source, + symbol=symbol, + epoch_id=epoch, + session=session, + event_state=event_state, + sparsity=sparsity, + broker_profile_id=broker, + horizon_ns=horizon, + window_count=len({item.alignment_window_id for item in items}), + ensemble_member_ids=tuple( + item.ensemble_member_id for item in items + ), + completed_outcome_count=sum( + item.completed_count for item in items + ), + mean_net_response_bps=_rounded(mean, policy.rounding_digits), + min_net_response_bps=min(values), + max_net_response_bps=max(values), + standard_deviation_bps=_rounded( + math.sqrt(variance), policy.rounding_digits + ), + ) + ) + return tuple(results) + + +def _build_restoration_results( + windows: Sequence[StrategyWindowResultV1], + policy: StrategyEvaluationPolicyV1, +) -> tuple[tuple[StrategyRestorationResultV1, ...], int]: + by_source: dict[ + StrategySourceKind, + dict[tuple[str, str, str, str, str, int], list[StrategySliceResultV1]], + ] = defaultdict(lambda: defaultdict(list)) + for window in windows: + for item in window.slices: + if item.completed_count: + by_source[item.source_kind][item.comparison_key].append(item) + dense = by_source.get(StrategySourceKind.OBSERVED, {}) + degraded = by_source.get(StrategySourceKind.DEGRADED_HOLDOUT, {}) + candidate_kinds = tuple( + item + for item in StrategySourceKind + if item + not in { + StrategySourceKind.OBSERVED, + StrategySourceKind.DEGRADED_HOLDOUT, + } + ) + results: list[StrategyRestorationResultV1] = [] + unavailable = 0 + for source in candidate_kinds: + for key, candidates in by_source.get(source, {}).items(): + reference_items = dense.get(key, ()) + degraded_items = degraded.get(key, ()) + if not reference_items or not degraded_items: + unavailable += len(candidates) + continue + reference = _weighted_response(reference_items) + degraded_response = _weighted_response(degraded_items) + degraded_error = abs(degraded_response - reference) + for candidate in candidates: + candidate_response = candidate.mean_net_execution_response_bps + candidate_error = abs(candidate_response - reference) + rounded_degraded_error = _rounded( + degraded_error, policy.rounding_digits + ) + rounded_candidate_error = _rounded( + candidate_error, policy.rounding_digits + ) + results.append( + StrategyRestorationResultV1( + alignment_window_id=candidate.alignment_window_id, + candidate_case_id=candidate.case_id, + candidate_source_kind=candidate.source_kind, + symbol=candidate.symbol, + epoch_id=candidate.epoch_id, + session=candidate.session, + event_state=candidate.event_state, + sparsity=candidate.sparsity, + broker_profile_id=candidate.broker_profile_id, + ensemble_member_id=candidate.ensemble_member_id, + horizon_ns=candidate.horizon_ns, + dense_reference_response_bps=_rounded( + reference, policy.rounding_digits + ), + degraded_response_bps=_rounded( + degraded_response, policy.rounding_digits + ), + candidate_response_bps=candidate_response, + degraded_absolute_error_bps=rounded_degraded_error, + candidate_absolute_error_bps=rounded_candidate_error, + restoration_gain_bps=_rounded( + rounded_degraded_error - rounded_candidate_error, + policy.rounding_digits, + ), + approaches_dense_reference=candidate_error + <= degraded_error, + ) + ) + if len(results) > policy.max_slices: + raise StrategyResourceLimitError( + "restoration results exceed max_slices" + ) + return tuple(results), unavailable + + +def _weighted_response(items: Sequence[StrategySliceResultV1]) -> float: + support = sum(item.completed_count for item in items) + if not support: + return 0.0 + return ( + sum( + item.mean_net_execution_response_bps * item.completed_count + for item in items + ) + / support + ) + + +def _entry_price( + side: StrategySide, + quote: StrategyQuoteV1, + execution: StrategyExecutionSpecificationV1, +) -> float: + slip = execution.slippage_bps_per_side / _BPS_SCALE + return ( + quote.ask * (1.0 + slip) + if side is StrategySide.LONG + else quote.bid * (1.0 - slip) + ) + + +def _exit_price( + side: StrategySide, + quote: StrategyQuoteV1, + execution: StrategyExecutionSpecificationV1, +) -> float: + slip = execution.slippage_bps_per_side / _BPS_SCALE + return ( + quote.bid * (1.0 - slip) + if side is StrategySide.LONG + else quote.ask * (1.0 + slip) + ) + + +def _validate_information_audit( + evaluation_case: StrategyEvaluationCaseV1, + audit: InformationAuditReportV1, +) -> None: + if not isinstance(audit, InformationAuditReportV1): + raise TypeError("strategy information audit must use report v1") + if audit.audit_id != evaluation_case.information_audit_id: + raise ValueError("strategy case information audit identity differs") + if audit.run_id != evaluation_case.run_id: + raise ValueError("strategy case information audit run differs") + if audit.manifest_id != evaluation_case.information_manifest_id: + raise ValueError("strategy case information manifest differs") + if audit.information_mode is not evaluation_case.information_mode: + raise ValueError("strategy case information mode differs from audit") + if not audit.accepted: + raise ValueError( + "strategy evaluation requires an accepted information audit" + ) + if ( + evaluation_case.valid_for_backtest + and not audit.valid_for_strategy_usefulness_claim + ): + raise ValueError( + "strategy-valid case lacks strategy-usefulness audit approval" + ) + + +def _validate_alignment_groups( + cases: Sequence[StrategyEvaluationCaseV1], +) -> None: + groups: dict[str, tuple[str, int, int]] = {} + seen_roles: set[tuple[str, StrategySourceKind, str, str | None]] = set() + for item in cases: + axis = (item.symbol, item.start_ns, item.end_ns) + previous = groups.setdefault(item.alignment_window_id, axis) + if previous != axis: + raise ValueError( + "aligned strategy cases differ in symbol or time bounds" + ) + role = ( + item.alignment_window_id, + item.source_kind, + item.ensemble_member_id, + item.broker_profile_id, + ) + if role in seen_roles: + raise ValueError( + "aligned strategy source/member role is duplicated" + ) + seen_roles.add(role) + + +def _validate_case_quote( + evaluation_case: StrategyEvaluationCaseV1, + quote: StrategyQuoteV1, + previous_key: tuple[int, int, str] | None, +) -> None: + if not isinstance(quote, StrategyQuoteV1): + raise TypeError("strategy quote stream must use quote v1") + if quote.symbol != evaluation_case.symbol: + raise ValueError("strategy quote symbol differs from case") + if ( + not evaluation_case.start_ns + <= quote.event_time_ns + < evaluation_case.end_ns + ): + raise ValueError( + "strategy quote falls outside aligned half-open window" + ) + if previous_key is not None and quote.order_key <= previous_key: + raise ValueError("strategy quote stream is not strictly ordered") + if ( + quote.ensemble_member_id is not None + and quote.ensemble_member_id != evaluation_case.ensemble_member_id + ): + raise ValueError("strategy quote ensemble member differs from case") + if ( + quote.broker_profile_id is not None + and evaluation_case.broker_profile_id is not None + and quote.broker_profile_id != evaluation_case.broker_profile_id + ): + raise ValueError("strategy quote broker profile differs from case") + if evaluation_case.source_kind is StrategySourceKind.DERIVED_BARS: + if quote.source_scope != evaluation_case.source_scope: + raise ValueError("strategy bar quote scope differs from case") + if quote.bar_interval_code != evaluation_case.bar_interval_code: + raise ValueError("strategy bar quote interval differs from case") + elif quote.source_scope is not None or quote.bar_interval_code is not None: + raise ValueError( + "non-bar strategy case received derived-bar quote metadata" + ) + + +def _validate_signal( + signal: StrategySignalV1, + quote: StrategyQuoteV1, + specification: StrategySpecificationV1, +) -> None: + if not isinstance(signal, StrategySignalV1): + raise TypeError("strategy plugin emitted a non-v1 signal") + if signal.strategy_specification_id != specification.specification_id: + raise ValueError("strategy signal specification differs from engine") + if signal.source_quote_id != quote.quote_id: + raise ValueError("strategy signal is not bound to the current quote") + if signal.decision_time_ns != quote.event_time_ns: + raise ValueError( + "strategy signal decision time differs from current quote" + ) + + +def _empty_window_result( + evaluation_case: StrategyEvaluationCaseV1, + status: StrategyWindowStatus, + *, + reason: str, + invalid_for_backtest_reason: str | None, +) -> StrategyWindowResultV1: + return StrategyWindowResultV1( + case_id=evaluation_case.case_id, + alignment_window_id=evaluation_case.alignment_window_id, + source_kind=evaluation_case.source_kind, + status=status, + quote_count=0, + signal_count=0, + outcome_count=0, + completed_outcome_count=0, + missing_support_count=0, + mean_interarrival_ns=0.0, + max_interarrival_ns=0, + mean_spread_bps=0.0, + reason=reason, + invalid_for_backtest_reason=invalid_for_backtest_reason, + ) + + +def _validate_window_status(result: StrategyWindowResultV1) -> None: + if result.status is StrategyWindowStatus.COMPLETED: + if result.completed_outcome_count < 1 or result.reason is not None: + raise ValueError("completed strategy window status is inconsistent") + return + if result.reason is None: + raise ValueError("non-completed strategy window requires a reason") + if result.status is StrategyWindowStatus.NO_TRADE and result.signal_count: + raise ValueError("no-trade strategy window cannot contain signals") + if result.status in { + StrategyWindowStatus.REFUSED, + StrategyWindowStatus.FAILED, + }: + if result.quote_count or result.signal_count or result.slices: + raise ValueError( + "failed/refused strategy window cannot claim partial results" + ) + + +def _bounded_mapping( + value: Mapping[str, JSONValue], label: str +) -> dict[str, JSONValue]: + if not isinstance(value, Mapping): + raise TypeError(f"{label} must be a mapping") + if len(value) > MAX_STRATEGY_PARAMETERS: + raise ValueError(f"{label} exceeds item limit") + normalized = {str(key): item for key, item in sorted(value.items())} + _ensure_payload_size(normalized, MAX_STRATEGY_PARAMETER_BYTES) + return normalized + + +def _ensure_payload_size(value: Mapping[str, JSONValue], maximum: int) -> None: + encoded = canonical_contract_json(value).encode("utf-8") + if len(encoded) > maximum: + raise ValueError("strategy payload exceeds configured byte limit") + + +def _stable_id(prefix: str, value: Mapping[str, JSONValue]) -> str: + digest = hashlib.sha256( + canonical_contract_json(value).encode("utf-8") + ).hexdigest() + return f"{prefix}:sha256:{digest}" + + +def _required_text(value: Any) -> str: + normalized = str(value).strip() + if not normalized: + raise ValueError("required strategy text is empty") + if len(normalized) > MAX_STRATEGY_TEXT: + raise ValueError("strategy text exceeds length limit") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + if not normalized: + return None + return _required_text(normalized) + + +def _normalized_symbol(value: Any) -> str: + return _required_text(value).upper() + + +def _normalized_text_tuple(values: Iterable[str]) -> tuple[str, ...]: + return tuple(sorted({_required_text(item) for item in values})) + + +def _single_or_mixed(values: Sequence[str], fallback: str | None) -> str | None: + normalized = _normalized_text_tuple(values) + if not normalized: + return fallback + if len(normalized) == 1: + return normalized[0] + return "mixed" + + +def _strict_int(value: Any, name: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise TypeError(f"{name} must be an integer") + return value + + +def _nonnegative_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _positive_int(value: Any, name: str) -> int: + result = _strict_int(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _bounded_int(value: Any, name: str, minimum: int, maximum: int) -> int: + result = _strict_int(value, name) + if not minimum <= result <= maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}") + return result + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{name} must be numeric") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _nonnegative_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if result < 0: + raise ValueError(f"{name} must be non-negative") + return result + + +def _positive_float(value: Any, name: str) -> float: + result = _finite_float(value, name) + if result <= 0: + raise ValueError(f"{name} must be positive") + return result + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise TypeError(f"{name} must be a boolean") + return value + + +def _ratio(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator else 0.0 + + +def _rounded(value: float, digits: int) -> float: + return round(float(value), digits) + + +def _enum_value(enum_type: type[_EnumT], value: Any, label: str) -> _EnumT: + if isinstance(value, enum_type): + return value + try: + return enum_type(str(value).strip().lower()) + except ValueError as err: + raise ValueError(f"unsupported {label}") from err + + +def _require_schema_version(value: str, expected: str, label: str) -> None: + if value != expected: + raise ValueError(f"unsupported {label} schema version") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError("unsupported strategy contract schema version") + + +def _require_derived(data: Mapping[str, Any], name: str, expected: Any) -> None: + if data.get(name) != expected: + raise ValueError(f"derived strategy field {name} differs") + + +def _mapping(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise TypeError(f"{name} must be a mapping") + return value + + +def _mapping_optional_text(data: Mapping[str, Any], name: str) -> str | None: + return _optional_text(data.get(name)) + + +def _sequence(value: Any, name: str) -> tuple[Any, ...]: + if not isinstance(value, (list, tuple)): + raise TypeError(f"{name} must be a sequence") + return tuple(value) + + +def _mapping_sequence(value: Any, name: str) -> tuple[Mapping[str, Any], ...]: + return tuple(_mapping(item, name) for item in _sequence(value, name)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + if not isinstance(value, Mapping): + raise TypeError("strategy JSON must contain an object") + return value + + +__all__ = [ + "DEFAULT_STRATEGY_HORIZONS_NS", + "REFERENCE_MOMENTUM_STRATEGY_ID", + "REFERENCE_MOMENTUM_STRATEGY_VERSION", + "STRATEGY_EVALUATION_CASE_SCHEMA_VERSION", + "STRATEGY_EVALUATION_PLAN_SCHEMA_VERSION", + "STRATEGY_EVALUATION_POLICY_SCHEMA_VERSION", + "STRATEGY_EXECUTION_SPECIFICATION_SCHEMA_VERSION", + "STRATEGY_INVALID_FOR_BACKTEST_LABEL", + "STRATEGY_QUOTE_SCHEMA_VERSION", + "STRATEGY_RESTORATION_RESULT_SCHEMA_VERSION", + "STRATEGY_SENSITIVITY_REPORT_SCHEMA_VERSION", + "STRATEGY_SIGNAL_SCHEMA_VERSION", + "STRATEGY_SLICE_RESULT_SCHEMA_VERSION", + "STRATEGY_SPECIFICATION_SCHEMA_VERSION", + "STRATEGY_UNCERTAINTY_SUMMARY_SCHEMA_VERSION", + "STRATEGY_WINDOW_RESULT_SCHEMA_VERSION", + "ReferenceMomentumStrategyV1", + "StrategyEvaluationCaseV1", + "StrategyEvaluationFailure", + "StrategyEvaluationPlanV1", + "StrategyEvaluationPolicyV1", + "StrategyExecutionSpecificationV1", + "StrategyQuoteV1", + "StrategyResourceLimitError", + "StrategyRestorationResultV1", + "StrategySensitivityReportV1", + "StrategySide", + "StrategySignalEngineV1", + "StrategySignalStateV1", + "StrategySignalV1", + "StrategySliceResultV1", + "StrategySourceKind", + "StrategySpecificationV1", + "StrategyUncertaintySummaryV1", + "StrategyWindowResultV1", + "StrategyWindowStatus", + "evaluate_strategy_sensitivity", + "strategy_sensitivity_benchmark_hooks", +] diff --git a/src/histdatacom/synthetic/streaming.py b/src/histdatacom/synthetic/streaming.py new file mode 100644 index 00000000..28d111d7 --- /dev/null +++ b/src/histdatacom/synthetic/streaming.py @@ -0,0 +1,2236 @@ +"""Bounded execution contracts for synthetic reconstruction streams. + +The version-one contracts in this module are the control-plane boundary +between narrow synthetic events and later production orchestration/storage. +They intentionally do not generate events, write final Parquet partitions, or +define Temporal workflows. Data-plane rows remain in process-local memory or +artifacts referenced by :class:`~histdatacom.runtime_contracts.ArtifactRef`. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, cast + +from histdatacom.runtime_contracts import ArtifactRef, JSONValue +from histdatacom.synthetic.contracts import canonical_contract_json + +RECONSTRUCTION_STORAGE_POLICY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-storage-policy.v1" +) +RECONSTRUCTION_RESOURCE_ESTIMATE_SCHEMA_VERSION = ( + "histdatacom.reconstruction-resource-estimate.v1" +) +RECONSTRUCTION_RUN_SCHEMA_VERSION = "histdatacom.reconstruction-run.v1" +RECONSTRUCTION_WINDOW_SCHEMA_VERSION = "histdatacom.reconstruction-window.v1" +EVENT_BATCH_SCHEMA_VERSION = "histdatacom.reconstruction-event-batch.v1" +CARRY_STATE_SCHEMA_VERSION = "histdatacom.reconstruction-carry-state.v1" +REJECTION_SUMMARY_SCHEMA_VERSION = ( + "histdatacom.reconstruction-rejection-summary.v1" +) +PARTITION_MANIFEST_SCHEMA_VERSION = ( + "histdatacom.reconstruction-partition-manifest.v1" +) +RECONSTRUCTION_CHECKPOINT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-checkpoint.v1" +) +RECONSTRUCTION_HEARTBEAT_SCHEMA_VERSION = ( + "histdatacom.reconstruction-heartbeat.v1" +) + +INT64_MIN = -(2**63) +INT64_MAX = 2**63 - 1 +UINT64_MAX = 2**64 - 1 +MAX_SYMBOLS_PER_SYNCHRONIZATION_UNIT = 64 +MAX_ARTIFACT_REFS_PER_CONTRACT = 256 +MAX_EVENT_BATCHES_PER_MANIFEST = 4096 +MAX_REJECTION_REASONS = 256 +MAX_ARTIFACT_METADATA_BYTES = 65_536 +MAX_CARRY_STATE_BYTES = 262_144 +MAX_HEARTBEAT_BYTES = 65_536 +DEFAULT_MAX_CHECKPOINT_BYTES = 524_288 + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_FORBIDDEN_INLINE_DATA_KEYS = frozenset( + { + "dataframe", + "events", + "records", + "rows", + "table", + } +) + + +class ReconstructionCommitPhase(str, Enum): + """Two-phase publication and interruption states for one window.""" + + PLANNED = "planned" + RUNNING = "running" + STAGED = "staged" + VALIDATED = "validated" + COMMITTED = "committed" + CANCELLED = "cancelled" + FAILED = "failed" + + @classmethod + def from_value( + cls, + value: str | "ReconstructionCommitPhase", + ) -> "ReconstructionCommitPhase": + """Return a strict normalized commit phase.""" + if isinstance(value, cls): + return value + try: + return cls(str(value).strip().lower()) + except ValueError as err: + raise ValueError("unsupported reconstruction commit phase") from err + + +class ReconstructionResourceLimitError(ValueError): + """Resource preflight refusal carrying the rejected estimate.""" + + def __init__( + self, + estimate: "ReconstructionResourceEstimateV1", + violations: Sequence[str], + ) -> None: + self.estimate = estimate + self.violations = tuple(violations) + super().__init__( + "reconstruction resource preflight failed: " + + "; ".join(self.violations) + ) + + +@dataclass(frozen=True, slots=True) +class ReconstructionStoragePolicyV1: + """Bounded scratch, memory, output, and checkpoint policy.""" + + max_events_per_batch: int = 100_000 + max_candidate_amplification: float = 25.0 + max_inflight_batches: int = 8 + max_memory_bytes: int = 2 * 1024**3 + max_scratch_bytes: int = 100 * 1024**3 + max_output_bytes: int = 100 * 1024**3 + max_retained_ensemble_members: int = 4 + checkpoint_every_batches: int = 1 + heartbeat_every_batches: int = 1 + max_checkpoint_bytes: int = DEFAULT_MAX_CHECKPOINT_BYTES + remove_uncommitted_on_cancel: bool = True + atomic_promotion_required: bool = True + advertise_only_committed: bool = True + policy_id: str = "" + schema_version: str = RECONSTRUCTION_STORAGE_POLICY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_STORAGE_POLICY_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction storage policy schema") + for name in ( + "max_events_per_batch", + "max_inflight_batches", + "max_memory_bytes", + "max_scratch_bytes", + "max_output_bytes", + "max_retained_ensemble_members", + "checkpoint_every_batches", + "heartbeat_every_batches", + "max_checkpoint_bytes", + ): + object.__setattr__( + self, + name, + _positive_int(getattr(self, name), name), + ) + amplification = _finite_float( + self.max_candidate_amplification, + "max_candidate_amplification", + ) + if amplification < 1.0: + raise ValueError("max_candidate_amplification must be at least one") + object.__setattr__( + self, + "max_candidate_amplification", + amplification, + ) + for name in ( + "remove_uncommitted_on_cancel", + "atomic_promotion_required", + "advertise_only_committed", + ): + object.__setattr__( + self, + name, + _strict_bool(getattr(self, name), name), + ) + if not self.remove_uncommitted_on_cancel: + raise ValueError( + "v1 requires uncommitted scratch cleanup on cancel" + ) + if not self.atomic_promotion_required: + raise ValueError("v1 requires atomic promotion") + if not self.advertise_only_committed: + raise ValueError("v1 advertises only committed artifacts") + expected = _stable_id("storage-policy", self.identity_payload()) + supplied = _optional_text(self.policy_id) + if supplied is not None and supplied != expected: + raise ValueError("policy_id does not match deterministic identity") + object.__setattr__(self, "policy_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return policy fields used for deterministic identity.""" + return { + "schema_version": self.schema_version, + "max_events_per_batch": self.max_events_per_batch, + "max_candidate_amplification": (self.max_candidate_amplification), + "max_inflight_batches": self.max_inflight_batches, + "max_memory_bytes": self.max_memory_bytes, + "max_scratch_bytes": self.max_scratch_bytes, + "max_output_bytes": self.max_output_bytes, + "max_retained_ensemble_members": ( + self.max_retained_ensemble_members + ), + "checkpoint_every_batches": self.checkpoint_every_batches, + "heartbeat_every_batches": self.heartbeat_every_batches, + "max_checkpoint_bytes": self.max_checkpoint_bytes, + "remove_uncommitted_on_cancel": (self.remove_uncommitted_on_cancel), + "atomic_promotion_required": self.atomic_promotion_required, + "advertise_only_committed": self.advertise_only_committed, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible policy metadata.""" + return {**self.identity_payload(), "policy_id": self.policy_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + def preflight( + self, + estimate: "ReconstructionResourceEstimateV1", + ) -> "ReconstructionResourceEstimateV1": + """Return an accepted estimate or fail early with full evidence.""" + violations: list[str] = [] + allowed_candidates = math.floor( + estimate.input_event_count * self.max_candidate_amplification + ) + if estimate.candidate_event_count > allowed_candidates: + violations.append( + "candidate_event_count " + f"{estimate.candidate_event_count} exceeds " + f"amplification limit {allowed_candidates}" + ) + limits = ( + ( + "peak_events_per_batch", + estimate.peak_events_per_batch, + self.max_events_per_batch, + ), + ( + "retained_ensemble_members", + estimate.retained_ensemble_members, + self.max_retained_ensemble_members, + ), + ( + "inflight_batches", + estimate.inflight_batches, + self.max_inflight_batches, + ), + ( + "estimated_memory_bytes", + estimate.estimated_memory_bytes, + self.max_memory_bytes, + ), + ( + "estimated_scratch_bytes", + estimate.estimated_scratch_bytes, + self.max_scratch_bytes, + ), + ( + "estimated_output_bytes", + estimate.estimated_output_bytes, + self.max_output_bytes, + ), + ) + for name, actual, limit in limits: + if actual > limit: + violations.append(f"{name} {actual} exceeds limit {limit}") + if violations: + raise ReconstructionResourceLimitError(estimate, violations) + return estimate + + @classmethod + def from_dict( + cls, + data: Mapping[str, Any], + ) -> "ReconstructionStoragePolicyV1": + """Restore and verify a version-one storage policy.""" + _require_schema(data, RECONSTRUCTION_STORAGE_POLICY_SCHEMA_VERSION) + return cls( + max_events_per_batch=cast( + int, + data.get("max_events_per_batch"), + ), + max_candidate_amplification=cast( + float, + data.get("max_candidate_amplification"), + ), + max_inflight_batches=cast( + int, + data.get("max_inflight_batches"), + ), + max_memory_bytes=cast(int, data.get("max_memory_bytes")), + max_scratch_bytes=cast(int, data.get("max_scratch_bytes")), + max_output_bytes=cast(int, data.get("max_output_bytes")), + max_retained_ensemble_members=cast( + int, + data.get("max_retained_ensemble_members"), + ), + checkpoint_every_batches=cast( + int, + data.get("checkpoint_every_batches"), + ), + heartbeat_every_batches=cast( + int, + data.get("heartbeat_every_batches"), + ), + max_checkpoint_bytes=cast( + int, + data.get("max_checkpoint_bytes"), + ), + remove_uncommitted_on_cancel=cast( + bool, + data.get("remove_uncommitted_on_cancel", True), + ), + atomic_promotion_required=cast( + bool, + data.get("atomic_promotion_required", True), + ), + advertise_only_committed=cast( + bool, + data.get("advertise_only_committed", True), + ), + policy_id=str(data.get("policy_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionStoragePolicyV1": + """Restore a policy from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionResourceEstimateV1: + """Pre-execution estimate retained when a run is accepted or refused.""" + + input_event_count: int + candidate_event_count: int + retained_ensemble_members: int + inflight_batches: int + peak_events_per_batch: int + estimated_memory_bytes: int + estimated_scratch_bytes: int + estimated_output_bytes: int + estimated_batch_count: int + estimate_id: str = "" + schema_version: str = RECONSTRUCTION_RESOURCE_ESTIMATE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if ( + self.schema_version + != RECONSTRUCTION_RESOURCE_ESTIMATE_SCHEMA_VERSION + ): + raise ValueError( + "unsupported reconstruction resource estimate schema" + ) + for name in ( + "input_event_count", + "candidate_event_count", + "retained_ensemble_members", + "inflight_batches", + "peak_events_per_batch", + "estimated_memory_bytes", + "estimated_scratch_bytes", + "estimated_output_bytes", + "estimated_batch_count", + ): + object.__setattr__( + self, + name, + _nonnegative_int(getattr(self, name), name), + ) + expected = _stable_id("resource-estimate", self.identity_payload()) + supplied = _optional_text(self.estimate_id) + if supplied is not None and supplied != expected: + raise ValueError( + "estimate_id does not match deterministic identity" + ) + object.__setattr__(self, "estimate_id", expected) + + @property + def candidate_amplification(self) -> float: + """Return the proposed candidate/input ratio.""" + if self.input_event_count == 0: + return 0.0 if self.candidate_event_count == 0 else math.inf + return self.candidate_event_count / self.input_event_count + + def identity_payload(self) -> dict[str, JSONValue]: + """Return estimate fields used for deterministic identity.""" + return { + "schema_version": self.schema_version, + "input_event_count": self.input_event_count, + "candidate_event_count": self.candidate_event_count, + "retained_ensemble_members": self.retained_ensemble_members, + "inflight_batches": self.inflight_batches, + "peak_events_per_batch": self.peak_events_per_batch, + "estimated_memory_bytes": self.estimated_memory_bytes, + "estimated_scratch_bytes": self.estimated_scratch_bytes, + "estimated_output_bytes": self.estimated_output_bytes, + "estimated_batch_count": self.estimated_batch_count, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible estimate metadata.""" + amplification: JSONValue = self.candidate_amplification + if isinstance(amplification, float) and not math.isfinite( + amplification + ): + amplification = "infinite" + return { + **self.identity_payload(), + "estimate_id": self.estimate_id, + "candidate_amplification": amplification, + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict( + cls, + data: Mapping[str, Any], + ) -> "ReconstructionResourceEstimateV1": + """Restore and verify a version-one resource estimate.""" + _require_schema(data, RECONSTRUCTION_RESOURCE_ESTIMATE_SCHEMA_VERSION) + return cls( + input_event_count=cast(int, data.get("input_event_count")), + candidate_event_count=cast( + int, + data.get("candidate_event_count"), + ), + retained_ensemble_members=cast( + int, + data.get("retained_ensemble_members"), + ), + inflight_batches=cast(int, data.get("inflight_batches")), + peak_events_per_batch=cast( + int, + data.get("peak_events_per_batch"), + ), + estimated_memory_bytes=cast( + int, + data.get("estimated_memory_bytes"), + ), + estimated_scratch_bytes=cast( + int, + data.get("estimated_scratch_bytes"), + ), + estimated_output_bytes=cast( + int, + data.get("estimated_output_bytes"), + ), + estimated_batch_count=cast( + int, + data.get("estimated_batch_count"), + ), + estimate_id=str(data.get("estimate_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionResourceEstimateV1": + """Restore a resource estimate from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionRunV1: + """Semantic run identity separated from execution/storage tuning.""" + + symbols: tuple[str, ...] + source_version_ids: tuple[str, ...] + configuration_ids: tuple[str, ...] + ensemble_member_ids: tuple[str, ...] + base_seed: int + storage_policy: ReconstructionStoragePolicyV1 = field( + default_factory=ReconstructionStoragePolicyV1 + ) + run_id: str = "" + schema_version: str = RECONSTRUCTION_RUN_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_RUN_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction run schema") + object.__setattr__(self, "symbols", _normalized_symbols(self.symbols)) + object.__setattr__( + self, + "source_version_ids", + _normalized_id_tuple(self.source_version_ids), + ) + object.__setattr__( + self, + "configuration_ids", + _normalized_id_tuple(self.configuration_ids), + ) + object.__setattr__( + self, + "ensemble_member_ids", + _normalized_id_tuple(self.ensemble_member_ids), + ) + if len(self.symbols) > MAX_SYMBOLS_PER_SYNCHRONIZATION_UNIT: + raise ValueError("run symbol count exceeds synchronization limit") + if not self.source_version_ids: + raise ValueError("run requires source_version_ids") + if not self.configuration_ids: + raise ValueError("run requires configuration_ids") + if not self.ensemble_member_ids: + raise ValueError("run requires ensemble_member_ids") + seed = _nonnegative_int(self.base_seed, "base_seed") + if seed > UINT64_MAX: + raise ValueError("base_seed is outside unsigned 64-bit range") + object.__setattr__(self, "base_seed", seed) + if not isinstance(self.storage_policy, ReconstructionStoragePolicyV1): + raise ValueError("storage_policy must be a v1 policy") + expected = _stable_id("reconstruction-run", self.identity_payload()) + supplied = _optional_text(self.run_id) + if supplied is not None and supplied != expected: + raise ValueError("run_id does not match deterministic identity") + object.__setattr__(self, "run_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return semantic inputs; execution policy is intentionally absent.""" + return { + "schema_version": self.schema_version, + "symbols": list(self.symbols), + "source_version_ids": list(self.source_version_ids), + "configuration_ids": list(self.configuration_ids), + "ensemble_member_ids": list(self.ensemble_member_ids), + "base_seed": self.base_seed, + } + + def seed_for(self, ensemble_member_id: str, semantic_key: str) -> int: + """Derive a partition-independent seed from stable semantic lineage.""" + member_id = _required_text(ensemble_member_id) + if member_id not in self.ensemble_member_ids: + raise ValueError("ensemble member is not part of this run") + key = _required_text(semantic_key) + encoded = canonical_contract_json( + { + "run_id": self.run_id, + "ensemble_member_id": member_id, + "semantic_key": key, + "base_seed": self.base_seed, + } + ).encode("utf-8") + return int.from_bytes(hashlib.sha256(encoded).digest()[:8], "big") + + def to_dict(self) -> dict[str, JSONValue]: + """Return deterministic JSON-compatible run metadata.""" + return { + **self.identity_payload(), + "run_id": self.run_id, + "storage_policy": self.storage_policy.to_dict(), + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionRunV1": + """Restore and verify a version-one run contract.""" + _require_schema(data, RECONSTRUCTION_RUN_SCHEMA_VERSION) + return cls( + symbols=_string_tuple(data.get("symbols")), + source_version_ids=_string_tuple(data.get("source_version_ids")), + configuration_ids=_string_tuple(data.get("configuration_ids")), + ensemble_member_ids=_string_tuple(data.get("ensemble_member_ids")), + base_seed=cast(int, data.get("base_seed")), + storage_policy=ReconstructionStoragePolicyV1.from_dict( + _mapping(data.get("storage_policy")) + ), + run_id=str(data.get("run_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionRunV1": + """Restore a run from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionWindowV1: + """One synchronized, half-open generation window and its input halo.""" + + run_id: str + ensemble_member_id: str + symbols: tuple[str, ...] + core_start_ns: int + core_end_ns: int + left_halo_ns: int = 0 + right_lookahead_ns: int = 0 + window_id: str = "" + synchronization_unit_id: str = "" + schema_version: str = RECONSTRUCTION_WINDOW_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_WINDOW_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction window schema") + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, + "ensemble_member_id", + _required_text(self.ensemble_member_id), + ) + object.__setattr__(self, "symbols", _normalized_symbols(self.symbols)) + if len(self.symbols) > MAX_SYMBOLS_PER_SYNCHRONIZATION_UNIT: + raise ValueError( + "window symbol count exceeds synchronization limit" + ) + start = _bounded_int64(self.core_start_ns, "core_start_ns") + end = _bounded_int64(self.core_end_ns, "core_end_ns") + if end <= start: + raise ValueError("core_end_ns must be greater than core_start_ns") + object.__setattr__(self, "core_start_ns", start) + object.__setattr__(self, "core_end_ns", end) + for name in ("left_halo_ns", "right_lookahead_ns"): + object.__setattr__( + self, + name, + _nonnegative_int(getattr(self, name), name), + ) + _bounded_int64(self.input_start_ns, "input_start_ns") + _bounded_int64(self.input_end_ns, "input_end_ns") + expected_sync = _stable_id( + "synchronization-unit", + self.identity_payload(), + ) + supplied_sync = _optional_text(self.synchronization_unit_id) + if supplied_sync is not None and supplied_sync != expected_sync: + raise ValueError( + "synchronization_unit_id does not match deterministic identity" + ) + object.__setattr__(self, "synchronization_unit_id", expected_sync) + expected_window = _stable_id( + "reconstruction-window", self.identity_payload() + ) + supplied_window = _optional_text(self.window_id) + if supplied_window is not None and supplied_window != expected_window: + raise ValueError("window_id does not match deterministic identity") + object.__setattr__(self, "window_id", expected_window) + + @property + def input_start_ns(self) -> int: + """Return the inclusive read start including left carry context.""" + return self.core_start_ns - self.left_halo_ns + + @property + def input_end_ns(self) -> int: + """Return the exclusive read end including declared future context.""" + return self.core_end_ns + self.right_lookahead_ns + + def owns_event_time(self, event_time_ns: int) -> bool: + """Return whether this window alone may generate at the timestamp.""" + value = _bounded_int64(event_time_ns, "event_time_ns") + return self.core_start_ns <= value < self.core_end_ns + + def reads_event_time(self, event_time_ns: int) -> bool: + """Return whether the timestamp lies in the bounded input interval.""" + value = _bounded_int64(event_time_ns, "event_time_ns") + return self.input_start_ns <= value < self.input_end_ns + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic synchronization-unit identity fields.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "symbols": list(self.symbols), + "core_start_ns": self.core_start_ns, + "core_end_ns": self.core_end_ns, + "left_halo_ns": self.left_halo_ns, + "right_lookahead_ns": self.right_lookahead_ns, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded JSON-compatible window metadata.""" + return { + **self.identity_payload(), + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "input_start_ns": self.input_start_ns, + "input_end_ns": self.input_end_ns, + "ownership_interval": "[core_start_ns,core_end_ns)", + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionWindowV1": + """Restore and verify a version-one window contract.""" + _require_schema(data, RECONSTRUCTION_WINDOW_SCHEMA_VERSION) + return cls( + run_id=str(data.get("run_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbols=_string_tuple(data.get("symbols")), + core_start_ns=cast(int, data.get("core_start_ns")), + core_end_ns=cast(int, data.get("core_end_ns")), + left_halo_ns=cast(int, data.get("left_halo_ns", 0)), + right_lookahead_ns=cast( + int, + data.get("right_lookahead_ns", 0), + ), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "ReconstructionWindowV1": + """Restore a window from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +def plan_reconstruction_windows( + run: ReconstructionRunV1, + *, + ensemble_member_id: str, + start_ns: int, + end_ns: int, + window_size_ns: int, + left_halo_ns: int = 0, + right_lookahead_ns: int = 0, +) -> tuple[ReconstructionWindowV1, ...]: + """Plan contiguous synchronized windows without worker-count inputs.""" + member_id = _required_text(ensemble_member_id) + if member_id not in run.ensemble_member_ids: + raise ValueError("ensemble member is not part of this run") + start = _bounded_int64(start_ns, "start_ns") + end = _bounded_int64(end_ns, "end_ns") + if end <= start: + raise ValueError("end_ns must be greater than start_ns") + size = _positive_int(window_size_ns, "window_size_ns") + windows: list[ReconstructionWindowV1] = [] + current = start + while current < end: + next_end = min(end, current + size) + windows.append( + ReconstructionWindowV1( + run_id=run.run_id, + ensemble_member_id=member_id, + symbols=run.symbols, + core_start_ns=current, + core_end_ns=next_end, + left_halo_ns=left_halo_ns, + right_lookahead_ns=right_lookahead_ns, + ) + ) + current = next_end + return validate_reconstruction_window_plan( + windows, + expected_start_ns=start, + expected_end_ns=end, + ) + + +def validate_reconstruction_window_plan( + windows: Sequence[ReconstructionWindowV1], + *, + expected_start_ns: int | None = None, + expected_end_ns: int | None = None, +) -> tuple[ReconstructionWindowV1, ...]: + """Validate one contiguous plan with exclusive generation ownership.""" + ordered = tuple(sorted(windows, key=lambda item: item.core_start_ns)) + if not ordered: + raise ValueError("window plan cannot be empty") + first = ordered[0] + for previous, current in zip(ordered, ordered[1:]): + if ( + current.run_id != first.run_id + or current.ensemble_member_id != first.ensemble_member_id + or current.symbols != first.symbols + ): + raise ValueError("window plan synchronization scope drifted") + if previous.core_end_ns != current.core_start_ns: + raise ValueError( + "window plan must be contiguous and non-overlapping" + ) + if ( + expected_start_ns is not None + and first.core_start_ns != expected_start_ns + ): + raise ValueError("window plan does not start at expected boundary") + if ( + expected_end_ns is not None + and ordered[-1].core_end_ns != expected_end_ns + ): + raise ValueError("window plan does not end at expected boundary") + return ordered + + +@dataclass(frozen=True, slots=True) +class EventBatchV1: + """Bounded metadata for an event batch stored outside workflow history.""" + + run_id: str + window_id: str + synchronization_unit_id: str + ensemble_member_id: str + symbol: str + batch_ordinal: int + event_count: int + ownership_start_ns: int + ownership_end_ns: int + first_event_time_ns: int + last_event_time_ns: int + content_sha256: str + artifact: ArtifactRef + batch_id: str = "" + schema_version: str = EVENT_BATCH_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != EVENT_BATCH_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction event batch schema") + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "ensemble_member_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__(self, "symbol", _normalized_symbol(self.symbol)) + object.__setattr__( + self, + "batch_ordinal", + _nonnegative_int(self.batch_ordinal, "batch_ordinal"), + ) + object.__setattr__( + self, + "event_count", + _positive_int(self.event_count, "event_count"), + ) + ownership_start = _bounded_int64( + self.ownership_start_ns, + "ownership_start_ns", + ) + ownership_end = _bounded_int64( + self.ownership_end_ns, + "ownership_end_ns", + ) + if ownership_end <= ownership_start: + raise ValueError( + "ownership_end_ns must be greater than ownership_start_ns" + ) + object.__setattr__(self, "ownership_start_ns", ownership_start) + object.__setattr__(self, "ownership_end_ns", ownership_end) + first_time = _bounded_int64( + self.first_event_time_ns, + "first_event_time_ns", + ) + last_time = _bounded_int64( + self.last_event_time_ns, + "last_event_time_ns", + ) + if last_time < first_time: + raise ValueError("last_event_time_ns precedes first_event_time_ns") + if first_time < ownership_start or last_time >= ownership_end: + raise ValueError( + "event batch lies outside half-open ownership interval" + ) + object.__setattr__(self, "first_event_time_ns", first_time) + object.__setattr__(self, "last_event_time_ns", last_time) + object.__setattr__( + self, + "content_sha256", + _required_sha256(self.content_sha256, "content_sha256"), + ) + object.__setattr__( + self, "artifact", _validated_artifact_ref(self.artifact) + ) + expected = _stable_id("event-batch", self.identity_payload()) + supplied = _optional_text(self.batch_id) + if supplied is not None and supplied != expected: + raise ValueError("batch_id does not match deterministic identity") + object.__setattr__(self, "batch_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return retry/worker-independent batch identity fields.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "ensemble_member_id": self.ensemble_member_id, + "symbol": self.symbol, + "batch_ordinal": self.batch_ordinal, + "event_count": self.event_count, + "ownership_start_ns": self.ownership_start_ns, + "ownership_end_ns": self.ownership_end_ns, + "first_event_time_ns": self.first_event_time_ns, + "last_event_time_ns": self.last_event_time_ns, + "content_sha256": self.content_sha256, + "artifact": _artifact_content_identity_payload(self.artifact), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded metadata and one artifact reference, never rows.""" + return { + **self.identity_payload(), + "batch_id": self.batch_id, + "artifact": self.artifact.to_dict(), + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "EventBatchV1": + """Restore and verify a version-one event batch.""" + _require_schema(data, EVENT_BATCH_SCHEMA_VERSION) + return cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbol=str(data.get("symbol", "")), + batch_ordinal=cast(int, data.get("batch_ordinal")), + event_count=cast(int, data.get("event_count")), + ownership_start_ns=cast(int, data.get("ownership_start_ns")), + ownership_end_ns=cast(int, data.get("ownership_end_ns")), + first_event_time_ns=cast(int, data.get("first_event_time_ns")), + last_event_time_ns=cast(int, data.get("last_event_time_ns")), + content_sha256=str(data.get("content_sha256", "")), + artifact=ArtifactRef.from_dict(_mapping(data.get("artifact"))), + batch_id=str(data.get("batch_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "EventBatchV1": + """Restore a batch from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class CarryStateV1: + """Bounded cross-window watermarks and references to larger state.""" + + run_id: str + ensemble_member_id: str + symbol_watermarks_ns: dict[str, int] + last_event_ids: dict[str, str] = field(default_factory=dict) + state_artifacts: tuple[ArtifactRef, ...] = () + carry_id: str = "" + schema_version: str = CARRY_STATE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != CARRY_STATE_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction carry-state schema") + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__( + self, + "ensemble_member_id", + _required_text(self.ensemble_member_id), + ) + watermarks = { + _normalized_symbol(symbol): _bounded_int64( + watermark, + f"symbol_watermarks_ns.{symbol}", + ) + for symbol, watermark in self.symbol_watermarks_ns.items() + } + if not watermarks: + raise ValueError("carry state requires symbol watermarks") + if len(watermarks) > MAX_SYMBOLS_PER_SYNCHRONIZATION_UNIT: + raise ValueError("carry-state symbol count exceeds limit") + object.__setattr__( + self, + "symbol_watermarks_ns", + dict(sorted(watermarks.items())), + ) + last_ids = { + _normalized_symbol(symbol): _required_text(event_id) + for symbol, event_id in self.last_event_ids.items() + } + if not set(last_ids).issubset(watermarks): + raise ValueError("last_event_ids contains an unknown symbol") + object.__setattr__( + self, + "last_event_ids", + dict(sorted(last_ids.items())), + ) + object.__setattr__( + self, + "state_artifacts", + _validated_artifact_refs(self.state_artifacts), + ) + expected = _stable_id("carry-state", self.identity_payload()) + supplied = _optional_text(self.carry_id) + if supplied is not None and supplied != expected: + raise ValueError("carry_id does not match deterministic identity") + object.__setattr__(self, "carry_id", expected) + _ensure_payload_size( + self.to_dict(), + MAX_CARRY_STATE_BYTES, + "carry state", + ) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic carry-state identity fields.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "ensemble_member_id": self.ensemble_member_id, + "symbol_watermarks_ns": dict(self.symbol_watermarks_ns), + "last_event_ids": dict(self.last_event_ids), + "state_artifacts": [ + _artifact_content_identity_payload(ref) + for ref in self.state_artifacts + ], + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded scalar state and artifact references.""" + return { + **self.identity_payload(), + "carry_id": self.carry_id, + "state_artifacts": [ + artifact.to_dict() for artifact in self.state_artifacts + ], + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CarryStateV1": + """Restore and verify a version-one carry state.""" + _require_schema(data, CARRY_STATE_SCHEMA_VERSION) + return cls( + run_id=str(data.get("run_id", "")), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbol_watermarks_ns={ + str(key): cast(int, value) + for key, value in _mapping( + data.get("symbol_watermarks_ns") + ).items() + }, + last_event_ids={ + str(key): str(value) + for key, value in _mapping(data.get("last_event_ids")).items() + }, + state_artifacts=tuple( + ArtifactRef.from_dict(_mapping(item)) + for item in _sequence(data.get("state_artifacts")) + ), + carry_id=str(data.get("carry_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "CarryStateV1": + """Restore carry state from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class RejectionSummaryV1: + """Bounded aggregate rejection evidence without candidate rows.""" + + run_id: str + window_id: str + candidate_count: int + accepted_count: int + rejected_count: int + reason_counts: dict[str, int] = field(default_factory=dict) + summary_id: str = "" + schema_version: str = REJECTION_SUMMARY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != REJECTION_SUMMARY_SCHEMA_VERSION: + raise ValueError("unsupported rejection-summary schema") + object.__setattr__(self, "run_id", _required_text(self.run_id)) + object.__setattr__(self, "window_id", _required_text(self.window_id)) + for name in ("candidate_count", "accepted_count", "rejected_count"): + object.__setattr__( + self, + name, + _nonnegative_int(getattr(self, name), name), + ) + if self.accepted_count + self.rejected_count != self.candidate_count: + raise ValueError( + "accepted plus rejected must equal candidate count" + ) + reasons = { + _required_text(reason): _positive_int(count, f"reason.{reason}") + for reason, count in self.reason_counts.items() + } + if len(reasons) > MAX_REJECTION_REASONS: + raise ValueError("rejection reason count exceeds bounded limit") + if sum(reasons.values()) != self.rejected_count: + raise ValueError("reason counts must reconcile with rejected_count") + object.__setattr__(self, "reason_counts", dict(sorted(reasons.items()))) + expected = _stable_id("rejection-summary", self.identity_payload()) + supplied = _optional_text(self.summary_id) + if supplied is not None and supplied != expected: + raise ValueError("summary_id does not match deterministic identity") + object.__setattr__(self, "summary_id", expected) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic rejection-summary identity fields.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "candidate_count": self.candidate_count, + "accepted_count": self.accepted_count, + "rejected_count": self.rejected_count, + "reason_counts": dict(self.reason_counts), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded rejection counts.""" + return {**self.identity_payload(), "summary_id": self.summary_id} + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "RejectionSummaryV1": + """Restore and verify a version-one rejection summary.""" + _require_schema(data, REJECTION_SUMMARY_SCHEMA_VERSION) + return cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + candidate_count=cast(int, data.get("candidate_count")), + accepted_count=cast(int, data.get("accepted_count")), + rejected_count=cast(int, data.get("rejected_count")), + reason_counts={ + str(key): cast(int, value) + for key, value in _mapping(data.get("reason_counts")).items() + }, + summary_id=str(data.get("summary_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + + @classmethod + def from_json(cls, text: str) -> "RejectionSummaryV1": + """Restore a summary from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class PartitionManifestV1: + """Compact manifest for one all-symbol synchronization unit.""" + + run_id: str + window_id: str + synchronization_unit_id: str + ensemble_member_id: str + symbols: tuple[str, ...] + symbol_event_counts: dict[str, int] + event_batches: tuple[EventBatchV1, ...] + rejection_summary_ref: ArtifactRef + carry_state_ref: ArtifactRef + manifest_id: str = "" + schema_version: str = PARTITION_MANIFEST_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != PARTITION_MANIFEST_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction partition manifest") + for name in ( + "run_id", + "window_id", + "synchronization_unit_id", + "ensemble_member_id", + ): + object.__setattr__(self, name, _required_text(getattr(self, name))) + symbols = _normalized_symbols(self.symbols) + object.__setattr__(self, "symbols", symbols) + counts = { + _normalized_symbol(symbol): _nonnegative_int( + count, + f"symbol_event_counts.{symbol}", + ) + for symbol, count in self.symbol_event_counts.items() + } + if set(counts) != set(symbols): + raise ValueError( + "symbol event counts must cover synchronization unit" + ) + object.__setattr__( + self, + "symbol_event_counts", + dict(sorted(counts.items())), + ) + batches = tuple( + sorted( + tuple(self.event_batches), + key=lambda item: ( + item.symbol, + item.batch_ordinal, + item.batch_id, + ), + ) + ) + if len(batches) > MAX_EVENT_BATCHES_PER_MANIFEST: + raise ValueError("partition manifest batch count exceeds limit") + if len({batch.batch_id for batch in batches}) != len(batches): + raise ValueError("partition manifest contains duplicate batch IDs") + calculated_counts = dict.fromkeys(symbols, 0) + for batch in batches: + if ( + batch.run_id != self.run_id + or batch.window_id != self.window_id + or batch.synchronization_unit_id != self.synchronization_unit_id + or batch.ensemble_member_id != self.ensemble_member_id + ): + raise ValueError("event batch scope does not match manifest") + if batch.symbol not in calculated_counts: + raise ValueError("event batch symbol is outside manifest") + calculated_counts[batch.symbol] += batch.event_count + if calculated_counts != counts: + raise ValueError("event batch counts do not reconcile by symbol") + object.__setattr__(self, "event_batches", batches) + object.__setattr__( + self, + "rejection_summary_ref", + _validated_artifact_ref(self.rejection_summary_ref), + ) + object.__setattr__( + self, + "carry_state_ref", + _validated_artifact_ref(self.carry_state_ref), + ) + expected = _stable_id("partition-manifest", self.identity_payload()) + supplied = _optional_text(self.manifest_id) + if supplied is not None and supplied != expected: + raise ValueError( + "manifest_id does not match deterministic identity" + ) + object.__setattr__(self, "manifest_id", expected) + + @property + def event_count(self) -> int: + """Return total events across the synchronized symbol unit.""" + return sum(self.symbol_event_counts.values()) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic manifest identity fields.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "ensemble_member_id": self.ensemble_member_id, + "symbols": list(self.symbols), + "symbol_event_counts": dict(self.symbol_event_counts), + "event_batches": [ + batch.identity_payload() for batch in self.event_batches + ], + "rejection_summary_ref": _artifact_content_identity_payload( + self.rejection_summary_ref + ), + "carry_state_ref": _artifact_content_identity_payload( + self.carry_state_ref + ), + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return compact all-symbol manifest metadata.""" + return { + **self.identity_payload(), + "manifest_id": self.manifest_id, + "event_count": self.event_count, + "event_batches": [batch.to_dict() for batch in self.event_batches], + "rejection_summary_ref": self.rejection_summary_ref.to_dict(), + "carry_state_ref": self.carry_state_ref.to_dict(), + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "PartitionManifestV1": + """Restore and verify a version-one partition manifest.""" + _require_schema(data, PARTITION_MANIFEST_SCHEMA_VERSION) + manifest = cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + ensemble_member_id=str(data.get("ensemble_member_id", "")), + symbols=_string_tuple(data.get("symbols")), + symbol_event_counts={ + str(key): cast(int, value) + for key, value in _mapping( + data.get("symbol_event_counts") + ).items() + }, + event_batches=tuple( + EventBatchV1.from_dict(_mapping(item)) + for item in _sequence(data.get("event_batches")) + ), + rejection_summary_ref=ArtifactRef.from_dict( + _mapping(data.get("rejection_summary_ref")) + ), + carry_state_ref=ArtifactRef.from_dict( + _mapping(data.get("carry_state_ref")) + ), + manifest_id=str(data.get("manifest_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + derived_count = data.get("event_count") + if derived_count is not None and derived_count != manifest.event_count: + raise ValueError("manifest event_count does not reconcile") + return manifest + + @classmethod + def from_json(cls, text: str) -> "PartitionManifestV1": + """Restore a manifest from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +_ALLOWED_PHASE_TRANSITIONS: dict[ + ReconstructionCommitPhase, + frozenset[ReconstructionCommitPhase], +] = { + ReconstructionCommitPhase.PLANNED: frozenset( + { + ReconstructionCommitPhase.RUNNING, + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + } + ), + ReconstructionCommitPhase.RUNNING: frozenset( + { + ReconstructionCommitPhase.RUNNING, + ReconstructionCommitPhase.STAGED, + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + } + ), + ReconstructionCommitPhase.STAGED: frozenset( + { + ReconstructionCommitPhase.VALIDATED, + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + } + ), + ReconstructionCommitPhase.VALIDATED: frozenset( + { + ReconstructionCommitPhase.COMMITTED, + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + } + ), + ReconstructionCommitPhase.COMMITTED: frozenset( + {ReconstructionCommitPhase.COMMITTED} + ), + ReconstructionCommitPhase.CANCELLED: frozenset( + {ReconstructionCommitPhase.RUNNING} + ), + ReconstructionCommitPhase.FAILED: frozenset( + {ReconstructionCommitPhase.RUNNING} + ), +} + + +@dataclass(frozen=True, slots=True) +class ReconstructionCheckpointV1: + """Bounded, chained recovery state for one synchronization unit.""" + + run_id: str + window_id: str + synchronization_unit_id: str + revision: int + phase: ReconstructionCommitPhase + input_watermark_ns: int | None = None + output_watermark_ns: int | None = None + completed_batch_ids: tuple[str, ...] = () + carry_state_ref: ArtifactRef | None = None + rejection_summary_ref: ArtifactRef | None = None + staged_manifest_ref: ArtifactRef | None = None + committed_manifest_ref: ArtifactRef | None = None + parent_checkpoint_id: str | None = None + interruption_reason: str = "" + checkpoint_id: str = "" + schema_version: str = RECONSTRUCTION_CHECKPOINT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_CHECKPOINT_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction checkpoint schema") + for name in ("run_id", "window_id", "synchronization_unit_id"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "revision", + _nonnegative_int(self.revision, "revision"), + ) + object.__setattr__( + self, + "phase", + ReconstructionCommitPhase.from_value(self.phase), + ) + for name in ("input_watermark_ns", "output_watermark_ns"): + value = getattr(self, name) + if value is not None: + object.__setattr__( + self, + name, + _bounded_int64(value, name), + ) + completed_values = tuple( + _required_text(value) for value in self.completed_batch_ids + ) + if len(set(completed_values)) != len(completed_values): + raise ValueError("checkpoint completed_batch_ids must be unique") + completed = tuple(sorted(completed_values)) + object.__setattr__(self, "completed_batch_ids", completed) + for name in ( + "carry_state_ref", + "rejection_summary_ref", + "staged_manifest_ref", + "committed_manifest_ref", + ): + value = getattr(self, name) + if value is not None: + object.__setattr__(self, name, _validated_artifact_ref(value)) + object.__setattr__( + self, + "parent_checkpoint_id", + _optional_text(self.parent_checkpoint_id), + ) + object.__setattr__( + self, + "interruption_reason", + str(self.interruption_reason or "").strip(), + ) + self._validate_phase_state() + expected = _stable_id("checkpoint", self.identity_payload()) + supplied = _optional_text(self.checkpoint_id) + if supplied is not None and supplied != expected: + raise ValueError( + "checkpoint_id does not match deterministic identity" + ) + object.__setattr__(self, "checkpoint_id", expected) + + @classmethod + def planned( + cls, + window: ReconstructionWindowV1, + ) -> "ReconstructionCheckpointV1": + """Create the initial checkpoint for a planned window.""" + return cls( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + revision=0, + phase=ReconstructionCommitPhase.PLANNED, + ) + + def _validate_phase_state(self) -> None: + if self.phase is ReconstructionCommitPhase.PLANNED: + if self.revision != 0 or self.completed_batch_ids: + raise ValueError( + "planned checkpoint must be an empty revision zero" + ) + if any( + ref is not None + for ref in ( + self.carry_state_ref, + self.rejection_summary_ref, + self.staged_manifest_ref, + self.committed_manifest_ref, + ) + ): + raise ValueError("planned checkpoint cannot reference outputs") + if ( + self.phase + in { + ReconstructionCommitPhase.STAGED, + ReconstructionCommitPhase.VALIDATED, + } + and self.staged_manifest_ref is None + ): + raise ValueError( + "staged/validated checkpoint requires staged manifest" + ) + if self.phase is ReconstructionCommitPhase.COMMITTED: + if self.committed_manifest_ref is None: + raise ValueError( + "committed checkpoint requires committed manifest" + ) + if self.staged_manifest_ref is not None: + raise ValueError( + "committed checkpoint cannot advertise staged path" + ) + elif self.committed_manifest_ref is not None: + raise ValueError( + "only committed checkpoint may reference final manifest" + ) + interrupted = self.phase in { + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + } + if interrupted != bool(self.interruption_reason): + raise ValueError( + "interrupted checkpoints require a reason and active ones reject it" + ) + + @property + def advertised_manifest_ref(self) -> ArtifactRef | None: + """Return a discoverable manifest only after atomic promotion.""" + if self.phase is ReconstructionCommitPhase.COMMITTED: + return self.committed_manifest_ref + return None + + def pending_batches( + self, + batches: Sequence[EventBatchV1], + ) -> tuple[EventBatchV1, ...]: + """Deduplicate retry delivery and return only unfinished batch work.""" + unique: dict[str, EventBatchV1] = {} + for batch in batches: + self._validate_batch_scope(batch) + existing = unique.get(batch.batch_id) + if ( + existing is not None + and existing.identity_payload() != batch.identity_payload() + ): + raise ValueError( + "batch ID collision contains different metadata" + ) + if existing is None: + unique[batch.batch_id] = batch + completed = set(self.completed_batch_ids) + return tuple( + batch + for batch in sorted( + unique.values(), + key=lambda item: ( + item.symbol, + item.batch_ordinal, + item.batch_id, + ), + ) + if batch.batch_id not in completed + ) + + def transition( + self, + phase: ReconstructionCommitPhase | str, + *, + expected_checkpoint_id: str, + completed_batches: Sequence[EventBatchV1] = (), + input_watermark_ns: int | None = None, + output_watermark_ns: int | None = None, + carry_state_ref: ArtifactRef | None = None, + rejection_summary_ref: ArtifactRef | None = None, + staged_manifest_ref: ArtifactRef | None = None, + committed_manifest_ref: ArtifactRef | None = None, + interruption_reason: str = "", + ) -> "ReconstructionCheckpointV1": + """Advance with optimistic concurrency and idempotent final commit.""" + if _required_text(expected_checkpoint_id) != self.checkpoint_id: + raise ValueError("stale checkpoint transition rejected") + target = ReconstructionCommitPhase.from_value(phase) + + if self.phase is ReconstructionCommitPhase.COMMITTED: + if target is not ReconstructionCommitPhase.COMMITTED: + raise ValueError("committed checkpoint is terminal") + requested_ref = ( + _validated_artifact_ref(committed_manifest_ref) + if committed_manifest_ref is not None + else self.committed_manifest_ref + ) + if requested_ref != self.committed_manifest_ref: + raise ValueError("idempotent commit cannot change manifest") + if completed_batches: + pending = self.pending_batches(completed_batches) + if pending: + raise ValueError("committed checkpoint rejects new batches") + return self + + if target not in _ALLOWED_PHASE_TRANSITIONS[self.phase]: + raise ValueError( + f"invalid checkpoint transition {self.phase.value}->{target.value}" + ) + new_batch_ids = set(self.completed_batch_ids) + for batch in completed_batches: + self._validate_batch_scope(batch) + new_batch_ids.add(batch.batch_id) + + input_watermark = _monotonic_optional_int64( + self.input_watermark_ns, + input_watermark_ns, + "input_watermark_ns", + ) + output_watermark = _monotonic_optional_int64( + self.output_watermark_ns, + output_watermark_ns, + "output_watermark_ns", + ) + next_carry = ( + _validated_artifact_ref(carry_state_ref) + if carry_state_ref is not None + else self.carry_state_ref + ) + next_rejections = ( + _validated_artifact_ref(rejection_summary_ref) + if rejection_summary_ref is not None + else self.rejection_summary_ref + ) + next_staged = self.staged_manifest_ref + next_committed: ArtifactRef | None = None + if target is ReconstructionCommitPhase.RUNNING and self.phase in { + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + }: + next_staged = None + if target is ReconstructionCommitPhase.STAGED: + if staged_manifest_ref is None: + raise ValueError( + "staging transition requires temporary manifest" + ) + next_staged = _validated_artifact_ref(staged_manifest_ref) + elif target is ReconstructionCommitPhase.VALIDATED: + if staged_manifest_ref is not None: + candidate = _validated_artifact_ref(staged_manifest_ref) + if next_staged is not None and candidate != next_staged: + raise ValueError( + "validation cannot replace staged manifest" + ) + next_staged = candidate + elif target is ReconstructionCommitPhase.COMMITTED: + if committed_manifest_ref is None: + raise ValueError("commit transition requires promoted manifest") + next_committed = _validated_artifact_ref(committed_manifest_ref) + if next_staged is None: + raise ValueError("commit transition requires validated staging") + if ( + next_committed.sha256 != next_staged.sha256 + or next_committed.size_bytes != next_staged.size_bytes + ): + raise ValueError( + "promoted manifest bytes do not match validated staging" + ) + next_staged = None + elif committed_manifest_ref is not None: + raise ValueError("final manifest is accepted only during commit") + + reason = str(interruption_reason or "").strip() + interrupted = target in { + ReconstructionCommitPhase.CANCELLED, + ReconstructionCommitPhase.FAILED, + } + if interrupted and not reason: + raise ValueError("cancelled/failed transition requires reason") + if not interrupted and reason: + raise ValueError("active transition rejects interruption reason") + + return ReconstructionCheckpointV1( + run_id=self.run_id, + window_id=self.window_id, + synchronization_unit_id=self.synchronization_unit_id, + revision=self.revision + 1, + phase=target, + input_watermark_ns=input_watermark, + output_watermark_ns=output_watermark, + completed_batch_ids=tuple(sorted(new_batch_ids)), + carry_state_ref=next_carry, + rejection_summary_ref=next_rejections, + staged_manifest_ref=next_staged, + committed_manifest_ref=next_committed, + parent_checkpoint_id=self.checkpoint_id, + interruption_reason=reason, + ) + + def _validate_batch_scope(self, batch: EventBatchV1) -> None: + if ( + batch.run_id != self.run_id + or batch.window_id != self.window_id + or batch.synchronization_unit_id != self.synchronization_unit_id + ): + raise ValueError("event batch scope does not match checkpoint") + + def assert_within( + self, + policy: ReconstructionStoragePolicyV1, + ) -> "ReconstructionCheckpointV1": + """Fail before persistence when checkpoint metadata exceeds policy.""" + _ensure_payload_size( + self.to_dict(), + policy.max_checkpoint_bytes, + "checkpoint", + ) + return self + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic chained checkpoint identity fields.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "revision": self.revision, + "phase": self.phase.value, + "input_watermark_ns": self.input_watermark_ns, + "output_watermark_ns": self.output_watermark_ns, + "completed_batch_ids": list(self.completed_batch_ids), + "carry_state_ref": _optional_artifact_identity_payload( + self.carry_state_ref + ), + "rejection_summary_ref": _optional_artifact_identity_payload( + self.rejection_summary_ref + ), + "staged_manifest_ref": _optional_artifact_identity_payload( + self.staged_manifest_ref + ), + "committed_manifest_ref": _optional_artifact_identity_payload( + self.committed_manifest_ref + ), + "parent_checkpoint_id": self.parent_checkpoint_id, + "interruption_reason": self.interruption_reason, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return bounded checkpoint state and artifact references.""" + return { + **self.identity_payload(), + "checkpoint_id": self.checkpoint_id, + "carry_state_ref": _optional_artifact_dict(self.carry_state_ref), + "rejection_summary_ref": _optional_artifact_dict( + self.rejection_summary_ref + ), + "staged_manifest_ref": _optional_artifact_dict( + self.staged_manifest_ref + ), + "committed_manifest_ref": _optional_artifact_dict( + self.committed_manifest_ref + ), + "advertised_manifest_ref": _optional_artifact_dict( + self.advertised_manifest_ref + ), + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionCheckpointV1": + """Restore and verify a version-one checkpoint.""" + _require_schema(data, RECONSTRUCTION_CHECKPOINT_SCHEMA_VERSION) + checkpoint = cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + revision=cast(int, data.get("revision")), + phase=ReconstructionCommitPhase.from_value( + str(data.get("phase", "")) + ), + input_watermark_ns=cast( + int | None, + data.get("input_watermark_ns"), + ), + output_watermark_ns=cast( + int | None, + data.get("output_watermark_ns"), + ), + completed_batch_ids=_string_tuple(data.get("completed_batch_ids")), + carry_state_ref=_optional_artifact_from_value( + data.get("carry_state_ref") + ), + rejection_summary_ref=_optional_artifact_from_value( + data.get("rejection_summary_ref") + ), + staged_manifest_ref=_optional_artifact_from_value( + data.get("staged_manifest_ref") + ), + committed_manifest_ref=_optional_artifact_from_value( + data.get("committed_manifest_ref") + ), + parent_checkpoint_id=_optional_text( + data.get("parent_checkpoint_id") + ), + interruption_reason=str(data.get("interruption_reason", "")), + checkpoint_id=str(data.get("checkpoint_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + advertised = _optional_artifact_from_value( + data.get("advertised_manifest_ref") + ) + if advertised != checkpoint.advertised_manifest_ref: + raise ValueError("advertised manifest does not match commit phase") + return checkpoint + + @classmethod + def from_json(cls, text: str) -> "ReconstructionCheckpointV1": + """Restore a checkpoint from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +@dataclass(frozen=True, slots=True) +class ReconstructionHeartbeatV1: + """Bounded activity progress suitable for Temporal heartbeat payloads.""" + + run_id: str + window_id: str + synchronization_unit_id: str + phase: ReconstructionCommitPhase + sequence: int + completed_units: int + total_units: int + observed_event_count: int = 0 + candidate_event_count: int = 0 + accepted_event_count: int = 0 + scratch_bytes: int = 0 + output_bytes: int = 0 + checkpoint_id: str | None = None + cancellation_requested: bool = False + message: str = "" + heartbeat_id: str = "" + schema_version: str = RECONSTRUCTION_HEARTBEAT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != RECONSTRUCTION_HEARTBEAT_SCHEMA_VERSION: + raise ValueError("unsupported reconstruction heartbeat schema") + for name in ("run_id", "window_id", "synchronization_unit_id"): + object.__setattr__(self, name, _required_text(getattr(self, name))) + object.__setattr__( + self, + "phase", + ReconstructionCommitPhase.from_value(self.phase), + ) + for name in ( + "sequence", + "completed_units", + "total_units", + "observed_event_count", + "candidate_event_count", + "accepted_event_count", + "scratch_bytes", + "output_bytes", + ): + object.__setattr__( + self, + name, + _nonnegative_int(getattr(self, name), name), + ) + if self.completed_units > self.total_units: + raise ValueError("completed_units cannot exceed total_units") + object.__setattr__( + self, + "checkpoint_id", + _optional_text(self.checkpoint_id), + ) + object.__setattr__(self, "message", str(self.message or "").strip()) + object.__setattr__( + self, + "cancellation_requested", + _strict_bool( + self.cancellation_requested, + "cancellation_requested", + ), + ) + expected = _stable_id("heartbeat", self.identity_payload()) + supplied = _optional_text(self.heartbeat_id) + if supplied is not None and supplied != expected: + raise ValueError( + "heartbeat_id does not match deterministic identity" + ) + object.__setattr__(self, "heartbeat_id", expected) + _ensure_payload_size( + self.to_dict(), + MAX_HEARTBEAT_BYTES, + "heartbeat", + ) + + @property + def percent_complete(self) -> float: + """Return bounded progress percentage.""" + if self.total_units == 0: + return 0.0 + return min(100.0, self.completed_units / self.total_units * 100.0) + + def identity_payload(self) -> dict[str, JSONValue]: + """Return deterministic bounded progress fields.""" + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "window_id": self.window_id, + "synchronization_unit_id": self.synchronization_unit_id, + "phase": self.phase.value, + "sequence": self.sequence, + "completed_units": self.completed_units, + "total_units": self.total_units, + "observed_event_count": self.observed_event_count, + "candidate_event_count": self.candidate_event_count, + "accepted_event_count": self.accepted_event_count, + "scratch_bytes": self.scratch_bytes, + "output_bytes": self.output_bytes, + "checkpoint_id": self.checkpoint_id, + "cancellation_requested": self.cancellation_requested, + "message": self.message, + } + + def to_dict(self) -> dict[str, JSONValue]: + """Return GUI/Temporal-ready progress metadata without rows.""" + return { + **self.identity_payload(), + "heartbeat_id": self.heartbeat_id, + "event_type": "reconstruction_progress", + "percent_complete": self.percent_complete, + "stops_future_work_on_cancel": True, + "resume_mode": "last_valid_checkpoint", + } + + def to_json(self) -> str: + """Return deterministic compact JSON.""" + return str(canonical_contract_json(self.to_dict())) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "ReconstructionHeartbeatV1": + """Restore and verify a version-one heartbeat.""" + _require_schema(data, RECONSTRUCTION_HEARTBEAT_SCHEMA_VERSION) + heartbeat = cls( + run_id=str(data.get("run_id", "")), + window_id=str(data.get("window_id", "")), + synchronization_unit_id=str( + data.get("synchronization_unit_id", "") + ), + phase=ReconstructionCommitPhase.from_value( + str(data.get("phase", "")) + ), + sequence=cast(int, data.get("sequence")), + completed_units=cast(int, data.get("completed_units")), + total_units=cast(int, data.get("total_units")), + observed_event_count=cast( + int, + data.get("observed_event_count", 0), + ), + candidate_event_count=cast( + int, + data.get("candidate_event_count", 0), + ), + accepted_event_count=cast( + int, + data.get("accepted_event_count", 0), + ), + scratch_bytes=cast(int, data.get("scratch_bytes", 0)), + output_bytes=cast(int, data.get("output_bytes", 0)), + checkpoint_id=_optional_text(data.get("checkpoint_id")), + cancellation_requested=cast( + bool, + data.get("cancellation_requested", False), + ), + message=str(data.get("message", "")), + heartbeat_id=str(data.get("heartbeat_id", "")), + schema_version=str(data.get("schema_version", "")), + ) + if data.get("event_type", "reconstruction_progress") != ( + "reconstruction_progress" + ): + raise ValueError("unsupported heartbeat event_type") + return heartbeat + + @classmethod + def from_json(cls, text: str) -> "ReconstructionHeartbeatV1": + """Restore a heartbeat from deterministic JSON.""" + return cls.from_dict(_json_mapping(text)) + + +def artifact_ref_for_json_contract( + contract: Any, + *, + kind: str, + path: str, + metadata: Mapping[str, JSONValue] | None = None, +) -> ArtifactRef: + """Build a strong artifact reference for deterministic contract JSON.""" + serializer = getattr(contract, "to_json", None) + if not callable(serializer): + raise ValueError("contract must provide to_json()") + encoded = str(serializer()).encode("utf-8") + return _validated_artifact_ref( + ArtifactRef( + kind=_required_text(kind), + path=_required_text(path), + size_bytes=len(encoded), + sha256=hashlib.sha256(encoded).hexdigest(), + metadata=dict(metadata or {}), + ) + ) + + +def _stable_id(prefix: str, payload: Mapping[str, JSONValue]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + return f"{prefix}:sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _required_text(value: Any) -> str: + normalized = str(value).strip() if value is not None else "" + if not normalized: + raise ValueError("required text value is empty") + return normalized + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _normalized_symbol(value: Any) -> str: + return _required_text(value).lower() + + +def _normalized_symbols(values: Sequence[str]) -> tuple[str, ...]: + normalized = tuple(sorted({_normalized_symbol(value) for value in values})) + if not normalized: + raise ValueError("at least one symbol is required") + return normalized + + +def _normalized_id_tuple(values: Sequence[str]) -> tuple[str, ...]: + return tuple(sorted({_required_text(value) for value in values})) + + +def _strict_bool(value: Any, name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{name} must be a boolean") + return value + + +def _positive_int(value: Any, name: str) -> int: + normalized = _nonnegative_int(value, name) + if normalized < 1: + raise ValueError(f"{name} must be positive") + return normalized + + +def _nonnegative_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + if value < 0: + raise ValueError(f"{name} must be non-negative") + return value + + +def _bounded_int64(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + if not INT64_MIN <= value <= INT64_MAX: + raise ValueError(f"{name} is outside signed 64-bit range") + return value + + +def _finite_float(value: Any, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{name} must be finite") + return number + + +def _required_sha256(value: Any, name: str) -> str: + normalized = _required_text(value).lower() + if not _SHA256_RE.fullmatch(normalized): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return normalized + + +def _validated_artifact_ref(value: ArtifactRef) -> ArtifactRef: + if not isinstance(value, ArtifactRef): + raise ValueError("artifact reference must be an ArtifactRef") + size = _nonnegative_int(value.size_bytes, "artifact.size_bytes") + metadata = dict(value.metadata) + _validate_json_value(metadata, "artifact.metadata") + _reject_inline_data(metadata) + _ensure_payload_size( + metadata, + MAX_ARTIFACT_METADATA_BYTES, + "artifact metadata", + ) + return ArtifactRef( + kind=_required_text(value.kind), + path=_required_text(value.path), + size_bytes=size, + sha256=_required_sha256(value.sha256, "artifact.sha256"), + metadata=metadata, + ) + + +def _validated_artifact_refs( + values: Sequence[ArtifactRef], +) -> tuple[ArtifactRef, ...]: + if len(values) > MAX_ARTIFACT_REFS_PER_CONTRACT: + raise ValueError("artifact reference count exceeds bounded limit") + normalized = tuple(_validated_artifact_ref(value) for value in values) + if len({(ref.kind, ref.path, ref.sha256) for ref in normalized}) != len( + normalized + ): + raise ValueError("duplicate artifact reference") + return tuple( + sorted(normalized, key=lambda ref: (ref.kind, ref.path, ref.sha256)) + ) + + +def _artifact_identity_payload(value: ArtifactRef) -> dict[str, JSONValue]: + return { + "kind": value.kind, + "path": value.path, + "size_bytes": value.size_bytes, + "sha256": value.sha256, + } + + +def _artifact_content_identity_payload( + value: ArtifactRef, +) -> dict[str, JSONValue]: + """Return content identity without worker-local artifact placement.""" + return { + "kind": value.kind, + "size_bytes": value.size_bytes, + "sha256": value.sha256, + } + + +def _optional_artifact_identity_payload( + value: ArtifactRef | None, +) -> dict[str, JSONValue] | None: + if value is None: + return None + return _artifact_identity_payload(value) + + +def _optional_artifact_dict( + value: ArtifactRef | None, +) -> dict[str, JSONValue] | None: + return value.to_dict() if value is not None else None + + +def _optional_artifact_from_value(value: Any) -> ArtifactRef | None: + if value is None: + return None + return ArtifactRef.from_dict(_mapping(value)) + + +def _reject_inline_data(value: JSONValue) -> None: + if isinstance(value, dict): + forbidden = _FORBIDDEN_INLINE_DATA_KEYS.intersection(value) + if forbidden: + raise ValueError( + "artifact metadata cannot contain inline data keys: " + + ", ".join(sorted(forbidden)) + ) + for nested in value.values(): + _reject_inline_data(nested) + elif isinstance(value, list): + for nested in value: + _reject_inline_data(nested) + + +def _validate_json_value(value: Any, path: str) -> None: + if value is None or isinstance(value, (str, bool, int)): + return + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError(f"{path} must contain only finite numbers") + return + if isinstance(value, list): + for index, nested in enumerate(value): + _validate_json_value(nested, f"{path}[{index}]") + return + if isinstance(value, dict): + for key, nested in value.items(): + if not isinstance(key, str): + raise ValueError(f"{path} keys must be strings") + _validate_json_value(nested, f"{path}.{key}") + return + raise ValueError(f"{path} contains unsupported JSON value") + + +def _monotonic_optional_int64( + previous: int | None, + requested: int | None, + name: str, +) -> int | None: + if requested is None: + return previous + normalized = _bounded_int64(requested, name) + if previous is not None and normalized < previous: + raise ValueError(f"{name} cannot move backwards") + return normalized + + +def _ensure_payload_size( + payload: Mapping[str, JSONValue], + limit: int, + name: str, +) -> None: + size = len(canonical_contract_json(payload).encode("utf-8")) + if size > limit: + raise ValueError(f"{name} payload {size} bytes exceeds limit {limit}") + + +def _require_schema(data: Mapping[str, Any], expected: str) -> None: + if str(data.get("schema_version", "")) != expected: + raise ValueError(f"unsupported schema version; expected {expected}") + + +def _mapping(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("expected a JSON object") + return value + + +def _sequence(value: Any) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError("expected a JSON array") + return value + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in _sequence(value)) + + +def _json_mapping(text: str) -> Mapping[str, Any]: + value = json.loads(text) + return _mapping(value) diff --git a/src/histdatacom/utils.py b/src/histdatacom/utils.py index a238b84d..ffe41e8d 100644 --- a/src/histdatacom/utils.py +++ b/src/histdatacom/utils.py @@ -37,6 +37,7 @@ } SUPPORTED_API_RETURN_TYPES = frozenset(API_RETURN_TYPE_MODULES) +OUTPUT_DATETIME_COLUMN = "datetime_local" def get_month_from_datemonth( @@ -222,6 +223,25 @@ def normalize_api_return_type(return_type: Optional[str]) -> Optional[str]: return normalized +def normalize_output_timezone(timezone_name: Optional[str]) -> str: + """Normalize and validate an optional IANA output timezone name.""" + if not timezone_name: + return "" + + normalized = timezone_name.strip() + if not normalized: + return "" + + try: + pytz.timezone(normalized) + except pytz.UnknownTimeZoneError as err: + raise ValueError( + f"unsupported output timezone '{timezone_name}'. " + "Use an IANA timezone name such as UTC or America/New_York." + ) from err + return normalized + + def check_installed_module( # noqa:CCR001,BLK100 module_name: str, load: bool = False ) -> bool: diff --git a/tests/architecture/classes_pyreverse.svg b/tests/architecture/classes_pyreverse.svg index 550ffc5f..591c74fd 100644 --- a/tests/architecture/classes_pyreverse.svg +++ b/tests/architecture/classes_pyreverse.svg @@ -4,6183 +4,18317 @@ - - + + classes_pyreverse - + histdatacom.APICaller - -APICaller - - -__call__(options: Options): 'list' | 'PolarsDataFrame' | 'DataFrame' | 'Table' + +APICaller + + +__call__(options: Options): 'list' | 'PolarsDataFrame' | 'DataFrame' | 'Table' - + +histdatacom.synthetic.activity.ActivityAggregationSemantics + +ActivityAggregationSemantics + +name + +from_value(value: str | 'ActivityAggregationSemantics'): 'ActivityAggregationSemantics' + + + histdatacom.orchestration.workflows.ActivityExecutionPolicy - -ActivityExecutionPolicy - -activity_name : str -heartbeat_timeout_seconds : int -retry_policy -start_to_close_timeout_seconds : int - -__init__(self, activity_name: str, start_to_close_timeout_seconds: int, heartbeat_timeout_seconds: int, retry_policy: ActivityRetryPolicy): None -to_dict(): dict[str, JSONValue] + +ActivityExecutionPolicy + +activity_name : str +heartbeat_timeout_seconds : int +retry_policy +start_to_close_timeout_seconds : int + +__init__(self, activity_name: str, start_to_close_timeout_seconds: int, heartbeat_timeout_seconds: int, retry_policy: ActivityRetryPolicy): None +to_dict(): dict[str, JSONValue] - + histdatacom.exceptions.ActivityRetryPolicy - -ActivityRetryPolicy - -backoff_coefficient : float -initial_interval_seconds : float -maximum_attempts : int -maximum_interval_seconds : float -name -non_retryable_error_types : tuple[str, ...] -retryable : bool - -__init__(self, name: RetryPolicyName, retryable: bool, maximum_attempts: int, initial_interval_seconds: float, backoff_coefficient: float, maximum_interval_seconds: float, non_retryable_error_types: tuple[str, ...]): None -to_dict(): dict[str, JSONValue] + +ActivityRetryPolicy + +backoff_coefficient : float +initial_interval_seconds : float +maximum_attempts : int +maximum_interval_seconds : float +name +non_retryable_error_types : tuple[str, ...] +retryable : bool + +__init__(self, name: RetryPolicyName, retryable: bool, maximum_attempts: int, initial_interval_seconds: float, backoff_coefficient: float, maximum_interval_seconds: float, non_retryable_error_types: tuple[str, ...]): None +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.workflows.ActivityExecutionPolicy->histdatacom.exceptions.ActivityRetryPolicy - - -retry_policy + + +retry_policy - + histdatacom.orchestration.workflows.ActivityExecutor - -ActivityExecutor - - -execute_activity -(activity_name: str, payload: Mapping[str, JSONValue]): Mapping[str, Any] + +ActivityExecutor + + +execute_activity +(activity_name: str, payload: Mapping[str, JSONValue]): Mapping[str, Any] + + + +histdatacom.synthetic.activity.ActivityMetricSemantics + +ActivityMetricSemantics + +name + +from_value(value: str | 'ActivityMetricSemantics'): 'ActivityMetricSemantics' - + histdatacom.exceptions.RetryPolicyName - -RetryPolicyName - -name - - + +RetryPolicyName + +name + + - + histdatacom.exceptions.ActivityRetryPolicy->histdatacom.exceptions.RetryPolicyName - - -name + + +name + + + +histdatacom.synthetic.activity.ActivitySliceScope + +ActivitySliceScope + +name + +from_value(value: str | 'ActivitySliceScope'): 'ActivitySliceScope' - + histdatacom.activity_stages.ActivityStageOutput - -ActivityStageOutput - -forward : bool -result -work_item - -__init__(self, work_item: WorkItem, result: StageResult, forward: bool): None -to_dict(): dict[str, Any] + +ActivityStageOutput + +forward : bool +result +work_item + +__init__(self, work_item: WorkItem, result: StageResult, forward: bool): None +to_dict(): dict[str, Any] - + histdatacom.runtime_contracts.StageResult - -StageResult - -artifacts : tuple[ArtifactRef, ...] -events : tuple[StatusEvent, ...] -failure : FailureInfo | None -metrics : dict[str, JSONValue] -stage : str -status -work_id : str - -__init__(self, work_id: str, stage: str, status: WorkStatus, artifacts: tuple[ArtifactRef, ...], events: tuple[StatusEvent, ...], failure: FailureInfo | None, metrics: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'StageResult' -to_dict(): dict[str, JSONValue] + +StageResult + +artifacts : tuple[ArtifactRef, ...] +events : tuple[StatusEvent, ...] +failure : FailureInfo | None +metrics : dict[str, JSONValue] +stage : str +status +work_id : str + +__init__(self, work_id: str, stage: str, status: WorkStatus, artifacts: tuple[ArtifactRef, ...], events: tuple[StatusEvent, ...], failure: FailureInfo | None, metrics: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'StageResult' +to_dict(): dict[str, JSONValue] - + histdatacom.activity_stages.ActivityStageOutput->histdatacom.runtime_contracts.StageResult - - -result + + +result - + histdatacom.runtime_contracts.WorkItem - -WorkItem - -bytes_length : str -cache_end : str -cache_filename : str -cache_line_count : str -cache_start : str -csv_filename : str -data_date : str -data_datemonth : str -data_dir : str -data_format : str -data_fxpair : str -data_month : str -data_timeframe : str -data_tk : str -data_year : str -encoding : str -legacy_status : str -metadata : dict[str, JSONValue] -status -status_text : str -url : str -work_id : str -zip_filename : str -zip_persist : str - -__init__(self, work_id: str, status: WorkStatus, status_text: str, url: str, encoding: str, bytes_length: str, data_date: str, data_year: str, data_month: str, data_datemonth: str, data_format: str, data_timeframe: str, data_fxpair: str, data_dir: str, data_tk: str, zip_filename: str, csv_filename: str, cache_filename: str, cache_line_count: str, cache_start: str, cache_end: str, zip_persist: str, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'WorkItem' -from_record(record: Any, work_id: str): 'WorkItem' -to_dict(): dict[str, JSONValue] -to_record_kwargs(): dict[str, str] -with_status(status: WorkStatus | str): 'WorkItem' + +WorkItem + +bytes_length : str +cache_end : str +cache_filename : str +cache_line_count : str +cache_start : str +csv_filename : str +data_date : str +data_datemonth : str +data_dir : str +data_format : str +data_fxpair : str +data_month : str +data_timeframe : str +data_tk : str +data_year : str +encoding : str +legacy_status : str +metadata : dict[str, JSONValue] +status +status_text : str +url : str +work_id : str +zip_filename : str +zip_persist : str + +__init__(self, work_id: str, status: WorkStatus, status_text: str, url: str, encoding: str, bytes_length: str, data_date: str, data_year: str, data_month: str, data_datemonth: str, data_format: str, data_timeframe: str, data_fxpair: str, data_dir: str, data_tk: str, zip_filename: str, csv_filename: str, cache_filename: str, cache_line_count: str, cache_start: str, cache_end: str, zip_persist: str, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'WorkItem' +from_record(record: Any, work_id: str): 'WorkItem' +to_dict(): dict[str, JSONValue] +to_record_kwargs(): dict[str, str] +with_status(status: WorkStatus | str): 'WorkItem' - + histdatacom.activity_stages.ActivityStageOutput->histdatacom.runtime_contracts.WorkItem - - -work_item + + +work_item + + + +histdatacom.synthetic.activity.ActivityVolumeState + +ActivityVolumeState + +name + +from_value(value: str | 'ActivityVolumeState'): 'ActivityVolumeState' - + histdatacom.data_analytics.feed_regimes.AnalyticsDiscoveryResult - -AnalyticsDiscoveryResult - -metadata : dict[str, JSONValue] -roots : tuple[str, ...] -target_count : int -targets : tuple[AnalyticsTarget, ...] - -__init__(self, roots: tuple[str, ...], targets: tuple[AnalyticsTarget, ...], metadata: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +AnalyticsDiscoveryResult + +metadata : dict[str, JSONValue] +roots : tuple[str, ...] +target_count : int +targets : tuple[AnalyticsTarget, ...] + +__init__(self, roots: tuple[str, ...], targets: tuple[AnalyticsTarget, ...], metadata: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_analytics.feed_regimes.AnalyticsTarget - -AnalyticsTarget - -data_format : str -is_supported_tick_target : bool -kind : str -metadata : dict[str, JSONValue] -path : str -period : str -symbol : str -timeframe : str - -__init__(self, path: str, kind: str, data_format: str, timeframe: str, symbol: str, period: str, metadata: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +AnalyticsTarget + +data_format : str +is_supported_tick_target : bool +kind : str +metadata : dict[str, JSONValue] +path : str +period : str +symbol : str +timeframe : str + +__init__(self, path: str, kind: str, data_format: str, timeframe: str, symbol: str, period: str, metadata: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] - + histdatacom.api.Api - -Api - -args : dict[str, Any] -return_type : str - -__init__(args: Mapping[str, Any] | None): None -_collate_sets_to_merge(records_to_merge: list): list -_create_cache(record: 'Record', args: dict): None -_extract_single_value_from_frame(frame: 'PolarsDataFrame', row: int, column: str): int -_import_file_to_polars(record: Record, zip_path: Path): 'PolarsDataFrame' -_import_frame_with_headers(timeframe: str, zip_path: Path): 'PolarsDataFrame' -_merge_records(tp_set_dict: dict): None -_validate_cache(record: Record, args: dict): Record | None -_write_cache_data(data_frame: 'PolarsDataFrame', file_path: str): None -import_cache_data(cache_path: str): 'PolarsDataFrame' -merge_caches(records_to_merge: list[Record] | None): list | PolarsDataFrame | DataFrame | Table -merge_records(records_to_merge: list): list | PolarsDataFrame | DataFrame | Table -test_for_cache_or_create(record: Record, args: dict): None -validate_caches(records: list[Record], args: Mapping[str, Any] | None): list[Record] + +Api + +args : dict[str, Any] +output_timezone : str +return_type : str + +__init__(args: Mapping[str, Any] | None): None +_collate_sets_to_merge(records_to_merge: list): list +_create_cache(record: 'Record', args: dict): None +_extract_single_value_from_frame(frame: 'PolarsDataFrame', row: int, column: str): int +_import_file_to_polars(record: Record, zip_path: Path): 'PolarsDataFrame' +_import_frame_with_headers(timeframe: str, zip_path: Path): 'PolarsDataFrame' +_merge_records(tp_set_dict: dict): None +_validate_cache(record: Record, args: dict): Record | None +_write_cache_data(data_frame: 'PolarsDataFrame', file_path: str): None +import_cache_data(cache_path: str): 'PolarsDataFrame' +merge_caches(records_to_merge: list[Record] | None): list | PolarsDataFrame | DataFrame | Table +merge_records(records_to_merge: list): list | PolarsDataFrame | DataFrame | Table +test_for_cache_or_create(record: Record, args: dict): None +validate_caches(records: list[Record], args: Mapping[str, Any] | None): list[Record] + + + +histdatacom.broker_capture.storage.AppendOnlyBrokerCaptureWriterV1 + +AppendOnlyBrokerCaptureWriterV1 + +_closed : bool +_event_kind_counts : dict[str, int] +_file : BinaryIO | None, NoneType +_last_monotonic_ns : int | None +_manifest +_partial_path : NoneType, Path | None +_partition_bytes : int +_partition_event_count : int +_partition_first_event : BrokerCaptureEventV1 | None, NoneType +_partition_kind_counts : dict[str, int] +_partition_last_event : BrokerCaptureEventV1 | None, NoneType +_partitions : list[BrokerCapturePartitionManifestV1] +_total_events : int +manifest : BrokerCaptureSessionManifestV1 +root : Path +session +session_directory +storage_policy + +__enter__(): 'AppendOnlyBrokerCaptureWriterV1' +__exit__(exc_type: object, exc: object, traceback: object): None +__init__(root: str | Path): None +_enforce_storage_limits(line_bytes: int, starting_partition: bool): None +_finalize_partition(): None +_open_partition(): None +_publish_session_manifest(state: BrokerCaptureSessionState): BrokerCaptureSessionManifestV1 +_should_rotate(event: BrokerCaptureEventV1, line_bytes: int): bool +append(event: BrokerCaptureEventV1): None +close(): BrokerCaptureSessionManifestV1 + + + +histdatacom.broker_capture.contracts.BrokerCapturePartitionManifestV1 + +BrokerCapturePartitionManifestV1 + +completed : bool +data_artifact +event_count : int +event_kind_counts : dict[str, int] +first_capture_sequence : int +first_receive_time_monotonic_ns : int +first_receive_time_utc_ns : int +last_capture_sequence : int +last_receive_time_monotonic_ns : int +last_receive_time_utc_ns : int +partition_id : str +partition_ordinal : int +policy_id : str +schema_version : str +session_id : str + +__init__(self, session_id: str, policy_id: str, partition_ordinal: int, data_artifact: ArtifactRef, event_count: int, first_capture_sequence: int, last_capture_sequence: int, first_receive_time_utc_ns: int, last_receive_time_utc_ns: int, first_receive_time_monotonic_ns: int, last_receive_time_monotonic_ns: int, event_kind_counts: dict[str, int], completed: bool, partition_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerCapturePartitionManifestV1' +from_json(text: str): 'BrokerCapturePartitionManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.storage.AppendOnlyBrokerCaptureWriterV1->histdatacom.broker_capture.contracts.BrokerCapturePartitionManifestV1 + + +_partitions - + histdatacom.exceptions.ArchiveDownloadError - -ArchiveDownloadError - - - + +ArchiveDownloadError + + + - + histdatacom.exceptions.ArchiveError - -ArchiveError - -category : ARCHIVE -code : str -retryable : bool - -__init__(code: str, message: str): None + +ArchiveError + +category : ARCHIVE +code : str +retryable : bool + +__init__(code: str, message: str): None - + histdatacom.exceptions.ArchiveDownloadError->histdatacom.exceptions.ArchiveError - - + + - + histdatacom.activity_stages.ArchiveDownloadResult - -ArchiveDownloadResult - -artifact_kind : str -filename : str -path : str -reused_existing : bool -sha256 : str -size_bytes : int - -__init__(self, filename: str, path: str, size_bytes: int, sha256: str, reused_existing: bool, artifact_kind: str): None -to_dict(): dict[str, JSONValue] + +ArchiveDownloadResult + +artifact_kind : str +filename : str +path : str +reused_existing : bool +sha256 : str +size_bytes : int + +__init__(self, filename: str, path: str, size_bytes: int, sha256: str, reused_existing: bool, artifact_kind: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.exceptions.HistDataOperationError - -HistDataOperationError - -category : UNKNOWN -code : str -detail : dict -exit_code : int | None -message : str -retryable : bool - -__init__(message: str): None -to_dict(): dict[str, JSONValue] -to_failure_info(): FailureInfo -to_reportable_error(): ReportableError + +HistDataOperationError + +category : UNKNOWN +code : str +detail : dict +exit_code : int | None +message : str +retryable : bool + +__init__(message: str): None +to_dict(): dict[str, JSONValue] +to_failure_info(): FailureInfo +to_reportable_error(): ReportableError - + histdatacom.exceptions.ArchiveError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.exceptions.ArchiveExtractionError - -ArchiveExtractionError - - - + +ArchiveExtractionError + + + - + histdatacom.exceptions.ArchiveExtractionError->histdatacom.exceptions.ArchiveError - - + + - + histdatacom.activity_stages.ArchiveExtractionResult - -ArchiveExtractionResult - -filename : str -path : str -reused_existing : bool -sha256 : str -size_bytes : int -zip_deleted : bool - -__init__(self, filename: str, path: str, size_bytes: int, sha256: str, reused_existing: bool, zip_deleted: bool): None -to_dict(): dict[str, JSONValue] + +ArchiveExtractionResult + +filename : str +path : str +reused_existing : bool +sha256 : str +size_bytes : int +zip_deleted : bool + +__init__(self, filename: str, path: str, size_bytes: int, sha256: str, reused_existing: bool, zip_deleted: bool): None +to_dict(): dict[str, JSONValue] - + histdatacom.cli.ArgParser - -ArgParser - -_default_args -arg_namespace -formatter_class - -__call__(): Options -__init__(options: Options | None) -_adjust_for_repo_data_request(): None -_check_cli_end_yearmonth(): None -_check_cli_start_yearmonth(): Tuple[Optional[str], Optional[str]] -_check_datetime_input(): None -_check_end_yearmonth_in_range(): None -_check_for_ascii_if_api(): None -_check_for_ascii_if_influx(): None -_check_for_now_in_yearmonth(): Tuple[Optional[str], Optional[str]] -_check_for_same_start_yearmonth(): None -_check_for_start_in_yearmonth(): Tuple[Optional[str], Optional[str]] | Any -_check_for_supported_cache_dimensions(): None -_check_for_supported_format_timeframe_combination(): None -_check_quality_mode(): None -_check_start_lessthan_end(): None -_check_start_yearmonth_in_range(): None -_clean_from_api_args(): list -_config_args_from_cli(cli_args: list[str]): list[str] -_expand_pair_groups(): None -_load_quality_profile(): None -_replace_falsey_yearmonth_with_none(): Tuple[Optional[str], Optional[str]] -_sanitize_input(): None -_set_args(): None -_validate_prerequisites(): None -_validate_yearmonth_format(yearmonth: str): str | Any -arg_list_to_set(args: dict): dict + +ArgParser + +_config_file_args : Tuple[str, ...], tuple +_default_args +_explicit_cli_args : Tuple[str, ...], tuple +arg_namespace +formatter_class + +__call__(): Options +__init__(options: Options | None) +_adjust_for_repo_data_request(): None +_check_cli_end_yearmonth(): None +_check_cli_start_yearmonth(): Tuple[Optional[str], Optional[str]] +_check_datetime_input(): None +_check_end_yearmonth_in_range(): None +_check_for_ascii_if_api(): None +_check_for_ascii_if_influx(): None +_check_for_now_in_yearmonth(): Tuple[Optional[str], Optional[str]] +_check_for_same_start_yearmonth(): None +_check_for_start_in_yearmonth(): Tuple[Optional[str], Optional[str]] | Any +_check_for_supported_cache_dimensions(): None +_check_for_supported_format_timeframe_combination(): None +_check_quality_mode(): None +_check_random_window_input(): None +_check_start_lessthan_end(): None +_check_start_yearmonth_in_range(): None +_clean_from_api_args(): list +_config_args_from_cli(cli_args: list[str]): list[str] +_expand_pair_groups(): None +_load_quality_profile(): None +_replace_falsey_yearmonth_with_none(): Tuple[Optional[str], Optional[str]] +_sanitize_input(): None +_set_args(): None +_validate_prerequisites(): None +_validate_yearmonth_format(yearmonth: str): str | Any +arg_list_to_set(args: dict): dict - + histdatacom.options.Options - -Options - -__slots__ : tuple -api_return_type : str | None -available_remote_data : bool -batch_size : str -build_cache : bool -by : str -config_path : str | None -cpu_utilization : str -data_directory : str -data_quality : bool -delete_after_influx : bool -download_data_archives : bool -end_yearmonth : NoneType, str | None -extract_csvs : bool -formats : set -from_api : bool -import_to_influxdb : bool -metadata : dict -no_overlap : bool -orchestration_keep_runtime : bool -orchestration_start : bool -orchestration_wait_result : bool -pair_groups : set -pairs : set -quality_check_groups : set[str] -quality_fail_on : str -quality_max_errors : int -quality_max_warnings : int -quality_paths : tuple[str, ...] -quality_preflight : bool -quality_preflight_evidence_allow_stale : bool -quality_preflight_evidence_max_age_seconds : int -quality_preflight_evidence_path : str | None -quality_preflight_markdown : bool -quality_preflight_markdown_report_path : str | None -quality_preflight_profile_preview_format : str -quality_preflight_profile_preview_output_path : str | None -quality_preflight_report_path : str | None -quality_preflight_run_validation : bool -quality_preflight_sample_size : int -quality_preflight_validation_report_path : str | None -quality_profile : dict -quality_profile_path : str | None -quality_profile_preview : bool -quality_profile_preview_format : str -quality_profile_preview_output_path : str | None -quality_remediation_catalog_audit : bool -quality_report_path : str | None -repo_quality_columns : bool -repo_quality_refresh : bool -request_bundle_out : str | None -request_json_out : str | None -schedule_key : str -start_yearmonth : NoneType, str | None -timeframes : set -update_remote_data : bool -use_orchestration : bool -validate_urls : bool -verbosity : int -version : bool -zip_persist : bool - -__init__(): None -to_dict(): dict[str, object] + +Options + +__slots__ : tuple +api_return_type : str | None +available_remote_data : bool +batch_size : str +build_cache : bool +by : str +config_path : str | None +cpu_utilization : str +data_directory : str +data_quality : bool +delete_after_influx : bool +download_data_archives : bool +end_yearmonth : NoneType, str | None +extract_csvs : bool +formats : set +from_api : bool +import_to_influxdb : bool +metadata : dict +no_overlap : bool +orchestration_keep_runtime : bool +orchestration_start : bool +orchestration_wait_result : bool +output_timezone : str +pair_groups : set +pairs : set +quality_check_groups : set[str] +quality_fail_on : str +quality_max_errors : int +quality_max_warnings : int +quality_paths : tuple[str, ...] +quality_preflight : bool +quality_preflight_evidence_allow_stale : bool +quality_preflight_evidence_max_age_seconds : int +quality_preflight_evidence_path : str | None +quality_preflight_markdown : bool +quality_preflight_markdown_report_path : str | None +quality_preflight_profile_preview_format : str +quality_preflight_profile_preview_output_path : str | None +quality_preflight_report_path : str | None +quality_preflight_run_validation : bool +quality_preflight_sample_size : int +quality_preflight_validation_evidence_path : str | None +quality_preflight_validation_report_path : str | None +quality_profile : dict +quality_profile_path : str | None +quality_profile_preview : bool +quality_profile_preview_format : str +quality_profile_preview_output_path : str | None +quality_profile_resolution : dict +quality_remediation_catalog_audit : bool +quality_report_path : str | None +random_seed : int | None +random_window : str +repo_quality_columns : bool +repo_quality_refresh : bool +request_bundle_out : str | None +request_json_out : str | None +schedule_key : str +start_yearmonth : NoneType, str | None +timeframes : set +update_remote_data : bool +use_orchestration : bool +validate_urls : bool +verbosity : int +version : bool +zip_persist : bool + +__init__(): None +to_dict(): dict[str, object] - + histdatacom.cli.ArgParser->histdatacom.options.Options - - -arg_namespace + + +arg_namespace - + histdatacom.runtime_contracts.ArtifactRef - -ArtifactRef - -kind : str -metadata : dict[str, JSONValue] -path : str -sha256 : str -size_bytes : int | None - -__init__(self, kind: str, path: str, size_bytes: int | None, sha256: str, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'ArtifactRef' -to_dict(): dict[str, JSONValue] + +ArtifactRef + +kind : str +metadata : dict[str, JSONValue] +path : str +sha256 : str +size_bytes : int | None + +__init__(self, kind: str, path: str, size_bytes: int | None, sha256: str, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'ArtifactRef' +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_quality.autoregressive.AutoregressiveProfile + +AutoregressiveProfile + +baseline_rolling_windows : tuple[int, ...] +compare_exponential_smoothing : bool +enabled : bool +projection_horizon : int +projection_specification_ids : tuple[str, ...] +rounding_digits : int +specifications : tuple[AutoregressiveSpecification, ...] + +__init__(self, enabled: bool, specifications: tuple[AutoregressiveSpecification, ...], projection_specification_ids: tuple[str, ...], projection_horizon: int, baseline_rolling_windows: tuple[int, ...], compare_exponential_smoothing: bool, rounding_digits: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.autoregressive.AutoregressiveResult + +AutoregressiveResult + +annotations : tuple[Mapping[str, Any], ...] +diagnostics : Mapping[str, JSONValue] +input_result + +__init__(self, diagnostics: Mapping[str, JSONValue], annotations: tuple[Mapping[str, Any], ...], input_result: ClassicalModelInputResult): None + + + +histdatacom.data_quality.classical_model_contracts.ClassicalModelInputResult + +ClassicalModelInputResult + +contract : Mapping[str, JSONValue] +folds : tuple[Mapping[str, JSONValue], ...] +regularized_frame : Any + +__init__(self, regularized_frame: Any, contract: Mapping[str, JSONValue], folds: tuple[Mapping[str, JSONValue], ...]): None + + + +histdatacom.data_quality.autoregressive.AutoregressiveResult->histdatacom.data_quality.classical_model_contracts.ClassicalModelInputResult + + +input_result + + + +histdatacom.data_quality.autoregressive.AutoregressiveSpecification + +AutoregressiveSpecification + +concentrate_scale : bool +d : int +enforce_invertibility : bool +enforce_stationarity : bool +estimation_method : str +family : str +fixed_parameters : tuple[tuple[str, float], ...] +initialization_method : str +max_iterations : int +p : int +q : int +specification_id : str +trend : str + +__init__(self, specification_id: str, family: str, p: int, d: int, q: int, trend: str, initialization_method: str, estimation_method: str, enforce_stationarity: bool, enforce_invertibility: bool, concentrate_scale: bool, fixed_parameters: tuple[tuple[str, float], ...], max_iterations: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.market_context.corpus.BankOfEnglandBankRateAdapterV1 + +BankOfEnglandBankRateAdapterV1 + +adapter_name : str +adapter_version : str +coverage_complete : bool +coverage_end_ns : int | None +coverage_start_ns : int | None +diagnostics : tuple[str, ...] +snapshot + +__init__(snapshot: MarketContextSourceSnapshotV1): None +load_events(): Iterable[MarketContextEventV1] + + + +histdatacom.synthetic.benchmark.BenchmarkCandidateKind + +BenchmarkCandidateKind + +name + +from_value(value: str | 'BenchmarkCandidateKind'): 'BenchmarkCandidateKind' + + + +histdatacom.synthetic.benchmark_corpus.BenchmarkCandidateReportV1 + +BenchmarkCandidateReportV1 + +candidate_id : str +ensemble_member_count : int +evaluated_window_count : int +failure_count : int +gate_decision +method_name : str +metrics : Mapping[str, JSONScalar] +provisional : bool +refusal_count : int +report_id : str +role : str +schema_version : str +uncertainty : Mapping[str, Mapping[str, float]] +window_metric_count : int + +__init__(self, candidate_id: str, method_name: str, role: str, metrics: Mapping[str, JSONScalar], uncertainty: Mapping[str, Mapping[str, float]], window_metric_count: int, ensemble_member_count: int, failure_count: int, refusal_count: int, evaluated_window_count: int, gate_decision: BenchmarkPromotionDecisionV1, provisional: bool, report_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkCandidateReportV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_gates.BenchmarkPromotionDecisionV1 + +BenchmarkPromotionDecisionV1 + +automatic_winner : bool +checks : tuple[BenchmarkGateCheckV1, ...] +decision_id : str +policy_id : str +promotion_eligible : bool +schema_version : str +scope +subject_id : str + +__init__(self, policy_id: str, scope: BenchmarkGateScope, subject_id: str, checks: tuple[BenchmarkGateCheckV1, ...], promotion_eligible: bool, automatic_winner: bool, decision_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkPromotionDecisionV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_corpus.BenchmarkCandidateReportV1->histdatacom.synthetic.benchmark_gates.BenchmarkPromotionDecisionV1 + + +gate_decision + + + +histdatacom.synthetic.benchmark.BenchmarkCandidateScoreV1 + +BenchmarkCandidateScoreV1 + +aggregate_metrics : Mapping[str, float] +automatic_winner : bool +candidate_id : str +candidate_score_id : str +cross_series_hooks : Mapping[str, JSONValue] +execution_summary : Mapping[str, JSONValue] +hard_constraint_violations : Mapping[str, int] +promotion_eligible : bool +relative_to_no_fill : Mapping[str, float] +scenario_id : str +schema_version : str +slice_scores : tuple[BenchmarkSliceScoreV1, ...] +split_kind +strategy_hooks : Mapping[str, JSONValue] +uncertainty_metrics : Mapping[str, JSONValue] + +__init__(self, scenario_id: str, candidate_id: str, split_kind: BenchmarkSplitKind, slice_scores: tuple[BenchmarkSliceScoreV1, ...], aggregate_metrics: Mapping[str, float], uncertainty_metrics: Mapping[str, JSONValue], execution_summary: Mapping[str, JSONValue], cross_series_hooks: Mapping[str, JSONValue], strategy_hooks: Mapping[str, JSONValue], hard_constraint_violations: Mapping[str, int], relative_to_no_fill: Mapping[str, float], promotion_eligible: bool, automatic_winner: bool, candidate_score_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkCandidateScoreV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark.BenchmarkSplitKind + +BenchmarkSplitKind + +name + +from_value(value: str | 'BenchmarkSplitKind'): 'BenchmarkSplitKind' + + + +histdatacom.synthetic.benchmark.BenchmarkCandidateScoreV1->histdatacom.synthetic.benchmark.BenchmarkSplitKind + + +split_kind + + + +histdatacom.synthetic.benchmark.BenchmarkCandidateV1 + +BenchmarkCandidateV1 + +candidate_id : str +control_kind : BenchmarkControlKind | None +ensemble_member_ids : tuple[str, ...] +event_schema_version : str +generator_config_id : str +implementation_version : str +kind +method_id : str +parameters : Mapping[str, JSONValue] +schema_version : str + +__init__(self, kind: BenchmarkCandidateKind, method_id: str, implementation_version: str, parameters: Mapping[str, JSONValue], ensemble_member_ids: tuple[str, ...], control_kind: BenchmarkControlKind | None, candidate_id: str, generator_config_id: str, event_schema_version: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkCandidateV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark.BenchmarkCandidateV1->histdatacom.synthetic.benchmark.BenchmarkCandidateKind + + +kind + + + +histdatacom.synthetic.benchmark.BenchmarkCandidateWindowV1 + +BenchmarkCandidateWindowV1 + +candidate_id : str +cross_series_hooks : Mapping[str, float] +ensemble_member_id : str +events : tuple[BenchmarkEventV1, ...] +execution +hard_constraint_violations : Mapping[str, int] +scenario_id : str +schema_version : str +strategy_hooks : Mapping[str, float] +window_id : str + +__init__(self, scenario_id: str, candidate_id: str, window_id: str, ensemble_member_id: str, events: tuple[BenchmarkEventV1, ...], execution: BenchmarkExecutionEvidenceV1, hard_constraint_violations: Mapping[str, int], cross_series_hooks: Mapping[str, float], strategy_hooks: Mapping[str, float], schema_version: str): None +__post_init__(): None +metadata(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark.BenchmarkExecutionEvidenceV1 + +BenchmarkExecutionEvidenceV1 + +attempted : bool +converged : bool +durable_bytes : int +evidence_id : str +failure_reason : str | None +peak_memory_bytes : int +schema_version : str +scratch_bytes : int +wall_time_ms : int + +__init__(self, attempted: bool, converged: bool, wall_time_ms: int, peak_memory_bytes: int, scratch_bytes: int, durable_bytes: int, failure_reason: str | None, evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkExecutionEvidenceV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark.BenchmarkCandidateWindowV1->histdatacom.synthetic.benchmark.BenchmarkExecutionEvidenceV1 + + +execution + + + +histdatacom.synthetic.benchmark.BenchmarkControlKind + +BenchmarkControlKind + +name + +from_value(value: str | 'BenchmarkControlKind'): 'BenchmarkControlKind' + + + +histdatacom.synthetic.benchmark.BenchmarkEventV1 + +BenchmarkEventV1 + +anchor_id : str | None +ask : float +benchmark_event_id : str +bid : float +ensemble_member_id : str | None +epoch_id : str +event_sequence : int +event_state : str +event_time_ns : int +mid : float +schema_version : str +session : str +slice_key : tuple[str, str, str, str, str] +source_event_id : str +sparsity : str +spread : float +support_lower_mid : float | None +support_upper_mid : float | None +symbol : str + +__init__(self, source_event_id: str, symbol: str, event_time_ns: int, event_sequence: int, bid: float, ask: float, epoch_id: str, session: str, event_state: str, sparsity: str, ensemble_member_id: str | None, anchor_id: str | None, support_lower_mid: float | None, support_upper_mid: float | None, benchmark_event_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkEventV1' +from_observation_input(event: ObservationInputEventV1): 'BenchmarkEventV1' +from_observation_output(event: ObservationOutputEventV1): 'BenchmarkEventV1' +from_synthetic_event(event: SyntheticEventV1): 'BenchmarkEventV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateCheckV1 + +BenchmarkGateCheckV1 + +blocking : bool +check_id : str +observation_id : str | None +reason : str +requirement_id : str +schema_version : str +status + +__init__(self, requirement_id: str, observation_id: str | None, status: BenchmarkGateStatus, blocking: bool, reason: str, check_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkGateCheckV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateStatus + +BenchmarkGateStatus + +name + + + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateCheckV1->histdatacom.synthetic.benchmark_gates.BenchmarkGateStatus + + +status + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateComparator + +BenchmarkGateComparator + +name + + + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateObservationV1 + +BenchmarkGateObservationV1 + +evidence_ids : tuple[str, ...] +metric_name : str +observation_id : str +schema_version : str +scope +subject_id : str +value + +__init__(self, scope: BenchmarkGateScope, subject_id: str, metric_name: str, value: JSONScalar, evidence_ids: tuple[str, ...], observation_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkGateObservationV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateScope + +BenchmarkGateScope + +name + + + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateObservationV1->histdatacom.synthetic.benchmark_gates.BenchmarkGateScope + + +scope + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateRequirementV1 + +BenchmarkGateRequirementV1 + +comparator +description : str +metric_name : str +requirement_id : str +schema_version : str +scope +severity +threshold + +__init__(self, requirement_id: str, scope: BenchmarkGateScope, severity: BenchmarkGateSeverity, metric_name: str, comparator: BenchmarkGateComparator, threshold: JSONScalar, description: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkGateRequirementV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateRequirementV1->histdatacom.synthetic.benchmark_gates.BenchmarkGateComparator + + +comparator + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateRequirementV1->histdatacom.synthetic.benchmark_gates.BenchmarkGateScope + + +scope + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateSeverity + +BenchmarkGateSeverity + +name + + + + + +histdatacom.synthetic.benchmark_gates.BenchmarkGateRequirementV1->histdatacom.synthetic.benchmark_gates.BenchmarkGateSeverity + + +severity + + + +histdatacom.synthetic.benchmark.BenchmarkGeneratorV1 + +BenchmarkGeneratorV1 + +candidate_id : str +event_schema_version : str + +generate +(degraded_events: Sequence[BenchmarkEventV1]): Sequence[BenchmarkEventV1] - + histdatacom.orchestration.performance.BenchmarkMeasurement - -BenchmarkMeasurement - -cpu_seconds : float -elapsed_seconds : float -metadata : dict[str, JSONValue] -name : str -peak_rss_bytes : int -retry_count : int -startup_seconds : float -throughput_per_second : float -work_item_count : int - -__init__(self, name: str, work_item_count: int, elapsed_seconds: float, cpu_seconds: float, peak_rss_bytes: int, retry_count: int, startup_seconds: float, metadata: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +BenchmarkMeasurement + +cpu_seconds : float +elapsed_seconds : float +metadata : dict[str, JSONValue] +name : str +peak_rss_bytes : int +retry_count : int +startup_seconds : float +throughput_per_second : float +work_item_count : int + +__init__(self, name: str, work_item_count: int, elapsed_seconds: float, cpu_seconds: float, peak_rss_bytes: int, retry_count: int, startup_seconds: float, metadata: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark.BenchmarkProfileV1 + +BenchmarkProfileV1 + +burst_threshold_ns : int +interarrival_buckets_ns : tuple[int, ...] +max_candidates : int +max_events_per_window : int +max_hook_metrics : int +max_payload_bytes : int +max_reason_codes : int +max_scenarios : int +max_slices : int +profile_id : str +quiet_threshold_ns : int +rounding_digits : int +schema_version : str +spread_buckets : tuple[float, ...] + +__init__(self, max_scenarios: int, max_candidates: int, max_slices: int, max_events_per_window: int, max_hook_metrics: int, max_reason_codes: int, max_payload_bytes: int, burst_threshold_ns: int, quiet_threshold_ns: int, interarrival_buckets_ns: tuple[int, ...], spread_buckets: tuple[float, ...], rounding_digits: int, profile_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkProfileV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_gates.BenchmarkPromotionDecisionV1->histdatacom.synthetic.benchmark_gates.BenchmarkGateScope + + +scope + + + +histdatacom.synthetic.benchmark_gates.BenchmarkPromotionGatePolicyV1 + +BenchmarkPromotionGatePolicyV1 + +frozen_before_candidate_results : bool +issue_number : int +policy_id : str +policy_name : str +policy_version : str +requirements : tuple[BenchmarkGateRequirementV1, ...] +schema_version : str + +__init__(self, policy_name: str, policy_version: str, issue_number: int, frozen_before_candidate_results: bool, requirements: tuple[BenchmarkGateRequirementV1, ...], policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkPromotionGatePolicyV1' +from_json(text: str): 'BenchmarkPromotionGatePolicyV1' +identity_payload(): dict[str, JSONValue] +requirements_for(scope: BenchmarkGateScope): tuple[BenchmarkGateRequirementV1, ...] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.benchmark.BenchmarkScenarioV1 + +BenchmarkScenarioV1 + +degradation_config_id : str +degradation_parameters : Mapping[str, JSONValue] +epoch_id : str +event_schema_version : str +observation_operator_id : str +scenario_id : str +schema_version : str +severity_id : str +split_kind + +__init__(self, split_kind: BenchmarkSplitKind, epoch_id: str, severity_id: str, observation_operator_id: str, degradation_parameters: Mapping[str, JSONValue], scenario_id: str, degradation_config_id: str, event_schema_version: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkScenarioV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark.BenchmarkScenarioV1->histdatacom.synthetic.benchmark.BenchmarkSplitKind + + +split_kind + + + +histdatacom.synthetic.benchmark.BenchmarkSliceScoreV1 + +BenchmarkSliceScoreV1 + +candidate_event_count_mean : float +candidate_id : str +degraded_event_count : int +epoch_id : str +event_state : str +metrics : Mapping[str, float] +reference_event_count : int +scenario_id : str +schema_version : str +session : str +slice_score_id : str +sparsity : str +support : Mapping[str, JSONValue] +symbol : str + +__init__(self, scenario_id: str, candidate_id: str, symbol: str, epoch_id: str, session: str, event_state: str, sparsity: str, reference_event_count: int, degraded_event_count: int, candidate_event_count_mean: float, metrics: Mapping[str, float], support: Mapping[str, JSONValue], slice_score_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkSliceScoreV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_corpus.BenchmarkSourcePartitionV1 + +BenchmarkSourcePartitionV1 + +partition_id : str +period : str +relative_path : str +row_count : int +schema_version : str +sha256 : str +size_bytes : int +symbol : str + +__init__(self, symbol: str, period: str, relative_path: str, size_bytes: int, row_count: int, sha256: str, partition_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkSourcePartitionV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark.BenchmarkSplitV1 + +BenchmarkSplitV1 + +end_ns : int +kind +schema_version : str +split_id : str +start_ns : int + +__init__(self, kind: BenchmarkSplitKind, start_ns: int, end_ns: int, split_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkSplitV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark.BenchmarkSplitV1->histdatacom.synthetic.benchmark.BenchmarkSplitKind + + +kind + + + +histdatacom.synthetic.benchmark_corpus.BenchmarkWindowPartitionV1 + +BenchmarkWindowPartitionV1 + +context_state : str +context_supported : bool +end_ns : int +epoch_label : str +event_state_counts : Mapping[str, int] +period : str +positioning_state : str +schema_version : str +selection_rule : str +session : str +source_partition_ids : tuple[str, ...] +split_kind : str +start_ns : int +symbol_event_counts : Mapping[str, int] +symbol_partition_sha256 : Mapping[str, str] +window_id : str + +__init__(self, split_kind: str, period: str, session: str, start_ns: int, end_ns: int, epoch_label: str, source_partition_ids: tuple[str, ...], symbol_event_counts: Mapping[str, int], symbol_partition_sha256: Mapping[str, str], event_state_counts: Mapping[str, int], context_state: str, positioning_state: str, context_supported: bool, selection_rule: str, window_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BenchmarkWindowPartitionV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.limits.BoundedReportLimit - -BoundedReportLimit - -default_limit : int -effective_limit : int -maximum_limit : int | None -minimum_limit : int -requested_limit : int | None -requested_or_default_limit : int -unbounded : bool - -__init__(self, requested_limit: int | None, default_limit: int, minimum_limit: int, maximum_limit: int | None, effective_limit: int, unbounded: bool): None -count_payload(total_count: int): dict[str, JSONValue] -included_count(total_count: int): int -limit_payload(): dict[str, JSONValue] -normalize(requested_limit: int | None): 'BoundedReportLimit' -slice(values: Sequence[_T]): list[_T] + +BoundedReportLimit + +default_limit : int +effective_limit : int +maximum_limit : int | None +minimum_limit : int +requested_limit : int | None +requested_or_default_limit : int +unbounded : bool + +__init__(self, requested_limit: int | None, default_limit: int, minimum_limit: int, maximum_limit: int | None, effective_limit: int, unbounded: bool): None +count_payload(total_count: int): dict[str, JSONValue] +included_count(total_count: int): int +limit_payload(): dict[str, JSONValue] +normalize(requested_limit: int | None): 'BoundedReportLimit' +slice(values: Sequence[_T]): list[_T] + + + +histdatacom.broker_capture.contracts.BrokerAdapterMessageV1 + +BrokerAdapterMessageV1 + +activity_semantics +activity_value : float | None +ask : float | None +ask_size : float | None +ask_text : str | None +bid : float | None +bid_size : float | None +bid_text : str | None +connection_id : str | None +gap_duration_ns : int | None +kind +message_id : str +price_text_semantics +public_metadata : dict[str, JSONValue] +raw_message_sha256 : str | None +reason_code : str | None +schema_version : str +size_semantics +source_batch_id : str | None +source_event_time_ns : int | None +source_message_id : str | None +source_sequence : int | None +source_timestamp_precision_ns : int | None +source_timestamp_semantics +subscription_id : str | None +symbol : str | None + +__init__(self, kind: BrokerCaptureEventKind, source_event_time_ns: int | None, source_timestamp_semantics: BrokerCaptureSourceTimestampSemantics, source_timestamp_precision_ns: int | None, source_sequence: int | None, source_message_id: str | None, source_batch_id: str | None, symbol: str | None, bid: float | None, ask: float | None, bid_text: str | None, ask_text: str | None, price_text_semantics: BrokerCapturePriceTextSemantics, bid_size: float | None, ask_size: float | None, size_semantics: BrokerCaptureSizeSemantics, activity_value: float | None, activity_semantics: BrokerCaptureActivitySemantics, connection_id: str | None, subscription_id: str | None, gap_duration_ns: int | None, reason_code: str | None, raw_message_sha256: str | None, public_metadata: dict[str, JSONValue], message_id: str, schema_version: str): None +__post_init__(): None +_validate_kind_fields(): None +from_dict(data: Mapping[str, Any]): 'BrokerAdapterMessageV1' +from_json(text: str): 'BrokerAdapterMessageV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.contracts.BrokerCaptureActivitySemantics + +BrokerCaptureActivitySemantics + +name + +from_value(value: str | 'BrokerCaptureActivitySemantics'): 'BrokerCaptureActivitySemantics' + + + +histdatacom.broker_capture.contracts.BrokerAdapterMessageV1->histdatacom.broker_capture.contracts.BrokerCaptureActivitySemantics + + +activity_semantics + + + +histdatacom.broker_capture.contracts.BrokerCaptureEventKind + +BrokerCaptureEventKind + +name + +from_value(value: str | 'BrokerCaptureEventKind'): 'BrokerCaptureEventKind' + + + +histdatacom.broker_capture.contracts.BrokerAdapterMessageV1->histdatacom.broker_capture.contracts.BrokerCaptureEventKind + + +kind + + + +histdatacom.broker_capture.contracts.BrokerCapturePriceTextSemantics + +BrokerCapturePriceTextSemantics + +name + +from_value(value: str | 'BrokerCapturePriceTextSemantics'): 'BrokerCapturePriceTextSemantics' + + + +histdatacom.broker_capture.contracts.BrokerAdapterMessageV1->histdatacom.broker_capture.contracts.BrokerCapturePriceTextSemantics + + +price_text_semantics + + + +histdatacom.broker_capture.contracts.BrokerCaptureSizeSemantics + +BrokerCaptureSizeSemantics + +name + +from_value(value: str | 'BrokerCaptureSizeSemantics'): 'BrokerCaptureSizeSemantics' + + + +histdatacom.broker_capture.contracts.BrokerAdapterMessageV1->histdatacom.broker_capture.contracts.BrokerCaptureSizeSemantics + + +size_semantics + + + +histdatacom.broker_capture.contracts.BrokerCaptureSourceTimestampSemantics + +BrokerCaptureSourceTimestampSemantics + +name + +from_value(value: str | 'BrokerCaptureSourceTimestampSemantics'): 'BrokerCaptureSourceTimestampSemantics' + + + +histdatacom.broker_capture.contracts.BrokerAdapterMessageV1->histdatacom.broker_capture.contracts.BrokerCaptureSourceTimestampSemantics + + +source_timestamp_semantics + + + +histdatacom.synthetic.broker_transfer.BrokerBenchmarkComparisonV1 + +BrokerBenchmarkComparisonV1 + +aggregate_metric_deltas : Mapping[str, Mapping[str, float]] +comparison_id : str +conditioned_candidate_id : str +conditioned_score_ids : tuple[str, ...] +scenario_ids : tuple[str, ...] +schema_version : str +scorecard_id : str +unconditioned_candidate_id : str +unconditioned_score_ids : tuple[str, ...] + +__init__(self, scorecard_id: str, conditioned_candidate_id: str, unconditioned_candidate_id: str, scenario_ids: tuple[str, ...], conditioned_score_ids: tuple[str, ...], unconditioned_score_ids: tuple[str, ...], aggregate_metric_deltas: Mapping[str, Mapping[str, float]], comparison_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerBenchmarkComparisonV1' +from_json(text: str): 'BrokerBenchmarkComparisonV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.adapters.BrokerCaptureAdapterV1 + +BrokerCaptureAdapterV1 + +adapter_id : str +adapter_version : str + +iter_messages +(): Iterable[BrokerAdapterMessageV1] + + + +histdatacom.broker_capture.storage.BrokerCaptureBackpressureError + +BrokerCaptureBackpressureError + + + + + + +histdatacom.broker_capture.storage.BrokerCaptureStorageError + +BrokerCaptureStorageError + + + + + + +histdatacom.broker_capture.storage.BrokerCaptureBackpressureError->histdatacom.broker_capture.storage.BrokerCaptureStorageError + + + + + +histdatacom.broker_capture.contracts.BrokerCaptureBackpressureMode + +BrokerCaptureBackpressureMode + +name + +from_value(value: str | 'BrokerCaptureBackpressureMode'): 'BrokerCaptureBackpressureMode' + + + +histdatacom.broker_capture.adapters.BrokerCaptureClockV1 + +BrokerCaptureClockV1 + + +sample +(): tuple[int, int] + + + +histdatacom.broker_capture.adapters.BrokerCaptureConsumeResultV1 + +BrokerCaptureConsumeResultV1 + +event_count : int +event_kind_counts : dict[str, int] +first_capture_sequence : int | None +last_capture_sequence : int | None +session_id : str + +__init__(self, session_id: str, event_count: int, event_kind_counts: dict[str, int], first_capture_sequence: int | None, last_capture_sequence: int | None): None + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerCaptureEligibilityStatus + +BrokerCaptureEligibilityStatus + +name + +from_value(value: str | 'BrokerCaptureEligibilityStatus'): 'BrokerCaptureEligibilityStatus' + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerCaptureEligibilityV1 + +BrokerCaptureEligibilityV1 + +clock_correction_count : int +config_id : str +decision_id : str +event_count : int +explained_wall_regression_count : int +first_receive_time_utc_ns : int | None +fit_allowed : bool +inspection_clean : bool +integrity_verified : bool +last_receive_time_utc_ns : int | None +logical_content_sha256 : str | None +manifest_complete : bool +manifest_id : str +max_abs_clock_correction_ns : int +quote_count : int +reason_codes : tuple[str, ...] +schema_version : str +session_id : str +status +unexplained_wall_regression_count : int + +__init__(self, session_id: str, manifest_id: str, config_id: str, status: BrokerCaptureEligibilityStatus, reason_codes: tuple[str, ...], manifest_complete: bool, inspection_clean: bool, integrity_verified: bool, event_count: int, quote_count: int, clock_correction_count: int, max_abs_clock_correction_ns: int, explained_wall_regression_count: int, unexplained_wall_regression_count: int, first_receive_time_utc_ns: int | None, last_receive_time_utc_ns: int | None, logical_content_sha256: str | None, decision_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerCaptureEligibilityV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerCaptureEligibilityV1->histdatacom.broker_capture.fingerprint_contracts.BrokerCaptureEligibilityStatus + + +status + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryIneligibleCaptureError + +BrokerDeliveryIneligibleCaptureError + +eligibility + +__init__(eligibility: BrokerCaptureEligibilityV1): None + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerCaptureEligibilityV1->histdatacom.broker_capture.fingerprints.BrokerDeliveryIneligibleCaptureError + + +eligibility + + + +histdatacom.broker_capture.adapters.BrokerCaptureEventConsumerV1 + +BrokerCaptureEventConsumerV1 + + +on_event +(event: BrokerCaptureEventV1): None + + + +histdatacom.broker_capture.adapters.BrokerCaptureEventSinkV1 + +BrokerCaptureEventSinkV1 + + +append +(event: BrokerCaptureEventV1): None + + + +histdatacom.broker_capture.adapters.BrokerCaptureEventSourceV1 + +BrokerCaptureEventSourceV1 + +session_id : str + +iter_events +(): Iterable[BrokerCaptureEventV1] + + + +histdatacom.broker_capture.contracts.BrokerCaptureEventV1 + +BrokerCaptureEventV1 + +capture_sequence : int +clock_offset_change_ns : int | None +event_id : str +kind : BrokerCaptureEventKind +message +receive_time_monotonic_ns : int +receive_time_utc_ns : int +schema_version : str +session_id : str + +__init__(self, session_id: str, capture_sequence: int, receive_time_utc_ns: int, receive_time_monotonic_ns: int, message: BrokerAdapterMessageV1, clock_offset_change_ns: int | None, event_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerCaptureEventV1' +from_json(text: str): 'BrokerCaptureEventV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.contracts.BrokerCaptureEventV1->histdatacom.broker_capture.storage.AppendOnlyBrokerCaptureWriterV1 + + +_partition_last_event + + + +histdatacom.broker_capture.contracts.BrokerCaptureEventV1->histdatacom.broker_capture.contracts.BrokerAdapterMessageV1 + + +message + + + +histdatacom.broker_capture.storage.BrokerCaptureExistingSessionError + +BrokerCaptureExistingSessionError + + + + + + +histdatacom.broker_capture.storage.BrokerCaptureExistingSessionError->histdatacom.broker_capture.storage.BrokerCaptureStorageError + + + + + +histdatacom.broker_capture.storage.BrokerCaptureIntegrityError + +BrokerCaptureIntegrityError + + + + + + +histdatacom.broker_capture.storage.BrokerCaptureIntegrityError->histdatacom.broker_capture.storage.BrokerCaptureStorageError + + + + + +histdatacom.broker_capture.contracts.BrokerCapturePartitionManifestV1->histdatacom.runtime_contracts.ArtifactRef + + +data_artifact + + + +histdatacom.broker_capture.storage.BrokerCaptureQuotaError + +BrokerCaptureQuotaError + + + + + + +histdatacom.broker_capture.storage.BrokerCaptureQuotaError->histdatacom.broker_capture.storage.BrokerCaptureStorageError + + + + + +histdatacom.broker_capture.storage.BrokerCaptureReplaySourceV1 + +BrokerCaptureReplaySourceV1 + +manifest +root : Path +session_id : str + +__init__(root: str | Path, manifest: BrokerCaptureSessionManifestV1): None +_event_iterator(): Iterator[BrokerCaptureEventV1] +iter_events(): Iterable[BrokerCaptureEventV1] + + + +histdatacom.broker_capture.contracts.BrokerCaptureSessionManifestV1 + +BrokerCaptureSessionManifestV1 + +complete : bool +event_count : int +event_kind_counts : dict[str, int] +first_capture_sequence : int | None +last_capture_sequence : int | None +limitations : tuple[str, ...] +manifest_id : str +partial_artifact_count : int +partitions : tuple[BrokerCapturePartitionManifestV1, ...] +schema_version : str +session +state +storage_policy + +__init__(self, session: BrokerCaptureSessionV1, storage_policy: BrokerCaptureStoragePolicyV1, state: BrokerCaptureSessionState, partitions: tuple[BrokerCapturePartitionManifestV1, ...], event_count: int, event_kind_counts: dict[str, int], first_capture_sequence: int | None, last_capture_sequence: int | None, partial_artifact_count: int, limitations: tuple[str, ...], manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerCaptureSessionManifestV1' +from_json(text: str): 'BrokerCaptureSessionManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.storage.BrokerCaptureReplaySourceV1->histdatacom.broker_capture.contracts.BrokerCaptureSessionManifestV1 + + +manifest + + + +histdatacom.broker_capture.contracts.BrokerCaptureReplaySummaryV1 + +BrokerCaptureReplaySummaryV1 + +event_count : int +event_kind_counts : dict[str, int] +first_capture_sequence : int | None +last_capture_sequence : int | None +logical_content_sha256 : str +manifest_id : str +partition_count : int +schema_version : str +session_id : str +summary_id : str + +__init__(self, session_id: str, manifest_id: str, partition_count: int, event_count: int, event_kind_counts: dict[str, int], first_capture_sequence: int | None, last_capture_sequence: int | None, logical_content_sha256: str, summary_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerCaptureReplaySummaryV1' +from_json(text: str): 'BrokerCaptureReplaySummaryV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.storage.BrokerCaptureRetentionError + +BrokerCaptureRetentionError + + + + + + +histdatacom.broker_capture.storage.BrokerCaptureRetentionError->histdatacom.broker_capture.storage.BrokerCaptureStorageError + + + + + +histdatacom.broker_capture.contracts.BrokerCaptureRetentionMode + +BrokerCaptureRetentionMode + +name + +from_value(value: str | 'BrokerCaptureRetentionMode'): 'BrokerCaptureRetentionMode' + + + +histdatacom.broker_capture.storage.BrokerCaptureSessionInspectionV1 + +BrokerCaptureSessionInspectionV1 + +advertised_partition_count : int +clean : bool +orphan_data_artifacts : tuple[str, ...] +orphan_manifest_artifacts : tuple[str, ...] +partial_artifacts : tuple[str, ...] +session_directory : str + +__init__(self, session_directory: str, advertised_partition_count: int, partial_artifacts: tuple[str, ...], orphan_data_artifacts: tuple[str, ...], orphan_manifest_artifacts: tuple[str, ...]): None + + + +histdatacom.broker_capture.contracts.BrokerCaptureSessionManifestV1->histdatacom.broker_capture.storage.AppendOnlyBrokerCaptureWriterV1 + + +_manifest + + + +histdatacom.broker_capture.contracts.BrokerCaptureSessionState + +BrokerCaptureSessionState + +name + +from_value(value: str | 'BrokerCaptureSessionState'): 'BrokerCaptureSessionState' + + + +histdatacom.broker_capture.contracts.BrokerCaptureSessionManifestV1->histdatacom.broker_capture.contracts.BrokerCaptureSessionState + + +state + + + +histdatacom.broker_capture.contracts.BrokerCaptureSessionV1 + +BrokerCaptureSessionV1 + +account_id_sha256 : str | None +adapter_config_sha256 : str +adapter_id : str +adapter_version : str +clock_resolution_ns : int +clock_source : str +collector_id : str +collector_version : str +environment_id : str +host_id_sha256 : str | None +protocol : str +public_metadata : dict[str, JSONValue] +schema_version : str +server_id : str +session_id : str +started_at_monotonic_ns : int +started_at_utc_ns : int + +__init__(self, adapter_id: str, adapter_version: str, adapter_config_sha256: str, protocol: str, environment_id: str, server_id: str, started_at_utc_ns: int, started_at_monotonic_ns: int, account_id_sha256: str | None, host_id_sha256: str | None, clock_source: str, clock_resolution_ns: int, collector_id: str, collector_version: str, public_metadata: dict[str, JSONValue], session_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerCaptureSessionV1' +from_json(text: str): 'BrokerCaptureSessionV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.contracts.BrokerCaptureSessionManifestV1->histdatacom.broker_capture.contracts.BrokerCaptureSessionV1 + + +session + + + +histdatacom.broker_capture.contracts.BrokerCaptureStoragePolicyV1 + +BrokerCaptureStoragePolicyV1 + +backpressure_mode +fsync_each_event : bool +high_watermark_bytes : int +manifest_reserve_bytes : int +max_partition_bytes : int +max_partition_duration_ns : int +max_partition_events : int +max_retained_partitions : int +max_session_bytes : int +policy_id : str +retention_mode +schema_version : str + +__init__(self, max_partition_events: int, max_partition_bytes: int, max_partition_duration_ns: int, max_session_bytes: int, high_watermark_bytes: int, max_retained_partitions: int, manifest_reserve_bytes: int, fsync_each_event: bool, retention_mode: BrokerCaptureRetentionMode, backpressure_mode: BrokerCaptureBackpressureMode, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerCaptureStoragePolicyV1' +from_json(text: str): 'BrokerCaptureStoragePolicyV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.contracts.BrokerCaptureSessionManifestV1->histdatacom.broker_capture.contracts.BrokerCaptureStoragePolicyV1 + + +storage_policy + + + +histdatacom.broker_capture.contracts.BrokerCaptureStoragePolicyV1->histdatacom.broker_capture.contracts.BrokerCaptureBackpressureMode + + +backpressure_mode + + + +histdatacom.broker_capture.contracts.BrokerCaptureStoragePolicyV1->histdatacom.broker_capture.contracts.BrokerCaptureRetentionMode + + +retention_mode + + + +histdatacom.synthetic.broker_transfer.BrokerConditionedProposalV1 + +BrokerConditionedProposalV1 + +conditioned_query : ReferenceMotifQueryV1 | None +metrics_after : Mapping[str, float] +metrics_before : Mapping[str, float] +original_query_id : str +proposal_id : str +schema_version : str +selection +status : BrokerTransferStatus +transfer_config + +__init__(self, selection: BrokerProfileSelectionV1, transfer_config: BrokerTransferConfigV1, original_query_id: str, conditioned_query: ReferenceMotifQueryV1 | None, metrics_before: Mapping[str, float], metrics_after: Mapping[str, float], proposal_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerConditionedProposalV1' +from_json(text: str): 'BrokerConditionedProposalV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.broker_transfer.BrokerProfileSelectionV1 + +BrokerProfileSelectionV1 + +applied : bool +drift_comparison_id : str | None +effective_condition_id : str | None +fingerprint_id : str +fingerprint_schema_version : str +material_drift_count : int | None +metric_condition_ids : Mapping[str, str] +metrics : Mapping[str, float] +profile_effective_end_utc_ns : int | None +profile_effective_start_utc_ns : int +reason_codes : tuple[str, ...] +requested_condition : Mapping[str, str] +requested_condition_id : str | None +schema_version : str +selected_at_utc_ns : int +selection_id : str +status +supersedes_fingerprint_id : str | None +support_status : str + +__init__(self, fingerprint_id: str, fingerprint_schema_version: str, requested_condition: Mapping[str, str], requested_condition_id: str | None, effective_condition_id: str | None, support_status: str, status: BrokerTransferStatus, selected_at_utc_ns: int, profile_effective_start_utc_ns: int, profile_effective_end_utc_ns: int | None, supersedes_fingerprint_id: str | None, metrics: Mapping[str, float], metric_condition_ids: Mapping[str, str], reason_codes: tuple[str, ...], drift_comparison_id: str | None, material_drift_count: int | None, selection_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerProfileSelectionV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.broker_transfer.BrokerConditionedProposalV1->histdatacom.synthetic.broker_transfer.BrokerProfileSelectionV1 + + +selection + + + +histdatacom.synthetic.broker_transfer.BrokerTransferConfigV1 + +BrokerTransferConfigV1 + +apply_batching : bool +apply_exact_duplicates : bool +apply_stale_behavior : bool +config_id : str +input_price_decimal_places : int +max_batch_size : int +max_events_per_group : int +max_spread_multiplier : float +max_timestamp_shift_ns : int +minimum_price_decimal_places : int +rounding_digits : int +schema_version : str +strength : float + +__init__(self, strength: float, max_timestamp_shift_ns: int, max_spread_multiplier: float, max_batch_size: int, max_events_per_group: int, input_price_decimal_places: int, minimum_price_decimal_places: int, apply_stale_behavior: bool, apply_exact_duplicates: bool, apply_batching: bool, rounding_digits: int, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerTransferConfigV1' +from_json(text: str): 'BrokerTransferConfigV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.broker_transfer.BrokerConditionedProposalV1->histdatacom.synthetic.broker_transfer.BrokerTransferConfigV1 + + +transfer_config + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryCaptureEvidenceV1 + +BrokerDeliveryCaptureEvidenceV1 + +eligibility_decision_id : str +event_count : int +evidence_id : str +first_receive_time_utc_ns : int +last_receive_time_utc_ns : int +logical_content_sha256 : str +manifest_id : str +partition_count : int +partition_hashes_sha256 : str +schema_version : str +session_id : str + +__init__(self, session_id: str, manifest_id: str, eligibility_decision_id: str, logical_content_sha256: str, partition_hashes_sha256: str, partition_count: int, event_count: int, first_receive_time_utc_ns: int, last_receive_time_utc_ns: int, evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryCaptureEvidenceV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryCellV1 + +BrokerDeliveryCellV1 + +backoff_condition_ids : tuple[str, ...] +cell_id : str +condition +effective_condition_id : str | None +limitations : tuple[str, ...] +metrics : tuple[BrokerDeliveryMetricV1, ...] +schema_version : str +support_count : int +support_status + +__init__(self, condition: BrokerDeliveryConditionV1, support_count: int, support_status: BrokerDeliverySupportStatus, backoff_condition_ids: tuple[str, ...], effective_condition_id: str | None, metrics: tuple[BrokerDeliveryMetricV1, ...], limitations: tuple[str, ...], cell_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryCellV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryConditionV1 + +BrokerDeliveryConditionV1 + +condition_id : str +dimensions : dict[str, str] +key : str +schema_version : str + +__init__(self, dimensions: dict[str, str], condition_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryConditionV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryCellV1->histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryConditionV1 + + +condition + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliverySupportStatus + +BrokerDeliverySupportStatus + +name + +from_value(value: str | 'BrokerDeliverySupportStatus'): 'BrokerDeliverySupportStatus' + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryCellV1->histdatacom.broker_capture.fingerprint_contracts.BrokerDeliverySupportStatus + + +support_status + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryDriftConfigV1 + +BrokerDeliveryDriftConfigV1 + +absolute_material_thresholds : dict[str, float] +config_id : str +default_absolute_material_threshold : float +max_comparisons : int +min_metric_support : int +relative_material_threshold : float +rounding_digits : int +schema_version : str + +__init__(self, min_metric_support: int, relative_material_threshold: float, default_absolute_material_threshold: float, absolute_material_thresholds: dict[str, float], max_comparisons: int, rounding_digits: int, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryDriftConfigV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryDriftStatus + +BrokerDeliveryDriftStatus + +name + +from_value(value: str | 'BrokerDeliveryDriftStatus'): 'BrokerDeliveryDriftStatus' + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintArtifactError + +BrokerDeliveryFingerprintArtifactError + + + + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintError + +BrokerDeliveryFingerprintError + + + + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintArtifactError->histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintError + + + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryFingerprintComparisonV1 + +BrokerDeliveryFingerprintComparisonV1 + +candidate_fingerprint_id : str +comparison_candidate_count : int +comparison_id : str +comparisons : tuple[BrokerDeliveryMetricComparisonV1, ...] +drift_config +material_drift_count : int +reference_fingerprint_id : str +schema_version : str +status_counts : dict[str, int] +truncated : bool + +__init__(self, reference_fingerprint_id: str, candidate_fingerprint_id: str, drift_config: BrokerDeliveryDriftConfigV1, comparison_candidate_count: int, comparisons: tuple[BrokerDeliveryMetricComparisonV1, ...], status_counts: dict[str, int], truncated: bool, comparison_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryFingerprintComparisonV1' +from_json(text: str): 'BrokerDeliveryFingerprintComparisonV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryFingerprintComparisonV1->histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryDriftConfigV1 + + +drift_config + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintIdentityError + +BrokerDeliveryFingerprintIdentityError + + + + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintIdentityError->histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintError + + + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryFingerprintV1 + +BrokerDeliveryFingerprintV1 + +account_id_sha256 : str | None +adapter_config_sha256 : str +adapter_id : str +adapter_version : str +capture_evidence : tuple[BrokerDeliveryCaptureEvidenceV1, ...] +cells : tuple[BrokerDeliveryCellV1, ...] +collector_id : str +collector_version : str +effective_end_utc_ns : int | None +effective_start_utc_ns : int +eligibility_decisions : tuple[BrokerCaptureEligibilityV1, ...] +environment_id : str +fingerprint_id : str +fit_config +limitations : tuple[str, ...] +protocol : str +schema_version : str +server_id : str +supersedes_fingerprint_id : str | None +support_end_utc_ns : int +support_start_utc_ns : int + +__init__(self, adapter_id: str, adapter_version: str, adapter_config_sha256: str, protocol: str, environment_id: str, server_id: str, account_id_sha256: str | None, collector_id: str, collector_version: str, fit_config: BrokerDeliveryFitConfigV1, capture_evidence: tuple[BrokerDeliveryCaptureEvidenceV1, ...], eligibility_decisions: tuple[BrokerCaptureEligibilityV1, ...], support_start_utc_ns: int, support_end_utc_ns: int, effective_start_utc_ns: int, effective_end_utc_ns: int | None, cells: tuple[BrokerDeliveryCellV1, ...], supersedes_fingerprint_id: str | None, limitations: tuple[str, ...], fingerprint_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryFingerprintV1' +from_json(text: str): 'BrokerDeliveryFingerprintV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryFitConfigV1 + +BrokerDeliveryFitConfigV1 + +burst_interval_ns : int +config_id : str +fatal_limitation_prefixes : tuple[str, ...] +max_abs_clock_correction_ns : int +max_capture_manifests : int +max_cells : int +max_clock_correction_events : int +max_input_events : int +max_market_context_events : int +max_market_matches_per_quote : int +max_samples_per_metric : int +max_transition_categories : int +max_unexplained_wall_regressions : int +min_capture_events : int +min_cell_support : int +min_quote_events : int +post_lifecycle_quote_count : int +quantiles : tuple[float, ...] +quiet_interval_ns : int +rounding_digits : int +schema_version : str +stale_max_interval_ns : int +supported_adapter_ids : tuple[str, ...] +supported_collector_versions : tuple[str, ...] + +__init__(self, min_capture_events: int, min_quote_events: int, min_cell_support: int, max_capture_manifests: int, max_input_events: int, max_cells: int, max_samples_per_metric: int, max_transition_categories: int, max_market_context_events: int, max_market_matches_per_quote: int, burst_interval_ns: int, quiet_interval_ns: int, stale_max_interval_ns: int, post_lifecycle_quote_count: int, max_clock_correction_events: int, max_abs_clock_correction_ns: int, max_unexplained_wall_regressions: int, quantiles: tuple[float, ...], supported_collector_versions: tuple[str, ...], supported_adapter_ids: tuple[str, ...], fatal_limitation_prefixes: tuple[str, ...], rounding_digits: int, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryFitConfigV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryFingerprintV1->histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryFitConfigV1 + + +fit_config + + + +histdatacom.broker_capture.fingerprints._FingerprintConsumer + +_FingerprintConsumer + +_batch_id : NoneType, str | None +_batch_run_count : int +_lifecycle : NoneType, str | None +_lifecycle_quotes_remaining : int +_previous_event_kind : BrokerCaptureEventKind | None, NoneType +_previous_event_monotonic_ns : NoneType, int | None +_previous_quotes : dict[str, _PreviousQuote] +_session_event_count : int +_session_first_monotonic_ns : NoneType, int | None +_session_id : str +_session_last_monotonic_ns : NoneType, int | None +_session_quote_count : int +calendar_profile +calendar_profile_complete : bool +cells : dict[str, _CellAccumulator] +config +context_events +event_count : int +quote_count : int + +__init__(config: BrokerDeliveryFitConfigV1): None +_append_condition_pair(conditions: list[dict[str, str]], symbol: str, dimension: str, tag: str): None +_ensure_cell(dimensions: Mapping[str, str]): _CellAccumulator +_flush_batch_run(evidence_key: str): None +_observe_batch(batch_id: str | None, evidence_key: str): None +_on_quote(event: BrokerCaptureEventV1): None +_quote_conditions(event: BrokerCaptureEventV1): tuple[dict[str, str], ...] +end_session(): None +finish(): None +on_event(event: BrokerCaptureEventV1): None +start_session(session_id: str): None + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryFitConfigV1->histdatacom.broker_capture.fingerprints._FingerprintConsumer + + +config + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryIneligibleCaptureError->histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintError + + + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryMetricComparisonV1 + +BrokerDeliveryMetricComparisonV1 + +absolute_difference : float | None +candidate_estimate : float | None +candidate_support_count : int +combined_uncertainty : float | None +comparison_id : str +condition_id : str +condition_key : str +metric_name : str +reason_codes : tuple[str, ...] +reference_estimate : float | None +reference_support_count : int +relative_difference : float | None +schema_version : str +status + +__init__(self, condition_id: str, condition_key: str, metric_name: str, reference_support_count: int, candidate_support_count: int, reference_estimate: float | None, candidate_estimate: float | None, absolute_difference: float | None, relative_difference: float | None, combined_uncertainty: float | None, status: BrokerDeliveryDriftStatus, reason_codes: tuple[str, ...], comparison_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryMetricComparisonV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryMetricComparisonV1->histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryDriftStatus + + +status + + + +histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryMetricV1 + +BrokerDeliveryMetricV1 + +category_counts : dict[str, int] +estimate : float | None +kind : str +limitations : tuple[str, ...] +lower : float | None +maximum : float | None +metric_id : str +minimum : float | None +name : str +quantiles : dict[str, float] +sample_count : int +schema_version : str +support_count : int +unit : str +upper : float | None + +__init__(self, name: str, kind: str, unit: str, support_count: int, sample_count: int, estimate: float | None, lower: float | None, upper: float | None, minimum: float | None, maximum: float | None, quantiles: dict[str, float], category_counts: dict[str, int], limitations: tuple[str, ...], metric_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerDeliveryMetricV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryResourceLimitError + +BrokerDeliveryResourceLimitError + + + + + + +histdatacom.broker_capture.fingerprints.BrokerDeliveryResourceLimitError->histdatacom.broker_capture.fingerprints.BrokerDeliveryFingerprintError + + + + + +histdatacom.synthetic.broker_transfer.BrokerTransferStatus + +BrokerTransferStatus + +name + + + + + +histdatacom.synthetic.broker_transfer.BrokerProfileSelectionV1->histdatacom.synthetic.broker_transfer.BrokerTransferStatus + + +status + + + +histdatacom.synthetic.broker_transfer.BrokerRenderLineageV1 + +BrokerRenderLineageV1 + +actions : tuple[str, ...] +input_event_id : str +input_event_time_ns : int +lineage_id : str +output_event_id : str +output_event_time_ns : int +schema_version : str +selection_id : str +symbol : str + +__init__(self, symbol: str, input_event_id: str, output_event_id: str, selection_id: str, input_event_time_ns: int, output_event_time_ns: int, actions: tuple[str, ...], lineage_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerRenderLineageV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.broker_transfer.BrokerRenderedGroupV1 + +BrokerRenderedGroupV1 + +cross_instrument_quality_payload : Mapping[str, JSONValue] | None +event_lineage : tuple[BrokerRenderLineageV1, ...] +manifest +post_broker_validation : CrossCurrencyValidationReportV1 | None +schema_version : str +status : BrokerTransferStatus +streams : tuple[SyntheticEventStreamV1, ...] + +__init__(self, manifest: BrokerTransferManifestV1, streams: tuple[SyntheticEventStreamV1, ...], event_lineage: tuple[BrokerRenderLineageV1, ...], post_broker_validation: CrossCurrencyValidationReportV1 | None, cross_instrument_quality_payload: Mapping[str, JSONValue] | None, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerRenderedGroupV1' +from_json(text: str): 'BrokerRenderedGroupV1' +metadata(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.broker_transfer.BrokerTransferManifestV1 + +BrokerTransferManifestV1 + +action_counts : Mapping[str, int] +benchmark_comparison_ids : tuple[str, ...] +cross_instrument_quality_sha256 : str | None +cross_instrument_quality_status : str | None +ensemble_member_id : str +fingerprint_id : str +input_content_sha256 : str +input_group_id : str +lineage_content_sha256 : str | None +lineage_count : int +local_validation_passed : bool +manifest_id : str +observed_event_count : int +output_content_sha256 : str | None +post_broker_validation_id : str | None +post_broker_validation_status : str | None +reason_codes : tuple[str, ...] +run_id : str +schema_version : str +selections : tuple[BrokerProfileSelectionV1, ...] +status +synchronization_unit_id : str +synthetic_event_count : int +transfer_config +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, ensemble_member_id: str, input_group_id: str, fingerprint_id: str, transfer_config: BrokerTransferConfigV1, selections: tuple[BrokerProfileSelectionV1, ...], status: BrokerTransferStatus, reason_codes: tuple[str, ...], input_content_sha256: str, output_content_sha256: str | None, observed_event_count: int, synthetic_event_count: int, action_counts: Mapping[str, int], lineage_count: int, lineage_content_sha256: str | None, local_validation_passed: bool, post_broker_validation_id: str | None, post_broker_validation_status: str | None, cross_instrument_quality_status: str | None, cross_instrument_quality_sha256: str | None, benchmark_comparison_ids: tuple[str, ...], manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'BrokerTransferManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.broker_transfer.BrokerRenderedGroupV1->histdatacom.synthetic.broker_transfer.BrokerTransferManifestV1 + + +manifest + + + +histdatacom.synthetic.broker_transfer.BrokerTransferManifestV1->histdatacom.synthetic.broker_transfer.BrokerTransferConfigV1 + + +transfer_config + + + +histdatacom.synthetic.broker_transfer.BrokerTransferManifestV1->histdatacom.synthetic.broker_transfer.BrokerTransferStatus + + +status - + histdatacom.orchestration.workflows.BuildCacheWorkflow - -BuildCacheWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +BuildCacheWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows._ActivityWorkflowBase - -_ActivityWorkflowBase - -_activity_executor : ActivityExecutor | None -_progress -workflow_name : str - -__init__(activity_executor: ActivityExecutor | None): None -_activity_executor_or_default(): ActivityExecutor -_run_activity(payload: dict[str, Any]): dict -status(): dict + +_ActivityWorkflowBase + +_activity_executor : ActivityExecutor | None +_progress +workflow_name : str + +__init__(activity_executor: ActivityExecutor | None): None +_activity_executor_or_default(): ActivityExecutor +_run_activity(payload: dict[str, Any]): dict +status(): dict - + histdatacom.orchestration.workflows.BuildCacheWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + - + histdatacom.exceptions.CacheBuildError - -CacheBuildError - -category : CACHE -code : str -retryable : bool - -__init__(code: str, message: str): None + +CacheBuildError + +category : CACHE +code : str +retryable : bool + +__init__(code: str, message: str): None - + histdatacom.exceptions.CacheBuildError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.activity_stages.CacheBuildResult - -CacheBuildResult - -end : str -filename : str -line_count : int -path : str -reused_existing : bool -schema : dict[str, str] -sha256 : str -size_bytes : int -start : str -timeframe : str - -__init__(self, filename: str, path: str, size_bytes: int, sha256: str, line_count: int, start: str, end: str, timeframe: str, schema: dict[str, str], reused_existing: bool): None -to_dict(): dict[str, JSONValue] + +CacheBuildResult + +end : str +filename : str +line_count : int +path : str +reused_existing : bool +schema : dict[str, str] +sha256 : str +size_bytes : int +start : str +timeframe : str + +__init__(self, filename: str, path: str, size_bytes: int, sha256: str, line_count: int, start: str, end: str, timeframe: str, schema: dict[str, str], reused_existing: bool): None +to_dict(): dict[str, JSONValue] - + histdatacom.activity_stages.CacheMergeSetSummary - -CacheMergeSetSummary - -artifacts : tuple[ArtifactRef, ...] -end : str -line_count : int -pair : str -record_count : int -start : str -timeframe : str -work_ids : tuple[str, ...] - -__init__(self, timeframe: str, pair: str, record_count: int, line_count: int, start: str, end: str, artifacts: tuple[ArtifactRef, ...], work_ids: tuple[str, ...]): None -to_dict(): dict[str, JSONValue] + +CacheMergeSetSummary + +artifacts : tuple[ArtifactRef, ...] +end : str +line_count : int +pair : str +record_count : int +start : str +timeframe : str +work_ids : tuple[str, ...] + +__init__(self, timeframe: str, pair: str, record_count: int, line_count: int, start: str, end: str, artifacts: tuple[ArtifactRef, ...], work_ids: tuple[str, ...]): None +to_dict(): dict[str, JSONValue] - + histdatacom.cache_status.CacheRunStatusResult - -CacheRunStatusResult - -cleanup : dict[str, Any] -disk : dict[str, Any] -errors : tuple[dict[str, str], ...] -filters : dict[str, Any] -groups : tuple[dict[str, Any], ...] -next_steps : tuple[str, ...] -root : str -runtime : dict[str, Any] -schema_version : int -status : str -summary : dict[str, Any] -symbols : tuple[dict[str, Any], ...] -workflows : dict[str, Any] - -__init__(self, root: str, status: str, filters: dict[str, Any], summary: dict[str, Any], disk: dict[str, Any], cleanup: dict[str, Any], runtime: dict[str, Any], workflows: dict[str, Any], groups: tuple[dict[str, Any], ...], symbols: tuple[dict[str, Any], ...], next_steps: tuple[str, ...], errors: tuple[dict[str, str], ...], schema_version: int): None -to_dict(): dict[str, Any] + +CacheRunStatusResult + +cleanup : dict[str, Any] +disk : dict[str, Any] +errors : tuple[dict[str, str], ...] +filters : dict[str, Any] +groups : tuple[dict[str, Any], ...] +next_steps : tuple[str, ...] +root : str +runtime : dict[str, Any] +schema_version : int +status : str +summary : dict[str, Any] +symbols : tuple[dict[str, Any], ...] +workflows : dict[str, Any] + +__init__(self, root: str, status: str, filters: dict[str, Any], summary: dict[str, Any], disk: dict[str, Any], cleanup: dict[str, Any], runtime: dict[str, Any], workflows: dict[str, Any], groups: tuple[dict[str, Any], ...], symbols: tuple[dict[str, Any], ...], next_steps: tuple[str, ...], errors: tuple[dict[str, str], ...], schema_version: int): None +to_dict(): dict[str, Any] - + histdatacom.histdata_ascii.CacheSummary - -CacheSummary - -end : int -line_count : int -start : int - -__init__(self, line_count: int, start: int, end: int): None + +CacheSummary + +end : int +line_count : int +start : int + +__init__(self, line_count: int, start: int, end: int): None + + + +histdatacom.data_quality.seasonal_exogenous.CalendarRegressorProfile + +CalendarRegressorProfile + +allow_partial_calendar : bool +max_regressors : int +require_complete_calendar_for : tuple[str, ...] + +__init__(self, allow_partial_calendar: bool, require_complete_calendar_for: tuple[str, ...], max_regressors: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] - + histdatacom.exceptions.CancellationOperationError - -CancellationOperationError - -category : CANCELLATION -code : str -retryable : bool - - + +CancellationOperationError + +category : CANCELLATION +code : str +retryable : bool + + - + histdatacom.exceptions.CancellationOperationError->histdatacom.exceptions.HistDataOperationError - - + + + + + +histdatacom.synthetic.streaming.CarryStateV1 + +CarryStateV1 + +carry_id : str +ensemble_member_id : str +last_event_ids : dict[str, str] +run_id : str +schema_version : str +state_artifacts : tuple[ArtifactRef, ...] +symbol_watermarks_ns : dict[str, int] + +__init__(self, run_id: str, ensemble_member_id: str, symbol_watermarks_ns: dict[str, int], last_event_ids: dict[str, str], state_artifacts: tuple[ArtifactRef, ...], carry_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CarryStateV1' +from_json(text: str): 'CarryStateV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.carving.CarvingBatchStatus + +CarvingBatchStatus + +name + + + + + +histdatacom.synthetic.carving.CarvingEventAction + +CarvingEventAction + +name + + + + + +histdatacom.synthetic.carving.CarvingFingerprintEvidenceV1 + +CarvingFingerprintEvidenceV1 + +candidate_batch_ids : tuple[str, ...] +candidate_report_id : str +evidence_id : str +reference_report_id : str +schema_version : str +status : str +validation_payload : Mapping[str, JSONValue] + +__init__(self, validation_payload: Mapping[str, JSONValue], candidate_batch_ids: tuple[str, ...], reference_report_id: str, candidate_report_id: str, evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CarvingFingerprintEvidenceV1' +from_json(text: str): 'CarvingFingerprintEvidenceV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.carving.CarvingReason + +CarvingReason + +name + + + + + +histdatacom.synthetic.certification.CertificationArtifactV1 + +CertificationArtifactV1 + +content_sha256 : str +evidence_id : str +kind : str +metadata : Mapping[str, JSONValue] +policy_id : str +relative_path : str +schema_version : str +size_bytes : int +subject_id : str +subject_schema_version : str +verified : bool + +__init__(self, policy_id: str, kind: str, subject_id: str, subject_schema_version: str, content_sha256: str, relative_path: str, size_bytes: int, verified: bool, metadata: Mapping[str, JSONValue], evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CertificationArtifactV1' +from_payload(): 'CertificationArtifactV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification_campaign.CertificationCampaignArtifactV1 + +CertificationCampaignArtifactV1 + +content_sha256 : str +evidence_key : str +kind : str +metadata : Mapping[str, JSONValue] +path : str +relative_path : str +schema_version : str +subject_id : str +subject_id_pointer : str +subject_schema_version : str + +__init__(self, evidence_key: str, kind: str, path: str, content_sha256: str, subject_id: str, subject_id_pointer: str, subject_schema_version: str, relative_path: str, metadata: Mapping[str, JSONValue], schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CertificationCampaignArtifactV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification_campaign.CertificationCampaignObservationV1 + +CertificationCampaignObservationV1 + +artifact_evidence_keys : tuple[str, ...] +check_id : str +measurement_evidence_key : str +measurement_pointer : str +note : str +schema_version : str + +__init__(self, check_id: str, measurement_evidence_key: str, measurement_pointer: str, artifact_evidence_keys: tuple[str, ...], note: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CertificationCampaignObservationV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification.CertificationCheckResultV1 + +CertificationCheckResultV1 + +actual +artifact_evidence_ids : tuple[str, ...] +check_id : str +comparator +expected +passed : bool +reason : str +requirement_id : str +result_id : str +schema_version : str +status + +__init__(self, requirement_id: str, check_id: str, status: CertificationCheckStatus, comparator: CertificationComparator, expected: JSONScalar, actual: JSONScalar, artifact_evidence_ids: tuple[str, ...], reason: str, result_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CertificationCheckResultV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification.CertificationCheckStatus + +CertificationCheckStatus + +name + + + + + +histdatacom.synthetic.certification.CertificationCheckResultV1->histdatacom.synthetic.certification.CertificationCheckStatus + + +status + + + +histdatacom.synthetic.certification.CertificationComparator + +CertificationComparator + +name + + + + + +histdatacom.synthetic.certification.CertificationCheckResultV1->histdatacom.synthetic.certification.CertificationComparator + + +comparator + + + +histdatacom.synthetic.certification.CertificationGate + +CertificationGate + +name + + + + + +histdatacom.synthetic.certification.CertificationGateResultV1 + +CertificationGateResultV1 + +check_results : tuple[CertificationCheckResultV1, ...] +gate +gate_result_id : str +schema_version : str +status : CertificationCheckStatus + +__init__(self, gate: CertificationGate, check_results: tuple[CertificationCheckResultV1, ...], gate_result_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CertificationGateResultV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification.CertificationGateResultV1->histdatacom.synthetic.certification.CertificationGate + + +gate + + + +histdatacom.synthetic.certification.CertificationObservationV1 + +CertificationObservationV1 + +actual +artifact_evidence_ids : tuple[str, ...] +check_id : str +note : str +observation_id : str +schema_version : str + +__init__(self, check_id: str, actual: JSONScalar, artifact_evidence_ids: tuple[str, ...], note: str, observation_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CertificationObservationV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification.CertificationRequirementV1 + +CertificationRequirementV1 + +check_id : str +comparator +description : str +expected +gate +required_artifact_kinds : tuple[str, ...] +requirement_id : str +schema_version : str + +__init__(self, gate: CertificationGate, check_id: str, comparator: CertificationComparator, expected: JSONScalar, required_artifact_kinds: tuple[str, ...], description: str, requirement_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CertificationRequirementV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification.CertificationRequirementV1->histdatacom.synthetic.certification.CertificationComparator + + +comparator + + + +histdatacom.synthetic.certification.CertificationRequirementV1->histdatacom.synthetic.certification.CertificationGate + + +gate + + + +histdatacom.synthetic.certification.CertificationState + +CertificationState + +name + + + + + +histdatacom.market_context.positioning.CftcArchiveConsistencyV1 + +CftcArchiveConsistencyV1 + +contract_name_change_count : int +evidence_id : str +limitations : tuple[str, ...] +matched_pre_rows : int +missing_pre_rows : int +open_interest_mismatch_count : int +report_family +report_scope +schema_version : str +selected_row_count : int +source_id : str + +__init__(self, source_id: str, report_family: CftcReportFamily, report_scope: CftcReportScope, selected_row_count: int, matched_pre_rows: int, missing_pre_rows: int, open_interest_mismatch_count: int, contract_name_change_count: int, limitations: tuple[str, ...], evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcArchiveConsistencyV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcReportFamily + +CftcReportFamily + +name + +from_value(value: str | 'CftcReportFamily'): 'CftcReportFamily' + + + +histdatacom.market_context.positioning.CftcArchiveConsistencyV1->histdatacom.market_context.positioning.CftcReportFamily + + +report_family + + + +histdatacom.market_context.positioning.CftcReportScope + +CftcReportScope + +name + +from_value(value: str | 'CftcReportScope'): 'CftcReportScope' + + + +histdatacom.market_context.positioning.CftcArchiveConsistencyV1->histdatacom.market_context.positioning.CftcReportScope + + +report_scope + + + +histdatacom.market_context.positioning.CftcAvailabilityConfidence + +CftcAvailabilityConfidence + +name + +from_value(value: str | 'CftcAvailabilityConfidence'): 'CftcAvailabilityConfidence' + + + +histdatacom.market_context.positioning.CftcMappingKind + +CftcMappingKind + +name + + + + + +histdatacom.market_context.positioning.CftcPositioningBenchmarkSmokeV1 + +CftcPositioningBenchmarkSmokeV1 + +benchmark_event_ids : tuple[str, ...] +binding_id : str +corpus_id : str +deterministic_reload : bool +logical_output_sha256 : str +query_id : str +reload_output_sha256 : str +schema_version : str +smoke_id : str +source_artifact_id : str +source_row_count : int +source_sha256 : str + +__init__(self, corpus_id: str, query_id: str, binding_id: str, source_artifact_id: str, source_sha256: str, source_row_count: int, benchmark_event_ids: tuple[str, ...], logical_output_sha256: str, reload_output_sha256: str, deterministic_reload: bool, smoke_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningBenchmarkSmokeV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningConsumer + +CftcPositioningConsumer + +name + + + + + +histdatacom.market_context.positioning.CftcPositioningConsumerBindingV1 + +CftcPositioningConsumerBindingV1 + +binding_id : str +consumer +consumer_artifact_id : str +corpus_id : str +information_input_ids : tuple[str, ...] +metrics : Mapping[str, float] +query_id : str +run_id : str +schema_version : str +snapshot_ids : tuple[str, ...] +state_label : str +window_id : str + +__init__(self, consumer: CftcPositioningConsumer, consumer_artifact_id: str, run_id: str, window_id: str, corpus_id: str, query_id: str, snapshot_ids: tuple[str, ...], information_input_ids: tuple[str, ...], state_label: str, metrics: Mapping[str, float], binding_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningConsumerBindingV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningConsumerBindingV1->histdatacom.market_context.positioning.CftcPositioningConsumer + + +consumer + + + +histdatacom.market_context.positioning.CftcPositioningCorpusBuildV1 + +CftcPositioningCorpusBuildV1 + +corpus +raw_sources : tuple[CftcPositioningRawSourceV1, ...] + +__init__(self, corpus: CftcPositioningCorpusV1, raw_sources: tuple[CftcPositioningRawSourceV1, ...]): None +__post_init__(): None + + + +histdatacom.market_context.positioning.CftcPositioningCorpusV1 + +CftcPositioningCorpusV1 + +archive_consistency : tuple[CftcArchiveConsistencyV1, ...] +corpus_id : str +coverage : tuple[CftcPositioningCoverageSliceV1, ...] +duplicate_key_count : int +limitations : tuple[str, ...] +peak_memory_bytes : int +profile +runtime_seconds : float +schema_version : str +snapshots : tuple[CftcPositioningSnapshotV1, ...] +sources : tuple[Mapping[str, JSONValue], ...] +symbol_mappings : tuple[CftcPositioningSymbolMappingV1, ...] + +__init__(self, profile: CftcPositioningFetchProfileV1, sources: tuple[Mapping[str, JSONValue], ...], symbol_mappings: tuple[CftcPositioningSymbolMappingV1, ...], snapshots: tuple[CftcPositioningSnapshotV1, ...], coverage: tuple[CftcPositioningCoverageSliceV1, ...], archive_consistency: tuple[CftcArchiveConsistencyV1, ...], duplicate_key_count: int, runtime_seconds: float, peak_memory_bytes: int, limitations: tuple[str, ...], corpus_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningCorpusV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningCorpusBuildV1->histdatacom.market_context.positioning.CftcPositioningCorpusV1 + + +corpus + + + +histdatacom.market_context.positioning.CftcPositioningFetchProfileV1 + +CftcPositioningFetchProfileV1 + +contract_codes : tuple[str, ...] +dataset_ids : tuple[str, ...] +end_date : str +historical_archives : tuple[str, ...] +max_pages_per_dataset : int +max_peak_memory_bytes : int +max_response_bytes : int +max_rows : int +max_runtime_seconds : float +max_staleness_days : int +max_total_source_bytes : int +page_size : int +schema_version : str +start_date : str +timeout_seconds : float +user_agent : str + +__init__(self, start_date: str, end_date: str, dataset_ids: tuple[str, ...], contract_codes: tuple[str, ...], historical_archives: tuple[str, ...], page_size: int, max_pages_per_dataset: int, timeout_seconds: float, max_response_bytes: int, max_total_source_bytes: int, max_rows: int, max_runtime_seconds: float, max_peak_memory_bytes: int, max_staleness_days: int, user_agent: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningFetchProfileV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningCorpusV1->histdatacom.market_context.positioning.CftcPositioningFetchProfileV1 + + +profile + + + +histdatacom.market_context.positioning.CftcPositioningCoverageSliceV1 + +CftcPositioningCoverageSliceV1 + +availability_counts : Mapping[str, int] +contract_code : str +contract_names : tuple[str, ...] +duplicate_key_count : int +first_report_date : str +last_report_date : str +market_names : tuple[str, ...] +missing_week_count : int +processing_seconds : float +report_family +report_scope +restatement_counts : Mapping[str, int] +row_count : int +schema_version : str +source_bytes : int +source_hashes : tuple[str, ...] +year : int + +__init__(self, year: int, report_family: CftcReportFamily, report_scope: CftcReportScope, contract_code: str, row_count: int, first_report_date: str, last_report_date: str, missing_week_count: int, duplicate_key_count: int, contract_names: tuple[str, ...], market_names: tuple[str, ...], availability_counts: Mapping[str, int], restatement_counts: Mapping[str, int], source_hashes: tuple[str, ...], source_bytes: int, processing_seconds: float, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningCoverageSliceV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningCoverageSliceV1->histdatacom.market_context.positioning.CftcReportFamily + + +report_family + + + +histdatacom.market_context.positioning.CftcPositioningCoverageSliceV1->histdatacom.market_context.positioning.CftcReportScope + + +report_scope + + + +histdatacom.market_context.positioning.CftcPositioningDiffV1 + +CftcPositioningDiffV1 + +added_keys : tuple[str, ...] +changed_keys : tuple[str, ...] +current_corpus_id : str +current_snapshot_ids : Mapping[str, str] +diff_id : str +previous_corpus_id : str +previous_snapshot_ids : Mapping[str, str] +removed_keys : tuple[str, ...] +schema_version : str + +__init__(self, previous_corpus_id: str, current_corpus_id: str, added_keys: tuple[str, ...], removed_keys: tuple[str, ...], changed_keys: tuple[str, ...], previous_snapshot_ids: Mapping[str, str], current_snapshot_ids: Mapping[str, str], diff_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningDiffV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningPreflightError + +CftcPositioningPreflightError + +decision + +__init__(decision: CftcPositioningPreflightV1): None + + + +histdatacom.market_context.positioning.CftcPositioningPreflightV1 + +CftcPositioningPreflightV1 + +as_of_ns : int | None +corpus_id : str +end_ns : int +information_mode +ready : bool +reasons : tuple[str, ...] +report_families : tuple[CftcReportFamily, ...] +report_scopes : tuple[CftcReportScope, ...] +schema_version : str +start_ns : int +symbols : tuple[str, ...] + +__init__(self, corpus_id: str, start_ns: int, end_ns: int, information_mode: InformationMode, as_of_ns: int | None, symbols: tuple[str, ...], report_families: tuple[CftcReportFamily, ...], report_scopes: tuple[CftcReportScope, ...], ready: bool, reasons: tuple[str, ...], schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningPreflightV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningPreflightV1->histdatacom.market_context.positioning.CftcPositioningPreflightError + + +decision + + + +histdatacom.synthetic.information.InformationMode + +InformationMode + +name + +from_value(value: str | 'InformationMode'): 'InformationMode' + + + +histdatacom.market_context.positioning.CftcPositioningPreflightV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.market_context.positioning.CftcPositioningQueryStatus + +CftcPositioningQueryStatus + +name + + + + + +histdatacom.market_context.positioning.CftcPositioningQueryV1 + +CftcPositioningQueryV1 + +age_seconds : Mapping[str, int] +as_of_ns : int | None +corpus_id : str +derived_values : Mapping[str, float] +end_ns : int +information_mode +mapping_kinds : Mapping[str, str] +query_id : str +reason : str +report_families : tuple[CftcReportFamily, ...] +report_scopes : tuple[CftcReportScope, ...] +schema_version : str +snapshots : tuple[CftcPositioningSnapshotV1, ...] +start_ns : int +status +symbol_snapshot_ids : Mapping[str, tuple[str, ...]] +symbols : tuple[str, ...] + +__init__(self, corpus_id: str, information_mode: InformationMode, start_ns: int, end_ns: int, as_of_ns: int | None, symbols: tuple[str, ...], report_families: tuple[CftcReportFamily, ...], report_scopes: tuple[CftcReportScope, ...], snapshots: tuple[CftcPositioningSnapshotV1, ...], symbol_snapshot_ids: Mapping[str, tuple[str, ...]], mapping_kinds: Mapping[str, str], derived_values: Mapping[str, float], status: CftcPositioningQueryStatus, reason: str, age_seconds: Mapping[str, int], query_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningQueryV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningQueryV1->histdatacom.market_context.positioning.CftcPositioningQueryStatus + + +status + + + +histdatacom.market_context.positioning.CftcPositioningQueryV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.market_context.positioning.CftcPositioningRawSourceV1 + +CftcPositioningRawSourceV1 + +content : bytes +content_sha256 : str +content_type : str +dataset_id : str | None +limitations : tuple[str, ...] +query_parameters : Mapping[str, str] +redistribution_allowed : bool +report_family : CftcReportFamily | None +report_scope : CftcReportScope | None +retrieved_at_ns : int +schema_version : str +source_id : str +source_key : str +source_kind : str +source_uri : str + +__init__(self, source_key: str, source_kind: str, source_uri: str, retrieved_at_ns: int, content: bytes, content_type: str, query_parameters: Mapping[str, str], dataset_id: str | None, report_family: CftcReportFamily | None, report_scope: CftcReportScope | None, redistribution_allowed: bool, limitations: tuple[str, ...], schema_version: str): None +__post_init__(): None +evidence_dict(): dict[str, JSONValue] +identity_payload(): dict[str, JSONValue] +restore(data: Mapping[str, Any], content: bytes): 'CftcPositioningRawSourceV1' + + + +histdatacom.market_context.positioning.CftcPositioningSnapshotV1 + +CftcPositioningSnapshotV1 + +contract_code : str +contract_name : str +dataset_id : str +logical_key : str +market_name : str +measurement_start_ns : int +pre_row_id : str +release_evidence +report_date : str +report_family +report_scope +restatement_status +schema_version : str +snapshot_id : str +source_id : str +source_row_sha256 : str +strict_ex_ante_eligible : bool +valid_from_ns : int | None +values : Mapping[str, float] + +__init__(self, report_family: CftcReportFamily, report_scope: CftcReportScope, contract_code: str, contract_name: str, market_name: str, report_date: str, dataset_id: str, source_id: str, source_row_sha256: str, pre_row_id: str, release_evidence: CftcReleaseEvidenceV1, restatement_status: CftcRestatementStatus, values: Mapping[str, float], snapshot_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningSnapshotV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcReleaseEvidenceV1 + +CftcReleaseEvidenceV1 + +confidence +evidence_id : str +knowledge_at_ns : int | None +notes : tuple[str, ...] +publication_at_ns : int | None +report_date : str +restatement_detected_at_ns : int | None +schema_version : str +source_id : str +strict_ex_ante_time_eligible : bool + +__init__(self, report_date: str, confidence: CftcAvailabilityConfidence, source_id: str, publication_at_ns: int | None, knowledge_at_ns: int | None, restatement_detected_at_ns: int | None, notes: tuple[str, ...], evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcReleaseEvidenceV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningSnapshotV1->histdatacom.market_context.positioning.CftcReleaseEvidenceV1 + + +release_evidence + + + +histdatacom.market_context.positioning.CftcPositioningSnapshotV1->histdatacom.market_context.positioning.CftcReportFamily + + +report_family + + + +histdatacom.market_context.positioning.CftcPositioningSnapshotV1->histdatacom.market_context.positioning.CftcReportScope + + +report_scope + + + +histdatacom.market_context.positioning.CftcRestatementStatus + +CftcRestatementStatus + +name + +from_value(value: str | 'CftcRestatementStatus'): 'CftcRestatementStatus' + + + +histdatacom.market_context.positioning.CftcPositioningSnapshotV1->histdatacom.market_context.positioning.CftcRestatementStatus + + +restatement_status + + + +histdatacom.market_context.positioning.CftcPositioningSymbolMappingV1 + +CftcPositioningSymbolMappingV1 + +contract_codes : tuple[str, ...] +mapping_id : str +mapping_kind +notes : tuple[str, ...] +quote_convention : str +schema_version : str +source_uris : tuple[str, ...] +symbol : str +valid_from_date : str | None + +__init__(self, symbol: str, contract_codes: tuple[str, ...], mapping_kind: CftcMappingKind, quote_convention: str, source_uris: tuple[str, ...], valid_from_date: str | None, notes: tuple[str, ...], mapping_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CftcPositioningSymbolMappingV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.positioning.CftcPositioningSymbolMappingV1->histdatacom.market_context.positioning.CftcMappingKind + + +mapping_kind + + + +histdatacom.market_context.positioning.CftcReleaseEvidenceV1->histdatacom.market_context.positioning.CftcAvailabilityConfidence + + +confidence + + + +histdatacom.market_context.positioning.CftcSourceFetcherV1 + +CftcSourceFetcherV1 + + +__call__(url: str): tuple[bytes, str, str] - + histdatacom.orchestration.workflows.ChildWorkflowExecutor - -ChildWorkflowExecutor - - -execute_child_workflow -(workflow_name: str, payload: Mapping[str, JSONValue]): Mapping[str, Any] + +ChildWorkflowExecutor + + +execute_child_workflow +(workflow_name: str, payload: Mapping[str, JSONValue]): Mapping[str, Any] + + + +histdatacom.data_quality.classical_baselines.ClassicalBaselineProfile + +ClassicalBaselineProfile + +enabled : bool +evaluation_fraction : float +minimum_evaluation_rows : int +minimum_training_rows : int +rolling_windows : tuple[int, ...] +rounding_digits : int +session_seasonal_enabled : bool + +__init__(self, enabled: bool, evaluation_fraction: float, minimum_training_rows: int, minimum_evaluation_rows: int, rolling_windows: tuple[int, ...], session_seasonal_enabled: bool, rounding_digits: int): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.classical_model_comparison.ClassicalModelComparisonProfile + +ClassicalModelComparisonProfile + +drift_tolerance : float +enabled : bool +max_comparisons : int +max_horizons : int +max_models : int +max_reason_codes : int +max_samples : int +mean_reference_baseline : str +minimum_stability_folds : int +near_zero_tolerance : float +rounding_digits : int +variance_reference_baseline : str + +__init__(self, enabled: bool, mean_reference_baseline: str, variance_reference_baseline: str, near_zero_tolerance: float, minimum_stability_folds: int, drift_tolerance: float, max_models: int, max_horizons: int, max_comparisons: int, max_reason_codes: int, max_samples: int, rounding_digits: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.classical_model_comparison.ClassicalModelComparisonResult + +ClassicalModelComparisonResult + +annotations : tuple[Mapping[str, Any], ...] +diagnostics : Mapping[str, JSONValue] + +__init__(self, diagnostics: Mapping[str, JSONValue], annotations: tuple[Mapping[str, Any], ...]): None + + + +histdatacom.data_quality.classical_model_contracts.ClassicalModelInputProfile + +ClassicalModelInputProfile + +alignment_epoch_ms : int +closed_side : str +differencing_order : int +embargo_observations : int +enabled : bool +expected_closure_policy : str +fold_kind : str +frequency_ms : int +horizons : tuple[int, ...] +label_side : str +midpoint_aggregation : str +minimum_evaluation_observations : int +minimum_observations_per_bin : int +minimum_training_observations : int +resources +rolling_window : int +rounding_digits : int +seasonal_differencing_order : int +seasonal_period : int +spread_aggregation : str +step_size : int +transform : str +unexpected_missing_policy : str + +__init__(self, enabled: bool, frequency_ms: int, alignment_epoch_ms: int, closed_side: str, label_side: str, midpoint_aggregation: str, spread_aggregation: str, minimum_observations_per_bin: int, expected_closure_policy: str, unexpected_missing_policy: str, transform: str, differencing_order: int, seasonal_differencing_order: int, seasonal_period: int, horizons: tuple[int, ...], fold_kind: str, minimum_training_observations: int, minimum_evaluation_observations: int, step_size: int, rolling_window: int, embargo_observations: int, rounding_digits: int, resources: ClassicalModelResourcePolicy): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.classical_model_contracts.ClassicalModelResourcePolicy + +ClassicalModelResourcePolicy + +max_candidate_orders : int +max_fit_attempts : int +max_folds : int +max_horizons : int +max_memory_bytes : int +max_regularized_observations : int +max_retained_diagnostics : int +max_source_rows : int +max_wall_time_seconds : int + +__init__(self, max_source_rows: int, max_regularized_observations: int, max_folds: int, max_horizons: int, max_candidate_orders: int, max_fit_attempts: int, max_wall_time_seconds: int, max_memory_bytes: int, max_retained_diagnostics: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.classical_model_contracts.ClassicalModelInputProfile->histdatacom.data_quality.classical_model_contracts.ClassicalModelResourcePolicy + + +resources - + histdatacom.cancellation.CleanupResult - -CleanupResult - -error : str -existed : bool -path : str -removed : bool - -__init__(self, path: str, existed: bool, removed: bool, error: str): None -to_dict(): dict[str, JSONValue] + +CleanupResult + +error : str +existed : bool +path : str +removed : bool + +__init__(self, path: str, existed: bool, removed: bool, error: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.cli_config.CliConfigError - -CliConfigError - - - + +CliConfigError + + + - + histdatacom.exceptions.CliValidationError - -CliValidationError - -category : VALIDATION -code : str -exit_code : int -retryable : bool - - + +CliValidationError + +category : VALIDATION +code : str +exit_code : int +retryable : bool + + - + histdatacom.exceptions.CliValidationError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.exceptions.ConfigurationError - -ConfigurationError - -category : CONFIGURATION -code : str -retryable : bool - - + +ConfigurationError + +category : CONFIGURATION +code : str +retryable : bool + + - + histdatacom.exceptions.ConfigurationError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.orchestration.control.ControlOperationName - -ControlOperationName - -name - - + +ControlOperationName + +name + + - + histdatacom.orchestration.control.ControlOperationPhase - -ControlOperationPhase - -name - - + +ControlOperationPhase + +name + + - + histdatacom.orchestration.control.ControlOperationState - -ControlOperationState - -available : bool -message : str -metadata : dict[str, JSONValue] -name -phase -reason : str -requested_at_utc : str - -__init__(self, name: ControlOperationName, phase: ControlOperationPhase, available: bool, requested_at_utc: str, reason: str, message: str, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'ControlOperationState' -to_dict(): dict[str, JSONValue] -transition(phase: ControlOperationPhase): 'ControlOperationState' -unavailable(name: ControlOperationName): 'ControlOperationState' + +ControlOperationState + +available : bool +message : str +metadata : dict[str, JSONValue] +name +phase +reason : str +requested_at_utc : str + +__init__(self, name: ControlOperationName, phase: ControlOperationPhase, available: bool, requested_at_utc: str, reason: str, message: str, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'ControlOperationState' +to_dict(): dict[str, JSONValue] +transition(phase: ControlOperationPhase): 'ControlOperationState' +unavailable(name: ControlOperationName): 'ControlOperationState' - + histdatacom.orchestration.control.ControlOperationState->histdatacom.orchestration.control.ControlOperationName - - -name + + +name - + histdatacom.orchestration.control.ControlOperationState->histdatacom.orchestration.control.ControlOperationPhase - - -phase + + +phase - + histdatacom.data_quality.manifest.CoverageDimension - -CoverageDimension - -data_format : str -period : str -symbol : str -timeframe : str - -__init__(self, data_format: str, timeframe: str, symbol: str, period: str): None -from_mapping(data: Mapping[str, Any]): 'CoverageDimension' -from_target(target: QualityTarget): 'CoverageDimension | None' -to_dict(): dict[str, JSONValue] + +CoverageDimension + +data_format : str +period : str +symbol : str +timeframe : str + +__init__(self, data_format: str, timeframe: str, symbol: str, period: str): None +from_mapping(data: Mapping[str, Any]): 'CoverageDimension' +from_target(target: QualityTarget): 'CoverageDimension | None' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencyConditionV1 + +CrossCurrencyConditionV1 + +condition_id : str +end_ns : int +event_key : str +feed_epoch_key : str +schema_version : str +session_key : str +start_ns : int + +__init__(self, start_ns: int, end_ns: int, session_key: str, event_key: str, feed_epoch_key: str, condition_id: str, schema_version: str): None +__post_init__(): None +covers(timestamp_ns: int): bool +from_dict(data: Mapping[str, Any]): 'CrossCurrencyConditionV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencyCoverageStatus + +CrossCurrencyCoverageStatus + +name + + + + + +histdatacom.synthetic.cross_currency.CrossCurrencyExcludedReason + +CrossCurrencyExcludedReason + +name + + + + + +histdatacom.synthetic.cross_currency.CrossCurrencyExcludedSpanV1 + +CrossCurrencyExcludedSpanV1 + +end_ns : int +reason +schema_version : str +start_ns : int +symbol : str + +__init__(self, symbol: str, start_ns: int, end_ns: int, reason: CrossCurrencyExcludedReason, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyExcludedSpanV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencyExcludedSpanV1->histdatacom.synthetic.cross_currency.CrossCurrencyExcludedReason + + +reason + + + +histdatacom.synthetic.cross_currency.CrossCurrencyGroupStatus + +CrossCurrencyGroupStatus + +name + + + + + +histdatacom.synthetic.cross_currency.CrossCurrencyProjectionLineageV1 + +CrossCurrencyProjectionLineageV1 + +allowed_residual : float +condition_ids : tuple[str, ...] +event_sequence : int +event_time_ns : int +input_content_sha256 : str +input_event_id : str +lineage_id : str +original_ask : float +original_bid : float +output_ask : float +output_bid : float +output_content_sha256 : str +output_event_id : str +post_residual : float +pre_residual : float +projection_relative : float +relationship_id : str +schema_version : str +symbol : str + +__init__(self, relationship_id: str, symbol: str, event_time_ns: int, event_sequence: int, input_event_id: str, output_event_id: str, input_content_sha256: str, output_content_sha256: str, original_bid: float, original_ask: float, output_bid: float, output_ask: float, pre_residual: float, post_residual: float, allowed_residual: float, projection_relative: float, condition_ids: tuple[str, ...], lineage_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyProjectionLineageV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencyReconciledGroupV1 + +CrossCurrencyReconciledGroupV1 + +condition_ids : tuple[str, ...] +config +ensemble_member_id : str +generation_ready : bool +generation_validation +group_id : str +input_stream_ids : dict[str, str] +missing_symbols : tuple[str, ...] +projection_lineage : tuple[CrossCurrencyProjectionLineageV1, ...] +requires_post_broker_validation : bool +run_id : str +schema_version : str +status +streams : tuple[SyntheticEventStreamV1, ...] +symbols : tuple[str, ...] +synchronization_unit_id : str +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, ensemble_member_id: str, symbols: tuple[str, ...], status: CrossCurrencyGroupStatus, streams: tuple[SyntheticEventStreamV1, ...], missing_symbols: tuple[str, ...], input_stream_ids: dict[str, str], config: CrossCurrencyReconciliationConfigV1, condition_ids: tuple[str, ...], projection_lineage: tuple[CrossCurrencyProjectionLineageV1, ...], generation_validation: CrossCurrencyValidationReportV1, group_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyReconciledGroupV1' +from_json(text: str): 'CrossCurrencyReconciledGroupV1' +metadata(): dict[str, JSONValue] +payload(): dict[str, JSONValue] +stream_for(symbol: str): SyntheticEventStreamV1 +to_dict(): dict[str, JSONValue] +to_json(): str +validate_atomic_manifest(manifest: PartitionManifestV1): None + + + +histdatacom.synthetic.cross_currency.CrossCurrencyReconciledGroupV1->histdatacom.synthetic.cross_currency.CrossCurrencyGroupStatus + + +status + + + +histdatacom.synthetic.cross_currency.CrossCurrencyReconciliationConfigV1 + +CrossCurrencyReconciliationConfigV1 + +config_id : str +max_projection_relative : float +relationships : tuple[CrossCurrencyRelationshipV1, ...] +residual_tolerance : float +rounding_digits : int +schema_version : str +spread_tolerance_multiplier : float +symbols : tuple[str, ...] + +__init__(self, relationships: tuple[CrossCurrencyRelationshipV1, ...], max_projection_relative: float, residual_tolerance: float, spread_tolerance_multiplier: float, rounding_digits: int, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyReconciliationConfigV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencyReconciledGroupV1->histdatacom.synthetic.cross_currency.CrossCurrencyReconciliationConfigV1 + + +config + + + +histdatacom.synthetic.cross_currency.CrossCurrencyValidationReportV1 + +CrossCurrencyValidationReportV1 + +anchor_preserved : bool +asynchronous_timestamp_count : int +common_timestamp_count : int +config_id : str +duplicate_timestamp_event_count : int +ensemble_member_id : str +failure_reasons : tuple[str, ...] +observed_event_count : int +output_content_sha256 : str +passed : bool +relationship_support : tuple[CrossCurrencyRelationshipSupportV1, ...] +residual_slices : tuple[CrossCurrencyResidualSliceV1, ...] +run_id : str +schema_version : str +stage +stale_join_risk_count : int +status +symbols : tuple[str, ...] +synchronization_unit_id : str +union_timestamp_count : int +validation_id : str +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, ensemble_member_id: str, symbols: tuple[str, ...], config_id: str, stage: CrossCurrencyValidationStage, status: CrossCurrencyValidationStatus, relationship_support: tuple[CrossCurrencyRelationshipSupportV1, ...], residual_slices: tuple[CrossCurrencyResidualSliceV1, ...], union_timestamp_count: int, common_timestamp_count: int, asynchronous_timestamp_count: int, duplicate_timestamp_event_count: int, stale_join_risk_count: int, observed_event_count: int, anchor_preserved: bool, output_content_sha256: str, failure_reasons: tuple[str, ...], validation_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyValidationReportV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencyReconciledGroupV1->histdatacom.synthetic.cross_currency.CrossCurrencyValidationReportV1 + + +generation_validation + + + +histdatacom.synthetic.cross_currency.CrossCurrencyRelationshipKind + +CrossCurrencyRelationshipKind + +name + + + + + +histdatacom.synthetic.cross_currency.CrossCurrencyRelationshipSupportV1 + +CrossCurrencyRelationshipSupportV1 + +allowed_residual_max : float +infeasible_count : int +post_residual_max : float +post_residual_mean : float +pre_residual_max : float +pre_residual_mean : float +projected_count : int +relationship_id : str +schema_version : str +support_count : int + +__init__(self, relationship_id: str, support_count: int, projected_count: int, infeasible_count: int, pre_residual_max: float, pre_residual_mean: float, post_residual_max: float, post_residual_mean: float, allowed_residual_max: float, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyRelationshipSupportV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencyRelationshipV1 + +CrossCurrencyRelationshipV1 + +kind +projection_priority : tuple[str, ...] +relationship_id : str +schema_version : str +symbols : tuple[str, ...] + +__init__(self, kind: CrossCurrencyRelationshipKind, symbols: tuple[str, ...], projection_priority: tuple[str, ...], relationship_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyRelationshipV1' +inverse(): 'CrossCurrencyRelationshipV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +triangle(): 'CrossCurrencyRelationshipV1' + + + +histdatacom.synthetic.cross_currency.CrossCurrencyRelationshipV1->histdatacom.synthetic.cross_currency.CrossCurrencyRelationshipKind + + +kind + + + +histdatacom.synthetic.cross_currency.CrossCurrencyResidualSliceV1 + +CrossCurrencyResidualSliceV1 + +dimension : str +infeasible_count : int +key : str +post_residual_max : float +pre_residual_max : float +projected_count : int +relationship_id : str +schema_version : str +support_count : int + +__init__(self, relationship_id: str, dimension: str, key: str, support_count: int, projected_count: int, infeasible_count: int, pre_residual_max: float, post_residual_max: float, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyResidualSliceV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencySymbolCoverageV1 + +CrossCurrencySymbolCoverageV1 + +end_ns : int | None +schema_version : str +source_periods : tuple[str, ...] +start_ns : int | None +status +symbol : str + +__init__(self, symbol: str, start_ns: int | None, end_ns: int | None, source_periods: tuple[str, ...], status: CrossCurrencyCoverageStatus, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencySymbolCoverageV1' +missing(symbol: str): 'CrossCurrencySymbolCoverageV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.cross_currency.CrossCurrencySymbolCoverageV1->histdatacom.synthetic.cross_currency.CrossCurrencyCoverageStatus + + +status + + + +histdatacom.synthetic.cross_currency.CrossCurrencyValidationStage + +CrossCurrencyValidationStage + +name + + + + + +histdatacom.synthetic.cross_currency.CrossCurrencyValidationReportV1->histdatacom.synthetic.cross_currency.CrossCurrencyValidationStage + + +stage + + + +histdatacom.synthetic.cross_currency.CrossCurrencyValidationStatus + +CrossCurrencyValidationStatus + +name + + + + + +histdatacom.synthetic.cross_currency.CrossCurrencyValidationReportV1->histdatacom.synthetic.cross_currency.CrossCurrencyValidationStatus + + +status + + + +histdatacom.synthetic.cross_currency.CrossCurrencyWindowPlanStatus + +CrossCurrencyWindowPlanStatus + +name + + + + + +histdatacom.synthetic.cross_currency.CrossCurrencyWindowPlanV1 + +CrossCurrencyWindowPlanV1 + +common_end_ns : int | None +common_start_ns : int | None +coverages : tuple[CrossCurrencySymbolCoverageV1, ...] +ensemble_member_id : str +excluded_spans : tuple[CrossCurrencyExcludedSpanV1, ...] +missing_symbols : tuple[str, ...] +plan_id : str +requested_end_ns : int +requested_start_ns : int +run_id : str +schema_version : str +status +symbols : tuple[str, ...] +windows : tuple[ReconstructionWindowV1, ...] + +__init__(self, run_id: str, ensemble_member_id: str, symbols: tuple[str, ...], requested_start_ns: int, requested_end_ns: int, coverages: tuple[CrossCurrencySymbolCoverageV1, ...], excluded_spans: tuple[CrossCurrencyExcludedSpanV1, ...], windows: tuple[ReconstructionWindowV1, ...], status: CrossCurrencyWindowPlanStatus, missing_symbols: tuple[str, ...], plan_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'CrossCurrencyWindowPlanV1' +from_json(text: str): 'CrossCurrencyWindowPlanV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.cross_currency.CrossCurrencyWindowPlanV1->histdatacom.synthetic.cross_currency.CrossCurrencyWindowPlanStatus + + +status + + + +histdatacom.data_quality.symbols.CrossInstrumentPointInput + +CrossInstrumentPointInput + +event_seq : int +price : float +row_id : int +source_row_number : int +timestamp_utc_ms : int + +__init__(self, timestamp_utc_ms: int, price: float, row_id: int, source_row_number: int, event_seq: int): None +__post_init__(): None + + + +histdatacom.data_quality.symbols.CrossInstrumentScanProvider + +CrossInstrumentScanProvider + +_scan : _CrossInstrumentScan | None +_signature : tuple[object, ...] | None + +__init__ +(): None +scan(targets: tuple[QualityTarget, ...]): _CrossInstrumentScan + + + +histdatacom.data_quality.symbols.CrossInstrumentSeriesInput + +CrossInstrumentSeriesInput + +computed_from : str +path : str +period : str +points : tuple[CrossInstrumentPointInput, ...] +series_id : str +symbol : str +timeframe : str + +__init__(self, symbol: str, timeframe: str, period: str, series_id: str, points: tuple[CrossInstrumentPointInput, ...], path: str, computed_from: str): None +__post_init__(): None - + histdatacom.csvs.Csv - -Csv - -args : dict[str, Any] - -__init__(args: Mapping[str, Any] | None): None -_extract_csv(record: Record, args: Mapping[str, Any]): Record | None -extract_csvs(records: Iterable[Record], args: Mapping[str, Any] | None): list[Record] + +Csv + +args : dict[str, Any] + +__init__(args: Mapping[str, Any] | None): None +_extract_csv(record: Record, args: Mapping[str, Any]): Record | None +extract_csvs(records: Iterable[Record], args: Mapping[str, Any] | None): list[Record] - + histdatacom.orchestration.workflows.DataQualityWorkflow - -DataQualityWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +DataQualityWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows.DataQualityWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + - + histdatacom.activity_stages.DatasetPeriod - -DatasetPeriod - -datemonth : str -month : str -url_path : str -year : str - -__init__(self, year: str, month: str, url_path: str, datemonth: str): None + +DatasetPeriod + +datemonth : str +month : str +url_path : str +year : str + +__init__(self, year: str, month: str, url_path: str, datemonth: str): None - + histdatacom.activity_stages.DatasetPlanOutput - -DatasetPlanOutput - -result -work_items : tuple[WorkItem, ...] - -__init__(self, work_items: tuple[WorkItem, ...], result: StageResult): None -to_dict(): dict[str, Any] + +DatasetPlanOutput + +result +work_items : tuple[WorkItem, ...] + +__init__(self, work_items: tuple[WorkItem, ...], result: StageResult): None +to_dict(): dict[str, Any] - + histdatacom.activity_stages.DatasetPlanOutput->histdatacom.runtime_contracts.StageResult - - -result + + +result - + histdatacom.orchestration.workflows.DatasetPlanWorkflow - -DatasetPlanWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +DatasetPlanWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows.DatasetPlanWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + - + histdatacom.exceptions.DependencyOperationError - -DependencyOperationError - -category : DEPENDENCY -code : str -retryable : bool - - + +DependencyOperationError + +category : DEPENDENCY +code : str +retryable : bool + + - + histdatacom.exceptions.DependencyOperationError->histdatacom.exceptions.HistDataOperationError - - + + + + + +histdatacom.synthetic.bars.DerivedBarIntervalV1 + +DerivedBarIntervalV1 + +alignment_epoch_ns : int +code : str +duration_ns : int +schema_version : str + +__init__(self, code: str, duration_ns: int, alignment_epoch_ns: int, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'DerivedBarIntervalV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.bars.DerivedBarPartitionV1 + +DerivedBarPartitionV1 + +bar_month : str +byte_sha256 : str +interval_code : str +logical_content_sha256 : str +max_bar_start_ns : int +min_bar_start_ns : int +partition_id : str +relative_path : str +row_count : int +row_group_count : int +schema_version : str +scope +size_bytes : int +symbol : str + +__init__(self, relative_path: str, symbol: str, scope: ActivitySliceScope, interval_code: str, bar_month: str, row_count: int, min_bar_start_ns: int, max_bar_start_ns: int, logical_content_sha256: str, byte_sha256: str, size_bytes: int, row_group_count: int, partition_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'DerivedBarPartitionV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.bars.DerivedBarPartitionV1->histdatacom.synthetic.activity.ActivitySliceScope + + +scope + + + +histdatacom.synthetic.bars.DerivedBarPersistenceError + +DerivedBarPersistenceError + + + + + + +histdatacom.synthetic.bars.DerivedBarPolicyV1 + +DerivedBarPolicyV1 + +interval_contracts : tuple[DerivedBarIntervalV1, ...] +intervals : tuple[str, ...] +max_bars : int +max_provenance_values : int +max_symbols : int +policy_id : str +rounding_digits : int +schema_version : str +scopes : tuple[ActivitySliceScope, ...] + +__init__(self, intervals: tuple[str, ...], scopes: tuple[ActivitySliceScope, ...], max_bars: int, max_symbols: int, max_provenance_values: int, rounding_digits: int, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'DerivedBarPolicyV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.bars.DerivedBarProductManifestV1 + +DerivedBarProductManifestV1 + +bar_count : int +compression : str +ensemble_member_id : str +logical_content_sha256 : str +manifest_id : str +partitions : tuple[DerivedBarPartitionV1, ...] +policy +publication_id : str +python_runtime : str +query_end_ns : int | None +query_start_ns : int | None +row_group_size : int +run_id : str +schema_version : str +source_product_logical_sha256 : str +source_product_manifest_id : str +source_product_publication_id : str +symbol_bar_counts : Mapping[str, int] +symbols : tuple[str, ...] +writer_id : str +writer_library_version : str + +__init__(self, source_product_manifest_id: str, source_product_publication_id: str, source_product_logical_sha256: str, run_id: str, ensemble_member_id: str, query_start_ns: int | None, query_end_ns: int | None, policy: DerivedBarPolicyV1, symbols: tuple[str, ...], symbol_bar_counts: Mapping[str, int], partitions: tuple[DerivedBarPartitionV1, ...], logical_content_sha256: str, writer_id: str, writer_library_version: str, python_runtime: str, compression: str, row_group_size: int, publication_id: str, manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'DerivedBarProductManifestV1' +from_json(text: str): 'DerivedBarProductManifestV1' +payload(): dict[str, JSONValue] +publication_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.bars.DerivedBarProductManifestV1->histdatacom.synthetic.bars.DerivedBarPolicyV1 + + +policy + + + +histdatacom.synthetic.bars.DerivedBarV1 + +DerivedBarV1 + +activity_duration_ns : int +ask_close : float +ask_high : float +ask_low : float +ask_open : float +bar_end_ns : int +bar_id : str +bar_start_ns : int +bid_close : float +bid_high : float +bid_low : float +bid_open : float +broker_profile_ids : tuple[str, ...] +confidence_support_count : int +constraint_set_ids : tuple[str, ...] +ensemble_member_id : str +event_content_sha256 : str +event_count : int +feed_epoch_ids : tuple[str, ...] +first_event_id : str +first_event_time_ns : int +generator_config_ids : tuple[str, ...] +generator_ids : tuple[str, ...] +generator_versions : tuple[str, ...] +interval_code : str +interval_ns : int +is_partial_end : bool +is_partial_start : bool +last_event_id : str +last_event_time_ns : int +mean_event_confidence : float | None +mean_spread : float +mid_close : float +mid_high : float +mid_low : float +mid_open : float +motif_ids : tuple[str, ...] +observed_event_count : int +policy_id : str +price_change_count : int +quote_update_count : int +reference_ids : tuple[str, ...] +rounding_digits : int +run_id : str +schema_version : str +scope +source_product_manifest_id : str +source_version_ids : tuple[str, ...] +spread_close : float +spread_high : float +spread_low : float +spread_open : float +stale_quote_count : int +stale_quote_rate : float | None +symbol : str +synthetic_event_count : int +tick_intensity_per_second : float | None +transition_count : int + +__init__(self, source_product_manifest_id: str, policy_id: str, rounding_digits: int, run_id: str, ensemble_member_id: str, symbol: str, scope: ActivitySliceScope, interval_code: str, interval_ns: int, bar_start_ns: int, bar_end_ns: int, first_event_id: str, last_event_id: str, first_event_time_ns: int, last_event_time_ns: int, event_count: int, observed_event_count: int, synthetic_event_count: int, quote_update_count: int, transition_count: int, bid_open: float, bid_high: float, bid_low: float, bid_close: float, ask_open: float, ask_high: float, ask_low: float, ask_close: float, mid_open: float, mid_high: float, mid_low: float, mid_close: float, spread_open: float, spread_high: float, spread_low: float, spread_close: float, mean_spread: float, activity_duration_ns: int, tick_intensity_per_second: float | None, price_change_count: int, stale_quote_count: int, stale_quote_rate: float | None, mean_event_confidence: float | None, confidence_support_count: int, is_partial_start: bool, is_partial_end: bool, source_version_ids: tuple[str, ...], generator_ids: tuple[str, ...], generator_versions: tuple[str, ...], generator_config_ids: tuple[str, ...], reference_ids: tuple[str, ...], motif_ids: tuple[str, ...], feed_epoch_ids: tuple[str, ...], broker_profile_ids: tuple[str, ...], constraint_set_ids: tuple[str, ...], event_content_sha256: str, bar_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'DerivedBarV1' +from_json(text: str): 'DerivedBarV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.bars.DerivedBarV1->histdatacom.synthetic.activity.ActivitySliceScope + + +scope - + histdatacom.orchestration.workflows.DownloadArchivesWorkflow - -DownloadArchivesWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +DownloadArchivesWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows.DownloadArchivesWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + + + + +histdatacom.market_context.corpus.EcbPolicyRateAdapterV1 + +EcbPolicyRateAdapterV1 + +adapter_name : str +adapter_version : str +coverage_complete : bool +coverage_end_ns : int | None +coverage_start_ns : int | None +diagnostics : tuple[str, ...] +snapshot + +__init__(snapshot: MarketContextSourceSnapshotV1): None +load_events(): Iterable[MarketContextEventV1] + + + +histdatacom.synthetic.generation.EmpiricalMotifBenchmarkGeneratorV1 + +EmpiricalMotifBenchmarkGeneratorV1 + +as_of_ns : int | None +candidate +candidate_id : str +condition +config +event_schema_version : str +information_mode +motif_index +run + +__init__(self, candidate: BenchmarkCandidateV1, run: ReconstructionRunV1, motif_index: ReferenceMotifIndexV1, condition: ReferenceMotifConditionV1, config: EmpiricalMotifGeneratorConfigV1, information_mode: InformationMode, as_of_ns: int | None, event_schema_version: str): None +__post_init__(): None +generate(degraded_events: Sequence[BenchmarkEventV1]): Sequence[BenchmarkEventV1] + + + +histdatacom.synthetic.generation.EmpiricalMotifBenchmarkGeneratorV1->histdatacom.synthetic.benchmark.BenchmarkCandidateV1 + + +candidate + + + +histdatacom.synthetic.generation.EmpiricalMotifBenchmarkGeneratorV1->histdatacom.synthetic.benchmark.BenchmarkGeneratorV1 + + + + + +histdatacom.synthetic.generation.EmpiricalMotifGeneratorConfigV1 + +EmpiricalMotifGeneratorConfigV1 + +closed_session_states : tuple[str, ...] +confidence_rounding_digits : int +config_id : str +constraint_set_id : str +estimated_bytes_per_event : int +fallback_price_precision_digits : int +max_events_per_interval : int +max_transformations_per_interval : int +schema_version : str + +__init__(self, max_events_per_interval: int, max_transformations_per_interval: int, estimated_bytes_per_event: int, fallback_price_precision_digits: int, confidence_rounding_digits: int, closed_session_states: tuple[str, ...], constraint_set_id: str, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EmpiricalMotifGeneratorConfigV1' +from_json(text: str): 'EmpiricalMotifGeneratorConfigV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.generation.EmpiricalMotifBenchmarkGeneratorV1->histdatacom.synthetic.generation.EmpiricalMotifGeneratorConfigV1 + + +config + + + +histdatacom.synthetic.generation.EmpiricalMotifBenchmarkGeneratorV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.streaming.ReconstructionRunV1 + +ReconstructionRunV1 + +base_seed : int +configuration_ids : tuple[str, ...] +ensemble_member_ids : tuple[str, ...] +run_id : str +schema_version : str +source_version_ids : tuple[str, ...] +storage_policy +symbols : tuple[str, ...] + +__init__(self, symbols: tuple[str, ...], source_version_ids: tuple[str, ...], configuration_ids: tuple[str, ...], ensemble_member_ids: tuple[str, ...], base_seed: int, storage_policy: ReconstructionStoragePolicyV1, run_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionRunV1' +from_json(text: str): 'ReconstructionRunV1' +identity_payload(): dict[str, JSONValue] +seed_for(ensemble_member_id: str, semantic_key: str): int +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.generation.EmpiricalMotifBenchmarkGeneratorV1->histdatacom.synthetic.streaming.ReconstructionRunV1 + + +run + + + +histdatacom.synthetic.motifs.ReferenceMotifConditionV1 + +ReferenceMotifConditionV1 + +active_sessions : tuple[str, ...] +activity_regime : str +currencies : tuple[str, ...] +event_tags : tuple[str, ...] +feed_epoch_id : str +holiday_tags : tuple[str, ...] +interarrival_regime : str +metrics : Mapping[str, float] +overlap_tags : tuple[str, ...] +price_precision : str +range_regime : str +return_regime : str +schema_version : str +session_state : str +source_quality_state : str +special_tags : tuple[str, ...] +spread_regime : str +symbol : str +timestamp_precision : str +volatility_regime : str + +__init__(self, symbol: str, feed_epoch_id: str, session_state: str, currencies: tuple[str, ...], active_sessions: tuple[str, ...], overlap_tags: tuple[str, ...], special_tags: tuple[str, ...], holiday_tags: tuple[str, ...], event_tags: tuple[str, ...], return_regime: str, range_regime: str, volatility_regime: str, spread_regime: str, activity_regime: str, interarrival_regime: str, timestamp_precision: str, price_precision: str, source_quality_state: str, metrics: Mapping[str, float], schema_version: str): None +__post_init__(): None +coordinates(): dict[str, str] +from_dict(data: Mapping[str, Any]): 'ReferenceMotifConditionV1' +matches(pattern: Mapping[str, str]): bool +pattern_for_level(level: str): dict[str, str] | None +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.generation.EmpiricalMotifBenchmarkGeneratorV1->histdatacom.synthetic.motifs.ReferenceMotifConditionV1 + + +condition + + + +histdatacom.synthetic.motifs.ReferenceMotifIndexV1 + +ReferenceMotifIndexV1 + +config +excluded_split_counts : Mapping[str, int] +fragments : tuple[ReferenceMotifFragmentV1, ...] +index_id : str +ineligible_window_count : int +leakage_comparison_count : int +lineage_artifacts : tuple[ArtifactRef, ...] +schema_version : str +selection_omitted_count : int +source_window_count : int +splits : tuple[ReferenceMotifSplitV1, ...] + +__init__(self, config: ReferenceMotifIndexConfigV1, splits: tuple[ReferenceMotifSplitV1, ...], fragments: tuple[ReferenceMotifFragmentV1, ...], source_window_count: int, excluded_split_counts: Mapping[str, int], ineligible_window_count: int, selection_omitted_count: int, leakage_comparison_count: int, lineage_artifacts: tuple[ArtifactRef, ...], index_id: str, schema_version: str): None +__post_init__(): None +fragment_by_id(fragment_id: str): ReferenceMotifFragmentV1 +from_dict(data: Mapping[str, Any]): 'ReferenceMotifIndexV1' +from_json(text: str): 'ReferenceMotifIndexV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.generation.EmpiricalMotifBenchmarkGeneratorV1->histdatacom.synthetic.motifs.ReferenceMotifIndexV1 + + +motif_index + + + +histdatacom.synthetic.generation.EmpiricalMotifCandidateBatchV1 + +EmpiricalMotifCandidateBatchV1 + +anchor_interval_id : str +batch_id : str +carry_state +decision +decision_details : tuple[str, ...] +ensemble_member_id : str +event_lineage : tuple[EmpiricalMotifEventLineageV1, ...] +events : tuple[SyntheticEventV1, ...] +generator_config +generator_config_id : str +left_anchor_event_id : str +query_result +resource_estimate +right_anchor_event_id : str +run_id : str +schema_version : str +status +symbol : str +target_event_count : int +transformations : tuple[EmpiricalMotifTransformationV1, ...] +window_id : str + +__init__(self, run_id: str, window_id: str, ensemble_member_id: str, symbol: str, anchor_interval_id: str, left_anchor_event_id: str, right_anchor_event_id: str, generator_config: EmpiricalMotifGeneratorConfigV1, query_result: ReferenceMotifQueryResultV1, status: MotifGenerationStatus, decision: MotifGenerationDecision, target_event_count: int, events: tuple[SyntheticEventV1, ...], transformations: tuple[EmpiricalMotifTransformationV1, ...], event_lineage: tuple[EmpiricalMotifEventLineageV1, ...], resource_estimate: ReconstructionResourceEstimateV1, carry_state: CarryStateV1, decision_details: tuple[str, ...], batch_id: str, schema_version: str): None +__post_init__(): None +lineage_for(event_id: str): EmpiricalMotifEventLineageV1 +merged_stream(observed_events: Sequence[SyntheticEventV1]): SyntheticEventStreamV1 +metadata(): dict[str, JSONValue] +payload(): dict[str, JSONValue] + + + +histdatacom.synthetic.generation.EmpiricalMotifCandidateBatchV1->histdatacom.synthetic.streaming.CarryStateV1 + + +carry_state + + + +histdatacom.synthetic.generation.EmpiricalMotifCandidateBatchV1->histdatacom.synthetic.generation.EmpiricalMotifGeneratorConfigV1 + + +generator_config + + + +histdatacom.synthetic.generation.MotifGenerationDecision + +MotifGenerationDecision + +name + + + + + +histdatacom.synthetic.generation.EmpiricalMotifCandidateBatchV1->histdatacom.synthetic.generation.MotifGenerationDecision + + +decision + + + +histdatacom.synthetic.generation.MotifGenerationStatus + +MotifGenerationStatus + +name + + + + + +histdatacom.synthetic.generation.EmpiricalMotifCandidateBatchV1->histdatacom.synthetic.generation.MotifGenerationStatus + + +status + + + +histdatacom.synthetic.streaming.ReconstructionResourceEstimateV1 + +ReconstructionResourceEstimateV1 + +candidate_amplification : float +candidate_event_count : int +estimate_id : str +estimated_batch_count : int +estimated_memory_bytes : int +estimated_output_bytes : int +estimated_scratch_bytes : int +inflight_batches : int +input_event_count : int +peak_events_per_batch : int +retained_ensemble_members : int +schema_version : str + +__init__(self, input_event_count: int, candidate_event_count: int, retained_ensemble_members: int, inflight_batches: int, peak_events_per_batch: int, estimated_memory_bytes: int, estimated_scratch_bytes: int, estimated_output_bytes: int, estimated_batch_count: int, estimate_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionResourceEstimateV1' +from_json(text: str): 'ReconstructionResourceEstimateV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.generation.EmpiricalMotifCandidateBatchV1->histdatacom.synthetic.streaming.ReconstructionResourceEstimateV1 + + +resource_estimate + + + +histdatacom.synthetic.motifs.ReferenceMotifQueryResultV1 + +ReferenceMotifQueryResultV1 + +backoff_attempts : tuple[ReferenceMotifBackoffAttemptV1, ...] +hidden_by_availability_count : int +index_id : str +matches : tuple[ReferenceMotifMatchV1, ...] +query +result_id : str +scanned_fragment_count : int +schema_version : str +status + +__init__(self, index_id: str, query: ReferenceMotifQueryV1, status: ReferenceMotifQueryStatus, matches: tuple[ReferenceMotifMatchV1, ...], backoff_attempts: tuple[ReferenceMotifBackoffAttemptV1, ...], scanned_fragment_count: int, hidden_by_availability_count: int, result_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifQueryResultV1' +from_json(text: str): 'ReferenceMotifQueryResultV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.generation.EmpiricalMotifCandidateBatchV1->histdatacom.synthetic.motifs.ReferenceMotifQueryResultV1 + + +query_result + + + +histdatacom.synthetic.generation.EmpiricalMotifEventLineageV1 + +EmpiricalMotifEventLineageV1 + +anchor_progress : float +event_id : str +global_event_ordinal : int +requested_event_time_ns : int +schema_version : str +segment_event_ordinal : int +source_progress : float +transformation_id : str + +__init__(self, event_id: str, transformation_id: str, global_event_ordinal: int, segment_event_ordinal: int, source_progress: float, anchor_progress: float, requested_event_time_ns: int, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EmpiricalMotifEventLineageV1' +from_json(text: str): 'EmpiricalMotifEventLineageV1' +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.generation.EmpiricalMotifTransformationV1 + +EmpiricalMotifTransformationV1 + +applied_price_scale : float +backoff_level : str +cell_support : int +confidence : float +index_id : str +match_distance : float +output_end_ordinal : int +output_start_ordinal : int +price_scale_clamped : bool +query_id : str +query_result_id : str +requested_price_scale : float +schema_version : str +seed : int +segment_ordinal : int +source_artifact_sha256 : str +source_event_count : int +source_fragment_id : str +source_period : str +source_series_id : str +source_window_id : str +spread_shape_applied : bool +time_scale : float +time_warp_ratio : float +transformation_id : str + +__init__(self, index_id: str, query_id: str, query_result_id: str, source_fragment_id: str, source_window_id: str, source_series_id: str, source_period: str, source_artifact_sha256: str, backoff_level: str, cell_support: int, match_distance: float, segment_ordinal: int, output_start_ordinal: int, output_end_ordinal: int, source_event_count: int, time_scale: float, time_warp_ratio: float, requested_price_scale: float, applied_price_scale: float, price_scale_clamped: bool, spread_shape_applied: bool, seed: int, confidence: float, transformation_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EmpiricalMotifTransformationV1' +from_json(text: str): 'EmpiricalMotifTransformationV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.ensembles.EnsembleArtifactDigestV1 + +EnsembleArtifactDigestV1 + +artifact_id : str +digest_id : str +kind +schema_version : str +sha256 : str + +__init__(self, artifact_id: str, sha256: str, kind: EnsembleArtifactKind, digest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleArtifactDigestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.EnsembleArtifactKind + +EnsembleArtifactKind + +name + + + + + +histdatacom.synthetic.ensembles.EnsembleArtifactDigestV1->histdatacom.synthetic.ensembles.EnsembleArtifactKind + + +kind + + + +histdatacom.synthetic.ensembles.EnsembleCalibrationConfigV1 + +EnsembleCalibrationConfigV1 + +config_id : str +estimated_bytes_per_event : int +failure_penalty : float +horizons_ns : tuple[int, ...] +logical_distance_tolerance : float +max_payload_bytes : int +max_samples : int +max_slices : int +maximum_collapse_rate : float +maximum_false_diversity_rate : float +member_count : int +minimum_achieved_coverage : float +minimum_fit_samples : int +nominal_coverage : float +retained_member_count : int +rounding_digits : int +schema_version : str + +__init__(self, member_count: int, retained_member_count: int, horizons_ns: tuple[int, ...], nominal_coverage: float, minimum_achieved_coverage: float, minimum_fit_samples: int, maximum_collapse_rate: float, maximum_false_diversity_rate: float, logical_distance_tolerance: float, failure_penalty: float, estimated_bytes_per_event: int, rounding_digits: int, max_samples: int, max_slices: int, max_payload_bytes: int, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleCalibrationConfigV1' +from_json(text: str): 'EnsembleCalibrationConfigV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.ensembles.EnsembleCalibrationReportV1 + +EnsembleCalibrationReportV1 + +automatic_winner : bool +benchmark_manifest_id : str +calibrated : bool +candidate_id : str +config_id : str +default_generator_id : str | None +diversity_summaries : tuple[EnsembleDiversitySummaryV1, ...] +evaluation_sample_count : int +fit_sample_count : int +member_selections : tuple[EnsembleMemberSelectionV1, ...] +metric_calibrations : tuple[EnsembleMetricCalibrationV1, ...] +outcome_summaries : tuple[EnsembleOutcomeSummaryV1, ...] +plan_id : str +primary_member_id : str | None +regenerable_member_ids : tuple[str, ...] +report_id : str +retained_member_count : int +retained_member_ids : tuple[str, ...] +run_id : str +schema_version : str +status +storage_estimate_id : str + +__init__(self, run_id: str, plan_id: str, config_id: str, benchmark_manifest_id: str, candidate_id: str, storage_estimate_id: str, retained_member_count: int, status: EnsembleReportStatus, primary_member_id: str | None, retained_member_ids: tuple[str, ...], regenerable_member_ids: tuple[str, ...], member_selections: tuple[EnsembleMemberSelectionV1, ...], metric_calibrations: tuple[EnsembleMetricCalibrationV1, ...], diversity_summaries: tuple[EnsembleDiversitySummaryV1, ...], outcome_summaries: tuple[EnsembleOutcomeSummaryV1, ...], fit_sample_count: int, evaluation_sample_count: int, automatic_winner: bool, default_generator_id: str | None, report_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleCalibrationReportV1' +from_json(text: str): 'EnsembleCalibrationReportV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.ensembles.EnsembleReportStatus + +EnsembleReportStatus + +name + + + + + +histdatacom.synthetic.ensembles.EnsembleCalibrationReportV1->histdatacom.synthetic.ensembles.EnsembleReportStatus + + +status + + + +histdatacom.synthetic.ensembles.EnsembleCalibrationSampleV1 + +EnsembleCalibrationSampleV1 + +benchmark_manifest_id : str +candidate_id : str +members : tuple[EnsembleMemberCalibrationV1, ...] +reference_content_sha256 : str +reference_metrics : Mapping[str, float] +sample_id : str +scenario_id : str +schema_version : str +split_kind +stratum +window_id : str + +__init__(self, benchmark_manifest_id: str, scenario_id: str, candidate_id: str, window_id: str, split_kind: BenchmarkSplitKind, stratum: EnsembleCalibrationStratumV1, reference_metrics: Mapping[str, float], reference_content_sha256: str, members: tuple[EnsembleMemberCalibrationV1, ...], sample_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleCalibrationSampleV1' +from_json(text: str): 'EnsembleCalibrationSampleV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.ensembles.EnsembleCalibrationSampleV1->histdatacom.synthetic.benchmark.BenchmarkSplitKind + + +split_kind + + + +histdatacom.synthetic.ensembles.EnsembleCalibrationStratumV1 + +EnsembleCalibrationStratumV1 + +epoch_id : str +event_state : str +horizon_ns : int +schema_version : str +session : str +sparsity : str +stratum_id : str +symbol : str + +__init__(self, epoch_id: str, session: str, event_state: str, symbol: str, horizon_ns: int, sparsity: str, stratum_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleCalibrationStratumV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.EnsembleCalibrationSampleV1->histdatacom.synthetic.ensembles.EnsembleCalibrationStratumV1 + + +stratum + + + +histdatacom.synthetic.ensembles.EnsembleDiversityStatus + +EnsembleDiversityStatus + +name + + + + + +histdatacom.synthetic.ensembles.EnsembleDiversitySummaryV1 + +EnsembleDiversitySummaryV1 + +collapse_rate : float | None +collapsed_pair_count : int +distinct_content_count : int +false_diversity_pair_count : int +false_diversity_rate : float | None +mean_normalized_metric_distance : float | None +pair_count : int +sample_count : int +schema_version : str +split_kind +status +stratum +summary_id : str + +__init__(self, split_kind: BenchmarkSplitKind, stratum: EnsembleCalibrationStratumV1, sample_count: int, pair_count: int, collapsed_pair_count: int, false_diversity_pair_count: int, distinct_content_count: int, mean_normalized_metric_distance: float | None, collapse_rate: float | None, false_diversity_rate: float | None, status: EnsembleDiversityStatus, summary_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleDiversitySummaryV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.EnsembleDiversitySummaryV1->histdatacom.synthetic.benchmark.BenchmarkSplitKind + + +split_kind + + + +histdatacom.synthetic.ensembles.EnsembleDiversitySummaryV1->histdatacom.synthetic.ensembles.EnsembleCalibrationStratumV1 + + +stratum + + + +histdatacom.synthetic.ensembles.EnsembleDiversitySummaryV1->histdatacom.synthetic.ensembles.EnsembleDiversityStatus + + +status + + + +histdatacom.synthetic.ensembles.EnsembleMemberCalibrationV1 + +EnsembleMemberCalibrationV1 + +logical_content_sha256 : str | None +member_id : str +metrics : Mapping[str, float] +reason : str | None +result_id : str +schema_version : str +status + +__init__(self, member_id: str, status: EnsembleMemberStatus, metrics: Mapping[str, float], logical_content_sha256: str | None, reason: str | None, result_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleMemberCalibrationV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.EnsembleMemberStatus + +EnsembleMemberStatus + +name + + + + + +histdatacom.synthetic.ensembles.EnsembleMemberCalibrationV1->histdatacom.synthetic.ensembles.EnsembleMemberStatus + + +status + + + +histdatacom.synthetic.ensembles.EnsembleMemberPlanV1 + +EnsembleMemberPlanV1 + +member_id : str +ordinal : int +schema_version : str +seed : int + +__init__(self, ordinal: int, member_id: str, seed: int, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleMemberPlanV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.EnsembleMemberSelectionV1 + +EnsembleMemberSelectionV1 + +completed_count : int +failed_count : int +member_id : str +primary : bool +refused_count : int +representative_distance : float | None +retained : bool +schema_version : str +selection_id : str +selection_rank : int + +__init__(self, member_id: str, selection_rank: int, representative_distance: float | None, completed_count: int, refused_count: int, failed_count: int, primary: bool, retained: bool, selection_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleMemberSelectionV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.EnsembleMetricCalibrationV1 + +EnsembleMetricCalibrationV1 + +calibrated_coverage_rate : float | None +calibrated_covered_count : int +calibration_adjustment : float | None +evaluation_sample_count : int +fit_sample_count : int +mean_absolute_median_error : float | None +mean_calibrated_interval_width : float | None +mean_raw_interval_width : float | None +metric_group : str +metric_name : str +minimum_achieved_coverage : float +nominal_coverage : float +raw_coverage_rate : float | None +raw_covered_count : int +schema_version : str +status +stratum +summary_id : str + +__init__(self, stratum: EnsembleCalibrationStratumV1, metric_name: str, metric_group: str, nominal_coverage: float, minimum_achieved_coverage: float, fit_sample_count: int, evaluation_sample_count: int, calibration_adjustment: float | None, raw_covered_count: int, calibrated_covered_count: int, raw_coverage_rate: float | None, calibrated_coverage_rate: float | None, mean_raw_interval_width: float | None, mean_calibrated_interval_width: float | None, mean_absolute_median_error: float | None, status: EnsembleMetricStatus, summary_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleMetricCalibrationV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.EnsembleMetricCalibrationV1->histdatacom.synthetic.ensembles.EnsembleCalibrationStratumV1 + + +stratum + + + +histdatacom.synthetic.ensembles.EnsembleMetricStatus + +EnsembleMetricStatus + +name + + + + + +histdatacom.synthetic.ensembles.EnsembleMetricCalibrationV1->histdatacom.synthetic.ensembles.EnsembleMetricStatus + + +status + + + +histdatacom.synthetic.ensembles.EnsembleOutcomeSummaryV1 + +EnsembleOutcomeSummaryV1 + +attempt_count : int +completed_count : int +completion_rate : float +failed_count : int +failure_rate : float +reason_counts : Mapping[str, int] +refusal_rate : float +refused_count : int +schema_version : str +split_kind +stratum +summary_id : str + +__init__(self, split_kind: BenchmarkSplitKind, stratum: EnsembleCalibrationStratumV1, attempt_count: int, completed_count: int, refused_count: int, failed_count: int, completion_rate: float, refusal_rate: float, failure_rate: float, reason_counts: Mapping[str, int], summary_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleOutcomeSummaryV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.EnsembleOutcomeSummaryV1->histdatacom.synthetic.benchmark.BenchmarkSplitKind + + +split_kind + + + +histdatacom.synthetic.ensembles.EnsembleOutcomeSummaryV1->histdatacom.synthetic.ensembles.EnsembleCalibrationStratumV1 + + +stratum + + + +histdatacom.synthetic.ensembles.EnsembleRegenerationRequestV1 + +EnsembleRegenerationRequestV1 + +configuration_artifact_hashes : Mapping[str, str] +member_ids : tuple[str, ...] +plan_id : str +report_id : str +request_id : str +schema_version : str +source_artifact_hashes : Mapping[str, str] + +__init__(self, plan_id: str, report_id: str, member_ids: tuple[str, ...], source_artifact_hashes: Mapping[str, str], configuration_artifact_hashes: Mapping[str, str], request_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleRegenerationRequestV1' +from_json(text: str): 'EnsembleRegenerationRequestV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.ensembles.EnsembleStorageEstimateV1 + +EnsembleStorageEstimateV1 + +conservative_retained_event_count : int +estimate_id : str +estimated_bytes_per_event : int +member_event_counts : Mapping[str, int] +plan_id : str +resource_estimate +retained_member_count : int +run_id : str +schema_version : str + +__init__(self, run_id: str, plan_id: str, member_event_counts: Mapping[str, int], retained_member_count: int, conservative_retained_event_count: int, estimated_bytes_per_event: int, resource_estimate: ReconstructionResourceEstimateV1, estimate_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EnsembleStorageEstimateV1' +from_json(text: str): 'EnsembleStorageEstimateV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.ensembles.EnsembleStorageEstimateV1->histdatacom.synthetic.streaming.ReconstructionResourceEstimateV1 + + +resource_estimate - + histdatacom.exceptions.ErrorCategory - -ErrorCategory - -name - - + +ErrorCategory + +name + + + + + +histdatacom.synthetic.streaming.EventBatchV1 + +EventBatchV1 + +artifact +batch_id : str +batch_ordinal : int +content_sha256 : str +ensemble_member_id : str +event_count : int +first_event_time_ns : int +last_event_time_ns : int +ownership_end_ns : int +ownership_start_ns : int +run_id : str +schema_version : str +symbol : str +synchronization_unit_id : str +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, ensemble_member_id: str, symbol: str, batch_ordinal: int, event_count: int, ownership_start_ns: int, ownership_end_ns: int, first_event_time_ns: int, last_event_time_ns: int, content_sha256: str, artifact: ArtifactRef, batch_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'EventBatchV1' +from_json(text: str): 'EventBatchV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.streaming.EventBatchV1->histdatacom.runtime_contracts.ArtifactRef + + +artifact + + + +histdatacom.data_quality.exponential_smoothing.ExponentialSmoothingProfile + +ExponentialSmoothingProfile + +baseline_rolling_windows : tuple[int, ...] +enabled : bool +projection_horizon : int +projection_specification_id : str +rounding_digits : int +specifications : tuple[ExponentialSmoothingSpecification, ...] + +__init__(self, enabled: bool, specifications: tuple[ExponentialSmoothingSpecification, ...], projection_specification_id: str, projection_horizon: int, baseline_rolling_windows: tuple[int, ...], rounding_digits: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.exponential_smoothing.ExponentialSmoothingResult + +ExponentialSmoothingResult + +annotations : tuple[Mapping[str, Any], ...] +diagnostics : Mapping[str, JSONValue] +input_result + +__init__(self, diagnostics: Mapping[str, JSONValue], annotations: tuple[Mapping[str, Any], ...], input_result: ClassicalModelInputResult): None + + + +histdatacom.data_quality.exponential_smoothing.ExponentialSmoothingResult->histdatacom.data_quality.classical_model_contracts.ClassicalModelInputResult + + +input_result + + + +histdatacom.data_quality.exponential_smoothing.ExponentialSmoothingSpecification + +ExponentialSmoothingSpecification + +damped_trend : bool +damping_trend : float | None +error : str +family : str +initial_level : float | None +initial_seasonal : tuple[float, ...] +initial_trend : float | None +initialization_method : str +level : bool +max_iterations : int +method : str +optimized : bool +parameter_bounds : tuple[tuple[str, float, float], ...] +remove_bias : bool +seasonal : str +seasonal_periods : int +smoothing_level : float | None +smoothing_seasonal : float | None +smoothing_trend : float | None +specification_id : str +trend : str +use_brute : bool + +__init__(self, specification_id: str, family: str, level: bool, error: str, trend: str, damped_trend: bool, seasonal: str, seasonal_periods: int, initialization_method: str, initial_level: float | None, initial_trend: float | None, initial_seasonal: tuple[float, ...], optimized: bool, method: str, use_brute: bool, remove_bias: bool, smoothing_level: float | None, smoothing_trend: float | None, smoothing_seasonal: float | None, damping_trend: float | None, parameter_bounds: tuple[tuple[str, float, float], ...], max_iterations: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] - + histdatacom.orchestration.workflows.ExtractCsvWorkflow - -ExtractCsvWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +ExtractCsvWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows.ExtractCsvWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + - + histdatacom.runtime_contracts.FailureInfo - -FailureInfo - -code : str -detail : dict[str, JSONValue] -message : str -retryable : bool - -__init__(self, code: str, message: str, retryable: bool, detail: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any] | None): 'FailureInfo | None' -to_dict(): dict[str, JSONValue] + +FailureInfo + +code : str +detail : dict[str, JSONValue] +message : str +retryable : bool + +__init__(self, code: str, message: str, retryable: bool, detail: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any] | None): 'FailureInfo | None' +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.corpus.FederalReserveFomcCalendarAdapterV1 + +FederalReserveFomcCalendarAdapterV1 + +adapter_name : str +adapter_version : str +diagnostics : tuple[str, ...] +snapshot + +__init__(snapshot: MarketContextSourceSnapshotV1): None +load_events(): Iterable[MarketContextEventV1] + + + +histdatacom.market_context.corpus.FederalReserveFomcHistoricalAdapterV1 + +FederalReserveFomcHistoricalAdapterV1 + +adapter_name : str +adapter_version : str +diagnostics : tuple[str, ...] +snapshot + +__init__(snapshot: MarketContextSourceSnapshotV1): None +load_events(): Iterable[MarketContextEventV1] + + + +histdatacom.data_analytics.feed_epochs.FeedEpochAssignmentV1 + +FeedEpochAssignmentV1 + +assignment_kind : str +boundary_id : str | None +definition_id : str +epoch_id : str | None +label : str +schema_version : str +symbol : str +timestamp_utc_ms : int + +__init__(self, definition_id: str, symbol: str, timestamp_utc_ms: int, assignment_kind: str, label: str, epoch_id: str | None, boundary_id: str | None, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochAssignmentV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochAssignmentV2 + +FeedEpochAssignmentV2 + +assignment_kind : str +boundary_id : str | None +definition_id : str +epoch_id : str | None +label : str +schema_version : str +symbol : str +timestamp_utc_ms : int + +__init__(self, definition_id: str, symbol: str, timestamp_utc_ms: int, assignment_kind: str, label: str, epoch_id: str | None, boundary_id: str | None, schema_version: str): None +__post_init__(): None + + + +histdatacom.data_analytics.feed_epochs.FeedEpochBoundaryV1 + +FeedEpochBoundaryV1 + +boundary_id : str +central_timestamp_utc_ms : int +change_score : float +confidence : float +contributing_features : tuple[str, ...] +left_period : str +right_period : str +schema_version : str +support : float +support_by_analysis : Mapping[str, float] +transition_label : str +uncertainty_end_utc_ms : int +uncertainty_start_utc_ms : int + +__init__(self, boundary_id: str, left_period: str, right_period: str, central_timestamp_utc_ms: int, uncertainty_start_utc_ms: int, uncertainty_end_utc_ms: int, change_score: float, confidence: float, support: float, support_by_analysis: Mapping[str, float], contributing_features: tuple[str, ...], transition_label: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochBoundaryV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochBoundaryV2 + +FeedEpochBoundaryV2 + +boundary_id : str +central_timestamp_utc_ms : int +left_period : str +objective_gain : float +right_period : str +schema_version : str +support : float +supporting_features : tuple[str, ...] +transition_label : str +uncertainty_end_period : str +uncertainty_start_period : str + +__init__(self, left_period: str, right_period: str, central_timestamp_utc_ms: int, support: float, uncertainty_start_period: str, uncertainty_end_period: str, objective_gain: float, supporting_features: tuple[str, ...], boundary_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochBoundaryV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochCampaignV2 + +FeedEpochCampaignV2 + +definition +evidence : tuple[FeedEpochEvidenceV2, ...] +peak_memory_bytes : int +runtime_seconds : float +schema_version : str +skipped_sources : tuple[Mapping[str, JSONValue], ...] +source_bytes : int +source_count : int + +__init__(self, definition: FeedEpochDefinitionV2, evidence: tuple[FeedEpochEvidenceV2, ...], source_count: int, source_bytes: int, runtime_seconds: float, peak_memory_bytes: int, skipped_sources: tuple[Mapping[str, JSONValue], ...], schema_version: str): None +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochDefinitionV2 + +FeedEpochDefinitionV2 + +boundaries : tuple[FeedEpochBoundaryV2, ...] +config +coverage_end_utc_ms : int +coverage_start_utc_ms : int +definition_id : str +epochs : tuple[FeedEpochIntervalV2, ...] +evidence_count : int +feature_names : tuple[str, ...] +lineage : Mapping[str, JSONValue] +period_count : int +schema_version : str +stability +symbol_deviations : tuple[FeedEpochSymbolDeviationV2, ...] +symbols : tuple[str, ...] +valid_for_observation_models : bool + +__init__(self, config: FeedEpochFitConfigV2, symbols: tuple[str, ...], coverage_start_utc_ms: int, coverage_end_utc_ms: int, evidence_count: int, period_count: int, feature_names: tuple[str, ...], boundaries: tuple[FeedEpochBoundaryV2, ...], epochs: tuple[FeedEpochIntervalV2, ...], symbol_deviations: tuple[FeedEpochSymbolDeviationV2, ...], stability: FeedEpochStabilityV2, lineage: Mapping[str, JSONValue], definition_id: str, schema_version: str): None +__post_init__(): None +assign(): FeedEpochAssignmentV2 +from_dict(data: Mapping[str, Any]): 'FeedEpochDefinitionV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochCampaignV2->histdatacom.data_analytics.feed_epochs_v2.FeedEpochDefinitionV2 + + +definition + + + +histdatacom.data_analytics.feed_epochs.FeedEpochDefinitionV1 + +FeedEpochDefinitionV1 + +boundaries : tuple[FeedEpochBoundaryV1, ...] +config +coverage_end_utc_ms : int +coverage_start_utc_ms : int +definition_id : str +epochs : tuple[FeedEpochIntervalV1, ...] +evidence_count : int +feature_names : tuple[str, ...] +lineage : Mapping[str, JSONValue] +period_count : int +schema_version : str +stability +symbols : tuple[str, ...] +valid_for_observation_models : bool + +__init__(self, config: FeedEpochFitConfigV1, symbols: tuple[str, ...], coverage_start_utc_ms: int, coverage_end_utc_ms: int, evidence_count: int, period_count: int, feature_names: tuple[str, ...], boundaries: tuple[FeedEpochBoundaryV1, ...], epochs: tuple[FeedEpochIntervalV1, ...], stability: FeedEpochStabilityV1, lineage: Mapping[str, JSONValue], definition_id: str, schema_version: str): None +__post_init__(): None +assign(): FeedEpochAssignmentV1 +from_dict(data: Mapping[str, Any]): 'FeedEpochDefinitionV1' +from_json(value: str): 'FeedEpochDefinitionV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.data_analytics.feed_epochs.FeedEpochFitConfigV1 + +FeedEpochFitConfigV1 + +boundary_match_tolerance_periods : int +config_id : str +feature_names : tuple[str, ...] +max_evidence : int +max_sensitivity_runs : int +min_boundary_support : float +min_change_score : float +min_evidence_periods : int +min_feature_coverage : float +min_segment_periods : int +rounding_digits : int +schema_version : str + +__init__(self, feature_names: tuple[str, ...], min_evidence_periods: int, min_segment_periods: int, min_feature_coverage: float, min_change_score: float, min_boundary_support: float, boundary_match_tolerance_periods: int, max_evidence: int, max_sensitivity_runs: int, rounding_digits: int, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochFitConfigV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs.FeedEpochDefinitionV1->histdatacom.data_analytics.feed_epochs.FeedEpochFitConfigV1 + + +config + + + +histdatacom.data_analytics.feed_epochs.FeedEpochStabilityV1 + +FeedEpochStabilityV1 + +limitations : tuple[str, ...] +run_counts : Mapping[str, int] +schema_version : str +status : str +unstable_boundary_ids : tuple[str, ...] +usable_run_counts : Mapping[str, int] + +__init__(self, status: str, run_counts: Mapping[str, int], usable_run_counts: Mapping[str, int], unstable_boundary_ids: tuple[str, ...], limitations: tuple[str, ...], schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochStabilityV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs.FeedEpochDefinitionV1->histdatacom.data_analytics.feed_epochs.FeedEpochStabilityV1 + + +stability + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochFitConfigV2 + +FeedEpochFitConfigV2 + +active_gap_cap_ms : int +activity_bin_ms : int +boundary_match_tolerance_periods : int +burst_interval_ms : int +config_id : str +feature_names : tuple[str, ...] +max_evidence : int +max_sensitivity_runs : int +min_boundary_support : float +min_evidence_periods : int +min_feature_coverage : float +min_segment_periods : int +min_symbol_count : int +penalty_multiplier : float +robust_clip : float +rounding_digits : int +schema_version : str +sensitivity_penalty_multipliers : tuple[float, ...] + +__init__(self, feature_names: tuple[str, ...], min_evidence_periods: int, min_segment_periods: int, min_feature_coverage: float, min_symbol_count: int, penalty_multiplier: float, robust_clip: float, min_boundary_support: float, boundary_match_tolerance_periods: int, sensitivity_penalty_multipliers: tuple[float, ...], active_gap_cap_ms: int, burst_interval_ms: int, activity_bin_ms: int, max_evidence: int, max_sensitivity_runs: int, rounding_digits: int, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochFitConfigV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochDefinitionV2->histdatacom.data_analytics.feed_epochs_v2.FeedEpochFitConfigV2 + + +config + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochStabilityV2 + +FeedEpochStabilityV2 + +boundary_support : Mapping[str, float] +boundary_support_by_family : Mapping[str, Mapping[str, float]] +common_period_count : int +feature_coverage : Mapping[str, float] +reasons : tuple[str, ...] +rejected_candidates : Mapping[str, Mapping[str, JSONValue]] +run_count : int +run_counts : Mapping[str, int] +schema_version : str +status : str +symbol_count : int + +__init__(self, status: str, reasons: tuple[str, ...], run_count: int, run_counts: Mapping[str, int], boundary_support: Mapping[str, float], boundary_support_by_family: Mapping[str, Mapping[str, float]], rejected_candidates: Mapping[str, Mapping[str, JSONValue]], feature_coverage: Mapping[str, float], common_period_count: int, symbol_count: int, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochStabilityV2' +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochDefinitionV2->histdatacom.data_analytics.feed_epochs_v2.FeedEpochStabilityV2 + + +stability + + + +histdatacom.data_analytics.feed_epochs.FeedEpochEvidenceV1 + +FeedEpochEvidenceV1 + +conditioning : Mapping[str, JSONValue] +end_timestamp_utc_ms : int +evidence_id : str +feature_provenance : Mapping[str, tuple[str, ...]] +feature_values : Mapping[str, float] +fingerprint_id : str +period : str +profile : Mapping[str, JSONValue] +quality : Mapping[str, JSONValue] +schema_version : str +source_artifact_sha256 : str +source_hash_basis : str +source_kind : str +start_timestamp_utc_ms : int +symbol : str + +__init__(self, symbol: str, period: str, start_timestamp_utc_ms: int, end_timestamp_utc_ms: int, fingerprint_id: str, source_artifact_sha256: str, source_hash_basis: str, source_kind: str, feature_values: Mapping[str, float], feature_provenance: Mapping[str, tuple[str, ...]], conditioning: Mapping[str, JSONValue], quality: Mapping[str, JSONValue], profile: Mapping[str, JSONValue], evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochEvidenceV1' +from_fingerprint(payload: Mapping[str, Any]): 'FeedEpochEvidenceV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochEvidenceV2 + +FeedEpochEvidenceV2 + +activity_bin_counts : Mapping[str, int] +calendar_policy : Mapping[str, JSONValue] +counts : Mapping[str, int] +denominators_ms : Mapping[str, int] +end_timestamp_utc_ms : int +evidence_id : str +feature_provenance : Mapping[str, tuple[str, ...]] +feature_values : Mapping[str, float] +limitations : tuple[str, ...] +period : str +profile : Mapping[str, JSONValue] +row_count : int +schema_version : str +source_artifact_sha256 : str +source_hash_basis : str +source_path : str +source_size_bytes : int +start_timestamp_utc_ms : int +symbol : str + +__init__(self, symbol: str, period: str, source_path: str, source_artifact_sha256: str, source_size_bytes: int, start_timestamp_utc_ms: int, end_timestamp_utc_ms: int, row_count: int, denominators_ms: Mapping[str, int], counts: Mapping[str, int], feature_values: Mapping[str, float], feature_provenance: Mapping[str, tuple[str, ...]], activity_bin_counts: Mapping[str, int], calendar_policy: Mapping[str, JSONValue], limitations: tuple[str, ...], evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochEvidenceV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs.FeedEpochIntervalV1 + +FeedEpochIntervalV1 + +end_timestamp_utc_ms : int +epoch_id : str +evidence_count : int +label : str +period_end : str +period_start : str +schema_version : str +start_timestamp_utc_ms : int + +__init__(self, epoch_id: str, label: str, period_start: str, period_end: str, start_timestamp_utc_ms: int, end_timestamp_utc_ms: int, evidence_count: int, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochIntervalV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochIntervalV2 + +FeedEpochIntervalV2 + +end_timestamp_utc_ms : int +epoch_id : str +evidence_count : int +feature_medians : Mapping[str, float] +label : str +period_end : str +period_start : str +schema_version : str +start_timestamp_utc_ms : int + +__init__(self, label: str, period_start: str, period_end: str, start_timestamp_utc_ms: int, end_timestamp_utc_ms: int, evidence_count: int, feature_medians: Mapping[str, float], epoch_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'FeedEpochIntervalV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.data_analytics.feed_epochs_v2.FeedEpochSymbolDeviationV2 + +FeedEpochSymbolDeviationV2 + +left_period : str +nearest_global_distance_periods : int | None +right_period : str +symbol : str + +__init__(self, symbol: str, left_period: str, right_period: str, nearest_global_distance_periods: int | None): None +from_dict(data: Mapping[str, Any]): 'FeedEpochSymbolDeviationV2' +to_dict(): dict[str, JSONValue] - + histdatacom.data_analytics.feed_regimes.FeedPeriodProfile - -FeedPeriodProfile - -bucket : str -end_utc_ms : int -max_interarrival_ms : int -median_interarrival_ms : float -p95_interarrival_ms : float -period : str -quiet_gap_count : int -quote_update_count : int -quote_update_ratio : float -row_count : int -session_counts : dict[str, int] -spread_max : float -spread_mean : float -spread_median : float -spread_min : float -start_utc_ms : int -symbol : str -target_paths : tuple[str, ...] -tick_rate_per_hour : float -zero_change_run_count : int -zero_change_tick_count : int - -__init__(self, symbol: str, period: str, bucket: str, row_count: int, start_utc_ms: int, end_utc_ms: int, tick_rate_per_hour: float, median_interarrival_ms: float, p95_interarrival_ms: float, max_interarrival_ms: int, quiet_gap_count: int, quote_update_count: int, quote_update_ratio: float, zero_change_run_count: int, zero_change_tick_count: int, spread_min: float, spread_median: float, spread_mean: float, spread_max: float, session_counts: dict[str, int], target_paths: tuple[str, ...]): None -to_dict(): dict[str, JSONValue] + +FeedPeriodProfile + +bucket : str +end_utc_ms : int +max_interarrival_ms : int +median_interarrival_ms : float +p95_interarrival_ms : float +period : str +quiet_gap_count : int +quote_update_count : int +quote_update_ratio : float +row_count : int +session_counts : dict[str, int] +spread_max : float +spread_mean : float +spread_median : float +spread_min : float +start_utc_ms : int +symbol : str +target_paths : tuple[str, ...] +tick_rate_per_hour : float +zero_change_run_count : int +zero_change_tick_count : int + +__init__(self, symbol: str, period: str, bucket: str, row_count: int, start_utc_ms: int, end_utc_ms: int, tick_rate_per_hour: float, median_interarrival_ms: float, p95_interarrival_ms: float, max_interarrival_ms: int, quiet_gap_count: int, quote_update_count: int, quote_update_ratio: float, zero_change_run_count: int, zero_change_tick_count: int, spread_min: float, spread_median: float, spread_mean: float, spread_max: float, session_counts: dict[str, int], target_paths: tuple[str, ...]): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_analytics.feed_regimes.FeedRegimeEra - -FeedRegimeEra - -bucket : str -end_utc_ms : int -label : str -mean_tick_rate_per_hour : float -median_interarrival_ms : float -metadata : dict[str, JSONValue] -period_end : str -period_start : str -profile_count : int -quiet_gap_count : int -quote_update_ratio : float -row_count : int -start_utc_ms : int -symbol : str - -__init__(self, symbol: str, label: str, bucket: str, period_start: str, period_end: str, start_utc_ms: int, end_utc_ms: int, profile_count: int, row_count: int, mean_tick_rate_per_hour: float, median_interarrival_ms: float, quote_update_ratio: float, quiet_gap_count: int, metadata: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +FeedRegimeEra + +bucket : str +end_utc_ms : int +label : str +mean_tick_rate_per_hour : float +median_interarrival_ms : float +metadata : dict[str, JSONValue] +period_end : str +period_start : str +profile_count : int +quiet_gap_count : int +quote_update_ratio : float +row_count : int +start_utc_ms : int +symbol : str + +__init__(self, symbol: str, label: str, bucket: str, period_start: str, period_end: str, start_utc_ms: int, end_utc_ms: int, profile_count: int, row_count: int, mean_tick_rate_per_hour: float, median_interarrival_ms: float, quote_update_ratio: float, quiet_gap_count: int, metadata: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_analytics.feed_regimes.FeedRegimeReport - -FeedRegimeReport - -discovery -metadata : dict[str, JSONValue] -period_profiles : tuple[FeedPeriodProfile, ...] -regimes : tuple[FeedRegimeEra, ...] - -__init__(self, discovery: AnalyticsDiscoveryResult, period_profiles: tuple[FeedPeriodProfile, ...], regimes: tuple[FeedRegimeEra, ...], metadata: dict[str, JSONValue]): None -summary(): dict[str, JSONValue] -to_dict(): dict[str, JSONValue] + +FeedRegimeReport + +discovery +epoch_definition : FeedEpochDefinitionV1 | None +metadata : dict[str, JSONValue] +period_profiles : tuple[FeedPeriodProfile, ...] +regimes : tuple[FeedRegimeEra, ...] + +__init__(self, discovery: AnalyticsDiscoveryResult, period_profiles: tuple[FeedPeriodProfile, ...], regimes: tuple[FeedRegimeEra, ...], epoch_definition: FeedEpochDefinitionV1 | None, metadata: dict[str, JSONValue]): None +summary(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] - + histdatacom.data_analytics.feed_regimes.FeedRegimeReport->histdatacom.data_analytics.feed_regimes.AnalyticsDiscoveryResult - - -discovery + + +discovery - + histdatacom.exceptions.FileSystemOperationError - -FileSystemOperationError - -category : FILESYSTEM -code : str -retryable : bool - - + +FileSystemOperationError + +category : FILESYSTEM +code : str +retryable : bool + + - + histdatacom.exceptions.FileSystemOperationError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.data_quality.fingerprint_contracts.FingerprintPlannedSectionContract - -FingerprintPlannedSectionContract - -issue : str -name : str - -__init__(self, name: str, issue: str): None -to_discovery_payload(): dict[str, JSONValue] + +FingerprintPlannedSectionContract + +issue : str +name : str + +__init__(self, name: str, issue: str): None +to_discovery_payload(): dict[str, JSONValue] - + histdatacom.data_quality.fingerprint_contracts.FingerprintReportSurfaceContract - -FingerprintReportSurfaceContract - -bounded_payload_key : str -cli_summary_heading : str -cli_summary_section : str -intentional_absence_reason : str -key : str -report_metadata_key : str -summary_schema_key : str - -__init__(self, key: str, summary_schema_key: str, report_metadata_key: str, bounded_payload_key: str, cli_summary_section: str, cli_summary_heading: str, intentional_absence_reason: str): None -to_discovery_payload(): dict[str, JSONValue] + +FingerprintReportSurfaceContract + +bounded_payload_key : str +cli_summary_heading : str +cli_summary_section : str +intentional_absence_reason : str +key : str +report_metadata_key : str +summary_schema_key : str + +__init__(self, key: str, summary_schema_key: str, report_metadata_key: str, bounded_payload_key: str, cli_summary_section: str, cli_summary_heading: str, intentional_absence_reason: str): None +to_discovery_payload(): dict[str, JSONValue] - + histdatacom.data_quality.fingerprint_contracts.FingerprintRunSectionContract - -FingerprintRunSectionContract - -issue : str -name : str -rule_id : str -status : str - -__init__(self, name: str, status: str, rule_id: str, issue: str): None -to_discovery_payload(): dict[str, JSONValue] + +FingerprintRunSectionContract + +issue : str +name : str +rule_id : str +status : str + +__init__(self, name: str, status: str, rule_id: str, issue: str): None +to_discovery_payload(): dict[str, JSONValue] - + histdatacom.data_quality.fingerprint_contracts.FingerprintSchemaContract - -FingerprintSchemaContract - -bounded_payload_key : str -issue : str -key : str -metadata_key : str -payload_path : str -rule_id : str -schema_version : str | None -status : str - -__init__(self, key: str, schema_version: str | None, rule_id: str, status: str, metadata_key: str, payload_path: str, bounded_payload_key: str, issue: str): None -to_discovery_payload(): dict[str, JSONValue] + +FingerprintSchemaContract + +bounded_payload_key : str +issue : str +key : str +metadata_key : str +payload_path : str +rule_id : str +schema_version : str | None +status : str + +__init__(self, key: str, schema_version: str | None, rule_id: str, status: str, metadata_key: str, payload_path: str, bounded_payload_key: str, issue: str): None +to_discovery_payload(): dict[str, JSONValue] - + histdatacom.data_quality.fingerprint_contracts.FingerprintTargetSectionContract - -FingerprintTargetSectionContract - -basis_values : tuple[str, ...] -description : str -extra : Mapping[str, JSONValue] -key_fields : tuple[str, ...] -name : str -row_order_values : tuple[str, ...] -schema_key : str -target_timeframes : tuple[str, ...] - -__init__(self, name: str, description: str, target_timeframes: tuple[str, ...], schema_key: str, key_fields: tuple[str, ...], basis_values: tuple[str, ...], row_order_values: tuple[str, ...], extra: Mapping[str, JSONValue]): None -to_discovery_payload(): dict[str, JSONValue] + +FingerprintTargetSectionContract + +basis_values : tuple[str, ...] +description : str +extra : Mapping[str, JSONValue] +key_fields : tuple[str, ...] +name : str +row_order_values : tuple[str, ...] +schema_key : str +target_timeframes : tuple[str, ...] + +__init__(self, name: str, description: str, target_timeframes: tuple[str, ...], schema_key: str, key_fields: tuple[str, ...], basis_values: tuple[str, ...], row_order_values: tuple[str, ...], extra: Mapping[str, JSONValue]): None +to_discovery_payload(): dict[str, JSONValue] - + histdatacom.fx_enums.Format - -Format - -name - -list_keys(): set -list_values(): set + +Format + +name + +list_keys(): set +list_values(): set - + histdatacom.data_quality.polars_cache.FreshPolarsCache - -FreshPolarsCache - -frame : Any -path : Path -source : str - -__init__(self, path: Path, frame: Any, source: str): None + +FreshPolarsCache + +cache_mtime_ns : int | None +frame : Any +fresh : bool | None +path : Path +source : str +source_mtime_ns : int | None + +__init__(self, path: Path, frame: Any, source: str, fresh: bool | None, source_mtime_ns: int | None, cache_mtime_ns: int | None): None - + histdatacom.data_quality.time.HistDataAsciiEstNoDstTimeRule - -HistDataAsciiEstNoDstTimeRule - -description : str -rule_id : str - -__init__(self, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiEstNoDstTimeRule + +description : str +rule_id : str + +__init__(self, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.ingestion.HistDataAsciiRowCountIngestionRule - -HistDataAsciiRowCountIngestionRule - -description : str -min_row_count : int -min_size_bytes : int -rule_id : str -size_severity -tiny_severity -truncation_severity - -__init__(self, min_row_count: int, min_size_bytes: int, tiny_severity: QualitySeverity, size_severity: QualitySeverity, truncation_severity: QualitySeverity, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiRowCountIngestionRule + +description : str +min_row_count : int +min_size_bytes : int +rule_id : str +size_severity +tiny_severity +truncation_severity + +__init__(self, min_row_count: int, min_size_bytes: int, tiny_severity: QualitySeverity, size_severity: QualitySeverity, truncation_severity: QualitySeverity, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.contracts.QualitySeverity - -QualitySeverity - -name -rank : int - -from_value(value: str | 'QualitySeverity' | None): 'QualitySeverity' -max(severities: Iterable[str | 'QualitySeverity' | None]): 'QualitySeverity' + +QualitySeverity + +name +rank : int + +from_value(value: str | 'QualitySeverity' | None): 'QualitySeverity' +max(severities: Iterable[str | 'QualitySeverity' | None]): 'QualitySeverity' - + histdatacom.data_quality.ingestion.HistDataAsciiRowCountIngestionRule->histdatacom.data_quality.contracts.QualitySeverity - - -tiny_severity + + +tiny_severity - + histdatacom.data_quality.ingestion.HistDataAsciiRowCountIngestionRule->histdatacom.data_quality.contracts.QualitySeverity - - -size_severity + + +size_severity - + histdatacom.data_quality.ingestion.HistDataAsciiRowCountIngestionRule->histdatacom.data_quality.contracts.QualitySeverity - - -truncation_severity + + +truncation_severity - + histdatacom.data_quality.ingestion.HistDataAsciiSchemaIngestionRule - -HistDataAsciiSchemaIngestionRule - -description : str -rule_id : str - -__init__(self, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiSchemaIngestionRule + +description : str +rule_id : str + +__init__(self, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.ingestion.HistDataAsciiTextIngestionRule - -HistDataAsciiTextIngestionRule - -description : str -rule_id : str - -__init__(self, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiTextIngestionRule + +description : str +rule_id : str + +__init__(self, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.ticks.HistDataAsciiTickMicrostructureRule - -HistDataAsciiTickMicrostructureRule - -description : str -rule_id : str -session_name : str -thresholds -thresholds_by_asset_class : Mapping[str, HistDataTickMicrostructureThresholds] -thresholds_by_session : Mapping[str, HistDataTickMicrostructureThresholds] -thresholds_by_symbol : Mapping[str, HistDataTickMicrostructureThresholds] -thresholds_by_symbol_session : Mapping[str, HistDataTickMicrostructureThresholds] -warning_severity - -__init__(self, thresholds: HistDataTickMicrostructureThresholds, thresholds_by_symbol: Mapping[str, HistDataTickMicrostructureThresholds], thresholds_by_session: Mapping[str, HistDataTickMicrostructureThresholds], thresholds_by_asset_class: Mapping[str, HistDataTickMicrostructureThresholds], thresholds_by_symbol_session: Mapping[str, HistDataTickMicrostructureThresholds], session_name: str, warning_severity: QualitySeverity, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiTickMicrostructureRule + +description : str +rule_id : str +session_name : str +thresholds +thresholds_by_asset_class : Mapping[str, HistDataTickMicrostructureThresholds] +thresholds_by_session : Mapping[str, HistDataTickMicrostructureThresholds] +thresholds_by_symbol : Mapping[str, HistDataTickMicrostructureThresholds] +thresholds_by_symbol_session : Mapping[str, HistDataTickMicrostructureThresholds] +warning_severity + +__init__(self, thresholds: HistDataTickMicrostructureThresholds, thresholds_by_symbol: Mapping[str, HistDataTickMicrostructureThresholds], thresholds_by_session: Mapping[str, HistDataTickMicrostructureThresholds], thresholds_by_asset_class: Mapping[str, HistDataTickMicrostructureThresholds], thresholds_by_symbol_session: Mapping[str, HistDataTickMicrostructureThresholds], session_name: str, warning_severity: QualitySeverity, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.ticks.HistDataTickMicrostructureThresholds - -HistDataTickMicrostructureThresholds - -burst_max_interval_ms : int -burst_run_length : int -one_sided_run_length : int -stale_max_gap_ms : int -stale_quote_run_length : int - -__init__(self, stale_quote_run_length: int, stale_max_gap_ms: int, burst_max_interval_ms: int, burst_run_length: int, one_sided_run_length: int): None -__post_init__(): None -to_metadata(): dict[str, JSONValue] + +HistDataTickMicrostructureThresholds + +burst_max_interval_ms : int +burst_run_length : int +one_sided_run_length : int +stale_max_gap_ms : int +stale_quote_run_length : int + +__init__(self, stale_quote_run_length: int, stale_max_gap_ms: int, burst_max_interval_ms: int, burst_run_length: int, one_sided_run_length: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.ticks.HistDataAsciiTickMicrostructureRule->histdatacom.data_quality.ticks.HistDataTickMicrostructureThresholds - - -thresholds + + +thresholds - + histdatacom.data_quality.ticks.HistDataAsciiTickMicrostructureRule->histdatacom.data_quality.contracts.QualitySeverity - - -warning_severity + + +warning_severity - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRegimeRule - -HistDataAsciiTickSpreadRegimeRule - -description : str -rule_id : str -schema_severity -thresholds -thresholds_by_asset_class : Mapping[str, HistDataTickSpreadRegimeThresholds] -warning_severity - -__init__(self, thresholds: HistDataTickSpreadRegimeThresholds, thresholds_by_asset_class: Mapping[str, HistDataTickSpreadRegimeThresholds], warning_severity: QualitySeverity, schema_severity: QualitySeverity, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiTickSpreadRegimeRule + +description : str +rule_id : str +schema_severity +thresholds +thresholds_by_asset_class : Mapping[str, HistDataTickSpreadRegimeThresholds] +warning_severity + +__init__(self, thresholds: HistDataTickSpreadRegimeThresholds, thresholds_by_asset_class: Mapping[str, HistDataTickSpreadRegimeThresholds], warning_severity: QualitySeverity, schema_severity: QualitySeverity, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.ticks.HistDataTickSpreadRegimeThresholds - -HistDataTickSpreadRegimeThresholds - -jump_spread_multiplier : float -minimum_spread_jump : float -minimum_wide_spread : float -regime_median_multiplier : float -wide_spread_multiplier : float - -__init__(self, wide_spread_multiplier: float, jump_spread_multiplier: float, regime_median_multiplier: float, minimum_wide_spread: float, minimum_spread_jump: float): None -__post_init__(): None -to_metadata(): dict[str, JSONValue] + +HistDataTickSpreadRegimeThresholds + +jump_spread_multiplier : float +minimum_spread_jump : float +minimum_wide_spread : float +regime_median_multiplier : float +wide_spread_multiplier : float + +__init__(self, wide_spread_multiplier: float, jump_spread_multiplier: float, regime_median_multiplier: float, minimum_wide_spread: float, minimum_spread_jump: float): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRegimeRule->histdatacom.data_quality.ticks.HistDataTickSpreadRegimeThresholds - - -thresholds + + +thresholds - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRegimeRule->histdatacom.data_quality.contracts.QualitySeverity - - -warning_severity + + +warning_severity - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRegimeRule->histdatacom.data_quality.contracts.QualitySeverity - - -schema_severity + + +schema_severity - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRule - -HistDataAsciiTickSpreadRule - -description : str -negative_spread_severity -rule_id : str -schema_severity -thresholds -thresholds_by_asset_class : Mapping[str, HistDataTickSpreadThresholds] -zero_spread_severity - -__init__(self, thresholds: HistDataTickSpreadThresholds, thresholds_by_asset_class: Mapping[str, HistDataTickSpreadThresholds], zero_spread_severity: QualitySeverity, negative_spread_severity: QualitySeverity, schema_severity: QualitySeverity, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiTickSpreadRule + +description : str +negative_spread_severity +rule_id : str +schema_severity +thresholds +thresholds_by_asset_class : Mapping[str, HistDataTickSpreadThresholds] +zero_spread_severity + +__init__(self, thresholds: HistDataTickSpreadThresholds, thresholds_by_asset_class: Mapping[str, HistDataTickSpreadThresholds], zero_spread_severity: QualitySeverity, negative_spread_severity: QualitySeverity, schema_severity: QualitySeverity, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.ticks.HistDataTickSpreadThresholds - -HistDataTickSpreadThresholds - -zero_spread_run_length : int - -__init__(self, zero_spread_run_length: int): None -__post_init__(): None -to_metadata(): dict[str, JSONValue] + +HistDataTickSpreadThresholds + +zero_spread_run_length : int + +__init__(self, zero_spread_run_length: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRule->histdatacom.data_quality.ticks.HistDataTickSpreadThresholds - - -thresholds + + +thresholds - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRule->histdatacom.data_quality.contracts.QualitySeverity - - -zero_spread_severity + + +zero_spread_severity - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRule->histdatacom.data_quality.contracts.QualitySeverity - - -negative_spread_severity + + +negative_spread_severity - + histdatacom.data_quality.ticks.HistDataAsciiTickSpreadRule->histdatacom.data_quality.contracts.QualitySeverity - - -schema_severity + + +schema_severity - + histdatacom.data_quality.time.HistDataAsciiTimestampContinuityRule - -HistDataAsciiTimestampContinuityRule - -description : str -rule_id : str -tolerance -warning_severity - -__init__(self, tolerance: HistDataGapTolerance, warning_severity: QualitySeverity, rule_id: str, description: str): None -evaluate_run(targets: Iterable[QualityTarget]): QualityReport + +HistDataAsciiTimestampContinuityRule + +description : str +rule_id : str +tolerance +warning_severity + +__init__(self, tolerance: HistDataGapTolerance, warning_severity: QualitySeverity, rule_id: str, description: str): None +evaluate_run(targets: Iterable[QualityTarget]): QualityReport - + histdatacom.data_quality.time.HistDataGapTolerance - -HistDataGapTolerance - -bucket_thresholds_ms : tuple[int, ...] -dynamic_window_growth_factor : float -dynamic_window_initial_ms : int -dynamic_window_max_ms : int -dynamic_window_shrink_factor : float -expected_interval_ms : int -session_boundary_grace_ms : int -suspicious_gap_ms : int - -__init__(self, expected_interval_ms: int, suspicious_gap_ms: int, bucket_thresholds_ms: tuple[int, ...], session_boundary_grace_ms: int, dynamic_window_initial_ms: int, dynamic_window_max_ms: int, dynamic_window_growth_factor: float, dynamic_window_shrink_factor: float): None -to_metadata(): dict[str, JSONValue] + +HistDataGapTolerance + +bucket_thresholds_ms : tuple[int, ...] +dynamic_window_growth_factor : float +dynamic_window_initial_ms : int +dynamic_window_max_ms : int +dynamic_window_shrink_factor : float +expected_interval_ms : int +session_boundary_grace_ms : int +suspicious_gap_ms : int + +__init__(self, expected_interval_ms: int, suspicious_gap_ms: int, bucket_thresholds_ms: tuple[int, ...], session_boundary_grace_ms: int, dynamic_window_initial_ms: int, dynamic_window_max_ms: int, dynamic_window_growth_factor: float, dynamic_window_shrink_factor: float): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.time.HistDataAsciiTimestampContinuityRule->histdatacom.data_quality.time.HistDataGapTolerance - - -tolerance + + +tolerance - + histdatacom.data_quality.time.HistDataAsciiTimestampContinuityRule->histdatacom.data_quality.contracts.QualitySeverity - - -warning_severity + + +warning_severity - + histdatacom.data_quality.time.HistDataAsciiTimestampGapRule - -HistDataAsciiTimestampGapRule - -description : str -rule_id : str -tolerance -warning_severity - -__init__(self, tolerance: HistDataGapTolerance, warning_severity: QualitySeverity, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiTimestampGapRule + +description : str +rule_id : str +tolerance +warning_severity + +__init__(self, tolerance: HistDataGapTolerance, warning_severity: QualitySeverity, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.time.HistDataAsciiTimestampGapRule->histdatacom.data_quality.time.HistDataGapTolerance - - -tolerance + + +tolerance - + histdatacom.data_quality.time.HistDataAsciiTimestampGapRule->histdatacom.data_quality.contracts.QualitySeverity - - -warning_severity + + +warning_severity - + histdatacom.data_quality.time.HistDataAsciiTimestampSequenceRule - -HistDataAsciiTimestampSequenceRule - -description : str -rule_id : str - -__init__(self, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataAsciiTimestampSequenceRule + +description : str +rule_id : str + +__init__(self, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.calendar.HistDataCalendarClassification - -HistDataCalendarClassification - -active_sessions : tuple[str, ...] -calendar_tags : tuple[str, ...] -clock_sessions : tuple[str, ...] -event_tags : tuple[str, ...] -holiday_tags : tuple[str, ...] -overlaps : tuple[str, ...] -session_state : str -source_datetime : datetime -source_timestamp : str -source_timestamp_iso : str -special_tags : tuple[str, ...] -timestamp_utc_ms : int -utc_datetime : datetime -utc_timestamp : str - -__init__(self, timestamp_utc_ms: int, source_timestamp: str, source_datetime: datetime, utc_datetime: datetime, session_state: str, clock_sessions: tuple[str, ...], active_sessions: tuple[str, ...], overlaps: tuple[str, ...], special_tags: tuple[str, ...], holiday_tags: tuple[str, ...], event_tags: tuple[str, ...], calendar_tags: tuple[str, ...]): None -to_metadata(): dict[str, JSONValue] + +HistDataCalendarClassification + +active_sessions : tuple[str, ...] +calendar_tags : tuple[str, ...] +clock_sessions : tuple[str, ...] +event_tags : tuple[str, ...] +holiday_tags : tuple[str, ...] +overlaps : tuple[str, ...] +session_state : str +source_datetime : datetime +source_timestamp : str +source_timestamp_iso : str +special_tags : tuple[str, ...] +timestamp_utc_ms : int +utc_datetime : datetime +utc_timestamp : str + +__init__(self, timestamp_utc_ms: int, source_timestamp: str, source_datetime: datetime, utc_datetime: datetime, session_state: str, clock_sessions: tuple[str, ...], active_sessions: tuple[str, ...], overlaps: tuple[str, ...], special_tags: tuple[str, ...], holiday_tags: tuple[str, ...], event_tags: tuple[str, ...], calendar_tags: tuple[str, ...]): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.calendar_profiles.HistDataCalendarDateTag - -HistDataCalendarDateTag - -asset_classes : tuple[str, ...] -category : str -day : int -description : str -is_movable : bool -markets : tuple[str, ...] -month : int -name : str -offset_days : int -rule : str -tag : str - -__init__(self, name: str, tag: str, month: int, day: int, rule: str, offset_days: int, category: str, description: str, markets: tuple[str, ...], asset_classes: tuple[str, ...]): None -__post_init__(): None -matches(source: datetime): bool -matches_fields(): bool -movable_date_for_year(year: int): date -to_metadata(): dict[str, JSONValue] + +HistDataCalendarDateTag + +asset_classes : tuple[str, ...] +category : str +day : int +description : str +is_movable : bool +markets : tuple[str, ...] +month : int +name : str +offset_days : int +rule : str +tag : str + +__init__(self, name: str, tag: str, month: int, day: int, rule: str, offset_days: int, category: str, description: str, markets: tuple[str, ...], asset_classes: tuple[str, ...]): None +__post_init__(): None +matches(source: datetime): bool +matches_fields(): bool +movable_date_for_year(year: int): date +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.calendar_profiles.HistDataCalendarProfile - -HistDataCalendarProfile - -complete : bool -date_tags : tuple[HistDataCalendarDateTag, ...] -limitations : tuple[str, ...] -name : str -schema_version : str -source : str -static_advisory : bool -version : str -window_tags : tuple[HistDataCalendarWindowTag, ...] - -__init__(self, name: str, source: str, version: str, schema_version: str, complete: bool, static_advisory: bool, limitations: tuple[str, ...], date_tags: tuple[HistDataCalendarDateTag, ...], window_tags: tuple[HistDataCalendarWindowTag, ...]): None -__post_init__(): None -applicable_date_tags(): tuple[HistDataCalendarDateTag, ...] -applicable_window_tags(): tuple[HistDataCalendarWindowTag, ...] -event_tags_for(source: datetime): tuple[str, ...] -event_tags_for_fields(): tuple[str, ...] -holiday_tags_for(source: datetime): tuple[str, ...] -holiday_tags_for_fields(): tuple[str, ...] -to_metadata(): dict[str, JSONValue] + +HistDataCalendarProfile + +complete : bool +date_tags : tuple[HistDataCalendarDateTag, ...] +expected_session_closure_policy : str +limitations : tuple[str, ...] +name : str +schema_version : str +source : str +static_advisory : bool +version : str +weekend_activity_policy : str +window_tags : tuple[HistDataCalendarWindowTag, ...] + +__init__(self, name: str, source: str, version: str, schema_version: str, complete: bool, static_advisory: bool, weekend_activity_policy: str, expected_session_closure_policy: str, limitations: tuple[str, ...], date_tags: tuple[HistDataCalendarDateTag, ...], window_tags: tuple[HistDataCalendarWindowTag, ...]): None +__post_init__(): None +applicable_date_tags(): tuple[HistDataCalendarDateTag, ...] +applicable_window_tags(): tuple[HistDataCalendarWindowTag, ...] +event_tags_for(source: datetime): tuple[str, ...] +event_tags_for_fields(): tuple[str, ...] +holiday_tags_for(source: datetime): tuple[str, ...] +holiday_tags_for_fields(): tuple[str, ...] +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.calendar.HistDataCalendarSessionRule - -HistDataCalendarSessionRule - -calendar_profile -description : str -profile_missing_severity -rule_id : str -source_severity -timestamp_severity - -__init__(self, calendar_profile: HistDataCalendarProfile, timestamp_severity: QualitySeverity, source_severity: QualitySeverity, profile_missing_severity: QualitySeverity, rule_id: str, description: str): None -_findings_for_scan(target: QualityTarget, scan: _CalendarScan): tuple[QualityFinding, ...] -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataCalendarSessionRule + +calendar_profile +description : str +profile_missing_severity +rule_id : str +source_severity +timestamp_severity + +__init__(self, calendar_profile: HistDataCalendarProfile, timestamp_severity: QualitySeverity, source_severity: QualitySeverity, profile_missing_severity: QualitySeverity, rule_id: str, description: str): None +_findings_for_scan(target: QualityTarget, scan: _CalendarScan): tuple[QualityFinding, ...] +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.calendar.HistDataCalendarSessionRule->histdatacom.data_quality.calendar_profiles.HistDataCalendarProfile - - -calendar_profile + + +calendar_profile - + histdatacom.data_quality.calendar.HistDataCalendarSessionRule->histdatacom.data_quality.contracts.QualitySeverity - - -timestamp_severity + + +timestamp_severity - + histdatacom.data_quality.calendar.HistDataCalendarSessionRule->histdatacom.data_quality.contracts.QualitySeverity - - -source_severity + + +source_severity - + histdatacom.data_quality.calendar.HistDataCalendarSessionRule->histdatacom.data_quality.contracts.QualitySeverity - - -profile_missing_severity + + +profile_missing_severity - + histdatacom.data_quality.calendar_profiles.HistDataCalendarWindowTag - -HistDataCalendarWindowTag - -asset_classes : tuple[str, ...] -category : str -description : str -end_date : str -end_day : int -end_day_number : int -end_month : int -markets : tuple[str, ...] -name : str -start_date : str -start_day : int -start_day_number : int -start_month : int -tag : str -uses_absolute_dates : bool - -__init__(self, name: str, tag: str, category: str, start_month: int, start_day: int, end_month: int, end_day: int, start_date: str, end_date: str, description: str, markets: tuple[str, ...], asset_classes: tuple[str, ...]): None -__post_init__(): None -matches(source: datetime): bool -matches_fields(): bool -to_metadata(): dict[str, JSONValue] + +HistDataCalendarWindowTag + +asset_classes : tuple[str, ...] +category : str +description : str +end_date : str +end_day : int +end_day_number : int +end_month : int +markets : tuple[str, ...] +name : str +start_date : str +start_day : int +start_day_number : int +start_month : int +tag : str +uses_absolute_dates : bool + +__init__(self, name: str, tag: str, category: str, start_month: int, start_day: int, end_month: int, end_day: int, start_date: str, end_date: str, description: str, markets: tuple[str, ...], asset_classes: tuple[str, ...]): None +__post_init__(): None +matches(source: datetime): bool +matches_fields(): bool +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.calendar.HistDataClockWindow - -HistDataClockWindow - -description : str -end_minute : int -name : str -start_minute : int -timezone : str - -__init__(self, name: str, start_minute: int, end_minute: int, timezone: str, description: str): None -contains(minute_of_day: int): bool -to_metadata(): dict[str, JSONValue] + +HistDataClockWindow + +description : str +end_minute : int +name : str +start_minute : int +timezone : str + +__init__(self, name: str, start_minute: int, end_minute: int, timezone: str, description: str): None +contains(minute_of_day: int): bool +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.manifest.HistDataCoverageManifestRule - -HistDataCoverageManifestRule - -description : str -rule_id : str - -__init__(self, rule_id: str, description: str): None -evaluate_run(targets: Iterable[QualityTarget]): QualityReport + +HistDataCoverageManifestRule + +description : str +rule_id : str + +__init__(self, rule_id: str, description: str): None +evaluate_run(targets: Iterable[QualityTarget]): QualityReport - + histdatacom.data_quality.symbols.HistDataCrossInstrumentConsistencyRule - -HistDataCrossInstrumentConsistencyRule - -description : str -error_severity -rule_id : str -tolerance -warning_severity - -__init__(self, tolerance: HistDataCrossInstrumentTolerance, warning_severity: QualitySeverity, error_severity: QualitySeverity, rule_id: str, description: str): None -evaluate_run(targets: Iterable[QualityTarget]): QualityReport + +HistDataCrossInstrumentConsistencyRule + +description : str +error_severity +rule_id : str +scan_provider : CrossInstrumentScanProvider | None +tolerance +warning_severity + +__init__(self, tolerance: HistDataCrossInstrumentTolerance, warning_severity: QualitySeverity, error_severity: QualitySeverity, scan_provider: CrossInstrumentScanProvider | None, rule_id: str, description: str): None +evaluate_run(targets: Iterable[QualityTarget]): QualityReport +evaluate_series(series: Iterable[CrossInstrumentSeriesInput]): QualityReport - + histdatacom.data_quality.symbols.HistDataCrossInstrumentTolerance - -HistDataCrossInstrumentTolerance - -inverse_error_relative_tolerance : float -inverse_warning_relative_tolerance : float -minimum_common_timestamp_ratio : float -stale_forward_fill_min_run : int -triangular_error_relative_tolerance : float -triangular_warning_relative_tolerance : float - -__init__(self, triangular_warning_relative_tolerance: float, triangular_error_relative_tolerance: float, inverse_warning_relative_tolerance: float, inverse_error_relative_tolerance: float, minimum_common_timestamp_ratio: float, stale_forward_fill_min_run: int): None -__post_init__(): None -to_metadata(): dict[str, JSONValue] + +HistDataCrossInstrumentTolerance + +inverse_error_relative_tolerance : float +inverse_warning_relative_tolerance : float +minimum_common_timestamp_ratio : float +stale_forward_fill_min_run : int +triangular_error_relative_tolerance : float +triangular_warning_relative_tolerance : float + +__init__(self, triangular_warning_relative_tolerance: float, triangular_error_relative_tolerance: float, inverse_warning_relative_tolerance: float, inverse_error_relative_tolerance: float, minimum_common_timestamp_ratio: float, stale_forward_fill_min_run: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.symbols.HistDataCrossInstrumentConsistencyRule->histdatacom.data_quality.symbols.HistDataCrossInstrumentTolerance - - -tolerance + + +tolerance - + histdatacom.data_quality.symbols.HistDataCrossInstrumentConsistencyRule->histdatacom.data_quality.contracts.QualitySeverity - - -warning_severity + + +warning_severity - + histdatacom.data_quality.symbols.HistDataCrossInstrumentConsistencyRule->histdatacom.data_quality.contracts.QualitySeverity - - -error_severity + + +error_severity + + + +histdatacom.data_quality.symbols.HistDataCrossSeriesFingerprintRule + +HistDataCrossSeriesFingerprintRule + +correlation_limit : int +description : str +error_severity +group_limit : int +rule_id : str +scan_provider : CrossInstrumentScanProvider | None +tolerance +warning_severity + +__init__(self, tolerance: HistDataCrossInstrumentTolerance, warning_severity: QualitySeverity, error_severity: QualitySeverity, scan_provider: CrossInstrumentScanProvider | None, group_limit: int, correlation_limit: int, rule_id: str, description: str): None +evaluate_run(targets: Iterable[QualityTarget]): QualityReport + + + +histdatacom.data_quality.symbols.HistDataCrossSeriesFingerprintRule->histdatacom.data_quality.symbols.HistDataCrossInstrumentTolerance + + +tolerance + + + +histdatacom.data_quality.symbols.HistDataCrossSeriesFingerprintRule->histdatacom.data_quality.contracts.QualitySeverity + + +warning_severity + + + +histdatacom.data_quality.symbols.HistDataCrossSeriesFingerprintRule->histdatacom.data_quality.contracts.QualitySeverity + + +error_severity - + histdatacom.data_quality.fingerprints.HistDataFingerprintDistributionAttentionProfile - -HistDataFingerprintDistributionAttentionProfile - -flag_cache_float_precision : bool -flag_truncated_distribution : bool -invalid_row_min_count : int -invalid_row_min_rate : float -negative_spread_min_count : int -negative_spread_min_rate : float -zero_spread_min_count : int -zero_spread_min_rate : float - -__init__(self, invalid_row_min_count: int, invalid_row_min_rate: float, zero_spread_min_count: int, zero_spread_min_rate: float, negative_spread_min_count: int, negative_spread_min_rate: float, flag_truncated_distribution: bool, flag_cache_float_precision: bool): None -to_metadata(): dict[str, JSONValue] + +HistDataFingerprintDistributionAttentionProfile + +flag_cache_float_precision : bool +flag_truncated_distribution : bool +invalid_row_min_count : int +invalid_row_min_rate : float +negative_spread_min_count : int +negative_spread_min_rate : float +zero_spread_min_count : int +zero_spread_min_rate : float + +__init__(self, invalid_row_min_count: int, invalid_row_min_rate: float, zero_spread_min_count: int, zero_spread_min_rate: float, negative_spread_min_count: int, negative_spread_min_rate: float, flag_truncated_distribution: bool, flag_cache_float_precision: bool): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintParityProfile + +HistDataFingerprintParityProfile + +enabled : bool +mismatch_limit : int + +__init__(self, enabled: bool, mismatch_limit: int): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.fingerprints.HistDataFingerprintProfile - -HistDataFingerprintProfile - -calendar_profile -distribution_attention -histogram_bins : int -lags : tuple[int, ...] -max_rows : int -quantiles : tuple[float, ...] -rolling_windows : tuple[int, ...] -rounding_digits : int - -__init__(self, quantiles: tuple[float, ...], lags: tuple[int, ...], rolling_windows: tuple[int, ...], histogram_bins: int, max_rows: int, rounding_digits: int, calendar_profile: HistDataCalendarProfile, distribution_attention: HistDataFingerprintDistributionAttentionProfile): None -to_metadata(): dict[str, JSONValue] + +HistDataFingerprintProfile + +autoregressive +cache_source_parity +calendar_profile +classical_baselines +classical_model_comparison +classical_model_input +distribution_attention +exponential_smoothing +histogram_bins : int +lags : tuple[int, ...] +max_rows : int +quantiles : tuple[float, ...] +rolling_windows : tuple[int, ...] +rounding_digits : int +seasonal_exogenous +state_space +topology_inspection_sample_limit : int +volatility + +__init__(self, quantiles: tuple[float, ...], lags: tuple[int, ...], rolling_windows: tuple[int, ...], histogram_bins: int, max_rows: int, rounding_digits: int, topology_inspection_sample_limit: int, calendar_profile: HistDataCalendarProfile, distribution_attention: HistDataFingerprintDistributionAttentionProfile, cache_source_parity: HistDataFingerprintParityProfile, classical_baselines: ClassicalBaselineProfile, classical_model_input: ClassicalModelInputProfile, exponential_smoothing: ExponentialSmoothingProfile, autoregressive: AutoregressiveProfile, seasonal_exogenous: SeasonalExogenousProfile, state_space: StateSpaceProfile, volatility: VolatilityProfile, classical_model_comparison: ClassicalModelComparisonProfile): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.autoregressive.AutoregressiveProfile + + +autoregressive + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.classical_baselines.ClassicalBaselineProfile + + +classical_baselines + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.classical_model_comparison.ClassicalModelComparisonProfile + + +classical_model_comparison + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.classical_model_contracts.ClassicalModelInputProfile + + +classical_model_input + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.exponential_smoothing.ExponentialSmoothingProfile + + +exponential_smoothing - + histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.calendar_profiles.HistDataCalendarProfile - - -calendar_profile + + +calendar_profile - + histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.fingerprints.HistDataFingerprintDistributionAttentionProfile - - -distribution_attention + + +distribution_attention + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.fingerprints.HistDataFingerprintParityProfile + + +cache_source_parity + + + +histdatacom.data_quality.seasonal_exogenous.SeasonalExogenousProfile + +SeasonalExogenousProfile + +baseline_rolling_windows : tuple[int, ...] +compare_autoregressive : bool +compare_exponential_smoothing : bool +enabled : bool +projection_horizon : int +projection_specification_ids : tuple[str, ...] +regressor_profile +rounding_digits : int +specifications : tuple[SeasonalExogenousSpecification, ...] + +__init__(self, enabled: bool, specifications: tuple[SeasonalExogenousSpecification, ...], projection_specification_ids: tuple[str, ...], projection_horizon: int, regressor_profile: CalendarRegressorProfile, baseline_rolling_windows: tuple[int, ...], compare_exponential_smoothing: bool, compare_autoregressive: bool, rounding_digits: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.seasonal_exogenous.SeasonalExogenousProfile + + +seasonal_exogenous + + + +histdatacom.data_quality.state_space.StateSpaceProfile + +StateSpaceProfile + +baseline_rolling_windows : tuple[int, ...] +compare_autoregressive : bool +compare_exponential_smoothing : bool +compare_seasonal_exogenous : bool +enabled : bool +max_component_count : int +max_prediction_only_gap : int +max_retained_states : int +max_state_dimension : int +projection_horizon : int +projection_specification_id : str +rounding_digits : int +specifications : tuple[StateSpaceSpecification, ...] + +__init__(self, enabled: bool, specifications: tuple[StateSpaceSpecification, ...], projection_specification_id: str, projection_horizon: int, max_state_dimension: int, max_component_count: int, max_prediction_only_gap: int, max_retained_states: int, baseline_rolling_windows: tuple[int, ...], compare_exponential_smoothing: bool, compare_autoregressive: bool, compare_seasonal_exogenous: bool, rounding_digits: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.state_space.StateSpaceProfile + + +state_space + + + +histdatacom.data_quality.volatility.VolatilityProfile + +VolatilityProfile + +annualization_periods : int +baseline_rolling_windows : tuple[int, ...] +boundary_tolerance : float +compare_autoregressive : bool +compare_exponential_smoothing : bool +compare_seasonal_exogenous : bool +compare_state_space : bool +enabled : bool +ewma_decay : float +maximum_covariance_condition_number : float +maximum_persistence : float +projection_horizon : int +projection_specification_ids : tuple[str, ...] +realized_variance_proxy : str +rounding_digits : int +specifications : tuple[VolatilitySpecification, ...] + +__init__(self, enabled: bool, specifications: tuple[VolatilitySpecification, ...], projection_specification_ids: tuple[str, ...], projection_horizon: int, realized_variance_proxy: str, annualization_periods: int, baseline_rolling_windows: tuple[int, ...], ewma_decay: float, maximum_persistence: float, maximum_covariance_condition_number: float, boundary_tolerance: float, compare_exponential_smoothing: bool, compare_autoregressive: bool, compare_seasonal_exogenous: bool, compare_state_space: bool, rounding_digits: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.fingerprints.HistDataFingerprintProfile->histdatacom.data_quality.volatility.VolatilityProfile + + +volatility - + histdatacom.data_quality.format_support.HistDataFormatSupport - -HistDataFormatSupport - -canonical_cache_supported : bool -data_format : str -format_code : str -inventory_supported : bool -level : str -message : str -parser_supported : bool -payload_extension : str -schema : str -status : str -supported_check_groups : tuple[str, ...] -supported_timeframes : tuple[str, ...] -timeframe : str - -__init__(self, data_format: str, format_code: str, timeframe: str, status: str, level: str, inventory_supported: bool, parser_supported: bool, canonical_cache_supported: bool, payload_extension: str, schema: str, supported_timeframes: tuple[str, ...], supported_check_groups: tuple[str, ...], message: str): None -to_metadata(): dict[str, JSONValue] + +HistDataFormatSupport + +canonical_cache_supported : bool +data_format : str +format_code : str +inventory_supported : bool +level : str +message : str +parser_supported : bool +payload_extension : str +schema : str +status : str +supported_check_groups : tuple[str, ...] +supported_timeframes : tuple[str, ...] +timeframe : str + +__init__(self, data_format: str, format_code: str, timeframe: str, status: str, level: str, inventory_supported: bool, parser_supported: bool, canonical_cache_supported: bool, payload_extension: str, schema: str, supported_timeframes: tuple[str, ...], supported_check_groups: tuple[str, ...], message: str): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.inventory.HistDataFormatSupportRule - -HistDataFormatSupportRule - -description : str -rule_id : str - -__init__(self, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataFormatSupportRule + +description : str +rule_id : str + +__init__(self, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.modeling.HistDataModelingReadinessRule - -HistDataModelingReadinessRule - -assumptions : Mapping[str, JSONValue] -description : str -rule_id : str -warning_severity - -__init__(self, assumptions: Mapping[str, JSONValue], warning_severity: QualitySeverity, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataModelingReadinessRule + +assumptions : Mapping[str, JSONValue] +description : str +rule_id : str +warning_severity + +__init__(self, assumptions: Mapping[str, JSONValue], warning_severity: QualitySeverity, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.modeling.HistDataModelingReadinessRule->histdatacom.data_quality.contracts.QualitySeverity - - -warning_severity + + +warning_severity - + histdatacom.exceptions.HistDataNoDataError - -HistDataNoDataError - -category : NO_DATA - -__init__(message: str): None + +HistDataNoDataError + +category : NO_DATA + +__init__(message: str): None - + histdatacom.exceptions.UrlValidationError - -UrlValidationError - -category : PARSE -code : str -no_data : bool -retryable : bool - -__init__(code: str, message: str): None + +UrlValidationError + +category : PARSE +code : str +no_data : bool +retryable : bool + +__init__(code: str, message: str): None - + histdatacom.exceptions.HistDataNoDataError->histdatacom.exceptions.UrlValidationError - - + + - + histdatacom.data_quality.provenance.HistDataProvenanceManifestRule - -HistDataProvenanceManifestRule - -description : str -explicit : bool -max_findings : int -rule_id : str - -__init__(self, explicit: bool, max_findings: int, rule_id: str, description: str): None -evaluate_run(targets: Iterable[QualityTarget]): QualityReport + +HistDataProvenanceManifestRule + +description : str +explicit : bool +max_findings : int +rule_id : str + +__init__(self, explicit: bool, max_findings: int, rule_id: str, description: str): None +evaluate_run(targets: Iterable[QualityTarget]): QualityReport - + histdatacom.data_quality.profiles.HistDataRowCountProfile - -HistDataRowCountProfile - -min_row_count : int -min_size_bytes : int - -__init__(self, min_row_count: int, min_size_bytes: int): None + +HistDataRowCountProfile + +min_row_count : int +min_size_bytes : int + +__init__(self, min_row_count: int, min_size_bytes: int): None - + histdatacom.orchestration.workflows.HistDataRunWorkflow - -HistDataRunWorkflow - -_executor : ChildWorkflowExecutor | None -_progress - -__init__(executor: ChildWorkflowExecutor | None): None -run(request_payload: dict[str, Any]): dict -status(): dict + +HistDataRunWorkflow + +_executor : ChildWorkflowExecutor | None +_progress + +__init__(executor: ChildWorkflowExecutor | None): None +run(request_payload: dict[str, Any]): dict +status(): dict - + histdatacom.data_quality.fingerprints.HistDataSeriesFingerprintRule - -HistDataSeriesFingerprintRule - -description : str -profile -rule_id : str - -__init__(self, profile: HistDataFingerprintProfile, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataSeriesFingerprintRule + +description : str +profile +rule_id : str + +__init__(self, profile: HistDataFingerprintProfile, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.fingerprints.HistDataSeriesFingerprintRule->histdatacom.data_quality.fingerprints.HistDataFingerprintProfile - - -profile + + +profile - + histdatacom.data_quality.calendar.HistDataSessionWindow - -HistDataSessionWindow - -description : str -end_minute_utc : int -name : str -start_minute_utc : int - -__init__(self, name: str, start_minute_utc: int, end_minute_utc: int, description: str): None -contains(minute_of_day: int): bool -to_metadata(): dict[str, JSONValue] + +HistDataSessionWindow + +description : str +end_minute_utc : int +name : str +start_minute_utc : int + +__init__(self, name: str, start_minute_utc: int, end_minute_utc: int, description: str): None +contains(minute_of_day: int): bool +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.calendar.HistDataStaticHoliday - -HistDataStaticHoliday - -day : int -description : str -month : int -name : str - -__init__(self, name: str, month: int, day: int, description: str): None -matches(source: datetime): bool -to_metadata(): dict[str, JSONValue] + +HistDataStaticHoliday + +day : int +description : str +month : int +name : str + +__init__(self, name: str, month: int, day: int, description: str): None +matches(source: datetime): bool +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.symbols.HistDataSymbolMetadata - -HistDataSymbolMetadata - -aliases : tuple[str, ...] -asset_class : str -base : str -known : bool -normalized_symbol : str -pair_key : str -precision_rule : HistDataSymbolPrecisionRule | None -quote : str -source : str -symbol : str - -__init__(self, symbol: str, normalized_symbol: str, asset_class: str, base: str, quote: str, pair_key: str, source: str, precision_rule: HistDataSymbolPrecisionRule | None, aliases: tuple[str, ...]): None -to_metadata(): dict[str, JSONValue] + +HistDataSymbolMetadata + +aliases : tuple[str, ...] +asset_class : str +base : str +known : bool +normalized_symbol : str +pair_key : str +precision_rule : HistDataSymbolPrecisionRule | None +quote : str +source : str +symbol : str + +__init__(self, symbol: str, normalized_symbol: str, asset_class: str, base: str, quote: str, pair_key: str, source: str, precision_rule: HistDataSymbolPrecisionRule | None, aliases: tuple[str, ...]): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.symbols.HistDataSymbolMetadataRule - -HistDataSymbolMetadataRule - -description : str -rule_id : str -warning_severity - -__init__(self, warning_severity: QualitySeverity, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataSymbolMetadataRule + +description : str +rule_id : str +warning_severity + +__init__(self, warning_severity: QualitySeverity, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] - + histdatacom.data_quality.symbols.HistDataSymbolMetadataRule->histdatacom.data_quality.contracts.QualitySeverity - - -warning_severity + + +warning_severity - + histdatacom.data_quality.symbols.HistDataSymbolPrecisionRule - -HistDataSymbolPrecisionRule - -expected_decimal_places : tuple[int, ...] -name : str -pip_size : str -quote_side : str -tick_size : str - -__init__(self, name: str, expected_decimal_places: tuple[int, ...], pip_size: str, tick_size: str, quote_side: str): None -to_metadata(): dict[str, JSONValue] + +HistDataSymbolPrecisionRule + +expected_decimal_places : tuple[int, ...] +name : str +pip_size : str +quote_side : str +tick_size : str + +__init__(self, name: str, expected_decimal_places: tuple[int, ...], pip_size: str, tick_size: str, quote_side: str): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.inventory.HistDataZipInventoryRule - -HistDataZipInventoryRule - -description : str -rule_id : str - -__init__(self, rule_id: str, description: str): None -evaluate(target: QualityTarget): tuple[QualityFinding, ...] + +HistDataZipInventoryRule + +description : str +rule_id : str + +__init__(self, rule_id: str, description: str): None +evaluate(target: QualityTarget): tuple[QualityFinding, ...] + + + +histdatacom.synthetic.carving.HistoricalCarvedCandidateBatchV1 + +HistoricalCarvedCandidateBatchV1 + +accepted_events : tuple[SyntheticEventV1, ...] +accepted_lineage : tuple[HistoricalCarvingEventLineageV1, ...] +anchor_interval_id : str +batch_id : str +carry_state +constraint_set_id : str +ensemble_member_id : str +input_candidate_batch_ids : tuple[str, ...] +left_anchor_event_id : str +market_context_query_id : str +market_context_timeline_id : str +projected_event_count : int +refusal_reason : CarvingReason | None +rejection_examples : tuple[HistoricalCarvingRejectionExampleV1, ...] +rejection_summary +right_anchor_event_id : str +run_id : str +schema_version : str +status +substituted_event_count : int +symbol : str +validation_evidence +window_id : str + +__init__(self, run_id: str, window_id: str, ensemble_member_id: str, symbol: str, anchor_interval_id: str, left_anchor_event_id: str, right_anchor_event_id: str, constraint_set_id: str, input_candidate_batch_ids: tuple[str, ...], market_context_query_id: str, market_context_timeline_id: str, status: CarvingBatchStatus, accepted_events: tuple[SyntheticEventV1, ...], accepted_lineage: tuple[HistoricalCarvingEventLineageV1, ...], rejection_summary: RejectionSummaryV1, rejection_examples: tuple[HistoricalCarvingRejectionExampleV1, ...], validation_evidence: HistoricalCarvingValidationEvidenceV1, carry_state: CarryStateV1, projected_event_count: int, substituted_event_count: int, refusal_reason: CarvingReason | None, batch_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'HistoricalCarvedCandidateBatchV1' +from_json(text: str): 'HistoricalCarvedCandidateBatchV1' +merged_stream(observed_events: Sequence[SyntheticEventV1]): SyntheticEventStreamV1 +metadata(): dict[str, JSONValue] +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.carving.HistoricalCarvedCandidateBatchV1->histdatacom.synthetic.streaming.CarryStateV1 + + +carry_state + + + +histdatacom.synthetic.carving.HistoricalCarvedCandidateBatchV1->histdatacom.synthetic.carving.CarvingBatchStatus + + +status + + + +histdatacom.synthetic.carving.HistoricalCarvingValidationEvidenceV1 + +HistoricalCarvingValidationEvidenceV1 + +evidence_id : str +fingerprint_evidence_id : str | None +local_validation_status : str +observed_anchor_content_sha256 : tuple[str, ...] +schema_version : str +validator_ids : tuple[str, ...] + +__init__(self, fingerprint_evidence_id: str | None, local_validation_status: str, observed_anchor_content_sha256: tuple[str, ...], validator_ids: tuple[str, ...], evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'HistoricalCarvingValidationEvidenceV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.carving.HistoricalCarvedCandidateBatchV1->histdatacom.synthetic.carving.HistoricalCarvingValidationEvidenceV1 + + +validation_evidence + + + +histdatacom.synthetic.streaming.RejectionSummaryV1 + +RejectionSummaryV1 + +accepted_count : int +candidate_count : int +reason_counts : dict[str, int] +rejected_count : int +run_id : str +schema_version : str +summary_id : str +window_id : str + +__init__(self, run_id: str, window_id: str, candidate_count: int, accepted_count: int, rejected_count: int, reason_counts: dict[str, int], summary_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'RejectionSummaryV1' +from_json(text: str): 'RejectionSummaryV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.carving.HistoricalCarvedCandidateBatchV1->histdatacom.synthetic.streaming.RejectionSummaryV1 + + +rejection_summary + + + +histdatacom.synthetic.carving.HistoricalCarvingConditionPolicyV1 + +HistoricalCarvingConditionPolicyV1 + +acceptance_rate : float +eligible_motif_ids : tuple[str, ...] +match_tags : tuple[str, ...] +name : str +policy_id : str +priority : int +schema_version : str +spread_multiplier : float + +__init__(self, name: str, match_tags: tuple[str, ...], acceptance_rate: float, spread_multiplier: float, eligible_motif_ids: tuple[str, ...], priority: int, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'HistoricalCarvingConditionPolicyV1' +from_json(text: str): 'HistoricalCarvingConditionPolicyV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.carving.HistoricalCarvingConstraintSetV1 + +HistoricalCarvingConstraintSetV1 + +closed_session_states : tuple[str, ...] +condition_policies : tuple[HistoricalCarvingConditionPolicyV1, ...] +constraint_set_id : str +fingerprint_constraint_id : str +max_anchor_gap_ns : int +max_combined_spread_multiplier : float +max_input_candidate_events : int +max_rejection_examples : int +price_precision_digits : int +quarantines : tuple[HistoricalCarvingQuarantineV1, ...] +require_complete_calendar_profile : bool +require_fingerprint_validation : bool +schema_version : str + +__init__(self, fingerprint_constraint_id: str, condition_policies: tuple[HistoricalCarvingConditionPolicyV1, ...], quarantines: tuple[HistoricalCarvingQuarantineV1, ...], closed_session_states: tuple[str, ...], require_complete_calendar_profile: bool, require_fingerprint_validation: bool, max_anchor_gap_ns: int, max_input_candidate_events: int, max_rejection_examples: int, max_combined_spread_multiplier: float, price_precision_digits: int, constraint_set_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'HistoricalCarvingConstraintSetV1' +from_json(text: str): 'HistoricalCarvingConstraintSetV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.carving.HistoricalCarvingEventLineageV1 + +HistoricalCarvingEventLineageV1 + +acceptance_score : float +action +candidate_batch_id : str +candidate_content_sha256 : str +candidate_event_id : str +candidate_transformation_id : str +context_event_ids : tuple[str, ...] +final_constraint_set_id : str +lineage_id : str +original_ask : float | None +original_bid : float | None +original_constraint_set_id : str +output_content_sha256 : str +output_event_id : str +policy_ids : tuple[str, ...] +rule_ids : tuple[str, ...] +schema_version : str +spread_multiplier : float + +__init__(self, output_event_id: str, output_content_sha256: str, candidate_event_id: str, candidate_content_sha256: str, candidate_batch_id: str, candidate_transformation_id: str, action: CarvingEventAction, rule_ids: tuple[str, ...], context_event_ids: tuple[str, ...], policy_ids: tuple[str, ...], original_constraint_set_id: str, final_constraint_set_id: str, acceptance_score: float, spread_multiplier: float, original_bid: float | None, original_ask: float | None, lineage_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'HistoricalCarvingEventLineageV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.carving.HistoricalCarvingEventLineageV1->histdatacom.synthetic.carving.CarvingEventAction + + +action + + + +histdatacom.synthetic.carving.HistoricalCarvingQuarantineV1 + +HistoricalCarvingQuarantineV1 + +end_ns : int +quarantine_id : str +reason : str +schema_version : str +source_id : str +start_ns : int +symbol : str + +__init__(self, symbol: str, start_ns: int, end_ns: int, reason: str, source_id: str, quarantine_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'HistoricalCarvingQuarantineV1' +from_json(text: str): 'HistoricalCarvingQuarantineV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.carving.HistoricalCarvingRejectionExampleV1 + +HistoricalCarvingRejectionExampleV1 + +candidate_batch_id : str +candidate_content_sha256 : str +candidate_event_id : str +event_sequence : int +event_time_ns : int +example_id : str +reason +rule_ids : tuple[str, ...] +schema_version : str + +__init__(self, candidate_event_id: str, candidate_content_sha256: str, candidate_batch_id: str, event_time_ns: int, event_sequence: int, reason: CarvingReason, rule_ids: tuple[str, ...], example_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'HistoricalCarvingRejectionExampleV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.carving.HistoricalCarvingRejectionExampleV1->histdatacom.synthetic.carving.CarvingReason + + +reason - + histdatacom.orchestration.workflows.ImportWorkflow - -ImportWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +ImportWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows.ImportWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + - + histdatacom.influx.Influx - -Influx - -args : dict[str, Any] - -__init__(args: Mapping[str, Any] | None): None -_import_cache(record: Record, args: Mapping[str, Any], emit_lines: LineSink | Any): None -_import_file(record: Record, args: Mapping[str, Any], emit_lines: LineSink | Any): Record -_parse_cache_row(row: Tuple[Any, ...], record: Record): str -_parse_cache_rows(iterable: Iterable, record: Record): list[str] -import_data(records: Iterable[Record], args: Mapping[str, Any] | None): list[Record] + +Influx + +args : dict[str, Any] + +__init__(args: Mapping[str, Any] | None): None +_import_cache(record: Record, args: Mapping[str, Any], emit_lines: LineSink | Any): None +_import_file(record: Record, args: Mapping[str, Any], emit_lines: LineSink | Any): Record +_parse_cache_row(row: Tuple[Any, ...], record: Record): str +_parse_cache_rows(iterable: Iterable, record: Record): list[str] +import_data(records: Iterable[Record], args: Mapping[str, Any] | None): list[Record] - + histdatacom.influx.InfluxBatchWriter - -InfluxBatchWriter - -args : dict -client : InfluxDBClient -write_api : WriteApi -write_precision : str - -__enter__(): 'InfluxBatchWriter' -__exit__(): None -__init__(args: Mapping[str, Any]) -close(): None -terminate(): None -write_lines(lines: list[str]): None + +InfluxBatchWriter + +args : dict +client : InfluxDBClient +write_api : WriteApi +write_precision : str + +__enter__(): 'InfluxBatchWriter' +__exit__(): None +__init__(args: Mapping[str, Any]) +close(): None +terminate(): None +write_lines(lines: list[str]): None - + histdatacom.exceptions.InfluxConfigurationError - -InfluxConfigurationError - -category : CONFIGURATION -code : str -retryable : bool - - + +InfluxConfigurationError + +category : CONFIGURATION +code : str +retryable : bool + + - + histdatacom.exceptions.InfluxError - -InfluxError - -category : INFLUX -code : str -retryable : bool - - + +InfluxError + +category : INFLUX +code : str +retryable : bool + + - + histdatacom.exceptions.InfluxConfigurationError->histdatacom.exceptions.InfluxError - - + + - + histdatacom.exceptions.InfluxDependencyError - -InfluxDependencyError - -category : DEPENDENCY -code : str -retryable : bool - - + +InfluxDependencyError + +category : DEPENDENCY +code : str +retryable : bool + + - + histdatacom.exceptions.InfluxDependencyError->histdatacom.exceptions.InfluxError - - + + - + histdatacom.exceptions.InfluxError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.exceptions.InfluxImportError - -InfluxImportError - -code : str - - + +InfluxImportError + +code : str + + - + histdatacom.exceptions.InfluxImportError->histdatacom.exceptions.InfluxError - - + + + + + +histdatacom.synthetic.information.InformationAuditFindingV1 + +InformationAuditFindingV1 + +evidence : Mapping[str, JSONValue] | None +finding_id : str +input_id : str | None +message : str +rule_id +schema_version : str + +__init__(self, rule_id: InformationAuditRule, message: str, input_id: str | None, evidence: Mapping[str, JSONValue] | None, finding_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'InformationAuditFindingV1' +from_json(text: str): 'InformationAuditFindingV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.information.InformationAuditRule + +InformationAuditRule + +name + +from_value(value: str | 'InformationAuditRule'): 'InformationAuditRule' + + + +histdatacom.synthetic.information.InformationAuditFindingV1->histdatacom.synthetic.information.InformationAuditRule + + +rule_id + + + +histdatacom.synthetic.information.InformationAuditReportV1 + +InformationAuditReportV1 + +accepted : bool +audit_id : str +evidence_truncated : bool +findings : tuple[InformationAuditFindingV1, ...] +information_mode +invalid_for : tuple[str, ...] +manifest_id : str +policy_id : str +run_id : str +schema_version : str +summary : str +total_violation_count : int +valid_for : tuple[str, ...] +valid_for_strategy_usefulness_claim : bool +window_plan_id : str + +__init__(self, run_id: str, policy_id: str, manifest_id: str, window_plan_id: str, information_mode: InformationMode, accepted: bool, total_violation_count: int, findings: tuple[InformationAuditFindingV1, ...], evidence_truncated: bool, valid_for: tuple[str, ...], invalid_for: tuple[str, ...], summary: str, audit_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'InformationAuditReportV1' +from_json(text: str): 'InformationAuditReportV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.information.InformationLeakageError + +InformationLeakageError + +report + +__init__(report: InformationAuditReportV1): None + + + +histdatacom.synthetic.information.InformationAuditReportV1->histdatacom.synthetic.information.InformationLeakageError + + +report + + + +histdatacom.synthetic.information.InformationAuditReportV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.information.InformationInputKind + +InformationInputKind + +name + +from_value(value: str | 'InformationInputKind'): 'InformationInputKind' + + + +histdatacom.synthetic.information.InformationScope + +InformationScope + +name + +from_value(value: str | 'InformationScope'): 'InformationScope' + + + +histdatacom.synthetic.information.InformationSplitKind + +InformationSplitKind + +name + +from_value(value: str | 'InformationSplitKind'): 'InformationSplitKind' + + + +histdatacom.synthetic.information.InformationStage + +InformationStage + +name + +from_value(value: str | 'InformationStage'): 'InformationStage' - + histdatacom.orchestration.control.JobControlAction - -JobControlAction - -name - - + +JobControlAction + +name + + - + histdatacom.orchestration.control.JobControlStates - -JobControlStates - -cancel -resume -retry - -__init__(self, cancel: ControlOperationState, retry: ControlOperationState, resume: ControlOperationState): None -for_status(status: WorkStatus, lifecycle: JobLifecycle): 'JobControlStates' -from_dict(data: Mapping[str, Any]): 'JobControlStates' -to_dict(): dict[str, JSONValue] -with_operation(operation: ControlOperationName, state: ControlOperationState): 'JobControlStates' + +JobControlStates + +cancel +resume +retry + +__init__(self, cancel: ControlOperationState, retry: ControlOperationState, resume: ControlOperationState): None +for_status(status: WorkStatus, lifecycle: JobLifecycle): 'JobControlStates' +from_dict(data: Mapping[str, Any]): 'JobControlStates' +to_dict(): dict[str, JSONValue] +with_operation(operation: ControlOperationName, state: ControlOperationState): 'JobControlStates' - + histdatacom.orchestration.control.JobControlStates->histdatacom.orchestration.control.ControlOperationState - - -cancel + + +cancel - + histdatacom.orchestration.control.JobControlStates->histdatacom.orchestration.control.ControlOperationState - - -retry + + +retry - + histdatacom.orchestration.control.JobControlStates->histdatacom.orchestration.control.ControlOperationState - - -resume + + +resume - + histdatacom.orchestration.control.JobLifecycle - -JobLifecycle - -name - - + +JobLifecycle + +name + + - + histdatacom.orchestration.control.JobLogEntry - -JobLogEntry - -level : str -message : str -metadata : dict[str, JSONValue] -source : str -timestamp_utc : str - -__init__(self, source: str, message: str, level: str, timestamp_utc: str, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'JobLogEntry' -to_dict(): dict[str, JSONValue] + +JobLogEntry + +level : str +message : str +metadata : dict[str, JSONValue] +source : str +timestamp_utc : str + +__init__(self, source: str, message: str, level: str, timestamp_utc: str, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'JobLogEntry' +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.control.JobProgressSnapshot - -JobProgressSnapshot - -artifacts : tuple[ArtifactRef, ...] -completed_children : int -completed_stages : tuple[str, ...] -current_stage : str -events : tuple[StatusEvent, ...] -last_error : str -percent_complete : float -planned_children : tuple[str, ...] -rate_per_second : float -request_id : str -started_at_utc : str -status -total_children : int -unit : str -updated_at_utc : str -workflow_name : str - -__init__(self, workflow_name: str, request_id: str, status: WorkStatus, current_stage: str, total_children: int, completed_children: int, unit: str, started_at_utc: str, updated_at_utc: str, rate_per_second: float, last_error: str, planned_children: tuple[str, ...], completed_stages: tuple[str, ...], events: tuple[StatusEvent, ...], artifacts: tuple[ArtifactRef, ...]): None -from_dict(data: Mapping[str, Any]): 'JobProgressSnapshot' -to_dict(): dict[str, JSONValue] + +JobProgressSnapshot + +artifacts : tuple[ArtifactRef, ...] +completed_children : int +completed_stages : tuple[str, ...] +current_stage : str +events : tuple[StatusEvent, ...] +last_error : str +percent_complete : float +planned_children : tuple[str, ...] +rate_per_second : float +request_id : str +started_at_utc : str +status +total_children : int +unit : str +updated_at_utc : str +workflow_name : str + +__init__(self, workflow_name: str, request_id: str, status: WorkStatus, current_stage: str, total_children: int, completed_children: int, unit: str, started_at_utc: str, updated_at_utc: str, rate_per_second: float, last_error: str, planned_children: tuple[str, ...], completed_stages: tuple[str, ...], events: tuple[StatusEvent, ...], artifacts: tuple[ArtifactRef, ...]): None +from_dict(data: Mapping[str, Any]): 'JobProgressSnapshot' +to_dict(): dict[str, JSONValue] - + histdatacom.runtime_contracts.WorkStatus - -WorkStatus - -name -terminal : bool - -from_value(value: str | 'WorkStatus' | None): 'WorkStatus' + +WorkStatus + +name +terminal : bool + +from_value(value: str | 'WorkStatus' | None): 'WorkStatus' - + histdatacom.orchestration.control.JobProgressSnapshot->histdatacom.runtime_contracts.WorkStatus - - -status + + +status - + histdatacom.data_quality.remediation_audit.KnownQualityFindingCode - -KnownQualityFindingCode - -finding_code : str -rule_id : str -severity -severity_source : str -source : str -source_family : str - -__init__(self, rule_id: str, finding_code: str, severity: QualitySeverity, source: str, severity_source: str, source_family: str): None + +KnownQualityFindingCode + +attribution_reason : str +attribution_status : str +finding_code : str +finding_code_prefix : str +rule_id : str +severity +severity_source : str +source : str +source_family : str +source_helper : str + +__init__(self, rule_id: str, finding_code: str, severity: QualitySeverity, source: str, severity_source: str, source_family: str, source_helper: str, finding_code_prefix: str, attribution_status: str, attribution_reason: str): None - + histdatacom.data_quality.remediation_audit.KnownQualityFindingCode->histdatacom.data_quality.contracts.QualitySeverity - - -severity + + +severity - + histdatacom.legacy_boundary.LegacyHelperSideEffectWarning - -LegacyHelperSideEffectWarning - - - + +LegacyHelperSideEffectWarning + + + - + histdatacom.records.LegacyManifestStatusAliasWarning - -LegacyManifestStatusAliasWarning - - - + +LegacyManifestStatusAliasWarning + + + + + + +histdatacom.broker_capture.adapters.LiveBrokerCaptureSourceV1 + +LiveBrokerCaptureSourceV1 + +adapter +clock +clock_correction_threshold_ns : int +session +session_id : str + +__init__(self, session: BrokerCaptureSessionV1, adapter: BrokerCaptureAdapterV1, clock: BrokerCaptureClockV1, clock_correction_threshold_ns: int): None +__post_init__(): None +_event_iterator(): Iterator[BrokerCaptureEventV1] +iter_events(): Iterable[BrokerCaptureEventV1] + + + +histdatacom.broker_capture.adapters.LiveBrokerCaptureSourceV1->histdatacom.broker_capture.adapters.BrokerCaptureAdapterV1 + + +adapter + + + +histdatacom.broker_capture.adapters.LiveBrokerCaptureSourceV1->histdatacom.broker_capture.adapters.BrokerCaptureClockV1 + + +clock + + + +histdatacom.broker_capture.adapters.LiveBrokerCaptureSourceV1->histdatacom.broker_capture.contracts.BrokerCaptureSessionV1 + + +session - + histdatacom.orchestration.rich_progress.LiveJobProgressRenderer - -LiveJobProgressRenderer - -_cached_health : dict[str, Any] | None -_health_provider : NoneType -_health_refresh_seconds -_last_health_refresh : float -_live : Live | None, NoneType -console : Console - -__enter__(): 'LiveJobProgressRenderer' -__exit__(): None -__init__(): None -_snapshot_with_health(snapshot: OrchestrationJobSnapshot): OrchestrationJobSnapshot -close(): None -update(snapshot: OrchestrationJobSnapshot): None + +LiveJobProgressRenderer + +_cached_health : dict[str, Any] | None +_health_provider : NoneType +_health_refresh_seconds +_last_health_refresh : float +_live : Live | None, NoneType +console : Console + +__enter__(): 'LiveJobProgressRenderer' +__exit__(): None +__init__(): None +_snapshot_with_health(snapshot: OrchestrationJobSnapshot): OrchestrationJobSnapshot +close(): None +update(snapshot: OrchestrationJobSnapshot): None - + histdatacom.orchestration.live_smoke.LiveOrchestrationSmokeError - -LiveOrchestrationSmokeError - -diagnostics : dict - -__init__(message: str, diagnostics: Mapping[str, Any]): None + +LiveOrchestrationSmokeError + +diagnostics : dict + +__init__(message: str, diagnostics: Mapping[str, Any]): None - + histdatacom.orchestration.live_smoke.LiveOrchestrationSmokeResult - -LiveOrchestrationSmokeResult - -client_routing : str -diagnostics : Mapping[str, Any] -doctor : Mapping[str, Any] -request -snapshot -started_status -stopped_status : OrchestrationStatus | None -worker_config - -__init__(self, request: RunRequest, worker_config: OrchestrationWorkerConfig, started_status: OrchestrationStatus, snapshot: OrchestrationJobSnapshot, doctor: Mapping[str, Any], stopped_status: OrchestrationStatus | None, client_routing: str, diagnostics: Mapping[str, Any]): None -to_dict(): dict[str, Any] + +LiveOrchestrationSmokeResult + +client_routing : str +diagnostics : Mapping[str, Any] +doctor : Mapping[str, Any] +request +snapshot +started_status +stopped_status : OrchestrationStatus | None +worker_config + +__init__(self, request: RunRequest, worker_config: OrchestrationWorkerConfig, started_status: OrchestrationStatus, snapshot: OrchestrationJobSnapshot, doctor: Mapping[str, Any], stopped_status: OrchestrationStatus | None, client_routing: str, diagnostics: Mapping[str, Any]): None +to_dict(): dict[str, Any] - + histdatacom.orchestration.control.OrchestrationJobSnapshot - -OrchestrationJobSnapshot - -artifacts : tuple[ArtifactRef, ...] -controls -job_id : str -lifecycle -logs : tuple[JobLogEntry, ...] -metadata : dict[str, JSONValue] -namespace : str -orchestration_message : str -orchestration_state : str -progress : JobProgressSnapshot | None -request_id : str -result : Optional[Any] -run_id : str -schema_version : int -status -task_queue : str -updated_at_utc : str -workflow_id : str - -__init__(self, job_id: str, request_id: str, workflow_id: str, run_id: str, namespace: str, task_queue: str, lifecycle: JobLifecycle, status: WorkStatus, progress: JobProgressSnapshot | None, controls: JobControlStates, logs: tuple[JobLogEntry, ...], artifacts: tuple[ArtifactRef, ...], result: Any, orchestration_state: str, orchestration_message: str, updated_at_utc: str, metadata: dict[str, JSONValue], schema_version: int): None -from_dict(data: Mapping[str, Any]): 'OrchestrationJobSnapshot' -from_handle(handle: Any): 'OrchestrationJobSnapshot' -from_workflow_status(handle: Any, status_payload: Mapping[str, Any]): 'OrchestrationJobSnapshot' -mark_cancelled(): 'OrchestrationJobSnapshot' -mark_resuming(): 'OrchestrationJobSnapshot' -mark_retrying(): 'OrchestrationJobSnapshot' -request_cancel(): 'OrchestrationJobSnapshot' -request_resume(): 'OrchestrationJobSnapshot' -request_retry(): 'OrchestrationJobSnapshot' -to_dict(): dict[str, Any] -with_orchestration_status(orchestration_status: Any): 'OrchestrationJobSnapshot' -with_progress(progress: JobProgressSnapshot): 'OrchestrationJobSnapshot' -with_result(result: Any): 'OrchestrationJobSnapshot' + +OrchestrationJobSnapshot + +artifacts : tuple[ArtifactRef, ...] +controls +job_id : str +lifecycle +logs : tuple[JobLogEntry, ...] +metadata : dict[str, JSONValue] +namespace : str +orchestration_message : str +orchestration_state : str +progress : JobProgressSnapshot | None +request_id : str +result : Optional[Any] +run_id : str +schema_version : int +status +task_queue : str +updated_at_utc : str +workflow_id : str + +__init__(self, job_id: str, request_id: str, workflow_id: str, run_id: str, namespace: str, task_queue: str, lifecycle: JobLifecycle, status: WorkStatus, progress: JobProgressSnapshot | None, controls: JobControlStates, logs: tuple[JobLogEntry, ...], artifacts: tuple[ArtifactRef, ...], result: Any, orchestration_state: str, orchestration_message: str, updated_at_utc: str, metadata: dict[str, JSONValue], schema_version: int): None +from_dict(data: Mapping[str, Any]): 'OrchestrationJobSnapshot' +from_handle(handle: Any): 'OrchestrationJobSnapshot' +from_workflow_status(handle: Any, status_payload: Mapping[str, Any]): 'OrchestrationJobSnapshot' +mark_cancelled(): 'OrchestrationJobSnapshot' +mark_resuming(): 'OrchestrationJobSnapshot' +mark_retrying(): 'OrchestrationJobSnapshot' +request_cancel(): 'OrchestrationJobSnapshot' +request_resume(): 'OrchestrationJobSnapshot' +request_retry(): 'OrchestrationJobSnapshot' +to_dict(): dict[str, Any] +with_orchestration_status(orchestration_status: Any): 'OrchestrationJobSnapshot' +with_progress(progress: JobProgressSnapshot): 'OrchestrationJobSnapshot' +with_result(result: Any): 'OrchestrationJobSnapshot' - + histdatacom.orchestration.live_smoke.LiveOrchestrationSmokeResult->histdatacom.orchestration.control.OrchestrationJobSnapshot - - -snapshot + + +snapshot - + histdatacom.orchestration.supervisor.OrchestrationStatus - -OrchestrationStatus - -command : tuple[str, ...] -components : dict[str, str] -disk : dict[str, Any] -lock_file : str -logs : dict[str, str] -message : str -pid_file : str -pids : dict[str, int] -ports : dict[str, int | str | list[int]] -running : bool -state : str -state_dir : str -worker_readiness : dict[str, dict[str, Any]] - -__init__(self, state: str, message: str, state_dir: str, pid_file: str, lock_file: str, logs: dict[str, str], pids: dict[str, int], command: tuple[str, ...], ports: dict[str, int | str | list[int]], components: dict[str, str], worker_readiness: dict[str, dict[str, Any]], disk: dict[str, Any]): None -to_dict(): dict[str, Any] + +OrchestrationStatus + +command : tuple[str, ...] +components : dict[str, str] +disk : dict[str, Any] +lock_file : str +logs : dict[str, str] +message : str +pid_file : str +pids : dict[str, int] +ports : dict[str, int | str | list[int]] +running : bool +state : str +state_dir : str +worker_readiness : dict[str, dict[str, Any]] + +__init__(self, state: str, message: str, state_dir: str, pid_file: str, lock_file: str, logs: dict[str, str], pids: dict[str, int], command: tuple[str, ...], ports: dict[str, int | str | list[int]], components: dict[str, str], worker_readiness: dict[str, dict[str, Any]], disk: dict[str, Any]): None +to_dict(): dict[str, Any] - + histdatacom.orchestration.live_smoke.LiveOrchestrationSmokeResult->histdatacom.orchestration.supervisor.OrchestrationStatus - - -started_status + + +started_status - + histdatacom.orchestration.queues.OrchestrationWorkerConfig - -OrchestrationWorkerConfig - -concurrency : OrchestrationConcurrencyProfile | None -concurrency_profile : OrchestrationConcurrencyProfile -lane -namespace : str -runtime_policy -target_host : str -task_queue : str -task_queues -worker_options : dict[str, int] - -__init__(self, runtime_policy: OrchestrationRuntimePolicy, namespace: str, task_queues: OrchestrationTaskQueues, lane: TaskQueueLane, concurrency: OrchestrationConcurrencyProfile | None): None -for_lane(lane: str | TaskQueueLane): 'OrchestrationWorkerConfig' -to_dict(): dict[str, Any] + +OrchestrationWorkerConfig + +concurrency : OrchestrationConcurrencyProfile | None +concurrency_profile : OrchestrationConcurrencyProfile +lane +namespace : str +runtime_policy +target_host : str +task_queue : str +task_queues +worker_options : dict[str, int] + +__init__(self, runtime_policy: OrchestrationRuntimePolicy, namespace: str, task_queues: OrchestrationTaskQueues, lane: TaskQueueLane, concurrency: OrchestrationConcurrencyProfile | None): None +for_lane(lane: str | TaskQueueLane): 'OrchestrationWorkerConfig' +to_dict(): dict[str, Any] - + histdatacom.orchestration.live_smoke.LiveOrchestrationSmokeResult->histdatacom.orchestration.queues.OrchestrationWorkerConfig - - -worker_config + + +worker_config - + histdatacom.runtime_contracts.RunRequest - -RunRequest - -api_return_type : str -available_remote_data : bool -batch_size : str -build_cache : bool -cpu_utilization : str -data_directory : str -data_quality : bool -delete_after_influx : bool -download_data_archives : bool -end_yearmonth : str -extract_csvs : bool -formats : tuple[str, ...] -import_to_influxdb : bool -metadata : dict[str, JSONValue] -pairs : tuple[str, ...] -quality_check_groups : tuple[str, ...] -quality_fail_on : str -quality_max_errors : int -quality_max_warnings : int -quality_paths : tuple[str, ...] -quality_profile : dict[str, JSONValue] -quality_profile_path : str -quality_report_path : str -repo_quality_columns : bool -repo_quality_refresh : bool -request_id : str -start_yearmonth : str -timeframes : tuple[str, ...] -update_remote_data : bool -validate_urls : bool -verbosity : int -zip_persist : bool - -__init__(self, request_id: str, pairs: tuple[str, ...], formats: tuple[str, ...], timeframes: tuple[str, ...], start_yearmonth: str, end_yearmonth: str, data_directory: str, api_return_type: str, cpu_utilization: str, batch_size: str, available_remote_data: bool, update_remote_data: bool, validate_urls: bool, download_data_archives: bool, extract_csvs: bool, build_cache: bool, import_to_influxdb: bool, delete_after_influx: bool, zip_persist: bool, data_quality: bool, quality_paths: tuple[str, ...], quality_check_groups: tuple[str, ...], quality_report_path: str, quality_fail_on: str, quality_max_errors: int, quality_max_warnings: int, quality_profile_path: str, quality_profile: dict[str, JSONValue], repo_quality_refresh: bool, repo_quality_columns: bool, verbosity: int, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'RunRequest' -from_options(options: Any, request_id: str): 'RunRequest' -to_dict(): dict[str, JSONValue] + +RunRequest + +api_return_type : str +available_remote_data : bool +batch_size : str +build_cache : bool +cpu_utilization : str +data_directory : str +data_quality : bool +delete_after_influx : bool +download_data_archives : bool +end_yearmonth : str +extract_csvs : bool +formats : tuple[str, ...] +import_to_influxdb : bool +metadata : dict[str, JSONValue] +pairs : tuple[str, ...] +quality_check_groups : tuple[str, ...] +quality_fail_on : str +quality_max_errors : int +quality_max_warnings : int +quality_paths : tuple[str, ...] +quality_profile : dict[str, JSONValue] +quality_profile_path : str +quality_report_path : str +random_seed : int | None +random_window : str +repo_quality_columns : bool +repo_quality_refresh : bool +request_id : str +start_yearmonth : str +timeframes : tuple[str, ...] +update_remote_data : bool +validate_urls : bool +verbosity : int +zip_persist : bool + +__init__(self, request_id: str, pairs: tuple[str, ...], formats: tuple[str, ...], timeframes: tuple[str, ...], start_yearmonth: str, end_yearmonth: str, random_window: str, random_seed: int | None, data_directory: str, api_return_type: str, cpu_utilization: str, batch_size: str, available_remote_data: bool, update_remote_data: bool, validate_urls: bool, download_data_archives: bool, extract_csvs: bool, build_cache: bool, import_to_influxdb: bool, delete_after_influx: bool, zip_persist: bool, data_quality: bool, quality_paths: tuple[str, ...], quality_check_groups: tuple[str, ...], quality_report_path: str, quality_fail_on: str, quality_max_errors: int, quality_max_warnings: int, quality_profile_path: str, quality_profile: dict[str, JSONValue], repo_quality_refresh: bool, repo_quality_columns: bool, verbosity: int, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'RunRequest' +from_options(options: Any, request_id: str): 'RunRequest' +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.live_smoke.LiveOrchestrationSmokeResult->histdatacom.runtime_contracts.RunRequest - - -request + + +request - + histdatacom.orchestration.live_smoke.LiveOrchestrationStopError - -LiveOrchestrationStopError - -status : OrchestrationStatus | None - -__init__(message: str, status: OrchestrationStatus | None): None + +LiveOrchestrationStopError + +status : OrchestrationStatus | None + +__init__(message: str, status: OrchestrationStatus | None): None - + histdatacom.orchestration.maintenance.LogMaintenanceResult - -LogMaintenanceResult - -action : str -error : str -exists : bool -path : str -reason : str -rotations_removed : int -size_after_bytes : int -size_before_bytes : int - -__init__(self, path: str, exists: bool, action: str, size_before_bytes: int, size_after_bytes: int, rotations_removed: int, reason: str, error: str): None -to_dict(): dict[str, Any] + +LogMaintenanceResult + +action : str +error : str +exists : bool +path : str +reason : str +rotations_removed : int +size_after_bytes : int +size_before_bytes : int + +__init__(self, path: str, exists: bool, action: str, size_before_bytes: int, size_after_bytes: int, rotations_removed: int, reason: str, error: str): None +to_dict(): dict[str, Any] - + histdatacom.manifest_store.ManifestMigrationResult - -ManifestMigrationResult - -error : str -migrated : bool -path : str -reason : str -work_id : str - -__init__(self, path: str, migrated: bool, reason: str, work_id: str, error: str): None + +ManifestMigrationResult + +error : str +migrated : bool +path : str +reason : str +work_id : str + +__init__(self, path: str, migrated: bool, reason: str, work_id: str, error: str): None - + histdatacom.manifest_store.ManifestStatusStore - -ManifestStatusStore - -db_path -root_dir : Path - -__init__(root_dir: str | Path) -_connect(): sqlite3.Connection -_dataset_plan_work_ids(plan_id: str): tuple[str, ...] -_ensure_schema(): None -_insert_artifact(conn: sqlite3.Connection): None -_insert_status_event(conn: sqlite3.Connection): None -dataset_plan_ref(plan_id: str): dict[str, JSONValue] -delete_record(record: Any): None -existing_for_record(record: Any): 'ManifestStatusStore | None' -get_dataset_plan(plan_id: str): dict[str, Any] | None -get_dataset_plan_work_items(plan_id: str): tuple[WorkItem, ...] -get_job_snapshot(job_id: str): dict[str, Any] | None -get_work_item(work_id: str): WorkItem | None -get_work_item_for_record(record: Any): WorkItem | None -import_legacy_meta_files(root_dir: str | Path | None): tuple[ManifestMigrationResult, ...] -import_meta_file(meta_path: str | Path): ManifestMigrationResult -inspect_schema(root_dir: str | Path): dict[str, Any] -list_artifacts(owner_id: str): tuple[dict[str, JSONValue], ...] -list_job_snapshots(): tuple[dict[str, Any], ...] -list_stage_results(work_id: str): tuple[dict[str, Any], ...] -list_work_items(): tuple[WorkItem, ...] -path_for_root(root_dir: str | Path): Path -prune_retention(): dict[str, int] -schema_status(): dict[str, Any] -status_history(owner_id: str): tuple[dict[str, Any], ...] -status_store_ref(): dict[str, JSONValue] -write_dataset_plan(): dict[str, JSONValue] -write_job_snapshot(snapshot: Any): None -write_live_stage_update(): dict[str, Any] -write_record(record: Any): WorkItem -write_stage_result(result: StageResult): None -write_work_item(work_item: WorkItem): None + +ManifestStatusStore + +db_path +root_dir : Path + +__init__(root_dir: str | Path) +_connect(): sqlite3.Connection +_dataset_plan_work_ids(plan_id: str): tuple[str, ...] +_ensure_schema(): None +_insert_artifact(conn: sqlite3.Connection): None +_insert_status_event(conn: sqlite3.Connection): None +compare_and_swap_job_snapshot(snapshot: Any): bool +dataset_plan_ref(plan_id: str): dict[str, JSONValue] +delete_record(record: Any): None +existing_for_record(record: Any): 'ManifestStatusStore | None' +get_dataset_plan(plan_id: str): dict[str, Any] | None +get_dataset_plan_work_items(plan_id: str): tuple[WorkItem, ...] +get_job_snapshot(job_id: str): dict[str, Any] | None +get_work_item(work_id: str): WorkItem | None +get_work_item_for_record(record: Any): WorkItem | None +import_legacy_meta_files(root_dir: str | Path | None): tuple[ManifestMigrationResult, ...] +import_meta_file(meta_path: str | Path): ManifestMigrationResult +inspect_schema(root_dir: str | Path): dict[str, Any] +list_artifacts(owner_id: str): tuple[dict[str, JSONValue], ...] +list_job_snapshots(): tuple[dict[str, Any], ...] +list_stage_results(work_id: str): tuple[dict[str, Any], ...] +list_work_items(): tuple[WorkItem, ...] +path_for_root(root_dir: str | Path): Path +prune_retention(): dict[str, int] +schema_status(): dict[str, Any] +status_history(owner_id: str): tuple[dict[str, Any], ...] +status_store_ref(): dict[str, JSONValue] +write_dataset_plan(): dict[str, JSONValue] +write_job_snapshot(snapshot: Any): None +write_live_stage_update(): dict[str, Any] +write_record(record: Any): WorkItem +write_stage_result(result: StageResult): None +write_work_item(work_item: WorkItem): None + + + +histdatacom.orchestration.reconstruction.ReconstructionCheckpointStore + +ReconstructionCheckpointStore + +store + +__init__(root: str | Path) +_snapshot(state: ReconstructionWindowStateV1): dict[str, JSONValue] +_validate_task(state: ReconstructionWindowStateV1, request_id: str, task: ReconstructionWindowTaskV1): None +initialize(request_id: str, task: ReconstructionWindowTaskV1): ReconstructionWindowStateV1 +job_id_for(window: ReconstructionWindowV1): str +load(window: ReconstructionWindowV1): ReconstructionWindowStateV1 | None +save(state: ReconstructionWindowStateV1): ReconstructionWindowStateV1 + + + +histdatacom.manifest_store.ManifestStatusStore->histdatacom.orchestration.reconstruction.ReconstructionCheckpointStore + + +store + + + +histdatacom.market_context.contracts.MarketContextCalendarStateV1 + +MarketContextCalendarStateV1 + +active_sessions : tuple[str, ...] +calendar_tags : tuple[str, ...] +clock_sessions : tuple[str, ...] +event_tags : tuple[str, ...] +holiday_tags : tuple[str, ...] +limitations : tuple[str, ...] +overlaps : tuple[str, ...] +profile_complete : bool +profile_source : str +profile_version : str +schema_version : str +session_state : str +special_tags : tuple[str, ...] +state_id : str +timestamp_utc_ns : int + +__init__(self, timestamp_utc_ns: int, session_state: str, clock_sessions: tuple[str, ...], active_sessions: tuple[str, ...], overlaps: tuple[str, ...], special_tags: tuple[str, ...], holiday_tags: tuple[str, ...], event_tags: tuple[str, ...], calendar_tags: tuple[str, ...], profile_source: str, profile_version: str, profile_complete: bool, limitations: tuple[str, ...], state_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextCalendarStateV1' +from_json(text: str): 'MarketContextCalendarStateV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.market_context.corpus.MarketContextCorpusBuildV1 + +MarketContextCorpusBuildV1 + +corpus +snapshots : tuple[MarketContextSourceSnapshotV1, ...] + +__init__(self, corpus: MarketContextCorpusV1, snapshots: tuple[MarketContextSourceSnapshotV1, ...]): None +__post_init__(): None + + + +histdatacom.market_context.corpus.MarketContextCorpusV1 + +MarketContextCorpusV1 + +corpus_id : str +counts_by_year_currency_kind : Mapping[str, int] +coverage : tuple[MarketContextCoverageSliceV1, ...] +duplicate_event_count : int +limitations : tuple[str, ...] +peak_memory_bytes : int +profile +runtime_seconds : float +schema_version : str +sources : tuple[MarketContextSourceEvidenceV1, ...] +timeline + +__init__(self, profile: MarketContextFetchProfileV1, timeline: MarketContextTimelineV1, sources: tuple[MarketContextSourceEvidenceV1, ...], coverage: tuple[MarketContextCoverageSliceV1, ...], duplicate_event_count: int, counts_by_year_currency_kind: Mapping[str, int], runtime_seconds: float, peak_memory_bytes: int, limitations: tuple[str, ...], corpus_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextCorpusV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.corpus.MarketContextCorpusBuildV1->histdatacom.market_context.corpus.MarketContextCorpusV1 + + +corpus + + + +histdatacom.market_context.corpus.MarketContextCorpusPreflightError + +MarketContextCorpusPreflightError + +decision + +__init__(decision: MarketContextCorpusPreflightV1): None + + + +histdatacom.market_context.corpus.MarketContextCorpusPreflightV1 + +MarketContextCorpusPreflightV1 + +corpus_id : str +currencies : tuple[str, ...] +end_ns : int +kinds : tuple[MarketContextKind, ...] +matched_coverage : tuple[str, ...] +ready : bool +reasons : tuple[str, ...] +schema_version : str +start_ns : int + +__init__(self, corpus_id: str, start_ns: int, end_ns: int, currencies: tuple[str, ...], kinds: tuple[MarketContextKind, ...], ready: bool, reasons: tuple[str, ...], matched_coverage: tuple[str, ...], schema_version: str): None +__post_init__(): None +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.corpus.MarketContextCorpusPreflightV1->histdatacom.market_context.corpus.MarketContextCorpusPreflightError + + +decision + + + +histdatacom.market_context.corpus.MarketContextFetchProfileV1 + +MarketContextFetchProfileV1 + +coverage_end_ns : int +coverage_start_ns : int +end_date : str +max_events : int +max_ons_pages_per_query : int +max_response_bytes : int +max_runtime_seconds : float +max_total_source_bytes : int +ons_queries : tuple[str, ...] +sources : tuple[str, ...] +start_date : str +timeout_seconds : float + +__init__(self, start_date: str, end_date: str, sources: tuple[str, ...], ons_queries: tuple[str, ...], timeout_seconds: float, max_response_bytes: int, max_total_source_bytes: int, max_ons_pages_per_query: int, max_events: int, max_runtime_seconds: float): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextFetchProfileV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.corpus.MarketContextCorpusV1->histdatacom.market_context.corpus.MarketContextFetchProfileV1 + + +profile + + + +histdatacom.market_context.contracts.MarketContextTimelineV1 + +MarketContextTimelineV1 + +complete : bool +coverage_end_ns : int +coverage_start_ns : int +events : tuple[MarketContextEventV1, ...] +limitations : tuple[str, ...] +schema_version : str +timeline_id : str +timeline_version : str + +__init__(self, timeline_version: str, coverage_start_ns: int, coverage_end_ns: int, complete: bool, events: tuple[MarketContextEventV1, ...], limitations: tuple[str, ...], timeline_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextTimelineV1' +from_json(text: str): 'MarketContextTimelineV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.market_context.corpus.MarketContextCorpusV1->histdatacom.market_context.contracts.MarketContextTimelineV1 + + +timeline + + + +histdatacom.market_context.corpus.MarketContextCoverageSliceV1 + +MarketContextCoverageSliceV1 + +complete : bool +coverage_end_ns : int +coverage_start_ns : int +currency : str +event_count : int +kind +missingness_reason : str +schema_version : str +source_family : str + +__init__(self, source_family: str, currency: str, kind: MarketContextKind, coverage_start_ns: int, coverage_end_ns: int, complete: bool, event_count: int, missingness_reason: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextCoverageSliceV1' +supports(start_ns: int, end_ns: int): bool +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.contracts.MarketContextKind + +MarketContextKind + +name + +from_value(value: str | 'MarketContextKind'): 'MarketContextKind' + + + +histdatacom.market_context.corpus.MarketContextCoverageSliceV1->histdatacom.market_context.contracts.MarketContextKind + + +kind + + + +histdatacom.market_context.contracts.MarketContextEventV1 + +MarketContextEventV1 + +actual_value : float | None +affected_currencies : tuple[str, ...] +affected_symbols : tuple[str, ...] +ambiguity_reason : str | None +available_at_ns : int +canonical_key : str +confidence : float +content_sha256 : str | None +event_id : str +event_time_ns : int +expected_value : float | None +first_known_at_ns : int +kind +limitations : tuple[str, ...] +post_event_ns : int +pre_event_ns : int +precision +previous_value : float | None +revised_previous_value : float | None +revision_sequence : int +schema_version : str +source +source_event_time : str +source_time_fold : int | None +source_timezone : str +supersedes_event_id : str | None +surprise_value : float | None +tags : tuple[str, ...] +title : str +value_unit : str | None +vintage_id : str +window_end_ns : int +window_start_ns : int + +__init__(self, canonical_key: str, kind: MarketContextKind, title: str, source: MarketContextSourceV1, source_event_time: str, source_timezone: str, event_time_ns: int, first_known_at_ns: int, available_at_ns: int, pre_event_ns: int, post_event_ns: int, affected_currencies: tuple[str, ...], affected_symbols: tuple[str, ...], confidence: float, precision: MarketContextPrecision, limitations: tuple[str, ...], vintage_id: str, revision_sequence: int, supersedes_event_id: str | None, source_time_fold: int | None, ambiguity_reason: str | None, expected_value: float | None, actual_value: float | None, previous_value: float | None, revised_previous_value: float | None, surprise_value: float | None, value_unit: str | None, content_sha256: str | None, tags: tuple[str, ...], event_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextEventV1' +from_json(text: str): 'MarketContextEventV1' +identity_payload(): dict[str, JSONValue] +overlaps(start_ns: int, end_ns: int): bool +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.market_context.contracts.MarketContextEventV1->histdatacom.market_context.contracts.MarketContextKind + + +kind + + + +histdatacom.market_context.contracts.MarketContextPrecision + +MarketContextPrecision + +name + +from_value(value: str | 'MarketContextPrecision'): 'MarketContextPrecision' + + + +histdatacom.market_context.contracts.MarketContextEventV1->histdatacom.market_context.contracts.MarketContextPrecision + + +precision + + + +histdatacom.market_context.contracts.MarketContextSourceV1 + +MarketContextSourceV1 + +adapter_name : str +adapter_version : str +content_sha256 : str +license_name : str +limitations : tuple[str, ...] +metadata : Mapping[str, JSONValue] | None +name : str +redistribution_allowed : bool +redistribution_constraints : tuple[str, ...] +retrieved_at_ns : int +schema_version : str +source_id : str +source_uri : str | None +source_version : str + +__init__(self, name: str, source_version: str, retrieved_at_ns: int, content_sha256: str, adapter_name: str, adapter_version: str, license_name: str, redistribution_allowed: bool, redistribution_constraints: tuple[str, ...], limitations: tuple[str, ...], source_uri: str | None, metadata: Mapping[str, JSONValue] | None, source_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextSourceV1' +from_json(text: str): 'MarketContextSourceV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.market_context.contracts.MarketContextEventV1->histdatacom.market_context.contracts.MarketContextSourceV1 + + +source + + + +histdatacom.market_context.contracts.MarketContextMissingReason + +MarketContextMissingReason + +name + +from_value(value: str | 'MarketContextMissingReason'): 'MarketContextMissingReason' + + + +histdatacom.market_context.contracts.MarketContextQueryLimitError + +MarketContextQueryLimitError + + + + + + +histdatacom.market_context.contracts.MarketContextQueryStatus + +MarketContextQueryStatus + +name + +from_value(value: str | 'MarketContextQueryStatus'): 'MarketContextQueryStatus' + + + +histdatacom.market_context.contracts.MarketContextQueryV1 + +MarketContextQueryV1 + +as_of_ns : int | None +calendar_state : MarketContextCalendarStateV1 | None +end_ns : int +events : tuple[MarketContextEventV1, ...] +information_mode : InformationMode +limitations : tuple[str, ...] +missing_reason : MarketContextMissingReason | None +query_id : str +requested_currencies : tuple[str, ...] +requested_kinds : tuple[MarketContextKind, ...] +requested_symbols : tuple[str, ...] +schema_version : str +start_ns : int +status +timeline_id : str +view +window_id : str | None + +__init__(self, timeline_id: str, view: MarketContextView, start_ns: int, end_ns: int, as_of_ns: int | None, events: tuple[MarketContextEventV1, ...], status: MarketContextQueryStatus, missing_reason: MarketContextMissingReason | None, calendar_state: MarketContextCalendarStateV1 | None, requested_currencies: tuple[str, ...], requested_symbols: tuple[str, ...], requested_kinds: tuple[MarketContextKind, ...], window_id: str | None, limitations: tuple[str, ...], query_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextQueryV1' +from_json(text: str): 'MarketContextQueryV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.market_context.contracts.MarketContextQueryV1->histdatacom.market_context.contracts.MarketContextQueryStatus + + +status + + + +histdatacom.market_context.contracts.MarketContextView + +MarketContextView + +name + +from_value(value: str | 'MarketContextView'): 'MarketContextView' + + + +histdatacom.market_context.contracts.MarketContextQueryV1->histdatacom.market_context.contracts.MarketContextView + + +view + + + +histdatacom.market_context.contracts.MarketContextSourceAdapterV1 + +MarketContextSourceAdapterV1 + +adapter_name : str +adapter_version : str + +load_events +(): Iterable[MarketContextEventV1] + + + +histdatacom.market_context.corpus.MarketContextSourceEvidenceV1 + +MarketContextSourceEvidenceV1 + +adapter_name : str +adapter_version : str +content_sha256 : str +content_type : str +diagnostics : tuple[str, ...] +emitted_event_count : int +license_name : str +limitations : tuple[str, ...] +metadata : Mapping[str, JSONValue] +redistribution_allowed : bool +redistribution_constraints : tuple[str, ...] +retrieved_at_ns : int +schema_version : str +size_bytes : int +source_key : str +source_name : str +source_uri : str + +__init__(self, source_key: str, source_name: str, source_uri: str, retrieved_at_ns: int, content_sha256: str, size_bytes: int, content_type: str, adapter_name: str, adapter_version: str, license_name: str, redistribution_allowed: bool, redistribution_constraints: tuple[str, ...], limitations: tuple[str, ...], metadata: Mapping[str, JSONValue], emitted_event_count: int, diagnostics: tuple[str, ...], schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'MarketContextSourceEvidenceV1' +from_snapshot(snapshot: MarketContextSourceSnapshotV1): 'MarketContextSourceEvidenceV1' +restore_snapshot(content: bytes): MarketContextSourceSnapshotV1 +source_contract(): MarketContextSourceV1 +to_dict(): dict[str, JSONValue] + + + +histdatacom.market_context.corpus.MarketContextSourceSnapshotV1 + +MarketContextSourceSnapshotV1 + +adapter_name : str +adapter_version : str +content : bytes +content_sha256 : str +content_type : str +license_name : str +limitations : tuple[str, ...] +metadata : Mapping[str, JSONValue] +redistribution_allowed : bool +redistribution_constraints : tuple[str, ...] +retrieved_at_ns : int +source_key : str +source_name : str +source_uri : str + +__init__(self, source_key: str, source_name: str, source_uri: str, retrieved_at_ns: int, content: bytes, content_type: str, adapter_name: str, adapter_version: str, license_name: str, redistribution_allowed: bool, redistribution_constraints: tuple[str, ...], limitations: tuple[str, ...], metadata: Mapping[str, JSONValue]): None +__post_init__(): None +source_contract(): MarketContextSourceV1 + + + +histdatacom.market_context.corpus.MarketContextSourceSnapshotV1->histdatacom.market_context.corpus.BankOfEnglandBankRateAdapterV1 + + +snapshot + + + +histdatacom.market_context.corpus.MarketContextSourceSnapshotV1->histdatacom.market_context.corpus.EcbPolicyRateAdapterV1 + + +snapshot + + + +histdatacom.market_context.corpus.MarketContextSourceSnapshotV1->histdatacom.market_context.corpus.FederalReserveFomcCalendarAdapterV1 + + +snapshot + + + +histdatacom.market_context.corpus.MarketContextSourceSnapshotV1->histdatacom.market_context.corpus.FederalReserveFomcHistoricalAdapterV1 + + +snapshot + + + +histdatacom.market_context.corpus.OnsReleaseCalendarAdapterV1 + +OnsReleaseCalendarAdapterV1 + +adapter_name : str +adapter_version : str +diagnostics : tuple[str, ...] +snapshot + +__init__(snapshot: MarketContextSourceSnapshotV1): None +load_events(): Iterable[MarketContextEventV1] + + + +histdatacom.market_context.corpus.MarketContextSourceSnapshotV1->histdatacom.market_context.corpus.OnsReleaseCalendarAdapterV1 + + +snapshot + + + +histdatacom.market_context.corpus.OperatorMarketContextCatalogAdapterV1 + +OperatorMarketContextCatalogAdapterV1 + +adapter_name : str +adapter_version : str +diagnostics : tuple[str, ...] +snapshot + +__init__(snapshot: MarketContextSourceSnapshotV1): None +load_events(): Iterable[MarketContextEventV1] + + + +histdatacom.market_context.corpus.MarketContextSourceSnapshotV1->histdatacom.market_context.corpus.OperatorMarketContextCatalogAdapterV1 + + +snapshot - + histdatacom.orchestration.workflows.MergeCacheWorkflow - -MergeCacheWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +MergeCacheWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows.MergeCacheWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + - + histdatacom.activity_stages.MergeStageOutput - -MergeStageOutput - -data : Any -merge_sets : tuple['CacheMergeSetSummary', ...] -result - -__init__(self, data: Any, result: StageResult, merge_sets: tuple['CacheMergeSetSummary', ...]): None -to_dict(): dict[str, Any] + +MergeStageOutput + +data : Any +merge_sets : tuple['CacheMergeSetSummary', ...] +result + +__init__(self, data: Any, result: StageResult, merge_sets: tuple['CacheMergeSetSummary', ...]): None +to_dict(): dict[str, Any] - + histdatacom.activity_stages.MergeStageOutput->histdatacom.runtime_contracts.StageResult - - -result + + +result + + + +histdatacom.synthetic.certification_campaign.ModernReferenceCertificationCampaignResultV1 + +ModernReferenceCertificationCampaignResultV1 + +campaign_id : str +dossier_id : str +dossier_json +dossier_markdown +observation_count : int +policy_id : str +schema_version : str +state +verified_input_count : int + +__init__(self, campaign_id: str, policy_id: str, dossier_id: str, state: CertificationState, dossier_json: ArtifactRef, dossier_markdown: ArtifactRef, verified_input_count: int, observation_count: int, schema_version: str): None +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification_campaign.ModernReferenceCertificationCampaignResultV1->histdatacom.runtime_contracts.ArtifactRef + + +dossier_json + + + +histdatacom.synthetic.certification_campaign.ModernReferenceCertificationCampaignResultV1->histdatacom.runtime_contracts.ArtifactRef + + +dossier_markdown + + + +histdatacom.synthetic.certification_campaign.ModernReferenceCertificationCampaignResultV1->histdatacom.synthetic.certification.CertificationState + + +state + + + +histdatacom.synthetic.certification_campaign.ModernReferenceCertificationCampaignSpecV1 + +ModernReferenceCertificationCampaignSpecV1 + +accepted_limitations : tuple[str, ...] +artifacts : tuple[CertificationCampaignArtifactV1, ...] +blocking_limitations : tuple[str, ...] +campaign_id : str +candidate_amplification_budget : float +common_end_period : str +methodology : str +observations : tuple[CertificationCampaignObservationV1, ...] +peak_memory_budget_bytes : int +promotion_boundary : bool +runtime_budget_seconds : float +schema_version : str +scratch_budget_bytes : int +storage_budget_bytes : int + +__init__(self, common_end_period: str, peak_memory_budget_bytes: int, scratch_budget_bytes: int, runtime_budget_seconds: float, storage_budget_bytes: int, candidate_amplification_budget: float, artifacts: tuple[CertificationCampaignArtifactV1, ...], observations: tuple[CertificationCampaignObservationV1, ...], methodology: str, accepted_limitations: tuple[str, ...], blocking_limitations: tuple[str, ...], promotion_boundary: bool, campaign_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ModernReferenceCertificationCampaignSpecV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.motif_library.ModernReferenceMotifBuildV1 + +ModernReferenceMotifBuildV1 + +coverage : Mapping[str, Any] +index +leakage_audit : Mapping[str, Any] +library_id : str +manifest : Mapping[str, Any] +profile +qualification : Mapping[str, Any] +resource_audit : Mapping[str, Any] + +__init__(self, profile: ModernReferenceMotifProfileV1, index: ReferenceMotifIndexV1, library_id: str, manifest: Mapping[str, Any], leakage_audit: Mapping[str, Any], coverage: Mapping[str, Any], qualification: Mapping[str, Any], resource_audit: Mapping[str, Any]): None + + + +histdatacom.synthetic.motif_library.ModernReferenceMotifProfileV1 + +ModernReferenceMotifProfileV1 + +fragment_event_count : int +max_artifact_bytes : int +max_events_per_symbol : int +max_fragments : int +max_matches : int +max_peak_memory_bytes : int +max_runtime_seconds : float +max_source_bytes : int +minimum_events_per_symbol : int +neighbor_guard_seconds : int +profile_id : str +schema_version : str +split_periods : Mapping[str, tuple[str, ...]] +symbols : tuple[str, ...] +synchronized_windows_per_period : int +window_duration_seconds : int + +__init__(self, symbols: tuple[str, ...], split_periods: Mapping[str, tuple[str, ...]], synchronized_windows_per_period: int, window_duration_seconds: int, minimum_events_per_symbol: int, max_events_per_symbol: int, fragment_event_count: int, max_fragments: int, max_matches: int, neighbor_guard_seconds: int, max_source_bytes: int, max_runtime_seconds: float, max_peak_memory_bytes: int, max_artifact_bytes: int, profile_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ModernReferenceMotifProfileV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motif_library.ModernReferenceMotifBuildV1->histdatacom.synthetic.motif_library.ModernReferenceMotifProfileV1 + + +profile + + + +histdatacom.synthetic.motif_library.ModernReferenceMotifBuildV1->histdatacom.synthetic.motifs.ReferenceMotifIndexV1 + + +index - + histdatacom.exceptions.NetworkOperationError - -NetworkOperationError - -category : NETWORK -code : str -retryable : bool - - + +NetworkOperationError + +category : NETWORK +code : str +retryable : bool + + - + histdatacom.exceptions.NetworkOperationError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.exceptions.NoDataOperationError - -NoDataOperationError - -category : NO_DATA -code : str -retryable : bool - - + +NoDataOperationError + +category : NO_DATA +code : str +retryable : bool + + - + histdatacom.exceptions.NoDataOperationError->histdatacom.exceptions.HistDataOperationError - - + + + + + +histdatacom.synthetic.observation.ObservationApplicationResultV1 + +ObservationApplicationResultV1 + +application_mode : str +carry_state +diagnostic_samples : tuple[Mapping[str, JSONValue], ...] +fallback_counts : Mapping[str, int] +input_count : int +operator_id : str +output_count : int +output_events : tuple[ObservationOutputEventV1, ...] +reason_counts : Mapping[str, int] +result_id : str +samples_truncated : bool +schema_version : str +symbol : str +window_id : str + +__init__(self, operator_id: str, window_id: str, symbol: str, application_mode: str, input_count: int, output_events: tuple[ObservationOutputEventV1, ...], reason_counts: Mapping[str, int], fallback_counts: Mapping[str, int], diagnostic_samples: tuple[Mapping[str, JSONValue], ...], samples_truncated: bool, carry_state: ObservationCarryStateV1, result_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationApplicationResultV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationCarryStateV1 + +ObservationCarryStateV1 + +carry_id : str +last_ask : float | None +last_bid : float | None +last_observed_time_ns : int | None +last_source_time_ns : int | None +operator_id : str +outage_active : bool +outage_bucket_start_ns : int | None +rate_bucket_count : int +rate_bucket_start_ns : int | None +reconnect_pending : bool +schema_version : str +symbol : str + +__init__(self, operator_id: str, symbol: str, last_source_time_ns: int | None, last_observed_time_ns: int | None, last_bid: float | None, last_ask: float | None, rate_bucket_start_ns: int | None, rate_bucket_count: int, outage_bucket_start_ns: int | None, outage_active: bool, reconnect_pending: bool, carry_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationCarryStateV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationApplicationResultV1->histdatacom.synthetic.observation.ObservationCarryStateV1 + + +carry_state + + + +histdatacom.synthetic.observation_calibration.ObservationCalibrationCampaignV2 + +ObservationCalibrationCampaignV2 + +calibration_corpus_sha256 : str +campaign_id : str +feed_epoch_definition_id : str +fit_evidence : tuple[ObservationFitEvidenceV1, ...] +operator +peak_memory_bytes : int +profile +readiness_reasons : tuple[str, ...] +readiness_status : str +runtime_seconds : float +schema_version : str +targets : tuple[ObservationCalibrationTargetV2, ...] +valid_for_application : bool +windows : tuple[ObservationCalibrationWindowV2, ...] + +__init__(self, feed_epoch_definition_id: str, calibration_corpus_sha256: str, profile: ObservationCalibrationProfileV2, operator: ObservationOperatorV1, targets: tuple[ObservationCalibrationTargetV2, ...], fit_evidence: tuple[ObservationFitEvidenceV1, ...], windows: tuple[ObservationCalibrationWindowV2, ...], readiness_status: str, readiness_reasons: tuple[str, ...], runtime_seconds: float, peak_memory_bytes: int, campaign_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationCalibrationCampaignV2' +identity_payload(): dict[str, JSONValue] +require_application_ready(context: ObservationContextV1): ObservationCalibrationTargetV2 +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation_calibration.ObservationCalibrationProfileV2 + +ObservationCalibrationProfileV2 + +duplicate_tolerance : float +max_events_per_window : int +max_peak_memory_bytes : int +max_runtime_seconds : float +max_source_bytes : int +minimum_events_per_window : int +profile_id : str +retention_tolerance : float +rounding_digits : int +schema_version : str +sessions : tuple[str, ...] +split_periods : Mapping[str, str] +timestamp_tolerance : float +update_mix_l1_tolerance : float + +__init__(self, split_periods: Mapping[str, str], sessions: tuple[str, ...], max_events_per_window: int, minimum_events_per_window: int, max_source_bytes: int, max_runtime_seconds: float, max_peak_memory_bytes: int, retention_tolerance: float, duplicate_tolerance: float, timestamp_tolerance: float, update_mix_l1_tolerance: float, rounding_digits: int, profile_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationCalibrationProfileV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation_calibration.ObservationCalibrationCampaignV2->histdatacom.synthetic.observation_calibration.ObservationCalibrationProfileV2 + + +profile + + + +histdatacom.synthetic.observation.ObservationOperatorV1 + +ObservationOperatorV1 + +carry_fields : tuple[str, ...] +carry_required_after_first_window : bool +diagnostics +feed_epoch_definition_id : str +feed_epoch_labels : tuple[str, ...] +fit_config +lineage : Mapping[str, JSONValue] +operator_id : str +required_left_halo_ns : int +schema_version : str +source_hashes : tuple[str, ...] +strata : tuple[ObservationStratumV1, ...] +valid_for_application : bool + +__init__(self, feed_epoch_definition_id: str, feed_epoch_labels: tuple[str, ...], fit_config: ObservationOperatorFitConfigV1, strata: tuple[ObservationStratumV1, ...], diagnostics: ObservationFitDiagnosticsV1, source_hashes: tuple[str, ...], lineage: Mapping[str, JSONValue], required_left_halo_ns: int, carry_required_after_first_window: bool, carry_fields: tuple[str, ...], operator_id: str, schema_version: str): None +__post_init__(): None +apply(events: Sequence['ObservationInputEventV1']): 'ObservationApplicationResultV1' +degrade(events: Sequence['ObservationInputEventV1']): 'ObservationApplicationResultV1' +from_dict(data: Mapping[str, Any]): 'ObservationOperatorV1' +from_json(text: str): 'ObservationOperatorV1' +identity_payload(): dict[str, JSONValue] +resolve_stratum(context: ObservationContextV1): tuple[ObservationStratumV1, tuple[str, ...]] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.observation_calibration.ObservationCalibrationCampaignV2->histdatacom.synthetic.observation.ObservationOperatorV1 + + +operator + + + +histdatacom.synthetic.observation_calibration.ObservationCalibrationTargetV2 + +ObservationCalibrationTargetV2 + +calibration_end_period : str +epoch_label : str +mechanism_diagnostics : Mapping[str, str] +parameter_lower_bounds : Mapping[str, float] +parameter_reasons : Mapping[str, str] +parameter_status : Mapping[str, str] +parameter_support_counts : Mapping[str, int] +parameter_upper_bounds : Mapping[str, float] +parameter_values : Mapping[str, float] +reference_epoch_label : str +schema_version : str +source_evidence_ids : tuple[str, ...] +source_hashes : tuple[str, ...] +symbol : str +target_id : str +target_statistics : Mapping[str, JSONValue] +unsupported_parameters : tuple[str, ...] + +__init__(self, symbol: str, epoch_label: str, reference_epoch_label: str, calibration_end_period: str, parameter_values: Mapping[str, float], parameter_lower_bounds: Mapping[str, float], parameter_upper_bounds: Mapping[str, float], parameter_support_counts: Mapping[str, int], parameter_status: Mapping[str, str], parameter_reasons: Mapping[str, str], target_statistics: Mapping[str, JSONValue], mechanism_diagnostics: Mapping[str, str], source_evidence_ids: tuple[str, ...], source_hashes: tuple[str, ...], target_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationCalibrationTargetV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation_calibration.ObservationCalibrationWindowV2 + +ObservationCalibrationWindowV2 + +absolute_errors : Mapping[str, float] +epoch_label : str +failure_reasons : tuple[str, ...] +input_count : int +observed_metrics : Mapping[str, float] +output_count : int +passed : bool +period : str +reason_counts : Mapping[str, int] +retained_source_count : int +schema_version : str +session : str +source_artifact_sha256 : str +split_kind +symbol : str +target_metrics : Mapping[str, float] +tolerances : Mapping[str, float] +transformation_counts : Mapping[str, int] +window_id : str + +__init__(self, split_kind: BenchmarkSplitKind, symbol: str, period: str, session: str, epoch_label: str, source_artifact_sha256: str, input_count: int, output_count: int, retained_source_count: int, target_metrics: Mapping[str, float], observed_metrics: Mapping[str, float], absolute_errors: Mapping[str, float], tolerances: Mapping[str, float], reason_counts: Mapping[str, int], transformation_counts: Mapping[str, int], passed: bool, failure_reasons: tuple[str, ...], window_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationCalibrationWindowV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation_calibration.ObservationCalibrationWindowV2->histdatacom.synthetic.benchmark.BenchmarkSplitKind + + +split_kind + + + +histdatacom.synthetic.observation.ObservationContextV1 + +ObservationContextV1 + +epoch_id : str +event_tag : str | None +schema_version : str +session : str | None +state : str | None +symbol : str + +__init__(self, symbol: str, epoch_id: str, state: str | None, session: str | None, event_tag: str | None, schema_version: str): None +__post_init__(): None +candidate_keys(levels: Sequence[str]): tuple[str, ...] +from_dict(data: Mapping[str, Any]): 'ObservationContextV1' +key_for_level(level: str): str | None +pattern_for_level(level: str): dict[str, str] | None +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationFitDiagnosticsV1 + +ObservationFitDiagnosticsV1 + +diagnostics_id : str +evidence_count : int +parameter_residual_medians : Mapping[str, float] +parameter_support_counts : Mapping[str, int] +samples : tuple[Mapping[str, JSONValue], ...] +samples_truncated : bool +schema_version : str +status_counts : Mapping[str, int] +stratum_count : int +unsupported_parameter_names : tuple[str, ...] + +__init__(self, evidence_count: int, stratum_count: int, status_counts: Mapping[str, int], parameter_support_counts: Mapping[str, int], parameter_residual_medians: Mapping[str, float], unsupported_parameter_names: tuple[str, ...], samples: tuple[Mapping[str, JSONValue], ...], samples_truncated: bool, diagnostics_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationFitDiagnosticsV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationFitEvidenceV1 + +ObservationFitEvidenceV1 + +context +end_timestamp_ns : int +evidence_id : str +evidence_kind : str +parameter_basis : Mapping[str, str] +parameter_lower_bounds : Mapping[str, float] +parameter_provenance : Mapping[str, tuple[str, ...]] +parameter_support_counts : Mapping[str, int] +parameter_upper_bounds : Mapping[str, float] +parameter_values : Mapping[str, float] +period : str +schema_version : str +source_artifact_sha256 : str +source_evidence_id : str +source_hash_basis : str +start_timestamp_ns : int +support_count : int + +__init__(self, context: ObservationContextV1, period: str, start_timestamp_ns: int, end_timestamp_ns: int, source_evidence_id: str, source_artifact_sha256: str, source_hash_basis: str, evidence_kind: str, parameter_values: Mapping[str, float], parameter_lower_bounds: Mapping[str, float], parameter_upper_bounds: Mapping[str, float], parameter_support_counts: Mapping[str, int], parameter_basis: Mapping[str, str], parameter_provenance: Mapping[str, tuple[str, ...]], evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationFitEvidenceV1' +from_feed_epoch_evidence(evidence: FeedEpochEvidenceV1 | FeedEpochEvidenceV2, epoch_definition: FeedEpochDefinitionV1 | FeedEpochDefinitionV2): 'ObservationFitEvidenceV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationFitEvidenceV1->histdatacom.synthetic.observation.ObservationContextV1 + + +context + + + +histdatacom.synthetic.observation.ObservationInputEventV1 + +ObservationInputEventV1 + +ask : float +bid : float +context +event_sequence : int +event_time_ns : int +protected_anchor : bool +schema_version : str +source_event_id : str +symbol : str + +__init__(self, source_event_id: str, symbol: str, event_time_ns: int, event_sequence: int, bid: float, ask: float, context: ObservationContextV1, protected_anchor: bool, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationInputEventV1' +from_synthetic_event(event: SyntheticEventV1): 'ObservationInputEventV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationInputEventV1->histdatacom.synthetic.observation.ObservationContextV1 + + +context + + + +histdatacom.synthetic.observation.ObservationOperatorFitConfigV1 + +ObservationOperatorFitConfigV1 + +backoff_levels : tuple[str, ...] +config_id : str +diagnostic_sample_limit : int +max_evidence : int +max_input_events : int +max_outputs_per_input : int +max_strata : int +min_parameter_support : int +min_stratum_support : int +min_supported_parameters : int +required_left_halo_ns : int +rounding_digits : int +schema_version : str + +__init__(self, min_stratum_support: int, min_parameter_support: int, min_supported_parameters: int, max_evidence: int, max_strata: int, max_input_events: int, max_outputs_per_input: int, diagnostic_sample_limit: int, required_left_halo_ns: int, backoff_levels: tuple[str, ...], rounding_digits: int, config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationOperatorFitConfigV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationOperatorV1->histdatacom.synthetic.observation.ObservationFitDiagnosticsV1 + + +diagnostics + + + +histdatacom.synthetic.observation.ObservationOperatorV1->histdatacom.synthetic.observation.ObservationOperatorFitConfigV1 + + +fit_config + + + +histdatacom.synthetic.observation.ObservationOutputEventV1 + +ObservationOutputEventV1 + +ask : float +bid : float +duplicate_ordinal : int +observation_id : str +observed_sequence : int +observed_time_ns : int +operator_id : str +protected_anchor : bool +schema_version : str +source_event_id : str +source_time_ns : int +stratum_id : str +symbol : str +transformations : tuple[str, ...] + +__init__(self, source_event_id: str, operator_id: str, stratum_id: str, symbol: str, source_time_ns: int, observed_time_ns: int, observed_sequence: int, bid: float, ask: float, duplicate_ordinal: int, transformations: tuple[str, ...], protected_anchor: bool, observation_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationOutputEventV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationParameterEstimateV1 + +ObservationParameterEstimateV1 + +estimation_bases : tuple[str, ...] +evidence_count : int +evidence_ids : tuple[str, ...] +lower : float +name : str +provenance : tuple[str, ...] +schema_version : str +support_count : int +support_status : str +upper : float +value : float + +__init__(self, name: str, value: float, lower: float, upper: float, support_count: int, evidence_count: int, support_status: str, estimation_bases: tuple[str, ...], evidence_ids: tuple[str, ...], provenance: tuple[str, ...], schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ObservationParameterEstimateV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.observation.ObservationStratumV1 + +ObservationStratumV1 + +evidence_ids : tuple[str, ...] +fallback_keys : tuple[str, ...] +key : str +level : str +parameter_map : dict[str, ObservationParameterEstimateV1] +parameters : tuple[ObservationParameterEstimateV1, ...] +pattern : Mapping[str, str] +schema_version : str +status : str +stratum_id : str +support_count : int + +__init__(self, level: str, key: str, pattern: Mapping[str, str], status: str, support_count: int, parameters: tuple[ObservationParameterEstimateV1, ...], evidence_ids: tuple[str, ...], fallback_keys: tuple[str, ...], stratum_id: str, schema_version: str): None +__post_init__(): None +effective_value(name: str): float +from_dict(data: Mapping[str, Any]): 'ObservationStratumV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] - + histdatacom.cancellation.OperationResumePolicy - -OperationResumePolicy - -cleanup_summary : str -partial_artifact_disposition -partial_artifact_patterns : tuple[str, ...] -resume_mode -resume_summary : str -retry_safe : bool -stage : str -stops_future_work : bool - -__init__(self, stage: str, partial_artifact_disposition: PartialArtifactDisposition, resume_mode: ResumeMode, cleanup_summary: str, resume_summary: str, retry_safe: bool, stops_future_work: bool, partial_artifact_patterns: tuple[str, ...]): None -to_dict(): dict[str, JSONValue] + +OperationResumePolicy + +cleanup_summary : str +partial_artifact_disposition +partial_artifact_patterns : tuple[str, ...] +resume_mode +resume_summary : str +retry_safe : bool +stage : str +stops_future_work : bool + +__init__(self, stage: str, partial_artifact_disposition: PartialArtifactDisposition, resume_mode: ResumeMode, cleanup_summary: str, resume_summary: str, retry_safe: bool, stops_future_work: bool, partial_artifact_patterns: tuple[str, ...]): None +to_dict(): dict[str, JSONValue] - + histdatacom.cancellation.PartialArtifactDisposition - -PartialArtifactDisposition - -name - - + +PartialArtifactDisposition + +name + + - + histdatacom.cancellation.OperationResumePolicy->histdatacom.cancellation.PartialArtifactDisposition - - -partial_artifact_disposition + + +partial_artifact_disposition - + histdatacom.cancellation.ResumeMode - -ResumeMode - -name - - + +ResumeMode + +name + + - + histdatacom.cancellation.OperationResumePolicy->histdatacom.cancellation.ResumeMode - - -resume_mode + + +resume_mode - + histdatacom.histdata_com._HistDataCom - -_HistDataCom - -context -options - -__init__(options: Options): None -_export_run_artifacts(): dict[str, JSONValue] -_export_run_request_json(): dict[str, JSONValue] -_materialize_orchestration_api_return(records: list[Record]): list | PolarsDataFrame | DataFrame | Table -_run_orchestration_job(): list | dict | PolarsDataFrame | DataFrame | Table -_run_quality_preflight(): dict[str, Any] -_should_materialize_orchestration_api_return(payload: dict): bool -_should_materialize_orchestration_repository_return(): bool -_submit_orchestration_job(): JobResult -_warn_before_large_quality_run_without_preflight(): None -run(): list | dict | PolarsDataFrame | DataFrame | Table | None + +_HistDataCom + +context +options + +__init__(options: Options): None +_export_run_artifacts(): dict[str, JSONValue] +_export_run_request_json(): dict[str, JSONValue] +_materialize_orchestration_api_return(records: list[Record]): list | PolarsDataFrame | DataFrame | Table +_run_orchestration_job(): list | dict | PolarsDataFrame | DataFrame | Table +_run_quality_preflight(): dict[str, Any] +_should_materialize_orchestration_api_return(payload: dict): bool +_should_materialize_orchestration_repository_return(): bool +_submit_orchestration_job(): JobResult +_warn_before_large_quality_run_without_preflight(): None +run(): list | dict | PolarsDataFrame | DataFrame | Table | None - + histdatacom.options.Options->histdatacom.histdata_com._HistDataCom - - -options + + +options - + histdatacom.orchestration.performance.OrchestrationConcurrencyProfile - -OrchestrationConcurrencyProfile - -base_workers : int -cpu_file_workers : int -cpu_utilization : str -influx_workers : int -network_multiplier : int -network_workers : int -orchestration_workers : int -source : str - -__init__(self, cpu_utilization: str, base_workers: int, orchestration_workers: int, network_workers: int, cpu_file_workers: int, influx_workers: int, network_multiplier: int, source: str): None -to_dict(): dict[str, JSONValue] -with_lane_override(lane: object, workers: int): 'OrchestrationConcurrencyProfile' -worker_options_for_lane(lane: object): dict[str, int] -workers_for_lane(lane: object): int + +OrchestrationConcurrencyProfile + +base_workers : int +cpu_file_workers : int +cpu_utilization : str +influx_workers : int +network_multiplier : int +network_workers : int +orchestration_workers : int +source : str + +__init__(self, cpu_utilization: str, base_workers: int, orchestration_workers: int, network_workers: int, cpu_file_workers: int, influx_workers: int, network_multiplier: int, source: str): None +to_dict(): dict[str, JSONValue] +with_lane_override(lane: object, workers: int): 'OrchestrationConcurrencyProfile' +worker_options_for_lane(lane: object): dict[str, int] +workers_for_lane(lane: object): int - + histdatacom.orchestration.resources.OrchestrationExecutableUnavailable - -OrchestrationExecutableUnavailable - - - + +OrchestrationExecutableUnavailable + + + - + histdatacom.orchestration.resources.OrchestrationResourceError - -OrchestrationResourceError - - - + +OrchestrationResourceError + + + - + histdatacom.orchestration.resources.OrchestrationExecutableUnavailable->histdatacom.orchestration.resources.OrchestrationResourceError - - + + - + histdatacom.orchestration.client.OrchestrationJobHandle - -OrchestrationJobHandle - -namespace : str -request_id : str -run_id : str -task_queue : str -workflow_id : str - -__init__(self, request_id: str, workflow_id: str, run_id: str, task_queue: str, namespace: str): None -to_dict(): dict[str, str] + +OrchestrationJobHandle + +namespace : str +request_id : str +run_id : str +task_queue : str +workflow_id : str + +__init__(self, request_id: str, workflow_id: str, run_id: str, task_queue: str, namespace: str): None +to_dict(): dict[str, str] - + histdatacom.orchestration.control.OrchestrationJobList - -OrchestrationJobList - -jobs : tuple[OrchestrationJobSnapshot, ...] -schema_version : int - -__init__(self, jobs: tuple[OrchestrationJobSnapshot, ...], schema_version: int): None -from_dict(data: Mapping[str, Any]): 'OrchestrationJobList' -to_dict(): dict[str, Any] + +OrchestrationJobList + +jobs : tuple[OrchestrationJobSnapshot, ...] +schema_version : int + +__init__(self, jobs: tuple[OrchestrationJobSnapshot, ...], schema_version: int): None +from_dict(data: Mapping[str, Any]): 'OrchestrationJobList' +to_dict(): dict[str, Any] - + histdatacom.orchestration.client.OrchestrationJobResult - -OrchestrationJobResult - -handle -orchestration_status : OrchestrationStatus | None -result : Optional[Any] -runtime_cleanup : dict[str, Any] | None -snapshot : OrchestrationJobSnapshot | None -status : str - -__init__(self, handle: OrchestrationJobHandle, status: str, result: Any, orchestration_status: OrchestrationStatus | None, snapshot: OrchestrationJobSnapshot | None, runtime_cleanup: dict[str, Any] | None): None -to_dict(): dict[str, Any] + +OrchestrationJobResult + +handle +orchestration_status : OrchestrationStatus | None +result : Optional[Any] +runtime_cleanup : dict[str, Any] | None +snapshot : OrchestrationJobSnapshot | None +status : str + +__init__(self, handle: OrchestrationJobHandle, status: str, result: Any, orchestration_status: OrchestrationStatus | None, snapshot: OrchestrationJobSnapshot | None, runtime_cleanup: dict[str, Any] | None): None +to_dict(): dict[str, Any] - + histdatacom.orchestration.client.OrchestrationJobResult->histdatacom.orchestration.client.OrchestrationJobHandle - - -handle + + +handle - + histdatacom.orchestration.control.OrchestrationJobSnapshot->histdatacom.orchestration.control.JobControlStates - - -controls + + +controls - + histdatacom.orchestration.control.OrchestrationJobSnapshot->histdatacom.orchestration.control.JobLifecycle - - -lifecycle + + +lifecycle - + histdatacom.orchestration.control.OrchestrationJobSnapshot->histdatacom.runtime_contracts.WorkStatus - - -status + + +status - + histdatacom.orchestration.maintenance.OrchestrationMaintenanceResult - -OrchestrationMaintenanceResult - -downloaded_artifacts_removed : bool -logs : tuple[LogMaintenanceResult, ...] -message : str -orchestration_state : str -retention_policy -runtime_policy -state : str -status_store -temporal_sqlite -warnings : tuple[str, ...] - -__init__(self, state: str, message: str, runtime_policy: OrchestrationRuntimePolicy, retention_policy: OrchestrationRetentionPolicy, orchestration_state: str, logs: tuple[LogMaintenanceResult, ...], status_store: StatusStoreMaintenanceResult, temporal_sqlite: TemporalSqliteMaintenanceResult, warnings: tuple[str, ...], downloaded_artifacts_removed: bool): None -to_dict(): dict[str, Any] + +OrchestrationMaintenanceResult + +downloaded_artifacts_removed : bool +logs : tuple[LogMaintenanceResult, ...] +message : str +orchestration_state : str +retention_policy +runtime_policy +state : str +status_store +temporal_sqlite +warnings : tuple[str, ...] + +__init__(self, state: str, message: str, runtime_policy: OrchestrationRuntimePolicy, retention_policy: OrchestrationRetentionPolicy, orchestration_state: str, logs: tuple[LogMaintenanceResult, ...], status_store: StatusStoreMaintenanceResult, temporal_sqlite: TemporalSqliteMaintenanceResult, warnings: tuple[str, ...], downloaded_artifacts_removed: bool): None +to_dict(): dict[str, Any] - + histdatacom.orchestration.maintenance.OrchestrationRetentionPolicy - -OrchestrationRetentionPolicy - -max_artifacts_per_owner : int -max_dataset_plans_per_request : int -max_job_snapshots : int -max_log_bytes : int -max_rotated_logs : int -max_stage_results_per_work_item : int -max_status_events_per_owner : int -max_temporal_sqlite_bytes : int - -__init__(self, max_log_bytes: int, max_rotated_logs: int, max_temporal_sqlite_bytes: int, max_job_snapshots: int, max_status_events_per_owner: int, max_stage_results_per_work_item: int, max_artifacts_per_owner: int, max_dataset_plans_per_request: int): None -__post_init__(): None -to_dict(): dict[str, int] + +OrchestrationRetentionPolicy + +max_artifacts_per_owner : int +max_dataset_plans_per_request : int +max_job_snapshots : int +max_log_bytes : int +max_rotated_logs : int +max_stage_results_per_work_item : int +max_status_events_per_owner : int +max_temporal_sqlite_bytes : int + +__init__(self, max_log_bytes: int, max_rotated_logs: int, max_temporal_sqlite_bytes: int, max_job_snapshots: int, max_status_events_per_owner: int, max_stage_results_per_work_item: int, max_artifacts_per_owner: int, max_dataset_plans_per_request: int): None +__post_init__(): None +to_dict(): dict[str, int] - + histdatacom.orchestration.maintenance.OrchestrationMaintenanceResult->histdatacom.orchestration.maintenance.OrchestrationRetentionPolicy - - -retention_policy + + +retention_policy - + histdatacom.orchestration.runtime.OrchestrationRuntimePolicy - -OrchestrationRuntimePolicy - -paths -ports -runtime_home : Path -workspace : Path -workspace_id : str -workspace_slug : str - -__init__(self, workspace: Path, workspace_id: str, workspace_slug: str, runtime_home: Path, paths: OrchestrationPaths, ports: OrchestrationPorts): None -ensure_directories(): None -temporal_start_args(): tuple[str, ...] -to_dict(): dict[str, Any] -with_available_ports(port_available: PortAvailabilityProbe): 'OrchestrationRuntimePolicy' -write_manifest(): None + +OrchestrationRuntimePolicy + +paths +ports +runtime_home : Path +workspace : Path +workspace_id : str +workspace_slug : str + +__init__(self, workspace: Path, workspace_id: str, workspace_slug: str, runtime_home: Path, paths: OrchestrationPaths, ports: OrchestrationPorts): None +ensure_directories(): None +temporal_start_args(): tuple[str, ...] +to_dict(): dict[str, Any] +with_available_ports(port_available: PortAvailabilityProbe): 'OrchestrationRuntimePolicy' +write_manifest(): None - + histdatacom.orchestration.maintenance.OrchestrationMaintenanceResult->histdatacom.orchestration.runtime.OrchestrationRuntimePolicy - - -runtime_policy + + +runtime_policy - + histdatacom.orchestration.maintenance.StatusStoreMaintenanceResult - -StatusStoreMaintenanceResult - -action : str -bytes_recovered : int -compacted : bool -error : str -exists : bool -expected_schema_version : int -path : str -reason : str -rows_deleted : dict[str, int] -schema_state : str -schema_version : int -size_after_bytes : int -size_before_bytes : int - -__init__(self, path: str, exists: bool, action: str, rows_deleted: dict[str, int], reason: str, error: str, schema_version: int, expected_schema_version: int, schema_state: str, size_before_bytes: int, size_after_bytes: int, bytes_recovered: int, compacted: bool): None -to_dict(): dict[str, Any] + +StatusStoreMaintenanceResult + +action : str +bytes_recovered : int +compacted : bool +error : str +exists : bool +expected_schema_version : int +path : str +reason : str +rows_deleted : dict[str, int] +schema_state : str +schema_version : int +size_after_bytes : int +size_before_bytes : int + +__init__(self, path: str, exists: bool, action: str, rows_deleted: dict[str, int], reason: str, error: str, schema_version: int, expected_schema_version: int, schema_state: str, size_before_bytes: int, size_after_bytes: int, bytes_recovered: int, compacted: bool): None +to_dict(): dict[str, Any] - + histdatacom.orchestration.maintenance.OrchestrationMaintenanceResult->histdatacom.orchestration.maintenance.StatusStoreMaintenanceResult - - -status_store + + +status_store - + histdatacom.orchestration.maintenance.TemporalSqliteMaintenanceResult - -TemporalSqliteMaintenanceResult - -action : str -error : str -exists : bool -files : tuple[str, ...] -max_bytes : int -path : str -reason : str -size_bytes : int -within_limit : bool - -__init__(self, path: str, files: tuple[str, ...], exists: bool, action: str, size_bytes: int, max_bytes: int, within_limit: bool, reason: str, error: str): None -to_dict(): dict[str, Any] + +TemporalSqliteMaintenanceResult + +action : str +error : str +exists : bool +files : tuple[str, ...] +max_bytes : int +path : str +reason : str +size_bytes : int +within_limit : bool + +__init__(self, path: str, files: tuple[str, ...], exists: bool, action: str, size_bytes: int, max_bytes: int, within_limit: bool, reason: str, error: str): None +to_dict(): dict[str, Any] - + histdatacom.orchestration.maintenance.OrchestrationMaintenanceResult->histdatacom.orchestration.maintenance.TemporalSqliteMaintenanceResult - - -temporal_sqlite + + +temporal_sqlite - + histdatacom.orchestration.resources.OrchestrationManifest - -OrchestrationManifest - -distribution_strategy : str -embedded_binary : bool -platforms : dict[str, OrchestrationPlatformResource] -resource_files : tuple[str, ...] -runtime : str -runtime_artifact_index : str -schema_version : int -sdist_fallback : str - -__init__(self, schema_version: int, runtime: str, distribution_strategy: str, embedded_binary: bool, resource_files: tuple[str, ...], runtime_artifact_index: str, platforms: dict[str, OrchestrationPlatformResource], sdist_fallback: str): None -from_dict(data: Mapping[str, Any]): 'OrchestrationManifest' + +OrchestrationManifest + +distribution_strategy : str +embedded_binary : bool +platforms : dict[str, OrchestrationPlatformResource] +resource_files : tuple[str, ...] +runtime : str +runtime_artifact_index : str +schema_version : int +sdist_fallback : str + +__init__(self, schema_version: int, runtime: str, distribution_strategy: str, embedded_binary: bool, resource_files: tuple[str, ...], runtime_artifact_index: str, platforms: dict[str, OrchestrationPlatformResource], sdist_fallback: str): None +from_dict(data: Mapping[str, Any]): 'OrchestrationManifest' - + histdatacom.orchestration.client.OrchestrationOverlapError - -OrchestrationOverlapError - -category : VALIDATION -code : str -exit_code : int -retryable : bool - - + +OrchestrationOverlapError + +category : VALIDATION +code : str +exit_code : int +retryable : bool + + - + histdatacom.orchestration.client.OrchestrationOverlapError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.orchestration.runtime.OrchestrationPaths - -OrchestrationPaths - -lock_file : Path -logs_dir : Path -manifests_dir : Path -pid_file : Path -runtime_dir : Path -runtime_manifest : Path -server_log : Path -sqlite_db : Path -sqlite_dir : Path -state_dir : Path -worker_log : Path - -__init__(self, runtime_dir: Path, state_dir: Path, logs_dir: Path, sqlite_dir: Path, manifests_dir: Path, pid_file: Path, lock_file: Path, server_log: Path, worker_log: Path, sqlite_db: Path, runtime_manifest: Path): None -ensure_directories(): None -from_runtime_dir(runtime_dir: Path | str): 'OrchestrationPaths' -from_state_dir(state_dir: Path | str): 'OrchestrationPaths' -to_dict(): dict[str, str] + +OrchestrationPaths + +lock_file : Path +logs_dir : Path +manifests_dir : Path +pid_file : Path +runtime_dir : Path +runtime_manifest : Path +server_log : Path +sqlite_db : Path +sqlite_dir : Path +state_dir : Path +worker_log : Path + +__init__(self, runtime_dir: Path, state_dir: Path, logs_dir: Path, sqlite_dir: Path, manifests_dir: Path, pid_file: Path, lock_file: Path, server_log: Path, worker_log: Path, sqlite_db: Path, runtime_manifest: Path): None +ensure_directories(): None +from_runtime_dir(runtime_dir: Path | str): 'OrchestrationPaths' +from_state_dir(state_dir: Path | str): 'OrchestrationPaths' +to_dict(): dict[str, str] - + histdatacom.orchestration.resources.OrchestrationPlatformResource - -OrchestrationPlatformResource - -bundled : bool -executable : str -key : str -notes : str -provenance : str -wheel_tags : tuple[str, ...] - -__init__(self, key: str, bundled: bool, executable: str, wheel_tags: tuple[str, ...], provenance: str, notes: str): None -from_dict(key: str, data: Mapping[str, Any]): 'OrchestrationPlatformResource' + +OrchestrationPlatformResource + +bundled : bool +executable : str +key : str +notes : str +provenance : str +wheel_tags : tuple[str, ...] + +__init__(self, key: str, bundled: bool, executable: str, wheel_tags: tuple[str, ...], provenance: str, notes: str): None +from_dict(key: str, data: Mapping[str, Any]): 'OrchestrationPlatformResource' - + histdatacom.orchestration.runtime.OrchestrationPorts - -OrchestrationPorts - -bind_ip : str -collisions : tuple[int, ...] -grpc : int -source : str -ui : int - -__init__(self, bind_ip: str, grpc: int, ui: int, source: str, collisions: tuple[int, ...]): None -temporal_start_args(): tuple[str, ...] -to_dict(): dict[str, Any] + +OrchestrationPorts + +bind_ip : str +collisions : tuple[int, ...] +grpc : int +source : str +ui : int + +__init__(self, bind_ip: str, grpc: int, ui: int, source: str, collisions: tuple[int, ...]): None +temporal_start_args(): tuple[str, ...] +to_dict(): dict[str, Any] - + histdatacom.orchestration.runtime.OrchestrationRuntimePolicy->histdatacom.orchestration.runtime.OrchestrationPaths - - -paths + + +paths - + histdatacom.orchestration.runtime.OrchestrationRuntimePolicy->histdatacom.orchestration.runtime.OrchestrationPorts - - -ports + + +ports - + histdatacom.orchestration.supervisor.OrchestrationSupervisor - -OrchestrationSupervisor - -_command_runner -_frontend_ready -_port_available -_process_exists -_process_factory : Popen -_process_kill -_process_terminate -_sleep -_worker_dependency_available -cpu_utilization : str -influx_workers : int -namespace : str -network_multiplier : int -orchestration_workers : int -paths -runtime_policy -task_queue_prefix : str -worker_lanes : tuple - -__init__(paths: OrchestrationPaths | None): None -_acquire_lock(): None -_attempt_process_action(action: Callable[..., None]): None -_cleanup_started_processes(processes: Mapping[str, Any], pids: Mapping[str, int]): None -_client_worker_config_from_running_state(status: OrchestrationStatus): OrchestrationWorkerConfig -_component_lane(component: str): TaskQueueLane | None -_component_states(pids: Mapping[str, int], worker_readiness: Mapping[str, Mapping[str, Any]] | None): dict[str, str] -_default_logs(): dict[str, str] -_ensure_namespace(executable: Path, runtime_policy: OrchestrationRuntimePolicy): None -_kill_pids(pids: Mapping[str, int]): None -_launch_component(command: Sequence[str], log_path: Path): Any -_normalize_state_schema(state: dict[str, Any]): dict[str, Any] -_path_dict(): dict[str, str] -_process_returncode(process: Any): int | None -_process_running(process: Any): bool -_read_running_state(): dict[str, Any] -_read_state(): dict[str, Any] -_readiness_pid(payload: Mapping[str, Any]): int -_release_lock(): None -_remove_state_files(): None -_required_components(): tuple[str, ...] -_run_temporal_command(command: Sequence[str]): subprocess.CompletedProcess[str] -_runtime_policy_from_running_state(state: Mapping[str, Any], status: OrchestrationStatus): OrchestrationRuntimePolicy -_runtime_policy_from_status(status: OrchestrationStatus): OrchestrationRuntimePolicy -_settle_before_worker_launch(): None -_start_process(executable: Path, extra_args: Sequence[str], startup_timeout: float, runtime_policy: OrchestrationRuntimePolicy): OrchestrationStatus -_start_worker_lane(lane: TaskQueueLane, worker_config: OrchestrationWorkerConfig, component: str, processes: dict[str, Any], pids: dict[str, int], deadline: float): tuple[Any, list[str], Path, dict[str, Any]] -_state_concurrency_profile(worker_fleet: Mapping[str, Any]): OrchestrationConcurrencyProfile | None -_state_int(payload: Mapping[str, Any], key: str): int -_state_logs(state: Mapping[str, Any]): dict[str, str] -_state_pids(state: Mapping[str, Any]): dict[str, int] -_state_ports(state: Mapping[str, Any]): dict[str, int | str | list[int]] -_state_ports_from_mapping(payload: Mapping[str, Any]): OrchestrationPorts -_state_schema_diagnostics(): dict[str, Any] -_state_task_queues(worker_fleet: Mapping[str, Any], runtime_policy: OrchestrationRuntimePolicy): OrchestrationTaskQueues -_state_worker_fleet(state: Mapping[str, Any]): Mapping[str, Any] -_status(state: str, message: str, pids: Mapping[str, int], command: Sequence[str], ports: Mapping[str, int | str | list[int]] | None, logs: Mapping[str, str] | None, components: Mapping[str, str] | None, worker_readiness: Mapping[str, Mapping[str, Any]] | None): OrchestrationStatus -_terminate_and_wait(pids: Mapping[str, int], stop_timeout: float): dict[str, int] -_terminate_pids(pids: Mapping[str, int]): None -_wait_for_frontend(server_process: Any, runtime_policy: OrchestrationRuntimePolicy, deadline: float): None -_wait_for_process_exit(pids: Mapping[str, int], timeout: float): dict[str, int] -_wait_for_started_processes(processes: Mapping[str, Any]): None -_wait_for_worker_ready(lane: TaskQueueLane, pid: int, worker_process: Any, worker_command: Sequence[str], deadline: float, log_path: Path): dict[str, Any] -_worker_component(lane: TaskQueueLane): str -_worker_config(runtime_policy: OrchestrationRuntimePolicy): OrchestrationWorkerConfig -_worker_exit_message(lane: TaskQueueLane, pid: int, worker_process: Any, command: Sequence[str], log_path: Path): str -_worker_exit_message_with_attempts(lane: TaskQueueLane, pid: int, worker_process: Any, command: Sequence[str], log_path: Path): str -_worker_fleet_metadata(config: OrchestrationWorkerConfig): dict[str, Any] -_worker_launch_attempt_diagnostic(attempt: int, pid: int, worker_process: Any): dict[str, int | str | None] -_worker_launch_attempts_suffix(attempts: Sequence[Mapping[str, int | str | None]]): str -_worker_launch_retry_delay(worker_process: Any, deadline: float): float -_worker_log_path(lane: TaskQueueLane): Path -_worker_readiness_state(lane: TaskQueueLane, pid: int): dict[str, Any] -_worker_readiness_states(pids: Mapping[str, int]): dict[str, dict[str, Any]] -_worker_status(pids: Mapping[str, int], component_states: Mapping[str, str], worker_readiness: Mapping[str, Mapping[str, Any]]): dict[str, dict[str, Any]] -_write_state(state: Mapping[str, Any]): None -client_worker_config(): OrchestrationWorkerConfig -doctor(): dict[str, Any] -restart(): OrchestrationStatus -start(): OrchestrationStatus -status(): OrchestrationStatus -stop(): OrchestrationStatus + +OrchestrationSupervisor + +_command_runner +_frontend_ready +_port_available +_process_exists +_process_factory : Popen +_process_kill +_process_terminate +_sleep +_worker_dependency_available +cpu_utilization : str +influx_workers : int +namespace : str +network_multiplier : int +orchestration_workers : int +paths +runtime_policy +task_queue_prefix : str +worker_lanes : tuple + +__init__(paths: OrchestrationPaths | None): None +_acquire_lock(): None +_attempt_process_action(action: Callable[..., None]): None +_cleanup_started_processes(processes: Mapping[str, Any], pids: Mapping[str, int]): None +_client_worker_config_from_running_state(status: OrchestrationStatus): OrchestrationWorkerConfig +_component_lane(component: str): TaskQueueLane | None +_component_states(pids: Mapping[str, int], worker_readiness: Mapping[str, Mapping[str, Any]] | None): dict[str, str] +_default_logs(): dict[str, str] +_ensure_namespace(executable: Path, runtime_policy: OrchestrationRuntimePolicy): None +_kill_pids(pids: Mapping[str, int]): None +_launch_component(command: Sequence[str], log_path: Path): Any +_normalize_state_schema(state: dict[str, Any]): dict[str, Any] +_path_dict(): dict[str, str] +_process_returncode(process: Any): int | None +_process_running(process: Any): bool +_read_running_state(): dict[str, Any] +_read_state(): dict[str, Any] +_readiness_pid(payload: Mapping[str, Any]): int +_release_lock(): None +_remove_state_files(): None +_required_components(): tuple[str, ...] +_run_temporal_command(command: Sequence[str]): subprocess.CompletedProcess[str] +_runtime_policy_from_running_state(state: Mapping[str, Any], status: OrchestrationStatus): OrchestrationRuntimePolicy +_runtime_policy_from_status(status: OrchestrationStatus): OrchestrationRuntimePolicy +_settle_before_worker_launch(): None +_start_process(executable: Path, extra_args: Sequence[str], startup_timeout: float, runtime_policy: OrchestrationRuntimePolicy): OrchestrationStatus +_start_worker_lane(lane: TaskQueueLane, worker_config: OrchestrationWorkerConfig, component: str, processes: dict[str, Any], pids: dict[str, int], deadline: float): tuple[Any, list[str], Path, dict[str, Any]] +_state_concurrency_profile(worker_fleet: Mapping[str, Any]): OrchestrationConcurrencyProfile | None +_state_int(payload: Mapping[str, Any], key: str): int +_state_logs(state: Mapping[str, Any]): dict[str, str] +_state_pids(state: Mapping[str, Any]): dict[str, int] +_state_ports(state: Mapping[str, Any]): dict[str, int | str | list[int]] +_state_ports_from_mapping(payload: Mapping[str, Any]): OrchestrationPorts +_state_schema_diagnostics(): dict[str, Any] +_state_task_queues(worker_fleet: Mapping[str, Any], runtime_policy: OrchestrationRuntimePolicy): OrchestrationTaskQueues +_state_worker_fleet(state: Mapping[str, Any]): Mapping[str, Any] +_status(state: str, message: str, pids: Mapping[str, int], command: Sequence[str], ports: Mapping[str, int | str | list[int]] | None, logs: Mapping[str, str] | None, components: Mapping[str, str] | None, worker_readiness: Mapping[str, Mapping[str, Any]] | None): OrchestrationStatus +_terminate_and_wait(pids: Mapping[str, int], stop_timeout: float): dict[str, int] +_terminate_pids(pids: Mapping[str, int]): None +_wait_for_frontend(server_process: Any, runtime_policy: OrchestrationRuntimePolicy, deadline: float): None +_wait_for_process_exit(pids: Mapping[str, int], timeout: float): dict[str, int] +_wait_for_started_processes(processes: Mapping[str, Any]): None +_wait_for_worker_ready(lane: TaskQueueLane, pid: int, worker_process: Any, worker_command: Sequence[str], deadline: float, log_path: Path): dict[str, Any] +_worker_component(lane: TaskQueueLane): str +_worker_config(runtime_policy: OrchestrationRuntimePolicy): OrchestrationWorkerConfig +_worker_exit_message(lane: TaskQueueLane, pid: int, worker_process: Any, command: Sequence[str], log_path: Path): str +_worker_exit_message_with_attempts(lane: TaskQueueLane, pid: int, worker_process: Any, command: Sequence[str], log_path: Path): str +_worker_fleet_metadata(config: OrchestrationWorkerConfig): dict[str, Any] +_worker_launch_attempt_diagnostic(attempt: int, pid: int, worker_process: Any): dict[str, int | str | None] +_worker_launch_attempts_suffix(attempts: Sequence[Mapping[str, int | str | None]]): str +_worker_launch_retry_delay(worker_process: Any, deadline: float): float +_worker_log_path(lane: TaskQueueLane): Path +_worker_readiness_state(lane: TaskQueueLane, pid: int): dict[str, Any] +_worker_readiness_states(pids: Mapping[str, int]): dict[str, dict[str, Any]] +_worker_status(pids: Mapping[str, int], component_states: Mapping[str, str], worker_readiness: Mapping[str, Mapping[str, Any]]): dict[str, dict[str, Any]] +_write_state(state: Mapping[str, Any]): None +client_worker_config(): OrchestrationWorkerConfig +doctor(): dict[str, Any] +restart(): OrchestrationStatus +start(): OrchestrationStatus +status(): OrchestrationStatus +stop(): OrchestrationStatus - + histdatacom.orchestration.runtime.OrchestrationRuntimePolicy->histdatacom.orchestration.supervisor.OrchestrationSupervisor - - -runtime_policy + + +runtime_policy - + histdatacom.orchestration.client.OrchestrationSubmissionPreflight - -OrchestrationSubmissionPreflight - -allowed : bool -blocking_snapshot : OrchestrationJobSnapshot | None -checked_job_count : int -checked_job_source : str -config -exit_code : int -message : str -offline : bool -overlap_guard_enabled : bool -request -schedule_identity : str -schedule_identity_source : str -state : str -status_store_path : Path - -__init__(self, request: RunRequest, config: OrchestrationWorkerConfig, status_store_path: Path, allowed: bool, overlap_guard_enabled: bool, schedule_identity_source: str, schedule_identity: str, checked_job_count: int, checked_job_source: str, offline: bool, blocking_snapshot: OrchestrationJobSnapshot | None): None -to_dict(): dict[str, JSONValue] + +OrchestrationSubmissionPreflight + +allowed : bool +blocking_snapshot : OrchestrationJobSnapshot | None +checked_job_count : int +checked_job_source : str +config +exit_code : int +message : str +offline : bool +overlap_guard_enabled : bool +request +schedule_identity : str +schedule_identity_source : str +state : str +status_store_path : Path + +__init__(self, request: RunRequest, config: OrchestrationWorkerConfig, status_store_path: Path, allowed: bool, overlap_guard_enabled: bool, schedule_identity_source: str, schedule_identity: str, checked_job_count: int, checked_job_source: str, offline: bool, blocking_snapshot: OrchestrationJobSnapshot | None): None +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.client.OrchestrationSubmissionPreflight->histdatacom.orchestration.queues.OrchestrationWorkerConfig - - -config + + +config - + histdatacom.orchestration.client.OrchestrationSubmissionPreflight->histdatacom.runtime_contracts.RunRequest - - -request + + +request - + histdatacom.orchestration.supervisor.OrchestrationSupervisor->histdatacom.orchestration.runtime.OrchestrationPaths - - -paths + + +paths - + histdatacom.orchestration.queues.OrchestrationTaskQueues - -OrchestrationTaskQueues - -cpu_file : str -influx : str -network : str -orchestration : str -prefix : str -workspace_id : str - -__init__(self, prefix: str, workspace_id: str, orchestration: str, network: str, cpu_file: str, influx: str): None -for_lane(lane: str | TaskQueueLane): str -to_dict(): dict[str, str] + +OrchestrationTaskQueues + +cpu_file : str +influx : str +network : str +orchestration : str +prefix : str +workspace_id : str + +__init__(self, prefix: str, workspace_id: str, orchestration: str, network: str, cpu_file: str, influx: str): None +for_lane(lane: str | TaskQueueLane): str +to_dict(): dict[str, str] - + histdatacom.orchestration.client.OrchestrationUnavailableError - -OrchestrationUnavailableError - -category : DEPENDENCY -code : str -exit_code : int -retryable : bool - - + +OrchestrationUnavailableError + +category : DEPENDENCY +code : str +exit_code : int +retryable : bool + + - + histdatacom.orchestration.client.OrchestrationUnavailableError->histdatacom.exceptions.DependencyOperationError - - + + - + histdatacom.orchestration.queues.OrchestrationWorkerConfig->histdatacom.orchestration.runtime.OrchestrationRuntimePolicy - - -runtime_policy + + +runtime_policy - + histdatacom.orchestration.queues.OrchestrationWorkerConfig->histdatacom.orchestration.queues.OrchestrationTaskQueues - - -task_queues + + +task_queues - + histdatacom.orchestration.queues.TaskQueueLane - -TaskQueueLane - -name - -from_value(value: str | 'TaskQueueLane'): 'TaskQueueLane' + +TaskQueueLane + +name + +from_value(value: str | 'TaskQueueLane'): 'TaskQueueLane' - + histdatacom.orchestration.queues.OrchestrationWorkerConfig->histdatacom.orchestration.queues.TaskQueueLane - - -lane + + +lane - + histdatacom.fx_enums.Pairs - -Pairs - -name - -list_keys(): set -list_values(): set + +Pairs + +name + +list_keys(): set +list_values(): set - + histdatacom.exceptions.ParseDataError - -ParseDataError - -category : PARSE -code : str -retryable : bool - - + +ParseDataError + +category : PARSE +code : str +retryable : bool + + - + histdatacom.exceptions.ParseDataError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.histdata_ascii.ParsedAsciiBatch - -ParsedAsciiBatch - -columns : tuple[str, ...] -rows : tuple[tuple[Any, ...], ...] -summary -timeframe : str - -__init__(self, timeframe: str, columns: tuple[str, ...], rows: tuple[tuple[Any, ...], ...], summary: CacheSummary): None + +ParsedAsciiBatch + +columns : tuple[str, ...] +rows : tuple[tuple[Any, ...], ...] +summary +timeframe : str + +__init__(self, timeframe: str, columns: tuple[str, ...], rows: tuple[tuple[Any, ...], ...], summary: CacheSummary): None - + histdatacom.histdata_ascii.ParsedAsciiBatch->histdatacom.histdata_ascii.CacheSummary - - -summary + + +summary + + + +histdatacom.synthetic.streaming.PartitionManifestV1 + +PartitionManifestV1 + +carry_state_ref +ensemble_member_id : str +event_batches : tuple[EventBatchV1, ...] +event_count : int +manifest_id : str +rejection_summary_ref +run_id : str +schema_version : str +symbol_event_counts : dict[str, int] +symbols : tuple[str, ...] +synchronization_unit_id : str +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, ensemble_member_id: str, symbols: tuple[str, ...], symbol_event_counts: dict[str, int], event_batches: tuple[EventBatchV1, ...], rejection_summary_ref: ArtifactRef, carry_state_ref: ArtifactRef, manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'PartitionManifestV1' +from_json(text: str): 'PartitionManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.streaming.PartitionManifestV1->histdatacom.runtime_contracts.ArtifactRef + + +rejection_summary_ref + + + +histdatacom.synthetic.streaming.PartitionManifestV1->histdatacom.runtime_contracts.ArtifactRef + + +carry_state_ref + + + +histdatacom.resource_usage.PeakRssMeasurement + +PeakRssMeasurement + +available : bool +bytes : int +source : str + +__init__(self, bytes: int, source: str, available: bool): None - + histdatacom.orchestration.runtime.PortAllocationError - -PortAllocationError - - - + +PortAllocationError + + + - + histdatacom.observability.ProgressState - -ProgressState - -completed : float -event_sink : ProgressEventSink | None -events : tuple[StatusEvent, ...] -last_error : str -percent_complete : float -rate_per_second : float -stage : str -started_at_utc : str -status -total : float -unit : str -updated_at_utc : str -work_id : str - -__init__(self, stage: str, total: float, completed: float, unit: str, status: WorkStatus, work_id: str, started_at_utc: str, updated_at_utc: str, last_error: str, events: tuple[StatusEvent, ...], event_sink: ProgressEventSink | None): None -_record_event(event: StatusEvent): StatusEvent -advance(increment: float): StatusEvent -fail(error: BaseException | str): StatusEvent -to_dict(): dict[str, JSONValue] + +ProgressState + +completed : float +event_sink : ProgressEventSink | None +events : tuple[StatusEvent, ...] +last_error : str +percent_complete : float +rate_per_second : float +stage : str +started_at_utc : str +status +total : float +unit : str +updated_at_utc : str +work_id : str + +__init__(self, stage: str, total: float, completed: float, unit: str, status: WorkStatus, work_id: str, started_at_utc: str, updated_at_utc: str, last_error: str, events: tuple[StatusEvent, ...], event_sink: ProgressEventSink | None): None +_record_event(event: StatusEvent): StatusEvent +advance(increment: float): StatusEvent +fail(error: BaseException | str): StatusEvent +to_dict(): dict[str, JSONValue] - + histdatacom.observability.ProgressState->histdatacom.runtime_contracts.WorkStatus - - -status + + +status + + + +histdatacom.synthetic.bars.PublishedDerivedBarsV1 + +PublishedDerivedBarsV1 + +idempotent_retry : bool +manifest +manifest_path : Path +manifest_ref + +__init__(self, manifest: DerivedBarProductManifestV1, manifest_path: Path, manifest_ref: ArtifactRef, idempotent_retry: bool): None + + + +histdatacom.synthetic.bars.PublishedDerivedBarsV1->histdatacom.runtime_contracts.ArtifactRef + + +manifest_ref + + + +histdatacom.synthetic.bars.PublishedDerivedBarsV1->histdatacom.synthetic.bars.DerivedBarProductManifestV1 + + +manifest + + + +histdatacom.synthetic.persistence.PublishedReconstructionV1 + +PublishedReconstructionV1 + +idempotent_retry : bool +manifest +manifest_path : Path +manifest_ref + +__init__(self, manifest: ReconstructionProductManifestV1, manifest_path: Path, manifest_ref: ArtifactRef, idempotent_retry: bool): None + + + +histdatacom.synthetic.persistence.PublishedReconstructionV1->histdatacom.runtime_contracts.ArtifactRef + + +manifest_ref + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV1 + +ReconstructionProductManifestV1 + +broker_profile_id : str +constraints +ensemble +ensemble_member_id : str +event_count : int +manifest_id : str +max_event_time_ns : int +min_event_time_ns : int +observed_event_count : int +partitions : tuple[ReconstructionProductPartitionV1, ...] +publication_id : str +quality +replay +retention +run_id : str +schema_version : str +source +symbol_event_counts : Mapping[str, int] +symbol_group_id : str +symbols : tuple[str, ...] +synchronization_unit_id : str +synthetic_event_count : int +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, ensemble_member_id: str, broker_profile_id: str, symbol_group_id: str, symbols: tuple[str, ...], symbol_event_counts: Mapping[str, int], partitions: tuple[ReconstructionProductPartitionV1, ...], source: ReconstructionSourceManifestV1, constraints: ReconstructionConstraintManifestV1, quality: ReconstructionQualityManifestV1, replay: ReconstructionReplayManifestV1, ensemble: ReconstructionEnsembleManifestV1, retention: ReconstructionRetentionPlanV1, publication_id: str, manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionProductManifestV1' +from_json(text: str): 'ReconstructionProductManifestV1' +payload(): dict[str, JSONValue] +publication_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.persistence.PublishedReconstructionV1->histdatacom.synthetic.persistence.ReconstructionProductManifestV1 + + +manifest + + + +histdatacom.synthetic.persistence.PublishedReconstructionV2 + +PublishedReconstructionV2 + +idempotent_retry : bool +manifest +manifest_path : Path +manifest_ref + +__init__(self, manifest: ReconstructionProductManifestV2, manifest_path: Path, manifest_ref: ArtifactRef, idempotent_retry: bool): None + + + +histdatacom.synthetic.persistence.PublishedReconstructionV2->histdatacom.runtime_contracts.ArtifactRef + + +manifest_ref + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV2 + +ReconstructionProductManifestV2 + +constraints +delivery_profile_id : str +ensemble +ensemble_member_id : str +event_count : int +manifest_id : str +max_event_time_ns : int +min_event_time_ns : int +observed_event_count : int +partitions : tuple[ReconstructionProductPartitionV1, ...] +publication_id : str +quality +replay +retention +run_id : str +schema_version : str +source +symbol_event_counts : Mapping[str, int] +symbol_group_id : str +symbols : tuple[str, ...] +synchronization_unit_id : str +synthetic_event_count : int +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, ensemble_member_id: str, delivery_profile_id: str, symbol_group_id: str, symbols: tuple[str, ...], symbol_event_counts: Mapping[str, int], partitions: tuple[ReconstructionProductPartitionV1, ...], source: ReconstructionSourceManifestV1, constraints: ReconstructionConstraintManifestV1, quality: ReconstructionDeliveryQualityManifestV1, replay: ReconstructionReplayManifestV1, ensemble: ReconstructionEnsembleManifestV1, retention: ReconstructionRetentionPlanV1, publication_id: str, manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionProductManifestV2' +from_json(text: str): 'ReconstructionProductManifestV2' +payload(): dict[str, JSONValue] +publication_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.persistence.PublishedReconstructionV2->histdatacom.synthetic.persistence.ReconstructionProductManifestV2 + + +manifest - + histdatacom.data_quality.discovery.QualityDiscoveryError - -QualityDiscoveryError - - - + +QualityDiscoveryError + + + - + histdatacom.data_quality.discovery.QualityDiscoveryResult - -QualityDiscoveryResult - -metadata : dict[str, JSONValue] -roots : tuple[str, ...] -target_count : int -targets : tuple[QualityTarget, ...] - -__init__(self, roots: tuple[str, ...], targets: tuple[QualityTarget, ...], metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'QualityDiscoveryResult' -to_dict(): dict[str, JSONValue] + +QualityDiscoveryResult + +metadata : dict[str, JSONValue] +roots : tuple[str, ...] +target_count : int +targets : tuple[QualityTarget, ...] + +__init__(self, roots: tuple[str, ...], targets: tuple[QualityTarget, ...], metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'QualityDiscoveryResult' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.reporting.QualityExitDecision - -QualityExitDecision - -exit_code : int -policy -reason : str - -__init__(self, exit_code: int, reason: str, policy: QualityExitPolicy): None -to_dict(): dict[str, JSONValue] + +QualityExitDecision + +exit_code : int +policy +reason : str + +__init__(self, exit_code: int, reason: str, policy: QualityExitPolicy): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.reporting.QualityExitPolicy - -QualityExitPolicy - -fail_on -max_errors : int -max_warnings : int - -__init__(self, fail_on: QualityExitTrigger, max_errors: int, max_warnings: int): None -evaluate(summary: QualityRunSummary): 'QualityExitDecision' -from_values(): 'QualityExitPolicy' -to_dict(): dict[str, JSONValue] + +QualityExitPolicy + +fail_on +max_errors : int +max_warnings : int + +__init__(self, fail_on: QualityExitTrigger, max_errors: int, max_warnings: int): None +evaluate(summary: QualityRunSummary): 'QualityExitDecision' +from_values(): 'QualityExitPolicy' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.reporting.QualityExitDecision->histdatacom.data_quality.reporting.QualityExitPolicy - - -policy + + +policy - + histdatacom.data_quality.reporting.QualityExitTrigger - -QualityExitTrigger - -name - -from_value(value: str | 'QualityExitTrigger' | None): 'QualityExitTrigger' + +QualityExitTrigger + +name + +from_value(value: str | 'QualityExitTrigger' | None): 'QualityExitTrigger' - + histdatacom.data_quality.reporting.QualityExitPolicy->histdatacom.data_quality.reporting.QualityExitTrigger - - -fail_on + + +fail_on - + histdatacom.data_quality.contracts.QualityFinding - -QualityFinding - -code : str -location -message : str -metadata : dict[str, JSONValue] -rule_id : str -severity -target - -__init__(self, severity: QualitySeverity, code: str, message: str, rule_id: str, target: QualityTarget, location: QualityLocation, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'QualityFinding' -to_dict(): dict[str, JSONValue] + +QualityFinding + +code : str +location +message : str +metadata : dict[str, JSONValue] +rule_id : str +severity +target + +__init__(self, severity: QualitySeverity, code: str, message: str, rule_id: str, target: QualityTarget, location: QualityLocation, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'QualityFinding' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.contracts.QualityLocation - -QualityLocation - -column : str -metadata : dict[str, JSONValue] -path : str -row_number : int | None -timestamp_source : str -timestamp_utc_ms : int | None - -__init__(self, path: str, row_number: int | None, timestamp_source: str, timestamp_utc_ms: int | None, column: str, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any] | None): 'QualityLocation' -to_dict(): dict[str, JSONValue] + +QualityLocation + +column : str +metadata : dict[str, JSONValue] +path : str +row_number : int | None +timestamp_source : str +timestamp_utc_ms : int | None + +__init__(self, path: str, row_number: int | None, timestamp_source: str, timestamp_utc_ms: int | None, column: str, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any] | None): 'QualityLocation' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.contracts.QualityFinding->histdatacom.data_quality.contracts.QualityLocation - - -location + + +location - + histdatacom.data_quality.contracts.QualityFinding->histdatacom.data_quality.contracts.QualitySeverity - - -severity + + +severity - + histdatacom.data_quality.contracts.QualityTarget - -QualityTarget - -data_format : str -kind -metadata : dict[str, JSONValue] -path : str -period : str -symbol : str -timeframe : str - -__init__(self, path: str, kind: QualityTargetKind, data_format: str, timeframe: str, symbol: str, period: str, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'QualityTarget' -to_dict(): dict[str, JSONValue] + +QualityTarget + +data_format : str +kind +metadata : dict[str, JSONValue] +path : str +period : str +symbol : str +timeframe : str + +__init__(self, path: str, kind: QualityTargetKind, data_format: str, timeframe: str, symbol: str, period: str, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'QualityTarget' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.contracts.QualityFinding->histdatacom.data_quality.contracts.QualityTarget - - -target + + +target - + histdatacom.data_quality.profiles.QualityProfile - -QualityProfile - -is_default : bool -modeling_assumptions : Mapping[str, JSONValue] -name : str -reporting : Mapping[str, JSONValue] -rules : Mapping[str, Mapping[str, JSONValue]] -schema_version : str -source : str -source_path : str - -__init__(self, schema_version: str, name: str, source: str, source_path: str, rules: Mapping[str, Mapping[str, JSONValue]], reporting: Mapping[str, JSONValue], modeling_assumptions: Mapping[str, JSONValue]): None -__post_init__(): None -calendar_profile(): HistDataCalendarProfile -cross_instrument_tolerance(): HistDataCrossInstrumentTolerance -fingerprint_profile(): HistDataFingerprintProfile -gap_tolerance(rule_id: str): HistDataGapTolerance -modeling_profile_assumptions(): dict[str, JSONValue] -reporting_profile(): QualityReportingProfile -row_count_profile(): HistDataRowCountProfile -rule_config(rule_id: str): Mapping[str, JSONValue] -severity(rule_id: str, key: str, default: QualitySeverity): QualitySeverity -tick_microstructure_session_name(): str -tick_microstructure_thresholds(): HistDataTickMicrostructureThresholds -tick_microstructure_thresholds_by_asset_class(): dict[str, HistDataTickMicrostructureThresholds] -tick_microstructure_thresholds_by_session(): dict[str, HistDataTickMicrostructureThresholds] -tick_microstructure_thresholds_by_symbol(): dict[str, HistDataTickMicrostructureThresholds] -tick_microstructure_thresholds_by_symbol_session(): dict[str, HistDataTickMicrostructureThresholds] -tick_spread_regime_thresholds(): HistDataTickSpreadRegimeThresholds -tick_spread_regime_thresholds_by_asset_class(): dict[str, HistDataTickSpreadRegimeThresholds] -tick_spread_thresholds(): HistDataTickSpreadThresholds -tick_spread_thresholds_by_asset_class(): dict[str, HistDataTickSpreadThresholds] -to_metadata(): dict[str, JSONValue] -to_request_payload(): dict[str, JSONValue] + +QualityProfile + +is_default : bool +modeling_assumptions : Mapping[str, JSONValue] +name : str +reporting : Mapping[str, JSONValue] +rules : Mapping[str, Mapping[str, JSONValue]] +schema_version : str +source : str +source_path : str + +__init__(self, schema_version: str, name: str, source: str, source_path: str, rules: Mapping[str, Mapping[str, JSONValue]], reporting: Mapping[str, JSONValue], modeling_assumptions: Mapping[str, JSONValue]): None +__post_init__(): None +calendar_profile(): HistDataCalendarProfile +cross_instrument_tolerance(): HistDataCrossInstrumentTolerance +fingerprint_profile(): HistDataFingerprintProfile +gap_tolerance(rule_id: str): HistDataGapTolerance +modeling_profile_assumptions(): dict[str, JSONValue] +reporting_profile(): QualityReportingProfile +row_count_profile(): HistDataRowCountProfile +rule_config(rule_id: str): Mapping[str, JSONValue] +severity(rule_id: str, key: str, default: QualitySeverity): QualitySeverity +tick_microstructure_session_name(): str +tick_microstructure_thresholds(): HistDataTickMicrostructureThresholds +tick_microstructure_thresholds_by_asset_class(): dict[str, HistDataTickMicrostructureThresholds] +tick_microstructure_thresholds_by_session(): dict[str, HistDataTickMicrostructureThresholds] +tick_microstructure_thresholds_by_symbol(): dict[str, HistDataTickMicrostructureThresholds] +tick_microstructure_thresholds_by_symbol_session(): dict[str, HistDataTickMicrostructureThresholds] +tick_spread_regime_thresholds(): HistDataTickSpreadRegimeThresholds +tick_spread_regime_thresholds_by_asset_class(): dict[str, HistDataTickSpreadRegimeThresholds] +tick_spread_thresholds(): HistDataTickSpreadThresholds +tick_spread_thresholds_by_asset_class(): dict[str, HistDataTickSpreadThresholds] +to_metadata(): dict[str, JSONValue] +to_request_payload(): dict[str, JSONValue] - + histdatacom.data_quality.profiles.QualityProfileError - -QualityProfileError - - - + +QualityProfileError + + + + + + +histdatacom.data_quality.profiles.QualityProfileResolution + +QualityProfileResolution + +input_channels : tuple[dict[str, JSONValue], ...] +profile +schema_version : str +value_sources : tuple[QualityProfileValueSource, ...] + +__init__(self, profile: QualityProfile, value_sources: tuple[QualityProfileValueSource, ...], input_channels: tuple[dict[str, JSONValue], ...], schema_version: str): None +to_payload(): dict[str, JSONValue] + + + +histdatacom.data_quality.profiles.QualityProfileResolution->histdatacom.data_quality.profiles.QualityProfile + + +profile + + + +histdatacom.data_quality.profiles.QualityProfileValueSource + +QualityProfileValueSource + +override : bool +path : str +previous_source : str +previous_value : JSONValue | None +previous_value_present : bool +profile_name : str +selected_by : str +source : str +source_path : str +value + +__init__(self, path: str, value: JSONValue, source: str, profile_name: str, source_path: str, selected_by: str, override: bool, previous_source: str, previous_value: JSONValue | None, previous_value_present: bool): None +to_payload(): dict[str, JSONValue] - + histdatacom.data_quality.profiles.QualityRemediationCatalogAuditProfile - -QualityRemediationCatalogAuditProfile - -enabled : bool - -__init__(self, enabled: bool): None -to_metadata(): dict[str, JSONValue] + +QualityRemediationCatalogAuditProfile + +enabled : bool + +__init__(self, enabled: bool): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.remediation.QualityRemediationHint - -QualityRemediationHint - -action_kind : str -code : str -finding_code : str -flag : str -message : str -rule_id : str - -__init__(self, code: str, message: str, action_kind: str, rule_id: str, flag: str, finding_code: str): None -to_payload(): dict[str, JSONValue] + +QualityRemediationHint + +action_kind : str +code : str +finding_code : str +flag : str +message : str +policy_context : Mapping[str, JSONValue] +rule_id : str + +__init__(self, code: str, message: str, action_kind: str, rule_id: str, flag: str, finding_code: str, policy_context: Mapping[str, JSONValue]): None +to_payload(): dict[str, JSONValue] - + histdatacom.data_quality.contracts.QualityReport - -QualityReport - -findings : tuple[QualityFinding, ...] -max_severity : QualitySeverity -metadata : dict[str, JSONValue] -rule_count : int -rule_results : tuple[QualityRuleResult, ...] -status : QualityStatus -target_summaries : tuple[QualityTargetSummary, ...] -targets : tuple[QualityTarget, ...] - -__init__(self, targets: tuple[QualityTarget, ...], rule_results: tuple[QualityRuleResult, ...], metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'QualityReport' -summary(): QualityRunSummary -to_dict(): dict[str, JSONValue] + +QualityReport + +findings : tuple[QualityFinding, ...] +max_severity : QualitySeverity +metadata : dict[str, JSONValue] +rule_count : int +rule_results : tuple[QualityRuleResult, ...] +status : QualityStatus +target_summaries : tuple[QualityTargetSummary, ...] +targets : tuple[QualityTarget, ...] + +__init__(self, targets: tuple[QualityTarget, ...], rule_results: tuple[QualityRuleResult, ...], metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'QualityReport' +summary(): QualityRunSummary +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.profiles.QualityReportingProfile - -QualityReportingProfile - -remediation_catalog_audit - -__init__(self, remediation_catalog_audit: QualityRemediationCatalogAuditProfile): None -to_metadata(): dict[str, JSONValue] + +QualityReportingProfile + +remediation_catalog_audit + +__init__(self, remediation_catalog_audit: QualityRemediationCatalogAuditProfile): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.profiles.QualityReportingProfile->histdatacom.data_quality.profiles.QualityRemediationCatalogAuditProfile - - -remediation_catalog_audit + + +remediation_catalog_audit - + histdatacom.data_quality.contracts.QualityRule - -QualityRule - -description : str -rule_id : str - -evaluate -(target: QualityTarget): Iterable[QualityFinding] + +QualityRule + +description : str +rule_id : str + +evaluate +(target: QualityTarget): Iterable[QualityFinding] - + histdatacom.data_quality.contracts.QualityRuleResult - -QualityRuleResult - -findings : tuple[QualityFinding, ...] -max_severity : QualitySeverity -rule_id : str -status : QualityStatus -target - -__init__(self, rule_id: str, target: QualityTarget, findings: tuple[QualityFinding, ...]): None -from_dict(data: Mapping[str, Any]): 'QualityRuleResult' -to_dict(): dict[str, JSONValue] + +QualityRuleResult + +findings : tuple[QualityFinding, ...] +max_severity : QualitySeverity +rule_id : str +status : QualityStatus +target + +__init__(self, rule_id: str, target: QualityTarget, findings: tuple[QualityFinding, ...]): None +from_dict(data: Mapping[str, Any]): 'QualityRuleResult' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.contracts.QualityRuleResult->histdatacom.data_quality.contracts.QualityTarget - - -target + + +target - + histdatacom.data_quality.contracts.QualityRunRule - -QualityRunRule - -description : str -rule_id : str - -evaluate_run -(targets: Iterable[QualityTarget]): 'QualityReport' + +QualityRunRule + +description : str +rule_id : str + +evaluate_run +(targets: Iterable[QualityTarget]): 'QualityReport' - + histdatacom.data_quality.contracts.QualityRunSummary - -QualityRunSummary - -error_count : int -finding_count : int -info_count : int -max_severity -rule_count : int -status -target_count : int -warning_count : int - -__init__(self, target_count: int, rule_count: int, finding_count: int, info_count: int, warning_count: int, error_count: int, status: QualityStatus, max_severity: QualitySeverity): None -from_dict(data: Mapping[str, Any]): 'QualityRunSummary' -to_dict(): dict[str, JSONValue] + +QualityRunSummary + +error_count : int +finding_count : int +info_count : int +max_severity +rule_count : int +status +target_count : int +warning_count : int + +__init__(self, target_count: int, rule_count: int, finding_count: int, info_count: int, warning_count: int, error_count: int, status: QualityStatus, max_severity: QualitySeverity): None +from_dict(data: Mapping[str, Any]): 'QualityRunSummary' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.contracts.QualityRunSummary->histdatacom.data_quality.contracts.QualitySeverity - - -max_severity + + +max_severity - + histdatacom.data_quality.contracts.QualityStatus - -QualityStatus - -name - -from_severity_counts(): 'QualityStatus' -from_value(value: str | 'QualityStatus' | None): 'QualityStatus' + +QualityStatus + +name + +from_severity_counts(): 'QualityStatus' +from_value(value: str | 'QualityStatus' | None): 'QualityStatus' - + histdatacom.data_quality.contracts.QualityRunSummary->histdatacom.data_quality.contracts.QualityStatus - - -status + + +status + + + +histdatacom.data_quality.contracts.QualitySkipEvent + +QualitySkipEvent + +data_format : str +period : str +reason_code : str +rule_id : str +sort_key : tuple[str, str, str, str, str, str, str] +symbol : str +target_kind +timeframe : str + +__init__(self, reason_code: str, rule_id: str, target_kind: QualityTargetKind, data_format: str, timeframe: str, symbol: str, period: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'QualitySkipEvent' +from_target(): 'QualitySkipEvent' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.contracts.QualityTargetKind - -QualityTargetKind - -name - -from_value(value: str | 'QualityTargetKind' | None): 'QualityTargetKind' + +QualityTargetKind + +name + +from_value(value: str | 'QualityTargetKind' | None): 'QualityTargetKind' + + + +histdatacom.data_quality.contracts.QualitySkipEvent->histdatacom.data_quality.contracts.QualityTargetKind + + +target_kind - + histdatacom.data_quality.contracts.QualityTarget->histdatacom.data_quality.contracts.QualityTargetKind - - -kind + + +kind - + histdatacom.data_quality.contracts.QualityTargetSummary - -QualityTargetSummary - -error_count : int -finding_count : int -info_count : int -max_severity -rule_count : int -status -target -warning_count : int - -__init__(self, target: QualityTarget, rule_count: int, finding_count: int, info_count: int, warning_count: int, error_count: int, status: QualityStatus, max_severity: QualitySeverity): None -from_dict(data: Mapping[str, Any]): 'QualityTargetSummary' -to_dict(): dict[str, JSONValue] + +QualityTargetSummary + +error_count : int +finding_count : int +info_count : int +max_severity +rule_count : int +status +target +warning_count : int + +__init__(self, target: QualityTarget, rule_count: int, finding_count: int, info_count: int, warning_count: int, error_count: int, status: QualityStatus, max_severity: QualitySeverity): None +from_dict(data: Mapping[str, Any]): 'QualityTargetSummary' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.contracts.QualityTargetSummary->histdatacom.data_quality.contracts.QualitySeverity - - -max_severity + + +max_severity - + histdatacom.data_quality.contracts.QualityTargetSummary->histdatacom.data_quality.contracts.QualityStatus - - -status + + +status - + histdatacom.data_quality.contracts.QualityTargetSummary->histdatacom.data_quality.contracts.QualityTarget - - -target + + +target + + + +histdatacom.random_windows.RandomWindowEmptySelectionError + +RandomWindowEmptySelectionError + + + + + + +histdatacom.random_windows.RandomWindowError + +RandomWindowError + + + + + + +histdatacom.random_windows.RandomWindowEmptySelectionError->histdatacom.random_windows.RandomWindowError + + + + + +histdatacom.random_windows.RandomWindowExpressionV1 + +RandomWindowExpressionV1 + +bridge_count : int | None +bridge_unit : str +duration_count : int | None +duration_unit : str +end_session : str +expression : str +has_session : bool +prefix_minutes : int +schema_version : str +start_session : str +suffix_minutes : int + +__init__(self, expression: str, duration_count: int | None, duration_unit: str, prefix_minutes: int, start_session: str, bridge_count: int | None, bridge_unit: str, end_session: str, suffix_minutes: int, schema_version: str): None +to_dict(): dict[str, Any] + + + +histdatacom.random_windows.RandomWindowSelectionV1 + +RandomWindowSelectionV1 + +expression : str +mode : str +occurrence_count : int +schema_version : str +seed : int | None +selected_end_utc_ms : int | None +selected_start_utc_ms : int | None +selection_id : str +support_end_utc_ms : int +support_start_utc_ms : int + +__init__(self, expression: str, mode: str, support_start_utc_ms: int, support_end_utc_ms: int, seed: int | None, selected_start_utc_ms: int | None, selected_end_utc_ms: int | None, occurrence_count: int, selection_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'RandomWindowSelectionV1' +identity_payload(): dict[str, Any] +to_dict(): dict[str, Any] + + + +histdatacom.random_windows.RandomWindowSessionProfileV1 + +RandomWindowSessionProfileV1 + +code : str +end_minute_local : int +label : str +schema_version : str +start_minute_local : int +timezone_name : str + +__init__(self, code: str, label: str, timezone_name: str, start_minute_local: int, end_minute_local: int, schema_version: str): None +__post_init__(): None +to_dict(): dict[str, Any] + + + +histdatacom.random_windows.RandomWindowSupportError + +RandomWindowSupportError + + + + + + +histdatacom.random_windows.RandomWindowSupportError->histdatacom.random_windows.RandomWindowError + + + + + +histdatacom.random_windows.RandomWindowSyntaxError + +RandomWindowSyntaxError + + + + + + +histdatacom.random_windows.RandomWindowSyntaxError->histdatacom.random_windows.RandomWindowError + + - + histdatacom.readme_help.ReadmeHelpSyncError - -ReadmeHelpSyncError - - - + +ReadmeHelpSyncError + + + + + + +histdatacom.synthetic.activity.ReconstructionActivityBenchmarkEvidenceV1 + +ReconstructionActivityBenchmarkEvidenceV1 + +calibration_supported_candidate_count : int +candidate_score_ids : tuple[str, ...] +evidence_id : str +execution_failure_count : int +mean_restoration_gain_vs_degraded : float +metric_mean_errors : Mapping[str, float] +metric_support_counts : Mapping[str, int] +promotion_eligible_candidate_count : int +schema_version : str +scorecard_id : str +split_kinds : tuple[str, ...] + +__init__(self, scorecard_id: str, candidate_score_ids: tuple[str, ...], split_kinds: tuple[str, ...], metric_support_counts: Mapping[str, int], metric_mean_errors: Mapping[str, float], mean_restoration_gain_vs_degraded: float, promotion_eligible_candidate_count: int, calibration_supported_candidate_count: int, execution_failure_count: int, evidence_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionActivityBenchmarkEvidenceV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.activity.ReconstructionActivityManifestV1 + +ReconstructionActivityManifestV1 + +as_of_ns : int | None +benchmark_evidence : ReconstructionActivityBenchmarkEvidenceV1 | None +calibration_report_id : str | None +ensemble_member_id : str +event_count : int +information_manifest_id : str +information_mode +input_content_sha256 : str +manifest_id : str +policy +product_manifest_id : str | None +run_id : str +schema_version : str +slices : tuple[ReconstructionActivitySliceV1, ...] +symbols : tuple[str, ...] +synchronization_unit_id : str | None +window_id : str | None + +__init__(self, run_id: str, ensemble_member_id: str, information_mode: InformationMode, information_manifest_id: str, as_of_ns: int | None, policy: ReconstructionActivityPolicyV1, slices: tuple[ReconstructionActivitySliceV1, ...], input_content_sha256: str, window_id: str | None, synchronization_unit_id: str | None, product_manifest_id: str | None, calibration_report_id: str | None, benchmark_evidence: ReconstructionActivityBenchmarkEvidenceV1 | None, manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionActivityManifestV1' +from_json(text: str): 'ReconstructionActivityManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.activity.ReconstructionActivityManifestV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.activity.ReconstructionActivityPolicyV1 + +ReconstructionActivityPolicyV1 + +max_payload_bytes : int +max_provenance_values : int +max_slices : int +max_symbols : int +policy_id : str +rounding_digits : int +schema_version : str +scopes : tuple[ActivitySliceScope, ...] +volume_state + +__init__(self, scopes: tuple[ActivitySliceScope, ...], volume_state: ActivityVolumeState, rounding_digits: int, max_symbols: int, max_slices: int, max_provenance_values: int, max_payload_bytes: int, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionActivityPolicyV1' +from_json(text: str): 'ReconstructionActivityPolicyV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.activity.ReconstructionActivityManifestV1->histdatacom.synthetic.activity.ReconstructionActivityPolicyV1 + + +policy + + + +histdatacom.synthetic.activity.ReconstructionActivityMetricV1 + +ReconstructionActivityMetricV1 + +aggregation +confidence : float | None +limitations : tuple[str, ...] +name : str +schema_version : str +semantics +support_count : int +unit : str +value : int | float | None + +__init__(self, name: str, value: int | float | None, unit: str, aggregation: ActivityAggregationSemantics, semantics: ActivityMetricSemantics, support_count: int, confidence: float | None, limitations: tuple[str, ...], schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionActivityMetricV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.activity.ReconstructionActivityMetricV1->histdatacom.synthetic.activity.ActivityAggregationSemantics + + +aggregation + + + +histdatacom.synthetic.activity.ReconstructionActivityMetricV1->histdatacom.synthetic.activity.ActivityMetricSemantics + + +semantics + + + +histdatacom.synthetic.activity.ReconstructionActivityPolicyV1->histdatacom.synthetic.activity.ActivityVolumeState + + +volume_state + + + +histdatacom.synthetic.activity.ReconstructionActivitySliceV1 + +ReconstructionActivitySliceV1 + +broker_profile_ids : tuple[str, ...] +confidence_support_count : int +constraint_set_ids : tuple[str, ...] +end_event_time_ns : int +event_content_sha256 : str +event_count : int +feed_epoch_ids : tuple[str, ...] +generator_config_ids : tuple[str, ...] +generator_ids : tuple[str, ...] +generator_versions : tuple[str, ...] +limitations : tuple[str, ...] +metric_by_name : dict[str, ReconstructionActivityMetricV1] +metrics : tuple[ReconstructionActivityMetricV1, ...] +motif_ids : tuple[str, ...] +reference_ids : tuple[str, ...] +schema_version : str +scope +slice_id : str +source_version_ids : tuple[str, ...] +start_event_time_ns : int +stream_ids : tuple[str, ...] +symbol : str + +__init__(self, symbol: str, scope: ActivitySliceScope, start_event_time_ns: int, end_event_time_ns: int, metrics: tuple[ReconstructionActivityMetricV1, ...], source_version_ids: tuple[str, ...], generator_ids: tuple[str, ...], generator_versions: tuple[str, ...], generator_config_ids: tuple[str, ...], reference_ids: tuple[str, ...], motif_ids: tuple[str, ...], feed_epoch_ids: tuple[str, ...], broker_profile_ids: tuple[str, ...], constraint_set_ids: tuple[str, ...], stream_ids: tuple[str, ...], event_content_sha256: str, confidence_support_count: int, limitations: tuple[str, ...], slice_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionActivitySliceV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.activity.ReconstructionActivitySliceV1->histdatacom.synthetic.activity.ActivitySliceScope + + +scope + + + +histdatacom.orchestration.reconstruction.ReconstructionArtifactError + +ReconstructionArtifactError + + + + + + +histdatacom.synthetic.certification.ReconstructionCertificationDossierV1 + +ReconstructionCertificationDossierV1 + +accepted_limitations : tuple[str, ...] +artifacts : tuple[CertificationArtifactV1, ...] +blocking_limitations : tuple[str, ...] +certified : bool +dossier_id : str +gate_results : tuple[CertificationGateResultV1, ...] +methodology : str +policy +ready_for_promotion : bool +schema_version : str +state +summary : dict[str, JSONValue] + +__init__(self, policy: ReconstructionCertificationPolicyV1, artifacts: tuple[CertificationArtifactV1, ...], gate_results: tuple[CertificationGateResultV1, ...], methodology: str, accepted_limitations: tuple[str, ...], blocking_limitations: tuple[str, ...], state: CertificationState, dossier_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionCertificationDossierV1' +from_json(text: str): 'ReconstructionCertificationDossierV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str +to_markdown(): str + + + +histdatacom.synthetic.certification.ReconstructionCertificationDossierV1->histdatacom.synthetic.certification.CertificationState + + +state + + + +histdatacom.synthetic.certification.ReconstructionCertificationPolicyV1 + +ReconstructionCertificationPolicyV1 + +broker_fingerprint_id : str +common_end_period : str +common_start_period : str +max_artifacts : int +max_observations : int +max_payload_bytes : int +policy_id : str +product_version : str +requirements : tuple[CertificationRequirementV1, ...] +schema_version : str +symbols : tuple[str, ...] + +__init__(self, product_version: str, symbols: tuple[str, ...], common_start_period: str, common_end_period: str, broker_fingerprint_id: str, requirements: tuple[CertificationRequirementV1, ...], max_artifacts: int, max_observations: int, max_payload_bytes: int, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionCertificationPolicyV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification.ReconstructionCertificationDossierV1->histdatacom.synthetic.certification.ReconstructionCertificationPolicyV1 + + +policy + + + +histdatacom.synthetic.certification.ReconstructionCertificationDossierV2 + +ReconstructionCertificationDossierV2 + +accepted_limitations : tuple[str, ...] +artifacts : tuple[CertificationArtifactV1, ...] +blocking_limitations : tuple[str, ...] +certified : bool +dossier_id : str +gate_results : tuple[CertificationGateResultV1, ...] +methodology : str +policy +ready_for_promotion : bool +schema_version : str +state +summary : dict[str, JSONValue] + +__init__(self, policy: ReconstructionCertificationPolicyV2, artifacts: tuple[CertificationArtifactV1, ...], gate_results: tuple[CertificationGateResultV1, ...], methodology: str, accepted_limitations: tuple[str, ...], blocking_limitations: tuple[str, ...], state: CertificationState, dossier_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionCertificationDossierV2' +from_json(text: str): 'ReconstructionCertificationDossierV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str +to_markdown(): str + + + +histdatacom.synthetic.certification.ReconstructionCertificationDossierV2->histdatacom.synthetic.certification.CertificationState + + +state + + + +histdatacom.synthetic.certification.ReconstructionCertificationPolicyV2 + +ReconstructionCertificationPolicyV2 + +common_end_period : str +common_start_period : str +delivery_claim : str +delivery_mode : str +max_artifacts : int +max_observations : int +max_payload_bytes : int +policy_id : str +product_version : str +requirements : tuple[CertificationRequirementV1, ...] +schema_version : str +symbols : tuple[str, ...] + +__init__(self, product_version: str, symbols: tuple[str, ...], common_start_period: str, common_end_period: str, delivery_mode: str, delivery_claim: str, requirements: tuple[CertificationRequirementV1, ...], max_artifacts: int, max_observations: int, max_payload_bytes: int, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionCertificationPolicyV2' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.certification.ReconstructionCertificationDossierV2->histdatacom.synthetic.certification.ReconstructionCertificationPolicyV2 + + +policy + + + +histdatacom.orchestration.reconstruction.ReconstructionCheckpointConflict + +ReconstructionCheckpointConflict + + + + + + +histdatacom.synthetic.streaming.ReconstructionCheckpointV1 + +ReconstructionCheckpointV1 + +advertised_manifest_ref : ArtifactRef | None +carry_state_ref : ArtifactRef | None +checkpoint_id : str +committed_manifest_ref : ArtifactRef | None +completed_batch_ids : tuple[str, ...] +input_watermark_ns : int | None +interruption_reason : str +output_watermark_ns : int | None +parent_checkpoint_id : str | None +phase +rejection_summary_ref : ArtifactRef | None +revision : int +run_id : str +schema_version : str +staged_manifest_ref : ArtifactRef | None +synchronization_unit_id : str +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, revision: int, phase: ReconstructionCommitPhase, input_watermark_ns: int | None, output_watermark_ns: int | None, completed_batch_ids: tuple[str, ...], carry_state_ref: ArtifactRef | None, rejection_summary_ref: ArtifactRef | None, staged_manifest_ref: ArtifactRef | None, committed_manifest_ref: ArtifactRef | None, parent_checkpoint_id: str | None, interruption_reason: str, checkpoint_id: str, schema_version: str): None +__post_init__(): None +_validate_batch_scope(batch: EventBatchV1): None +_validate_phase_state(): None +assert_within(policy: ReconstructionStoragePolicyV1): 'ReconstructionCheckpointV1' +from_dict(data: Mapping[str, Any]): 'ReconstructionCheckpointV1' +from_json(text: str): 'ReconstructionCheckpointV1' +identity_payload(): dict[str, JSONValue] +pending_batches(batches: Sequence[EventBatchV1]): tuple[EventBatchV1, ...] +planned(window: ReconstructionWindowV1): 'ReconstructionCheckpointV1' +to_dict(): dict[str, JSONValue] +to_json(): str +transition(phase: ReconstructionCommitPhase | str): 'ReconstructionCheckpointV1' + + + +histdatacom.synthetic.streaming.ReconstructionCommitPhase + +ReconstructionCommitPhase + +name + +from_value(value: str | 'ReconstructionCommitPhase'): 'ReconstructionCommitPhase' + + + +histdatacom.synthetic.streaming.ReconstructionCheckpointV1->histdatacom.synthetic.streaming.ReconstructionCommitPhase + + +phase + + + +histdatacom.reconstruction.ReconstructionClient + +ReconstructionClient + +config : NoneType +supervisor : NoneType +temporal_client : NoneType + +__init__(): None +_construct_plan_model(spec: ReconstructionPlanSpecV1): SyntheticInfillPlanV1 +_executable_plan(request: ReconstructionExecutionRequestV1): SyntheticInfillPlanV1 +cancel(receipt: ReconstructionOperationReceiptV1): ReconstructionOperationReceiptV1 +cancel_async(receipt: ReconstructionOperationReceiptV1): ReconstructionOperationReceiptV1 +certify(spec_path: str | Path): tuple[ReconstructionCertificationDossierV2, ModernReferenceCertificationCampaignResultV1] +construct_plan(spec: ReconstructionPlanSpecV1): ArtifactRef +construct_plan_set(spec: ReconstructionPlanSpecV1): ArtifactRef +create_request(plan_path: str | Path): ReconstructionExecutionRequestV1 +execute_local(request: ReconstructionExecutionRequestV1): ReconstructionOperationReceiptV1 +inspect(receipt: ReconstructionOperationReceiptV1): ReconstructionOperationReceiptV1 +inspect_async(receipt: ReconstructionOperationReceiptV1): ReconstructionOperationReceiptV1 +outputs(request: ReconstructionExecutionRequestV1): dict[str, JSONValue] +preflight(request: ReconstructionExecutionRequestV1): ReconstructionPreflightV1 +preflight_plan_set(plan_set_path: str | Path): ReconstructionPlanSetPreflightV1 +preview(manifest_path: str | Path): dict[str, JSONValue] +replay(manifest_path: str | Path): dict[str, JSONValue] +resume(receipt: ReconstructionOperationReceiptV1): ReconstructionOperationReceiptV1 +submit(request: ReconstructionExecutionRequestV1): ReconstructionOperationReceiptV1 +submit_async(request: ReconstructionExecutionRequestV1): ReconstructionOperationReceiptV1 + + + +histdatacom.synthetic.persistence.ReconstructionConstraintManifestV1 + +ReconstructionConstraintManifestV1 + +constraint_manifest_id : str +constraint_set_ids : tuple[str, ...] +feed_epoch_ids : tuple[str, ...] +generator_config_ids : tuple[str, ...] +generator_ids : tuple[str, ...] +generator_versions : tuple[str, ...] +motif_assignment_count : int +motif_assignments_sha256 : str +reference_assignment_count : int +reference_assignments_sha256 : str +schema_version : str +synthetic_event_count : int + +__init__(self, synthetic_event_count: int, constraint_set_ids: tuple[str, ...], generator_ids: tuple[str, ...], generator_versions: tuple[str, ...], generator_config_ids: tuple[str, ...], feed_epoch_ids: tuple[str, ...], reference_assignment_count: int, reference_assignments_sha256: str, motif_assignment_count: int, motif_assignments_sha256: str, constraint_manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionConstraintManifestV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.delivery.ReconstructionDeliveredGroupV1 + +ReconstructionDeliveredGroupV1 + +manifest +schema_version : str +status : ReconstructionDeliveryStatus +streams : tuple[SyntheticEventStreamV1, ...] + +__init__(self, manifest: ReconstructionDeliveryManifestV1, streams: tuple[SyntheticEventStreamV1, ...], schema_version: str): None +__post_init__(): None +metadata(): dict[str, JSONValue] + + + +histdatacom.synthetic.delivery.ReconstructionDeliveryManifestV1 + +ReconstructionDeliveryManifestV1 + +delivery_mode +delivery_profile_id : str +ensemble_member_id : str +identity_event_count : int +identity_lineage_sha256 : str | None +input_content_sha256 : str +input_group_id : str +manifest_id : str +observed_event_count : int +output_content_sha256 : str | None +reason_codes : tuple[str, ...] +run_id : str +schema_version : str +status +synchronization_unit_id : str +synthetic_event_count : int +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, ensemble_member_id: str, input_group_id: str, delivery_mode: ReconstructionDeliveryMode, delivery_profile_id: str, status: ReconstructionDeliveryStatus, reason_codes: tuple[str, ...], input_content_sha256: str, output_content_sha256: str | None, observed_event_count: int, synthetic_event_count: int, identity_event_count: int, identity_lineage_sha256: str | None, manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionDeliveryManifestV1' +from_json(text: str): 'ReconstructionDeliveryManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.delivery.ReconstructionDeliveredGroupV1->histdatacom.synthetic.delivery.ReconstructionDeliveryManifestV1 + + +manifest + + + +histdatacom.synthetic.delivery.ReconstructionDeliveryMode + +ReconstructionDeliveryMode + +name + +from_value(value: str | 'ReconstructionDeliveryMode'): 'ReconstructionDeliveryMode' + + + +histdatacom.synthetic.delivery.ReconstructionDeliveryManifestV1->histdatacom.synthetic.delivery.ReconstructionDeliveryMode + + +delivery_mode + + + +histdatacom.synthetic.delivery.ReconstructionDeliveryStatus + +ReconstructionDeliveryStatus + +name + + + + + +histdatacom.synthetic.delivery.ReconstructionDeliveryManifestV1->histdatacom.synthetic.delivery.ReconstructionDeliveryStatus + + +status + + + +histdatacom.synthetic.persistence.ReconstructionDeliveryQualityManifestV1 + +ReconstructionDeliveryQualityManifestV1 + +benchmark_artifact_ids : tuple[str, ...] +cross_instrument_quality_sha256 : str +cross_instrument_quality_status : str +delivery_action_counts : Mapping[str, int] +delivery_manifest_id : str +delivery_mode +delivery_output_content_sha256 : str +delivery_profile_id : str +final_validation_id : str +final_validation_status : str +identity_event_count : int +identity_lineage_sha256 : str +observed_event_count : int +quality_manifest_id : str +schema_version : str +synthetic_event_count : int + +__init__(self, delivery_manifest_id: str, delivery_profile_id: str, delivery_mode: ReconstructionDeliveryMode, delivery_output_content_sha256: str, final_validation_id: str, final_validation_status: str, cross_instrument_quality_status: str, cross_instrument_quality_sha256: str, observed_event_count: int, synthetic_event_count: int, identity_event_count: int, identity_lineage_sha256: str, delivery_action_counts: Mapping[str, int], benchmark_artifact_ids: tuple[str, ...], quality_manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionDeliveryQualityManifestV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.persistence.ReconstructionDeliveryQualityManifestV1->histdatacom.synthetic.delivery.ReconstructionDeliveryMode + + +delivery_mode + + + +histdatacom.synthetic.persistence.ReconstructionEnsembleManifestV1 + +ReconstructionEnsembleManifestV1 + +ensemble_manifest_id : str +materialized_member_id : str +member_event_estimates : Mapping[str, int] +primary_member_id : str +retained_member_ids : tuple[str, ...] +retention_plan_id : str +run_id : str +schema_version : str + +__init__(self, run_id: str, materialized_member_id: str, primary_member_id: str, retained_member_ids: tuple[str, ...], member_event_estimates: Mapping[str, int], retention_plan_id: str, ensemble_manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionEnsembleManifestV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.ensembles.ReconstructionEnsemblePlanV1 + +ReconstructionEnsemblePlanV1 + +config +configuration_artifacts : tuple[EnsembleArtifactDigestV1, ...] +configuration_hashes : dict[str, str] +members : tuple[EnsembleMemberPlanV1, ...] +plan_id : str +run +schema_version : str +source_artifacts : tuple[EnsembleArtifactDigestV1, ...] +source_hashes : dict[str, str] + +__init__(self, run: ReconstructionRunV1, config: EnsembleCalibrationConfigV1, source_artifacts: tuple[EnsembleArtifactDigestV1, ...], configuration_artifacts: tuple[EnsembleArtifactDigestV1, ...], members: tuple[EnsembleMemberPlanV1, ...], plan_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionEnsemblePlanV1' +from_json(text: str): 'ReconstructionEnsemblePlanV1' +identity_payload(): dict[str, JSONValue] +member(member_id: str): EnsembleMemberPlanV1 +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.ensembles.ReconstructionEnsemblePlanV1->histdatacom.synthetic.ensembles.EnsembleCalibrationConfigV1 + + +config + + + +histdatacom.synthetic.ensembles.ReconstructionEnsemblePlanV1->histdatacom.synthetic.streaming.ReconstructionRunV1 + + +run + + + +histdatacom.reconstruction.ReconstructionExecutionRequestV1 + +ReconstructionExecutionRequestV1 + +allow_refusals : bool +information_mode +plan_id : str +plan_path : str +request_id : str +schema_version : str +scientific_nonclaim_acknowledged : bool +source_format : str +symbols : tuple[str, ...] +timeframe : str + +__init__(self, plan_path: str, plan_id: str, information_mode: InformationMode, scientific_nonclaim_acknowledged: bool, source_format: str, timeframe: str, symbols: tuple[str, ...], allow_refusals: bool, request_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionExecutionRequestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.reconstruction.ReconstructionExecutionRequestV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.reconstruction.ReconstructionExitCode + +ReconstructionExitCode + +name + + + + + +histdatacom.synthetic.streaming.ReconstructionHeartbeatV1 + +ReconstructionHeartbeatV1 + +accepted_event_count : int +cancellation_requested : bool +candidate_event_count : int +checkpoint_id : str | None +completed_units : int +heartbeat_id : str +message : str +observed_event_count : int +output_bytes : int +percent_complete : float +phase +run_id : str +schema_version : str +scratch_bytes : int +sequence : int +synchronization_unit_id : str +total_units : int +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, phase: ReconstructionCommitPhase, sequence: int, completed_units: int, total_units: int, observed_event_count: int, candidate_event_count: int, accepted_event_count: int, scratch_bytes: int, output_bytes: int, checkpoint_id: str | None, cancellation_requested: bool, message: str, heartbeat_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionHeartbeatV1' +from_json(text: str): 'ReconstructionHeartbeatV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.streaming.ReconstructionHeartbeatV1->histdatacom.synthetic.streaming.ReconstructionCommitPhase + + +phase + + + +histdatacom.synthetic.information.ReconstructionInformationInputV1 + +ReconstructionInformationInputV1 + +allowed_lookahead_ns : int +artifact_id : str +available_at_ns : int +event_time_ns : int +information_mode +input_id : str +input_kind +observation_end_ns : int +observation_start_ns : int +parent_input_ids : tuple[str, ...] +reason : str +revision_sequence : int +run_id : str +schema_version : str +scope +split_kind : InformationSplitKind | None +stage +supersedes_input_id : str | None +used_at_ns : int +vintage_id : str + +__init__(self, run_id: str, artifact_id: str, information_mode: InformationMode, input_kind: InformationInputKind, stage: InformationStage, scope: InformationScope, event_time_ns: int, available_at_ns: int, used_at_ns: int, observation_start_ns: int, observation_end_ns: int, vintage_id: str, reason: str, revision_sequence: int, supersedes_input_id: str | None, allowed_lookahead_ns: int, parent_input_ids: tuple[str, ...], split_kind: InformationSplitKind | None, input_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionInformationInputV1' +from_json(text: str): 'ReconstructionInformationInputV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.information.ReconstructionInformationInputV1->histdatacom.synthetic.information.InformationInputKind + + +input_kind + + + +histdatacom.synthetic.information.ReconstructionInformationInputV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.information.ReconstructionInformationInputV1->histdatacom.synthetic.information.InformationScope + + +scope + + + +histdatacom.synthetic.information.ReconstructionInformationInputV1->histdatacom.synthetic.information.InformationStage + + +stage + + + +histdatacom.synthetic.information.ReconstructionInformationManifestV1 + +ReconstructionInformationManifestV1 + +information_mode +inputs : tuple[ReconstructionInformationInputV1, ...] +manifest_id : str +policy_id : str +run_id : str +schema_version : str +splits : tuple[ReconstructionInformationSplitV1, ...] +window_plan_id : str + +__init__(self, run_id: str, policy_id: str, information_mode: InformationMode, window_plan_id: str, inputs: tuple[ReconstructionInformationInputV1, ...], splits: tuple[ReconstructionInformationSplitV1, ...], manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionInformationManifestV1' +from_json(text: str): 'ReconstructionInformationManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.information.ReconstructionInformationManifestV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.information.ReconstructionInformationPolicyV1 + +ReconstructionInformationPolicyV1 + +fail_closed : bool +information_mode +max_allowed_lookahead_ns : int +max_retained_findings : int +policy_id : str +require_time_ordered_splits : bool +schema_version : str + +__init__(self, information_mode: InformationMode, max_allowed_lookahead_ns: int, max_retained_findings: int, fail_closed: bool, require_time_ordered_splits: bool, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionInformationPolicyV1' +from_json(text: str): 'ReconstructionInformationPolicyV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.information.ReconstructionInformationPolicyV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.information.ReconstructionInformationSplitV1 + +ReconstructionInformationSplitV1 + +end_ns : int +kind +schema_version : str +split_id : str +start_ns : int + +__init__(self, kind: InformationSplitKind, start_ns: int, end_ns: int, split_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionInformationSplitV1' +from_json(text: str): 'ReconstructionInformationSplitV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.information.ReconstructionInformationSplitV1->histdatacom.synthetic.information.InformationSplitKind + + +kind + + + +histdatacom.reconstruction.ReconstructionOperationReceiptV1 + +ReconstructionOperationReceiptV1 + +execution_attempt_id : str +handles : tuple[OrchestrationJobHandle, ...] +job_snapshots : tuple[Mapping[str, JSONValue], ...] +operation : str +receipt_id : str +report_refs : tuple[ArtifactRef, ...] +reports : tuple[ReconstructionRunReportV1, ...] +request +schema_version : str +status : str +status_store_roots : tuple[str, ...] + +__init__(self, operation: str, request: ReconstructionExecutionRequestV1, status: str, handles: tuple[OrchestrationJobHandle, ...], status_store_roots: tuple[str, ...], execution_attempt_id: str, job_snapshots: tuple[Mapping[str, JSONValue], ...], reports: tuple[ReconstructionRunReportV1, ...], report_refs: tuple[ArtifactRef, ...], receipt_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionOperationReceiptV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.reconstruction.ReconstructionOperationReceiptV1->histdatacom.reconstruction.ReconstructionExecutionRequestV1 + + +request + + + +histdatacom.synthetic.persistence.ReconstructionPersistenceError + +ReconstructionPersistenceError + + + + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanCompatibilityError + +ReconstructionPlanCompatibilityError + + + + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1 + +ReconstructionPlanConfigurationV1 + +carving_constraints +configuration_id : str +cross_currency_config +delivery_mode +ensemble_config +generator_config +information_policy +left_halo_ns : int +max_parallel_windows : int +right_lookahead_ns : int +schema_version : str +storage_policy +window_size_ns : int + +__init__(self, delivery_mode: ReconstructionDeliveryMode, information_policy: ReconstructionInformationPolicyV1, generator_config: EmpiricalMotifGeneratorConfigV1, carving_constraints: HistoricalCarvingConstraintSetV1, cross_currency_config: CrossCurrencyReconciliationConfigV1, ensemble_config: EnsembleCalibrationConfigV1, storage_policy: ReconstructionStoragePolicyV1, window_size_ns: int, left_halo_ns: int, right_lookahead_ns: int, max_parallel_windows: int, configuration_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionPlanConfigurationV1' +from_json(text: str): 'ReconstructionPlanConfigurationV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1->histdatacom.synthetic.cross_currency.CrossCurrencyReconciliationConfigV1 + + +cross_currency_config + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1->histdatacom.synthetic.generation.EmpiricalMotifGeneratorConfigV1 + + +generator_config + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1->histdatacom.synthetic.ensembles.EnsembleCalibrationConfigV1 + + +ensemble_config + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1->histdatacom.synthetic.carving.HistoricalCarvingConstraintSetV1 + + +carving_constraints + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1->histdatacom.synthetic.delivery.ReconstructionDeliveryMode + + +delivery_mode + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1->histdatacom.synthetic.information.ReconstructionInformationPolicyV1 + + +information_policy + + + +histdatacom.synthetic.streaming.ReconstructionStoragePolicyV1 + +ReconstructionStoragePolicyV1 + +advertise_only_committed : bool +atomic_promotion_required : bool +checkpoint_every_batches : int +heartbeat_every_batches : int +max_candidate_amplification : float +max_checkpoint_bytes : int +max_events_per_batch : int +max_inflight_batches : int +max_memory_bytes : int +max_output_bytes : int +max_retained_ensemble_members : int +max_scratch_bytes : int +policy_id : str +remove_uncommitted_on_cancel : bool +schema_version : str + +__init__(self, max_events_per_batch: int, max_candidate_amplification: float, max_inflight_batches: int, max_memory_bytes: int, max_scratch_bytes: int, max_output_bytes: int, max_retained_ensemble_members: int, checkpoint_every_batches: int, heartbeat_every_batches: int, max_checkpoint_bytes: int, remove_uncommitted_on_cancel: bool, atomic_promotion_required: bool, advertise_only_committed: bool, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionStoragePolicyV1' +from_json(text: str): 'ReconstructionStoragePolicyV1' +identity_payload(): dict[str, JSONValue] +preflight(estimate: 'ReconstructionResourceEstimateV1'): 'ReconstructionResourceEstimateV1' +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1->histdatacom.synthetic.streaming.ReconstructionStoragePolicyV1 + + +storage_policy + + + +histdatacom.reconstruction.ReconstructionPlanError + +ReconstructionPlanError + +exit_code : INVALID_PLAN +reason_code : str + + + + + +histdatacom.reconstruction.ReconstructionPublicError + +ReconstructionPublicError + +exit_code : RUNTIME_FAILURE +reason_code : str + + + + + +histdatacom.reconstruction.ReconstructionPlanError->histdatacom.reconstruction.ReconstructionPublicError + + + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanExecutionManifestV1 + +ReconstructionPlanExecutionManifestV1 + +artifacts : Mapping[str, ArtifactRef] +checkpoint_root : str +configuration_id : str +delivery_mode +ensemble_plan_id : str +executable_window_count : int +information_audit_id : str +information_manifest_id : str +manifest_id : str +output_root : str +planned_window_count : int +refusal_ids : tuple[str, ...] +retention_plan_id : str +run_id : str +schema_version : str +scratch_root : str +source_inventory_id : str + +__init__(self, run_id: str, configuration_id: str, source_inventory_id: str, information_manifest_id: str, information_audit_id: str, ensemble_plan_id: str, retention_plan_id: str, delivery_mode: ReconstructionDeliveryMode, artifacts: Mapping[str, ArtifactRef], output_root: str, checkpoint_root: str, scratch_root: str, planned_window_count: int, executable_window_count: int, refusal_ids: tuple[str, ...], manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionPlanExecutionManifestV1' +from_json(text: str): 'ReconstructionPlanExecutionManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanExecutionManifestV1->histdatacom.synthetic.delivery.ReconstructionDeliveryMode + + +delivery_mode + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanRefusalCode + +ReconstructionPlanRefusalCode + +name + + + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanRefusalV1 + +ReconstructionPlanRefusalV1 + +code +end_ns : int +reason : str +refusal_id : str +schema_version : str +start_ns : int +symbols : tuple[str, ...] + +__init__(self, start_ns: int, end_ns: int, code: ReconstructionPlanRefusalCode, reason: str, symbols: tuple[str, ...], refusal_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionPlanRefusalV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanRefusalV1->histdatacom.synthetic.reconstruction_plan.ReconstructionPlanRefusalCode + + +code + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionPlanResourceSummaryV1 + +ReconstructionPlanResourceSummaryV1 + +candidate_amplification : float +ensemble_member_count : int +estimated_candidate_bytes : int +estimated_candidate_event_count : int +estimated_input_event_count : int +estimated_output_bytes : int +estimated_partition_count : int +estimated_peak_memory_bytes : int +estimated_peak_scratch_bytes : int +executable_window_count : int +planned_window_count : int +refused_window_count : int +retained_member_count : int +schema_version : str +source_event_count : int +source_size_bytes : int +summary_id : str +workflow_request_count : int + +__init__(self, source_event_count: int, source_size_bytes: int, planned_window_count: int, executable_window_count: int, refused_window_count: int, ensemble_member_count: int, retained_member_count: int, workflow_request_count: int, estimated_input_event_count: int, estimated_candidate_event_count: int, estimated_candidate_bytes: int, estimated_peak_memory_bytes: int, estimated_peak_scratch_bytes: int, estimated_output_bytes: int, estimated_partition_count: int, summary_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionPlanResourceSummaryV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.reconstruction.ReconstructionPlanSetPreflightV1 + +ReconstructionPlanSetPreflightV1 + +executable : bool +plan_set_id : str +refusal_count : int +resource_summary : Mapping[str, JSONValue] +schema_version : str +shard_count : int +shard_preflights : tuple[Mapping[str, JSONValue], ...] +status : str +verified_shard_count : int + +__init__(self, plan_set_id: str, status: str, executable: bool, shard_count: int, verified_shard_count: int, refusal_count: int, resource_summary: Mapping[str, JSONValue], shard_preflights: tuple[Mapping[str, JSONValue], ...], schema_version: str): None +to_dict(): dict[str, JSONValue] + + + +histdatacom.reconstruction.ReconstructionPlanSetV1 + +ReconstructionPlanSetV1 + +executable : bool +plan_set_id : str +requested_end_ns : int +requested_start_ns : int +resource_summary : Mapping[str, JSONValue] +schema_version : str +shards : tuple[ReconstructionPlanShardV1, ...] +source_spec +status : str + +__init__(self, source_spec: ReconstructionPlanSpecV1, shards: tuple[ReconstructionPlanShardV1, ...], requested_start_ns: int, requested_end_ns: int, resource_summary: Mapping[str, JSONValue], status: str, plan_set_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionPlanSetV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.reconstruction.ReconstructionPlanSpecV1 + +ReconstructionPlanSpecV1 + +artifact_root : str +benchmark_manifest_path : str +broker_delivery_artifact : ArtifactRef | None +cftc_positioning_corpus_path : str +checkpoint_root : str +delivery_mode +end_period : str | None +feed_epoch_definition_path : str +information_mode +market_context_corpus_path : str +motif_index_path : str +motif_leakage_audit_path : str +motif_manifest_path : str +motif_qualification_path : str +observation_operator_path : str +output_root : str +requested_end_ns : int | None +requested_start_ns : int | None +schema_version : str +scratch_root : str +source_format : str +source_root : str +start_period : str | None +symbols : tuple[str, ...] +timeframe : str +window_size_ns : int + +__init__(self, source_root: str, feed_epoch_definition_path: str, observation_operator_path: str, market_context_corpus_path: str, cftc_positioning_corpus_path: str, benchmark_manifest_path: str, motif_manifest_path: str, motif_index_path: str, motif_qualification_path: str, motif_leakage_audit_path: str, artifact_root: str, output_root: str, checkpoint_root: str, scratch_root: str, information_mode: InformationMode, start_period: str | None, end_period: str | None, requested_start_ns: int | None, requested_end_ns: int | None, window_size_ns: int, delivery_mode: ReconstructionDeliveryMode, broker_delivery_artifact: ArtifactRef | None, source_format: str, timeframe: str, symbols: tuple[str, ...], schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionPlanSpecV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.reconstruction.ReconstructionPlanSetV1->histdatacom.reconstruction.ReconstructionPlanSpecV1 + + +source_spec + + + +histdatacom.reconstruction.ReconstructionPlanShardV1 + +ReconstructionPlanShardV1 + +end_period : str +executable : bool +plan_id : str +plan_ref +preflight_status : str +refusal_count : int +requested_end_ns : int +requested_start_ns : int +resource_summary : Mapping[str, JSONValue] +schema_version : str +shard_id : str +start_period : str + +__init__(self, start_period: str, end_period: str, requested_start_ns: int, requested_end_ns: int, plan_id: str, plan_ref: ArtifactRef, preflight_status: str, executable: bool, refusal_count: int, resource_summary: Mapping[str, JSONValue], shard_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionPlanShardV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.reconstruction.ReconstructionPlanShardV1->histdatacom.runtime_contracts.ArtifactRef + + +plan_ref + + + +histdatacom.reconstruction.ReconstructionPlanSpecV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.reconstruction.ReconstructionPlanSpecV1->histdatacom.synthetic.delivery.ReconstructionDeliveryMode + + +delivery_mode + + + +histdatacom.reconstruction.ReconstructionPreflightV1 + +ReconstructionPreflightV1 + +dry_run : Mapping[str, JSONValue] +evidence_refs : Mapping[str, ArtifactRef] +executable : bool +plan_id : str +plan_status : str +refusal_reasons : tuple[Mapping[str, JSONValue], ...] +request_id : str +schema_version : str +status : str + +__init__(self, request_id: str, plan_id: str, status: str, executable: bool, plan_status: str, dry_run: Mapping[str, JSONValue], evidence_refs: Mapping[str, ArtifactRef], refusal_reasons: tuple[Mapping[str, JSONValue], ...], schema_version: str): None +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV1->histdatacom.synthetic.persistence.ReconstructionConstraintManifestV1 + + +constraints + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV1->histdatacom.synthetic.persistence.ReconstructionEnsembleManifestV1 + + +ensemble + + + +histdatacom.synthetic.persistence.ReconstructionQualityManifestV1 + +ReconstructionQualityManifestV1 + +benchmark_comparison_ids : tuple[str, ...] +broker_action_counts : Mapping[str, int] +broker_fingerprint_id : str +broker_lineage_content_sha256 : str +broker_lineage_count : int +broker_observed_event_count : int +broker_synthetic_event_count : int +broker_transfer_manifest_id : str +cross_instrument_quality_sha256 : str +cross_instrument_quality_status : str +post_broker_validation_id : str +post_broker_validation_status : str +quality_manifest_id : str +schema_version : str +transfer_output_content_sha256 : str + +__init__(self, broker_transfer_manifest_id: str, broker_fingerprint_id: str, transfer_output_content_sha256: str, post_broker_validation_id: str, post_broker_validation_status: str, cross_instrument_quality_status: str, cross_instrument_quality_sha256: str, broker_observed_event_count: int, broker_synthetic_event_count: int, broker_lineage_count: int, broker_lineage_content_sha256: str, broker_action_counts: Mapping[str, int], benchmark_comparison_ids: tuple[str, ...], quality_manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionQualityManifestV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV1->histdatacom.synthetic.persistence.ReconstructionQualityManifestV1 + + +quality + + + +histdatacom.synthetic.persistence.ReconstructionReplayManifestV1 + +ReconstructionReplayManifestV1 + +byte_hash_algorithm : str +canonicalized_metadata_exclusions : tuple[str, ...] +compression : str +logical_content_sha256 : str +logical_hash_algorithm : str +partition_byte_sha256 : str +python_runtime : str +replay_manifest_id : str +row_group_size : int +schema_version : str +writer_id : str +writer_library : str +writer_library_version : str + +__init__(self, logical_content_sha256: str, partition_byte_sha256: str, logical_hash_algorithm: str, byte_hash_algorithm: str, writer_id: str, writer_library: str, writer_library_version: str, python_runtime: str, compression: str, row_group_size: int, canonicalized_metadata_exclusions: tuple[str, ...], replay_manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionReplayManifestV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV1->histdatacom.synthetic.persistence.ReconstructionReplayManifestV1 + + +replay + + + +histdatacom.synthetic.persistence.ReconstructionRetentionPlanV1 + +ReconstructionRetentionPlanV1 + +estimated_bytes_per_event : int +estimated_compression_ratio : float +estimated_manifest_bytes : int +estimated_partition_count : int +estimated_primary_bytes : int +estimated_retained_bytes : int +estimated_total_output_bytes : int +member_event_counts : Mapping[str, int] +plan_id : str +primary_member_id : str +retained_member_ids : tuple[str, ...] +run_id : str +schema_version : str +storage_policy_id : str + +__init__(self, run_id: str, primary_member_id: str, retained_member_ids: tuple[str, ...], member_event_counts: Mapping[str, int], estimated_partition_count: int, estimated_bytes_per_event: int, estimated_compression_ratio: float, estimated_primary_bytes: int, estimated_retained_bytes: int, estimated_manifest_bytes: int, estimated_total_output_bytes: int, storage_policy_id: str, plan_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionRetentionPlanV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV1->histdatacom.synthetic.persistence.ReconstructionRetentionPlanV1 + + +retention + + + +histdatacom.synthetic.persistence.ReconstructionSourceManifestV1 + +ReconstructionSourceManifestV1 + +observed_content_sha256 : str +observed_event_count : int +observed_event_ids_sha256 : str +schema_version : str +source_manifest_id : str +source_periods : tuple[str, ...] +source_series_ids : tuple[str, ...] +source_version_ids : tuple[str, ...] + +__init__(self, source_version_ids: tuple[str, ...], source_series_ids: tuple[str, ...], source_periods: tuple[str, ...], observed_event_count: int, observed_content_sha256: str, observed_event_ids_sha256: str, source_manifest_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionSourceManifestV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV1->histdatacom.synthetic.persistence.ReconstructionSourceManifestV1 + + +source + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV2->histdatacom.synthetic.persistence.ReconstructionConstraintManifestV1 + + +constraints + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV2->histdatacom.synthetic.persistence.ReconstructionDeliveryQualityManifestV1 + + +quality + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV2->histdatacom.synthetic.persistence.ReconstructionEnsembleManifestV1 + + +ensemble + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV2->histdatacom.synthetic.persistence.ReconstructionReplayManifestV1 + + +replay + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV2->histdatacom.synthetic.persistence.ReconstructionRetentionPlanV1 + + +retention + + + +histdatacom.synthetic.persistence.ReconstructionProductManifestV2->histdatacom.synthetic.persistence.ReconstructionSourceManifestV1 + + +source + + + +histdatacom.synthetic.persistence.ReconstructionProductPartitionV1 + +ReconstructionProductPartitionV1 + +byte_sha256 : str +event_date : str +logical_content_sha256 : str +max_event_time_ns : int +min_event_time_ns : int +observed_event_count : int +partition_id : str +relative_path : str +row_count : int +row_group_count : int +schema_version : str +size_bytes : int +source_version_ids : tuple[str, ...] +stream_id : str +symbol : str +synthetic_event_count : int + +__init__(self, relative_path: str, symbol: str, event_date: str, stream_id: str, source_version_ids: tuple[str, ...], row_count: int, observed_event_count: int, synthetic_event_count: int, min_event_time_ns: int, max_event_time_ns: int, logical_content_sha256: str, byte_sha256: str, size_bytes: int, row_group_count: int, partition_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionProductPartitionV1' +payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.reconstruction.ReconstructionRefusedError + +ReconstructionRefusedError + +exit_code : REFUSED +reason_code : str + + + + + +histdatacom.reconstruction.ReconstructionRefusedError->histdatacom.reconstruction.ReconstructionPublicError + + + + + +histdatacom.orchestration.reconstruction.ReconstructionReportMismatch + +ReconstructionReportMismatch + + + + + + +histdatacom.synthetic.streaming.ReconstructionResourceLimitError + +ReconstructionResourceLimitError + +estimate : str +violations : tuple + +__init__(estimate: 'ReconstructionResourceEstimateV1', violations: Sequence[str]): None + + + +histdatacom.orchestration.reconstruction.ReconstructionRunReportV1 + +ReconstructionRunReportV1 + +cancelled_window_count : int +committed_manifest_refs : tuple[ArtifactRef, ...] +committed_window_count : int +failed_window_count : int +observed_event_count : int +report_id : str +request_id : str +run_id : str +schema_version : str +status : str +synthetic_event_count : int +window_count : int +window_states : tuple[dict[str, JSONValue], ...] + +__init__(self, request_id: str, run_id: str, status: str, window_count: int, committed_window_count: int, cancelled_window_count: int, failed_window_count: int, observed_event_count: int, synthetic_event_count: int, committed_manifest_refs: tuple[ArtifactRef, ...], window_states: tuple[dict[str, JSONValue], ...], report_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionRunReportV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.streaming.ReconstructionRunV1->histdatacom.synthetic.streaming.ReconstructionStoragePolicyV1 + + +storage_policy + + + +histdatacom.orchestration.workflows.ReconstructionRunWorkflow + +ReconstructionRunWorkflow + +_status : dict[str, JSONValue] + +__init__(): None +run(payload: dict[str, Any]): dict[str, Any] +status(): dict[str, JSONValue] + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionSourceInventoryV1 + +ReconstructionSourceInventoryV1 + +inventory_id : str +partitions : tuple[ReconstructionSourcePartitionV1, ...] +periods : tuple[str, ...] +requested_end_ns : int +requested_start_ns : int +schema_version : str +source_root : str +symbols : tuple[str, ...] +total_row_count : int +total_size_bytes : int + +__init__(self, source_root: str, symbols: tuple[str, ...], periods: tuple[str, ...], partitions: tuple[ReconstructionSourcePartitionV1, ...], requested_start_ns: int, requested_end_ns: int, total_row_count: int, total_size_bytes: int, inventory_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionSourceInventoryV1' +from_json(text: str): 'ReconstructionSourceInventoryV1' +identity_payload(): dict[str, JSONValue] +partitions_for_window(window: ReconstructionWindowV1): tuple[ReconstructionSourcePartitionV1, ...] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionSourcePartitionV1 + +ReconstructionSourcePartitionV1 + +artifact +coverage_end_ns : int +coverage_start_ns : int +feed_epoch_evidence_id : str +first_timestamp_ms : int +last_timestamp_ms : int +partition_id : str +period : str +row_count : int +schema_version : str +symbol : str + +__init__(self, symbol: str, period: str, artifact: ArtifactRef, row_count: int, coverage_start_ns: int, coverage_end_ns: int, first_timestamp_ms: int, last_timestamp_ms: int, feed_epoch_evidence_id: str, partition_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionSourcePartitionV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionSourcePartitionV1->histdatacom.runtime_contracts.ArtifactRef + + +artifact + + + +histdatacom.orchestration.reconstruction.ReconstructionStage + +ReconstructionStage + +name + +from_value(value: str | 'ReconstructionStage'): 'ReconstructionStage' + + + +histdatacom.orchestration.reconstruction.ReconstructionStageCommandV1 + +ReconstructionStageCommandV1 + +command_id : str +configuration_refs : tuple[ArtifactRef, ...] +handler_name : str +input_manifest_refs : tuple[ArtifactRef, ...] +receipt_path : str +schema_version : str +stage + +__init__(self, stage: ReconstructionStage, handler_name: str, receipt_path: str, input_manifest_refs: tuple[ArtifactRef, ...], configuration_refs: tuple[ArtifactRef, ...], command_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionStageCommandV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.orchestration.reconstruction.ReconstructionStageCommandV1->histdatacom.orchestration.reconstruction.ReconstructionStage + + +stage + + + +histdatacom.orchestration.reconstruction.ReconstructionStageExecutor + +ReconstructionStageExecutor + + +execute +(invocation: ReconstructionStageInvocationV1): ReconstructionStageOutcomeV1 + + + +histdatacom.orchestration.reconstruction.ReconstructionStageInvocationV1 + +ReconstructionStageInvocationV1 + +cancellation_check : Callable[[], bool] | None +cancellation_requested : bool +command +heartbeat_callback : Callable[[ReconstructionHeartbeatV1], Any] | None +input_fingerprint : str +prior_outcomes : tuple[ReconstructionStageOutcomeV1, ...] +run +task + +__init__(self, run: ReconstructionRunV1, task: ReconstructionWindowTaskV1, command: ReconstructionStageCommandV1, prior_outcomes: tuple[ReconstructionStageOutcomeV1, ...], heartbeat_callback: Callable[[ReconstructionHeartbeatV1], Any] | None, cancellation_check: Callable[[], bool] | None): None +completed(): ReconstructionStageOutcomeV1 +heartbeat(): None +refused(): ReconstructionStageOutcomeV1 + + + +histdatacom.orchestration.reconstruction.ReconstructionStageInvocationV1->histdatacom.synthetic.streaming.ReconstructionRunV1 + + +run + + + +histdatacom.orchestration.reconstruction.ReconstructionStageInvocationV1->histdatacom.orchestration.reconstruction.ReconstructionStageCommandV1 + + +command + + + +histdatacom.orchestration.reconstruction.ReconstructionWindowTaskV1 + +ReconstructionWindowTaskV1 + +commands : tuple[ReconstructionStageCommandV1, ...] +resource_estimate +schema_version : str +scratch_directory : str +task_id : str +window + +__init__(self, window: ReconstructionWindowV1, resource_estimate: ReconstructionResourceEstimateV1, commands: tuple[ReconstructionStageCommandV1, ...], scratch_directory: str, task_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionWindowTaskV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.orchestration.reconstruction.ReconstructionStageInvocationV1->histdatacom.orchestration.reconstruction.ReconstructionWindowTaskV1 + + +task + + + +histdatacom.orchestration.reconstruction.ReconstructionStageOutcomeV1 + +ReconstructionStageOutcomeV1 + +accepted_event_count : int +candidate_event_count : int +command_id : str +input_fingerprint : str +message : str +observed_event_count : int +outcome_id : str +output_bytes : int +output_refs : tuple[ArtifactRef, ...] +refusal_reasons : tuple[str, ...] +reused : bool +run_id : str +schema_version : str +scratch_bytes : int +stage +status +synchronization_unit_id : str +window_id : str + +__init__(self, run_id: str, window_id: str, synchronization_unit_id: str, stage: ReconstructionStage, command_id: str, input_fingerprint: str, status: ReconstructionStageStatus, output_refs: tuple[ArtifactRef, ...], observed_event_count: int, candidate_event_count: int, accepted_event_count: int, scratch_bytes: int, output_bytes: int, refusal_reasons: tuple[str, ...], message: str, reused: bool, outcome_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionStageOutcomeV1' +from_json(text: str): 'ReconstructionStageOutcomeV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.orchestration.reconstruction.ReconstructionStageOutcomeV1->histdatacom.orchestration.reconstruction.ReconstructionStage + + +stage + + + +histdatacom.orchestration.reconstruction.ReconstructionStageStatus + +ReconstructionStageStatus + +name + +from_value(value: str | 'ReconstructionStageStatus'): 'ReconstructionStageStatus' + + + +histdatacom.orchestration.reconstruction.ReconstructionStageOutcomeV1->histdatacom.orchestration.reconstruction.ReconstructionStageStatus + + +status + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionStagePlanV1 + +ReconstructionStagePlanV1 + +command +configuration +execution_manifest +execution_manifest_ref +source_inventory + +__init__(self, command: ReconstructionStageCommandV1, execution_manifest_ref: ArtifactRef, execution_manifest: ReconstructionPlanExecutionManifestV1, configuration: ReconstructionPlanConfigurationV1, source_inventory: ReconstructionSourceInventoryV1): None +__post_init__(): None + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionStagePlanV1->histdatacom.runtime_contracts.ArtifactRef + + +execution_manifest_ref + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionStagePlanV1->histdatacom.synthetic.reconstruction_plan.ReconstructionPlanConfigurationV1 + + +configuration + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionStagePlanV1->histdatacom.synthetic.reconstruction_plan.ReconstructionPlanExecutionManifestV1 + + +execution_manifest + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionStagePlanV1->histdatacom.synthetic.reconstruction_plan.ReconstructionSourceInventoryV1 + + +source_inventory + + + +histdatacom.synthetic.reconstruction_plan.ReconstructionStagePlanV1->histdatacom.orchestration.reconstruction.ReconstructionStageCommandV1 + + +command + + + +histdatacom.synthetic.persistence.ReconstructionStoragePreflightError + +ReconstructionStoragePreflightError + +estimate : str +violations : tuple + +__init__(estimate: 'ReconstructionRetentionPlanV1', violations: Sequence[str]): None + + + +histdatacom.synthetic.persistence.ReconstructionStoragePreflightError->histdatacom.synthetic.persistence.ReconstructionPersistenceError + + + + + +histdatacom.reconstruction.ReconstructionUnsupportedError + +ReconstructionUnsupportedError + +exit_code : INVALID_PLAN +reason_code : str + + + + + +histdatacom.reconstruction.ReconstructionUnsupportedError->histdatacom.reconstruction.ReconstructionPublicError + + + + + +histdatacom.reconstruction.ReconstructionValidationError + +ReconstructionValidationError + +exit_code : VALIDATION_FAILURE +reason_code : str + + + + + +histdatacom.reconstruction.ReconstructionValidationError->histdatacom.reconstruction.ReconstructionPublicError + + + + + +histdatacom.orchestration.reconstruction.ReconstructionWindowStateV1 + +ReconstructionWindowStateV1 + +checkpoint +committed_manifest_ref : ArtifactRef | None +outcomes : tuple[ReconstructionStageOutcomeV1, ...] +request_id : str +schema_version : str +state_id : str +task +terminal : bool + +__init__(self, request_id: str, task: ReconstructionWindowTaskV1, checkpoint: ReconstructionCheckpointV1, outcomes: tuple[ReconstructionStageOutcomeV1, ...], state_id: str, schema_version: str): None +__post_init__(): None +_validate_phase(): None +complete(outcome: ReconstructionStageOutcomeV1): 'ReconstructionWindowStateV1' +from_dict(data: Mapping[str, Any]): 'ReconstructionWindowStateV1' +identity_payload(): dict[str, JSONValue] +interrupted(phase: ReconstructionCommitPhase, reason: str): 'ReconstructionWindowStateV1' +outcome_for(stage: ReconstructionStage): ReconstructionStageOutcomeV1 | None +planned(request_id: str, task: ReconstructionWindowTaskV1): 'ReconstructionWindowStateV1' +running(): 'ReconstructionWindowStateV1' +to_dict(): dict[str, JSONValue] +validated(): 'ReconstructionWindowStateV1' + + + +histdatacom.orchestration.reconstruction.ReconstructionWindowStateV1->histdatacom.synthetic.streaming.ReconstructionCheckpointV1 + + +checkpoint + + + +histdatacom.orchestration.reconstruction.ReconstructionWindowStateV1->histdatacom.orchestration.reconstruction.ReconstructionWindowTaskV1 + + +task + + + +histdatacom.orchestration.reconstruction.ReconstructionWindowTaskV1->histdatacom.synthetic.streaming.ReconstructionResourceEstimateV1 + + +resource_estimate + + + +histdatacom.synthetic.streaming.ReconstructionWindowV1 + +ReconstructionWindowV1 + +core_end_ns : int +core_start_ns : int +ensemble_member_id : str +input_end_ns : int +input_start_ns : int +left_halo_ns : int +right_lookahead_ns : int +run_id : str +schema_version : str +symbols : tuple[str, ...] +synchronization_unit_id : str +window_id : str + +__init__(self, run_id: str, ensemble_member_id: str, symbols: tuple[str, ...], core_start_ns: int, core_end_ns: int, left_halo_ns: int, right_lookahead_ns: int, window_id: str, synchronization_unit_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReconstructionWindowV1' +from_json(text: str): 'ReconstructionWindowV1' +identity_payload(): dict[str, JSONValue] +owns_event_time(event_time_ns: int): bool +reads_event_time(event_time_ns: int): bool +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.orchestration.reconstruction.ReconstructionWindowTaskV1->histdatacom.synthetic.streaming.ReconstructionWindowV1 + + +window + + + +histdatacom.orchestration.workflows.ReconstructionWindowWorkflow + +ReconstructionWindowWorkflow + +_status : dict[str, JSONValue] + +__init__(): None +run(payload: dict[str, Any]): dict[str, Any] +status(): dict[str, JSONValue] + + + +histdatacom.orchestration.reconstruction.ReconstructionWorkflowRequestV1 + +ReconstructionWorkflowRequestV1 + +manifest_store_root : str +max_inflight_memory_bytes : int | None +max_parallel_windows : int +report_root : str +request_fingerprint : str +request_id : str +run +schema_version : str +task_queues : dict[str, str] +tasks : tuple[ReconstructionWindowTaskV1, ...] + +__init__(self, request_id: str, run: ReconstructionRunV1, tasks: tuple[ReconstructionWindowTaskV1, ...], manifest_store_root: str, report_root: str, task_queues: dict[str, str], max_parallel_windows: int, max_inflight_memory_bytes: int | None, request_fingerprint: str, schema_version: str): None +__post_init__(): None +for_task(task: ReconstructionWindowTaskV1): 'ReconstructionWorkflowRequestV1' +from_dict(data: Mapping[str, Any]): 'ReconstructionWorkflowRequestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.orchestration.reconstruction.ReconstructionWorkflowRequestV1->histdatacom.synthetic.streaming.ReconstructionRunV1 + + +run - + histdatacom.records.Record - -Record - -_status : PLANNED -bytes_length : str -cache_end : str -cache_filename : str -cache_line_count : str -cache_start : str -csv_filename : str -data_date : str -data_datemonth : str -data_dir : str -data_format : str -data_fxpair : str -data_month : str -data_timeframe : str -data_tk : str -data_year : str -encoding : str -legacy_status : str -status : WorkStatus -status_text : str -url : str -zip_filename : str -zip_persist : str - -__call__(): Any -__init__(): None -_create_record_data_dir(base_dir: str): None -_set_record_data_dir(base_dir: str): str -_to_dict(): dict -delete_manifest_status(base_dir: str): None -delete_momento_file(base_dir: str): None -restore_manifest_status(base_dir: str): bool -restore_momento(base_dir: str): bool -write_manifest_status(base_dir: str): None -write_memento_file(base_dir: str): None + +Record + +_status : PLANNED +bytes_length : str +cache_end : str +cache_filename : str +cache_line_count : str +cache_start : str +csv_filename : str +data_date : str +data_datemonth : str +data_dir : str +data_format : str +data_fxpair : str +data_month : str +data_timeframe : str +data_tk : str +data_year : str +encoding : str +legacy_status : str +status : WorkStatus +status_text : str +url : str +zip_filename : str +zip_persist : str + +__call__(): Any +__init__(): None +_create_record_data_dir(base_dir: str): None +_set_record_data_dir(base_dir: str): str +_to_dict(): dict +delete_manifest_status(base_dir: str): None +delete_momento_file(base_dir: str): None +restore_manifest_status(base_dir: str): bool +restore_momento(base_dir: str): bool +write_manifest_status(base_dir: str): None +write_memento_file(base_dir: str): None + + + +histdatacom.synthetic.strategy_sensitivity.ReferenceMomentumStrategyV1 + +ReferenceMomentumStrategyV1 + +_decision_interval_ns +_lookback_ns +_max_state_quotes +_threshold_bps : float +specification + +__init__(): None +start_window(evaluation_case: StrategyEvaluationCaseV1): StrategySignalStateV1 + + + +histdatacom.synthetic.motifs.ReferenceMotifBackoffAttemptV1 + +ReferenceMotifBackoffAttemptV1 + +available_count : int +candidate_count : int +level : str +minimum_support : int +outcome : str +pattern : Mapping[str, str] +schema_version : str + +__init__(self, level: str, pattern: Mapping[str, str], candidate_count: int, available_count: int, minimum_support: int, outcome: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifBackoffAttemptV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifFragmentV1 + +ReferenceMotifFragmentV1 + +ask_deltas : tuple[float, ...] +available_at_ns : int +bid_deltas : tuple[float, ...] +condition +data_quality_eligible : bool +data_quality_reasons : tuple[str, ...] +duration_ns : int +end_ask : float +end_bid : float +event_offsets_ns : tuple[int, ...] +first_known_at_ns : int +fragment_id : str +near_duplicate_signature : str +period : str +schema_version : str +source_artifact +source_end_ns : int +source_event_ids : tuple[str, ...] +source_row_ids : tuple[int, ...] +source_series_id : str +source_start_ns : int +source_window_id : str +split_kind +start_ask : float +start_bid : float +transform_policy +transitions : tuple[ReferenceMotifTransition, ...] + +__init__(self, source_window_id: str, source_series_id: str, period: str, source_artifact: ArtifactRef, split_kind: ReferenceMotifSplitKind, condition: ReferenceMotifConditionV1, source_start_ns: int, source_end_ns: int, source_row_ids: tuple[int, ...], source_event_ids: tuple[str, ...], event_offsets_ns: tuple[int, ...], bid_deltas: tuple[float, ...], ask_deltas: tuple[float, ...], transitions: tuple[ReferenceMotifTransition, ...], start_bid: float, start_ask: float, first_known_at_ns: int, available_at_ns: int, data_quality_eligible: bool, data_quality_reasons: tuple[str, ...], transform_policy: ReferenceMotifTransformPolicyV1, near_duplicate_signature: str, fragment_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifFragmentV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifFragmentV1->histdatacom.runtime_contracts.ArtifactRef + + +source_artifact + + + +histdatacom.synthetic.motifs.ReferenceMotifFragmentV1->histdatacom.synthetic.motifs.ReferenceMotifConditionV1 + + +condition + + + +histdatacom.synthetic.motifs.ReferenceMotifSplitKind + +ReferenceMotifSplitKind + +name + +from_value(value: str | 'ReferenceMotifSplitKind'): 'ReferenceMotifSplitKind' + + + +histdatacom.synthetic.motifs.ReferenceMotifFragmentV1->histdatacom.synthetic.motifs.ReferenceMotifSplitKind + + +split_kind + + + +histdatacom.synthetic.motifs.ReferenceMotifTransformPolicyV1 + +ReferenceMotifTransformPolicyV1 + +allow_price_translation : bool +allow_spread_scaling : bool +max_price_scale : float +max_time_scale : float +max_time_warp_ratio : float +min_price_scale : float +min_time_scale : float +policy_id : str +schema_version : str + +__init__(self, min_time_scale: float, max_time_scale: float, min_price_scale: float, max_price_scale: float, max_time_warp_ratio: float, allow_price_translation: bool, allow_spread_scaling: bool, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifTransformPolicyV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifFragmentV1->histdatacom.synthetic.motifs.ReferenceMotifTransformPolicyV1 + + +transform_policy + + + +histdatacom.synthetic.motifs.ReferenceMotifIndexConfigV1 + +ReferenceMotifIndexConfigV1 + +backoff_levels : tuple[str, ...] +categorical_mismatch_penalty : float +config_id : str +max_artifact_bytes : int +max_events_per_fragment : int +max_fragments : int +max_matches : int +max_source_windows : int +metric_scales : Mapping[str, float] +metric_weights : Mapping[str, float] +min_cell_support : int +min_events_per_fragment : int +missing_metric_penalty : float +near_duplicate_rounding_digits : int +rounding_digits : int +schema_version : str +source_overlap_guard_ns : int + +__init__(self, min_events_per_fragment: int, max_events_per_fragment: int, max_source_windows: int, max_fragments: int, min_cell_support: int, max_matches: int, source_overlap_guard_ns: int, near_duplicate_rounding_digits: int, rounding_digits: int, max_artifact_bytes: int, categorical_mismatch_penalty: float, missing_metric_penalty: float, metric_scales: Mapping[str, float], metric_weights: Mapping[str, float], backoff_levels: tuple[str, ...], config_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifIndexConfigV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifIndexV1->histdatacom.synthetic.motifs.ReferenceMotifIndexConfigV1 + + +config + + + +histdatacom.synthetic.motifs.ReferenceMotifLeakageError + +ReferenceMotifLeakageError + +findings : tuple + +__init__(findings: Sequence[Mapping[str, JSONValue]]): None + + + +histdatacom.synthetic.motifs.ReferenceMotifMatchV1 + +ReferenceMotifMatchV1 + +backoff_level : str +cell_support : int +distance : float +fragment +rank : int +schema_version : str + +__init__(self, fragment: ReferenceMotifFragmentV1, distance: float, cell_support: int, backoff_level: str, rank: int, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifMatchV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifMatchV1->histdatacom.synthetic.motifs.ReferenceMotifFragmentV1 + + +fragment + + + +histdatacom.synthetic.motifs.ReferenceMotifQueryStatus + +ReferenceMotifQueryStatus + +name + + + + + +histdatacom.synthetic.motifs.ReferenceMotifQueryResultV1->histdatacom.synthetic.motifs.ReferenceMotifQueryStatus + + +status + + + +histdatacom.synthetic.motifs.ReferenceMotifQueryV1 + +ReferenceMotifQueryV1 + +as_of_ns : int | None +condition +excluded_source_window_ids : tuple[str, ...] +information_mode +max_distance : float | None +max_results : int +min_cell_support : int | None +query_id : str +schema_version : str +used_at_ns : int + +__init__(self, condition: ReferenceMotifConditionV1, information_mode: InformationMode, used_at_ns: int, as_of_ns: int | None, max_results: int, min_cell_support: int | None, max_distance: float | None, excluded_source_window_ids: tuple[str, ...], query_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifQueryV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifQueryResultV1->histdatacom.synthetic.motifs.ReferenceMotifQueryV1 + + +query + + + +histdatacom.synthetic.motifs.ReferenceMotifQueryV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.motifs.ReferenceMotifQueryV1->histdatacom.synthetic.motifs.ReferenceMotifConditionV1 + + +condition + + + +histdatacom.synthetic.motifs.ReferenceMotifResourceLimitError + +ReferenceMotifResourceLimitError + + + + + + +histdatacom.synthetic.motifs.ReferenceMotifSourceEventV1 + +ReferenceMotifSourceEventV1 + +ask : float +bid : float +event_sequence : int +event_time_ns : int +schema_version : str +source_event_id : str +source_row_id : int + +__init__(self, event_time_ns: int, event_sequence: int, bid: float, ask: float, source_row_id: int, source_event_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifSourceEventV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifSourceWindowV1 + +ReferenceMotifSourceWindowV1 + +available_at_ns : int +condition +data_quality_eligible : bool +data_quality_reasons : tuple[str, ...] +end_ns : int +events : tuple[ReferenceMotifSourceEventV1, ...] +first_known_at_ns : int +period : str +schema_version : str +source_artifact +source_series_id : str +source_window_id : str +split_kind +start_ns : int +transform_policy + +__init__(self, source_series_id: str, period: str, source_artifact: ArtifactRef, split_kind: ReferenceMotifSplitKind, condition: ReferenceMotifConditionV1, events: tuple[ReferenceMotifSourceEventV1, ...], first_known_at_ns: int, available_at_ns: int, data_quality_eligible: bool, data_quality_reasons: tuple[str, ...], transform_policy: ReferenceMotifTransformPolicyV1, source_window_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifSourceWindowV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifSourceWindowV1->histdatacom.runtime_contracts.ArtifactRef + + +source_artifact + + + +histdatacom.synthetic.motifs.ReferenceMotifSourceWindowV1->histdatacom.synthetic.motifs.ReferenceMotifConditionV1 + + +condition + + + +histdatacom.synthetic.motifs.ReferenceMotifSourceWindowV1->histdatacom.synthetic.motifs.ReferenceMotifSplitKind + + +split_kind + + + +histdatacom.synthetic.motifs.ReferenceMotifSourceWindowV1->histdatacom.synthetic.motifs.ReferenceMotifTransformPolicyV1 + + +transform_policy + + + +histdatacom.synthetic.motifs.ReferenceMotifSplitV1 + +ReferenceMotifSplitV1 + +end_ns : int +kind +schema_version : str +split_id : str +start_ns : int + +__init__(self, kind: ReferenceMotifSplitKind, start_ns: int, end_ns: int, split_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReferenceMotifSplitV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.motifs.ReferenceMotifSplitV1->histdatacom.synthetic.motifs.ReferenceMotifSplitKind + + +kind + + + +histdatacom.synthetic.motifs.ReferenceMotifTransition + +ReferenceMotifTransition + +name + +from_value(value: str | 'ReferenceMotifTransition'): 'ReferenceMotifTransition' + + + +histdatacom.orchestration.reconstruction.RegisteredReconstructionStageExecutor + +RegisteredReconstructionStageExecutor + + +execute(invocation: ReconstructionStageInvocationV1): ReconstructionStageOutcomeV1 + + + +histdatacom.data_quality.remediation.RemediationActionability + +RemediationActionability + +name + + + + + +histdatacom.data_quality.remediation.RemediationActionabilityDecision + +RemediationActionabilityDecision + +actionability +reason : str + +__init__(self, actionability: RemediationActionability, reason: str): None +to_payload(): dict[str, JSONValue] + + + +histdatacom.data_quality.remediation.RemediationActionabilityDecision->histdatacom.data_quality.remediation.RemediationActionability + + +actionability - + histdatacom.scraper.repo.Repo - -Repo - -args : dict[str, Any] -filter_pairs : NoneType, set[str] | None -repo_data : dict[str, Any] -repo_file_exists : bool -repo_local_path : Path -repo_url : str - -__init__(args: Mapping[str, Any] | None): None -_check_for_create_or_update(args: Mapping[str, Any]): bool -_ensure_repo_data_loaded(): None -_filter_pairs_for_action(args: Mapping[str, Any]): set[str] | None -_filter_repo_dict_by_pairs(repo_dict_copy: dict, filter_pairs: set): dict -_hash_repo(): None -_print_refresh_failure(code: str): None -_print_repo_data_table(args: Mapping[str, Any]): None -_repo_local_path(args: Mapping[str, Any]): Path -_runtime_args(args: Mapping[str, Any] | None): dict[str, Any] -_sort_repo_dict_by(repo_dict_copy: dict, filter_pairs: set): dict -_validate_repository_coverage(args: Mapping[str, Any]): None -_write_repo_data_file(): None -check_for_repo_action(args: Mapping[str, Any]): bool -check_if_repo_validation_is_needed(args: Mapping[str, Any]): bool -get_available_repo_data(args: Mapping[str, Any] | None): dict | None -read_repo_data_file(): None -set_repo_datum(record: 'Record'): None -test_for_repo_data_file(): bool -update_repo_from_github(args: Mapping[str, Any] | None): None + +Repo + +args : dict[str, Any] +filter_pairs : NoneType, set[str] | None +repo_data : dict[str, Any] +repo_file_exists : bool +repo_local_path : Path +repo_url : str + +__init__(args: Mapping[str, Any] | None): None +_check_for_create_or_update(args: Mapping[str, Any]): bool +_ensure_repo_data_loaded(): None +_filter_pairs_for_action(args: Mapping[str, Any]): set[str] | None +_filter_repo_dict_by_pairs(repo_dict_copy: dict, filter_pairs: set): dict +_hash_repo(): None +_print_refresh_failure(code: str): None +_print_repo_data_table(args: Mapping[str, Any]): None +_repo_local_path(args: Mapping[str, Any]): Path +_runtime_args(args: Mapping[str, Any] | None): dict[str, Any] +_sort_repo_dict_by(repo_dict_copy: dict, filter_pairs: set): dict +_validate_repository_coverage(args: Mapping[str, Any]): None +_write_repo_data_file(): None +check_for_repo_action(args: Mapping[str, Any]): bool +check_if_repo_validation_is_needed(args: Mapping[str, Any]): bool +get_available_repo_data(args: Mapping[str, Any] | None): dict | None +read_repo_data_file(): None +set_repo_datum(record: 'Record'): None +test_for_repo_data_file(): bool +update_repo_from_github(args: Mapping[str, Any] | None): None - + histdatacom.scraper.scraper.Scraper - -Scraper - -args : dict[str, Any] -check_for_repo_action : Callable -check_if_repo_validation_is_needed : Callable -filter_pairs : set[str] | None -repo -set_repo_datum : Callable -urls - -__init__(args: Mapping[str, Any] | None): None -_check_for_existing_archives_on_disk(record: Record): bool -_check_for_valid_download(record: Record): None -_download_zip(record: Record, args: Mapping[str, Any]): Record | None -_ensure_pairs(): None -_fetch_form_values(page_data: dict, record: Record): Record -_get_page_data(url: str, timeout: int): dict -_get_zip_file_name(response: requests.Response): str -_init_record(url: str, args: Mapping[str, Any] | None): Record -_init_record_from_work_item(work_item: WorkItem, args: Mapping[str, Any] | None): Record -_request_file(record: Record, timeout: int): requests.Response -_runtime_args(args: Mapping[str, Any] | None): dict[str, Any] -_scrape_record_info(record: Record): Record -_sync_filter_pairs(args: Mapping[str, Any]): None -_validate_url(record: Record, args: Mapping[str, Any]): Record | None -_write_file(record: Record, zip_content: bytes): None -download_zips(records: Iterable[Record], args: Mapping[str, Any] | None): list[Record] -get_zip_file(record: Record): None -plan_initial_records(args: Mapping[str, Any] | None): list[Record] -validate_urls(records: Iterable[Record], args: Mapping[str, Any] | None): list[Record] + +Scraper + +args : dict[str, Any] +check_for_repo_action : Callable +check_if_repo_validation_is_needed : Callable +filter_pairs : set[str] | None +repo +set_repo_datum : Callable +urls + +__init__(args: Mapping[str, Any] | None): None +_check_for_existing_archives_on_disk(record: Record): bool +_check_for_valid_download(record: Record): None +_download_zip(record: Record, args: Mapping[str, Any]): Record | None +_ensure_pairs(): None +_fetch_form_values(page_data: dict, record: Record): Record +_get_page_data(url: str, timeout: int): dict +_get_zip_file_name(response: requests.Response): str +_init_record(url: str, args: Mapping[str, Any] | None): Record +_init_record_from_work_item(work_item: WorkItem, args: Mapping[str, Any] | None): Record +_request_file(record: Record, timeout: int): requests.Response +_runtime_args(args: Mapping[str, Any] | None): dict[str, Any] +_scrape_record_info(record: Record): Record +_sync_filter_pairs(args: Mapping[str, Any]): None +_validate_url(record: Record, args: Mapping[str, Any]): Record | None +_write_file(record: Record, zip_content: bytes): None +download_zips(records: Iterable[Record], args: Mapping[str, Any] | None): list[Record] +get_zip_file(record: Record): None +plan_initial_records(args: Mapping[str, Any] | None): list[Record] +validate_urls(records: Iterable[Record], args: Mapping[str, Any] | None): list[Record] - + histdatacom.scraper.repo.Repo->histdatacom.scraper.scraper.Scraper - - -repo + + +repo - + histdatacom.exceptions.ReportableError - -ReportableError - -action : str -category -code : str -detail : dict[str, JSONValue] -message : str -retryable : bool - -__init__(self, code: str, message: str, category: ErrorCategory, retryable: bool, action: str, detail: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +ReportableError + +action : str +category +code : str +detail : dict[str, JSONValue] +message : str +retryable : bool + +__init__(self, code: str, message: str, category: ErrorCategory, retryable: bool, action: str, detail: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] - + histdatacom.exceptions.ReportableError->histdatacom.exceptions.ErrorCategory - - -category + + +category - + histdatacom.exceptions.RepositoryError - -RepositoryError - -category : REPOSITORY -code : str -retryable : bool - - + +RepositoryError + +category : REPOSITORY +code : str +retryable : bool + + - + histdatacom.exceptions.RepositoryError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.orchestration.workflows.RepositoryRefreshWorkflow - -RepositoryRefreshWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +RepositoryRefreshWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows.RepositoryRefreshWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + - + histdatacom.activity_stages.RepositoryStageOutput - -RepositoryStageOutput - -available_data : dict[str, Any] -filter_pairs : tuple[str, ...] -repo_data : dict[str, Any] -repo_file_exists : bool -result - -__init__(self, repo_data: dict[str, Any], available_data: dict[str, Any], filter_pairs: tuple[str, ...], repo_file_exists: bool, result: StageResult): None -to_dict(): dict[str, Any] + +RepositoryStageOutput + +available_data : dict[str, Any] +filter_pairs : tuple[str, ...] +repo_data : dict[str, Any] +repo_file_exists : bool +result + +__init__(self, repo_data: dict[str, Any], available_data: dict[str, Any], filter_pairs: tuple[str, ...], repo_file_exists: bool, result: StageResult): None +to_dict(): dict[str, Any] - + histdatacom.activity_stages.RepositoryStageOutput->histdatacom.runtime_contracts.StageResult - - -result + + +result + + + +histdatacom.synthetic.benchmark_corpus.ReverseDegradationBenchmarkCampaignV1 + +ReverseDegradationBenchmarkCampaignV1 + +artifact_bytes : int +campaign_gate_decision +campaign_id : str +campaign_metrics : Mapping[str, JSONScalar] +candidate_reports : tuple[BenchmarkCandidateReportV1, ...] +completed_at_utc : str +context_slice_counts : Mapping[str, int] +corpus_id : str +degradation_coverage : Mapping[str, int] +motif_index_id : str +peak_memory_bytes : int +runtime_seconds : float +schema_version : str +source_replay_verified : bool +started_at_utc : str + +__init__(self, corpus_id: str, motif_index_id: str, candidate_reports: tuple[BenchmarkCandidateReportV1, ...], campaign_metrics: Mapping[str, JSONScalar], degradation_coverage: Mapping[str, int], context_slice_counts: Mapping[str, int], campaign_gate_decision: BenchmarkPromotionDecisionV1, source_replay_verified: bool, runtime_seconds: float, peak_memory_bytes: int, artifact_bytes: int, started_at_utc: str, completed_at_utc: str, campaign_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReverseDegradationBenchmarkCampaignV1' +from_json(text: str): 'ReverseDegradationBenchmarkCampaignV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.benchmark_corpus.ReverseDegradationBenchmarkCampaignV1->histdatacom.synthetic.benchmark_gates.BenchmarkPromotionDecisionV1 + + +campaign_gate_decision + + + +histdatacom.synthetic.benchmark_corpus.ReverseDegradationBenchmarkCorpusV1 + +ReverseDegradationBenchmarkCorpusV1 + +cftc_positioning_corpus_id : str +corpus_id : str +degradation_configs : tuple[Mapping[str, JSONValue], ...] +dependency_artifacts : Mapping[str, ArtifactRef] +feed_epoch_definition_id : str +gate_policy_commit : str +gate_policy_id : str +market_context_corpus_id : str +metric_registry : tuple[str, ...] +neighbor_leakage_count : int +observation_operator_id : str +profile +schema_version : str +sources : tuple[BenchmarkSourcePartitionV1, ...] +split_hashes : Mapping[str, str] +windows : tuple[BenchmarkWindowPartitionV1, ...] + +__init__(self, profile: ReverseDegradationCorpusProfileV1, sources: tuple[BenchmarkSourcePartitionV1, ...], windows: tuple[BenchmarkWindowPartitionV1, ...], split_hashes: Mapping[str, str], degradation_configs: tuple[Mapping[str, JSONValue], ...], metric_registry: tuple[str, ...], dependency_artifacts: Mapping[str, ArtifactRef], feed_epoch_definition_id: str, observation_operator_id: str, market_context_corpus_id: str, cftc_positioning_corpus_id: str, gate_policy_id: str, gate_policy_commit: str, neighbor_leakage_count: int, corpus_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReverseDegradationBenchmarkCorpusV1' +from_json(text: str): 'ReverseDegradationBenchmarkCorpusV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.benchmark_corpus.ReverseDegradationCorpusProfileV1 + +ReverseDegradationCorpusProfileV1 + +ensemble_member_ids : tuple[str, ...] +max_artifact_bytes : int +max_events_per_symbol : int +max_peak_memory_bytes : int +max_runtime_seconds : float +max_source_bytes : int +minimum_events_per_symbol : int +neighbor_guard_seconds : int +profile_id : str +schema_version : str +split_periods : Mapping[str, str] +symbols : tuple[str, ...] +synchronized_windows_per_split : int +window_duration_seconds : int + +__init__(self, symbols: tuple[str, ...], split_periods: Mapping[str, str], synchronized_windows_per_split: int, window_duration_seconds: int, minimum_events_per_symbol: int, max_events_per_symbol: int, neighbor_guard_seconds: int, ensemble_member_ids: tuple[str, ...], max_source_bytes: int, max_runtime_seconds: float, max_peak_memory_bytes: int, max_artifact_bytes: int, profile_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReverseDegradationCorpusProfileV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.benchmark_corpus.ReverseDegradationBenchmarkCorpusV1->histdatacom.synthetic.benchmark_corpus.ReverseDegradationCorpusProfileV1 + + +profile + + + +histdatacom.synthetic.benchmark.ReverseDegradationBenchmarkManifestV1 + +ReverseDegradationBenchmarkManifestV1 + +automatic_winner : bool +candidates : tuple[BenchmarkCandidateV1, ...] +information_manifest_id : str +manifest_id : str +profile +run_id : str +scenarios : tuple[BenchmarkScenarioV1, ...] +schema_version : str +splits : tuple[BenchmarkSplitV1, ...] + +__init__(self, run_id: str, information_manifest_id: str, profile: BenchmarkProfileV1, splits: tuple[BenchmarkSplitV1, ...], scenarios: tuple[BenchmarkScenarioV1, ...], candidates: tuple[BenchmarkCandidateV1, ...], automatic_winner: bool, manifest_id: str, schema_version: str): None +__post_init__(): None +candidate_by_id(candidate_id: str): BenchmarkCandidateV1 +from_dict(data: Mapping[str, Any]): 'ReverseDegradationBenchmarkManifestV1' +from_json(text: str): 'ReverseDegradationBenchmarkManifestV1' +identity_payload(): dict[str, JSONValue] +scenario_by_id(scenario_id: str): BenchmarkScenarioV1 +split_for(kind: BenchmarkSplitKind): BenchmarkSplitV1 +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.benchmark.ReverseDegradationBenchmarkManifestV1->histdatacom.synthetic.benchmark.BenchmarkProfileV1 + + +profile + + + +histdatacom.synthetic.benchmark.ReverseDegradationBenchmarkV1 + +ReverseDegradationBenchmarkV1 + +_finalized : bool +_slice_count : int +_states : dict[tuple[str, str], _ComparisonState] +manifest + +__init__(manifest: ReverseDegradationBenchmarkManifestV1): None +_candidate_score(scenario: BenchmarkScenarioV1, candidate: BenchmarkCandidateV1, state: _ComparisonState): BenchmarkCandidateScoreV1 +_consume_candidate_cell(state: _ComparisonState, candidate: BenchmarkCandidateV1, reference: Sequence[BenchmarkEventV1], degraded: Sequence[BenchmarkEventV1], windows: Sequence[BenchmarkCandidateWindowV1]): None +_empty_stats(): _StreamStats +_slice_score(scenario: BenchmarkScenarioV1, candidate: BenchmarkCandidateV1, state: _ComparisonState, key: tuple[str, str, str, str, str]): BenchmarkSliceScoreV1 +_stream_stats(target: dict[tuple[str, str, str, str, str], _StreamStats], key: tuple[str, str, str, str, str]): _StreamStats +_update_diversity(state: _ComparisonState, windows: Sequence[BenchmarkCandidateWindowV1]): None +_update_hooks(target: dict[str, _OnlineMetric], values: Mapping[str, float]): None +_validate_candidate_windows(values: Sequence[BenchmarkCandidateWindowV1]): dict[str, tuple[BenchmarkCandidateWindowV1, ...]] +_validate_window_events(values: Sequence[BenchmarkEventV1], window: ReconstructionWindowV1, scenario: BenchmarkScenarioV1, label: str): tuple[BenchmarkEventV1, ...] +consume_window(): None +finalize(): ReverseDegradationScorecardV1 + + + +histdatacom.synthetic.benchmark.ReverseDegradationBenchmarkManifestV1->histdatacom.synthetic.benchmark.ReverseDegradationBenchmarkV1 + + +manifest + + + +histdatacom.synthetic.benchmark._ComparisonState + +_ComparisonState + +attempted_count : int +converged_count : int +cross_series_hooks : dict[str, _OnlineMetric] +degraded_slices : dict[tuple[str, str, str, str, str], _StreamStats] +diversity_by_slice : dict[tuple[str, str, str, str, str], _OnlineMetric] +durable_bytes : int +failure_reasons : Counter[str] +first_window_start_ns : int | None +hard_violations : Counter[str] +last_window_end_ns : int | None +member_states : dict[str, _MemberState] +peak_memory_bytes : int +reference_slices : dict[tuple[str, str, str, str, str], _StreamStats] +scratch_bytes : int +strategy_hooks : dict[str, _OnlineMetric] +wall_time_ms : int +window_count : int + +__init__(self, reference_slices: dict[tuple[str, str, str, str, str], _StreamStats], degraded_slices: dict[tuple[str, str, str, str, str], _StreamStats], member_states: dict[str, _MemberState], hard_violations: Counter[str], failure_reasons: Counter[str], attempted_count: int, converged_count: int, wall_time_ms: int, peak_memory_bytes: int, scratch_bytes: int, durable_bytes: int, cross_series_hooks: dict[str, _OnlineMetric], strategy_hooks: dict[str, _OnlineMetric], diversity_by_slice: dict[tuple[str, str, str, str, str], _OnlineMetric], first_window_start_ns: int | None, last_window_end_ns: int | None, window_count: int): None + + + +histdatacom.synthetic.benchmark.ReverseDegradationBenchmarkV1->histdatacom.synthetic.benchmark._ComparisonState + + +_states + + + +histdatacom.synthetic.benchmark.ReverseDegradationScorecardV1 + +ReverseDegradationScorecardV1 + +automatic_winner : bool +candidate_scores : tuple[BenchmarkCandidateScoreV1, ...] +manifest_id : str +schema_version : str +scorecard_id : str + +__init__(self, manifest_id: str, candidate_scores: tuple[BenchmarkCandidateScoreV1, ...], automatic_winner: bool, scorecard_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'ReverseDegradationScorecardV1' +from_json(text: str): 'ReverseDegradationScorecardV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str - + histdatacom.orchestration.throughput.RuntimeBenchmarkResult - -RuntimeBenchmarkResult - -artifact_count : int -cpu_utilization_ratio : float -failure_count : int -measurement -metadata : dict[str, JSONValue] -retry_count : int -runtime : str -scenario : str -stage_counts : dict[str, int] -status : str - -__init__(self, runtime: str, scenario: str, measurement: BenchmarkMeasurement, status: str, stage_counts: dict[str, int], artifact_count: int, failure_count: int, retry_count: int, metadata: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +RuntimeBenchmarkResult + +artifact_count : int +cpu_utilization_ratio : float +failure_count : int +measurement +metadata : dict[str, JSONValue] +retry_count : int +runtime : str +scenario : str +stage_counts : dict[str, int] +status : str + +__init__(self, runtime: str, scenario: str, measurement: BenchmarkMeasurement, status: str, stage_counts: dict[str, int], artifact_count: int, failure_count: int, retry_count: int, metadata: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.throughput.RuntimeBenchmarkResult->histdatacom.orchestration.performance.BenchmarkMeasurement - - -measurement + + +measurement - + histdatacom.histdata_com.RuntimeContext - -RuntimeContext - -api_return_type : str | None -args : Mapping[str, Any] -available_remote_data : bool -data_quality : bool -from_api : bool -import_to_influxdb : bool -orchestration_keep_runtime : bool -orchestration_start : bool -orchestration_wait_result : bool -quality_check_groups : tuple[str, ...] -quality_fail_on : str -quality_max_errors : int -quality_max_warnings : int -quality_paths : tuple[str, ...] -quality_preflight : bool -quality_preflight_evidence_allow_stale : bool -quality_preflight_evidence_max_age_seconds : int -quality_preflight_evidence_path : str | None -quality_preflight_markdown : bool -quality_preflight_markdown_report_path : str | None -quality_preflight_profile_preview_format : str -quality_preflight_profile_preview_output_path : str -quality_preflight_report_path : str | None -quality_preflight_run_validation : bool -quality_preflight_sample_size : int -quality_preflight_validation_report_path : str | None -quality_profile : Mapping[str, Any] -quality_profile_path : str -quality_profile_preview : bool -quality_profile_preview_format : str -quality_profile_preview_output_path : str -quality_report_path : str | None -repo_quality_columns : bool -repo_quality_refresh : bool -request -request_bundle_out : str -request_json_out : str -update_remote_data : bool -verbosity : int -version : bool - -__init__(self, args: Mapping[str, Any], request: RunRequest, version: bool, from_api: bool, orchestration_start: bool, orchestration_keep_runtime: bool, orchestration_wait_result: bool, api_return_type: str | None, data_quality: bool, quality_paths: tuple[str, ...], quality_check_groups: tuple[str, ...], quality_report_path: str | None, quality_fail_on: str, quality_max_errors: int, quality_max_warnings: int, quality_preflight: bool, quality_preflight_evidence_allow_stale: bool, quality_preflight_evidence_max_age_seconds: int, quality_preflight_evidence_path: str | None, quality_preflight_markdown: bool, quality_preflight_markdown_report_path: str | None, quality_preflight_profile_preview_format: str, quality_preflight_profile_preview_output_path: str, quality_preflight_report_path: str | None, quality_preflight_run_validation: bool, quality_preflight_sample_size: int, quality_preflight_validation_report_path: str | None, quality_profile_path: str, quality_profile: Mapping[str, Any], quality_profile_preview: bool, quality_profile_preview_format: str, quality_profile_preview_output_path: str, repo_quality_refresh: bool, repo_quality_columns: bool, available_remote_data: bool, update_remote_data: bool, import_to_influxdb: bool, verbosity: int, request_bundle_out: str, request_json_out: str): None + +RuntimeContext + +api_return_type : str | None +args : Mapping[str, Any] +available_remote_data : bool +data_quality : bool +from_api : bool +import_to_influxdb : bool +orchestration_keep_runtime : bool +orchestration_start : bool +orchestration_wait_result : bool +output_timezone : str +quality_check_groups : tuple[str, ...] +quality_fail_on : str +quality_max_errors : int +quality_max_warnings : int +quality_paths : tuple[str, ...] +quality_preflight : bool +quality_preflight_evidence_allow_stale : bool +quality_preflight_evidence_max_age_seconds : int +quality_preflight_evidence_path : str | None +quality_preflight_markdown : bool +quality_preflight_markdown_report_path : str | None +quality_preflight_profile_preview_format : str +quality_preflight_profile_preview_output_path : str +quality_preflight_report_path : str | None +quality_preflight_run_validation : bool +quality_preflight_sample_size : int +quality_preflight_validation_evidence_path : str +quality_preflight_validation_report_path : str | None +quality_profile : Mapping[str, Any] +quality_profile_path : str +quality_profile_preview : bool +quality_profile_preview_format : str +quality_profile_preview_output_path : str +quality_profile_resolution : Mapping[str, Any] +quality_report_path : str | None +repo_quality_columns : bool +repo_quality_refresh : bool +request +request_bundle_out : str +request_json_out : str +update_remote_data : bool +verbosity : int +version : bool + +__init__(self, args: Mapping[str, Any], request: RunRequest, version: bool, from_api: bool, orchestration_start: bool, orchestration_keep_runtime: bool, orchestration_wait_result: bool, api_return_type: str | None, output_timezone: str, data_quality: bool, quality_paths: tuple[str, ...], quality_check_groups: tuple[str, ...], quality_report_path: str | None, quality_fail_on: str, quality_max_errors: int, quality_max_warnings: int, quality_preflight: bool, quality_preflight_evidence_allow_stale: bool, quality_preflight_evidence_max_age_seconds: int, quality_preflight_evidence_path: str | None, quality_preflight_markdown: bool, quality_preflight_markdown_report_path: str | None, quality_preflight_profile_preview_format: str, quality_preflight_profile_preview_output_path: str, quality_preflight_report_path: str | None, quality_preflight_run_validation: bool, quality_preflight_sample_size: int, quality_preflight_validation_report_path: str | None, quality_preflight_validation_evidence_path: str, quality_profile_path: str, quality_profile: Mapping[str, Any], quality_profile_resolution: Mapping[str, Any], quality_profile_preview: bool, quality_profile_preview_format: str, quality_profile_preview_output_path: str, repo_quality_refresh: bool, repo_quality_columns: bool, available_remote_data: bool, update_remote_data: bool, import_to_influxdb: bool, verbosity: int, request_bundle_out: str, request_json_out: str): None - + histdatacom.histdata_com.RuntimeContext->histdatacom.runtime_contracts.RunRequest - - -request + + +request - + histdatacom.histdata_com.RuntimeContext->histdatacom.histdata_com._HistDataCom - - -context + + +context - + histdatacom.orchestration.cutover.RuntimeCutoverPolicy - -RuntimeCutoverPolicy - -config_globals_lifecycle : str -default_runtime : str -foreground_lifecycle : str -orchestration_activation : str -unavailable_runtime_behavior : str -unsupported_artifact_behavior : str - -__init__(self, default_runtime: str, orchestration_activation: str, foreground_lifecycle: str, config_globals_lifecycle: str, unavailable_runtime_behavior: str, unsupported_artifact_behavior: str): None -to_dict(): dict[str, str] + +RuntimeCutoverPolicy + +config_globals_lifecycle : str +default_runtime : str +foreground_lifecycle : str +orchestration_activation : str +unavailable_runtime_behavior : str +unsupported_artifact_behavior : str + +__init__(self, default_runtime: str, orchestration_activation: str, foreground_lifecycle: str, config_globals_lifecycle: str, unavailable_runtime_behavior: str, unsupported_artifact_behavior: str): None +to_dict(): dict[str, str] - + histdatacom.scheduled_run_bundle.ScheduledRunBundle - -ScheduledRunBundle - -request -schedule - -__init__(self, request: RunRequest, schedule: ScheduledRunSchedule): None -from_dict(data: Mapping[str, Any]): 'ScheduledRunBundle' -request_for_jobs(): RunRequest -to_dict(): dict[str, JSONValue] + +ScheduledRunBundle + +request +schedule + +__init__(self, request: RunRequest, schedule: ScheduledRunSchedule): None +from_dict(data: Mapping[str, Any]): 'ScheduledRunBundle' +request_for_jobs(): RunRequest +to_dict(): dict[str, JSONValue] - + histdatacom.scheduled_run_bundle.ScheduledRunBundle->histdatacom.runtime_contracts.RunRequest - - -request + + +request - + histdatacom.scheduled_run_bundle.ScheduledRunSchedule - -ScheduledRunSchedule - -no_overlap : bool -schedule_key : str - -__init__(self, no_overlap: bool, schedule_key: str): None -from_dict(data: Mapping[str, Any]): 'ScheduledRunSchedule' -to_dict(): dict[str, JSONValue] + +ScheduledRunSchedule + +no_overlap : bool +schedule_key : str + +__init__(self, no_overlap: bool, schedule_key: str): None +from_dict(data: Mapping[str, Any]): 'ScheduledRunSchedule' +to_dict(): dict[str, JSONValue] - + histdatacom.scheduled_run_bundle.ScheduledRunBundle->histdatacom.scheduled_run_bundle.ScheduledRunSchedule - - -schedule + + +schedule - + histdatacom.scheduled_run_bundle.ScheduledRunBundleError - -ScheduledRunBundleError - - - + +ScheduledRunBundleError + + + + + + +histdatacom.data_quality.seasonal_exogenous.SeasonalExogenousProfile->histdatacom.data_quality.seasonal_exogenous.CalendarRegressorProfile + + +regressor_profile + + + +histdatacom.data_quality.seasonal_exogenous.SeasonalExogenousResult + +SeasonalExogenousResult + +annotations : tuple[Mapping[str, Any], ...] +diagnostics : Mapping[str, JSONValue] +input_result + +__init__(self, diagnostics: Mapping[str, JSONValue], annotations: tuple[Mapping[str, Any], ...], input_result: ClassicalModelInputResult): None + + + +histdatacom.data_quality.seasonal_exogenous.SeasonalExogenousResult->histdatacom.data_quality.classical_model_contracts.ClassicalModelInputResult + + +input_result + + + +histdatacom.data_quality.seasonal_exogenous.SeasonalExogenousSpecification + +SeasonalExogenousSpecification + +concentrate_scale : bool +d : int +enforce_invertibility : bool +enforce_stationarity : bool +family : str +fixed_parameters : tuple[tuple[str, float], ...] +initialization_method : str +max_iterations : int +optimizer : str +p : int +q : int +regressor_names : tuple[str, ...] +seasonal_cycle_ms : int +seasonal_d : int +seasonal_p : int +seasonal_period : int +seasonal_q : int +specification_id : str +trend : str +use_exact_diffuse : bool + +__init__(self, specification_id: str, family: str, p: int, d: int, q: int, seasonal_p: int, seasonal_d: int, seasonal_q: int, seasonal_period: int, seasonal_cycle_ms: int, trend: str, initialization_method: str, optimizer: str, enforce_stationarity: bool, enforce_invertibility: bool, concentrate_scale: bool, use_exact_diffuse: bool, regressor_names: tuple[str, ...], fixed_parameters: tuple[tuple[str, float], ...], max_iterations: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.broker_capture.adapters.SequenceBrokerCaptureAdapterV1 + +SequenceBrokerCaptureAdapterV1 + +adapter_id : str +adapter_version : str +messages : tuple[BrokerAdapterMessageV1, ...] + +__init__(self, adapter_id: str, adapter_version: str, messages: tuple[BrokerAdapterMessageV1, ...]): None +iter_messages(): Iterable[BrokerAdapterMessageV1] + + + +histdatacom.broker_capture.adapters.SequenceBrokerCaptureClockV1 + +SequenceBrokerCaptureClockV1 + +_index : int +samples : Sequence[tuple[int, int]] + +__init__(self, samples: Sequence[tuple[int, int]], _index: int): None +sample(): tuple[int, int] - + histdatacom.source_cleanup.SourceCleanupResult - -SourceCleanupResult - -by_suffix : dict[str, dict[str, int]] -deleted_count : int -deleted_size_bytes : int -dry_run : bool -errors : tuple[dict[str, str], ...] -matched_count : int -matched_size_bytes : int -root : str -suffixes : tuple[str, ...] - -__init__(self, root: str, dry_run: bool, suffixes: tuple[str, ...], matched_count: int, matched_size_bytes: int, deleted_count: int, deleted_size_bytes: int, by_suffix: dict[str, dict[str, int]], errors: tuple[dict[str, str], ...]): None -to_dict(): dict[str, Any] + +SourceCleanupResult + +by_suffix : dict[str, dict[str, int]] +deleted_count : int +deleted_size_bytes : int +dry_run : bool +errors : tuple[dict[str, str], ...] +matched_count : int +matched_size_bytes : int +root : str +suffixes : tuple[str, ...] + +__init__(self, root: str, dry_run: bool, suffixes: tuple[str, ...], matched_count: int, matched_size_bytes: int, deleted_count: int, deleted_size_bytes: int, by_suffix: dict[str, dict[str, int]], errors: tuple[dict[str, str], ...]): None +to_dict(): dict[str, Any] - + histdatacom.runtime_contracts.StageResult->histdatacom.runtime_contracts.WorkStatus - - -status + + +status + + + +histdatacom.synthetic.bars.StagedDerivedBarPublicationV1 + +StagedDerivedBarPublicationV1 + +committed_directory : Path +manifest +manifest_path : Path +manifest_ref : ArtifactRef +root : Path +staging_directory : Path + +__init__(self, root: Path, staging_directory: Path, committed_directory: Path, manifest: DerivedBarProductManifestV1): None + + + +histdatacom.synthetic.bars.StagedDerivedBarPublicationV1->histdatacom.synthetic.bars.DerivedBarProductManifestV1 + + +manifest + + + +histdatacom.synthetic.persistence.StagedReconstructionPublicationV1 + +StagedReconstructionPublicationV1 + +committed_directory : Path +manifest +manifest_path : Path +manifest_ref : ArtifactRef +root : Path +staging_directory : Path + +__init__(self, root: Path, staging_directory: Path, committed_directory: Path, manifest: ReconstructionProductManifestV1): None + + + +histdatacom.synthetic.persistence.StagedReconstructionPublicationV1->histdatacom.synthetic.persistence.ReconstructionProductManifestV1 + + +manifest + + + +histdatacom.synthetic.persistence.StagedReconstructionPublicationV2 + +StagedReconstructionPublicationV2 + +committed_directory : Path +manifest +manifest_path : Path +root : Path +staging_directory : Path + +__init__(self, root: Path, staging_directory: Path, committed_directory: Path, manifest: ReconstructionProductManifestV2): None + + + +histdatacom.synthetic.persistence.StagedReconstructionPublicationV2->histdatacom.synthetic.persistence.ReconstructionProductManifestV2 + + +manifest + + + +histdatacom.data_quality.state_space.StateSpaceResult + +StateSpaceResult + +annotations : tuple[Mapping[str, Any], ...] +diagnostics : Mapping[str, JSONValue] +input_result + +__init__(self, diagnostics: Mapping[str, JSONValue], annotations: tuple[Mapping[str, Any], ...], input_result: ClassicalModelInputResult): None + + + +histdatacom.data_quality.state_space.StateSpaceResult->histdatacom.data_quality.classical_model_contracts.ClassicalModelInputResult + + +input_result + + + +histdatacom.data_quality.state_space.StateSpaceSpecification + +StateSpaceSpecification + +approximate_diffuse_variance : float +autoregressive_order : int +component_count : int +cycle : bool +damped_cycle : bool +family : str +fixed_parameters : tuple[tuple[str, float], ...] +initialization_method : str +irregular : bool +level : bool +max_iterations : int +optimizer : str +seasonal_cycle_ms : int +seasonal_period : int +specification_id : str +stochastic_cycle : bool +stochastic_level : bool +stochastic_seasonal : bool +stochastic_trend : bool +trend : bool + +__init__(self, specification_id: str, family: str, irregular: bool, stochastic_level: bool, stochastic_trend: bool, seasonal_period: int, seasonal_cycle_ms: int, stochastic_seasonal: bool, cycle: bool, stochastic_cycle: bool, damped_cycle: bool, autoregressive_order: int, initialization_method: str, approximate_diffuse_variance: float, optimizer: str, fixed_parameters: tuple[tuple[str, float], ...], max_iterations: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.market_context.contracts.StaticMarketContextSourceAdapterV1 + +StaticMarketContextSourceAdapterV1 + +adapter_name : str +adapter_version : str +events : tuple[MarketContextEventV1, ...] + +__init__(self, adapter_name: str, adapter_version: str, events: tuple[MarketContextEventV1, ...]): None +__post_init__(): None +load_events(): Iterable[MarketContextEventV1] - + histdatacom.runtime_contracts.StatusEvent - -StatusEvent - -message : str -metadata : dict[str, JSONValue] -stage : str -status -timestamp_utc : str -work_id : str - -__init__(self, status: WorkStatus, stage: str, message: str, work_id: str, timestamp_utc: str, metadata: dict[str, JSONValue]): None -from_dict(data: Mapping[str, Any]): 'StatusEvent' -to_dict(): dict[str, JSONValue] + +StatusEvent + +message : str +metadata : dict[str, JSONValue] +stage : str +status +timestamp_utc : str +work_id : str + +__init__(self, status: WorkStatus, stage: str, message: str, work_id: str, timestamp_utc: str, metadata: dict[str, JSONValue]): None +from_dict(data: Mapping[str, Any]): 'StatusEvent' +to_dict(): dict[str, JSONValue] - + histdatacom.runtime_contracts.StatusEvent->histdatacom.runtime_contracts.WorkStatus - - -status + + +status + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationCaseV1 + +StrategyEvaluationCaseV1 + +alignment_window_id : str +bar_interval_code : str | None +broker_profile_id : str | None +case_id : str +end_ns : int +ensemble_member_id : str +information_audit_id : str +information_manifest_id : str +information_mode +invalid_for_backtest_reason : str | None +run_id : str +schema_version : str +source_artifact_id : str +source_kind +source_scope : str | None +start_ns : int +symbol : str +valid_for_backtest : bool + +__init__(self, run_id: str, alignment_window_id: str, source_kind: StrategySourceKind, source_artifact_id: str, symbol: str, start_ns: int, end_ns: int, information_mode: InformationMode, information_manifest_id: str, information_audit_id: str, ensemble_member_id: str, broker_profile_id: str | None, source_scope: str | None, bar_interval_code: str | None, invalid_for_backtest_reason: str | None, case_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategyEvaluationCaseV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationCaseV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.strategy_sensitivity.StrategySourceKind + +StrategySourceKind + +name + +from_value(value: str | 'StrategySourceKind'): 'StrategySourceKind' + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationCaseV1->histdatacom.synthetic.strategy_sensitivity.StrategySourceKind + + +source_kind + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationFailure + +StrategyEvaluationFailure + + + + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationPlanV1 + +StrategyEvaluationPlanV1 + +cases : tuple[StrategyEvaluationCaseV1, ...] +execution +invalid_for_backtest_reason : str | None +plan_id : str +policy +run_id : str +schema_version : str +strategy +valid_for_backtest : bool + +__init__(self, run_id: str, strategy: StrategySpecificationV1, execution: StrategyExecutionSpecificationV1, policy: StrategyEvaluationPolicyV1, cases: tuple[StrategyEvaluationCaseV1, ...], invalid_for_backtest_reason: str | None, plan_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategyEvaluationPlanV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationPolicyV1 + +StrategyEvaluationPolicyV1 + +horizons_ns : tuple[int, ...] +max_cases : int +max_payload_bytes : int +max_pending_signals : int +max_quotes_per_window : int +max_signals_per_window : int +max_slices : int +policy_id : str +rounding_digits : int +schema_version : str + +__init__(self, horizons_ns: tuple[int, ...], max_cases: int, max_quotes_per_window: int, max_signals_per_window: int, max_pending_signals: int, max_slices: int, max_payload_bytes: int, rounding_digits: int, policy_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategyEvaluationPolicyV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationPlanV1->histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationPolicyV1 + + +policy + + + +histdatacom.synthetic.strategy_sensitivity.StrategyExecutionSpecificationV1 + +StrategyExecutionSpecificationV1 + +entry_latency_ns : int +exposure_semantics : str +fixed_cost_bps_per_side : float +max_execution_wait_ns : int +price_semantics : str +schema_version : str +slippage_bps_per_side : float +specification_id : str + +__init__(self, entry_latency_ns: int, max_execution_wait_ns: int, slippage_bps_per_side: float, fixed_cost_bps_per_side: float, price_semantics: str, exposure_semantics: str, specification_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategyExecutionSpecificationV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationPlanV1->histdatacom.synthetic.strategy_sensitivity.StrategyExecutionSpecificationV1 + + +execution + + + +histdatacom.synthetic.strategy_sensitivity.StrategySpecificationV1 + +StrategySpecificationV1 + +implementation_version : str +method_id : str +parameters : Mapping[str, JSONValue] +schema_version : str +specification_id : str + +__init__(self, method_id: str, implementation_version: str, parameters: Mapping[str, JSONValue], specification_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategySpecificationV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationPlanV1->histdatacom.synthetic.strategy_sensitivity.StrategySpecificationV1 + + +strategy + + + +histdatacom.synthetic.strategy_sensitivity.StrategyQuoteV1 + +StrategyQuoteV1 + +ask : float +bar_interval_code : str | None +bid : float +broker_profile_id : str | None +ensemble_member_id : str | None +epoch_id : str +event_sequence : int +event_state : str +event_time_ns : int +mid : float +order_key : tuple[int, int, str] +quote_id : str +schema_version : str +session : str +source_event_id : str +source_scope : str | None +sparsity : str +spread : float +symbol : str + +__init__(self, source_event_id: str, symbol: str, event_time_ns: int, event_sequence: int, bid: float, ask: float, epoch_id: str, session: str, event_state: str, sparsity: str, ensemble_member_id: str | None, broker_profile_id: str | None, source_scope: str | None, bar_interval_code: str | None, quote_id: str, schema_version: str): None +__post_init__(): None +from_benchmark_event(event: BenchmarkEventV1): 'StrategyQuoteV1' +from_derived_bar(bar: DerivedBarV1): 'StrategyQuoteV1' +from_dict(data: Mapping[str, Any]): 'StrategyQuoteV1' +from_synthetic_event(event: SyntheticEventV1): 'StrategyQuoteV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyResourceLimitError + +StrategyResourceLimitError + + + + + + +histdatacom.synthetic.strategy_sensitivity.StrategyResourceLimitError->histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationFailure + + + + + +histdatacom.synthetic.strategy_sensitivity.StrategyRestorationResultV1 + +StrategyRestorationResultV1 + +alignment_window_id : str +approaches_dense_reference : bool +broker_profile_id : str +candidate_absolute_error_bps : float +candidate_case_id : str +candidate_response_bps : float +candidate_source_kind +degraded_absolute_error_bps : float +degraded_response_bps : float +dense_reference_response_bps : float +ensemble_member_id : str +epoch_id : str +event_state : str +horizon_ns : int +restoration_gain_bps : float +result_id : str +schema_version : str +session : str +sparsity : str +symbol : str + +__init__(self, alignment_window_id: str, candidate_case_id: str, candidate_source_kind: StrategySourceKind, symbol: str, epoch_id: str, session: str, event_state: str, sparsity: str, broker_profile_id: str, ensemble_member_id: str, horizon_ns: int, dense_reference_response_bps: float, degraded_response_bps: float, candidate_response_bps: float, degraded_absolute_error_bps: float, candidate_absolute_error_bps: float, restoration_gain_bps: float, approaches_dense_reference: bool, result_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategyRestorationResultV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyRestorationResultV1->histdatacom.synthetic.strategy_sensitivity.StrategySourceKind + + +candidate_source_kind + + + +histdatacom.synthetic.strategy_sensitivity.StrategySensitivityReportV1 + +StrategySensitivityReportV1 + +plan +report_id : str +restoration_results : tuple[StrategyRestorationResultV1, ...] +restoration_unavailable_count : int +schema_version : str +summary : dict[str, JSONValue] +uncertainty_summaries : tuple[StrategyUncertaintySummaryV1, ...] +valid_for_backtest : bool +window_results : tuple[StrategyWindowResultV1, ...] + +__init__(self, plan: StrategyEvaluationPlanV1, window_results: tuple[StrategyWindowResultV1, ...], uncertainty_summaries: tuple[StrategyUncertaintySummaryV1, ...], restoration_results: tuple[StrategyRestorationResultV1, ...], restoration_unavailable_count: int, report_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategySensitivityReportV1' +from_json(text: str): 'StrategySensitivityReportV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.strategy_sensitivity.StrategySensitivityReportV1->histdatacom.synthetic.strategy_sensitivity.StrategyEvaluationPlanV1 + + +plan + + + +histdatacom.synthetic.strategy_sensitivity.StrategySide + +StrategySide + +name +sign : float + +from_value(value: str | 'StrategySide'): 'StrategySide' + + + +histdatacom.synthetic.strategy_sensitivity.StrategySignalEngineV1 + +StrategySignalEngineV1 + +specification + +start_window +(evaluation_case: StrategyEvaluationCaseV1): StrategySignalStateV1 + + + +histdatacom.synthetic.strategy_sensitivity.StrategySignalEngineV1->histdatacom.synthetic.strategy_sensitivity.StrategySpecificationV1 + + +specification + + + +histdatacom.synthetic.strategy_sensitivity.StrategySignalStateV1 + +StrategySignalStateV1 + + +observe +(quote: StrategyQuoteV1): Sequence[StrategySignalV1] + + + +histdatacom.synthetic.strategy_sensitivity.StrategySignalV1 + +StrategySignalV1 + +decision_time_ns : int +schema_version : str +side +signal_id : str +source_quote_id : str +strategy_specification_id : str + +__init__(self, strategy_specification_id: str, source_quote_id: str, decision_time_ns: int, side: StrategySide, signal_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategySignalV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategySignalV1->histdatacom.synthetic.strategy_sensitivity.StrategySide + + +side + + + +histdatacom.synthetic.strategy_sensitivity.StrategySliceResultV1 + +StrategySliceResultV1 + +alignment_window_id : str +broker_profile_id : str +case_id : str +comparison_key : tuple[str, str, str, str, str, int] +completed_count : int +ensemble_member_id : str +epoch_id : str +event_state : str +favorable_response_rate : float +horizon_ns : int +mean_cost_drag_bps : float +mean_entry_delay_ns : float +mean_gross_response_bps : float +mean_net_execution_response_bps : float +missing_support_count : int +schema_version : str +session : str +signal_count : int +slice_result_id : str +source_kind +sparsity : str +symbol : str + +__init__(self, case_id: str, alignment_window_id: str, source_kind: StrategySourceKind, symbol: str, epoch_id: str, session: str, event_state: str, sparsity: str, broker_profile_id: str, ensemble_member_id: str, horizon_ns: int, signal_count: int, completed_count: int, missing_support_count: int, mean_gross_response_bps: float, mean_net_execution_response_bps: float, mean_cost_drag_bps: float, mean_entry_delay_ns: float, favorable_response_rate: float, slice_result_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategySliceResultV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategySliceResultV1->histdatacom.synthetic.strategy_sensitivity.StrategySourceKind + + +source_kind + + + +histdatacom.synthetic.strategy_sensitivity.StrategySpecificationV1->histdatacom.synthetic.strategy_sensitivity.ReferenceMomentumStrategyV1 + + +specification + + + +histdatacom.synthetic.strategy_sensitivity.StrategyUncertaintySummaryV1 + +StrategyUncertaintySummaryV1 + +broker_profile_id : str +completed_outcome_count : int +ensemble_member_ids : tuple[str, ...] +epoch_id : str +event_state : str +horizon_ns : int +max_net_response_bps : float +mean_net_response_bps : float +min_net_response_bps : float +schema_version : str +session : str +source_kind +sparsity : str +standard_deviation_bps : float +summary_id : str +symbol : str +window_count : int + +__init__(self, source_kind: StrategySourceKind, symbol: str, epoch_id: str, session: str, event_state: str, sparsity: str, broker_profile_id: str, horizon_ns: int, window_count: int, ensemble_member_ids: tuple[str, ...], completed_outcome_count: int, mean_net_response_bps: float, min_net_response_bps: float, max_net_response_bps: float, standard_deviation_bps: float, summary_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategyUncertaintySummaryV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyUncertaintySummaryV1->histdatacom.synthetic.strategy_sensitivity.StrategySourceKind + + +source_kind + + + +histdatacom.synthetic.strategy_sensitivity.StrategyWindowResultV1 + +StrategyWindowResultV1 + +alignment_window_id : str +case_id : str +completed_outcome_count : int +invalid_for_backtest_reason : str | None +max_interarrival_ns : int +mean_interarrival_ns : float +mean_spread_bps : float +missing_support_count : int +outcome_count : int +quote_count : int +reason : str | None +schema_version : str +signal_count : int +slices : tuple[StrategySliceResultV1, ...] +source_kind +status +valid_for_backtest : bool +window_result_id : str + +__init__(self, case_id: str, alignment_window_id: str, source_kind: StrategySourceKind, status: StrategyWindowStatus, quote_count: int, signal_count: int, outcome_count: int, completed_outcome_count: int, missing_support_count: int, mean_interarrival_ns: float, max_interarrival_ns: int, mean_spread_bps: float, slices: tuple[StrategySliceResultV1, ...], reason: str | None, invalid_for_backtest_reason: str | None, window_result_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'StrategyWindowResultV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.strategy_sensitivity.StrategyWindowResultV1->histdatacom.synthetic.strategy_sensitivity.StrategySourceKind + + +source_kind + + + +histdatacom.synthetic.strategy_sensitivity.StrategyWindowStatus + +StrategyWindowStatus + +name + +from_value(value: str | 'StrategyWindowStatus'): 'StrategyWindowStatus' + + + +histdatacom.synthetic.strategy_sensitivity.StrategyWindowResultV1->histdatacom.synthetic.strategy_sensitivity.StrategyWindowStatus + + +status - + histdatacom.orchestration.workflows.SymbolTimeframeWorkflow - -SymbolTimeframeWorkflow - -_executor : ChildWorkflowExecutor | None -_progress - -__init__(executor: ChildWorkflowExecutor | None): None -run(payload: dict[str, Any]): dict -status(): dict + +SymbolTimeframeWorkflow + +_executor : ChildWorkflowExecutor | None +_progress + +__init__(executor: ChildWorkflowExecutor | None): None +run(payload: dict[str, Any]): dict +status(): dict + + + +histdatacom.synthetic.contracts.SyntheticEnsembleManifestV1 + +SyntheticEnsembleManifestV1 + +configuration_ids : tuple[str, ...] +ensemble_id : str +members : tuple[SyntheticEnsembleMemberV1, ...] +primary_member_id : str +run_id : str +schema_version : str +source_version_ids : tuple[str, ...] + +__init__(self, run_id: str, primary_member_id: str, members: tuple[SyntheticEnsembleMemberV1, ...], source_version_ids: tuple[str, ...], configuration_ids: tuple[str, ...], ensemble_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'SyntheticEnsembleManifestV1' +from_json(text: str): 'SyntheticEnsembleManifestV1' +from_streams(streams: Iterable[SyntheticEventStreamV1]): 'SyntheticEnsembleManifestV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.contracts.SyntheticEnsembleMemberV1 + +SyntheticEnsembleMemberV1 + +content_sha256 : str +event_count : int +member_id : str +observed_event_count : int +stream_id : str +synthetic_event_count : int + +__init__(self, member_id: str, stream_id: str, event_count: int, observed_event_count: int, synthetic_event_count: int, content_sha256: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'SyntheticEnsembleMemberV1' +from_stream(stream: SyntheticEventStreamV1): 'SyntheticEnsembleMemberV1' +to_dict(): dict[str, JSONValue] + + + +histdatacom.synthetic.contracts.SyntheticEventOrigin + +SyntheticEventOrigin + +name + +from_value(value: str | 'SyntheticEventOrigin'): 'SyntheticEventOrigin' + + + +histdatacom.synthetic.contracts.SyntheticEventStreamV1 + +SyntheticEventStreamV1 + +ensemble_member_id : str +events : tuple[SyntheticEventV1, ...] +observed_event_count : int +run_id : str +schema_version : str +source_version_ids : tuple[str, ...] +stream_id : str +symbol : str +synthetic_event_count : int + +__init__(self, run_id: str, ensemble_member_id: str, symbol: str, events: tuple[SyntheticEventV1, ...], source_version_ids: tuple[str, ...], stream_id: str, schema_version: str): None +__post_init__(): None +from_dict(data: Mapping[str, Any]): 'SyntheticEventStreamV1' +from_json(text: str): 'SyntheticEventStreamV1' +header_dict(): dict[str, JSONValue] +identity_payload(): dict[str, JSONValue] +merge(): 'SyntheticEventStreamV1' +to_dict(): dict[str, JSONValue] +to_json(): str +to_json_header(): str + + + +histdatacom.synthetic.contracts.SyntheticEventV1 + +SyntheticEventV1 + +anchor_interval_id : str | None +ask : float +bid : float +broker_profile_id : str | None +confidence : float | None +constraint_set_id : str | None +ensemble_member_id : str +event_id : str +event_sequence : int +event_time_ns : int +feed_epoch_id : str | None +generator_config_id : str | None +generator_id : str | None +generator_version : str | None +left_anchor_event_id : str | None +motif_id : str | None +origin +reference_id : str | None +right_anchor_event_id : str | None +run_id : str +schema_version : str +source_period : str | None +source_row_id : int | None +source_series_id : str | None +source_version_id : str +symbol : str + +__init__(self, origin: SyntheticEventOrigin, symbol: str, event_time_ns: int, event_sequence: int, bid: float, ask: float, run_id: str, ensemble_member_id: str, source_version_id: str, source_series_id: str | None, source_period: str | None, source_row_id: int | None, anchor_interval_id: str | None, left_anchor_event_id: str | None, right_anchor_event_id: str | None, generator_id: str | None, generator_version: str | None, generator_config_id: str | None, reference_id: str | None, motif_id: str | None, feed_epoch_id: str | None, broker_profile_id: str | None, constraint_set_id: str | None, confidence: float | None, event_id: str, schema_version: str): None +__post_init__(): None +_validate_lineage(): None +from_dict(data: Mapping[str, Any]): 'SyntheticEventV1' +from_json(text: str): 'SyntheticEventV1' +generated(): 'SyntheticEventV1' +identity_payload(): dict[str, JSONValue] +observed(): 'SyntheticEventV1' +to_dict(): dict[str, JSONValue] +to_json(): str + + + +histdatacom.synthetic.contracts.SyntheticEventV1->histdatacom.synthetic.contracts.SyntheticEventOrigin + + +origin + + + +histdatacom.synthetic.reconstruction_plan.SyntheticInfillPlanV1 + +SyntheticInfillPlanV1 + +artifact_graph : Mapping[str, ArtifactRef] +configuration_id : str +delivery_mode +execution_manifest_id : str +information_mode +plan_id : str +refusals : tuple[ReconstructionPlanRefusalV1, ...] +requested_end_ns : int +requested_start_ns : int +resources +run +schema_version : str +status : str +workflow_requests : tuple[ReconstructionWorkflowRequestV1, ...] + +__init__(self, run: ReconstructionRunV1, configuration_id: str, execution_manifest_id: str, information_mode: InformationMode, delivery_mode: ReconstructionDeliveryMode, requested_start_ns: int, requested_end_ns: int, workflow_requests: tuple[ReconstructionWorkflowRequestV1, ...], artifact_graph: Mapping[str, ArtifactRef], resources: ReconstructionPlanResourceSummaryV1, refusals: tuple[ReconstructionPlanRefusalV1, ...], plan_id: str, schema_version: str): None +__post_init__(): None +dry_run_payload(): dict[str, JSONValue] +from_dict(data: Mapping[str, Any]): 'SyntheticInfillPlanV1' +from_json(text: str): 'SyntheticInfillPlanV1' +identity_payload(): dict[str, JSONValue] +to_dict(): dict[str, JSONValue] +to_dry_run_json(): str +to_json(): str + + + +histdatacom.synthetic.reconstruction_plan.SyntheticInfillPlanV1->histdatacom.synthetic.information.InformationMode + + +information_mode + + + +histdatacom.synthetic.reconstruction_plan.SyntheticInfillPlanV1->histdatacom.synthetic.delivery.ReconstructionDeliveryMode + + +delivery_mode + + + +histdatacom.synthetic.reconstruction_plan.SyntheticInfillPlanV1->histdatacom.synthetic.reconstruction_plan.ReconstructionPlanResourceSummaryV1 + + +resources + + + +histdatacom.synthetic.reconstruction_plan.SyntheticInfillPlanV1->histdatacom.synthetic.streaming.ReconstructionRunV1 + + +run + + + +histdatacom.data_quality.synthetic_generation.SyntheticTickGenerationProfile + +SyntheticTickGenerationProfile + +anchor_mode : str +block_size : int +diagnostic_sample_limit : int +max_abs_log_return : float +max_generated_rows : int +max_reference_rows : int +method : str +minimum_reference_rows : int +overwrite_existing : bool +rounding_digits : int +seed : int + +__init__(self, method: str, seed: int, block_size: int, minimum_reference_rows: int, max_reference_rows: int, max_generated_rows: int, max_abs_log_return: float, rounding_digits: int, diagnostic_sample_limit: int, anchor_mode: str, overwrite_existing: bool): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] + + + +histdatacom.data_quality.synthetic_generation.SyntheticTickGenerationResult + +SyntheticTickGenerationResult + +candidate_report : QualityReport | None +diagnostics : Mapping[str, JSONValue] +frame : Any + +__init__(self, frame: Any, diagnostics: Mapping[str, JSONValue], candidate_report: QualityReport | None): None + + + +histdatacom.broker_capture.adapters.SystemBrokerCaptureClockV1 + +SystemBrokerCaptureClockV1 + + +sample(): tuple[int, int] - + histdatacom.orchestration.workflows.TemporalActivityExecutor - -TemporalActivityExecutor - - -execute_activity(activity_name: str, payload: Mapping[str, JSONValue]): Mapping[str, Any] + +TemporalActivityExecutor + + +execute_activity(activity_name: str, payload: Mapping[str, JSONValue]): Mapping[str, Any] - + histdatacom.orchestration.workflows.TemporalChildWorkflowExecutor - -TemporalChildWorkflowExecutor - - -execute_child_workflow(workflow_name: str, payload: Mapping[str, JSONValue]): Mapping[str, Any] + +TemporalChildWorkflowExecutor + + +execute_child_workflow(workflow_name: str, payload: Mapping[str, JSONValue]): Mapping[str, Any] - + histdatacom.orchestration.client.TemporalDependencyError - -TemporalDependencyError - -code : str - - + +TemporalDependencyError + +code : str + + - + histdatacom.orchestration.client.TemporalDependencyError->histdatacom.orchestration.client.OrchestrationUnavailableError - - + + - + histdatacom.orchestration.resources.TemporalRuntimeArtifact - -TemporalRuntimeArtifact - -archive_format : str -archive_name : str -archive_sha256 : str -archive_size_bytes : int -archive_url : str -executable_name : str -license : str -machine : str -platform_key : str -system : str -upstream_repository : str - -__init__(self, platform_key: str, system: str, machine: str, archive_name: str, archive_format: str, archive_url: str, archive_sha256: str, archive_size_bytes: int, executable_name: str, license: str, upstream_repository: str): None -from_dict(platform_key: str, data: Mapping[str, Any]): 'TemporalRuntimeArtifact' + +TemporalRuntimeArtifact + +archive_format : str +archive_name : str +archive_sha256 : str +archive_size_bytes : int +archive_url : str +executable_name : str +license : str +machine : str +platform_key : str +system : str +upstream_repository : str + +__init__(self, platform_key: str, system: str, machine: str, archive_name: str, archive_format: str, archive_url: str, archive_sha256: str, archive_size_bytes: int, executable_name: str, license: str, upstream_repository: str): None +from_dict(platform_key: str, data: Mapping[str, Any]): 'TemporalRuntimeArtifact' - + histdatacom.orchestration.resources.TemporalRuntimeCacheEntry - -TemporalRuntimeCacheEntry - -archive_sha256 : str -executable : Path -path : Path -platform_key : str -provenance_path : Path -reason : str -valid : bool -version : str - -__init__(self, path: Path, executable: Path, provenance_path: Path, platform_key: str, version: str, archive_sha256: str, valid: bool, reason: str): None -to_dict(): dict[str, Any] + +TemporalRuntimeCacheEntry + +archive_sha256 : str +executable : Path +path : Path +platform_key : str +provenance_path : Path +reason : str +valid : bool +version : str + +__init__(self, path: Path, executable: Path, provenance_path: Path, platform_key: str, version: str, archive_sha256: str, valid: bool, reason: str): None +to_dict(): dict[str, Any] - + histdatacom.orchestration.resources.TemporalRuntimeChecksumError - -TemporalRuntimeChecksumError - - - + +TemporalRuntimeChecksumError + + + - + histdatacom.orchestration.resources.TemporalRuntimeProvisioningError - -TemporalRuntimeProvisioningError - - - + +TemporalRuntimeProvisioningError + + + - + histdatacom.orchestration.resources.TemporalRuntimeChecksumError->histdatacom.orchestration.resources.TemporalRuntimeProvisioningError - - + + - + histdatacom.orchestration.resources.TemporalRuntimeIndex - -TemporalRuntimeIndex - -component : str -platforms : dict[str, TemporalRuntimeArtifact] -release_base_url : str -schema_version : int -version : str - -__init__(self, schema_version: int, component: str, version: str, release_base_url: str, platforms: dict[str, TemporalRuntimeArtifact]): None -from_dict(data: Mapping[str, Any]): 'TemporalRuntimeIndex' + +TemporalRuntimeIndex + +component : str +platforms : dict[str, TemporalRuntimeArtifact] +release_base_url : str +schema_version : int +version : str + +__init__(self, schema_version: int, component: str, version: str, release_base_url: str, platforms: dict[str, TemporalRuntimeArtifact]): None +from_dict(data: Mapping[str, Any]): 'TemporalRuntimeIndex' - + histdatacom.orchestration.resources.TemporalRuntimeOfflineError - -TemporalRuntimeOfflineError - - - + +TemporalRuntimeOfflineError + + + - + histdatacom.orchestration.resources.TemporalRuntimeOfflineError->histdatacom.orchestration.resources.TemporalRuntimeProvisioningError - - + + - + histdatacom.orchestration.resources.TemporalRuntimeProvisioningError->histdatacom.orchestration.resources.OrchestrationResourceError - - + + - + histdatacom.orchestration.resources.TemporalRuntimeResolution - -TemporalRuntimeResolution - -archive_sha256 : str -cache_entry : Path | None -executable : Path -network_fetch : bool -platform_key : str -provenance_path : Path | None -source : str -version : str - -__init__(self, executable: Path, source: str, platform_key: str, version: str, archive_sha256: str, cache_entry: Path | None, provenance_path: Path | None, network_fetch: bool): None -to_dict(): dict[str, Any] + +TemporalRuntimeResolution + +archive_sha256 : str +cache_entry : Path | None +executable : Path +network_fetch : bool +platform_key : str +provenance_path : Path | None +source : str +version : str + +__init__(self, executable: Path, source: str, platform_key: str, version: str, archive_sha256: str, cache_entry: Path | None, provenance_path: Path | None, network_fetch: bool): None +to_dict(): dict[str, Any] - + histdatacom.orchestration.throughput.ThroughputBenchmarkReport - -ThroughputBenchmarkReport - -comparisons : tuple[ThroughputComparison, ...] -notes : tuple[str, ...] -runtime_startup -started_status -stopped_status : OrchestrationStatus | None -worker_config - -__init__(self, comparisons: tuple[ThroughputComparison, ...], runtime_startup: BenchmarkMeasurement, worker_config: OrchestrationWorkerConfig, started_status: OrchestrationStatus, stopped_status: OrchestrationStatus | None, notes: tuple[str, ...]): None -to_dict(): dict[str, JSONValue] -to_json(): str + +ThroughputBenchmarkReport + +comparisons : tuple[ThroughputComparison, ...] +notes : tuple[str, ...] +runtime_startup +started_status +stopped_status : OrchestrationStatus | None +worker_config + +__init__(self, comparisons: tuple[ThroughputComparison, ...], runtime_startup: BenchmarkMeasurement, worker_config: OrchestrationWorkerConfig, started_status: OrchestrationStatus, stopped_status: OrchestrationStatus | None, notes: tuple[str, ...]): None +to_dict(): dict[str, JSONValue] +to_json(): str - + histdatacom.orchestration.throughput.ThroughputBenchmarkReport->histdatacom.orchestration.performance.BenchmarkMeasurement - - -runtime_startup + + +runtime_startup - + histdatacom.orchestration.throughput.ThroughputBenchmarkReport->histdatacom.orchestration.supervisor.OrchestrationStatus - - -started_status + + +started_status - + histdatacom.orchestration.throughput.ThroughputBenchmarkReport->histdatacom.orchestration.queues.OrchestrationWorkerConfig - - -worker_config + + +worker_config - + histdatacom.orchestration.throughput.ThroughputBenchmarkScenario - -ThroughputBenchmarkScenario - -name : str -notes : str -operations : tuple[str, ...] -request -work_item_count : int - -__init__(self, name: str, request: RunRequest, operations: tuple[str, ...], work_item_count: int, notes: str): None -to_dict(): dict[str, JSONValue] + +ThroughputBenchmarkScenario + +name : str +notes : str +operations : tuple[str, ...] +request +work_item_count : int + +__init__(self, name: str, request: RunRequest, operations: tuple[str, ...], work_item_count: int, notes: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.throughput.ThroughputBenchmarkScenario->histdatacom.runtime_contracts.RunRequest - - -request + + +request - + histdatacom.orchestration.throughput.ThroughputComparison - -ThroughputComparison - -baseline : RuntimeBenchmarkResult | None -runtime -runtime_to_baseline_elapsed_ratio : float | None -scenario - -__init__(self, scenario: ThroughputBenchmarkScenario, runtime: RuntimeBenchmarkResult, baseline: RuntimeBenchmarkResult | None): None -to_dict(): dict[str, JSONValue] + +ThroughputComparison + +baseline : RuntimeBenchmarkResult | None +runtime +runtime_to_baseline_elapsed_ratio : float | None +scenario + +__init__(self, scenario: ThroughputBenchmarkScenario, runtime: RuntimeBenchmarkResult, baseline: RuntimeBenchmarkResult | None): None +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.throughput.ThroughputComparison->histdatacom.orchestration.throughput.RuntimeBenchmarkResult - - -runtime + + +runtime - + histdatacom.orchestration.throughput.ThroughputComparison->histdatacom.orchestration.throughput.ThroughputBenchmarkScenario - - -scenario + + +scenario - + histdatacom.fx_enums.TimeFormat - -TimeFormat - -name - -list_keys(): set -list_values(): set + +TimeFormat + +name + +list_keys(): set +list_values(): set - + histdatacom.fx_enums.TimePrecision - -TimePrecision - -name - -list_keys(): set -list_values(): set + +TimePrecision + +name + +list_keys(): set +list_values(): set - + histdatacom.fx_enums.Timeframe - -Timeframe - -name - -convert_to_values(timeframe_set: set): set -list_keys(): set -list_values(): set + +Timeframe + +name + +convert_to_values(timeframe_set: set): set +list_keys(): set +list_values(): set + + + +histdatacom.data_quality.training_features.TrainingFeatureDefinition + +TrainingFeatureDefinition + +default : JSONValue | None +description : str +dtype : str +grain : str +name : str +nullable : bool +source : str + +__init__(self, name: str, dtype: str, default: JSONValue | None, description: str, source: str, grain: str, nullable: bool): None - + histdatacom.orchestration.resources.UnsupportedOrchestrationPlatform - -UnsupportedOrchestrationPlatform - - - + +UnsupportedOrchestrationPlatform + + + - + histdatacom.orchestration.resources.UnsupportedOrchestrationPlatform->histdatacom.orchestration.resources.OrchestrationResourceError - - + + - + histdatacom.activity_stages.UrlFormMetadata - -UrlFormMetadata - -bytes_length : str -data_date : str -data_datemonth : str -data_format : str -data_fxpair : str -data_timeframe : str -data_tk : str -encoding : str - -__init__(self, data_tk: str, data_date: str, data_datemonth: str, data_format: str, data_timeframe: str, data_fxpair: str, encoding: str, bytes_length: str): None -to_dict(): dict[str, JSONValue] + +UrlFormMetadata + +bytes_length : str +data_date : str +data_datemonth : str +data_format : str +data_fxpair : str +data_timeframe : str +data_tk : str +encoding : str + +__init__(self, data_tk: str, data_date: str, data_datemonth: str, data_format: str, data_timeframe: str, data_fxpair: str, encoding: str, bytes_length: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.activity_stages.UrlPageData - -UrlPageData - -bytes_length : str -encoding : str -headers : dict[str, str] -html : str - -__init__(self, html: str, encoding: str, bytes_length: str, headers: dict[str, str]): None -from_mapping(data: Mapping[str, Any]): 'UrlPageData' -to_dict(): dict[str, JSONValue] + +UrlPageData + +bytes_length : str +encoding : str +headers : dict[str, str] +html : str + +__init__(self, html: str, encoding: str, bytes_length: str, headers: dict[str, str]): None +from_mapping(data: Mapping[str, Any]): 'UrlPageData' +to_dict(): dict[str, JSONValue] - + histdatacom.exceptions.UrlValidationError->histdatacom.exceptions.HistDataOperationError - - + + - + histdatacom.scraper.urls.Urls - -Urls - -base_url : str - -__init__(): None -generate_form_urls(start_yearmonth: str, end_yearmonth: str, formats: set, pairs: Optional[Set[Any]], timeframes: set): Generator[str, None, None] + +Urls + +base_url : str + +__init__(): None +generate_form_urls(start_yearmonth: str, end_yearmonth: str, formats: set, pairs: Optional[Set[Any]], timeframes: set): Generator[str, None, None] - + histdatacom.scraper.urls.Urls->histdatacom.scraper.scraper.Scraper - - -urls + + +urls - + histdatacom.orchestration.workflows.ValidateUrlsWorkflow - -ValidateUrlsWorkflow - -status -workflow_name : str - -run(payload: dict[str, Any]): dict + +ValidateUrlsWorkflow + +status +workflow_name : str + +run(payload: dict[str, Any]): dict - + histdatacom.orchestration.workflows.ValidateUrlsWorkflow->histdatacom.orchestration.workflows._ActivityWorkflowBase - - + + + + + +histdatacom.data_quality.volatility.VolatilityResult + +VolatilityResult + +annotations : tuple[Mapping[str, Any], ...] +diagnostics : Mapping[str, JSONValue] +input_result + +__init__(self, diagnostics: Mapping[str, JSONValue], annotations: tuple[Mapping[str, Any], ...], input_result: ClassicalModelInputResult): None + + + +histdatacom.data_quality.volatility.VolatilityResult->histdatacom.data_quality.classical_model_contracts.ClassicalModelInputResult + + +input_result + + + +histdatacom.data_quality.volatility.VolatilitySpecification + +VolatilitySpecification + +covariance_type : str +distribution : str +family : str +initial_variance : float | None +innovation_order : int +input_definition : str +max_iterations : int +mean_model : str +mean_model_reference_id : str +optimizer_tolerance : float | None +parameter_bounds : tuple[tuple[str, float | None, float | None], ...] +scale_factor : float +specification_id : str +variance_initialization : str +variance_order : int + +__init__(self, specification_id: str, family: str, input_definition: str, mean_model: str, mean_model_reference_id: str, distribution: str, innovation_order: int, variance_order: int, scale_factor: float, variance_initialization: str, initial_variance: float | None, covariance_type: str, optimizer_tolerance: float | None, parameter_bounds: tuple[tuple[str, float | None, float | None], ...], max_iterations: int): None +__post_init__(): None +to_metadata(): dict[str, JSONValue] - + histdatacom.runtime_contracts.WorkItem->histdatacom.runtime_contracts.WorkStatus - - -status + + +status - + histdatacom.runtime_contracts.WorkStatus->histdatacom.records.Record - - -_status + + +_status - + histdatacom.orchestration.workflows.WorkflowInvocation - -WorkflowInvocation - -payload : dict[str, JSONValue] -task_queue : str -task_queue_lane -workflow_id : str -workflow_name : str - -__init__(self, workflow_name: str, workflow_id: str, task_queue_lane: TaskQueueLane, task_queue: str, payload: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +WorkflowInvocation + +payload : dict[str, JSONValue] +task_queue : str +task_queue_lane +workflow_id : str +workflow_name : str + +__init__(self, workflow_name: str, workflow_id: str, task_queue_lane: TaskQueueLane, task_queue: str, payload: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.workflows.WorkflowInvocation->histdatacom.orchestration.queues.TaskQueueLane - - -task_queue_lane + + +task_queue_lane - + histdatacom.orchestration.workflows.WorkflowProgress - -WorkflowProgress - -artifact_count : int -artifacts : tuple[ArtifactRef, ...] -completed_children : int -completed_stages : tuple[str, ...] -current_stage : str -event_count : int -events : tuple[StatusEvent, ...] -last_error : str -planned_children : tuple[str, ...] -rate_per_second : float -request_id : str -started_at_utc : str -status -total_children : int -unit : str -updated_at_utc : str -workflow_name : str - -__init__(self, workflow_name: str, request_id: str, status: WorkStatus, current_stage: str, total_children: int, completed_children: int, unit: str, started_at_utc: str, updated_at_utc: str, rate_per_second: float, last_error: str, event_count: int, artifact_count: int, planned_children: tuple[str, ...], completed_stages: tuple[str, ...], events: tuple[StatusEvent, ...], artifacts: tuple[ArtifactRef, ...]): None -_child_progress_event(stage: str, result: StageResult): StatusEvent -finish(status: WorkStatus): None -record_child(stage: str, result: StageResult): None -start(): None -to_dict(): dict[str, JSONValue] + +WorkflowProgress + +artifact_count : int +artifacts : tuple[ArtifactRef, ...] +completed_children : int +completed_stages : tuple[str, ...] +current_stage : str +event_count : int +events : tuple[StatusEvent, ...] +last_error : str +planned_children : tuple[str, ...] +rate_per_second : float +request_id : str +started_at_utc : str +status +total_children : int +unit : str +updated_at_utc : str +workflow_name : str + +__init__(self, workflow_name: str, request_id: str, status: WorkStatus, current_stage: str, total_children: int, completed_children: int, unit: str, started_at_utc: str, updated_at_utc: str, rate_per_second: float, last_error: str, event_count: int, artifact_count: int, planned_children: tuple[str, ...], completed_stages: tuple[str, ...], events: tuple[StatusEvent, ...], artifacts: tuple[ArtifactRef, ...]): None +_child_progress_event(stage: str, result: StageResult): StatusEvent +finish(status: WorkStatus): None +record_child(stage: str, result: StageResult): None +start(): None +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.workflows.WorkflowProgress->histdatacom.orchestration.workflows.HistDataRunWorkflow - - -_progress + + +_progress - + histdatacom.orchestration.workflows.WorkflowProgress->histdatacom.orchestration.workflows.SymbolTimeframeWorkflow - - -_progress + + +_progress - + histdatacom.orchestration.workflows.WorkflowProgress->histdatacom.runtime_contracts.WorkStatus - - -status + + +status - + histdatacom.orchestration.workflows.WorkflowProgress->histdatacom.orchestration.workflows._ActivityWorkflowBase - - -_progress + + +_progress - + histdatacom.orchestration.workflows.WorkflowSpec - -WorkflowSpec - -children : tuple[str, ...] -history_policy : str -lane -name : str -operation_family : str - -__init__(self, name: str, lane: TaskQueueLane, operation_family: str, children: tuple[str, ...], history_policy: str): None -to_dict(): dict[str, JSONValue] + +WorkflowSpec + +children : tuple[str, ...] +history_policy : str +lane +name : str +operation_family : str + +__init__(self, name: str, lane: TaskQueueLane, operation_family: str, children: tuple[str, ...], history_policy: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.orchestration.workflows.WorkflowSpec->histdatacom.orchestration.queues.TaskQueueLane - - -lane + + +lane + + + +histdatacom.market_context.corpus._AcquisitionBudget + +_AcquisitionBudget + +consumed_bytes : int +profile +remaining_bytes : int +remaining_seconds : float +started : float + +__init__(self, profile: MarketContextFetchProfileV1, started: float, consumed_bytes: int): None +check(): None +consume(size: int): None + + + +histdatacom.market_context.corpus._AcquisitionBudget->histdatacom.market_context.corpus.MarketContextFetchProfileV1 + + +profile + + + +histdatacom.synthetic.activity._ActivityAccumulator + +_ActivityAccumulator + +ensemble_member_id +last_positions : dict[str, tuple[int, int]] +policy +run_id +states : dict[tuple[str, ActivitySliceScope], _ActivitySliceState] +symbols : set[str] + +__init__(): None +add(event: _ActivityEventView): None +finalize(): tuple[ReconstructionActivitySliceV1, ...] + + + +histdatacom.synthetic.activity._ActivitySliceState + +_ActivitySliceState + +broker_profile_ids : set[str] +confidence_count : int +confidence_total : float +constraint_set_ids : set[str] +digest : Any +event_count : int +feed_epoch_ids : set[str] +first_time_ns : int | None +generator_config_ids : set[str] +generator_ids : set[str] +generator_versions : set[str] +interarrival_count : int +interarrival_total_ns : int +last_ask : float | None +last_bid : float | None +last_time_ns : int | None +max_interarrival_ns : int | None +max_spread : float | None +min_interarrival_ns : int | None +min_spread : float | None +motif_ids : set[str] +policy +price_change_count : int +provenance_digests : dict[str, Any] +provenance_occurrence_counts : dict[str, int] +reference_ids : set[str] +scope +source_version_ids : set[str] +spread_total : float +stale_quote_count : int +stream_ids : set[str] +symbol : str +truncated_provenance : set[str] + +__init__(self, symbol: str, scope: ActivitySliceScope, policy: ReconstructionActivityPolicyV1, event_count: int, first_time_ns: int | None, last_time_ns: int | None, last_bid: float | None, last_ask: float | None, interarrival_count: int, interarrival_total_ns: int, min_interarrival_ns: int | None, max_interarrival_ns: int | None, price_change_count: int, stale_quote_count: int, spread_total: float, min_spread: float | None, max_spread: float | None, confidence_total: float, confidence_count: int, source_version_ids: set[str], generator_ids: set[str], generator_versions: set[str], generator_config_ids: set[str], reference_ids: set[str], motif_ids: set[str], feed_epoch_ids: set[str], broker_profile_ids: set[str], constraint_set_ids: set[str], stream_ids: set[str], provenance_occurrence_counts: dict[str, int], provenance_digests: dict[str, Any], truncated_provenance: set[str], digest: Any): None +_add_provenance(name: str, value: str): None +add(event: _ActivityEventView): None +finalize(): ReconstructionActivitySliceV1 + + + +histdatacom.synthetic.activity._ActivityAccumulator->histdatacom.synthetic.activity._ActivitySliceState + + +states + + + +histdatacom.synthetic.activity._ActivityEventView + +_ActivityEventView + +ask : float +bid : float +broker_profile_id : str | None +confidence : float | None +constraint_set_id : str | None +ensemble_member_id : str +event_id : str +event_sequence : int +event_time_ns : int +feed_epoch_id : str | None +generator_config_id : str | None +generator_id : str | None +generator_version : str | None +motif_id : str | None +origin +reference_id : str | None +run_id : str +source_version_id : str +stream_ids : tuple[str, ...] +symbol : str + +__init__(self, event_id: str, origin: SyntheticEventOrigin, symbol: str, event_time_ns: int, event_sequence: int, bid: float, ask: float, run_id: str, ensemble_member_id: str, source_version_id: str, generator_id: str | None, generator_version: str | None, generator_config_id: str | None, reference_id: str | None, motif_id: str | None, feed_epoch_id: str | None, broker_profile_id: str | None, constraint_set_id: str | None, confidence: float | None, stream_ids: tuple[str, ...]): None + + + +histdatacom.synthetic.activity._ActivityEventView->histdatacom.synthetic.contracts.SyntheticEventOrigin + + +origin + + + +histdatacom.synthetic.activity._ActivitySliceState->histdatacom.synthetic.activity.ActivitySliceScope + + +scope + + + +histdatacom.synthetic.activity._ActivitySliceState->histdatacom.synthetic.activity.ReconstructionActivityPolicyV1 + + +policy - + histdatacom.cache_status._Artifact - -_Artifact - -file_format : str -path : Path -size_bytes : int -symbol : str -timeframe : str - -__init__(self, path: Path, size_bytes: int, file_format: str, timeframe: str, symbol: str): None + +_Artifact + +file_format : str +path : Path +size_bytes : int +symbol : str +timeframe : str + +__init__(self, path: Path, size_bytes: int, file_format: str, timeframe: str, symbol: str): None + + + +histdatacom.data_quality.exponential_smoothing._Backend + +_Backend + +ets_model : Any +holt_winters : Any +version : str + +__init__(self, version: str, holt_winters: Any, ets_model: Any): None + + + +histdatacom.data_quality.autoregressive._Backend + +_Backend + +arima : Any +version : str + +__init__(self, version: str, arima: Any): None + + + +histdatacom.data_quality.seasonal_exogenous._Backend + +_Backend + +sarimax : Any +version : str + +__init__(self, version: str, sarimax: Any): None + + + +histdatacom.data_quality.state_space._Backend + +_Backend + +unobserved_components : Any +version : str + +__init__(self, version: str, unobserved_components: Any): None + + + +histdatacom.data_quality.volatility._Backend + +_Backend + +arch_model : Any +version : str + +__init__(self, version: str, arch_model: Any): None + + + +histdatacom.market_context.corpus._BankRateTableParser + +_BankRateTableParser + +cell_parts : list[str] +current_row : list[str] +in_cell : bool +in_table : bool +rows : list[tuple[str, ...]] + +__init__(): None +handle_data(data: str): None +handle_endtag(tag: str): None +handle_starttag(tag: str, attrs: list[tuple[str, str | None]]): None + + + +histdatacom.synthetic.bars._BarAccumulator + +_BarAccumulator + +bar_count : int +current_symbol : str | None +ensemble_member_id : str +grouped_symbols : bool +last_positions : dict[str, tuple[int, int, str]] +policy +previous_quotes : dict[tuple[str, ActivitySliceScope, str], tuple[float, float]] +query_end_ns +query_start_ns +run_id : str +source_product_manifest_id : str +states : dict[tuple[str, ActivitySliceScope, str], _BarState] +symbols : set[str] + +__init__(): None +_finalize(key: tuple[str, ActivitySliceScope, str], state: _BarState): DerivedBarV1 +_finish_symbol(symbol: str): tuple[DerivedBarV1, ...] +add(event: _BarEventView): tuple[DerivedBarV1, ...] +finish(): tuple[DerivedBarV1, ...] + + + +histdatacom.synthetic.bars._BarState + +_BarState + +ask_close : float +ask_high : float +ask_low : float +ask_open : float +bar_start_ns : int +bid_close : float +bid_high : float +bid_low : float +bid_open : float +broker_profile_ids : set[str] +confidence_count : int +confidence_total : float +constraint_set_ids : set[str] +digest : Any +ensemble_member_id : str +event_count : int +feed_epoch_ids : set[str] +first_event : _BarEventView | None +generator_config_ids : set[str] +generator_ids : set[str] +generator_versions : set[str] +interval +last_event : _BarEventView | None +mid_close : float +mid_high : float +mid_low : float +mid_open : float +motif_ids : set[str] +observed_event_count : int +policy +previous_ask : float | None +previous_bid : float | None +price_change_count : int +query_end_ns : int | None +query_start_ns : int | None +reference_ids : set[str] +run_id : str +scope +source_product_manifest_id : str +source_version_ids : set[str] +spread_close : float +spread_high : float +spread_low : float +spread_open : float +spread_total : float +stale_quote_count : int +symbol : str +synthetic_event_count : int +transition_count : int + +__init__(self, source_product_manifest_id: str, run_id: str, ensemble_member_id: str, symbol: str, scope: ActivitySliceScope, interval: DerivedBarIntervalV1, bar_start_ns: int, policy: DerivedBarPolicyV1, query_start_ns: int | None, query_end_ns: int | None, previous_bid: float | None, previous_ask: float | None, first_event: _BarEventView | None, last_event: _BarEventView | None, event_count: int, observed_event_count: int, synthetic_event_count: int, transition_count: int, price_change_count: int, stale_quote_count: int, confidence_total: float, confidence_count: int, spread_total: float, bid_open: float, bid_high: float, bid_low: float, bid_close: float, ask_open: float, ask_high: float, ask_low: float, ask_close: float, mid_open: float, mid_high: float, mid_low: float, mid_close: float, spread_open: float, spread_high: float, spread_low: float, spread_close: float, source_version_ids: set[str], generator_ids: set[str], generator_versions: set[str], generator_config_ids: set[str], reference_ids: set[str], motif_ids: set[str], feed_epoch_ids: set[str], broker_profile_ids: set[str], constraint_set_ids: set[str], digest: Any): None +_add_lineage(name: str, value: str): None +add(event: _BarEventView): None +finalize(): DerivedBarV1 + + + +histdatacom.synthetic.bars._BarAccumulator->histdatacom.synthetic.bars._BarState + + +states + + + +histdatacom.synthetic.bars._BarEventView + +_BarEventView + +ask : float +bid : float +broker_profile_id : str | None +confidence : float | None +constraint_set_id : str | None +ensemble_member_id : str +event_id : str +event_sequence : int +event_time_ns : int +feed_epoch_id : str | None +generator_config_id : str | None +generator_id : str | None +generator_version : str | None +motif_id : str | None +origin +reference_id : str | None +run_id : str +source_version_id : str +symbol : str + +__init__(self, event_id: str, origin: SyntheticEventOrigin, symbol: str, event_time_ns: int, event_sequence: int, bid: float, ask: float, run_id: str, ensemble_member_id: str, source_version_id: str, generator_id: str | None, generator_version: str | None, generator_config_id: str | None, reference_id: str | None, motif_id: str | None, feed_epoch_id: str | None, broker_profile_id: str | None, constraint_set_id: str | None, confidence: float | None): None + + + +histdatacom.synthetic.bars._BarEventView->histdatacom.synthetic.contracts.SyntheticEventOrigin + + +origin + + + +histdatacom.synthetic.bars._BarState->histdatacom.synthetic.activity.ActivitySliceScope + + +scope + + + +histdatacom.synthetic.bars._BarState->histdatacom.synthetic.bars.DerivedBarIntervalV1 + + +interval + + + +histdatacom.synthetic.bars._BarState->histdatacom.synthetic.bars.DerivedBarPolicyV1 + + +policy + + + +histdatacom.data_quality.classical_baselines._BaselineRow + +_BaselineRow + +mid : float +period : str +row_id : int +series_id : str +session_state : int + + - + histdatacom.data_quality.bounded_payload_contracts._BoundedSequenceContract - -_BoundedSequenceContract - -included_path : tuple[str, ...] -metadata_path : tuple[str, ...] -name : str -omitted_path : tuple[str, ...] -sequence_path : tuple[str, ...] -total_path : tuple[str, ...] -truncated_path : tuple[str, ...] - -__init__(self, name: str, metadata_path: tuple[str, ...], sequence_path: tuple[str, ...], total_path: tuple[str, ...], included_path: tuple[str, ...], omitted_path: tuple[str, ...], truncated_path: tuple[str, ...]): None + +_BoundedSequenceContract + +included_path : tuple[str, ...] +metadata_path : tuple[str, ...] +name : str +omitted_path : tuple[str, ...] +sequence_path : tuple[str, ...] +total_path : tuple[str, ...] +truncated_path : tuple[str, ...] + +__init__(self, name: str, metadata_path: tuple[str, ...], sequence_path: tuple[str, ...], total_path: tuple[str, ...], included_path: tuple[str, ...], omitted_path: tuple[str, ...], truncated_path: tuple[str, ...]): None - + histdatacom.data_quality.preflight._CacheTarget - -_CacheTarget - -path : Path -size_bytes : int -target - -__init__(self, target: QualityTarget, path: Path, size_bytes: int): None + +_CacheTarget + +path : Path +size_bytes : int +target + +__init__(self, target: QualityTarget, path: Path, size_bytes: int): None - + histdatacom.data_quality.preflight._CacheTarget->histdatacom.data_quality.contracts.QualityTarget - - -target + + +target - + histdatacom.data_quality.calendar._CalendarSample - -_CalendarSample - -classification -row_number : int -source_member : str - -__init__(self, row_number: int, classification: HistDataCalendarClassification, source_member: str): None -to_dict(): dict[str, JSONValue] + +_CalendarSample + +classification +row_number : int +source_member : str + +__init__(self, row_number: int, classification: HistDataCalendarClassification, source_member: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.calendar._CalendarSample->histdatacom.data_quality.calendar.HistDataCalendarClassification - - -classification + + +classification - + histdatacom.data_quality.calendar._CalendarScan - -_CalendarScan - -active_session_counts : Counter[str] -calendar_tag_counts : Counter[str] -clock_session_counts : Counter[str] -day_of_week_counts : Counter[str] -event_tag_counts : Counter[str] -holiday_tag_counts : Counter[str] -hour_of_day_counts : Counter[str] -invalid_timestamp_count : int -invalid_timestamps : list[_InvalidTimestampSample] -overlap_counts : Counter[str] -parsed_row_count : int -row_count : int -samples : list[_CalendarSample] -session_state_counts : Counter[str] -special_tag_counts : Counter[str] - -__init__(self, row_count: int, parsed_row_count: int, invalid_timestamp_count: int, session_state_counts: Counter[str], clock_session_counts: Counter[str], active_session_counts: Counter[str], overlap_counts: Counter[str], special_tag_counts: Counter[str], holiday_tag_counts: Counter[str], event_tag_counts: Counter[str], calendar_tag_counts: Counter[str], hour_of_day_counts: Counter[str], day_of_week_counts: Counter[str], samples: list[_CalendarSample], invalid_timestamps: list[_InvalidTimestampSample]): None + +_CalendarScan + +active_session_counts : Counter[str] +calendar_tag_counts : Counter[str] +clock_session_counts : Counter[str] +day_of_week_counts : Counter[str] +event_tag_counts : Counter[str] +holiday_tag_counts : Counter[str] +hour_of_day_counts : Counter[str] +invalid_timestamp_count : int +invalid_timestamps : list[_InvalidTimestampSample] +overlap_counts : Counter[str] +parsed_row_count : int +row_count : int +samples : list[_CalendarSample] +session_state_counts : Counter[str] +special_tag_counts : Counter[str] + +__init__(self, row_count: int, parsed_row_count: int, invalid_timestamp_count: int, session_state_counts: Counter[str], clock_session_counts: Counter[str], active_session_counts: Counter[str], overlap_counts: Counter[str], special_tag_counts: Counter[str], holiday_tag_counts: Counter[str], event_tag_counts: Counter[str], calendar_tag_counts: Counter[str], hour_of_day_counts: Counter[str], day_of_week_counts: Counter[str], samples: list[_CalendarSample], invalid_timestamps: list[_InvalidTimestampSample]): None + + + +histdatacom.data_analytics.feed_epochs._Candidate + +_Candidate + +feature_differences : Mapping[str, float] +left_end_utc_ms : int +left_index : int +left_period : str +right_index : int +right_period : str +right_start_utc_ms : int +score : float + +__init__(self, left_index: int, right_index: int, left_period: str, right_period: str, left_end_utc_ms: int, right_start_utc_ms: int, score: float, feature_differences: Mapping[str, float]): None + + + +histdatacom.synthetic.benchmark_corpus._CandidateAccumulator + +_CandidateAccumulator + +failures : int +refusals : int +values : dict[str, list[float]] + +__init__(self, values: dict[str, list[float]], failures: int, refusals: int): None +consume(values: Mapping[str, float]): None + + + +histdatacom.broker_capture.fingerprints._CellAccumulator + +_CellAccumulator + +active_runs : dict[str, int] +condition +metrics : dict[str, _SampleAccumulator] +rounding_digits : int +sample_limit : int +support_count : int + +__init__(self, condition: BrokerDeliveryConditionV1, sample_limit: int, rounding_digits: int, support_count: int, metrics: dict[str, _SampleAccumulator], active_runs: dict[str, int]): None +add(name: str, value: float, evidence_key: str): None +flush_runs(evidence_key: str): None +observe_quote(): None +observe_run(name: str, active: bool, evidence_key: str): None + + + +histdatacom.broker_capture.fingerprints._CellAccumulator->histdatacom.broker_capture.fingerprint_contracts.BrokerDeliveryConditionV1 + + +condition - + histdatacom.data_quality.remediation_audit._CodeAggregate - -_CodeAggregate - -finding_code : str -mapped : bool -max_severity : str -occurrence_count : int -rule_id : str -severity_counts : Counter[str] -source_counts : Counter[str] -source_family_counts : Counter[str] - -__init__(self, rule_id: str, finding_code: str, mapped: bool, occurrence_count: int, severity_counts: Counter[str], source_counts: Counter[str], source_family_counts: Counter[str]): None + +_CodeAggregate + +attribution_reason_counts : Counter[str] +attribution_status_counts : Counter[str] +finding_code : str +finding_code_prefix_counts : Counter[str] +mapped : bool +max_severity : str +occurrence_count : int +rule_id : str +severity_counts : Counter[str] +source_counts : Counter[str] +source_family_counts : Counter[str] +source_helper_counts : Counter[str] + +__init__(self, rule_id: str, finding_code: str, mapped: bool, occurrence_count: int, severity_counts: Counter[str], source_counts: Counter[str], source_family_counts: Counter[str], source_helper_counts: Counter[str], finding_code_prefix_counts: Counter[str], attribution_status_counts: Counter[str], attribution_reason_counts: Counter[str]): None - + histdatacom.data_quality.time._ContinuityBoundary - -_ContinuityBoundary - -first -invalid_timestamp_count : int -last -parsed_row_count : int -path : str -period : str -source_member : str -target - -__init__(self, target: QualityTarget, first: _TimestampSample, last: _TimestampSample, parsed_row_count: int, invalid_timestamp_count: int, source_member: str): None -to_dict(): dict[str, JSONValue] + +_ContinuityBoundary + +first +invalid_timestamp_count : int +last +parsed_row_count : int +path : str +period : str +source_member : str +target + +__init__(self, target: QualityTarget, first: _TimestampSample, last: _TimestampSample, parsed_row_count: int, invalid_timestamp_count: int, source_member: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.time._ContinuityBoundary->histdatacom.data_quality.contracts.QualityTarget - - -target + + +target - + histdatacom.data_quality.time._TimestampSample - -_TimestampSample - -row_number : int -row_values : tuple[str, ...] -source_day : int -source_day_ordinal : int -source_member : str -source_month : int -source_month_length : int -source_period : str -source_time_of_day_ms : int -source_weekday : int -source_year : int -timestamp_source : str -timestamp_utc_ms : int -utc_period : str -utc_timestamp : str - -__init__(self, row_number: int, timestamp_source: str, timestamp_utc_ms: int, source_period: str, utc_period: str, source_year: int, source_month: int, source_day: int, source_month_length: int, source_weekday: int, source_day_ordinal: int, source_time_of_day_ms: int, source_member: str, row_values: tuple[str, ...]): None -to_dict(): dict[str, JSONValue] + +_TimestampSample + +row_number : int +row_values : tuple[str, ...] +source_day : int +source_day_ordinal : int +source_member : str +source_month : int +source_month_length : int +source_period : str +source_time_of_day_ms : int +source_weekday : int +source_year : int +timestamp_source : str +timestamp_utc_ms : int +utc_period : str +utc_timestamp : str + +__init__(self, row_number: int, timestamp_source: str, timestamp_utc_ms: int, source_period: str, utc_period: str, source_year: int, source_month: int, source_day: int, source_month_length: int, source_weekday: int, source_day_ordinal: int, source_time_of_day_ms: int, source_member: str, row_values: tuple[str, ...]): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.time._ContinuityBoundary->histdatacom.data_quality.time._TimestampSample - - -first + + +first - + histdatacom.data_quality.time._ContinuityBoundary->histdatacom.data_quality.time._TimestampSample - - -last + + +last - + histdatacom.data_quality.time._ContinuityComparison - -_ContinuityComparison - -classification : str -current -current_path : str -gap_ms : int -key -missing_periods : tuple[str, ...] -previous -previous_path : str - -__init__(self, key: _ContinuityGroupKey, previous: _ContinuityBoundary, current: _ContinuityBoundary, gap_ms: int, classification: str, missing_periods: tuple[str, ...]): None -to_dict(): dict[str, JSONValue] + +_ContinuityComparison + +classification : str +current +current_path : str +gap_ms : int +key +missing_periods : tuple[str, ...] +previous +previous_path : str + +__init__(self, key: _ContinuityGroupKey, previous: _ContinuityBoundary, current: _ContinuityBoundary, gap_ms: int, classification: str, missing_periods: tuple[str, ...]): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.time._ContinuityComparison->histdatacom.data_quality.time._ContinuityBoundary - - -previous + + +previous - + histdatacom.data_quality.time._ContinuityComparison->histdatacom.data_quality.time._ContinuityBoundary - - -current + + +current - + histdatacom.data_quality.time._ContinuityGroupKey - -_ContinuityGroupKey - -data_format : str -symbol : str -timeframe : str - -__init__(self, data_format: str, timeframe: str, symbol: str): None -from_target(target: QualityTarget): '_ContinuityGroupKey | None' -to_dict(): dict[str, JSONValue] + +_ContinuityGroupKey + +data_format : str +symbol : str +timeframe : str + +__init__(self, data_format: str, timeframe: str, symbol: str): None +from_target(target: QualityTarget): '_ContinuityGroupKey | None' +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.time._ContinuityComparison->histdatacom.data_quality.time._ContinuityGroupKey - - -key + + +key - + histdatacom.data_quality.time._ContinuityScan - -_ContinuityScan - -adjacent_pair_count : int -candidate_target_count : int -clean_boundary_count : int -comparable_target_count : int -duplicate_overlap_count : int -duplicate_overlaps : list[_ContinuityComparison] -expected_session_closure_count : int -expected_session_closures : list[_ContinuityComparison] -group_count : int -missing_period_count : int -missing_periods : list[_ContinuityComparison] -period_gap_count : int -reversed_order : list[_ContinuityComparison] -reversed_order_count : int -skipped_target_count : int -suspicious_gap_count : int -suspicious_gaps : list[_ContinuityComparison] -target_count : int - -__init__(self, target_count: int, candidate_target_count: int, comparable_target_count: int, skipped_target_count: int, group_count: int, adjacent_pair_count: int, clean_boundary_count: int, missing_period_count: int, period_gap_count: int, duplicate_overlap_count: int, reversed_order_count: int, suspicious_gap_count: int, expected_session_closure_count: int, missing_periods: list[_ContinuityComparison], duplicate_overlaps: list[_ContinuityComparison], reversed_order: list[_ContinuityComparison], suspicious_gaps: list[_ContinuityComparison], expected_session_closures: list[_ContinuityComparison]): None + +_ContinuityScan + +adjacent_pair_count : int +candidate_target_count : int +clean_boundary_count : int +comparable_target_count : int +duplicate_overlap_count : int +duplicate_overlaps : list[_ContinuityComparison] +expected_session_closure_count : int +expected_session_closures : list[_ContinuityComparison] +group_count : int +missing_period_count : int +missing_periods : list[_ContinuityComparison] +period_gap_count : int +reversed_order : list[_ContinuityComparison] +reversed_order_count : int +skipped_target_count : int +suspicious_gap_count : int +suspicious_gaps : list[_ContinuityComparison] +target_count : int + +__init__(self, target_count: int, candidate_target_count: int, comparable_target_count: int, skipped_target_count: int, group_count: int, adjacent_pair_count: int, clean_boundary_count: int, missing_period_count: int, period_gap_count: int, duplicate_overlap_count: int, reversed_order_count: int, suspicious_gap_count: int, expected_session_closure_count: int, missing_periods: list[_ContinuityComparison], duplicate_overlaps: list[_ContinuityComparison], reversed_order: list[_ContinuityComparison], suspicious_gaps: list[_ContinuityComparison], expected_session_closures: list[_ContinuityComparison]): None - + histdatacom.data_quality.fingerprints._CoverageScan - -_CoverageScan - -end_timestamp_utc_ms : int | None -parsed_row_count : int -row_count : int -start_timestamp_utc_ms : int | None - -__init__(self, row_count: int, parsed_row_count: int, start_timestamp_utc_ms: int | None, end_timestamp_utc_ms: int | None): None -to_payload(): dict[str, JSONValue] + +_CoverageScan + +end_timestamp_utc_ms : int | None +parsed_row_count : int +row_count : int +start_timestamp_utc_ms : int | None + +__init__(self, row_count: int, parsed_row_count: int, start_timestamp_utc_ms: int | None, end_timestamp_utc_ms: int | None): None +to_payload(): dict[str, JSONValue] - + histdatacom.data_quality.symbols._CrossInstrumentPoint - -_CrossInstrumentPoint - -price : float -row_number : int -timestamp_utc_ms : int - -__init__(self, timestamp_utc_ms: int, price: float, row_number: int): None + +_CrossInstrumentPoint + +event_seq : int +price : float +row_id : int +row_number : int +source_row_number : int +timestamp_utc_ms : int + +__init__(self, timestamp_utc_ms: int, price: float, row_number: int, row_id: int, source_row_number: int, event_seq: int): None - + histdatacom.data_quality.symbols._CrossInstrumentScan - -_CrossInstrumentScan - -ascii_target_count : int -fx_series : list[_CrossInstrumentSeries] -invalid_price_count : int -inverse_candidate_count : int -inverse_compared_timestamp_count : int -inverse_error_count : int -inverse_errors : list[_InverseComparisonSample] -inverse_warning_count : int -inverse_warnings : list[_InverseComparisonSample] -source_error_count : int -sparse_grid_count : int -sparse_grids : list[_TimestampGridSample] -stale_join_risk_count : int -stale_join_risks : list[_StaleJoinSample] -target_count : int -timestamp_grid_group_count : int -triangular_candidate_count : int -triangular_compared_timestamp_count : int -triangular_error_count : int -triangular_errors : list[_TriangularComparisonSample] -triangular_warning_count : int -triangular_warnings : list[_TriangularComparisonSample] -unavailable : list[_CrossInstrumentUnavailable] - -__init__(self, target_count: int, ascii_target_count: int, fx_series: list[_CrossInstrumentSeries], source_error_count: int, invalid_price_count: int, triangular_candidate_count: int, triangular_compared_timestamp_count: int, triangular_warning_count: int, triangular_error_count: int, inverse_candidate_count: int, inverse_compared_timestamp_count: int, inverse_warning_count: int, inverse_error_count: int, timestamp_grid_group_count: int, sparse_grid_count: int, stale_join_risk_count: int, unavailable: list[_CrossInstrumentUnavailable], triangular_warnings: list[_TriangularComparisonSample], triangular_errors: list[_TriangularComparisonSample], inverse_warnings: list[_InverseComparisonSample], inverse_errors: list[_InverseComparisonSample], sparse_grids: list[_TimestampGridSample], stale_join_risks: list[_StaleJoinSample]): None + +_CrossInstrumentScan + +ascii_target_count : int +fx_series : list[_CrossInstrumentSeries] +invalid_price_count : int +inverse_candidate_count : int +inverse_compared_timestamp_count : int +inverse_error_count : int +inverse_errors : list[_InverseComparisonSample] +inverse_warning_count : int +inverse_warnings : list[_InverseComparisonSample] +source_error_count : int +sparse_grid_count : int +sparse_grids : list[_TimestampGridSample] +stale_join_risk_count : int +stale_join_risks : list[_StaleJoinSample] +target_count : int +timestamp_grid_group_count : int +triangular_candidate_count : int +triangular_compared_timestamp_count : int +triangular_error_count : int +triangular_errors : list[_TriangularComparisonSample] +triangular_warning_count : int +triangular_warnings : list[_TriangularComparisonSample] +unavailable : list[_CrossInstrumentUnavailable] + +__init__(self, target_count: int, ascii_target_count: int, fx_series: list[_CrossInstrumentSeries], source_error_count: int, invalid_price_count: int, triangular_candidate_count: int, triangular_compared_timestamp_count: int, triangular_warning_count: int, triangular_error_count: int, inverse_candidate_count: int, inverse_compared_timestamp_count: int, inverse_warning_count: int, inverse_error_count: int, timestamp_grid_group_count: int, sparse_grid_count: int, stale_join_risk_count: int, unavailable: list[_CrossInstrumentUnavailable], triangular_warnings: list[_TriangularComparisonSample], triangular_errors: list[_TriangularComparisonSample], inverse_warnings: list[_InverseComparisonSample], inverse_errors: list[_InverseComparisonSample], sparse_grids: list[_TimestampGridSample], stale_join_risks: list[_StaleJoinSample]): None - + histdatacom.data_quality.symbols._CrossInstrumentSeries - -_CrossInstrumentSeries - -base : str -end_utc_ms : int | None -metadata -period : str -points : dict[int, _CrossInstrumentPoint] -price_kind : str -quote : str -start_utc_ms : int | None -symbol : str -target -timeframe : str -timestamp_count : int -timestamps : set[int] - -__init__(self, target: QualityTarget, metadata: HistDataSymbolMetadata, timeframe: str, period: str, price_kind: str, points: dict[int, _CrossInstrumentPoint]): None -to_metadata(): dict[str, JSONValue] + +_CrossInstrumentSeries + +base : str +cache_source : str +computed_from : str +duplicate_timestamp_count : int +end_utc_ms : int | None +metadata +period : str +points : dict[int, _CrossInstrumentPoint] +price_kind : str +quote : str +row_count : int +series_id : str +start_utc_ms : int | None +symbol : str +target +timeframe : str +timestamp_count : int +timestamps : set[int] +training_schema_version : str + +__init__(self, target: QualityTarget, metadata: HistDataSymbolMetadata, timeframe: str, period: str, price_kind: str, points: dict[int, _CrossInstrumentPoint], row_count: int, duplicate_timestamp_count: int, computed_from: str, series_id: str, cache_source: str, training_schema_version: str): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.symbols._CrossInstrumentSeries->histdatacom.data_quality.symbols.HistDataSymbolMetadata - - -metadata + + +metadata - + histdatacom.data_quality.symbols._CrossInstrumentSeries->histdatacom.data_quality.contracts.QualityTarget - - -target + + +target - + histdatacom.data_quality.symbols._CrossInstrumentUnavailable - -_CrossInstrumentUnavailable - -metadata : dict[str, JSONValue] -period : str -reason : str -symbols : tuple[str, ...] -timeframe : str - -__init__(self, reason: str, symbols: tuple[str, ...], timeframe: str, period: str, metadata: dict[str, JSONValue]): None -to_metadata(): dict[str, JSONValue] + +_CrossInstrumentUnavailable + +metadata : dict[str, JSONValue] +period : str +reason : str +symbols : tuple[str, ...] +timeframe : str + +__init__(self, reason: str, symbols: tuple[str, ...], timeframe: str, period: str, metadata: dict[str, JSONValue]): None +to_metadata(): dict[str, JSONValue] - + histdatacom.orchestration.resources._DirectoryLock - -_DirectoryLock - -path : Path -timeout - -__enter__(): '_DirectoryLock' -__exit__(): None -__init__(path: Path): None + +_DirectoryLock + +path : Path +timeout + +__enter__(): '_DirectoryLock' +__exit__(): None +__init__(path: Path): None + + + +histdatacom.data_quality.time._DuplicateTimestampSample + +_DuplicateTimestampSample + +exact_row_group_count : int +occurrence_count : int +row_number : int +source_member : str +timestamp_source : str +timestamp_utc_ms : int + +__init__(self, row_number: int, timestamp_source: str, timestamp_utc_ms: int, occurrence_count: int, exact_row_group_count: int, source_member: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.provenance._FindingTemplate - -_FindingTemplate - -code : str -location_path : str -message : str -metadata : Mapping[str, JSONValue] | None -severity - -__init__(self, severity: QualitySeverity, code: str, message: str, location_path: str, metadata: Mapping[str, JSONValue] | None): None + +_FindingTemplate + +code : str +location_path : str +message : str +metadata : Mapping[str, JSONValue] | None +severity + +__init__(self, severity: QualitySeverity, code: str, message: str, location_path: str, metadata: Mapping[str, JSONValue] | None): None - + histdatacom.data_quality.provenance._FindingTemplate->histdatacom.data_quality.contracts.QualitySeverity - - -severity + + +severity + + + +histdatacom.broker_capture.fingerprints._FingerprintConsumer->histdatacom.broker_capture.fingerprints._CellAccumulator + + +cells + + + +histdatacom.broker_capture.fingerprints._PreviousQuote + +_PreviousQuote + +ask : float +bid : float +message_id : str +monotonic_ns : int +spread : float + +__init__(self, monotonic_ns: int, bid: float, ask: float, spread: float, message_id: str): None + + + +histdatacom.broker_capture.fingerprints._FingerprintConsumer->histdatacom.broker_capture.fingerprints._PreviousQuote + + +_previous_quotes + + + +histdatacom.data_quality.exponential_smoothing._FitOutcome + +_FitOutcome + +converged : bool +fit_seconds : float +forecasts : tuple[float, ...] +parameters : Mapping[str, float] +reason : str +status : str +warning_codes : tuple[str, ...] + +__init__(self, status: str, reason: str, forecasts: tuple[float, ...], parameters: Mapping[str, float], warning_codes: tuple[str, ...], converged: bool, fit_seconds: float): None + + + +histdatacom.data_quality.autoregressive._FitOutcome + +_FitOutcome + +ar_root_min_modulus : float | None +converged : bool +covariance_condition_number : float | None +effective_observation_count : int +forecasts : tuple[float, ...] +invertible : bool | None +ma_root_min_modulus : float | None +parameters : Mapping[str, float] +reason : str +stationary : bool | None +status : str +warning_codes : tuple[str, ...] + +__init__(self, status: str, reason: str, forecasts: tuple[float, ...], parameters: Mapping[str, float], warning_codes: tuple[str, ...], converged: bool, stationary: bool | None, invertible: bool | None, ar_root_min_modulus: float | None, ma_root_min_modulus: float | None, covariance_condition_number: float | None, effective_observation_count: int): None + + + +histdatacom.data_quality.seasonal_exogenous._FitOutcome + +_FitOutcome + +ar_root_min_modulus : float | None +converged : bool +covariance_condition_number : float | None +effective_observation_count : int +forecasts : tuple[float, ...] +invertible : bool | None +ma_root_min_modulus : float | None +parameters : Mapping[str, float] +reason : str +residual_summary : Mapping[str, JSONValue] +stationary : bool | None +status : str +warning_codes : tuple[str, ...] + +__init__(self, status: str, reason: str, forecasts: tuple[float, ...], parameters: Mapping[str, float], warning_codes: tuple[str, ...], converged: bool, stationary: bool | None, invertible: bool | None, ar_root_min_modulus: float | None, ma_root_min_modulus: float | None, covariance_condition_number: float | None, effective_observation_count: int, residual_summary: Mapping[str, JSONValue]): None + + + +histdatacom.data_quality.state_space._FitOutcome + +_FitOutcome + +aic : float | None +bic : float | None +converged : bool +covariance_condition_number : float | None +effective_observation_count : int +filtered_state : tuple[float | None, ...] +filtered_variance : tuple[float | None, ...] +forecasts : tuple[float, ...] +innovation_summary : Mapping[str, JSONValue] +log_likelihood : float | None +lower_bounds : tuple[float, ...] +max_prediction_only_gap : int +missing_observation_count : int +parameters : Mapping[str, float] +prediction_only_transition_count : int +reason : str +smoothed_state : tuple[float | None, ...] +smoothed_variance : tuple[float | None, ...] +standard_errors : tuple[float, ...] +state_dimension : int +state_names : tuple[str, ...] +status : str +upper_bounds : tuple[float, ...] +warning_codes : tuple[str, ...] + +__init__(self, status: str, reason: str, forecasts: tuple[float, ...], standard_errors: tuple[float, ...], lower_bounds: tuple[float, ...], upper_bounds: tuple[float, ...], parameters: Mapping[str, float], warning_codes: tuple[str, ...], converged: bool, state_dimension: int, state_names: tuple[str, ...], filtered_state: tuple[float | None, ...], filtered_variance: tuple[float | None, ...], smoothed_state: tuple[float | None, ...], smoothed_variance: tuple[float | None, ...], effective_observation_count: int, missing_observation_count: int, prediction_only_transition_count: int, max_prediction_only_gap: int, log_likelihood: float | None, aic: float | None, bic: float | None, covariance_condition_number: float | None, innovation_summary: Mapping[str, JSONValue]): None + + + +histdatacom.data_quality.volatility._FitOutcome + +_FitOutcome + +aic : float | None +bic : float | None +boundary_parameter : bool +converged : bool +covariance_condition_number : float | None +effective_observation_count : int +log_likelihood : float | None +mean_forecasts : tuple[float, ...] +missing_reset_count : int +parameters : Mapping[str, float] +persistence : float | None +reason : str +standardized_residual_summary : Mapping[str, JSONValue] +status : str +unconditional_variance : float | None +variance_forecasts : tuple[float, ...] +warning_codes : tuple[str, ...] + +__init__(self, status: str, reason: str, mean_forecasts: tuple[float, ...], variance_forecasts: tuple[float, ...], parameters: Mapping[str, float], warning_codes: tuple[str, ...], converged: bool, effective_observation_count: int, missing_reset_count: int, persistence: float | None, unconditional_variance: float | None, covariance_condition_number: float | None, boundary_parameter: bool, standardized_residual_summary: Mapping[str, JSONValue], log_likelihood: float | None, aic: float | None, bic: float | None): None + + + +histdatacom.data_quality.classical_model_contracts._GridRow + +_GridRow + +cm_input_available_at_utc_ms : int +cm_input_bin_end_utc_ms : int +cm_input_bin_start_utc_ms : int +cm_input_expected_closure : bool +cm_input_mid_close : float | None +cm_input_mid_high : float | None +cm_input_mid_low : float | None +cm_input_mid_mean : float | None +cm_input_mid_median : float | None +cm_input_mid_open : float | None +cm_input_observation_count : int +cm_input_observed_value : float | None +cm_input_source_first_row_id : int | None +cm_input_source_last_row_id : int | None +cm_input_spread : float | None +cm_input_status_code : int +cm_input_transform_valid : bool +cm_input_unexpected_missing : bool +cm_input_value : float | None +period : str +series_id : str + + + + + +histdatacom.broker_capture.fingerprints._HealthConsumer + +_HealthConsumer + +_previous_wall_ns : int | None +clock_correction_count : int +event_count : int +explained_wall_regression_count : int +first_receive_time_utc_ns : int | None +last_receive_time_utc_ns : int | None +max_abs_clock_correction_ns : int +quote_count : int +unexplained_wall_regression_count : int + +__init__(self, event_count: int, quote_count: int, clock_correction_count: int, max_abs_clock_correction_ns: int, explained_wall_regression_count: int, unexplained_wall_regression_count: int, first_receive_time_utc_ns: int | None, last_receive_time_utc_ns: int | None, _previous_wall_ns: int | None): None +on_event(event: BrokerCaptureEventV1): None - + histdatacom.activity_stages._HistDataDownloadFormParser - -_HistDataDownloadFormParser - -in_download_form : bool -seen_download_form : bool -values : dict[str, str] - -__init__(): None -handle_endtag(tag: str): None -handle_startendtag(tag: str, attrs: list[tuple[str, str | None]]): None -handle_starttag(tag: str, attrs: list[tuple[str, str | None]]): None + +_HistDataDownloadFormParser + +in_download_form : bool +seen_download_form : bool +values : dict[str, str] + +__init__(): None +handle_endtag(tag: str): None +handle_startendtag(tag: str, attrs: list[tuple[str, str | None]]): None +handle_starttag(tag: str, attrs: list[tuple[str, str | None]]): None - + histdatacom.data_quality.ingestion._IngestionProfile - -_IngestionProfile - -container_size_bytes : int -empty_line_count : int | None -end_timestamp_utc_ms : int | None -final_line_terminated : bool | None -kind : str -line_count : int | None -payload_size_bytes : int -row_count : int -schema : dict[str, str] -source_member : str -start_timestamp_utc_ms : int | None - -__init__(self, kind: str, row_count: int, payload_size_bytes: int, container_size_bytes: int, source_member: str, line_count: int | None, empty_line_count: int | None, final_line_terminated: bool | None, schema: dict[str, str], start_timestamp_utc_ms: int | None, end_timestamp_utc_ms: int | None): None -to_metadata(target: QualityTarget): dict[str, JSONValue] + +_IngestionProfile + +container_size_bytes : int +empty_line_count : int | None +end_timestamp_utc_ms : int | None +final_line_terminated : bool | None +kind : str +line_count : int | None +payload_size_bytes : int +row_count : int +schema : dict[str, str] +source_member : str +start_timestamp_utc_ms : int | None + +__init__(self, kind: str, row_count: int, payload_size_bytes: int, container_size_bytes: int, source_member: str, line_count: int | None, empty_line_count: int | None, final_line_terminated: bool | None, schema: dict[str, str], start_timestamp_utc_ms: int | None, end_timestamp_utc_ms: int | None): None +to_metadata(target: QualityTarget): dict[str, JSONValue] - + histdatacom.data_quality.calendar._InvalidTimestampSample - -_InvalidTimestampSample - -error : str -row_number : int -source_member : str -timestamp_source : str - -__init__(self, row_number: int, timestamp_source: str, source_member: str, error: str): None -to_dict(): dict[str, JSONValue] + +_InvalidTimestampSample + +error : str +row_number : int +source_member : str +timestamp_source : str + +__init__(self, row_number: int, timestamp_source: str, source_member: str, error: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.symbols._InverseComparisonSample - -_InverseComparisonSample - -left -left_price : float -product : float -relative_difference : float -right -right_price : float -severity -timestamp_utc_ms : int - -__init__(self, left: _CrossInstrumentSeries, right: _CrossInstrumentSeries, timestamp_utc_ms: int, left_price: float, right_price: float, product: float, relative_difference: float, severity: QualitySeverity): None -to_metadata(): dict[str, JSONValue] + +_InverseComparisonSample + +left +left_price : float +product : float +relative_difference : float +right +right_price : float +severity +timestamp_utc_ms : int + +__init__(self, left: _CrossInstrumentSeries, right: _CrossInstrumentSeries, timestamp_utc_ms: int, left_price: float, right_price: float, product: float, relative_difference: float, severity: QualitySeverity): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.symbols._InverseComparisonSample->histdatacom.data_quality.contracts.QualitySeverity - - -severity + + +severity - + histdatacom.data_quality.symbols._InverseComparisonSample->histdatacom.data_quality.symbols._CrossInstrumentSeries - - -left + + +left - + histdatacom.data_quality.symbols._InverseComparisonSample->histdatacom.data_quality.symbols._CrossInstrumentSeries - - -right + + +right - + histdatacom.data_quality.ingestion._LineEndingScan - -_LineEndingScan - -counts : dict[str, int] -has_malformed : bool -is_inconsistent : bool -used_styles : tuple[str, ...] - -__init__(self, counts: dict[str, int]): None -to_dict(): dict[str, JSONValue] + +_LineEndingScan + +counts : dict[str, int] +has_malformed : bool +is_inconsistent : bool +used_styles : tuple[str, ...] + +__init__(self, counts: dict[str, int]): None +to_dict(): dict[str, JSONValue] + + + +histdatacom.broker_capture.storage._LogicalDigestConsumer + +_LogicalDigestConsumer + +_digest : sha256 + +__init__(): None +hexdigest(): str +on_event(event: BrokerCaptureEventV1): None + + + +histdatacom.synthetic.benchmark._MemberState + +_MemberState + +anchor_preserved_count : int +anchor_reference_count : int +covered_reference_count : int +slices : dict[tuple[str, str, str, str, str], _StreamStats] +support_interval_count : int + +__init__(self, slices: dict[tuple[str, str, str, str, str], _StreamStats], support_interval_count: int, covered_reference_count: int, anchor_reference_count: int, anchor_preserved_count: int): None + + + +histdatacom.synthetic.observation._MutableApplicationState + +_MutableApplicationState + +last_ask : float | None +last_bid : float | None +last_observed_time_ns : int | None +last_source_time_ns : int | None +outage_active : bool +outage_bucket_start_ns : int | None +rate_bucket_count : int +rate_bucket_start_ns : int | None +reconnect_pending : bool + +__init__(self, last_source_time_ns: int | None, last_observed_time_ns: int | None, last_bid: float | None, last_ask: float | None, rate_bucket_start_ns: int | None, rate_bucket_count: int, outage_bucket_start_ns: int | None, outage_active: bool, reconnect_pending: bool): None - + histdatacom.data_quality.reporting._NextActionAggregate - -_NextActionAggregate - -action_kind : str -attention_level_counts : Counter[str] -code : str -finding_code_counts : Counter[str] -flag_counts : Counter[str] -message : str -occurrence_count : int -rule_id : str -severity_counts : Counter[str] -source_counts : Counter[str] -target_axis_counts : Counter[tuple[str, str, str, str, str]] - -__init__(self, code: str, message: str, action_kind: str, rule_id: str, occurrence_count: int, severity_counts: Counter[str], attention_level_counts: Counter[str], source_counts: Counter[str], finding_code_counts: Counter[str], flag_counts: Counter[str], target_axis_counts: Counter[tuple[str, str, str, str, str]]): None + +_NextActionAggregate + +action_kind : str +attention_level_counts : Counter[str] +code : str +finding_code_counts : Counter[str] +flag_counts : Counter[str] +message : str +occurrence_count : int +policy_context : dict[str, JSONValue] +rule_id : str +severity_counts : Counter[str] +source_counts : Counter[str] +target_axis_counts : Counter[tuple[str, str, str, str, str]] + +__init__(self, code: str, message: str, action_kind: str, rule_id: str, policy_context: dict[str, JSONValue], occurrence_count: int, severity_counts: Counter[str], attention_level_counts: Counter[str], source_counts: Counter[str], finding_code_counts: Counter[str], flag_counts: Counter[str], target_axis_counts: Counter[tuple[str, str, str, str, str]]): None - + histdatacom.orchestration.activities._NoopActivityApi - -_NoopActivityApi - -logger : NoneType, RootLogger - -defn(decorated: Any | None): Any + +_NoopActivityApi + +logger : NoneType, RootLogger + +defn(decorated: Any | None): Any - + histdatacom.orchestration.workflows._NoopWorkflowApi - -_NoopWorkflowApi - -logger : NoneType, RootLogger - -defn(decorated: Any | None): Any -execute_child_workflow(): Any -query(decorated: Any): Any -run(decorated: Any): Any + +_NoopWorkflowApi + +logger : NoneType, RootLogger + +defn(decorated: Any | None): Any +execute_child_workflow(): Any +query(decorated: Any): Any +run(decorated: Any): Any + + + +histdatacom.synthetic.benchmark._OnlineMetric + +_OnlineMetric + +count : int +maximum : float | None +mean : float +minimum : float | None +total : float + +__init__(self, count: int, total: float, minimum: float | None, maximum: float | None): None +payload(digits: int): dict[str, JSONValue] +update(value: float): None + + + +histdatacom.data_quality.repair_plan._OperationSpec + +_OperationSpec + +category : str +evidence_needed : tuple[str, ...] +next_step : str +preconditions : tuple[str, ...] +priority : int +required_evidence_groups : tuple[tuple[str, ...], ...] + +__init__(self, category: str, next_step: str, preconditions: tuple[str, ...], evidence_needed: tuple[str, ...], required_evidence_groups: tuple[tuple[str, ...], ...], priority: int): None - + histdatacom.orchestration.client._OrchestrationAvailability - -_OrchestrationAvailability - -started_by_invocation : bool -status - -__init__(self, status: OrchestrationStatus, started_by_invocation: bool): None + +_OrchestrationAvailability + +started_by_invocation : bool +status + +__init__(self, status: OrchestrationStatus, started_by_invocation: bool): None - + histdatacom.orchestration.client._OrchestrationAvailability->histdatacom.orchestration.supervisor.OrchestrationStatus - - -status + + +status + + + +histdatacom.synthetic.bars._PartitionManager + +_PartitionManager + +active : dict[tuple[str, ActivitySliceScope, str], _PartitionWriter] +buffer_rows +current_symbol : str | None +partitions : list[DerivedBarPartitionV1] +row_group_size +staging_directory : Path + +__init__(staging_directory: Path): None +_close(axis: tuple[str, ActivitySliceScope, str], writer: _PartitionWriter): None +_close_symbol(symbol: str): None +abort(): None +add(bar: DerivedBarV1): None +finish(): tuple[DerivedBarPartitionV1, ...] + + + +histdatacom.synthetic.bars._PartitionManager->histdatacom.synthetic.bars.DerivedBarPartitionV1 + + +partitions + + + +histdatacom.synthetic.bars._PartitionWriter + +_PartitionWriter + +bar_month +buffer : list[DerivedBarV1] +buffer_rows +digest : sha256 +interval_code +last_order : tuple[int, str] | None +maximum : int | None +minimum : int | None +path +relative_path : str +row_count : int +row_group_size +scope +symbol +writer : ParquetWriter + +__init__(staging_directory: Path, first: DerivedBarV1): None +abort(): None +add(bar: DerivedBarV1): None +close(): DerivedBarPartitionV1 +flush(): None + + + +histdatacom.synthetic.bars._PartitionManager->histdatacom.synthetic.bars._PartitionWriter + + +active + + + +histdatacom.synthetic.bars._PartitionWriter->histdatacom.synthetic.bars.DerivedBarV1 + + +buffer + + + +histdatacom.synthetic.broker_transfer._PendingRender + +_PendingRender + +actions : list[str] +ask : float +bid : float +event_time_ns : int +source + +__init__(self, source: SyntheticEventV1, event_time_ns: int, bid: float, ask: float, actions: list[str]): None + + + +histdatacom.synthetic.broker_transfer._PendingRender->histdatacom.synthetic.contracts.SyntheticEventV1 + + +source + + + +histdatacom.synthetic.strategy_sensitivity._PendingSignal + +_PendingSignal + +decision_quote +entry_price : float | None +entry_quote : StrategyQuoteV1 | None +signal +unresolved_horizons : set[int] + +__init__(self, signal: StrategySignalV1, decision_quote: StrategyQuoteV1, unresolved_horizons: set[int], entry_quote: StrategyQuoteV1 | None, entry_price: float | None): None + + + +histdatacom.synthetic.strategy_sensitivity._PendingSignal->histdatacom.synthetic.strategy_sensitivity.StrategyQuoteV1 + + +decision_quote + + + +histdatacom.synthetic.strategy_sensitivity._PendingSignal->histdatacom.synthetic.strategy_sensitivity.StrategySignalV1 + + +signal + + + +histdatacom.data_analytics.feed_epochs._PeriodPoint + +_PeriodPoint + +end_utc_ms : int +evidence_count : int +period : str +start_utc_ms : int +values : Mapping[str, float] + +__init__(self, period: str, start_utc_ms: int, end_utc_ms: int, evidence_count: int, values: Mapping[str, float]): None + + + +histdatacom.data_quality.repair_plan._PlanCandidate + +_PlanCandidate + +finding +hint : QualityRemediationHint | None +operation : _OperationSpec | None +sort_key : tuple[str, str, str, str, str, str, int, int, str, str, str] + +__init__(self, finding: QualityFinding, hint: QualityRemediationHint | None, operation: _OperationSpec | None): None + + + +histdatacom.data_quality.repair_plan._PlanCandidate->histdatacom.data_quality.contracts.QualityFinding + + +finding + + + +histdatacom.synthetic.generation._PlannedTransform + +_PlannedTransform + +event_count : int +match +record +start_index : int + +__init__(self, match: ReferenceMotifMatchV1, record: EmpiricalMotifTransformationV1, start_index: int, event_count: int): None + + + +histdatacom.synthetic.generation._PlannedTransform->histdatacom.synthetic.generation.EmpiricalMotifTransformationV1 + + +record + + + +histdatacom.synthetic.generation._PlannedTransform->histdatacom.synthetic.motifs.ReferenceMotifMatchV1 + + +match - + histdatacom.orchestration.workflows._PreparedChildInvocation - -_PreparedChildInvocation - -invocation -payload : dict[str, JSONValue] | None -skipped_result : StageResult | None - -__init__(self, invocation: WorkflowInvocation, payload: dict[str, JSONValue] | None, skipped_result: StageResult | None): None + +_PreparedChildInvocation + +invocation +payload : dict[str, JSONValue] | None +skipped_result : StageResult | None + +__init__(self, invocation: WorkflowInvocation, payload: dict[str, JSONValue] | None, skipped_result: StageResult | None): None - + histdatacom.orchestration.workflows._PreparedChildInvocation->histdatacom.orchestration.workflows.WorkflowInvocation - - -invocation + + +invocation - + histdatacom.readme_help._ReadmeHelpFormatter - -_ReadmeHelpFormatter - - -_format_action_invocation(action: argparse.Action): str + +_ReadmeHelpFormatter + + +_format_action_invocation(action: argparse.Action): str - + histdatacom.readme_help._ReadmeHelpFormatter->histdatacom.cli.ArgParser - - -formatter_class + + +formatter_class + + + +histdatacom.synthetic.strategy_sensitivity._ReferenceMomentumState + +_ReferenceMomentumState + +decision_interval_ns : int +history : deque[StrategyQuoteV1] +last_decision_ns : int | None +lookback_ns : int +max_state_quotes : int +specification_id : str +threshold_bps : float + +__init__(self, specification_id: str, lookback_ns: int, decision_interval_ns: int, threshold_bps: float, max_state_quotes: int, history: deque[StrategyQuoteV1], last_decision_ns: int | None): None +observe(quote: StrategyQuoteV1): Sequence[StrategySignalV1] + + + +histdatacom.data_quality.seasonal_exogenous._RegressorRow + +_RegressorRow + +active_sessions : tuple[str, ...] +availability : str +available : bool +event_tags : tuple[str, ...] +holiday_tags : tuple[str, ...] +reason : str +special_tags : tuple[str, ...] +values : Mapping[str, float] + +__init__(self, values: Mapping[str, float], available: bool, availability: str, reason: str, active_sessions: tuple[str, ...], special_tags: tuple[str, ...], holiday_tags: tuple[str, ...], event_tags: tuple[str, ...]): None - + histdatacom.data_quality.reporting._RemediationCoverageAggregate - -_RemediationCoverageAggregate - -finding_code : str -mapped : bool -occurrence_count : int -rule_id : str -severity_counts : Counter[str] -target_axis_counts : Counter[tuple[str, str, str, str, str]] - -__init__(self, rule_id: str, finding_code: str, mapped: bool, occurrence_count: int, severity_counts: Counter[str], target_axis_counts: Counter[tuple[str, str, str, str, str]]): None + +_RemediationCoverageAggregate + +actionability : str +actionability_reason : str +finding_code : str +mapped : bool +occurrence_count : int +rule_id : str +severity_counts : Counter[str] +target_axis_counts : Counter[tuple[str, str, str, str, str]] + +__init__(self, rule_id: str, finding_code: str, mapped: bool, actionability: str, actionability_reason: str, occurrence_count: int, severity_counts: Counter[str], target_axis_counts: Counter[tuple[str, str, str, str, str]]): None - + histdatacom.data_quality.remediation_audit._ReportGapAggregate - -_ReportGapAggregate - -finding_code : str -group_count : int -max_severity : str -occurrence_count : int -report_source_counts : Counter[str] -rule_id : str -severity_counts : Counter[str] - -__init__(self, rule_id: str, finding_code: str, occurrence_count: int, group_count: int, severity_counts: Counter[str], report_source_counts: Counter[str]): None + +_ReportGapAggregate + +finding_code : str +group_count : int +max_severity : str +occurrence_count : int +report_source_counts : Counter[str] +rule_id : str +severity_counts : Counter[str] + +__init__(self, rule_id: str, finding_code: str, occurrence_count: int, group_count: int, severity_counts: Counter[str], report_source_counts: Counter[str]): None - + histdatacom.data_quality.remediation_audit._ReportGapEvidence - -_ReportGapEvidence - -aggregates : dict[tuple[str, str], _ReportGapAggregate] -exact_counts : Counter[str] -finding_code_counts : Counter[str] -group_counts : Counter[str] -severity_counts : Counter[str] - -__init__(self, exact_counts: Counter[str], finding_code_counts: Counter[str], group_counts: Counter[str], severity_counts: Counter[str], aggregates: dict[tuple[str, str], _ReportGapAggregate]): None + +_ReportGapEvidence + +aggregates : dict[tuple[str, str], _ReportGapAggregate] +exact_counts : Counter[str] +finding_code_counts : Counter[str] +group_counts : Counter[str] +severity_counts : Counter[str] + +__init__(self, exact_counts: Counter[str], finding_code_counts: Counter[str], group_counts: Counter[str], severity_counts: Counter[str], aggregates: dict[tuple[str, str], _ReportGapAggregate]): None + + + +histdatacom.data_quality.bounded_payload_contracts._RepresentativeQualitySkipRule + +_RepresentativeQualitySkipRule + +description : str +rule_id : str + +evaluate(target: QualityTarget): tuple[QualityFinding, ...] + + + +histdatacom.synthetic.cross_currency._ResidualAccumulator + +_ResidualAccumulator + +allowed : list[float] +infeasible_count : int +post : list[float] +pre : list[float] +projected_count : int + +__init__(self, pre: list[float], post: list[float], allowed: list[float], projected_count: int, infeasible_count: int): None + + + +histdatacom.synthetic.reconstruction_plan._ResolvedPlanInputs + +_ResolvedPlanInputs + +artifacts : Mapping[str, ArtifactRef] +benchmark_corpus : Any +cftc_positioning +feed_epoch_definition : Any +market_context +motif_index : Any +motif_leakage_audit : Mapping[str, Any] +motif_manifest : Mapping[str, Any] +motif_profile +motif_qualification : Mapping[str, Any] +observation_operator + +__init__(self, feed_epoch_definition: Any, observation_operator: ObservationOperatorV1, market_context: MarketContextCorpusV1, cftc_positioning: CftcPositioningCorpusV1, benchmark_corpus: Any, motif_profile: ModernReferenceMotifProfileV1, motif_index: Any, artifacts: Mapping[str, ArtifactRef], motif_manifest: Mapping[str, Any], motif_qualification: Mapping[str, Any], motif_leakage_audit: Mapping[str, Any]): None + + + +histdatacom.synthetic.reconstruction_plan._ResolvedPlanInputs->histdatacom.market_context.positioning.CftcPositioningCorpusV1 + + +cftc_positioning + + + +histdatacom.synthetic.reconstruction_plan._ResolvedPlanInputs->histdatacom.market_context.corpus.MarketContextCorpusV1 + + +market_context + + + +histdatacom.synthetic.reconstruction_plan._ResolvedPlanInputs->histdatacom.synthetic.motif_library.ModernReferenceMotifProfileV1 + + +motif_profile + + + +histdatacom.synthetic.reconstruction_plan._ResolvedPlanInputs->histdatacom.synthetic.observation.ObservationOperatorV1 + + +observation_operator - + histdatacom.data_quality.ingestion._RowSample - -_RowSample - -field_count : int -raw : str -row_number : int - -__init__(self, row_number: int, field_count: int, raw: str): None -to_dict(): dict[str, JSONValue] + +_RowSample + +field_count : int +raw : str +row_number : int + +__init__(self, row_number: int, field_count: int, raw: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.ingestion._RowScan - -_RowScan - -delimiter_count : int -delimiter_samples : list[_RowSample] -field_count_error_count : int -field_count_samples : list[_RowSample] -header_row_number : int | None -row_count : int - -__init__(self, row_count: int, header_row_number: int | None, delimiter_samples: list[_RowSample], field_count_samples: list[_RowSample], delimiter_count: int, field_count_error_count: int): None + +_RowScan + +delimiter_count : int +delimiter_samples : list[_RowSample] +field_count_error_count : int +field_count_samples : list[_RowSample] +header_row_number : int | None +row_count : int + +__init__(self, row_count: int, header_row_number: int | None, delimiter_samples: list[_RowSample], field_count_samples: list[_RowSample], delimiter_count: int, field_count_error_count: int): None + + + +histdatacom.data_quality.remediation_audit._RuleAttribution + +_RuleAttribution + +reason : str +rule_id : str +status : str + +__init__(self, rule_id: str, status: str, reason: str): None + + + +histdatacom.broker_capture.fingerprints._SampleAccumulator + +_SampleAccumulator + +_samples : list[tuple[int, str, float]] +kind : str +maximum : float | None +minimum : float | None +name : str +rounding_digits : int +sample_limit : int +support_count : int +total : float +total_squares : float +unit : str + +__init__(self, name: str, kind: str, unit: str, sample_limit: int, rounding_digits: int, support_count: int, total: float, total_squares: float, minimum: float | None, maximum: float | None, _samples: list[tuple[int, str, float]]): None +add(value: float, evidence_key: str): None +to_metric(quantiles: Sequence[float]): BrokerDeliveryMetricV1 - + histdatacom.data_quality.ingestion._SchemaSample - -_SchemaSample - -column : str -error : str -raw_value : str -row_number : int - -__init__(self, row_number: int, column: str, raw_value: str, error: str): None -to_dict(): dict[str, JSONValue] + +_SchemaSample + +column : str +error : str +raw_value : str +row_number : int + +__init__(self, row_number: int, column: str, raw_value: str, error: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.ingestion._SchemaScan - -_SchemaScan - -bad_dtype_count : int -bad_dtypes : list[_SchemaSample] -bad_numeric_count : int -bad_numerics : list[_SchemaSample] -bad_timestamp_count : int -bad_timestamps : list[_SchemaSample] -bad_volume_count : int -bad_volumes : list[_SchemaSample] -invalid_row_count : int -invalid_rows : list[_SchemaSample] -missing_columns : list[str] -parsed_row_count : int -shifted_row_count : int -shifted_rows : list[_SchemaSample] - -__init__(self, parsed_row_count: int, missing_columns: list[str], bad_timestamp_count: int, bad_numeric_count: int, bad_volume_count: int, bad_dtype_count: int, shifted_row_count: int, invalid_row_count: int, bad_dtypes: list[_SchemaSample], bad_timestamps: list[_SchemaSample], bad_numerics: list[_SchemaSample], bad_volumes: list[_SchemaSample], shifted_rows: list[_SchemaSample], invalid_rows: list[_SchemaSample]): None + +_SchemaScan + +bad_dtype_count : int +bad_dtypes : list[_SchemaSample] +bad_numeric_count : int +bad_numerics : list[_SchemaSample] +bad_timestamp_count : int +bad_timestamps : list[_SchemaSample] +bad_volume_count : int +bad_volumes : list[_SchemaSample] +invalid_row_count : int +invalid_rows : list[_SchemaSample] +missing_columns : list[str] +parsed_row_count : int +shifted_row_count : int +shifted_rows : list[_SchemaSample] + +__init__(self, parsed_row_count: int, missing_columns: list[str], bad_timestamp_count: int, bad_numeric_count: int, bad_volume_count: int, bad_dtype_count: int, shifted_row_count: int, invalid_row_count: int, bad_dtypes: list[_SchemaSample], bad_timestamps: list[_SchemaSample], bad_numerics: list[_SchemaSample], bad_volumes: list[_SchemaSample], shifted_rows: list[_SchemaSample], invalid_rows: list[_SchemaSample]): None + + + +histdatacom.data_analytics.feed_epochs._SensitivityVariant + +_SensitivityVariant + +analysis : str +features : tuple[str, ...] +points : tuple[_PeriodPoint, ...] + +__init__(self, analysis: str, points: tuple[_PeriodPoint, ...], features: tuple[str, ...]): None + + + +histdatacom.synthetic.strategy_sensitivity._SliceAccumulator + +_SliceAccumulator + +completed_count : int +cost_total : float +entry_delay_total : int +favorable_count : int +gross_total : float +missing_support_count : int +net_total : float +signal_count : int + +__init__(self, signal_count: int, completed_count: int, missing_support_count: int, gross_total: float, net_total: float, cost_total: float, entry_delay_total: int, favorable_count: int): None +add_completed(): None +add_missing(): None +add_signal(): None - + histdatacom.data_quality.time._SourceDateInfo - -_SourceDateInfo - -day : int -month : int -month_length : int -month_period : str -ordinal : int -weekday : int -year : int -year_period : str - -__init__(self, year: int, month: int, day: int, weekday: int, ordinal: int, year_period: str, month_period: str, month_length: int): None + +_SourceDateInfo + +day : int +month : int +month_length : int +month_period : str +ordinal : int +weekday : int +year : int +year_period : str + +__init__(self, year: int, month: int, day: int, weekday: int, ordinal: int, year_period: str, month_period: str, month_length: int): None - + histdatacom.data_quality.time._SourceReadError - -_SourceReadError - -code : str -message : str -metadata : dict[str, JSONValue] - -__init__(self, code: str, message: str, metadata: dict[str, JSONValue]): None + +_SourceReadError + +code : str +message : str +metadata : dict[str, JSONValue] + +__init__(self, code: str, message: str, metadata: dict[str, JSONValue]): None - + histdatacom.data_quality.ingestion._SourceReadError - -_SourceReadError - -code : str -message : str -metadata : dict[str, JSONValue] - -__init__(self, code: str, message: str, metadata: dict[str, JSONValue]): None + +_SourceReadError + +code : str +message : str +metadata : dict[str, JSONValue] + +__init__(self, code: str, message: str, metadata: dict[str, JSONValue]): None - + histdatacom.data_quality.ticks._SourceReadError - -_SourceReadError - -code : str -message : str -metadata : dict[str, JSONValue] - -__init__(self, code: str, message: str, metadata: dict[str, JSONValue]): None + +_SourceReadError + +code : str +message : str +metadata : dict[str, JSONValue] + +__init__(self, code: str, message: str, metadata: dict[str, JSONValue]): None - + histdatacom.data_quality.calendar._SourceReadError - -_SourceReadError - -code : str -message : str -metadata : dict[str, JSONValue] - -__init__(self, code: str, message: str, metadata: dict[str, JSONValue]): None + +_SourceReadError + +code : str +message : str +metadata : dict[str, JSONValue] + +__init__(self, code: str, message: str, metadata: dict[str, JSONValue]): None - + histdatacom.data_quality.ticks._SpreadRegimeProfile - -_SpreadRegimeProfile - -max_sample : _TickSpreadRegimeSample | None -samples : list[_TickSpreadRegimeSample] -values : list[float] - -__init__(self, values: list[float], samples: list[_TickSpreadRegimeSample], max_sample: _TickSpreadRegimeSample | None): None -add(sample: _TickSpreadRegimeSample): None -to_metadata(): dict[str, JSONValue] + +_SpreadRegimeProfile + +max_sample : _TickSpreadRegimeSample | None +samples : list[_TickSpreadRegimeSample] +values : list[float] + +__init__(self, values: list[float], samples: list[_TickSpreadRegimeSample], max_sample: _TickSpreadRegimeSample | None): None +add(sample: _TickSpreadRegimeSample): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.symbols._StaleJoinSample - -_StaleJoinSample - -active_series -affected_timestamp_count : int -end_timestamp_utc_ms : int -stale_series -stale_value_timestamp_utc_ms : int -start_timestamp_utc_ms : int - -__init__(self, stale_series: _CrossInstrumentSeries, active_series: _CrossInstrumentSeries, stale_value_timestamp_utc_ms: int, start_timestamp_utc_ms: int, end_timestamp_utc_ms: int, affected_timestamp_count: int): None -to_metadata(): dict[str, JSONValue] + +_StaleJoinSample + +active_series +affected_timestamp_count : int +end_timestamp_utc_ms : int +stale_series +stale_value_timestamp_utc_ms : int +start_timestamp_utc_ms : int + +__init__(self, stale_series: _CrossInstrumentSeries, active_series: _CrossInstrumentSeries, stale_value_timestamp_utc_ms: int, start_timestamp_utc_ms: int, end_timestamp_utc_ms: int, affected_timestamp_count: int): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.symbols._StaleJoinSample->histdatacom.data_quality.symbols._CrossInstrumentSeries - - -stale_series + + +stale_series - + histdatacom.data_quality.symbols._StaleJoinSample->histdatacom.data_quality.symbols._CrossInstrumentSeries - - -active_series + + +active_series - + histdatacom.data_quality.provenance._StoreEvaluation - -_StoreEvaluation - -artifact_count : int -data_artifact_count : int -finding_count : int -findings : list[_FindingTemplate] | None -job_snapshot_count : int -store -target_count : int -work_item_count : int - -__init__(self, store: _StoreRef, work_item_count: int, artifact_count: int, data_artifact_count: int, target_count: int, job_snapshot_count: int, findings: list[_FindingTemplate] | None): None -to_metadata(): dict[str, JSONValue] + +_StoreEvaluation + +artifact_count : int +data_artifact_count : int +finding_count : int +findings : list[_FindingTemplate] | None +job_snapshot_count : int +store +target_count : int +work_item_count : int + +__init__(self, store: _StoreRef, work_item_count: int, artifact_count: int, data_artifact_count: int, target_count: int, job_snapshot_count: int, findings: list[_FindingTemplate] | None): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.provenance._StoreRef - -_StoreRef - -db_path : Path -root : Path -source : str - -__init__(self, root: Path, db_path: Path, source: str): None -to_metadata(): dict[str, JSONValue] + +_StoreRef + +db_path : Path +root : Path +source : str + +__init__(self, root: Path, db_path: Path, source: str): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.provenance._StoreEvaluation->histdatacom.data_quality.provenance._StoreRef - - -store + + +store + + + +histdatacom.synthetic.benchmark._StreamStats + +_StreamStats + +ask_mean : float +ask_total : float +ask_transition_mean : float +ask_transition_total : float +bid_mean : float +bid_total : float +bid_transition_mean : float +bid_transition_total : float +burst_count : int +burst_duration_count : int +burst_duration_mean_ns : float +burst_duration_total_ns : int +burst_rate : float +burst_threshold_ns : int +current_burst_duration_ns : int +event_count : int +exposure_ns : int +first_mid : float | None +intensity_per_second : float +interarrival_buckets : tuple[int, ...] +interarrival_count : int +interarrival_histogram : list[int] +last_mid : float | None +maximum_mid : float | None +mid_range : float +mid_transition_mean : float +mid_transition_total : float +minimum_mid : float | None +previous_ask : float | None +previous_bid : float | None +previous_mid : float | None +previous_spread : float | None +previous_time_ns : int | None +quiet_count : int +quiet_duration_count : int +quiet_duration_mean_ns : float +quiet_duration_total_ns : int +quiet_rate : float +quiet_threshold_ns : int +quote_transition_count : int +spread_buckets : tuple[float, ...] +spread_histogram : list[int] +spread_mean : float +spread_total : float +spread_transition_count : int +spread_transition_mean : float +spread_transition_total : float + +__init__(self, interarrival_buckets: tuple[int, ...], spread_buckets: tuple[float, ...], burst_threshold_ns: int, quiet_threshold_ns: int, event_count: int, exposure_ns: int, interarrival_count: int, burst_count: int, quiet_count: int, burst_duration_total_ns: int, burst_duration_count: int, current_burst_duration_ns: int, quiet_duration_total_ns: int, quiet_duration_count: int, bid_total: float, ask_total: float, spread_total: float, bid_transition_total: float, ask_transition_total: float, mid_transition_total: float, spread_transition_total: float, quote_transition_count: int, spread_transition_count: int, first_mid: float | None, last_mid: float | None, minimum_mid: float | None, maximum_mid: float | None, previous_time_ns: int | None, previous_bid: float | None, previous_ask: float | None, previous_mid: float | None, previous_spread: float | None, interarrival_histogram: list[int], spread_histogram: list[int]): None +__post_init__(): None +add_exposure(duration_ns: int): None +update(events: Sequence[BenchmarkEventV1]): None - + histdatacom.cache_status._SymbolAccumulator - -_SymbolAccumulator - -cache_count : int -cache_size_bytes : int -formats : set[str] -source_artifact_count : int -source_artifact_size_bytes : int -symbol : str -timeframes : set[str] - -__init__(self, symbol: str, cache_count: int, cache_size_bytes: int, source_artifact_count: int, source_artifact_size_bytes: int, formats: set[str], timeframes: set[str]): None -to_dict(): dict[str, Any] + +_SymbolAccumulator + +cache_count : int +cache_size_bytes : int +formats : set[str] +source_artifact_count : int +source_artifact_size_bytes : int +symbol : str +timeframes : set[str] + +__init__(self, symbol: str, cache_count: int, cache_size_bytes: int, source_artifact_count: int, source_artifact_size_bytes: int, formats: set[str], timeframes: set[str]): None +to_dict(): dict[str, Any] - + histdatacom.data_quality.time._TextPayload - -_TextPayload - -data : bytes -source_member : str - -__init__(self, data: bytes, source_member: str): None + +_TextPayload + +data : bytes +source_member : str + +__init__(self, data: bytes, source_member: str): None - + histdatacom.data_quality.ingestion._TextPayload - -_TextPayload - -data : bytes -source_member : str - -__init__(self, data: bytes, source_member: str): None + +_TextPayload + +data : bytes +source_member : str + +__init__(self, data: bytes, source_member: str): None - + histdatacom.data_quality.ticks._TextPayload - -_TextPayload - -data : bytes -source_member : str - -__init__(self, data: bytes, source_member: str): None + +_TextPayload + +data : bytes +source_member : str + +__init__(self, data: bytes, source_member: str): None - + histdatacom.data_quality.calendar._TextPayload - -_TextPayload - -data : bytes -source_member : str - -__init__(self, data: bytes, source_member: str): None + +_TextPayload + +data : bytes +source_member : str + +__init__(self, data: bytes, source_member: str): None - + histdatacom.data_quality.fingerprints._TextPayload - -_TextPayload - -source_member : str -text : str - -__init__(self, text: str, source_member: str): None + +_TextPayload + +source_member : str +text : str + +__init__(self, text: str, source_member: str): None - + histdatacom.data_quality.fingerprints._TickDynamicsRow - -_TickDynamicsRow - -ask : float -bid : float -spread : float -timestamp_utc_ms : int - -__init__(self, timestamp_utc_ms: int, bid: float, ask: float): None + +_TickDynamicsRow + +ask : float +bid : float +spread : float +timestamp_utc_ms : int + +__init__(self, timestamp_utc_ms: int, bid: float, ask: float): None - + histdatacom.data_quality.ticks._TickLineSource - -_TickLineSource - -iter_lines : Callable[[], Iterable[str]] -source_member : str - -__init__(self, iter_lines: Callable[[], Iterable[str]], source_member: str): None + +_TickLineSource + +iter_lines : Callable[[], Iterable[str]] +source_member : str + +__init__(self, iter_lines: Callable[[], Iterable[str]], source_member: str): None - + histdatacom.data_quality.ticks._TickMicrostructureRunSample - -_TickMicrostructureRunSample - -direction : str -end -metadata : dict[str, JSONValue] -metric : str -row_number : int -run_length : int -source_member : str -start -timestamp_source : str -timestamp_utc_ms : int | None -utc_timestamp : str - -__init__(self, start: _TickSpreadSample, end: _TickSpreadSample, run_length: int, metric: str, direction: str, metadata: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +_TickMicrostructureRunSample + +direction : str +end +metadata : dict[str, JSONValue] +metric : str +row_number : int +run_length : int +source_member : str +start +timestamp_source : str +timestamp_utc_ms : int | None +utc_timestamp : str + +__init__(self, start: _TickSpreadSample, end: _TickSpreadSample, run_length: int, metric: str, direction: str, metadata: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.ticks._TickSpreadSample - -_TickSpreadSample - -ask : float | None -bid : float | None -column : str -metadata : dict[str, JSONValue] -raw_values : tuple[str, ...] -row_number : int -source_member : str -spread : float | None -timestamp_source : str -timestamp_utc_ms : int | None -utc_timestamp : str - -__init__(self, row_number: int, timestamp_source: str, timestamp_utc_ms: int | None, column: str, bid: float | None, ask: float | None, spread: float | None, raw_values: tuple[str, ...], source_member: str, metadata: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] -values_metadata(): dict[str, JSONValue] + +_TickSpreadSample + +ask : float | None +bid : float | None +column : str +metadata : dict[str, JSONValue] +raw_values : tuple[str, ...] +row_number : int +source_member : str +spread : float | None +timestamp_source : str +timestamp_utc_ms : int | None +utc_timestamp : str + +__init__(self, row_number: int, timestamp_source: str, timestamp_utc_ms: int | None, column: str, bid: float | None, ask: float | None, spread: float | None, raw_values: tuple[str, ...], source_member: str, metadata: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] +values_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.ticks._TickMicrostructureRunSample->histdatacom.data_quality.ticks._TickSpreadSample - - -start + + +start - + histdatacom.data_quality.ticks._TickMicrostructureRunSample->histdatacom.data_quality.ticks._TickSpreadSample - - -end + + +end - + histdatacom.data_quality.ticks._TickMicrostructureScan - -_TickMicrostructureScan - -ask_only_movement_count : int -bid_only_movement_count : int -burst_interval_count : int -burst_run_count : int -burst_runs : list[_TickMicrostructureRunSample] -burst_tick_count : int -duplicate_row_count : int -duplicate_rows : list[_TickSpreadSample] -invalid_tick_count : int -one_sided_movement_count : int -one_sided_run_count : int -one_sided_runs : list[_TickMicrostructureRunSample] -parsed_row_count : int -row_count : int -stale_quote_repeat_count : int -stale_quote_run_count : int -stale_quote_run_row_count : int -stale_quote_runs : list[_TickMicrostructureRunSample] - -__init__(self, row_count: int, parsed_row_count: int, invalid_tick_count: int, duplicate_row_count: int, stale_quote_repeat_count: int, stale_quote_run_count: int, stale_quote_run_row_count: int, burst_interval_count: int, burst_run_count: int, burst_tick_count: int, one_sided_movement_count: int, one_sided_run_count: int, bid_only_movement_count: int, ask_only_movement_count: int, duplicate_rows: list[_TickSpreadSample], stale_quote_runs: list[_TickMicrostructureRunSample], burst_runs: list[_TickMicrostructureRunSample], one_sided_runs: list[_TickMicrostructureRunSample]): None + +_TickMicrostructureScan + +ask_only_movement_count : int +bid_only_movement_count : int +burst_interval_count : int +burst_run_count : int +burst_runs : list[_TickMicrostructureRunSample] +burst_tick_count : int +duplicate_row_count : int +duplicate_rows : list[_TickSpreadSample] +invalid_tick_count : int +one_sided_movement_count : int +one_sided_run_count : int +one_sided_runs : list[_TickMicrostructureRunSample] +parsed_row_count : int +row_count : int +stale_quote_repeat_count : int +stale_quote_run_count : int +stale_quote_run_row_count : int +stale_quote_runs : list[_TickMicrostructureRunSample] + +__init__(self, row_count: int, parsed_row_count: int, invalid_tick_count: int, duplicate_row_count: int, stale_quote_repeat_count: int, stale_quote_run_count: int, stale_quote_run_row_count: int, burst_interval_count: int, burst_run_count: int, burst_tick_count: int, one_sided_movement_count: int, one_sided_run_count: int, bid_only_movement_count: int, ask_only_movement_count: int, duplicate_rows: list[_TickSpreadSample], stale_quote_runs: list[_TickMicrostructureRunSample], burst_runs: list[_TickMicrostructureRunSample], one_sided_runs: list[_TickMicrostructureRunSample]): None - + histdatacom.data_quality.ticks._TickMicrostructureThresholdSelection - -_TickMicrostructureThresholdSelection - -asset_class_key : str -profile_key : str -session_key : str -source : str -symbol_key : str -thresholds - -__init__(self, thresholds: HistDataTickMicrostructureThresholds, source: str, symbol_key: str, session_key: str, asset_class_key: str, profile_key: str): None -to_metadata(): dict[str, JSONValue] + +_TickMicrostructureThresholdSelection + +asset_class_key : str +profile_key : str +session_key : str +source : str +symbol_key : str +thresholds + +__init__(self, thresholds: HistDataTickMicrostructureThresholds, source: str, symbol_key: str, session_key: str, asset_class_key: str, profile_key: str): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.ticks._TickMicrostructureThresholdSelection->histdatacom.data_quality.ticks.HistDataTickMicrostructureThresholds - - -thresholds - - - -histdatacom.data_analytics.feed_regimes._TickObservation - -_TickObservation - -ask : float -bid : float -symbol : str -target_path : str -utc_ms : int - -__init__(self, utc_ms: int, bid: float, ask: float, symbol: str, target_path: str): None + + +thresholds - + histdatacom.data_quality.ticks._TickQualityScans - -_TickQualityScans - -microstructure -source_member : str -spread -spread_regime - -__init__(self, spread: _TickSpreadScan, microstructure: _TickMicrostructureScan, spread_regime: _TickSpreadRegimeScan, source_member: str): None + +_TickQualityScans + +microstructure +source_member : str +spread +spread_regime + +__init__(self, spread: _TickSpreadScan, microstructure: _TickMicrostructureScan, spread_regime: _TickSpreadRegimeScan, source_member: str): None - + histdatacom.data_quality.ticks._TickQualityScans->histdatacom.data_quality.ticks._TickMicrostructureScan - - -microstructure + + +microstructure - + histdatacom.data_quality.ticks._TickSpreadRegimeScan - -_TickSpreadRegimeScan - -global_profile -invalid_tick_count : int -invalid_timestamp_count : int -liquid_profile -negative_spread_count : int -parsed_row_count : int -profiled_row_count : int -regime_shift_count : int -regime_shifts : list[_TickSpreadRegimeSample] -row_count : int -session_profiles : dict[str, _SpreadRegimeProfile] -source_hour_profiles : dict[str, _SpreadRegimeProfile] -special_regime_profiles : dict[str, _SpreadRegimeProfile] -spread_jump_candidates : list[_TickSpreadRegimeSample] -spread_jump_count : int -spread_jump_values : list[float] -spread_jumps : list[_TickSpreadRegimeSample] -symbol_profiles : dict[str, _SpreadRegimeProfile] -wide_candidates : list[_TickSpreadRegimeSample] -wide_spread_count : int -wide_spreads : list[_TickSpreadRegimeSample] - -__init__(self, row_count: int, parsed_row_count: int, profiled_row_count: int, invalid_tick_count: int, invalid_timestamp_count: int, negative_spread_count: int, symbol_profiles: dict[str, _SpreadRegimeProfile], source_hour_profiles: dict[str, _SpreadRegimeProfile], session_profiles: dict[str, _SpreadRegimeProfile], special_regime_profiles: dict[str, _SpreadRegimeProfile], liquid_profile: _SpreadRegimeProfile, global_profile: _SpreadRegimeProfile, wide_spread_count: int, spread_jump_count: int, regime_shift_count: int, spread_jump_values: list[float], wide_candidates: list[_TickSpreadRegimeSample], spread_jump_candidates: list[_TickSpreadRegimeSample], wide_spreads: list[_TickSpreadRegimeSample], spread_jumps: list[_TickSpreadRegimeSample], regime_shifts: list[_TickSpreadRegimeSample]): None + +_TickSpreadRegimeScan + +global_profile +invalid_tick_count : int +invalid_timestamp_count : int +liquid_profile +negative_spread_count : int +parsed_row_count : int +profiled_row_count : int +regime_shift_count : int +regime_shifts : list[_TickSpreadRegimeSample] +row_count : int +session_profiles : dict[str, _SpreadRegimeProfile] +source_hour_profiles : dict[str, _SpreadRegimeProfile] +special_regime_profiles : dict[str, _SpreadRegimeProfile] +spread_jump_candidates : list[_TickSpreadRegimeSample] +spread_jump_count : int +spread_jump_values : list[float] +spread_jumps : list[_TickSpreadRegimeSample] +symbol_profiles : dict[str, _SpreadRegimeProfile] +wide_candidates : list[_TickSpreadRegimeSample] +wide_spread_count : int +wide_spreads : list[_TickSpreadRegimeSample] + +__init__(self, row_count: int, parsed_row_count: int, profiled_row_count: int, invalid_tick_count: int, invalid_timestamp_count: int, negative_spread_count: int, symbol_profiles: dict[str, _SpreadRegimeProfile], source_hour_profiles: dict[str, _SpreadRegimeProfile], session_profiles: dict[str, _SpreadRegimeProfile], special_regime_profiles: dict[str, _SpreadRegimeProfile], liquid_profile: _SpreadRegimeProfile, global_profile: _SpreadRegimeProfile, wide_spread_count: int, spread_jump_count: int, regime_shift_count: int, spread_jump_values: list[float], wide_candidates: list[_TickSpreadRegimeSample], spread_jump_candidates: list[_TickSpreadRegimeSample], wide_spreads: list[_TickSpreadRegimeSample], spread_jumps: list[_TickSpreadRegimeSample], regime_shifts: list[_TickSpreadRegimeSample]): None - + histdatacom.data_quality.ticks._TickQualityScans->histdatacom.data_quality.ticks._TickSpreadRegimeScan - - -spread_regime + + +spread_regime - + histdatacom.data_quality.ticks._TickSpreadScan - -_TickSpreadScan - -invalid_bid_ask : list[_TickSpreadSample] -invalid_bid_ask_count : int -max_spread : float | None -min_spread : float | None -missing_bid_ask : list[_TickSpreadSample] -missing_bid_ask_count : int -negative_spread_count : int -negative_spreads : list[_TickSpreadSample] -parsed_row_count : int -row_count : int -zero_spread_count : int -zero_spread_run_count : int -zero_spread_runs : list[_ZeroSpreadRunSample] - -__init__(self, row_count: int, parsed_row_count: int, missing_bid_ask_count: int, invalid_bid_ask_count: int, negative_spread_count: int, zero_spread_count: int, zero_spread_run_count: int, min_spread: float | None, max_spread: float | None, missing_bid_ask: list[_TickSpreadSample], invalid_bid_ask: list[_TickSpreadSample], negative_spreads: list[_TickSpreadSample], zero_spread_runs: list[_ZeroSpreadRunSample]): None + +_TickSpreadScan + +invalid_bid_ask : list[_TickSpreadSample] +invalid_bid_ask_count : int +max_spread : float | None +min_spread : float | None +missing_bid_ask : list[_TickSpreadSample] +missing_bid_ask_count : int +negative_spread_count : int +negative_spreads : list[_TickSpreadSample] +parsed_row_count : int +row_count : int +zero_spread_count : int +zero_spread_run_count : int +zero_spread_runs : list[_ZeroSpreadRunSample] + +__init__(self, row_count: int, parsed_row_count: int, missing_bid_ask_count: int, invalid_bid_ask_count: int, negative_spread_count: int, zero_spread_count: int, zero_spread_run_count: int, min_spread: float | None, max_spread: float | None, missing_bid_ask: list[_TickSpreadSample], invalid_bid_ask: list[_TickSpreadSample], negative_spreads: list[_TickSpreadSample], zero_spread_runs: list[_ZeroSpreadRunSample]): None - + histdatacom.data_quality.ticks._TickQualityScans->histdatacom.data_quality.ticks._TickSpreadScan - - -spread + + +spread + + + +histdatacom.synthetic.benchmark_corpus._TickRow + +_TickRow + +ask : float +bid : float +row_id : int +timestamp_ms : int + +__init__(self, row_id: int, timestamp_ms: int, bid: float, ask: float): None + + + +histdatacom.synthetic.motif_library._TickRow + +_TickRow + +ask : float +bid : float +row_id : int +timestamp_ms : int + +__init__(self, row_id: int, timestamp_ms: int, bid: float, ask: float): None - + histdatacom.data_quality.ticks._TickSpreadRegimeProjection - -_TickSpreadRegimeProjection - -active_sessions : tuple[str, ...] -session_state : str -source_hour : int -special_tags : tuple[str, ...] -utc_hour : int - -__init__(self, source_hour: int, utc_hour: int, session_state: str, active_sessions: tuple[str, ...], special_tags: tuple[str, ...]): None + +_TickSpreadRegimeProjection + +active_sessions : tuple[str, ...] +session_state : str +source_hour : int +special_tags : tuple[str, ...] +utc_hour : int + +__init__(self, source_hour: int, utc_hour: int, session_state: str, active_sessions: tuple[str, ...], special_tags: tuple[str, ...]): None - + histdatacom.data_quality.ticks._TickSpreadRegimeSample - -_TickSpreadRegimeSample - -previous : _TickSpreadSample | None -profile_key : str -row_number : int -sample -session_keys : tuple[str, ...] -session_state : str -source_hour : str -source_member : str -special_regime_keys : tuple[str, ...] -spread : float | None -spread_delta : float | None -symbol_key : str -threshold : float | None -timestamp_source : str -timestamp_utc_ms : int | None -utc_hour : str -utc_timestamp : str - -__init__(self, sample: _TickSpreadSample, symbol_key: str, source_hour: str, utc_hour: str, session_state: str, session_keys: tuple[str, ...], special_regime_keys: tuple[str, ...], previous: _TickSpreadSample | None, spread_delta: float | None, threshold: float | None, profile_key: str): None -to_dict(): dict[str, JSONValue] + +_TickSpreadRegimeSample + +previous : _TickSpreadSample | None +profile_key : str +row_number : int +sample +session_keys : tuple[str, ...] +session_state : str +source_hour : str +source_member : str +special_regime_keys : tuple[str, ...] +spread : float | None +spread_delta : float | None +symbol_key : str +threshold : float | None +timestamp_source : str +timestamp_utc_ms : int | None +utc_hour : str +utc_timestamp : str + +__init__(self, sample: _TickSpreadSample, symbol_key: str, source_hour: str, utc_hour: str, session_state: str, session_keys: tuple[str, ...], special_regime_keys: tuple[str, ...], previous: _TickSpreadSample | None, spread_delta: float | None, threshold: float | None, profile_key: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.ticks._TickSpreadRegimeSample->histdatacom.data_quality.ticks._TickSpreadSample - - -sample + + +sample - + histdatacom.data_quality.ticks._TickSpreadRegimeScan->histdatacom.data_quality.ticks._SpreadRegimeProfile - - -liquid_profile + + +liquid_profile - + histdatacom.data_quality.ticks._TickSpreadRegimeScan->histdatacom.data_quality.ticks._SpreadRegimeProfile - - -global_profile + + +global_profile - + histdatacom.data_quality.ticks._TickSpreadRegimeThresholdSelection - -_TickSpreadRegimeThresholdSelection - -asset_class_key : str -profile_key : str -source : str -symbol_key : str -thresholds - -__init__(self, thresholds: HistDataTickSpreadRegimeThresholds, source: str, symbol_key: str, asset_class_key: str, profile_key: str): None -to_metadata(): dict[str, JSONValue] + +_TickSpreadRegimeThresholdSelection + +asset_class_key : str +profile_key : str +source : str +symbol_key : str +thresholds + +__init__(self, thresholds: HistDataTickSpreadRegimeThresholds, source: str, symbol_key: str, asset_class_key: str, profile_key: str): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.ticks._TickSpreadRegimeThresholdSelection->histdatacom.data_quality.ticks.HistDataTickSpreadRegimeThresholds - - -thresholds + + +thresholds - + histdatacom.data_quality.ticks._TickSpreadThresholdSelection - -_TickSpreadThresholdSelection - -asset_class_key : str -profile_key : str -source : str -symbol_key : str -thresholds - -__init__(self, thresholds: HistDataTickSpreadThresholds, source: str, symbol_key: str, asset_class_key: str, profile_key: str): None -to_metadata(): dict[str, JSONValue] + +_TickSpreadThresholdSelection + +asset_class_key : str +profile_key : str +source : str +symbol_key : str +thresholds + +__init__(self, thresholds: HistDataTickSpreadThresholds, source: str, symbol_key: str, asset_class_key: str, profile_key: str): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.ticks._TickSpreadThresholdSelection->histdatacom.data_quality.ticks.HistDataTickSpreadThresholds - - -thresholds + + +thresholds - + histdatacom.data_quality.time._TimestampGapSample - -_TimestampGapSample - -classification : str -current -dynamic_score_increment : float -dynamic_window_ms : int -gap_ms : int -previous -row_number : int -source_member : str -timestamp_source : str -timestamp_utc_ms : int -utc_timestamp : str - -__init__(self, previous: _TimestampSample, current: _TimestampSample, gap_ms: int, classification: str, dynamic_window_ms: int, dynamic_score_increment: float): None -to_dict(): dict[str, JSONValue] + +_TimestampGapSample + +classification : str +current +dynamic_score_increment : float +dynamic_window_ms : int +gap_ms : int +previous +row_number : int +source_member : str +timestamp_source : str +timestamp_utc_ms : int +utc_timestamp : str + +__init__(self, previous: _TimestampSample, current: _TimestampSample, gap_ms: int, classification: str, dynamic_window_ms: int, dynamic_score_increment: float): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.time._TimestampGapSample->histdatacom.data_quality.time._TimestampSample - - -previous + + +previous - + histdatacom.data_quality.time._TimestampGapSample->histdatacom.data_quality.time._TimestampSample - - -current + + +current - + histdatacom.data_quality.time._TimestampGapScan - -_TimestampGapScan - -bucket_counts : dict[str, int] -dynamic_gap_score : float -expected_session_closure_count : int -expected_session_closures : list[_TimestampGapSample] -final_dynamic_window_ms : int -max_gap : _TimestampGapSample | None -max_gap_ms : int -pair_count : int -suspicious_gap_count : int -suspicious_gaps : list[_TimestampGapSample] -tracked_gap_count : int -weekend_activity : list[_WeekendActivitySample] -weekend_activity_count : int - -__init__(self, pair_count: int, tracked_gap_count: int, max_gap_ms: int, max_gap: _TimestampGapSample | None, bucket_counts: dict[str, int], expected_session_closure_count: int, expected_session_closures: list[_TimestampGapSample], suspicious_gap_count: int, suspicious_gaps: list[_TimestampGapSample], weekend_activity_count: int, weekend_activity: list[_WeekendActivitySample], dynamic_gap_score: float, final_dynamic_window_ms: int): None + +_TimestampGapScan + +bucket_counts : dict[str, int] +dynamic_gap_score : float +expected_session_closure_count : int +expected_session_closures : list[_TimestampGapSample] +final_dynamic_window_ms : int +max_gap : _TimestampGapSample | None +max_gap_ms : int +pair_count : int +suspicious_gap_count : int +suspicious_gaps : list[_TimestampGapSample] +tracked_gap_count : int +weekend_activity : list[_WeekendActivitySample] +weekend_activity_count : int + +__init__(self, pair_count: int, tracked_gap_count: int, max_gap_ms: int, max_gap: _TimestampGapSample | None, bucket_counts: dict[str, int], expected_session_closure_count: int, expected_session_closures: list[_TimestampGapSample], suspicious_gap_count: int, suspicious_gaps: list[_TimestampGapSample], weekend_activity_count: int, weekend_activity: list[_WeekendActivitySample], dynamic_gap_score: float, final_dynamic_window_ms: int): None - + histdatacom.data_quality.symbols._TimestampGridSample - -_TimestampGridSample - -common_timestamp_count : int -common_timestamp_ratio : float -missing_by_symbol : dict[str, int] -period : str -symbols : tuple[str, ...] -timeframe : str -union_timestamp_count : int - -__init__(self, timeframe: str, period: str, symbols: tuple[str, ...], union_timestamp_count: int, common_timestamp_count: int, common_timestamp_ratio: float, missing_by_symbol: dict[str, int]): None -to_metadata(): dict[str, JSONValue] + +_TimestampGridSample + +common_timestamp_count : int +common_timestamp_ratio : float +missing_by_symbol : dict[str, int] +period : str +symbols : tuple[str, ...] +timeframe : str +union_timestamp_count : int + +__init__(self, timeframe: str, period: str, symbols: tuple[str, ...], union_timestamp_count: int, common_timestamp_count: int, common_timestamp_ratio: float, missing_by_symbol: dict[str, int]): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.time._TimestampIssueSample - -_TimestampIssueSample - -metadata : dict[str, JSONValue] -row_number : int -source_member : str -timestamp_source : str -timestamp_utc_ms : int | None -utc_timestamp : str - -__init__(self, row_number: int, timestamp_source: str, timestamp_utc_ms: int | None, source_member: str, metadata: dict[str, JSONValue]): None -to_dict(): dict[str, JSONValue] + +_TimestampIssueSample + +metadata : dict[str, JSONValue] +row_number : int +source_member : str +timestamp_source : str +timestamp_utc_ms : int | None +utc_timestamp : str + +__init__(self, row_number: int, timestamp_source: str, timestamp_utc_ms: int | None, source_member: str, metadata: dict[str, JSONValue]): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.time._TimestampScan - -_TimestampScan - -cache_source : str -field_count_error_count : int -first_valid_row : _TimestampSample | None -gap_scans : dict[tuple[JSONValue, ...], '_TimestampGapScan'] -header_row_count : int -invalid_timestamp_count : int -last_valid_row : _TimestampSample | None -parsed_row_count : int -period_mismatch_count : int -period_mismatches : list[_TimestampSample] -polars_frame : Any | None -row_count : int -samples : list[_TimestampSample] -sequence_scan : str -tick_precision_mismatch_count : int -tick_precision_mismatches : list['_TimestampIssueSample'] -topology_source : str -used_timestamp_projection : bool -utc_month_boundaries : list[_TimestampSample] -utc_month_boundary_count : int -valid_rows : list[_TimestampSample] - -__init__(self, row_count: int, parsed_row_count: int, invalid_timestamp_count: int, field_count_error_count: int, header_row_count: int, valid_rows: list[_TimestampSample], samples: list[_TimestampSample], period_mismatch_count: int, period_mismatches: list[_TimestampSample], utc_month_boundary_count: int, utc_month_boundaries: list[_TimestampSample], tick_precision_mismatch_count: int, tick_precision_mismatches: list['_TimestampIssueSample'], first_valid_row: _TimestampSample | None, last_valid_row: _TimestampSample | None, sequence_scan: '_TimestampSequenceScan | None', gap_scans: dict[tuple[JSONValue, ...], '_TimestampGapScan'], polars_frame: Any | None, topology_source: str, cache_source: str, used_timestamp_projection: bool): None + +_TimestampScan + +cache_source : str +field_count_error_count : int +first_valid_row : _TimestampSample | None +gap_scans : dict[tuple[JSONValue, ...], '_TimestampGapScan'] +header_row_count : int +invalid_timestamp_count : int +invalid_timestamps : list['_TimestampIssueSample'] +last_valid_row : _TimestampSample | None +parsed_row_count : int +period_mismatch_count : int +period_mismatches : list[_TimestampSample] +polars_frame : Any | None +row_count : int +samples : list[_TimestampSample] +sequence_scan : str +tick_precision_mismatch_count : int +tick_precision_mismatches : list['_TimestampIssueSample'] +topology_source : str +used_timestamp_projection : bool +utc_month_boundaries : list[_TimestampSample] +utc_month_boundary_count : int +valid_rows : list[_TimestampSample] + +__init__(self, row_count: int, parsed_row_count: int, invalid_timestamp_count: int, invalid_timestamps: list['_TimestampIssueSample'], field_count_error_count: int, header_row_count: int, valid_rows: list[_TimestampSample], samples: list[_TimestampSample], period_mismatch_count: int, period_mismatches: list[_TimestampSample], utc_month_boundary_count: int, utc_month_boundaries: list[_TimestampSample], tick_precision_mismatch_count: int, tick_precision_mismatches: list['_TimestampIssueSample'], first_valid_row: _TimestampSample | None, last_valid_row: _TimestampSample | None, sequence_scan: '_TimestampSequenceScan | None', gap_scans: dict[tuple[JSONValue, ...], '_TimestampGapScan'], polars_frame: Any | None, topology_source: str, cache_source: str, used_timestamp_projection: bool): None - + histdatacom.data_quality.time._TimestampScanCacheKey - -_TimestampScanCacheKey - -kind : str -mtime_ns : int -path : str -period : str -size_bytes : int -timeframe : str - -__init__(self, path: str, kind: str, timeframe: str, period: str, size_bytes: int, mtime_ns: int): None + +_TimestampScanCacheKey + +kind : str +mtime_ns : int +path : str +period : str +prefer_cache : bool +size_bytes : int +timeframe : str + +__init__(self, path: str, kind: str, timeframe: str, period: str, size_bytes: int, mtime_ns: int, prefer_cache: bool): None - + histdatacom.data_quality.time._TimestampSequenceScan - -_TimestampSequenceScan - -non_monotonic_count : int -non_monotonic_rows : list[_TimestampIssueSample] -tick_duplicate_row_count : int -tick_duplicate_rows : list[_TimestampIssueSample] - -__init__(self, non_monotonic_count: int, non_monotonic_rows: list[_TimestampIssueSample], tick_duplicate_row_count: int, tick_duplicate_rows: list[_TimestampIssueSample]): None + +_TimestampSequenceScan + +non_monotonic_count : int +non_monotonic_rows : list[_TimestampIssueSample] +tick_duplicate_row_count : int +tick_duplicate_rows : list[_TimestampIssueSample] +tick_duplicate_timestamp_count : int +tick_duplicate_timestamps : list['_DuplicateTimestampSample'] + +__init__(self, non_monotonic_count: int, non_monotonic_rows: list[_TimestampIssueSample], tick_duplicate_row_count: int, tick_duplicate_rows: list[_TimestampIssueSample], tick_duplicate_timestamp_count: int, tick_duplicate_timestamps: list['_DuplicateTimestampSample']): None - + histdatacom.data_quality.symbols._TriangularComparisonSample - -_TriangularComparisonSample - -denominator -direct -direct_price : float -implied_price : float -numerator -relative_difference : float -severity -timestamp_utc_ms : int - -__init__(self, direct: _CrossInstrumentSeries, numerator: _CrossInstrumentSeries, denominator: _CrossInstrumentSeries, timestamp_utc_ms: int, direct_price: float, implied_price: float, relative_difference: float, severity: QualitySeverity): None -to_metadata(): dict[str, JSONValue] + +_TriangularComparisonSample + +denominator +direct +direct_price : float +implied_price : float +numerator +relative_difference : float +severity +timestamp_utc_ms : int + +__init__(self, direct: _CrossInstrumentSeries, numerator: _CrossInstrumentSeries, denominator: _CrossInstrumentSeries, timestamp_utc_ms: int, direct_price: float, implied_price: float, relative_difference: float, severity: QualitySeverity): None +to_metadata(): dict[str, JSONValue] - + histdatacom.data_quality.symbols._TriangularComparisonSample->histdatacom.data_quality.contracts.QualitySeverity - - -severity + + +severity - + histdatacom.data_quality.symbols._TriangularComparisonSample->histdatacom.data_quality.symbols._CrossInstrumentSeries - - -direct + + +direct - + histdatacom.data_quality.symbols._TriangularComparisonSample->histdatacom.data_quality.symbols._CrossInstrumentSeries - - -numerator + + +numerator - + histdatacom.data_quality.symbols._TriangularComparisonSample->histdatacom.data_quality.symbols._CrossInstrumentSeries - - -denominator + + +denominator - + histdatacom.data_quality.preflight._ValidationReportCandidate - -_ValidationReportCandidate - -generated_at_utc : datetime -path : Path -payload : dict[str, JSONValue] -result_count : int - -__init__(self, path: Path, payload: dict[str, JSONValue], generated_at_utc: datetime, result_count: int): None + +_ValidationReportCandidate + +generated_at_utc : datetime +path : Path +payload : dict[str, JSONValue] +result_count : int + +__init__(self, path: Path, payload: dict[str, JSONValue], generated_at_utc: datetime, result_count: int): None - + histdatacom.data_quality.time._WeekendActivitySample - -_WeekendActivitySample - -row -session_state : str -utc_timestamp : str - -__init__(self, row: _TimestampSample, session_state: str): None -to_dict(): dict[str, JSONValue] + +_WeekendActivitySample + +row +session_state : str +utc_timestamp : str + +__init__(self, row: _TimestampSample, session_state: str): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.time._WeekendActivitySample->histdatacom.data_quality.time._TimestampSample - - -row + + +row + + + +histdatacom.resource_usage._WindowsProcessMemoryCounters + +_WindowsProcessMemoryCounters + +_fields_ : list +cb + + - + histdatacom.data_quality.ticks._ZeroSpreadRunSample - -_ZeroSpreadRunSample - -end -row_number : int -run_length : int -source_member : str -start -timestamp_source : str -timestamp_utc_ms : int | None -utc_timestamp : str - -__init__(self, start: _TickSpreadSample, end: _TickSpreadSample, run_length: int): None -to_dict(): dict[str, JSONValue] + +_ZeroSpreadRunSample + +end +row_number : int +run_length : int +source_member : str +start +timestamp_source : str +timestamp_utc_ms : int | None +utc_timestamp : str + +__init__(self, start: _TickSpreadSample, end: _TickSpreadSample, run_length: int): None +to_dict(): dict[str, JSONValue] - + histdatacom.data_quality.ticks._ZeroSpreadRunSample->histdatacom.data_quality.ticks._TickSpreadSample - - -start + + +start - + histdatacom.data_quality.ticks._ZeroSpreadRunSample->histdatacom.data_quality.ticks._TickSpreadSample - - -end + + +end diff --git a/tests/architecture/packages_pyreverse.svg b/tests/architecture/packages_pyreverse.svg index 304283fb..9c91e307 100644 --- a/tests/architecture/packages_pyreverse.svg +++ b/tests/architecture/packages_pyreverse.svg @@ -4,2968 +4,5770 @@ - - + + packages_pyreverse - + histdatacom - -histdatacom + +histdatacom - + histdatacom.fx_enums - -histdatacom.fx_enums + +histdatacom.fx_enums histdatacom->histdatacom.fx_enums - - + + - + histdatacom.options - -histdatacom.options + +histdatacom.options histdatacom->histdatacom.options - - + + histdatacom.__main__ - -histdatacom.__main__ + +histdatacom.__main__ histdatacom.activity_stages - -histdatacom.activity_stages + +histdatacom.activity_stages histdatacom.activity_stages->histdatacom - - + + - + histdatacom.cancellation - -histdatacom.cancellation + +histdatacom.cancellation histdatacom.activity_stages->histdatacom.cancellation - - + + - + histdatacom.config - -histdatacom.config + +histdatacom.config histdatacom.activity_stages->histdatacom.config - - + + + + + +histdatacom.data_quality.training_features + +histdatacom.data_quality.training_features + + + +histdatacom.activity_stages->histdatacom.data_quality.training_features + + - + histdatacom.exceptions - -histdatacom.exceptions + +histdatacom.exceptions - + histdatacom.activity_stages->histdatacom.exceptions - - + + - + histdatacom.activity_stages->histdatacom.fx_enums - - + + - + histdatacom.histdata_ascii - -histdatacom.histdata_ascii + +histdatacom.histdata_ascii - + histdatacom.activity_stages->histdatacom.histdata_ascii - - + + + + + +histdatacom.random_windows + +histdatacom.random_windows + + + +histdatacom.activity_stages->histdatacom.random_windows + + - + histdatacom.records - -histdatacom.records + +histdatacom.records - + histdatacom.activity_stages->histdatacom.records - - + + - + histdatacom.repository_quality - -histdatacom.repository_quality + +histdatacom.repository_quality - + histdatacom.activity_stages->histdatacom.repository_quality - - + + - + histdatacom.runtime_contracts - -histdatacom.runtime_contracts + +histdatacom.runtime_contracts - + histdatacom.activity_stages->histdatacom.runtime_contracts - - + + - + histdatacom.utils - -histdatacom.utils + +histdatacom.utils - + histdatacom.activity_stages->histdatacom.utils - - + + histdatacom.api - -histdatacom.api + +histdatacom.api - + histdatacom.api->histdatacom.activity_stages - - + + - + histdatacom.helper_args - -histdatacom.helper_args + +histdatacom.helper_args - + histdatacom.api->histdatacom.helper_args - - + + - + histdatacom.api->histdatacom.histdata_ascii - - + + - + histdatacom.legacy_boundary - -histdatacom.legacy_boundary + +histdatacom.legacy_boundary - + histdatacom.api->histdatacom.legacy_boundary - - + + - + histdatacom.observability - -histdatacom.observability + +histdatacom.observability - + histdatacom.api->histdatacom.observability - - + + + + + +histdatacom.api->histdatacom.random_windows + + - + histdatacom.api->histdatacom.records - - + + - + histdatacom.api->histdatacom.runtime_contracts - - + + - + histdatacom.scraper.scraper - -histdatacom.scraper.scraper + +histdatacom.scraper.scraper - + histdatacom.api->histdatacom.scraper.scraper - - + + - + histdatacom.api->histdatacom.utils - - + + - + +histdatacom.broker_capture + +histdatacom.broker_capture + + + +histdatacom.broker_capture.adapters + +histdatacom.broker_capture.adapters + + + +histdatacom.broker_capture->histdatacom.broker_capture.adapters + + + + + +histdatacom.broker_capture.contracts + +histdatacom.broker_capture.contracts + + + +histdatacom.broker_capture->histdatacom.broker_capture.contracts + + + + + +histdatacom.broker_capture.fingerprint_contracts + +histdatacom.broker_capture.fingerprint_contracts + + + +histdatacom.broker_capture->histdatacom.broker_capture.fingerprint_contracts + + + + + +histdatacom.broker_capture.fingerprints + +histdatacom.broker_capture.fingerprints + + + +histdatacom.broker_capture->histdatacom.broker_capture.fingerprints + + + + + +histdatacom.broker_capture.storage + +histdatacom.broker_capture.storage + + + +histdatacom.broker_capture->histdatacom.broker_capture.storage + + + + + +histdatacom.broker_capture.adapters->histdatacom.broker_capture.contracts + + + + + +histdatacom.broker_capture.contracts->histdatacom.runtime_contracts + + + + + +histdatacom.broker_capture.fingerprint_contracts->histdatacom.broker_capture.contracts + + + + + +histdatacom.broker_capture.fingerprint_contracts->histdatacom.runtime_contracts + + + + + +histdatacom.broker_capture.fingerprints->histdatacom.broker_capture.contracts + + + + + +histdatacom.broker_capture.fingerprints->histdatacom.broker_capture.fingerprint_contracts + + + + + +histdatacom.broker_capture.fingerprints->histdatacom.broker_capture.storage + + + + + +histdatacom.data_quality.calendar_profiles + +histdatacom.data_quality.calendar_profiles + + + +histdatacom.broker_capture.fingerprints->histdatacom.data_quality.calendar_profiles + + + + + +histdatacom.market_context.contracts + +histdatacom.market_context.contracts + + + +histdatacom.broker_capture.fingerprints->histdatacom.market_context.contracts + + + + + +histdatacom.broker_capture.fingerprints->histdatacom.runtime_contracts + + + + + +histdatacom.broker_capture.storage->histdatacom.broker_capture.adapters + + + + + +histdatacom.broker_capture.storage->histdatacom.broker_capture.contracts + + + + + +histdatacom.broker_capture.storage->histdatacom.runtime_contracts + + + + + histdatacom.cache_status - -histdatacom.cache_status + +histdatacom.cache_status - + histdatacom.cache_status->histdatacom.fx_enums - - + + - + histdatacom.cache_status->histdatacom.histdata_ascii - - + + - + histdatacom.source_cleanup - -histdatacom.source_cleanup + +histdatacom.source_cleanup - + histdatacom.cache_status->histdatacom.source_cleanup - - + + - + histdatacom.cancellation->histdatacom.runtime_contracts - - + + - + histdatacom.cleanup_cli - -histdatacom.cleanup_cli + +histdatacom.cleanup_cli - + histdatacom.cleanup_cli->histdatacom.cache_status - - + + - + histdatacom.cli_config - -histdatacom.cli_config + +histdatacom.cli_config - + histdatacom.cleanup_cli->histdatacom.cli_config - - + + - + histdatacom.manifest_store - -histdatacom.manifest_store + +histdatacom.manifest_store - + histdatacom.cleanup_cli->histdatacom.manifest_store - - + + - + histdatacom.orchestration.runtime - -histdatacom.orchestration.runtime + +histdatacom.orchestration.runtime - + histdatacom.cleanup_cli->histdatacom.orchestration.runtime - - + + - + histdatacom.orchestration.supervisor - -histdatacom.orchestration.supervisor + +histdatacom.orchestration.supervisor - + histdatacom.cleanup_cli->histdatacom.orchestration.supervisor - - + + - + histdatacom.cleanup_cli->histdatacom.source_cleanup - - + + - + histdatacom.cli - -histdatacom.cli + +histdatacom.cli - + histdatacom.cli->histdatacom - - + + - + histdatacom.cli->histdatacom.cli_config - - + + - + histdatacom.concurrency - -histdatacom.concurrency + +histdatacom.concurrency - + histdatacom.cli->histdatacom.concurrency - - + + - + histdatacom.data_quality - -histdatacom.data_quality + +histdatacom.data_quality - + histdatacom.cli->histdatacom.data_quality - - + + - + histdatacom.data_quality.preflight - -histdatacom.data_quality.preflight + +histdatacom.data_quality.preflight - + histdatacom.cli->histdatacom.data_quality.preflight - - + + - + histdatacom.data_quality.profiles - -histdatacom.data_quality.profiles + +histdatacom.data_quality.profiles - + histdatacom.cli->histdatacom.data_quality.profiles - - + + - + histdatacom.cli->histdatacom.fx_enums - - + + + + + +histdatacom.cli->histdatacom.random_windows + + - + histdatacom.cli->histdatacom.utils - - + + - + histdatacom.verbosity - -histdatacom.verbosity + +histdatacom.verbosity - + histdatacom.cli->histdatacom.verbosity - - + + - + histdatacom.csvs - -histdatacom.csvs + +histdatacom.csvs - + histdatacom.csvs->histdatacom.activity_stages - - + + - + histdatacom.csvs->histdatacom.helper_args - - + + - + histdatacom.csvs->histdatacom.records - - + + - + histdatacom.csvs->histdatacom.runtime_contracts - - + + - + histdatacom.data_analytics - -histdatacom.data_analytics + +histdatacom.data_analytics + + + +histdatacom.data_analytics.feed_epochs + +histdatacom.data_analytics.feed_epochs + + + +histdatacom.data_analytics->histdatacom.data_analytics.feed_epochs + + + + + +histdatacom.data_analytics.feed_epochs_v2 + +histdatacom.data_analytics.feed_epochs_v2 + + + +histdatacom.data_analytics->histdatacom.data_analytics.feed_epochs_v2 + + - + histdatacom.data_analytics.feed_regimes - -histdatacom.data_analytics.feed_regimes + +histdatacom.data_analytics.feed_regimes - + histdatacom.data_analytics->histdatacom.data_analytics.feed_regimes - - + + - + histdatacom.data_analytics.cli - -histdatacom.data_analytics.cli + +histdatacom.data_analytics.cli - + histdatacom.data_analytics.cli->histdatacom.cli_config - - + + + + + +histdatacom.data_analytics.cli->histdatacom.data_analytics.feed_epochs + + + + + +histdatacom.data_analytics.cli->histdatacom.data_analytics.feed_epochs_v2 + + - + histdatacom.data_analytics.cli->histdatacom.data_analytics.feed_regimes - - + + + + + +histdatacom.market_context + +histdatacom.market_context + + + +histdatacom.data_analytics.cli->histdatacom.market_context + + + + + +histdatacom.synthetic.benchmark_corpus + +histdatacom.synthetic.benchmark_corpus + + + +histdatacom.data_analytics.cli->histdatacom.synthetic.benchmark_corpus + + + + + +histdatacom.synthetic.motif_library + +histdatacom.synthetic.motif_library + + + +histdatacom.data_analytics.cli->histdatacom.synthetic.motif_library + + + + + +histdatacom.synthetic.observation_calibration + +histdatacom.synthetic.observation_calibration + + + +histdatacom.data_analytics.cli->histdatacom.synthetic.observation_calibration + + - + histdatacom.data_analytics.cli->histdatacom.verbosity - - + + - - -histdatacom.data_analytics.feed_regimes->histdatacom.histdata_ascii - - + + +histdatacom.data_quality.fingerprints + +histdatacom.data_quality.fingerprints - - -histdatacom.data_analytics.feed_regimes->histdatacom.runtime_contracts - - + + +histdatacom.data_analytics.feed_epochs->histdatacom.data_quality.fingerprints + + - - -histdatacom.data_quality.bounded_payload_contracts - -histdatacom.data_quality.bounded_payload_contracts + + +histdatacom.data_analytics.feed_epochs->histdatacom.runtime_contracts + + - - -histdatacom.data_quality->histdatacom.data_quality.bounded_payload_contracts - - + + +histdatacom.synthetic.contracts + +histdatacom.synthetic.contracts + + + +histdatacom.data_analytics.feed_epochs->histdatacom.synthetic.contracts + + - + histdatacom.data_quality.calendar - -histdatacom.data_quality.calendar + +histdatacom.data_quality.calendar - - -histdatacom.data_quality->histdatacom.data_quality.calendar - - - - - -histdatacom.data_quality.calendar_profiles - -histdatacom.data_quality.calendar_profiles + + +histdatacom.data_analytics.feed_epochs_v2->histdatacom.data_quality.calendar + + + + + +histdatacom.data_quality.contracts + +histdatacom.data_quality.contracts + + + +histdatacom.data_analytics.feed_epochs_v2->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.discovery + +histdatacom.data_quality.discovery + + + +histdatacom.data_analytics.feed_epochs_v2->histdatacom.data_quality.discovery + + + + + +histdatacom.data_analytics.feed_epochs_v2->histdatacom.histdata_ascii + + + + + +histdatacom.resource_usage + +histdatacom.resource_usage + + + +histdatacom.data_analytics.feed_epochs_v2->histdatacom.resource_usage + + + + + +histdatacom.data_analytics.feed_epochs_v2->histdatacom.runtime_contracts + + + + + +histdatacom.data_analytics.feed_regimes->histdatacom.data_analytics.feed_epochs + + + + + +histdatacom.data_analytics.feed_regimes->histdatacom.data_quality.discovery + + + + + +histdatacom.data_quality.engine + +histdatacom.data_quality.engine + + + +histdatacom.data_analytics.feed_regimes->histdatacom.data_quality.engine + + + + + +histdatacom.data_analytics.feed_regimes->histdatacom.data_quality.fingerprints + + + + + +histdatacom.data_analytics.feed_regimes->histdatacom.histdata_ascii + + + + + +histdatacom.data_analytics.feed_regimes->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.autoregressive + +histdatacom.data_quality.autoregressive + + + +histdatacom.data_quality->histdatacom.data_quality.autoregressive + + + + + +histdatacom.data_quality.bounded_payload_contracts + +histdatacom.data_quality.bounded_payload_contracts + + + +histdatacom.data_quality->histdatacom.data_quality.bounded_payload_contracts + + + + + +histdatacom.data_quality->histdatacom.data_quality.calendar + + - + histdatacom.data_quality->histdatacom.data_quality.calendar_profiles - - + + - + histdatacom.data_quality.campaign - -histdatacom.data_quality.campaign + +histdatacom.data_quality.campaign - + histdatacom.data_quality->histdatacom.data_quality.campaign - - + + - - -histdatacom.data_quality.contracts - -histdatacom.data_quality.contracts + + +histdatacom.data_quality.classical_baselines + +histdatacom.data_quality.classical_baselines + + + +histdatacom.data_quality->histdatacom.data_quality.classical_baselines + + + + + +histdatacom.data_quality.classical_model_comparison + +histdatacom.data_quality.classical_model_comparison + + + +histdatacom.data_quality->histdatacom.data_quality.classical_model_comparison + + + + + +histdatacom.data_quality.classical_model_contracts + +histdatacom.data_quality.classical_model_contracts + + + +histdatacom.data_quality->histdatacom.data_quality.classical_model_contracts + + - + histdatacom.data_quality->histdatacom.data_quality.contracts - - - - - -histdatacom.data_quality.discovery - -histdatacom.data_quality.discovery + + - + histdatacom.data_quality->histdatacom.data_quality.discovery - - - - - -histdatacom.data_quality.engine - -histdatacom.data_quality.engine + + - + histdatacom.data_quality->histdatacom.data_quality.engine - - + + + + + +histdatacom.data_quality.exponential_smoothing + +histdatacom.data_quality.exponential_smoothing + + + +histdatacom.data_quality->histdatacom.data_quality.exponential_smoothing + + - + histdatacom.data_quality.fingerprint_discovery - -histdatacom.data_quality.fingerprint_discovery + +histdatacom.data_quality.fingerprint_discovery - + histdatacom.data_quality->histdatacom.data_quality.fingerprint_discovery - - + + - - -histdatacom.data_quality.fingerprints - -histdatacom.data_quality.fingerprints + + +histdatacom.data_quality.fingerprint_next_work + +histdatacom.data_quality.fingerprint_next_work + + + +histdatacom.data_quality->histdatacom.data_quality.fingerprint_next_work + + - + histdatacom.data_quality->histdatacom.data_quality.fingerprints - - + + - + histdatacom.data_quality.format_support - -histdatacom.data_quality.format_support + +histdatacom.data_quality.format_support - + histdatacom.data_quality->histdatacom.data_quality.format_support - - + + - + histdatacom.data_quality.ingestion - -histdatacom.data_quality.ingestion + +histdatacom.data_quality.ingestion - + histdatacom.data_quality->histdatacom.data_quality.ingestion - - + + - + histdatacom.data_quality.inventory - -histdatacom.data_quality.inventory + +histdatacom.data_quality.inventory - + histdatacom.data_quality->histdatacom.data_quality.inventory - - + + - + histdatacom.data_quality.limits - -histdatacom.data_quality.limits + +histdatacom.data_quality.limits - + histdatacom.data_quality->histdatacom.data_quality.limits - - + + - + histdatacom.data_quality.manifest - -histdatacom.data_quality.manifest + +histdatacom.data_quality.manifest - + histdatacom.data_quality->histdatacom.data_quality.manifest - - + + - + histdatacom.data_quality.modeling - -histdatacom.data_quality.modeling + +histdatacom.data_quality.modeling - + histdatacom.data_quality->histdatacom.data_quality.modeling - - + + - + histdatacom.data_quality->histdatacom.data_quality.profiles - - + + - + histdatacom.data_quality.provenance - -histdatacom.data_quality.provenance + +histdatacom.data_quality.provenance - + histdatacom.data_quality->histdatacom.data_quality.provenance - - + + - + histdatacom.data_quality.remediation - -histdatacom.data_quality.remediation + +histdatacom.data_quality.remediation - + histdatacom.data_quality->histdatacom.data_quality.remediation - - + + - + histdatacom.data_quality.remediation_audit - -histdatacom.data_quality.remediation_audit + +histdatacom.data_quality.remediation_audit - + histdatacom.data_quality->histdatacom.data_quality.remediation_audit - - + + + + + +histdatacom.data_quality.repair_plan + +histdatacom.data_quality.repair_plan + + + +histdatacom.data_quality->histdatacom.data_quality.repair_plan + + - + histdatacom.data_quality.reporting - -histdatacom.data_quality.reporting + +histdatacom.data_quality.reporting - + histdatacom.data_quality->histdatacom.data_quality.reporting - - + + - + histdatacom.data_quality.rules - -histdatacom.data_quality.rules + +histdatacom.data_quality.rules - + histdatacom.data_quality->histdatacom.data_quality.rules - - + + + + + +histdatacom.data_quality.seasonal_exogenous + +histdatacom.data_quality.seasonal_exogenous + + + +histdatacom.data_quality->histdatacom.data_quality.seasonal_exogenous + + + + + +histdatacom.data_quality.state_space + +histdatacom.data_quality.state_space + + + +histdatacom.data_quality->histdatacom.data_quality.state_space + + - + histdatacom.data_quality.symbols - -histdatacom.data_quality.symbols + +histdatacom.data_quality.symbols - + histdatacom.data_quality->histdatacom.data_quality.symbols - - + + + + + +histdatacom.data_quality.synthetic_constraints + +histdatacom.data_quality.synthetic_constraints + + + +histdatacom.data_quality->histdatacom.data_quality.synthetic_constraints + + + + + +histdatacom.data_quality.synthetic_generation + +histdatacom.data_quality.synthetic_generation + + + +histdatacom.data_quality->histdatacom.data_quality.synthetic_generation + + - + histdatacom.data_quality.ticks - -histdatacom.data_quality.ticks + +histdatacom.data_quality.ticks - + histdatacom.data_quality->histdatacom.data_quality.ticks - - + + - + histdatacom.data_quality.time - -histdatacom.data_quality.time + +histdatacom.data_quality.time - + histdatacom.data_quality->histdatacom.data_quality.time - - + + + + + +histdatacom.data_quality->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.volatility + +histdatacom.data_quality.volatility + + + +histdatacom.data_quality->histdatacom.data_quality.volatility + + - + histdatacom.publication_safety - -histdatacom.publication_safety + +histdatacom.publication_safety - + histdatacom.data_quality->histdatacom.publication_safety - - + + + + + +histdatacom.data_quality.autoregressive->histdatacom.data_quality.classical_model_contracts + + + + + +histdatacom.data_quality.autoregressive->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.autoregressive->histdatacom.data_quality.exponential_smoothing + + + + + +histdatacom.data_quality.autoregressive->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.autoregressive->histdatacom.data_quality.time + + + + + +histdatacom.data_quality.autoregressive->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.autoregressive->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.autoregressive + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.classical_baselines + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.classical_model_comparison + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.classical_model_contracts + + - + histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.contracts - - + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.engine + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.exponential_smoothing + + - + histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.fingerprints - - + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.limits + + - + histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.profiles - - + + - + histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.reporting - - + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.seasonal_exogenous + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.state_space + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.synthetic_constraints + + + + + +histdatacom.data_quality.bounded_payload_contracts->histdatacom.data_quality.volatility + + - + histdatacom.data_quality.bounded_payload_contracts->histdatacom.publication_safety - - + + - + histdatacom.data_quality.bounded_payload_contracts->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.calendar->histdatacom.data_quality.calendar_profiles - - + + - + histdatacom.data_quality.calendar->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.calendar->histdatacom.data_quality.symbols - - + + - + histdatacom.data_quality.calendar->histdatacom.data_quality.time - - + + - + histdatacom.data_quality.calendar->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.calendar->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.calendar_profiles->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.campaign->histdatacom.activity_stages - - + + - + histdatacom.data_quality.campaign->histdatacom.data_quality.format_support - - + + - + histdatacom.data_quality.campaign->histdatacom.fx_enums - - + + - + histdatacom.data_quality.campaign->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.campaign->histdatacom.utils - - + + + + + +histdatacom.data_quality.classical_baselines->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.classical_baselines->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.classical_baselines->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.classical_baselines->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.classical_model_comparison->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.classical_model_comparison->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.classical_model_comparison->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.classical_model_comparison->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.classical_model_contracts->histdatacom.data_quality.calendar + + + + + +histdatacom.data_quality.classical_model_contracts->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.classical_model_contracts->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.classical_model_contracts->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.classical_model_contracts->histdatacom.runtime_contracts + + - + histdatacom.data_quality.contracts->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.discovery->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.discovery->histdatacom.data_quality.format_support - - + + - + histdatacom.data_quality.discovery->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.discovery->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.engine->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.engine->histdatacom.data_quality.fingerprints - - + + + + + +histdatacom.data_quality.engine->histdatacom.data_quality.limits + + - + histdatacom.data_quality.engine->histdatacom.data_quality.ticks - - + + - + histdatacom.data_quality.engine->histdatacom.runtime_contracts - - + + + + + +histdatacom.data_quality.exponential_smoothing->histdatacom.data_quality.calendar + + + + + +histdatacom.data_quality.exponential_smoothing->histdatacom.data_quality.classical_model_contracts + + + + + +histdatacom.data_quality.exponential_smoothing->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.exponential_smoothing->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.exponential_smoothing->histdatacom.data_quality.time + + + + + +histdatacom.data_quality.exponential_smoothing->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.exponential_smoothing->histdatacom.runtime_contracts + + - + histdatacom.data_quality.fingerprint_contracts - -histdatacom.data_quality.fingerprint_contracts + +histdatacom.data_quality.fingerprint_contracts + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.autoregressive + + - + histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.calendar - - + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.classical_baselines + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.classical_model_comparison + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.classical_model_contracts + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.exponential_smoothing + + - + histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.fingerprints - - + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.seasonal_exogenous + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.state_space + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.synthetic_constraints + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.synthetic_generation + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.time + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.fingerprint_contracts->histdatacom.data_quality.volatility + + - + histdatacom.data_quality.fingerprint_contracts->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.fingerprint_contracts->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom.data_quality.bounded_payload_contracts - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom.data_quality.fingerprint_contracts - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom.data_quality.fingerprints - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom.data_quality.profiles - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom.data_quality.reporting - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom.publication_safety - - + + - + histdatacom.data_quality.fingerprint_discovery->histdatacom.runtime_contracts - - + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.data_quality.fingerprint_discovery + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.data_quality.fingerprints + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.data_quality.reporting + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.histdata_ascii + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.publication_safety + + + + + +histdatacom.data_quality.fingerprint_next_work->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.autoregressive + + - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.calendar - - + + - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.calendar_profiles - - + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.classical_baselines + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.classical_model_comparison + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.classical_model_contracts + + - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.contracts - - + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.exponential_smoothing + + - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.limits - - + + - + histdatacom.data_quality.polars_cache - -histdatacom.data_quality.polars_cache + +histdatacom.data_quality.polars_cache - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.polars_cache - - + + - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.remediation - - + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.seasonal_exogenous + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.state_space + + - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.symbols - - + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.synthetic_constraints + + - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.ticks - - + + - + histdatacom.data_quality.fingerprints->histdatacom.data_quality.time - - + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.fingerprints->histdatacom.data_quality.volatility + + - + histdatacom.data_quality.fingerprints->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.fingerprints->histdatacom.publication_safety - - + + - + histdatacom.data_quality.fingerprints->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.format_support->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.ingestion->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.ingestion->histdatacom.data_quality.polars_cache - - + + - + histdatacom.data_quality.ingestion->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.ingestion->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.inventory->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.inventory->histdatacom.data_quality.discovery - - + + - + histdatacom.data_quality.inventory->histdatacom.data_quality.format_support - - + + - + histdatacom.data_quality.inventory->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.limits->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.manifest->histdatacom.activity_stages - - + + - + histdatacom.data_quality.manifest->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.manifest->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.modeling->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.modeling->histdatacom.data_quality.symbols - - + + - + histdatacom.data_quality.modeling->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.modeling->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.polars_cache->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.polars_cache->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.preflight->histdatacom - - + + - + histdatacom.data_quality.preflight->histdatacom.cache_status - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.bounded_payload_contracts - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.discovery - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.engine - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.fingerprint_discovery - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.polars_cache - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.reporting - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.rules - - + + - + histdatacom.data_quality.preflight->histdatacom.data_quality.time - - + + - + histdatacom.data_quality.preflight->histdatacom.fx_enums - - + + - + histdatacom.orchestration.workflows - -histdatacom.orchestration.workflows + +histdatacom.orchestration.workflows - + histdatacom.data_quality.preflight->histdatacom.orchestration.workflows - - + + - + histdatacom.data_quality.preflight->histdatacom.publication_safety - - + + - + histdatacom.data_quality.preflight->histdatacom.runtime_contracts - - + + + + + +histdatacom.data_quality.profiles->histdatacom.data_quality.autoregressive + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.calendar - - + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.calendar_profiles - - + + + + + +histdatacom.data_quality.profiles->histdatacom.data_quality.classical_baselines + + + + + +histdatacom.data_quality.profiles->histdatacom.data_quality.classical_model_comparison + + + + + +histdatacom.data_quality.profiles->histdatacom.data_quality.classical_model_contracts + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.contracts - - + + + + + +histdatacom.data_quality.profiles->histdatacom.data_quality.exponential_smoothing + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.fingerprints - - + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.ingestion - - + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.modeling - - + + + + + +histdatacom.data_quality.profiles->histdatacom.data_quality.seasonal_exogenous + + + + + +histdatacom.data_quality.profiles->histdatacom.data_quality.state_space + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.symbols - - + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.ticks - - + + - + histdatacom.data_quality.profiles->histdatacom.data_quality.time - - + + + + + +histdatacom.data_quality.profiles->histdatacom.data_quality.volatility + + - + histdatacom.data_quality.profiles->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.provenance->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.provenance->histdatacom.manifest_store - - + + - + histdatacom.data_quality.provenance->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.remediation->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.remediation->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.remediation_audit->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.remediation_audit->histdatacom.data_quality.limits - - + + - + histdatacom.data_quality.remediation_audit->histdatacom.data_quality.remediation - - + + - + histdatacom.data_quality.remediation_audit->histdatacom.data_quality.reporting - - + + - + histdatacom.data_quality.remediation_audit->histdatacom.publication_safety - - + + - + histdatacom.data_quality.remediation_audit->histdatacom.runtime_contracts - - + + + + + +histdatacom.data_quality.repair_plan->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.repair_plan->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.repair_plan->histdatacom.data_quality.remediation + + + + + +histdatacom.data_quality.repair_plan->histdatacom.publication_safety + + + + + +histdatacom.data_quality.repair_plan->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.autoregressive + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.classical_baselines + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.classical_model_comparison + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.classical_model_contracts + + - + histdatacom.data_quality.reporting->histdatacom.data_quality.contracts - - + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.engine + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.exponential_smoothing + + - + histdatacom.data_quality.reporting->histdatacom.data_quality.fingerprint_contracts - - + + - + histdatacom.data_quality.reporting->histdatacom.data_quality.fingerprint_discovery - - + + - + histdatacom.data_quality.reporting->histdatacom.data_quality.fingerprints - - + + - + histdatacom.data_quality.reporting->histdatacom.data_quality.limits - - + + - + histdatacom.data_quality.reporting->histdatacom.data_quality.profiles - - + + - + histdatacom.data_quality.reporting->histdatacom.data_quality.remediation - - + + - + histdatacom.data_quality.reporting->histdatacom.data_quality.remediation_audit - - + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.seasonal_exogenous + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.state_space + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.synthetic_constraints + + + + + +histdatacom.data_quality.reporting->histdatacom.data_quality.volatility + + - + histdatacom.data_quality.reporting->histdatacom.publication_safety - - + + - + histdatacom.data_quality.reporting->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.calendar - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.discovery - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.fingerprints - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.ingestion - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.inventory - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.manifest - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.modeling - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.profiles - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.provenance - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.symbols - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.ticks - - + + - + histdatacom.data_quality.rules->histdatacom.data_quality.time - - + + - + histdatacom.data_quality.rules->histdatacom.runtime_contracts - - + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.autoregressive + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.calendar + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.calendar_profiles + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.classical_model_contracts + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.exponential_smoothing + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.time + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.seasonal_exogenous->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.state_space->histdatacom.data_quality.autoregressive + + + + + +histdatacom.data_quality.state_space->histdatacom.data_quality.classical_model_contracts + + + + + +histdatacom.data_quality.state_space->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.state_space->histdatacom.data_quality.exponential_smoothing + + + + + +histdatacom.data_quality.state_space->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.state_space->histdatacom.data_quality.seasonal_exogenous + + + + + +histdatacom.data_quality.state_space->histdatacom.data_quality.time + + + + + +histdatacom.data_quality.state_space->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.state_space->histdatacom.runtime_contracts + + - + histdatacom.data_quality.symbols->histdatacom.data_quality.contracts - - + + + + + +histdatacom.data_quality.symbols->histdatacom.data_quality.limits + + - + histdatacom.data_quality.symbols->histdatacom.data_quality.polars_cache - - + + + + + +histdatacom.data_quality.symbols->histdatacom.data_quality.training_features + + - + histdatacom.data_quality.symbols->histdatacom.fx_enums - - + + - + histdatacom.data_quality.symbols->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.symbols->histdatacom.runtime_contracts - - + + + + + +histdatacom.data_quality.synthetic_constraints->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.synthetic_constraints->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.synthetic_constraints->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.synthetic_constraints->histdatacom.histdata_ascii + + + + + +histdatacom.data_quality.synthetic_constraints->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.synthetic_generation->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.synthetic_generation->histdatacom.data_quality.fingerprints + + + + + +histdatacom.data_quality.synthetic_generation->histdatacom.data_quality.synthetic_constraints + + + + + +histdatacom.data_quality.synthetic_generation->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.synthetic_generation->histdatacom.histdata_ascii + + + + + +histdatacom.data_quality.synthetic_generation->histdatacom.runtime_contracts + + - + histdatacom.data_quality.ticks->histdatacom.data_quality.calendar - - + + - + histdatacom.data_quality.ticks->histdatacom.data_quality.contracts - - + + - + histdatacom.data_quality.ticks->histdatacom.data_quality.polars_cache - - + + - + histdatacom.data_quality.ticks->histdatacom.data_quality.symbols - - + + - + histdatacom.data_quality.ticks->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.ticks->histdatacom.runtime_contracts - - + + - + histdatacom.data_quality.time->histdatacom.data_quality.contracts - - + + + + + +histdatacom.data_quality.time->histdatacom.data_quality.limits + + - + histdatacom.data_quality.time->histdatacom.data_quality.polars_cache - - + + - + histdatacom.data_quality.time->histdatacom.histdata_ascii - - + + - + histdatacom.data_quality.time->histdatacom.runtime_contracts - - + + + + + +histdatacom.data_quality.training_features->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.training_features->histdatacom.histdata_ascii + + + + + +histdatacom.data_quality.training_features->histdatacom.runtime_contracts + + + + + +histdatacom.data_quality.volatility->histdatacom.data_quality.autoregressive + + + + + +histdatacom.data_quality.volatility->histdatacom.data_quality.classical_model_contracts + + + + + +histdatacom.data_quality.volatility->histdatacom.data_quality.contracts + + + + + +histdatacom.data_quality.volatility->histdatacom.data_quality.limits + + + + + +histdatacom.data_quality.volatility->histdatacom.data_quality.seasonal_exogenous + + + + + +histdatacom.data_quality.volatility->histdatacom.data_quality.time + + + + + +histdatacom.data_quality.volatility->histdatacom.data_quality.training_features + + + + + +histdatacom.data_quality.volatility->histdatacom.runtime_contracts + + - + histdatacom.exceptions->histdatacom.runtime_contracts - - + + - + histdatacom.group_catalog - -histdatacom.group_catalog + +histdatacom.group_catalog - + histdatacom.group_catalog->histdatacom.fx_enums - - + + - + histdatacom.groups_cli - -histdatacom.groups_cli + +histdatacom.groups_cli - + histdatacom.groups_cli->histdatacom.cli_config - - + + - + histdatacom.groups_cli->histdatacom.fx_enums - - + + - + histdatacom.groups_cli->histdatacom.group_catalog - - + + - + histdatacom.histdata_com - -histdatacom.histdata_com + +histdatacom.histdata_com - + histdatacom.histdata_com->histdatacom - - + + - + histdatacom.histdata_com->histdatacom.api - - + + - + histdatacom.histdata_com->histdatacom.cleanup_cli - - + + - + histdatacom.histdata_com->histdatacom.cli - - + + - + histdatacom.histdata_com->histdatacom.cli_config - - + + - + histdatacom.histdata_com->histdatacom.data_analytics.cli - - + + - + histdatacom.histdata_com->histdatacom.data_quality.preflight - - + + - + histdatacom.histdata_com->histdatacom.data_quality.profiles - - + + - + histdatacom.histdata_com->histdatacom.data_quality.reporting - - + + - + histdatacom.histdata_com->histdatacom.exceptions - - + + - + histdatacom.histdata_com->histdatacom.fx_enums - - + + - + histdatacom.histdata_com->histdatacom.groups_cli - - + + - + histdatacom.histdata_com->histdatacom.histdata_ascii - - + + - + histdatacom.operational_health - -histdatacom.operational_health + +histdatacom.operational_health - + histdatacom.histdata_com->histdatacom.operational_health - - + + - + histdatacom.orchestration.cli - -histdatacom.orchestration.cli + +histdatacom.orchestration.cli - + histdatacom.histdata_com->histdatacom.orchestration.cli - - + + - + histdatacom.orchestration.client - -histdatacom.orchestration.client + +histdatacom.orchestration.client - + histdatacom.histdata_com->histdatacom.orchestration.client - - + + - + histdatacom.orchestration.cutover - -histdatacom.orchestration.cutover + +histdatacom.orchestration.cutover - + histdatacom.histdata_com->histdatacom.orchestration.cutover - - + + - + histdatacom.orchestration.rich_progress - -histdatacom.orchestration.rich_progress + +histdatacom.orchestration.rich_progress - + histdatacom.histdata_com->histdatacom.orchestration.rich_progress - - + + - + histdatacom.histdata_com->histdatacom.publication_safety - - + + - + histdatacom.quality_cli - -histdatacom.quality_cli + +histdatacom.quality_cli - + histdatacom.histdata_com->histdatacom.quality_cli - - + + + + + +histdatacom.histdata_com->histdatacom.random_windows + + + + + +histdatacom.reconstruction_cli + +histdatacom.reconstruction_cli + + + +histdatacom.histdata_com->histdatacom.reconstruction_cli + + - + histdatacom.histdata_com->histdatacom.records - - + + - + histdatacom.repository_output - -histdatacom.repository_output + +histdatacom.repository_output - + histdatacom.histdata_com->histdatacom.repository_output - - + + - + histdatacom.histdata_com->histdatacom.runtime_contracts - - + + - + histdatacom.scheduled_run_bundle - -histdatacom.scheduled_run_bundle + +histdatacom.scheduled_run_bundle - + histdatacom.histdata_com->histdatacom.scheduled_run_bundle - - + + - + histdatacom.histdata_com->histdatacom.utils - - + + - + histdatacom.histdata_com->histdatacom.verbosity - - + + - + histdatacom.influx - -histdatacom.influx + +histdatacom.influx - + histdatacom.influx->histdatacom.activity_stages - - + + - + histdatacom.influx->histdatacom.exceptions - - + + - + histdatacom.influx->histdatacom.helper_args - - + + - + histdatacom.influx->histdatacom.histdata_ascii - - + + - + histdatacom.influx->histdatacom.legacy_boundary - - + + - + histdatacom.influx->histdatacom.observability - - + + - + histdatacom.influx->histdatacom.records - - + + - + histdatacom.influx->histdatacom.runtime_contracts - - + + - + histdatacom.influx->histdatacom.utils - - + + - + histdatacom.manifest_store->histdatacom.runtime_contracts - - + + + + + +histdatacom.market_context->histdatacom.market_context.contracts + + + + + +histdatacom.market_context.corpus + +histdatacom.market_context.corpus + + + +histdatacom.market_context->histdatacom.market_context.corpus + + + + + +histdatacom.market_context.positioning + +histdatacom.market_context.positioning + + + +histdatacom.market_context->histdatacom.market_context.positioning + + + + + +histdatacom.market_context.contracts->histdatacom.data_quality.calendar + + + + + +histdatacom.market_context.contracts->histdatacom.data_quality.calendar_profiles + + + + + +histdatacom.market_context.contracts->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.information + +histdatacom.synthetic.information + + + +histdatacom.market_context.contracts->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.streaming + +histdatacom.synthetic.streaming + + + +histdatacom.market_context.contracts->histdatacom.synthetic.streaming + + + + + +histdatacom.market_context.corpus->histdatacom.market_context.contracts + + + + + +histdatacom.market_context.corpus->histdatacom.resource_usage + + + + + +histdatacom.market_context.corpus->histdatacom.runtime_contracts + + + + + +histdatacom.market_context.positioning->histdatacom.market_context.contracts + + + + + +histdatacom.market_context.positioning->histdatacom.resource_usage + + + + + +histdatacom.market_context.positioning->histdatacom.runtime_contracts + + + + + +histdatacom.market_context.positioning->histdatacom.synthetic.information + + - + histdatacom.observability->histdatacom.runtime_contracts - - + + - + histdatacom.operational_health->histdatacom.cache_status - - + + - + histdatacom.orchestration.control - -histdatacom.orchestration.control + +histdatacom.orchestration.control - + histdatacom.operational_health->histdatacom.orchestration.control - - + + - + histdatacom.operational_health->histdatacom.runtime_contracts - - + + - + histdatacom.options->histdatacom.fx_enums - - + + - + histdatacom.orchestration - -histdatacom.orchestration + +histdatacom.orchestration - + histdatacom.orchestration.activities - -histdatacom.orchestration.activities + +histdatacom.orchestration.activities - + histdatacom.orchestration.activities->histdatacom.activity_stages - - + + - + histdatacom.orchestration.activities->histdatacom.cancellation - - + + - + histdatacom.orchestration.activities->histdatacom.data_quality - - + + - + histdatacom.orchestration.activities->histdatacom.exceptions - - + + - + histdatacom.orchestration.activities->histdatacom.influx - - + + - + histdatacom.orchestration.activities->histdatacom.manifest_store - - + + - + histdatacom.orchestration.activities->histdatacom.observability - - + + + + + +histdatacom.orchestration.reconstruction + +histdatacom.orchestration.reconstruction + + + +histdatacom.orchestration.activities->histdatacom.orchestration.reconstruction + + - + histdatacom.orchestration.workflow_metadata - -histdatacom.orchestration.workflow_metadata + +histdatacom.orchestration.workflow_metadata - + histdatacom.orchestration.activities->histdatacom.orchestration.workflow_metadata - - + + - + histdatacom.orchestration.activities->histdatacom.orchestration.workflows - - + + + + + +histdatacom.orchestration.activities->histdatacom.random_windows + + - + histdatacom.orchestration.activities->histdatacom.repository_quality - - + + - + histdatacom.orchestration.activities->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.activities->histdatacom.source_cleanup - - + + - + histdatacom.orchestration.activities->histdatacom.utils - - + + - + histdatacom.orchestration.activities->histdatacom.verbosity - - + + - + histdatacom.orchestration.cli->histdatacom.cli_config - - + + - + histdatacom.orchestration.cli->histdatacom.operational_health - - + + - + histdatacom.orchestration.cli->histdatacom.orchestration.client - - + + - + histdatacom.orchestration.cli->histdatacom.orchestration.control - - + + - + histdatacom.orchestration.maintenance - -histdatacom.orchestration.maintenance + +histdatacom.orchestration.maintenance - + histdatacom.orchestration.cli->histdatacom.orchestration.maintenance - - + + - + histdatacom.orchestration.performance - -histdatacom.orchestration.performance + +histdatacom.orchestration.performance - + histdatacom.orchestration.cli->histdatacom.orchestration.performance - - + + - + histdatacom.orchestration.queues - -histdatacom.orchestration.queues + +histdatacom.orchestration.queues - + histdatacom.orchestration.cli->histdatacom.orchestration.queues - - + + - + histdatacom.orchestration.resources - -histdatacom.orchestration.resources + +histdatacom.orchestration.resources - + histdatacom.orchestration.cli->histdatacom.orchestration.resources - - + + - + histdatacom.orchestration.cli->histdatacom.orchestration.rich_progress - - + + - + histdatacom.orchestration.cli->histdatacom.orchestration.runtime - - + + - + histdatacom.orchestration.cli->histdatacom.orchestration.supervisor - - + + - + histdatacom.orchestration.cli->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.cli->histdatacom.scheduled_run_bundle - - + + - + histdatacom.orchestration.client->histdatacom.cancellation - - + + - + histdatacom.orchestration.client->histdatacom.exceptions - - + + - + histdatacom.orchestration.client->histdatacom.manifest_store - - + + - + histdatacom.orchestration.client->histdatacom.orchestration.control - - + + - + histdatacom.orchestration.client->histdatacom.orchestration.queues - - + + + + + +histdatacom.orchestration.client->histdatacom.orchestration.reconstruction + + - + histdatacom.orchestration.client->histdatacom.orchestration.resources - - + + - + histdatacom.orchestration.client->histdatacom.orchestration.supervisor - - + + - + histdatacom.orchestration.client->histdatacom.orchestration.workflow_metadata - - + + - + histdatacom.orchestration.client->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.client->histdatacom.verbosity - - + + - + histdatacom.orchestration.contracts - -histdatacom.orchestration.contracts + +histdatacom.orchestration.contracts - + histdatacom.orchestration.contracts->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.control->histdatacom.cancellation - - + + - + histdatacom.orchestration.control->histdatacom.publication_safety - - + + - + histdatacom.orchestration.control->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.live_smoke - -histdatacom.orchestration.live_smoke + +histdatacom.orchestration.live_smoke - + histdatacom.orchestration.live_smoke->histdatacom.orchestration.client - - + + - + histdatacom.orchestration.live_smoke->histdatacom.orchestration.control - - + + - + histdatacom.orchestration.live_smoke->histdatacom.orchestration.queues - - + + - + histdatacom.orchestration.live_smoke->histdatacom.orchestration.runtime - - + + - + histdatacom.orchestration.live_smoke->histdatacom.orchestration.supervisor - - + + - + histdatacom.orchestration.live_smoke->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.maintenance->histdatacom.manifest_store - - + + - + histdatacom.orchestration.maintenance->histdatacom.orchestration.runtime - - + + - + histdatacom.orchestration.performance->histdatacom.concurrency - - + + - + histdatacom.orchestration.performance->histdatacom.orchestration.workflows - - + + + + + +histdatacom.orchestration.performance->histdatacom.resource_usage + + - + histdatacom.orchestration.performance->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.queues->histdatacom.orchestration.performance - - + + - + histdatacom.orchestration.queues->histdatacom.orchestration.runtime - - + + - + histdatacom.orchestration.readiness - -histdatacom.orchestration.readiness + +histdatacom.orchestration.readiness - + histdatacom.orchestration.readiness->histdatacom.orchestration.queues - - + + + + + +histdatacom.orchestration.reconstruction->histdatacom.manifest_store + + + + + +histdatacom.orchestration.reconstruction->histdatacom.runtime_contracts + + + + + +histdatacom.orchestration.reconstruction->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.persistence + +histdatacom.synthetic.persistence + + + +histdatacom.orchestration.reconstruction->histdatacom.synthetic.persistence + + + + + +histdatacom.orchestration.reconstruction->histdatacom.synthetic.streaming + + - + histdatacom.orchestration.rich_progress->histdatacom.orchestration.control - - + + - + histdatacom.orchestration.rich_progress->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.supervisor->histdatacom.manifest_store - - + + - + histdatacom.orchestration.supervisor->histdatacom.orchestration.performance - - + + - + histdatacom.orchestration.supervisor->histdatacom.orchestration.queues - - + + - + histdatacom.orchestration.supervisor->histdatacom.orchestration.readiness - - + + - + histdatacom.orchestration.supervisor->histdatacom.orchestration.resources - - + + - + histdatacom.orchestration.supervisor->histdatacom.orchestration.runtime - - + + - + histdatacom.orchestration.telemetry - -histdatacom.orchestration.telemetry + +histdatacom.orchestration.telemetry - + histdatacom.orchestration.telemetry->histdatacom.orchestration.client - - + + - + histdatacom.orchestration.telemetry->histdatacom.orchestration.control - - + + - + histdatacom.orchestration.throughput - -histdatacom.orchestration.throughput + +histdatacom.orchestration.throughput - + histdatacom.orchestration.throughput->histdatacom.orchestration.client - - + + - + histdatacom.orchestration.throughput->histdatacom.orchestration.control - - + + - + histdatacom.orchestration.throughput->histdatacom.orchestration.live_smoke - - + + - + histdatacom.orchestration.throughput->histdatacom.orchestration.performance - - + + - + histdatacom.orchestration.throughput->histdatacom.orchestration.queues - - + + - + histdatacom.orchestration.throughput->histdatacom.orchestration.runtime - - + + - + histdatacom.orchestration.throughput->histdatacom.orchestration.supervisor - - + + - + histdatacom.orchestration.throughput->histdatacom.orchestration.workflows - - + + - + histdatacom.orchestration.throughput->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.worker - -histdatacom.orchestration.worker + +histdatacom.orchestration.worker - + histdatacom.orchestration.worker->histdatacom.cli_config - - + + - + histdatacom.orchestration.worker->histdatacom.orchestration.activities - - + + - + histdatacom.orchestration.worker->histdatacom.orchestration.client - - + + - + histdatacom.orchestration.worker->histdatacom.orchestration.queues - - + + - + histdatacom.orchestration.worker->histdatacom.orchestration.readiness - - + + - + histdatacom.orchestration.worker->histdatacom.orchestration.runtime - - + + - + histdatacom.orchestration.worker->histdatacom.orchestration.workflows - - + + + + + +histdatacom.synthetic.reconstruction_handlers + +histdatacom.synthetic.reconstruction_handlers + + + +histdatacom.orchestration.worker->histdatacom.synthetic.reconstruction_handlers + + - + histdatacom.orchestration.worker->histdatacom.verbosity - - + + - + histdatacom.orchestration.workflows->histdatacom.exceptions - - + + - + histdatacom.orchestration.workflows->histdatacom.manifest_store - - + + - + histdatacom.orchestration.workflows->histdatacom.observability - - + + - + histdatacom.orchestration.workflows->histdatacom.orchestration.client - - + + - + histdatacom.orchestration.workflows->histdatacom.orchestration.queues - - + + + + + +histdatacom.orchestration.workflows->histdatacom.orchestration.reconstruction + + - + histdatacom.orchestration.workflows->histdatacom.orchestration.workflow_metadata - - + + - + histdatacom.orchestration.workflows->histdatacom.runtime_contracts - - + + - + histdatacom.orchestration.workflows->histdatacom.verbosity - - + + - + histdatacom.publication_safety->histdatacom.runtime_contracts - - + + - + histdatacom.quality_cli->histdatacom.cli_config - - + + - + histdatacom.quality_cli->histdatacom.data_quality - - + + - + histdatacom.quality_cli->histdatacom.data_quality.bounded_payload_contracts - - + + + + + +histdatacom.quality_cli->histdatacom.data_quality.contracts + + + + + +histdatacom.quality_cli->histdatacom.data_quality.discovery + + - + histdatacom.quality_cli->histdatacom.data_quality.fingerprint_discovery - - + + + + + +histdatacom.quality_cli->histdatacom.data_quality.fingerprint_next_work + + - + histdatacom.quality_cli->histdatacom.data_quality.preflight - - + + - + histdatacom.quality_cli->histdatacom.data_quality.profiles - - + + - + histdatacom.quality_cli->histdatacom.data_quality.remediation_audit - - + + + + + +histdatacom.quality_cli->histdatacom.data_quality.repair_plan + + - + histdatacom.quality_cli->histdatacom.data_quality.reporting - - + + + + + +histdatacom.quality_cli->histdatacom.data_quality.synthetic_constraints + + + + + +histdatacom.quality_cli->histdatacom.data_quality.synthetic_generation + + - + histdatacom.quality_cli->histdatacom.fx_enums - - + + + + + +histdatacom.quality_cli->histdatacom.histdata_ascii + + - + histdatacom.quality_cli->histdatacom.publication_safety - - + + - + histdatacom.quality_cli->histdatacom.runtime_contracts - - + + - + histdatacom.quality_cli->histdatacom.verbosity - - + + - + histdatacom.readme_help - -histdatacom.readme_help + +histdatacom.readme_help - + histdatacom.readme_help->histdatacom - - + + - + histdatacom.readme_help->histdatacom.cli - - + + + + + +histdatacom.reconstruction + +histdatacom.reconstruction + + + +histdatacom.reconstruction->histdatacom.manifest_store + + + + + +histdatacom.reconstruction->histdatacom.orchestration.client + + + + + +histdatacom.reconstruction->histdatacom.orchestration.queues + + + + + +histdatacom.reconstruction->histdatacom.orchestration.reconstruction + + + + + +histdatacom.reconstruction->histdatacom.orchestration.supervisor + + + + + +histdatacom.reconstruction->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.certification + +histdatacom.synthetic.certification + + + +histdatacom.reconstruction->histdatacom.synthetic.certification + + + + + +histdatacom.synthetic.certification_campaign + +histdatacom.synthetic.certification_campaign + + + +histdatacom.reconstruction->histdatacom.synthetic.certification_campaign + + + + + +histdatacom.reconstruction->histdatacom.synthetic.contracts + + + + + +histdatacom.reconstruction->histdatacom.synthetic.information + + + + + +histdatacom.reconstruction->histdatacom.synthetic.persistence + + + + + +histdatacom.reconstruction->histdatacom.synthetic.reconstruction_handlers + + + + + +histdatacom.synthetic.reconstruction_plan + +histdatacom.synthetic.reconstruction_plan + + + +histdatacom.reconstruction->histdatacom.synthetic.reconstruction_plan + + + + + +histdatacom.reconstruction_cli->histdatacom.cli_config + + + + + +histdatacom.reconstruction_cli->histdatacom.orchestration.supervisor + + + + + +histdatacom.reconstruction_cli->histdatacom.reconstruction + + + + + +histdatacom.reconstruction_cli->histdatacom.synthetic.certification + + + + + +histdatacom.reconstruction_cli->histdatacom.synthetic.information + + - + histdatacom.records->histdatacom.fx_enums - - + + - + histdatacom.records->histdatacom.manifest_store - - + + - + histdatacom.records->histdatacom.runtime_contracts - - + + - + histdatacom.records->histdatacom.utils - - + + - + histdatacom.repository_output->histdatacom.repository_quality - - + + - + histdatacom.repository_output->histdatacom.utils - - + + - + histdatacom.repository_quality->histdatacom.observability - - + + - + histdatacom.repository_quality->histdatacom.publication_safety - - + + - + histdatacom.repository_quality->histdatacom.runtime_contracts - - + + - + histdatacom.runtime_contracts->histdatacom.fx_enums - - + + - + histdatacom.scheduled_run_bundle->histdatacom.runtime_contracts - - + + - + histdatacom.scraper - -histdatacom.scraper + +histdatacom.scraper - + histdatacom.scraper.repo - -histdatacom.scraper.repo + +histdatacom.scraper.repo - + histdatacom.scraper.repo->histdatacom.activity_stages - - + + - + histdatacom.scraper.repo->histdatacom.helper_args - - + + - + histdatacom.scraper.repo->histdatacom.legacy_boundary - - + + - + histdatacom.scraper.repo->histdatacom.records - - + + - + histdatacom.scraper.repo->histdatacom.repository_quality - - + + - + histdatacom.scraper.repo->histdatacom.utils - - + + - + histdatacom.scraper.scraper->histdatacom - - + + - + histdatacom.scraper.scraper->histdatacom.activity_stages - - + + - + histdatacom.scraper.scraper->histdatacom.config - - + + + + + +histdatacom.scraper.scraper->histdatacom.exceptions + + - + histdatacom.scraper.scraper->histdatacom.helper_args - - + + - + histdatacom.scraper.scraper->histdatacom.legacy_boundary - - + + - + histdatacom.scraper.scraper->histdatacom.observability - - + + - + histdatacom.scraper.scraper->histdatacom.records - - + + - + histdatacom.scraper.scraper->histdatacom.runtime_contracts - - + + - + histdatacom.scraper.scraper->histdatacom.scraper.repo - - + + - + histdatacom.scraper.urls - -histdatacom.scraper.urls + +histdatacom.scraper.urls - + histdatacom.scraper.scraper->histdatacom.scraper.urls - - + + - + histdatacom.scraper.urls->histdatacom.activity_stages - - + + - + histdatacom.source_cleanup->histdatacom.publication_safety - - + + + + + +histdatacom.synthetic + +histdatacom.synthetic + + + +histdatacom.synthetic.activity + +histdatacom.synthetic.activity + + + +histdatacom.synthetic->histdatacom.synthetic.activity + + + + + +histdatacom.synthetic.bars + +histdatacom.synthetic.bars + + + +histdatacom.synthetic->histdatacom.synthetic.bars + + + + + +histdatacom.synthetic.benchmark + +histdatacom.synthetic.benchmark + + + +histdatacom.synthetic->histdatacom.synthetic.benchmark + + + + + +histdatacom.synthetic->histdatacom.synthetic.benchmark_corpus + + + + + +histdatacom.synthetic.benchmark_gates + +histdatacom.synthetic.benchmark_gates + + + +histdatacom.synthetic->histdatacom.synthetic.benchmark_gates + + + + + +histdatacom.synthetic.broker_transfer + +histdatacom.synthetic.broker_transfer + + + +histdatacom.synthetic->histdatacom.synthetic.broker_transfer + + + + + +histdatacom.synthetic.carving + +histdatacom.synthetic.carving + + + +histdatacom.synthetic->histdatacom.synthetic.carving + + + + + +histdatacom.synthetic->histdatacom.synthetic.certification + + + + + +histdatacom.synthetic->histdatacom.synthetic.certification_campaign + + + + + +histdatacom.synthetic->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.cross_currency + +histdatacom.synthetic.cross_currency + + + +histdatacom.synthetic->histdatacom.synthetic.cross_currency + + + + + +histdatacom.synthetic.delivery + +histdatacom.synthetic.delivery + + + +histdatacom.synthetic->histdatacom.synthetic.delivery + + + + + +histdatacom.synthetic.ensembles + +histdatacom.synthetic.ensembles + + + +histdatacom.synthetic->histdatacom.synthetic.ensembles + + + + + +histdatacom.synthetic.generation + +histdatacom.synthetic.generation + + + +histdatacom.synthetic->histdatacom.synthetic.generation + + + + + +histdatacom.synthetic->histdatacom.synthetic.information + + + + + +histdatacom.synthetic->histdatacom.synthetic.motif_library + + + + + +histdatacom.synthetic.motifs + +histdatacom.synthetic.motifs + + + +histdatacom.synthetic->histdatacom.synthetic.motifs + + + + + +histdatacom.synthetic.observation + +histdatacom.synthetic.observation + + + +histdatacom.synthetic->histdatacom.synthetic.observation + + + + + +histdatacom.synthetic->histdatacom.synthetic.observation_calibration + + + + + +histdatacom.synthetic->histdatacom.synthetic.persistence + + + + + +histdatacom.synthetic.strategy_sensitivity + +histdatacom.synthetic.strategy_sensitivity + + + +histdatacom.synthetic->histdatacom.synthetic.strategy_sensitivity + + + + + +histdatacom.synthetic->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.activity->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.activity->histdatacom.synthetic.benchmark + + + + + +histdatacom.synthetic.activity->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.activity->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.activity->histdatacom.synthetic.persistence + + + + + +histdatacom.synthetic.bars->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.bars->histdatacom.synthetic.activity + + + + + +histdatacom.synthetic.bars->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.bars->histdatacom.synthetic.persistence + + + + + +histdatacom.synthetic.benchmark->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.benchmark->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.benchmark->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.benchmark->histdatacom.synthetic.observation + + + + + +histdatacom.synthetic.benchmark->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.data_analytics.feed_epochs_v2 + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.market_context + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.resource_usage + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.benchmark + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.benchmark_gates + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.generation + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.motifs + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.observation + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.observation_calibration + + + + + +histdatacom.synthetic.benchmark_corpus->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.benchmark_gates->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.benchmark_gates->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.broker_capture.fingerprint_contracts + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.data_quality.contracts + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.synthetic.benchmark + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.synthetic.carving + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.synthetic.cross_currency + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.synthetic.motifs + + + + + +histdatacom.synthetic.broker_transfer->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.carving->histdatacom.data_quality.synthetic_constraints + + + + + +histdatacom.synthetic.carving->histdatacom.market_context.contracts + + + + + +histdatacom.synthetic.carving->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.carving->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.carving->histdatacom.synthetic.generation + + + + + +histdatacom.synthetic.carving->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.certification->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.certification->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.certification_campaign->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.certification_campaign->histdatacom.synthetic.certification + + + + + +histdatacom.synthetic.certification_campaign->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.contracts->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.cross_currency->histdatacom.data_quality.contracts + + + + + +histdatacom.synthetic.cross_currency->histdatacom.data_quality.symbols + + + + + +histdatacom.synthetic.cross_currency->histdatacom.histdata_ascii + + + + + +histdatacom.synthetic.cross_currency->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.cross_currency->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.cross_currency->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.delivery->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.delivery->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.delivery->histdatacom.synthetic.cross_currency + + + + + +histdatacom.synthetic.ensembles->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.ensembles->histdatacom.synthetic.benchmark + + + + + +histdatacom.synthetic.ensembles->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.ensembles->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.generation->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.generation->histdatacom.synthetic.benchmark + + + + + +histdatacom.synthetic.generation->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.generation->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.generation->histdatacom.synthetic.motifs + + + + + +histdatacom.synthetic.generation->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.information->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.information->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.information->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.motif_library->histdatacom.data_analytics.feed_epochs_v2 + + + + + +histdatacom.synthetic.motif_library->histdatacom.market_context + + + + + +histdatacom.synthetic.motif_library->histdatacom.resource_usage + + + + + +histdatacom.synthetic.motif_library->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.motif_library->histdatacom.synthetic.benchmark_corpus + + + + + +histdatacom.synthetic.motif_library->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.motif_library->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.motif_library->histdatacom.synthetic.motifs + + + + + +histdatacom.synthetic.motifs->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.motifs->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.observation->histdatacom.data_analytics.feed_epochs + + + + + +histdatacom.synthetic.observation->histdatacom.data_analytics.feed_epochs_v2 + + + + + +histdatacom.synthetic.observation->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.observation->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.observation->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.observation->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.observation_calibration->histdatacom.data_analytics.feed_epochs_v2 + + + + + +histdatacom.synthetic.observation_calibration->histdatacom.resource_usage + + + + + +histdatacom.synthetic.observation_calibration->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.observation_calibration->histdatacom.synthetic.benchmark + + + + + +histdatacom.synthetic.observation_calibration->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.observation_calibration->histdatacom.synthetic.observation + + + + + +histdatacom.synthetic.observation_calibration->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.persistence->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.persistence->histdatacom.synthetic.broker_transfer + + + + + +histdatacom.synthetic.persistence->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.persistence->histdatacom.synthetic.cross_currency + + + + + +histdatacom.synthetic.persistence->histdatacom.synthetic.delivery + + + + + +histdatacom.synthetic.persistence->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.data_analytics.feed_epochs_v2 + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.market_context + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.orchestration.reconstruction + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.resource_usage + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.benchmark_corpus + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.carving + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.cross_currency + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.delivery + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.generation + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.motif_library + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.motifs + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.observation + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.persistence + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.reconstruction_plan + + + + + +histdatacom.synthetic.reconstruction_handlers->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.data_analytics.feed_epochs_v2 + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.market_context + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.orchestration.reconstruction + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.benchmark_corpus + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.benchmark_gates + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.carving + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.cross_currency + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.delivery + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.ensembles + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.generation + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.motif_library + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.observation + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.persistence + + + + + +histdatacom.synthetic.reconstruction_plan->histdatacom.synthetic.streaming + + + + + +histdatacom.synthetic.strategy_sensitivity->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.strategy_sensitivity->histdatacom.synthetic.bars + + + + + +histdatacom.synthetic.strategy_sensitivity->histdatacom.synthetic.benchmark + + + + + +histdatacom.synthetic.strategy_sensitivity->histdatacom.synthetic.contracts + + + + + +histdatacom.synthetic.strategy_sensitivity->histdatacom.synthetic.information + + + + + +histdatacom.synthetic.streaming->histdatacom.runtime_contracts + + + + + +histdatacom.synthetic.streaming->histdatacom.synthetic.contracts + + diff --git a/tests/fixtures/data_quality_reports/autoregressive_summary.json b/tests/fixtures/data_quality_reports/autoregressive_summary.json new file mode 100644 index 00000000..99566b71 --- /dev/null +++ b/tests/fixtures/data_quality_reports/autoregressive_summary.json @@ -0,0 +1,39 @@ +{ + "advisory": true, + "included_target_count": 1, + "limit_metadata": { + "targets": { + "default_limit": 16, + "effective_limit": 1, + "limit": 1, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 1, + "unbounded": false + } + }, + "omitted_target_count": 2, + "schema_version": "histdatacom.autoregressive-summary.v1", + "status_counts": { + "ready": 3 + }, + "target_count": 3, + "target_summaries": [ + { + "evaluated_fold_count": 6, + "failed_fit_count": 0, + "fit_attempt_count": 3, + "model_count": 3, + "reason": null, + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "AUDUSD", + "timeframe": "T" + } + } + ], + "truncated": true +} diff --git a/tests/fixtures/data_quality_reports/cache_target_report.json b/tests/fixtures/data_quality_reports/cache_target_report.json index 22bc7e69..22a4a7ad 100644 --- a/tests/fixtures/data_quality_reports/cache_target_report.json +++ b/tests/fixtures/data_quality_reports/cache_target_report.json @@ -5,6 +5,11 @@ ], "operation": "data-quality", "remediation_coverage": { + "actionability_counts": { + "informational_only": 1 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -92,8 +97,10 @@ } ], "finding_count": 1, + "included_unmapped_actionable_warning_error_group_count": 0, "included_unmapped_group_count": 1, "included_unmapped_warning_error_group_count": 0, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -118,6 +125,7 @@ "mapped_finding_count": 0, "mapped_rule_id_counts": [], "mapped_severity_counts": {}, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -130,6 +138,8 @@ "severity_counts": { "info": 1 }, + "unmapped_actionable_warning_error_finding_count": 0, + "unmapped_actionable_warning_error_group_count": 0, "unmapped_finding_code_counts": [ { "count": 1, @@ -140,6 +150,8 @@ "unmapped_group_count": 1, "unmapped_groups": [ { + "actionability": "informational_only", + "actionability_reason": "informational_severity", "finding_code": "ASCII_CACHE_SCHEMA_SUMMARY", "included_target_axis_count": 1, "limit_metadata": { diff --git a/tests/fixtures/data_quality_reports/classical_baseline.json b/tests/fixtures/data_quality_reports/classical_baseline.json new file mode 100644 index 00000000..3531cf84 --- /dev/null +++ b/tests/fixtures/data_quality_reports/classical_baseline.json @@ -0,0 +1,211 @@ +{ + "advisory": true, + "base_grain": { + "data_format": "ascii", + "timeframe": "T" + }, + "baseline_id": "d0f808a4a27b83581bcd4b429ebaad2fa641e5bf88ece4f4e702f935bd54a6fd", + "configuration": { + "enabled": true, + "evaluation_fraction": 0.2, + "minimum_evaluation_rows": 5, + "minimum_training_rows": 10, + "rolling_windows": [ + 3 + ], + "rounding_digits": 6, + "session_seasonal_enabled": true + }, + "deferred_model_families": [ + "ets", + "arima", + "sarima", + "state_space", + "garch" + ], + "evaluation": { + "best_model": { + "coverage_rate": 1.0, + "evaluation_row_count": 6, + "forecast_count": 6, + "mae": 0.01, + "mean_error": -0.01, + "model": "naive_random_walk", + "model_code": 1, + "reason": null, + "rmse": 0.01, + "status": "evaluated" + }, + "calculation_basis": "observed_sequence_walk_forward", + "evaluated_model_count": 4, + "guard_codes": [], + "metric": "mid", + "model_count": 4, + "models": [ + { + "coverage_rate": 1.0, + "evaluation_row_count": 6, + "forecast_count": 6, + "mae": 0.01, + "mean_error": -0.01, + "model": "naive_random_walk", + "model_code": 1, + "reason": null, + "rmse": 0.01, + "status": "evaluated" + }, + { + "coverage_rate": 1.0, + "evaluation_row_count": 6, + "forecast_count": 6, + "mae": 0.02, + "mean_error": -0.02, + "model": "rolling_mean", + "model_code": 2, + "reason": null, + "rmse": 0.02, + "status": "evaluated", + "window": 3 + }, + { + "coverage_rate": 1.0, + "evaluation_row_count": 6, + "forecast_count": 6, + "mae": 0.02, + "mean_error": -0.02, + "model": "rolling_median", + "model_code": 3, + "reason": null, + "rmse": 0.02, + "status": "evaluated", + "window": 3 + }, + { + "coverage_rate": 1.0, + "evaluation_row_count": 6, + "forecast_count": 6, + "mae": 0.03, + "mean_error": -0.03, + "model": "session_seasonal_naive", + "model_code": 4, + "reason": null, + "rmse": 0.03, + "status": "evaluated" + } + ], + "recommended_transforms": [], + "skipped_model_count": 0, + "status": "evaluated", + "transforms_applied": [] + }, + "limitations": [], + "non_goals": [ + "forecasting_leaderboard", + "automatic_model_selection", + "synthetic_generation", + "hard_fail_quality_gate" + ], + "prerequisite_readiness": { + "computed_window_count": 1, + "distribution_shift_status": "computed", + "limited_sections": [], + "missing_sections": [], + "recommended_transforms": [], + "required_sections": [ + "coverage", + "temporal_topology", + "calendar_regimes", + "tick_distribution", + "microstructure_dynamics", + "dependence", + "stationarity_diagnostics", + "decomposition" + ], + "rolling_drift_status": "available", + "skipped_window_count": 0, + "stationarity_reason": null, + "stationarity_status": "valid", + "status": "valid", + "unavailable_sections": [], + "zero_variance_metrics": [] + }, + "reason": null, + "reference_fingerprint_id": "fingerprint-eurusd", + "schema_version": "histdatacom.classical-baselines.v1", + "split_policy": { + "evaluation_fraction": 0.2, + "evaluation_row_count": 6, + "future_values_visible": false, + "kind": "chronological_holdout", + "metrics_emitted": true, + "order_by": [ + "series_id", + "period", + "row_id" + ], + "row_count": 30, + "shuffle": false, + "split_index": 24, + "split_row_id": 25, + "timestamp_required": false, + "training_row_count": 24, + "walk_forward": true + }, + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "training_projection": { + "grain": "period", + "identity_fields": [ + "series_id", + "period", + "row_id" + ], + "schema_version": "histdatacom.classical-baseline-training-projection.v1", + "timestamp_required": false, + "values": { + "baseline_best_mae": 0.01, + "baseline_best_model_code": 1, + "baseline_best_model_window": null, + "baseline_best_rmse": 0.01, + "baseline_evaluation_row_count": 6, + "baseline_exclusion_reason_code": 0, + "baseline_split_row_id": 25, + "baseline_stationarity_status_code": 3, + "baseline_status_code": 3, + "baseline_training_ready": true, + "baseline_training_row_count": 24, + "baseline_transform_advisory_code": 0 + } + }, + "training_substrate": { + "identity_fields": [ + "series_id", + "period", + "row_id" + ], + "legacy_cache_enriched_on_read": false, + "metric": "mid", + "observed_bid_ask_preserved": true, + "ordering_fields": [ + "series_id", + "period", + "row_id" + ], + "required_columns": [ + "series_id", + "period", + "row_id", + "mid", + "training_usable" + ], + "schema_version": "histdatacom.ascii-tick-training-features.v1", + "status": "available", + "timestamp_required": false + } +} diff --git a/tests/fixtures/data_quality_reports/classical_model_comparison_cli.golden b/tests/fixtures/data_quality_reports/classical_model_comparison_cli.golden new file mode 100644 index 00000000..d0797f32 --- /dev/null +++ b/tests/fixtures/data_quality_reports/classical_model_comparison_cli.golden @@ -0,0 +1,4 @@ +Classical model comparison +targets: 1; eligible comparisons: 6 +statuses: ready=1 +selection policy: none (descriptive comparisons only) diff --git a/tests/fixtures/data_quality_reports/classical_model_comparison_summary.json b/tests/fixtures/data_quality_reports/classical_model_comparison_summary.json new file mode 100644 index 00000000..89c16bd9 --- /dev/null +++ b/tests/fixtures/data_quality_reports/classical_model_comparison_summary.json @@ -0,0 +1,48 @@ +{ + "advisory": true, + "eligible_comparison_count": 6, + "included_target_count": 1, + "limit_metadata": { + "default_limit": 25, + "effective_limit": 1, + "included_count": 1, + "limit": 1, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 1, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "omitted_target_count": 0, + "schema_version": "histdatacom.classical-model-comparison-summary.v1", + "selection_policy": "none", + "status_counts": { + "ready": 1 + }, + "target_count": 1, + "target_summaries": [ + { + "comparison_count": 6, + "comparison_id": "classical-model-comparison:sha256:cef1d1048f0d41d98f52aca22ecbcce45e3048f2113e9e8b0f2a947ee46aff5e", + "eligible_comparison_count": 6, + "horizons": [ + 1 + ], + "ineligible_comparison_count": 0, + "model_count": 2, + "reason": null, + "selection_policy": "none", + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + } + } + ], + "truncated": false +} diff --git a/tests/fixtures/data_quality_reports/classical_model_comparison_surfaces.json b/tests/fixtures/data_quality_reports/classical_model_comparison_surfaces.json new file mode 100644 index 00000000..2090f286 --- /dev/null +++ b/tests/fixtures/data_quality_reports/classical_model_comparison_surfaces.json @@ -0,0 +1,99 @@ +{ + "bounded_payload": { + "advisory": true, + "eligible_comparison_count": 6, + "included_target_count": 1, + "limit_metadata": { + "default_limit": 25, + "effective_limit": 25, + "included_count": 1, + "limit": 25, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 25, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "omitted_target_count": 0, + "schema_version": "histdatacom.classical-model-comparison-summary.v1", + "selection_policy": "none", + "status_counts": { + "ready": 1 + }, + "target_count": 1, + "target_summaries": [ + { + "comparison_count": 6, + "comparison_id": "classical-model-comparison:sha256:cef1d1048f0d41d98f52aca22ecbcce45e3048f2113e9e8b0f2a947ee46aff5e", + "eligible_comparison_count": 6, + "horizons": [ + 1 + ], + "ineligible_comparison_count": 0, + "model_count": 2, + "reason": null, + "selection_policy": "none", + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + } + } + ], + "truncated": false + }, + "cli_contains_summary": true, + "full_report_metadata": { + "advisory": true, + "eligible_comparison_count": 6, + "included_target_count": 1, + "limit_metadata": { + "default_limit": 25, + "effective_limit": 25, + "included_count": 1, + "limit": 25, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 25, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "omitted_target_count": 0, + "schema_version": "histdatacom.classical-model-comparison-summary.v1", + "selection_policy": "none", + "status_counts": { + "ready": 1 + }, + "target_count": 1, + "target_summaries": [ + { + "comparison_count": 6, + "comparison_id": "classical-model-comparison:sha256:cef1d1048f0d41d98f52aca22ecbcce45e3048f2113e9e8b0f2a947ee46aff5e", + "eligible_comparison_count": 6, + "horizons": [ + 1 + ], + "ineligible_comparison_count": 0, + "model_count": 2, + "reason": null, + "selection_policy": "none", + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + } + } + ], + "truncated": false + } +} diff --git a/tests/fixtures/data_quality_reports/classical_model_input.json b/tests/fixtures/data_quality_reports/classical_model_input.json new file mode 100644 index 00000000..8ab8ed6f --- /dev/null +++ b/tests/fixtures/data_quality_reports/classical_model_input.json @@ -0,0 +1,530 @@ +{ + "advisory": true, + "base_grain": { + "data_format": "ascii", + "timeframe": "T" + }, + "calculation_basis": "regular_grid_from_enriched_ascii_ticks", + "canonical_row_identity": [ + "series_id", + "period", + "row_id" + ], + "configuration": { + "alignment_epoch_ms": 0, + "closed_side": "left", + "differencing_order": 0, + "embargo_observations": 0, + "enabled": true, + "expected_closure_policy": "mark", + "fold_kind": "expanding", + "frequency_ms": 500, + "horizons": [ + 1, + 2 + ], + "label_side": "left", + "midpoint_aggregation": "last", + "minimum_evaluation_observations": 2, + "minimum_observations_per_bin": 1, + "minimum_training_observations": 4, + "resources": { + "max_candidate_orders": 32, + "max_fit_attempts": 64, + "max_folds": 64, + "max_horizons": 16, + "max_memory_bytes": 536870912, + "max_regularized_observations": 1000, + "max_retained_diagnostics": 64, + "max_source_rows": 1000, + "max_wall_time_seconds": 300 + }, + "rolling_window": 0, + "rounding_digits": 12, + "seasonal_differencing_order": 0, + "seasonal_period": 0, + "spread_aggregation": "last", + "step_size": 1, + "timezone": "UTC", + "transform": "level", + "unexpected_missing_policy": "mark" + }, + "dependency_policy": { + "availability_basis": "not_probed", + "contract_available_without_optional_dependencies": true, + "core_dependency_added": false, + "dependencies": [ + { + "available": null, + "package": "statsmodels", + "purpose": "ETS, ARIMA, SARIMAX, and state-space families" + }, + { + "available": null, + "package": "arch", + "purpose": "ARCH and GARCH volatility families" + } + ], + "future_install_extra": "models", + "rich_model_fitting_available": false + }, + "derivation_id": "sha256:5fcb4fd5a520f13c3db3a5b38ee9477b21e5dc79aae2999996b722547f0f0155", + "derived_ohlc_is_canonical_source": false, + "fold_policy": { + "embargo_observations": 0, + "fold_count": 10, + "fold_samples": [ + { + "embargo_observations": 0, + "evaluation_end_index": 5, + "evaluation_end_row_id": 30, + "evaluation_start_index": 4, + "evaluation_start_row_id": 21, + "fold_id": 1, + "future_values_visible": false, + "horizon": 1, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 3000, + "origin_row_id": 20, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 3500, + "target_index": 4, + "target_row_id": 25, + "timestamp_required_as_identity": false, + "training_end_index": 3, + "training_end_row_id": 20, + "training_observation_count": 4, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 5, + "evaluation_end_row_id": 30, + "evaluation_start_index": 4, + "evaluation_start_row_id": 21, + "fold_id": 2, + "future_values_visible": false, + "horizon": 2, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 3000, + "origin_row_id": 20, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 4000, + "target_index": 5, + "target_row_id": 30, + "timestamp_required_as_identity": false, + "training_end_index": 3, + "training_end_row_id": 20, + "training_observation_count": 4, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 6, + "evaluation_end_row_id": 35, + "evaluation_start_index": 5, + "evaluation_start_row_id": 26, + "fold_id": 3, + "future_values_visible": false, + "horizon": 1, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 3500, + "origin_row_id": 25, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 4000, + "target_index": 5, + "target_row_id": 30, + "timestamp_required_as_identity": false, + "training_end_index": 4, + "training_end_row_id": 25, + "training_observation_count": 5, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 6, + "evaluation_end_row_id": 35, + "evaluation_start_index": 5, + "evaluation_start_row_id": 26, + "fold_id": 4, + "future_values_visible": false, + "horizon": 2, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 3500, + "origin_row_id": 25, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 4500, + "target_index": 6, + "target_row_id": 35, + "timestamp_required_as_identity": false, + "training_end_index": 4, + "training_end_row_id": 25, + "training_observation_count": 5, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 7, + "evaluation_end_row_id": 40, + "evaluation_start_index": 6, + "evaluation_start_row_id": 31, + "fold_id": 5, + "future_values_visible": false, + "horizon": 1, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 4000, + "origin_row_id": 30, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 4500, + "target_index": 6, + "target_row_id": 35, + "timestamp_required_as_identity": false, + "training_end_index": 5, + "training_end_row_id": 30, + "training_observation_count": 6, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 7, + "evaluation_end_row_id": 40, + "evaluation_start_index": 6, + "evaluation_start_row_id": 31, + "fold_id": 6, + "future_values_visible": false, + "horizon": 2, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 4000, + "origin_row_id": 30, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 5000, + "target_index": 7, + "target_row_id": 40, + "timestamp_required_as_identity": false, + "training_end_index": 5, + "training_end_row_id": 30, + "training_observation_count": 6, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 8, + "evaluation_end_row_id": 45, + "evaluation_start_index": 7, + "evaluation_start_row_id": 36, + "fold_id": 7, + "future_values_visible": false, + "horizon": 1, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 4500, + "origin_row_id": 35, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 5000, + "target_index": 7, + "target_row_id": 40, + "timestamp_required_as_identity": false, + "training_end_index": 6, + "training_end_row_id": 35, + "training_observation_count": 7, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 8, + "evaluation_end_row_id": 45, + "evaluation_start_index": 7, + "evaluation_start_row_id": 36, + "fold_id": 8, + "future_values_visible": false, + "horizon": 2, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 4500, + "origin_row_id": 35, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 5500, + "target_index": 8, + "target_row_id": 45, + "timestamp_required_as_identity": false, + "training_end_index": 6, + "training_end_row_id": 35, + "training_observation_count": 7, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 9, + "evaluation_end_row_id": 50, + "evaluation_start_index": 8, + "evaluation_start_row_id": 41, + "fold_id": 9, + "future_values_visible": false, + "horizon": 1, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 5000, + "origin_row_id": 40, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 5500, + "target_index": 8, + "target_row_id": 45, + "timestamp_required_as_identity": false, + "training_end_index": 7, + "training_end_row_id": 40, + "training_observation_count": 8, + "training_start_index": 0, + "training_start_row_id": 1 + }, + { + "embargo_observations": 0, + "evaluation_end_index": 9, + "evaluation_end_row_id": 50, + "evaluation_start_index": 8, + "evaluation_start_row_id": 41, + "fold_id": 10, + "future_values_visible": false, + "horizon": 2, + "kind": "expanding", + "kind_code": 1, + "origin_bin_end_utc_ms": 5000, + "origin_row_id": 40, + "period": "", + "reason": null, + "schema_version": "histdatacom.classical-model-fold.v1", + "series_id": "ascii:T::histdata.com", + "shuffle": false, + "status": "valid", + "target_bin_end_utc_ms": 6000, + "target_index": 9, + "target_row_id": 50, + "timestamp_required_as_identity": false, + "training_end_index": 7, + "training_end_row_id": 40, + "training_observation_count": 8, + "training_start_index": 0, + "training_start_row_id": 1 + } + ], + "folds_truncated": false, + "future_values_visible": false, + "horizon_unit": "regularized_grid_steps", + "horizons": [ + 1, + 2 + ], + "incomplete_horizon_policy": "omit_unavailable_targets", + "kind": "expanding", + "minimum_evaluation_observations": 2, + "minimum_training_observations": 4, + "rolling_window": 0, + "schema_version": "histdatacom.classical-model-fold.v1", + "shuffle": false, + "skipped_fold_count": 0, + "step_size": 1, + "timestamp_required_as_identity": false, + "valid_fold_count": 10 + }, + "hard_fail_quality_gate": false, + "limitations": [], + "model_fitting_in_scope": false, + "reason": null, + "reference_fingerprint_basis": "provided_fingerprint_id", + "reference_fingerprint_id": "fingerprint-eurusd", + "regularization": { + "alignment_epoch_ms": 0, + "basis": "regular_grid", + "bin_interval": "[start,end)", + "closed_side": "left", + "derived_ohlc_fields": [ + "cm_input_mid_open", + "cm_input_mid_high", + "cm_input_mid_low", + "cm_input_mid_close" + ], + "duplicate_timestamp_policy": "preserve_and_aggregate_in_row_id_order", + "empty_bin_value_policy": "explicit_null", + "expected_closure_count": 0, + "expected_closure_grid_rows_retained": true, + "expected_closure_model_observations_omitted": false, + "expected_closure_policy": "mark", + "forward_fill_policy": "never", + "frequency_ms": 500, + "insufficient_bin_value_policy": "explicit_null", + "insufficient_observation_bin_count": 0, + "label_side": "left", + "midpoint_aggregation": "last", + "minimum_observations_per_bin": 1, + "observed_bin_count": 10, + "partial_or_unusable_row_policy": "exclude_via_training_usable_before_regularization", + "period_boundary_crossing": false, + "period_partitioned": true, + "regularized_observation_count": 10, + "rounding_applies_to": [ + "cm_input_value" + ], + "rounding_digits": 12, + "row_mapping_policy": "availability_safe_repetition_after_bin_close", + "schema_version": "histdatacom.classical-model-input.v1", + "source_basis": "enriched_ascii_tick_rows", + "spread_aggregation": "last", + "structurally_unavailable_count": 0, + "timezone": "UTC", + "truncated": false, + "unexpected_missing_count": 0, + "unexpected_missing_policy": "mark" + }, + "regularized_view_replaces_source_cache": false, + "row_selection_policy": "training_usable_and_finite_mid_spread", + "schema_version": "histdatacom.classical-model-input.v1", + "source": { + "bounded_row_count": 50, + "kind": "enriched_training_frame", + "legacy_cache_enriched_on_read": true, + "row_count": 50, + "source_rows_truncated": false, + "structurally_unusable_row_count": 0, + "usable_row_count": 50 + }, + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "timestamp_is_sole_identity": false, + "training_projection": { + "column_names": [ + "cm_input_schema_version", + "cm_input_derivation_id", + "cm_input_status_code", + "cm_input_ready", + "cm_input_exclusion_reason_code", + "cm_input_frequency_ms", + "cm_input_bin_start_utc_ms", + "cm_input_bin_end_utc_ms", + "cm_input_available_at_utc_ms", + "cm_input_observation_count", + "cm_input_source_first_row_id", + "cm_input_source_last_row_id", + "cm_input_observed_value", + "cm_input_value", + "cm_input_spread", + "cm_input_transform_code", + "cm_input_calculation_basis_code", + "cm_input_available", + "cm_input_expected_closure", + "cm_input_unexpected_missing", + "cm_fold_schema_version", + "cm_fold_id", + "cm_fold_kind_code", + "cm_fold_origin_row_id", + "cm_fold_target_row_id", + "cm_fold_horizon", + "cm_fold_training_start_row_id", + "cm_fold_training_end_row_id", + "cm_fold_evaluation_start_row_id", + "cm_fold_evaluation_end_row_id", + "cm_evaluation_schema_version", + "cm_evaluation_status_code", + "cm_evaluation_target_available", + "cm_evaluation_forecast", + "cm_evaluation_actual", + "cm_evaluation_error", + "cm_evaluation_diagnostic_only" + ], + "fold_count": 10, + "forecast_columns_populated": false, + "frequency_ms": 500, + "grain": "row", + "identity_fields": [ + "series_id", + "period", + "row_id" + ], + "mapping_policy": "availability_safe_repetition_after_bin_close", + "observed_columns_overwritten": false, + "ready": true, + "reason_code": 0, + "schema_version": "histdatacom.classical-model-training-projection.v1", + "status_code": 3, + "timestamp_required_as_identity": false, + "transform_code": 1 + }, + "training_schema_version": "histdatacom.ascii-tick-training-features.v1", + "transform_policy": { + "applied_explicitly": true, + "cross_period_state": false, + "differencing_order": 0, + "invalid_domain_count": 0, + "inverse_transform": "identity", + "original_scale_metrics_required": true, + "seasonal_differencing_order": 0, + "seasonal_period": 0, + "transform": "level", + "transform_code": 1, + "transformed_observation_count": 10, + "warmup_loss": 0 + } +} diff --git a/tests/fixtures/data_quality_reports/clean_csv_report.json b/tests/fixtures/data_quality_reports/clean_csv_report.json index 6d7b7425..8c1c8fe4 100644 --- a/tests/fixtures/data_quality_reports/clean_csv_report.json +++ b/tests/fixtures/data_quality_reports/clean_csv_report.json @@ -5,6 +5,11 @@ ], "operation": "data-quality", "remediation_coverage": { + "actionability_counts": { + "informational_only": 1 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -92,8 +97,10 @@ } ], "finding_count": 1, + "included_unmapped_actionable_warning_error_group_count": 0, "included_unmapped_group_count": 1, "included_unmapped_warning_error_group_count": 0, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -118,6 +125,7 @@ "mapped_finding_count": 0, "mapped_rule_id_counts": [], "mapped_severity_counts": {}, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -130,6 +138,8 @@ "severity_counts": { "info": 1 }, + "unmapped_actionable_warning_error_finding_count": 0, + "unmapped_actionable_warning_error_group_count": 0, "unmapped_finding_code_counts": [ { "count": 1, @@ -140,6 +150,8 @@ "unmapped_group_count": 1, "unmapped_groups": [ { + "actionability": "informational_only", + "actionability_reason": "informational_severity", "finding_code": "ASCII_SCHEMA_SUMMARY", "included_target_axis_count": 1, "limit_metadata": { diff --git a/tests/fixtures/data_quality_reports/corrupt_zip_repair_plan.json b/tests/fixtures/data_quality_reports/corrupt_zip_repair_plan.json new file mode 100644 index 00000000..89a81c6c --- /dev/null +++ b/tests/fixtures/data_quality_reports/corrupt_zip_repair_plan.json @@ -0,0 +1,117 @@ +{ + "apply_supported": false, + "included_plan_item_count": 1, + "input_report": { + "finding_count": 1, + "path": "tests/fixtures/data_quality_reports/corrupt_zip_report.json", + "status": "failed", + "target_count": 1 + }, + "items": [ + { + "action_kind": "rebuild", + "confidence": { + "basis": [ + "exact rule and finding operation mapping", + "required diagnostic evidence is present", + "operation remains manual and non-mutating in the application" + ], + "level": "high" + }, + "evidence": { + "default_limit": 8, + "effective_limit": 8, + "included_count": 1, + "items": [ + { + "kind": "error_type", + "value": "BadZipFile" + } + ], + "limit": 8, + "maximum_limit": 32, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 8, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "evidence_needed": [ + "archive target identity", + "replacement provenance" + ], + "finding_code": "ZIP_CORRUPT", + "missing_evidence": [], + "operation": { + "automation_status": "manual_only", + "category": "redownload_archive", + "next_step": "Replace the corrupt archive with a trusted copy from the original source.", + "proposal_status": "proposed", + "reason": "exact_rule_finding_operation_mapping", + "specificity": "exact" + }, + "preconditions": [ + "Confirm the archive target identity and period.", + "Retain the original until the replacement passes ZIP integrity checks." + ], + "rank": 1, + "remediation_hint_code": "redownload_corrupt_zip_archive", + "rule_id": "inventory.zip.integrity", + "severity": "error", + "target": { + "path": "quality-fixtures/DAT_ASCII_EURUSD_T_201202.zip", + "target_axis": { + "data_format": "ascii", + "kind": "zip", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + } + } + } + ], + "mode": "non_mutating", + "mutating_operations_performed": false, + "omitted_plan_item_count": 0, + "operation_category_counts": { + "redownload_archive": 1 + }, + "payload_limits": { + "evidence_per_item": { + "default_limit": 8, + "effective_limit": 8, + "limit": 8, + "maximum_limit": 32, + "minimum_limit": 0, + "requested_limit": 8, + "unbounded": false + }, + "items": { + "default_limit": 16, + "effective_limit": 16, + "included_count": 1, + "limit": 16, + "maximum_limit": 64, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 16, + "total_count": 1, + "truncated": false, + "unbounded": false + } + }, + "plan_item_count": 1, + "proposal_status_counts": { + "proposed": 1 + }, + "safety": { + "advisory_only": true, + "automatic_execution": "unsupported", + "filesystem_mutation_performed": false, + "network_access_performed": false, + "requires_user_verification_before_action": true + }, + "schema_version": "histdatacom.quality-repair-plan.v1", + "truncated": false +} diff --git a/tests/fixtures/data_quality_reports/corrupt_zip_report.json b/tests/fixtures/data_quality_reports/corrupt_zip_report.json index 75a9dbc5..6fac1cbd 100644 --- a/tests/fixtures/data_quality_reports/corrupt_zip_report.json +++ b/tests/fixtures/data_quality_reports/corrupt_zip_report.json @@ -86,6 +86,11 @@ "truncated": false }, "remediation_coverage": { + "actionability_counts": { + "remediable_defect": 1 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -173,8 +178,10 @@ } ], "finding_count": 1, + "included_unmapped_actionable_warning_error_group_count": 0, "included_unmapped_group_count": 0, "included_unmapped_warning_error_group_count": 0, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -211,6 +218,7 @@ "mapped_severity_counts": { "error": 1 }, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -223,6 +231,8 @@ "severity_counts": { "error": 1 }, + "unmapped_actionable_warning_error_finding_count": 0, + "unmapped_actionable_warning_error_group_count": 0, "unmapped_finding_code_counts": [], "unmapped_finding_count": 0, "unmapped_group_count": 0, diff --git a/tests/fixtures/data_quality_reports/coverage_manifest_failure_report.json b/tests/fixtures/data_quality_reports/coverage_manifest_failure_report.json index 7797f2c0..d5677cae 100644 --- a/tests/fixtures/data_quality_reports/coverage_manifest_failure_report.json +++ b/tests/fixtures/data_quality_reports/coverage_manifest_failure_report.json @@ -22,6 +22,11 @@ }, "operation": "data-quality", "remediation_coverage": { + "actionability_counts": { + "remediable_defect": 1 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -109,8 +114,10 @@ } ], "finding_count": 1, + "included_unmapped_actionable_warning_error_group_count": 1, "included_unmapped_group_count": 1, "included_unmapped_warning_error_group_count": 1, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -135,6 +142,7 @@ "mapped_finding_count": 0, "mapped_rule_id_counts": [], "mapped_severity_counts": {}, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -147,6 +155,8 @@ "severity_counts": { "error": 1 }, + "unmapped_actionable_warning_error_finding_count": 1, + "unmapped_actionable_warning_error_group_count": 1, "unmapped_finding_code_counts": [ { "count": 1, @@ -157,6 +167,8 @@ "unmapped_group_count": 1, "unmapped_groups": [ { + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", "finding_code": "COVERAGE_PERIOD_MISSING", "included_target_axis_count": 1, "limit_metadata": { diff --git a/tests/fixtures/data_quality_reports/dirty_csv_report.json b/tests/fixtures/data_quality_reports/dirty_csv_report.json index a572b955..eb066eb7 100644 --- a/tests/fixtures/data_quality_reports/dirty_csv_report.json +++ b/tests/fixtures/data_quality_reports/dirty_csv_report.json @@ -6,6 +6,11 @@ ], "operation": "data-quality", "remediation_coverage": { + "actionability_counts": { + "remediable_defect": 2 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -97,8 +102,10 @@ } ], "finding_count": 2, + "included_unmapped_actionable_warning_error_group_count": 2, "included_unmapped_group_count": 2, "included_unmapped_warning_error_group_count": 2, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -123,6 +130,7 @@ "mapped_finding_count": 0, "mapped_rule_id_counts": [], "mapped_severity_counts": {}, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -140,6 +148,8 @@ "error": 1, "warning": 1 }, + "unmapped_actionable_warning_error_finding_count": 2, + "unmapped_actionable_warning_error_group_count": 2, "unmapped_finding_code_counts": [ { "count": 1, @@ -154,6 +164,8 @@ "unmapped_group_count": 2, "unmapped_groups": [ { + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", "finding_code": "ASCII_TICK_NEGATIVE_SPREAD", "included_target_axis_count": 1, "limit_metadata": { @@ -191,6 +203,8 @@ "target_axis_truncated": false }, { + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", "finding_code": "ASCII_TIMESTAMP_DUPLICATE", "included_target_axis_count": 1, "limit_metadata": { diff --git a/tests/fixtures/data_quality_reports/exponential_smoothing_summary.json b/tests/fixtures/data_quality_reports/exponential_smoothing_summary.json new file mode 100644 index 00000000..cbea64a9 --- /dev/null +++ b/tests/fixtures/data_quality_reports/exponential_smoothing_summary.json @@ -0,0 +1,39 @@ +{ + "advisory": true, + "included_target_count": 1, + "limit_metadata": { + "targets": { + "default_limit": 16, + "effective_limit": 1, + "limit": 1, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 1, + "unbounded": false + } + }, + "omitted_target_count": 2, + "schema_version": "histdatacom.exponential-smoothing-summary.v1", + "status_counts": { + "ready": 3 + }, + "target_count": 3, + "target_summaries": [ + { + "evaluated_fold_count": 2, + "failed_fit_count": 0, + "fit_attempt_count": 1, + "model_count": 1, + "reason": null, + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "AUDUSD", + "timeframe": "T" + } + } + ], + "truncated": true +} diff --git a/tests/fixtures/data_quality_reports/fingerprint_bounded_payload.json b/tests/fixtures/data_quality_reports/fingerprint_bounded_payload.json index e455b4e2..7f02bce1 100644 --- a/tests/fixtures/data_quality_reports/fingerprint_bounded_payload.json +++ b/tests/fixtures/data_quality_reports/fingerprint_bounded_payload.json @@ -48,6 +48,221 @@ "unavailable_count": 0, "unavailable_reason_counts": {} }, + "fingerprint_cross_series": { + "ascii_tick_target_count": 3, + "cache_source_counts": {}, + "calculation_basis": "shared_cross_instrument_scan", + "data_format": "ascii", + "fx_series_count": 3, + "group_count": 1, + "groups": [ + { + "complete": true, + "coverage_ranges": { + "common_end_timestamp_utc_ms": 1328072403000, + "common_start_timestamp_utc_ms": 1328072401000, + "limiting_end_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "limiting_start_symbols": [ + "EURGBP" + ], + "unequal_ranges": true + }, + "expected_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "group_id": "ascii:T:201202", + "missing_symbols": [], + "return_correlation": { + "included_pair_count": 3, + "limit_metadata": { + "pairs": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + } + }, + "omitted_pair_count": 0, + "pair_count": 3, + "pairs": [], + "truncated": false + }, + "series": [], + "series_count": 3, + "symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "target_axis": { + "data_format": "ascii", + "period": "201202", + "timeframe": "T" + }, + "timestamp_grid": { + "common_timestamp_count": 3, + "common_timestamp_ratio": 0.75, + "missing_by_symbol": { + "EURGBP": 1, + "EURUSD": 0, + "GBPUSD": 0 + }, + "union_timestamp_count": 4 + }, + "topology": { + "cache_source_counts": {}, + "computed_from_counts": { + "text_scan": 3 + }, + "duplicate_timestamp_row_count": 2, + "mixed_cache_source": false, + "mixed_computation_basis": false, + "source_series_count": 3, + "target_count": 3, + "topology_cache_source_counts": {}, + "topology_computed_from_counts": { + "text_scan": 3 + } + } + } + ], + "included_group_count": 1, + "incomplete_group_count": 0, + "inverse_consistency": { + "candidate_count": 0, + "compared_timestamp_count": 0, + "error_count": 0, + "error_samples": [], + "warning_count": 0, + "warning_samples": [] + }, + "limit_metadata": { + "correlations_per_group": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + }, + "groups": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + } + }, + "omitted_group_count": 0, + "panel_coverage": [ + { + "common_first_period": "201202", + "common_last_period": "201202", + "common_period_count": 1, + "first_period_by_symbol": { + "EURGBP": "201202", + "EURUSD": "201202", + "GBPUSD": "201202" + }, + "last_period_by_symbol": { + "EURGBP": "201202", + "EURUSD": "201202", + "GBPUSD": "201202" + }, + "limiting_end_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "limiting_start_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "missing_period_count_by_symbol": { + "EURGBP": 0, + "EURUSD": 0, + "GBPUSD": 0 + }, + "symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "timeframe": "T", + "unequal_period_ranges": false, + "union_period_count": 1 + } + ], + "return_correlation_status_counts": { + "unavailable": 1, + "valid": 2 + }, + "row_identity": { + "columns": [ + "series_id", + "period", + "row_id", + "source_row_number", + "event_seq" + ], + "duplicate_timestamp_row_count": 2, + "legacy_cache_enrichment_required": true, + "timestamp_is_durable_identity": false, + "training_schema_version": "histdatacom.tick-training-row.v1" + }, + "rule_id": "fingerprint.cross_series", + "schema_version": "histdatacom.cross-series-fingerprint.v1", + "source_basis_counts": { + "text_scan": 3 + }, + "stale_join_risk": { + "risk_count": 0, + "samples": [] + }, + "status": "limited", + "target_count": 3, + "timeframe": "T", + "tolerance": { + "inverse_error_relative_tolerance": 0.02, + "inverse_warning_relative_tolerance": 0.005, + "minimum_common_timestamp_ratio": 0.8, + "stale_forward_fill_min_run": 2, + "triangular_error_relative_tolerance": 0.02, + "triangular_warning_relative_tolerance": 0.005 + }, + "topology_basis": { + "included_target_count": 3, + "schema_version": "histdatacom.time-series-fingerprint-topology-summary.v1", + "target_count": 3, + "truncated": false + }, + "triangular_consistency": { + "candidate_count": 1, + "compared_timestamp_count": 3, + "error_count": 0, + "error_samples": [], + "warning_count": 0, + "warning_samples": [] + }, + "truncated": false, + "unavailable": { + "count": 1, + "samples": [] + } + }, "fingerprint_distribution": { "cache_backed_distribution_target_count": 0, "cache_source_counts": {}, @@ -163,6 +378,24 @@ }, "cache_source_counts": {}, "computed_from_counts": {}, + "decomposition_basis_counts": {}, + "decomposition_computed_window_count": 0, + "decomposition_limitation_counts": {}, + "decomposition_reason_counts": { + "not_emitted": 1 + }, + "decomposition_skipped_window_count": 0, + "decomposition_skipped_window_reason_counts": {}, + "decomposition_stationarity_status_counts": { + "unknown": 1 + }, + "decomposition_status_counts": { + "skipped": 1 + }, + "decomposition_structural_break_candidate_count": 0, + "decomposition_structural_break_status_counts": { + "unknown": 1 + }, "dependence_acf_basis_counts": {}, "dependence_computed_lag_count": 0, "dependence_limitation_counts": {}, @@ -244,6 +477,35 @@ "applicable_dynamics_reason": "not_emitted", "applicable_dynamics_section": "microstructure_dynamics", "applicable_dynamics_status": "skipped", + "decomposition": { + "basis": "unknown", + "cache_source": null, + "calculation_basis": "unknown", + "computed_from": "unknown", + "computed_window_count": 0, + "invalid_row_count": 0, + "level_sample_count": 0, + "limitations": [], + "metric": "unknown", + "partial_row_count": 0, + "reason": "not_emitted", + "regular_grid": false, + "return_sample_count": 0, + "rounding_digits": 0, + "row_count": 0, + "row_order": "unknown", + "sampled_row_count": 0, + "skipped_window_count": 0, + "skipped_window_reason_counts": {}, + "stationarity": {}, + "status": "skipped", + "structural_break": {}, + "training_projection": {}, + "trend": {}, + "truncated": false, + "usable_row_count": 0, + "windows": [] + }, "dependence": { "acf_basis": "unknown", "basis": "unknown", @@ -369,7 +631,7 @@ "topology": { "cache_source": null, "computed_from": "text_scan", - "duplicate_timestamp_count": 0, + "duplicate_timestamp_count": 1, "expected_session_closure_count": 0, "invalid_timestamp_count": 0, "non_monotonic_count": 0, @@ -379,13 +641,17 @@ "suspicious_gap_count": 0, "weekend_activity_count": 0 }, - "topology_limitations": [] + "topology_limitations": [ + "duplicate_timestamps" + ] } ], "tick_spread_conditioning_status_counts": { "eligible": 1 }, - "topology_limitation_counts": {}, + "topology_limitation_counts": { + "duplicate_timestamps": 1 + }, "truncated": false }, "fingerprint_readiness_risk": { @@ -423,21 +689,22 @@ "omitted_target_count": 0, "reason_counts": { "conditional_distribution_absent": 1, + "duplicate_timestamps": 1, "missing_regime_summary": 1, "not_emitted": 2 }, "report_surface_evidence": { "bounded_payload_state_counts": { - "present": 8 + "present": 19 }, "cli_summary_state_counts": { - "present": 8 + "present": 19 }, "report_metadata_state_counts": { - "present": 8 + "present": 19 }, "schema_version": "histdatacom.time-series-fingerprint-report-surface-evidence.v1", - "surface_count": 8 + "surface_count": 19 }, "risk_level_counts": { "high": 1 @@ -449,7 +716,8 @@ "calendar_regimes": 1, "conditional_distributions": 1, "dependence": 1, - "microstructure_dynamics": 1 + "microstructure_dynamics": 1, + "temporal_topology": 1 }, "section_status_counts": { "calendar_regimes": { @@ -472,22 +740,24 @@ "target_count": 1, "target_risks": [ { - "included_section_risk_count": 4, + "included_section_risk_count": 5, "omitted_section_risk_count": 0, "rank": 1, "reason_codes": [ "not_emitted", "conditional_distribution_absent", + "duplicate_timestamps", "missing_regime_summary" ], "reason_counts": { "conditional_distribution_absent": 1, + "duplicate_timestamps": 1, "missing_regime_summary": 1, "not_emitted": 2 }, "risk_level": "high", - "risk_score": 125, - "section_risk_count": 4, + "risk_score": 160, + "section_risk_count": 5, "section_risks": [ { "included_reason_count": 1, @@ -522,6 +792,17 @@ "section": "microstructure_dynamics", "status": "skipped" }, + { + "included_reason_count": 1, + "omitted_reason_count": 0, + "reason_count": 1, + "reasons": [ + "duplicate_timestamps" + ], + "score": 35, + "section": "temporal_topology", + "status": "limited" + }, { "included_reason_count": 1, "omitted_reason_count": 0, @@ -650,7 +931,9 @@ "computed_from_counts": { "text_scan": 1 }, - "flag_counts": {}, + "flag_counts": { + "duplicate_timestamps": 1 + }, "included_target_count": 1, "limit_metadata": { "targets": { @@ -670,16 +953,54 @@ }, "schema_version": "histdatacom.time-series-fingerprint-topology-summary.v1", "status_counts": { - "regular": 1 + "irregular": 1 }, "target_count": 1, "target_summaries": [ { "cache_source": null, "computed_from": "text_scan", - "duplicate_timestamp_count": 0, + "duplicate_timestamp_count": 1, "expected_session_closure_count": 0, - "flags": [], + "flags": [ + "duplicate_timestamps" + ], + "inspection_context": { + "duplicate_timestamps": { + "duplicate_row_count": 1, + "included_count": 1, + "limit_metadata": { + "samples": { + "default_limit": 5, + "effective_limit": 5, + "included_count": 1, + "limit": 5, + "maximum_limit": 5, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 5, + "total_count": 1, + "truncated": false, + "unbounded": false + } + }, + "omitted_count": 0, + "samples": [ + { + "exact_row_group_count": 1, + "occurrence_count": 2, + "row_number": 2, + "timestamp_source": "20120201 000000000", + "timestamp_source_truncated": false, + "timestamp_utc_ms": 1328072400000, + "utc_timestamp": "2012-02-01T05:00:00Z" + } + ], + "total_count": 1, + "truncated": false + }, + "schema_version": "histdatacom.timestamp-topology-inspection.v1" + }, "invalid_timestamp_count": 0, "max_gap_ms": 60000, "median_interval_ms": 60000, @@ -687,7 +1008,7 @@ "parsed_row_count": 3, "row_count": 3, "sampling_basis": "observed_sequence", - "status": "regular", + "status": "irregular", "suspicious_gap_count": 0, "target_axis": { "data_format": "ascii", @@ -702,10 +1023,14 @@ "truncated": false }, "fingerprint_topology_attention": { - "attention_flag_counts": {}, - "attention_level_counts": {}, - "attention_target_count": 0, - "included_attention_target_count": 0, + "attention_flag_counts": { + "duplicate_timestamps": 1 + }, + "attention_level_counts": { + "sequence": 1 + }, + "attention_target_count": 1, + "included_attention_target_count": 1, "limit_metadata": { "targets": { "default_limit": 32, @@ -720,10 +1045,178 @@ "omitted_attention_target_count": 0, "rule_id": "fingerprint.series", "schema_version": "histdatacom.time-series-fingerprint-topology-attention.v1", - "target_summaries": [], + "target_summaries": [ + { + "attention_flags": [ + "duplicate_timestamps" + ], + "attention_level": "sequence", + "cache_source": null, + "computed_from": "text_scan", + "duplicate_timestamp_count": 1, + "expected_session_closure_count": 0, + "flags": [ + "duplicate_timestamps" + ], + "inspection_context": { + "duplicate_timestamps": { + "actionable": true, + "duplicate_row_count": 1, + "included_count": 1, + "limit_metadata": { + "samples": { + "default_limit": 5, + "effective_limit": 5, + "included_count": 1, + "limit": 5, + "maximum_limit": 5, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 5, + "total_count": 1, + "truncated": false, + "unbounded": false + } + }, + "next_action": { + "action_kind": "inspect", + "code": "inspect_duplicate_timestamp_rows", + "flag": "duplicate_timestamps", + "message": "inspect duplicate timestamp rows", + "rule_id": "fingerprint.series" + }, + "omitted_count": 0, + "samples": [ + { + "exact_row_group_count": 1, + "occurrence_count": 2, + "row_number": 2, + "timestamp_source": "20120201 000000000", + "timestamp_source_truncated": false, + "timestamp_utc_ms": 1328072400000, + "utc_timestamp": "2012-02-01T05:00:00Z" + } + ], + "target_axis": { + "data_format": "ascii", + "kind": "csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "total_count": 1, + "truncated": false + }, + "schema_version": "histdatacom.timestamp-topology-inspection.v1" + }, + "invalid_timestamp_count": 0, + "max_gap_ms": 60000, + "non_monotonic_count": 0, + "remediation_hints": [ + { + "action_kind": "inspect", + "code": "inspect_duplicate_timestamp_rows", + "flag": "duplicate_timestamps", + "message": "inspect duplicate timestamp rows", + "rule_id": "fingerprint.series" + } + ], + "status": "irregular", + "suspicious_gap_count": 0, + "target_axis": { + "data_format": "ascii", + "kind": "csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "weekend_activity_count": 0 + } + ], "topology_target_count": 1, "truncated": false }, + "next_actions": { + "action_count": 1, + "actions": [ + { + "action_kind": "inspect", + "affected_target_count": 1, + "attention_level_counts": { + "sequence": 1 + }, + "code": "inspect_duplicate_timestamp_rows", + "finding_code_counts": {}, + "flag_counts": { + "duplicate_timestamps": 1 + }, + "included_target_axis_count": 1, + "limit_metadata": { + "target_axes": { + "default_limit": 8, + "effective_limit": 8, + "limit": 8, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 8, + "unbounded": false + } + }, + "max_attention_level": "sequence", + "max_severity": null, + "message": "inspect duplicate timestamp rows", + "occurrence_count": 1, + "omitted_target_axis_count": 0, + "rule_id": "fingerprint.series", + "severity_counts": {}, + "source_counts": { + "fingerprint_topology_attention": 1 + }, + "target_axis_count": 1, + "target_axis_counts": [ + { + "count": 1, + "target_axis": { + "data_format": "ascii", + "kind": "csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + } + } + ], + "target_axis_truncated": false, + "urgency": "medium" + } + ], + "included_action_count": 1, + "limit_metadata": { + "actions": { + "default_limit": 16, + "effective_limit": 16, + "limit": 16, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 16, + "unbounded": false + }, + "target_axes": { + "default_limit": 8, + "effective_limit": 8, + "limit": 8, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 8, + "unbounded": false + } + }, + "omitted_action_count": 0, + "schema_version": "histdatacom.quality-next-actions.v1", + "source_counts": { + "fingerprint_topology_attention": 1 + }, + "truncated": false + }, "operation": "data-quality", "payload_limits": { "cross_target_summaries": { @@ -755,7 +1248,7 @@ "next_actions": { "default_limit": 16, "effective_limit": 16, - "included_count": 0, + "included_count": 1, "limit": 16, "maximum_limit": null, "minimum_limit": 0, @@ -771,7 +1264,7 @@ "unbounded": false }, "target_axis_limit": 8, - "total_count": 0, + "total_count": 1, "truncated": false, "unbounded": false }, @@ -814,6 +1307,11 @@ }, "quality_profile": {}, "remediation_coverage": { + "actionability_counts": { + "informational_only": 1 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -901,8 +1399,10 @@ } ], "finding_count": 1, + "included_unmapped_actionable_warning_error_group_count": 0, "included_unmapped_group_count": 1, "included_unmapped_warning_error_group_count": 0, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -927,6 +1427,7 @@ "mapped_finding_count": 0, "mapped_rule_id_counts": [], "mapped_severity_counts": {}, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -939,6 +1440,8 @@ "severity_counts": { "info": 1 }, + "unmapped_actionable_warning_error_finding_count": 0, + "unmapped_actionable_warning_error_group_count": 0, "unmapped_finding_code_counts": [ { "count": 1, @@ -949,6 +1452,8 @@ "unmapped_group_count": 1, "unmapped_groups": [ { + "actionability": "informational_only", + "actionability_reason": "informational_severity", "finding_code": "FINGERPRINT_SERIES_SUMMARY", "included_target_axis_count": 1, "limit_metadata": { diff --git a/tests/fixtures/data_quality_reports/fingerprint_report.json b/tests/fixtures/data_quality_reports/fingerprint_report.json index 5e490c06..ee5214eb 100644 --- a/tests/fixtures/data_quality_reports/fingerprint_report.json +++ b/tests/fixtures/data_quality_reports/fingerprint_report.json @@ -3,8 +3,309 @@ "check_groups": [ "fingerprint" ], + "cross_series_fingerprint": { + "ascii_tick_target_count": 3, + "cache_source_counts": {}, + "calculation_basis": "shared_cross_instrument_scan", + "data_format": "ascii", + "fx_series_count": 3, + "group_count": 1, + "groups": [ + { + "complete": true, + "coverage_ranges": { + "common_end_timestamp_utc_ms": 1328072403000, + "common_start_timestamp_utc_ms": 1328072401000, + "limiting_end_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "limiting_start_symbols": [ + "EURGBP" + ], + "unequal_ranges": true + }, + "expected_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "group_id": "ascii:T:201202", + "missing_symbols": [], + "return_correlation": { + "included_pair_count": 3, + "limit_metadata": { + "pairs": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + } + }, + "omitted_pair_count": 0, + "pair_count": 3, + "pairs": [], + "truncated": false + }, + "series": [], + "series_count": 3, + "symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "target_axis": { + "data_format": "ascii", + "period": "201202", + "timeframe": "T" + }, + "timestamp_grid": { + "common_timestamp_count": 3, + "common_timestamp_ratio": 0.75, + "missing_by_symbol": { + "EURGBP": 1, + "EURUSD": 0, + "GBPUSD": 0 + }, + "union_timestamp_count": 4 + }, + "topology": { + "cache_source_counts": {}, + "computed_from_counts": { + "text_scan": 3 + }, + "duplicate_timestamp_row_count": 2, + "mixed_cache_source": false, + "mixed_computation_basis": false, + "source_series_count": 3, + "target_count": 3, + "topology_cache_source_counts": {}, + "topology_computed_from_counts": { + "text_scan": 3 + } + } + } + ], + "included_group_count": 1, + "incomplete_group_count": 0, + "inverse_consistency": { + "candidate_count": 0, + "compared_timestamp_count": 0, + "error_count": 0, + "error_samples": [], + "warning_count": 0, + "warning_samples": [] + }, + "limit_metadata": { + "correlations_per_group": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + }, + "groups": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + } + }, + "omitted_group_count": 0, + "panel_coverage": [ + { + "common_first_period": "201202", + "common_last_period": "201202", + "common_period_count": 1, + "first_period_by_symbol": { + "EURGBP": "201202", + "EURUSD": "201202", + "GBPUSD": "201202" + }, + "last_period_by_symbol": { + "EURGBP": "201202", + "EURUSD": "201202", + "GBPUSD": "201202" + }, + "limiting_end_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "limiting_start_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "missing_period_count_by_symbol": { + "EURGBP": 0, + "EURUSD": 0, + "GBPUSD": 0 + }, + "symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD" + ], + "timeframe": "T", + "unequal_period_ranges": false, + "union_period_count": 1 + } + ], + "return_correlation_status_counts": { + "unavailable": 1, + "valid": 2 + }, + "row_identity": { + "columns": [ + "series_id", + "period", + "row_id", + "source_row_number", + "event_seq" + ], + "duplicate_timestamp_row_count": 2, + "legacy_cache_enrichment_required": true, + "timestamp_is_durable_identity": false, + "training_schema_version": "histdatacom.tick-training-row.v1" + }, + "rule_id": "fingerprint.cross_series", + "schema_version": "histdatacom.cross-series-fingerprint.v1", + "source_basis_counts": { + "text_scan": 3 + }, + "stale_join_risk": { + "risk_count": 0, + "samples": [] + }, + "status": "limited", + "target_count": 3, + "timeframe": "T", + "tolerance": { + "inverse_error_relative_tolerance": 0.02, + "inverse_warning_relative_tolerance": 0.005, + "minimum_common_timestamp_ratio": 0.8, + "stale_forward_fill_min_run": 2, + "triangular_error_relative_tolerance": 0.02, + "triangular_warning_relative_tolerance": 0.005 + }, + "topology_basis": { + "included_target_count": 3, + "schema_version": "histdatacom.time-series-fingerprint-topology-summary.v1", + "target_count": 3, + "truncated": false + }, + "triangular_consistency": { + "candidate_count": 1, + "compared_timestamp_count": 3, + "error_count": 0, + "error_samples": [], + "warning_count": 0, + "warning_samples": [] + }, + "truncated": false, + "unavailable": { + "count": 1, + "samples": [] + } + }, "operation": "data-quality", + "quality_next_actions": { + "action_count": 1, + "actions": [ + { + "action_kind": "inspect", + "affected_target_count": 1, + "attention_level_counts": { + "sequence": 1 + }, + "code": "inspect_duplicate_timestamp_rows", + "finding_code_counts": {}, + "flag_counts": { + "duplicate_timestamps": 1 + }, + "included_target_axis_count": 1, + "limit_metadata": { + "target_axes": { + "default_limit": 8, + "effective_limit": 8, + "limit": 8, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 8, + "unbounded": false + } + }, + "max_attention_level": "sequence", + "max_severity": null, + "message": "inspect duplicate timestamp rows", + "occurrence_count": 1, + "omitted_target_axis_count": 0, + "rule_id": "fingerprint.series", + "severity_counts": {}, + "source_counts": { + "fingerprint_topology_attention": 1 + }, + "target_axis_count": 1, + "target_axis_counts": [ + { + "count": 1, + "target_axis": { + "data_format": "ascii", + "kind": "csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + } + } + ], + "target_axis_truncated": false, + "urgency": "medium" + } + ], + "included_action_count": 1, + "limit_metadata": { + "actions": { + "default_limit": 16, + "effective_limit": 16, + "limit": 16, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 16, + "unbounded": false + }, + "target_axes": { + "default_limit": 8, + "effective_limit": 8, + "limit": 8, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 8, + "unbounded": false + } + }, + "omitted_action_count": 0, + "schema_version": "histdatacom.quality-next-actions.v1", + "source_counts": { + "fingerprint_topology_attention": 1 + }, + "truncated": false + }, "remediation_coverage": { + "actionability_counts": { + "informational_only": 1 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -92,8 +393,10 @@ } ], "finding_count": 1, + "included_unmapped_actionable_warning_error_group_count": 0, "included_unmapped_group_count": 1, "included_unmapped_warning_error_group_count": 0, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -118,6 +421,7 @@ "mapped_finding_count": 0, "mapped_rule_id_counts": [], "mapped_severity_counts": {}, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -130,6 +434,8 @@ "severity_counts": { "info": 1 }, + "unmapped_actionable_warning_error_finding_count": 0, + "unmapped_actionable_warning_error_group_count": 0, "unmapped_finding_code_counts": [ { "count": 1, @@ -140,6 +446,8 @@ "unmapped_group_count": 1, "unmapped_groups": [ { + "actionability": "informational_only", + "actionability_reason": "informational_severity", "finding_code": "FINGERPRINT_SERIES_SUMMARY", "included_target_axis_count": 1, "limit_metadata": { @@ -357,21 +665,22 @@ "omitted_target_count": 0, "reason_counts": { "conditional_distribution_absent": 1, + "duplicate_timestamps": 1, "missing_regime_summary": 1, "not_emitted": 2 }, "report_surface_evidence": { "bounded_payload_state_counts": { - "present": 8 + "present": 19 }, "cli_summary_state_counts": { - "present": 8 + "present": 19 }, "report_metadata_state_counts": { - "present": 8 + "present": 19 }, "schema_version": "histdatacom.time-series-fingerprint-report-surface-evidence.v1", - "surface_count": 8 + "surface_count": 19 }, "risk_level_counts": { "high": 1 @@ -383,7 +692,8 @@ "calendar_regimes": 1, "conditional_distributions": 1, "dependence": 1, - "microstructure_dynamics": 1 + "microstructure_dynamics": 1, + "temporal_topology": 1 }, "section_status_counts": { "calendar_regimes": { @@ -406,22 +716,24 @@ "target_count": 1, "target_risks": [ { - "included_section_risk_count": 4, + "included_section_risk_count": 5, "omitted_section_risk_count": 0, "rank": 1, "reason_codes": [ "not_emitted", "conditional_distribution_absent", + "duplicate_timestamps", "missing_regime_summary" ], "reason_counts": { "conditional_distribution_absent": 1, + "duplicate_timestamps": 1, "missing_regime_summary": 1, "not_emitted": 2 }, "risk_level": "high", - "risk_score": 125, - "section_risk_count": 4, + "risk_score": 160, + "section_risk_count": 5, "section_risks": [ { "included_reason_count": 1, @@ -456,6 +768,17 @@ "section": "microstructure_dynamics", "status": "skipped" }, + { + "included_reason_count": 1, + "omitted_reason_count": 0, + "reason_count": 1, + "reasons": [ + "duplicate_timestamps" + ], + "score": 35, + "section": "temporal_topology", + "status": "limited" + }, { "included_reason_count": 1, "omitted_reason_count": 0, @@ -486,6 +809,24 @@ }, "cache_source_counts": {}, "computed_from_counts": {}, + "decomposition_basis_counts": {}, + "decomposition_computed_window_count": 0, + "decomposition_limitation_counts": {}, + "decomposition_reason_counts": { + "not_emitted": 1 + }, + "decomposition_skipped_window_count": 0, + "decomposition_skipped_window_reason_counts": {}, + "decomposition_stationarity_status_counts": { + "unknown": 1 + }, + "decomposition_status_counts": { + "skipped": 1 + }, + "decomposition_structural_break_candidate_count": 0, + "decomposition_structural_break_status_counts": { + "unknown": 1 + }, "dependence_acf_basis_counts": {}, "dependence_computed_lag_count": 0, "dependence_limitation_counts": {}, @@ -567,6 +908,35 @@ "applicable_dynamics_reason": "not_emitted", "applicable_dynamics_section": "microstructure_dynamics", "applicable_dynamics_status": "skipped", + "decomposition": { + "basis": "unknown", + "cache_source": null, + "calculation_basis": "unknown", + "computed_from": "unknown", + "computed_window_count": 0, + "invalid_row_count": 0, + "level_sample_count": 0, + "limitations": [], + "metric": "unknown", + "partial_row_count": 0, + "reason": "not_emitted", + "regular_grid": false, + "return_sample_count": 0, + "rounding_digits": 0, + "row_count": 0, + "row_order": "unknown", + "sampled_row_count": 0, + "skipped_window_count": 0, + "skipped_window_reason_counts": {}, + "stationarity": {}, + "status": "skipped", + "structural_break": {}, + "training_projection": {}, + "trend": {}, + "truncated": false, + "usable_row_count": 0, + "windows": [] + }, "dependence": { "acf_basis": "unknown", "basis": "unknown", @@ -692,7 +1062,7 @@ "topology": { "cache_source": null, "computed_from": "text_scan", - "duplicate_timestamp_count": 0, + "duplicate_timestamp_count": 1, "expected_session_closure_count": 0, "invalid_timestamp_count": 0, "non_monotonic_count": 0, @@ -702,13 +1072,17 @@ "suspicious_gap_count": 0, "weekend_activity_count": 0 }, - "topology_limitations": [] + "topology_limitations": [ + "duplicate_timestamps" + ] } ], "tick_spread_conditioning_status_counts": { "eligible": 1 }, - "topology_limitation_counts": {}, + "topology_limitation_counts": { + "duplicate_timestamps": 1 + }, "truncated": false }, "time_series_fingerprint_regime_summary": { @@ -811,10 +1185,14 @@ "truncated": false }, "time_series_fingerprint_topology_attention": { - "attention_flag_counts": {}, - "attention_level_counts": {}, - "attention_target_count": 0, - "included_attention_target_count": 0, + "attention_flag_counts": { + "duplicate_timestamps": 1 + }, + "attention_level_counts": { + "sequence": 1 + }, + "attention_target_count": 1, + "included_attention_target_count": 1, "limit_metadata": { "targets": { "default_limit": 32, @@ -829,7 +1207,94 @@ "omitted_attention_target_count": 0, "rule_id": "fingerprint.series", "schema_version": "histdatacom.time-series-fingerprint-topology-attention.v1", - "target_summaries": [], + "target_summaries": [ + { + "attention_flags": [ + "duplicate_timestamps" + ], + "attention_level": "sequence", + "cache_source": null, + "computed_from": "text_scan", + "duplicate_timestamp_count": 1, + "expected_session_closure_count": 0, + "flags": [ + "duplicate_timestamps" + ], + "inspection_context": { + "duplicate_timestamps": { + "actionable": true, + "duplicate_row_count": 1, + "included_count": 1, + "limit_metadata": { + "samples": { + "default_limit": 5, + "effective_limit": 5, + "included_count": 1, + "limit": 5, + "maximum_limit": 5, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 5, + "total_count": 1, + "truncated": false, + "unbounded": false + } + }, + "next_action": { + "action_kind": "inspect", + "code": "inspect_duplicate_timestamp_rows", + "flag": "duplicate_timestamps", + "message": "inspect duplicate timestamp rows", + "rule_id": "fingerprint.series" + }, + "omitted_count": 0, + "samples": [ + { + "exact_row_group_count": 1, + "occurrence_count": 2, + "row_number": 2, + "timestamp_source": "20120201 000000000", + "timestamp_source_truncated": false, + "timestamp_utc_ms": 1328072400000, + "utc_timestamp": "2012-02-01T05:00:00Z" + } + ], + "target_axis": { + "data_format": "ascii", + "kind": "csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "total_count": 1, + "truncated": false + }, + "schema_version": "histdatacom.timestamp-topology-inspection.v1" + }, + "invalid_timestamp_count": 0, + "max_gap_ms": 60000, + "non_monotonic_count": 0, + "remediation_hints": [ + { + "action_kind": "inspect", + "code": "inspect_duplicate_timestamp_rows", + "flag": "duplicate_timestamps", + "message": "inspect duplicate timestamp rows", + "rule_id": "fingerprint.series" + } + ], + "status": "irregular", + "suspicious_gap_count": 0, + "target_axis": { + "data_format": "ascii", + "kind": "csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "weekend_activity_count": 0 + } + ], "topology_target_count": 1, "truncated": false }, @@ -838,7 +1303,9 @@ "computed_from_counts": { "text_scan": 1 }, - "flag_counts": {}, + "flag_counts": { + "duplicate_timestamps": 1 + }, "included_target_count": 1, "limit_metadata": { "targets": { @@ -858,16 +1325,54 @@ }, "schema_version": "histdatacom.time-series-fingerprint-topology-summary.v1", "status_counts": { - "regular": 1 + "irregular": 1 }, "target_count": 1, "target_summaries": [ { "cache_source": null, "computed_from": "text_scan", - "duplicate_timestamp_count": 0, + "duplicate_timestamp_count": 1, "expected_session_closure_count": 0, - "flags": [], + "flags": [ + "duplicate_timestamps" + ], + "inspection_context": { + "duplicate_timestamps": { + "duplicate_row_count": 1, + "included_count": 1, + "limit_metadata": { + "samples": { + "default_limit": 5, + "effective_limit": 5, + "included_count": 1, + "limit": 5, + "maximum_limit": 5, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 5, + "total_count": 1, + "truncated": false, + "unbounded": false + } + }, + "omitted_count": 0, + "samples": [ + { + "exact_row_group_count": 1, + "occurrence_count": 2, + "row_number": 2, + "timestamp_source": "20120201 000000000", + "timestamp_source_truncated": false, + "timestamp_utc_ms": 1328072400000, + "utc_timestamp": "2012-02-01T05:00:00Z" + } + ], + "total_count": 1, + "truncated": false + }, + "schema_version": "histdatacom.timestamp-topology-inspection.v1" + }, "invalid_timestamp_count": 0, "max_gap_ms": 60000, "median_interval_ms": 60000, @@ -875,7 +1380,7 @@ "parsed_row_count": 3, "row_count": 3, "sampling_basis": "observed_sequence", - "status": "regular", + "status": "irregular", "suspicious_gap_count": 0, "target_axis": { "data_format": "ascii", @@ -977,7 +1482,7 @@ "unsupported_reason": null } }, - "fingerprint_id": "sha256:12c919fc872938c599980788b207700680013580935c6de1fcb0176475bd84bb", + "fingerprint_id": "sha256:1cffba0884465e650a34438b83051773e62b94895cc6a790a712c980b7d8dc77", "schema_version": "histdatacom.time-series-fingerprint.v1", "source": { "kind": "csv_text", @@ -993,9 +1498,9 @@ "temporal_topology": { "cache_source": null, "computed_from": "text_scan", - "duplicate_timestamp_count": 0, + "duplicate_timestamp_count": 1, "duplicate_timestamp_source_counts": { - "tick_duplicate_row": 0 + "tick_duplicate_row": 1 }, "expected_session_closure_count": 0, "gap_bucket_counts": { @@ -1021,6 +1526,42 @@ "session_boundary_grace_ms": 3600000, "suspicious_gap_ms": 300000 }, + "inspection_context": { + "duplicate_timestamps": { + "duplicate_row_count": 1, + "included_count": 1, + "limit_metadata": { + "samples": { + "default_limit": 5, + "effective_limit": 5, + "included_count": 1, + "limit": 5, + "maximum_limit": 5, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 5, + "total_count": 1, + "truncated": false, + "unbounded": false + } + }, + "omitted_count": 0, + "samples": [ + { + "exact_row_group_count": 1, + "occurrence_count": 2, + "row_number": 2, + "timestamp_source": "20120201 000000000", + "timestamp_source_truncated": false, + "timestamp_utc_ms": 1328072400000, + "utc_timestamp": "2012-02-01T05:00:00Z" + } + ], + "total_count": 1, + "truncated": false + }, + "schema_version": "histdatacom.timestamp-topology-inspection.v1" + }, "interval_count": 2, "invalid_timestamp_count": 0, "max_gap_ms": 60000, @@ -1031,7 +1572,7 @@ "row_count": 3, "sampling_basis": "observed_sequence", "suspicious_gap_count": 0, - "tick_duplicate_row_count": 0, + "tick_duplicate_row_count": 1, "timestamp_projection": "text_scan", "weekend_activity_count": 0 }, diff --git a/tests/fixtures/data_quality_reports/orchestration_bounded_payload.json b/tests/fixtures/data_quality_reports/orchestration_bounded_payload.json index 0086e440..159cf209 100644 --- a/tests/fixtures/data_quality_reports/orchestration_bounded_payload.json +++ b/tests/fixtures/data_quality_reports/orchestration_bounded_payload.json @@ -179,6 +179,11 @@ }, "quality_profile": {}, "remediation_coverage": { + "actionability_counts": { + "remediable_defect": 1 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -266,8 +271,10 @@ } ], "finding_count": 1, + "included_unmapped_actionable_warning_error_group_count": 1, "included_unmapped_group_count": 1, "included_unmapped_warning_error_group_count": 1, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -292,6 +299,7 @@ "mapped_finding_count": 0, "mapped_rule_id_counts": [], "mapped_severity_counts": {}, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -304,6 +312,8 @@ "severity_counts": { "error": 1 }, + "unmapped_actionable_warning_error_finding_count": 1, + "unmapped_actionable_warning_error_group_count": 1, "unmapped_finding_code_counts": [ { "count": 1, @@ -314,6 +324,8 @@ "unmapped_group_count": 1, "unmapped_groups": [ { + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", "finding_code": "DOMAIN_CROSS_INSTRUMENT_TRIANGULAR_ERROR", "included_target_axis_count": 1, "limit_metadata": { diff --git a/tests/fixtures/data_quality_reports/quality_engine_skip_bounded_payload.json b/tests/fixtures/data_quality_reports/quality_engine_skip_bounded_payload.json new file mode 100644 index 00000000..164cd4d7 --- /dev/null +++ b/tests/fixtures/data_quality_reports/quality_engine_skip_bounded_payload.json @@ -0,0 +1,281 @@ +{ + "check_groups": [ + "time" + ], + "cross_target_summaries": [], + "discovery": { + "metadata": { + "supported_kinds": [ + "zip", + "csv", + "cache" + ] + }, + "roots": [ + "quality-fixtures" + ], + "target_count": 2 + }, + "exit_decision": { + "exit_code": 0, + "policy": { + "fail_on": "error", + "max_errors": 0, + "max_warnings": 0 + }, + "reason": "quality report is within configured exit policy" + }, + "operation": "data-quality", + "payload_limits": { + "cross_target_summaries": { + "default_limit": 128, + "effective_limit": 128, + "included_count": 0, + "limit": 128, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 128, + "total_count": 0, + "truncated": false, + "unbounded": false + }, + "discovery_targets": { + "default_limit": 128, + "effective_limit": 128, + "included_count": 0, + "limit": 128, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 128, + "total_count": 0, + "truncated": false, + "unbounded": false + }, + "next_actions": { + "default_limit": 16, + "effective_limit": 16, + "included_count": 0, + "limit": 16, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 16, + "target_axes": { + "default_limit": 8, + "effective_limit": 8, + "limit": 8, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 8, + "unbounded": false + }, + "target_axis_limit": 8, + "total_count": 0, + "truncated": false, + "unbounded": false + }, + "remediation_coverage": { + "default_limit": 16, + "effective_limit": 16, + "included_count": 0, + "limit": 16, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 16, + "target_axes": { + "default_limit": 8, + "effective_limit": 8, + "limit": 8, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 8, + "unbounded": false + }, + "target_axis_limit": 8, + "total_count": 0, + "truncated": false, + "unbounded": false + }, + "target_summaries": { + "default_limit": 128, + "effective_limit": 128, + "included_count": 2, + "limit": 128, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 128, + "total_count": 2, + "truncated": false, + "unbounded": false + } + }, + "quality_engine": { + "duplicate_archive_scan_policy": "prefer_extracted_csv_for_non_inventory_rules", + "planned_target_rule_evaluation_count": 2, + "rule_count": 1, + "run_rule_count": 0, + "schema_version": "histdatacom.quality-engine.v1", + "skip_events": { + "event_count": 1, + "events": [ + { + "reason_code": "duplicate_archive_preferred_csv", + "rule_id": "time.ascii.gaps", + "target_axis": { + "data_format": "ascii", + "kind": "zip", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "target_kind": "zip" + } + ], + "included_event_count": 1, + "limit_metadata": { + "events": { + "default_limit": 128, + "effective_limit": 128, + "included_count": 1, + "limit": 128, + "maximum_limit": 128, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": null, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "reasons": { + "default_limit": 64, + "effective_limit": 64, + "included_count": 1, + "limit": 64, + "maximum_limit": 64, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": null, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "rules": { + "default_limit": 64, + "effective_limit": 64, + "included_count": 1, + "limit": 64, + "maximum_limit": 64, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": null, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "target_kinds": { + "default_limit": 64, + "effective_limit": 64, + "included_count": 1, + "limit": 64, + "maximum_limit": 64, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": null, + "total_count": 1, + "truncated": false, + "unbounded": false + } + }, + "omitted_event_count": 0, + "reason_counts": { + "duplicate_archive_preferred_csv": 1 + }, + "rule_id_counts": { + "time.ascii.gaps": 1 + }, + "schema_version": "histdatacom.quality-skip-events.v1", + "target_kind_counts": { + "zip": 1 + }, + "truncated": false + }, + "skipped_duplicate_archive_rule_evaluation_count": 1, + "skipped_rule_evaluation_count": 1, + "target_count": 2, + "target_rule_evaluation_count": 1 + }, + "quality_profile": {}, + "report_artifact": { + "kind": "quality-report", + "metadata": { + "error_count": 0, + "finding_count": 0, + "max_severity": "info", + "schema_version": "histdatacom.quality-report.v1", + "status": "clean", + "target_count": 2, + "warning_count": 0 + }, + "path": "quality-fixtures/reports/quality-engine-skip-report.json", + "sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "size_bytes": 2048 + }, + "report_schema_version": "histdatacom.quality-report.v1", + "summary": { + "error_count": 0, + "finding_count": 0, + "info_count": 0, + "max_severity": "info", + "rule_count": 1, + "status": "clean", + "target_count": 2, + "warning_count": 0 + }, + "target_status_counts": { + "clean": 2, + "failed": 0, + "warning": 0 + }, + "target_summaries": [ + { + "error_count": 0, + "finding_count": 0, + "info_count": 0, + "max_severity": "info", + "rule_count": 1, + "status": "clean", + "target": { + "data_format": "ascii", + "kind": "csv", + "metadata": {}, + "path": "quality-fixtures/DAT_ASCII_EURUSD_T_201202.csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "warning_count": 0 + }, + { + "error_count": 0, + "finding_count": 0, + "info_count": 0, + "max_severity": "info", + "rule_count": 0, + "status": "clean", + "target": { + "data_format": "ascii", + "kind": "zip", + "metadata": {}, + "path": "quality-fixtures/DAT_ASCII_EURUSD_T_201202.zip", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "warning_count": 0 + } + ] +} diff --git a/tests/fixtures/data_quality_reports/quality_engine_skip_report.json b/tests/fixtures/data_quality_reports/quality_engine_skip_report.json new file mode 100644 index 00000000..77fc6915 --- /dev/null +++ b/tests/fixtures/data_quality_reports/quality_engine_skip_report.json @@ -0,0 +1,189 @@ +{ + "metadata": { + "check_groups": [ + "time" + ], + "operation": "data-quality", + "quality_engine": { + "duplicate_archive_scan_policy": "prefer_extracted_csv_for_non_inventory_rules", + "planned_target_rule_evaluation_count": 2, + "rule_count": 1, + "run_rule_count": 0, + "schema_version": "histdatacom.quality-engine.v1", + "skip_events": { + "event_count": 1, + "events": [ + { + "reason_code": "duplicate_archive_preferred_csv", + "rule_id": "time.ascii.gaps", + "target_axis": { + "data_format": "ascii", + "kind": "zip", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "target_kind": "zip" + } + ], + "included_event_count": 1, + "limit_metadata": { + "events": { + "default_limit": 128, + "effective_limit": 128, + "included_count": 1, + "limit": 128, + "maximum_limit": 128, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": null, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "reasons": { + "default_limit": 64, + "effective_limit": 64, + "included_count": 1, + "limit": 64, + "maximum_limit": 64, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": null, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "rules": { + "default_limit": 64, + "effective_limit": 64, + "included_count": 1, + "limit": 64, + "maximum_limit": 64, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": null, + "total_count": 1, + "truncated": false, + "unbounded": false + }, + "target_kinds": { + "default_limit": 64, + "effective_limit": 64, + "included_count": 1, + "limit": 64, + "maximum_limit": 64, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": null, + "total_count": 1, + "truncated": false, + "unbounded": false + } + }, + "omitted_event_count": 0, + "reason_counts": { + "duplicate_archive_preferred_csv": 1 + }, + "rule_id_counts": { + "time.ascii.gaps": 1 + }, + "schema_version": "histdatacom.quality-skip-events.v1", + "target_kind_counts": { + "zip": 1 + }, + "truncated": false + }, + "skipped_duplicate_archive_rule_evaluation_count": 1, + "skipped_rule_evaluation_count": 1, + "target_count": 2, + "target_rule_evaluation_count": 1 + } + }, + "rule_results": [ + { + "findings": [], + "max_severity": "info", + "rule_id": "time.ascii.gaps", + "status": "clean", + "target": { + "data_format": "ascii", + "kind": "csv", + "metadata": {}, + "path": "quality-fixtures/DAT_ASCII_EURUSD_T_201202.csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + } + } + ], + "schema_version": "histdatacom.quality-report.v1", + "summary": { + "error_count": 0, + "finding_count": 0, + "info_count": 0, + "max_severity": "info", + "rule_count": 1, + "status": "clean", + "target_count": 2, + "warning_count": 0 + }, + "target_summaries": [ + { + "error_count": 0, + "finding_count": 0, + "info_count": 0, + "max_severity": "info", + "rule_count": 0, + "status": "clean", + "target": { + "data_format": "ascii", + "kind": "zip", + "metadata": {}, + "path": "quality-fixtures/DAT_ASCII_EURUSD_T_201202.zip", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "warning_count": 0 + }, + { + "error_count": 0, + "finding_count": 0, + "info_count": 0, + "max_severity": "info", + "rule_count": 1, + "status": "clean", + "target": { + "data_format": "ascii", + "kind": "csv", + "metadata": {}, + "path": "quality-fixtures/DAT_ASCII_EURUSD_T_201202.csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "warning_count": 0 + } + ], + "targets": [ + { + "data_format": "ascii", + "kind": "zip", + "metadata": {}, + "path": "quality-fixtures/DAT_ASCII_EURUSD_T_201202.zip", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + { + "data_format": "ascii", + "kind": "csv", + "metadata": {}, + "path": "quality-fixtures/DAT_ASCII_EURUSD_T_201202.csv", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + } + ] +} diff --git a/tests/fixtures/data_quality_reports/remediation_catalog_audit.json b/tests/fixtures/data_quality_reports/remediation_catalog_audit.json index 1a3c937b..787ad3ed 100644 --- a/tests/fixtures/data_quality_reports/remediation_catalog_audit.json +++ b/tests/fixtures/data_quality_reports/remediation_catalog_audit.json @@ -1,5 +1,14 @@ { "known_code_counts": { + "attribution_reason_counts": [ + { + "attribution_reason": "provided_rule_id", + "count": 3 + } + ], + "attribution_status_counts": { + "exact": 3 + }, "finding_code_counts": [ { "count": 1, @@ -72,11 +81,30 @@ "count": 1, "source_family": "time" } - ] + ], + "unresolved_finding_code_prefix_counts": [], + "unresolved_source_family_counts": [], + "unresolved_source_helper_counts": [] }, "known_unmapped_codes": [ { + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", + "attribution_reason": "provided_rule_id", + "attribution_reason_counts": { + "provided_rule_id": 1 + }, + "attribution_status": "exact", + "attribution_status_counts": { + "exact": 1 + }, "finding_code": "ASCII_TIMESTAMP_SOURCE_PERIOD_MISMATCH", + "finding_code_prefix_counts": [ + { + "count": 1, + "finding_code_prefix": "ASCII_TIMESTAMP_SOURCE" + } + ], "included_source_count": 1, "mapped": false, "max_severity": "error", @@ -94,6 +122,7 @@ "source_family": "time" } ], + "source_helper_counts": [], "sources": [ { "count": 1, @@ -102,7 +131,23 @@ ] }, { + "actionability": "informational_only", + "actionability_reason": "informational_severity", + "attribution_reason": "provided_rule_id", + "attribution_reason_counts": { + "provided_rule_id": 1 + }, + "attribution_status": "exact", + "attribution_status_counts": { + "exact": 1 + }, "finding_code": "FINGERPRINT_SERIES_SUMMARY", + "finding_code_prefix_counts": [ + { + "count": 1, + "finding_code_prefix": "FINGERPRINT" + } + ], "included_source_count": 1, "mapped": false, "max_severity": "info", @@ -120,6 +165,7 @@ "source_family": "fingerprint" } ], + "source_helper_counts": [], "sources": [ { "count": 1, @@ -129,6 +175,19 @@ } ], "payload_limits": { + "attribution_reason_counts": { + "default_limit": 16, + "effective_limit": 4, + "included_count": 1, + "limit": 4, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 4, + "total_count": 1, + "truncated": false, + "unbounded": false + }, "known_code_sources": { "applies_per_code": true, "default_limit": 8, @@ -211,6 +270,19 @@ "truncated": false, "unbounded": false }, + "remediation_plan": { + "default_limit": 16, + "effective_limit": 4, + "included_count": 2, + "limit": 4, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 4, + "total_count": 2, + "truncated": false, + "unbounded": false + }, "report_unmapped_groups": { "applies_per_report": true, "default_limit": 16, @@ -230,11 +302,53 @@ }, "target_axis_limit": 8, "unbounded": false + }, + "unresolved_finding_code_prefix_counts": { + "default_limit": 16, + "effective_limit": 4, + "included_count": 0, + "limit": 4, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 4, + "total_count": 0, + "truncated": false, + "unbounded": false + }, + "unresolved_source_helper_counts": { + "default_limit": 16, + "effective_limit": 4, + "included_count": 0, + "limit": 4, + "maximum_limit": null, + "minimum_limit": 0, + "omitted_count": 0, + "requested_limit": 4, + "total_count": 0, + "truncated": false, + "unbounded": false } }, "ranked_gaps": [ { + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", + "attribution_reason": "provided_rule_id", + "attribution_reason_counts": { + "provided_rule_id": 1 + }, + "attribution_status": "exact", + "attribution_status_counts": { + "exact": 1 + }, "finding_code": "ASCII_TIMESTAMP_SOURCE_PERIOD_MISMATCH", + "finding_code_prefix_counts": [ + { + "count": 1, + "finding_code_prefix": "ASCII_TIMESTAMP_SOURCE" + } + ], "included_source_count": 1, "known_source_occurrence_count": 1, "mapped": false, @@ -242,6 +356,7 @@ "omitted_source_count": 0, "rank": 1, "rank_reasons": [ + "actionability=remediable_defect", "severity=error", "source_family=time", "known_sources=1" @@ -260,6 +375,7 @@ "source_family": "time" } ], + "source_helper_counts": [], "sources": [ { "count": 1, @@ -268,7 +384,23 @@ ] }, { + "actionability": "informational_only", + "actionability_reason": "informational_severity", + "attribution_reason": "provided_rule_id", + "attribution_reason_counts": { + "provided_rule_id": 1 + }, + "attribution_status": "exact", + "attribution_status_counts": { + "exact": 1 + }, "finding_code": "FINGERPRINT_SERIES_SUMMARY", + "finding_code_prefix_counts": [ + { + "count": 1, + "finding_code_prefix": "FINGERPRINT" + } + ], "included_source_count": 1, "known_source_occurrence_count": 1, "mapped": false, @@ -276,6 +408,7 @@ "omitted_source_count": 0, "rank": 2, "rank_reasons": [ + "actionability=informational_only", "severity=info", "source_family=fingerprint", "known_sources=1" @@ -294,6 +427,7 @@ "source_family": "fingerprint" } ], + "source_helper_counts": [], "sources": [ { "count": 1, @@ -302,22 +436,195 @@ ] } ], + "remediation_plan": { + "actionability_counts": { + "informational_only": 1, + "remediable_defect": 1 + }, + "fixability_counts": { + "high": 1, + "low": 1 + }, + "included_plan_item_count": 2, + "items": [ + { + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", + "attribution_reason": "provided_rule_id", + "attribution_status": "exact", + "catalog_gap_rank": 1, + "draft_hint_code": "repair_ascii_timestamp_source_period_mismatch", + "evidence": { + "known_source_occurrence_count": 1, + "omitted_report_source_count": 0, + "omitted_source_count": 0, + "report_group_count": 0, + "report_occurrence_count": 0, + "reports": [], + "sources": [ + { + "count": 1, + "source": "data_quality/time.ascii.est_no_dst.py:1" + } + ] + }, + "finding_code": "ASCII_TIMESTAMP_SOURCE_PERIOD_MISMATCH", + "fixability": { + "basis": [ + "actionability=remediable_defect", + "selector=exact_rule_and_finding", + "attribution=exact", + "severity=error", + "concrete_action=true", + "known_sources=1", + "report_occurrences=0" + ], + "confidence": "high", + "level": "high", + "score": 96 + }, + "mapped": false, + "max_severity": "error", + "missing_fields": [ + "message" + ], + "rank": 1, + "rank_reasons": [ + "actionability=remediable_defect", + "severity=error", + "source_family=time", + "known_sources=1" + ], + "rule_id": "time.ascii.est_no_dst", + "severity_counts": { + "error": 1 + }, + "source_family": "time", + "suggested_action": { + "action_kind": "repair", + "basis": "finding_code_marker=mismatch", + "concrete": true, + "confidence": "high" + }, + "suggested_hint_code_prefix": "repair_ascii_timestamp_source", + "suggested_selector": { + "basis": "exact_rule_attribution", + "confidence": "high", + "finding_code": "ASCII_TIMESTAMP_SOURCE_PERIOD_MISMATCH", + "finding_code_prefix": "ASCII_TIMESTAMP_SOURCE", + "rule_id": "time.ascii.est_no_dst", + "shape": "exact_rule_and_finding" + } + }, + { + "actionability": "informational_only", + "actionability_reason": "informational_severity", + "attribution_reason": "provided_rule_id", + "attribution_status": "exact", + "catalog_gap_rank": 2, + "draft_hint_code": "inspect_fingerprint_series_summary", + "evidence": { + "known_source_occurrence_count": 1, + "omitted_report_source_count": 0, + "omitted_source_count": 0, + "report_group_count": 0, + "report_occurrence_count": 0, + "reports": [], + "sources": [ + { + "count": 1, + "source": "data_quality/fingerprint.series.py:1" + } + ] + }, + "finding_code": "FINGERPRINT_SERIES_SUMMARY", + "fixability": { + "basis": [ + "actionability=informational_only", + "selector=exact_rule_and_finding", + "attribution=exact", + "severity=info", + "concrete_action=false", + "known_sources=1", + "report_occurrences=0" + ], + "confidence": "low", + "level": "low", + "score": 36 + }, + "mapped": false, + "max_severity": "info", + "missing_fields": [ + "message", + "action_kind_confirmation", + "boundary_decision" + ], + "rank": 2, + "rank_reasons": [ + "actionability=informational_only", + "severity=info", + "source_family=fingerprint", + "known_sources=1" + ], + "rule_id": "fingerprint.series", + "severity_counts": { + "info": 1 + }, + "source_family": "fingerprint", + "suggested_action": { + "action_kind": "inspect", + "basis": "actionability_boundary=informational_only", + "concrete": false, + "confidence": "low" + }, + "suggested_hint_code_prefix": "inspect_fingerprint", + "suggested_selector": { + "basis": "exact_rule_attribution", + "confidence": "high", + "finding_code": "FINGERPRINT_SERIES_SUMMARY", + "finding_code_prefix": "FINGERPRINT", + "rule_id": "fingerprint.series", + "shape": "exact_rule_and_finding" + } + } + ], + "omitted_plan_item_count": 0, + "plan_item_count": 2, + "schema_version": "histdatacom.quality-remediation-plan.v1", + "truncated": false + }, "report_coverage": [], "schema_version": "histdatacom.quality-remediation-catalog-audit.v1", "status": "needs-remediation-guidance", "summary": { + "actionability_counts": { + "informational_only": 1, + "remediable_defect": 2 + }, + "blocked_by_attribution_warning_error_code_count": 0, + "blocked_by_missing_diagnostics_warning_error_code_count": 0, + "exact_attribution_occurrence_count": 3, + "inferred_attribution_occurrence_count": 0, + "intentionally_unremediable_warning_error_code_count": 0, "known_code_count": 3, "known_finding_occurrence_count": 3, "known_warning_error_code_count": 2, "mapped_known_code_count": 1, + "report_blocked_by_attribution_warning_error_finding_count": 0, + "report_blocked_by_missing_diagnostics_warning_error_finding_count": 0, "report_count": 0, "report_finding_count": 0, + "report_intentionally_unremediable_warning_error_finding_count": 0, "report_mapped_finding_count": 0, + "report_unmapped_actionable_warning_error_group_count": 0, "report_unmapped_finding_count": 0, "report_unmapped_warning_error_group_count": 0, + "unmapped_actionable_warning_error_code_count": 1, + "unmapped_actionable_warning_error_gap_count": 1, "unmapped_info_only_code_count": 1, "unmapped_known_code_count": 2, "unmapped_warning_error_code_count": 1, - "unmapped_warning_error_gap_count": 1 + "unmapped_warning_error_gap_count": 1, + "unresolved_attribution_occurrence_count": 0 } } diff --git a/tests/fixtures/data_quality_reports/run_scoped_report.json b/tests/fixtures/data_quality_reports/run_scoped_report.json index 0b5d6afe..18e674a0 100644 --- a/tests/fixtures/data_quality_reports/run_scoped_report.json +++ b/tests/fixtures/data_quality_reports/run_scoped_report.json @@ -5,6 +5,11 @@ ], "operation": "data-quality", "remediation_coverage": { + "actionability_counts": { + "remediable_defect": 1 + }, + "blocked_by_attribution_warning_error_finding_count": 0, + "blocked_by_missing_diagnostics_warning_error_finding_count": 0, "count_limits": { "finding_code_counts": { "default_limit": 16, @@ -92,8 +97,10 @@ } ], "finding_count": 1, + "included_unmapped_actionable_warning_error_group_count": 1, "included_unmapped_group_count": 1, "included_unmapped_warning_error_group_count": 1, + "intentionally_unremediable_warning_error_finding_count": 0, "limit_metadata": { "groups": { "default_limit": 16, @@ -118,6 +125,7 @@ "mapped_finding_count": 0, "mapped_rule_id_counts": [], "mapped_severity_counts": {}, + "omitted_unmapped_actionable_warning_error_group_count": 0, "omitted_unmapped_group_count": 0, "omitted_unmapped_warning_error_group_count": 0, "rule_id_counts": [ @@ -130,6 +138,8 @@ "severity_counts": { "error": 1 }, + "unmapped_actionable_warning_error_finding_count": 1, + "unmapped_actionable_warning_error_group_count": 1, "unmapped_finding_code_counts": [ { "count": 1, @@ -140,6 +150,8 @@ "unmapped_group_count": 1, "unmapped_groups": [ { + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", "finding_code": "DOMAIN_CROSS_INSTRUMENT_TRIANGULAR_ERROR", "included_target_axis_count": 1, "limit_metadata": { diff --git a/tests/fixtures/data_quality_reports/seasonal_exogenous_summary.json b/tests/fixtures/data_quality_reports/seasonal_exogenous_summary.json new file mode 100644 index 00000000..ddefd273 --- /dev/null +++ b/tests/fixtures/data_quality_reports/seasonal_exogenous_summary.json @@ -0,0 +1,40 @@ +{ + "advisory": true, + "included_target_count": 1, + "limit_metadata": { + "targets": { + "default_limit": 16, + "effective_limit": 1, + "limit": 1, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 1, + "unbounded": false + } + }, + "omitted_target_count": 2, + "schema_version": "histdatacom.seasonal-exogenous-summary.v1", + "status_counts": { + "limited": 1, + "ready": 2 + }, + "target_count": 3, + "target_summaries": [ + { + "evaluated_fold_count": 6, + "failed_fit_count": 0, + "fit_attempt_count": 6, + "model_count": 3, + "reason": null, + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "csv", + "period": "201201", + "symbol": "AUDUSD", + "timeframe": "T" + } + } + ], + "truncated": true +} diff --git a/tests/fixtures/data_quality_reports/state_space_summary.json b/tests/fixtures/data_quality_reports/state_space_summary.json new file mode 100644 index 00000000..8d1d3979 --- /dev/null +++ b/tests/fixtures/data_quality_reports/state_space_summary.json @@ -0,0 +1,40 @@ +{ + "advisory": true, + "included_target_count": 1, + "limit_metadata": { + "targets": { + "default_limit": 16, + "effective_limit": 1, + "limit": 1, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 1, + "unbounded": false + } + }, + "omitted_target_count": 2, + "schema_version": "histdatacom.state-space-summary.v1", + "status_counts": { + "limited": 1, + "ready": 2 + }, + "target_count": 3, + "target_summaries": [ + { + "evaluated_fold_count": 6, + "failed_fit_count": 0, + "fit_attempt_count": 6, + "model_count": 3, + "reason": null, + "status": "ready", + "target_axis": { + "data_format": "ascii", + "kind": "csv", + "period": "201201", + "symbol": "AUDUSD", + "timeframe": "T" + } + } + ], + "truncated": true +} diff --git a/tests/fixtures/data_quality_reports/synthetic_tick_generation.json b/tests/fixtures/data_quality_reports/synthetic_tick_generation.json new file mode 100644 index 00000000..72412e33 --- /dev/null +++ b/tests/fixtures/data_quality_reports/synthetic_tick_generation.json @@ -0,0 +1,330 @@ +{ + "advisory": true, + "base_grain": { + "data_format": "ascii", + "timeframe": "T" + }, + "configuration": { + "anchor_mode": "first_valid_mid", + "block_size": 4, + "diagnostic_sample_limit": 4, + "max_abs_log_return": 0.25, + "max_generated_rows": 1000, + "max_reference_rows": 1000, + "method": "empirical_block_bootstrap", + "method_code": 1, + "minimum_reference_rows": 8, + "overwrite_existing": false, + "rounding_digits": 8, + "schema_version": "histdatacom.synthetic-tick-generation-configuration.v1", + "seed": 17 + }, + "constraint_application": { + "defects_to_avoid": { + "application": "filter_flagged_reference_rows_and_enforce_quote_invariants", + "codes": [ + "avoid_negative_spread", + "avoid_duplicate_timestamps", + "avoid_non_monotonic_timestamps", + "avoid_suspicious_non_session_gaps", + "avoid_invalid_rows", + "avoid_partial_rows", + "avoid_topology_unavailable", + "avoid_fingerprint_unready", + "avoid_unsupported_schema", + "avoid_structurally_invalid_timestamps" + ], + "count": 10 + }, + "source_artifacts_to_parameterize": { + "application": "preserve_reference_row_topology_and_expose_configuration", + "codes": [ + "parameterize_source_provenance", + "parameterize_source_coverage", + "parameterize_gap_topology", + "parameterize_calendar_policy", + "parameterize_dynamics_limitations", + "parameterize_stationarity_limitations", + "parameterize_training_substrate", + "defer_raw_m1_ohlc_constraints" + ], + "count": 8 + }, + "stylized_facts_to_preserve": { + "application": "paired_empirical_blocks_then_fingerprint_validation", + "codes": [ + "session_activity_mix", + "active_session_mix", + "calendar_special_tag_mix", + "gap_bucket_shape", + "interval_topology", + "spread_distribution", + "precision_regime", + "spread_jump_behavior", + "stale_quote_runs", + "burst_behavior", + "one_sided_movement", + "volatility_clustering_proxy", + "return_tail_shape", + "rolling_drift", + "stationarity_transform_policy", + "structural_regime_proxy" + ], + "count": 16 + } + }, + "generation": { + "block_size": 4, + "calculation_basis": "paired_mid_log_return_and_spread_blocks", + "confidence": 1.0, + "generated_row_count": 80, + "input_row_count": 80, + "method": "empirical_block_bootstrap", + "method_code": 1, + "observed_columns_overwritten": false, + "omitted_row_count": 0, + "price_bounds": { + "maximum": 1000000000000.0, + "minimum": 1e-08 + }, + "price_clamp_count": 0, + "return_clip_count": 0, + "same_row_grain": true, + "sample_count": 4, + "samples": [ + { + "output_row_offset": 0, + "reference_source_row_id": 29, + "reference_transition_index": 27 + }, + { + "output_row_offset": 1, + "reference_source_row_id": 30, + "reference_transition_index": 28 + }, + { + "output_row_offset": 2, + "reference_source_row_id": 31, + "reference_transition_index": 29 + }, + { + "output_row_offset": 3, + "reference_source_row_id": 32, + "reference_transition_index": 30 + } + ], + "samples_truncated": true, + "seed": 17, + "synthetic_columns_only": true, + "timestamp_generation": false + }, + "generation_id": "synthetic-generation:sha256:8b28a63d7391c88986b2c2cfea3a3cb19be845648bc5ad9d1a4808d1f5e5d066", + "input_row_count": 80, + "output_contract": { + "columns": [ + "synth_bid", + "synth_ask", + "synth_spread", + "synth_mid", + "synth_method_code", + "synth_confidence", + "synth_usable" + ], + "durable_identity": [ + "series_id", + "period", + "row_id" + ], + "observed_bid_ask_preserved": true, + "raw_m1_generation": false, + "same_frame_projection": true, + "timestamp_is_sole_identity": false, + "volume_generation": false + }, + "quality_gate": false, + "reason": null, + "reason_code": 0, + "reference_constraint_id": "sha256:c27e09c72afacd22c755c666ee281b9f94fbafc71d9e186def5a3e7e8698134a", + "reference_constraint_status": "ready", + "reference_evidence": { + "defective_rows_used": false, + "filtered_reference_row_count": 0, + "hard_issue_columns": [ + "dq_issue_non_monotonic_timestamp", + "dq_issue_negative_spread", + "dq_issue_invalid_row", + "dq_issue_source_unavailable", + "dq_issue_topology_unavailable" + ], + "inspected_row_count": 80, + "reference_rows_truncated": false, + "transition_count": 79, + "valid_reference_row_count": 80 + }, + "reference_fingerprint_count": 1, + "reference_fingerprint_id": "sha256:3975187060b6b1f376a039cc2d0c788040fdada0752e816176a66de6b04ce7a9", + "reference_fingerprint_ids": [ + "sha256:3975187060b6b1f376a039cc2d0c788040fdada0752e816176a66de6b04ce7a9" + ], + "reference_fingerprint_required": true, + "resource_policy": { + "bounded": true, + "max_generated_rows": 1000, + "max_reference_rows": 1000 + }, + "schema_version": "histdatacom.synthetic-tick-generation.v1", + "status": "ready", + "status_code": 3, + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "validation": { + "candidate_fingerprint_id": "sha256:9fbb7310d9af4d08ca9afb5735e454faf3bb044e192cff45b44d20bbfc8a5549", + "candidate_fingerprint_schema_version": "histdatacom.time-series-fingerprint.v1", + "constraint_validation": { + "advisory": true, + "base_grain": { + "data_format": "ascii", + "timeframe": "T" + }, + "candidate_target_count": 1, + "compared_target_count": 1, + "included_target_count": 1, + "limit_metadata": { + "mismatches": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + }, + "targets": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + } + }, + "matching_target_count": 0, + "mismatch_code_counts": { + "synthetic_candidate_one_sided_movement_mismatch": 1, + "synthetic_candidate_precision_regime_mismatch": 1, + "synthetic_candidate_return_tail_shape_mismatch": 1, + "synthetic_candidate_spread_distribution_mismatch": 1, + "synthetic_candidate_structural_regime_proxy_mismatch": 1, + "synthetic_candidate_volatility_clustering_proxy_mismatch": 1 + }, + "mismatch_count": 6, + "mismatched_target_count": 1, + "not_compared_target_count": 0, + "omitted_target_count": 0, + "reference_target_count": 1, + "schema_version": "histdatacom.synthetic-fingerprint-validation.v1", + "status": "mismatch", + "target_results": [ + { + "compared_fact_count": 16, + "included_mismatch_code_count": 6, + "included_mismatch_count": 6, + "mismatch_code_count": 6, + "mismatch_code_counts": { + "synthetic_candidate_one_sided_movement_mismatch": 1, + "synthetic_candidate_precision_regime_mismatch": 1, + "synthetic_candidate_return_tail_shape_mismatch": 1, + "synthetic_candidate_spread_distribution_mismatch": 1, + "synthetic_candidate_structural_regime_proxy_mismatch": 1, + "synthetic_candidate_volatility_clustering_proxy_mismatch": 1 + }, + "mismatch_codes": [ + "synthetic_candidate_one_sided_movement_mismatch", + "synthetic_candidate_precision_regime_mismatch", + "synthetic_candidate_return_tail_shape_mismatch", + "synthetic_candidate_spread_distribution_mismatch", + "synthetic_candidate_structural_regime_proxy_mismatch", + "synthetic_candidate_volatility_clustering_proxy_mismatch" + ], + "mismatch_count": 6, + "mismatch_details": [ + { + "category": "stylized_fact", + "code": "synthetic_candidate_one_sided_movement_mismatch", + "comparison": "numeric_relative_tolerance", + "constraint_code": "one_sided_movement", + "tolerance": 0.1 + }, + { + "category": "stylized_fact", + "code": "synthetic_candidate_precision_regime_mismatch", + "comparison": "distribution_l1", + "constraint_code": "precision_regime", + "tolerance": 0.0 + }, + { + "category": "stylized_fact", + "code": "synthetic_candidate_return_tail_shape_mismatch", + "comparison": "numeric_relative_tolerance", + "constraint_code": "return_tail_shape", + "tolerance": 0.1 + }, + { + "category": "stylized_fact", + "code": "synthetic_candidate_spread_distribution_mismatch", + "comparison": "numeric_relative_tolerance", + "constraint_code": "spread_distribution", + "tolerance": 0.1 + }, + { + "category": "stylized_fact", + "code": "synthetic_candidate_structural_regime_proxy_mismatch", + "comparison": "numeric_relative_tolerance", + "constraint_code": "structural_regime_proxy", + "tolerance": 0.1 + }, + { + "category": "stylized_fact", + "code": "synthetic_candidate_volatility_clustering_proxy_mismatch", + "comparison": "numeric_absolute_tolerance", + "constraint_code": "volatility_clustering_proxy", + "tolerance": 0.2 + } + ], + "omitted_mismatch_code_count": 0, + "omitted_mismatch_count": 0, + "status": "mismatch", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "truncated": false + } + ], + "truncated": false + }, + "hard_quality_gate": false, + "same_fingerprint_path_used": true, + "same_point_influx_projection": { + "line_count": 1, + "observed_fields_present": true, + "raw_line_included": false, + "reason": null, + "same_measurement": true, + "status": "valid", + "synthetic_fields_present": true, + "training_fields_present": true + }, + "schema_version": "histdatacom.synthetic-tick-generation-validation.v1", + "status": "mismatch" + } +} diff --git a/tests/fixtures/data_quality_reports/synthetic_validation.json b/tests/fixtures/data_quality_reports/synthetic_validation.json new file mode 100644 index 00000000..963efacd --- /dev/null +++ b/tests/fixtures/data_quality_reports/synthetic_validation.json @@ -0,0 +1,86 @@ +{ + "advisory": true, + "base_grain": { + "data_format": "ascii", + "timeframe": "T" + }, + "candidate_target_count": 1, + "compared_target_count": 1, + "included_target_count": 1, + "limit_metadata": { + "mismatches": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + }, + "targets": { + "default_limit": 32, + "effective_limit": 32, + "limit": 32, + "maximum_limit": null, + "minimum_limit": 0, + "requested_limit": 32, + "unbounded": false + } + }, + "matching_target_count": 0, + "mismatch_code_counts": { + "synthetic_candidate_avoid_duplicate_timestamps_present": 1, + "synthetic_candidate_gap_bucket_shape_mismatch": 1 + }, + "mismatch_count": 2, + "mismatched_target_count": 1, + "not_compared_target_count": 0, + "omitted_target_count": 0, + "reference_target_count": 1, + "schema_version": "histdatacom.synthetic-fingerprint-validation.v1", + "status": "mismatch", + "target_results": [ + { + "compared_fact_count": 15, + "included_mismatch_code_count": 2, + "included_mismatch_count": 2, + "mismatch_code_count": 2, + "mismatch_code_counts": { + "synthetic_candidate_avoid_duplicate_timestamps_present": 1, + "synthetic_candidate_gap_bucket_shape_mismatch": 1 + }, + "mismatch_codes": [ + "synthetic_candidate_avoid_duplicate_timestamps_present", + "synthetic_candidate_gap_bucket_shape_mismatch" + ], + "mismatch_count": 2, + "mismatch_details": [ + { + "category": "defect", + "code": "synthetic_candidate_avoid_duplicate_timestamps_present", + "constraint_code": "avoid_duplicate_timestamps", + "observed_count": 2 + }, + { + "category": "stylized_fact", + "code": "synthetic_candidate_gap_bucket_shape_mismatch", + "comparison": "distribution_l1", + "constraint_code": "gap_bucket_shape", + "tolerance": 0.1 + } + ], + "omitted_mismatch_code_count": 0, + "omitted_mismatch_count": 0, + "status": "mismatch", + "target_axis": { + "data_format": "ascii", + "kind": "cache", + "period": "201202", + "symbol": "EURUSD", + "timeframe": "T" + }, + "truncated": false + } + ], + "truncated": false +} diff --git a/tests/fixtures/reconstruction_checkpoint_v1.json b/tests/fixtures/reconstruction_checkpoint_v1.json new file mode 100644 index 00000000..13dc3032 --- /dev/null +++ b/tests/fixtures/reconstruction_checkpoint_v1.json @@ -0,0 +1,19 @@ +{ + "advertised_manifest_ref": null, + "carry_state_ref": null, + "checkpoint_id": "checkpoint:sha256:360f81eec78eeaf06514f41790e2339fcba24ba2c7665da7a9a7009a96591e5f", + "committed_manifest_ref": null, + "completed_batch_ids": [], + "input_watermark_ns": null, + "interruption_reason": "", + "output_watermark_ns": null, + "parent_checkpoint_id": null, + "phase": "planned", + "rejection_summary_ref": null, + "revision": 0, + "run_id": "reconstruction-run:sha256:0219b294392b6c6f67153513f8cb95ab6090f8f760a8b6f7d11437bb65266775", + "schema_version": "histdatacom.reconstruction-checkpoint.v1", + "staged_manifest_ref": null, + "synchronization_unit_id": "synchronization-unit:sha256:2c3242707c28a462c62ace3d92883a8dba84596e464d00daee202afbb7d7e179", + "window_id": "reconstruction-window:sha256:2c3242707c28a462c62ace3d92883a8dba84596e464d00daee202afbb7d7e179" +} diff --git a/tests/fixtures/synthetic_event_stream_v1.json b/tests/fixtures/synthetic_event_stream_v1.json new file mode 100644 index 00000000..602c2a4e --- /dev/null +++ b/tests/fixtures/synthetic_event_stream_v1.json @@ -0,0 +1,101 @@ +{ + "ensemble_member_id": "member-000", + "event_count": 3, + "event_schema_version": "histdatacom.synthetic-event.v1", + "events": [ + { + "anchor_interval_id": null, + "ask": 1.0801, + "bid": 1.08, + "broker_profile_id": null, + "confidence": null, + "constraint_set_id": null, + "ensemble_member_id": "member-000", + "event_id": "event:sha256:3b0b77fabed45983450328b1e191052f6f10ebc00146a8967c7f32ca45b44db4", + "event_sequence": 0, + "event_time_ns": 1700000000000000000, + "feed_epoch_id": null, + "generator_config_id": null, + "generator_id": null, + "generator_version": null, + "left_anchor_event_id": null, + "motif_id": null, + "origin": "observed", + "reference_id": null, + "right_anchor_event_id": null, + "run_id": "run-example-v1", + "schema_version": "histdatacom.synthetic-event.v1", + "source_period": "202401", + "source_row_id": 1, + "source_series_id": "ascii:T:eurusd", + "source_version_id": "source:example-v1", + "symbol": "eurusd" + }, + { + "anchor_interval_id": "anchor-interval:sha256:5bc6800ce217669a477ebfaff592ddcaf07265e1631a7b5ef6e224941fcb1da8", + "ask": 1.0802, + "bid": 1.0801, + "broker_profile_id": null, + "confidence": 0.75, + "constraint_set_id": "constraint:example-v1", + "ensemble_member_id": "member-000", + "event_id": "event:sha256:b0c54d64acbfc74bedae292612198fa057967b6992ed976a1c843f7c892eccff", + "event_sequence": 0, + "event_time_ns": 1700000000000000500, + "feed_epoch_id": null, + "generator_config_id": "config:example-v1", + "generator_id": "example-generator", + "generator_version": "1.0.0", + "left_anchor_event_id": "event:sha256:3b0b77fabed45983450328b1e191052f6f10ebc00146a8967c7f32ca45b44db4", + "motif_id": null, + "origin": "synthetic", + "reference_id": "reference:example-v1", + "right_anchor_event_id": "event:sha256:eab055c8a91bfaba60ae4d3e2a925bb3d1275c7de61abf3ce67dc1da4ebb05ee", + "run_id": "run-example-v1", + "schema_version": "histdatacom.synthetic-event.v1", + "source_period": null, + "source_row_id": null, + "source_series_id": null, + "source_version_id": "source:example-inputs-v1", + "symbol": "eurusd" + }, + { + "anchor_interval_id": null, + "ask": 1.0803, + "bid": 1.0802, + "broker_profile_id": null, + "confidence": null, + "constraint_set_id": null, + "ensemble_member_id": "member-000", + "event_id": "event:sha256:eab055c8a91bfaba60ae4d3e2a925bb3d1275c7de61abf3ce67dc1da4ebb05ee", + "event_sequence": 0, + "event_time_ns": 1700000000000001000, + "feed_epoch_id": null, + "generator_config_id": null, + "generator_id": null, + "generator_version": null, + "left_anchor_event_id": null, + "motif_id": null, + "origin": "observed", + "reference_id": null, + "right_anchor_event_id": null, + "run_id": "run-example-v1", + "schema_version": "histdatacom.synthetic-event.v1", + "source_period": "202401", + "source_row_id": 2, + "source_series_id": "ascii:T:eurusd", + "source_version_id": "source:example-v1", + "symbol": "eurusd" + } + ], + "observed_event_count": 2, + "run_id": "run-example-v1", + "schema_version": "histdatacom.synthetic-event-stream.v1", + "source_version_ids": [ + "source:example-inputs-v1", + "source:example-v1" + ], + "stream_id": "stream:sha256:168b811195850a017958e874e43d3111bb40c05127c414f960de5d1af67881b8", + "symbol": "eurusd", + "synthetic_event_count": 1 +} diff --git a/tests/integration/test_reconstruction_handlers_real.py b/tests/integration/test_reconstruction_handlers_real.py new file mode 100644 index 00000000..2f4a731b --- /dev/null +++ b/tests/integration/test_reconstruction_handlers_real.py @@ -0,0 +1,572 @@ +"""Real-artifact closure gates for first-party reconstruction handlers. + +Set ``HISTDATACOM_REAL_RECONSTRUCTION_PLAN`` to a #465 plan artifact. These +tests never replace scientific handlers; they re-home only execution roots so +the committed products and injected failures remain isolated under pytest's +temporary directory. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, replace +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +from histdatacom import reconstruction_cli +from histdatacom.reconstruction import ( + ReconstructionClient, + read_operation_receipt, + write_execution_request, +) + +from histdatacom.orchestration.reconstruction import ( + RegisteredReconstructionStageExecutor, + ReconstructionCheckpointStore, + ReconstructionStage, + ReconstructionStageCommandV1, + ReconstructionStageInvocationV1, + ReconstructionStageOutcomeV1, + ReconstructionStageStatus, + ReconstructionWindowTaskV1, + ReconstructionWorkflowRequestV1, + run_reconstruction_window, +) +from histdatacom.runtime_contracts import ArtifactRef +from histdatacom.synthetic.contracts import canonical_contract_json +from histdatacom.synthetic.persistence import ( + ReconstructionProductManifestV2, + ReconstructionRetentionPlanV1, + discover_reconstruction_manifests, + load_reconstruction_manifest, + read_reconstruction_streams, +) +from histdatacom.synthetic.reconstruction_handlers import ( + atomic_commit_handler, + carving_handler, + cross_series_reconciliation_handler, + delivery_projection_handler, + proposal_handler, + register_first_party_reconstruction_handlers, + source_enrichment_handler, + validation_handler, +) +from histdatacom.synthetic.reconstruction_plan import ( + ReconstructionPlanResourceSummaryV1, + ReconstructionPlanExecutionManifestV1, + SyntheticInfillPlanV1, + read_reconstruction_plan_execution_manifest, + read_synthetic_infill_plan, + write_synthetic_infill_plan, +) +from histdatacom.synthetic.streaming import ( + ReconstructionCommitPhase, + ReconstructionResourceEstimateV1, + ReconstructionWindowV1, +) + +_PLAN_ENV = "HISTDATACOM_REAL_RECONSTRUCTION_PLAN" +_START_ENV = "HISTDATACOM_REAL_RECONSTRUCTION_START" + + +@dataclass(frozen=True) +class _Case: + plan: SyntheticInfillPlanV1 + execution: ReconstructionPlanExecutionManifestV1 + request: ReconstructionWorkflowRequestV1 + task: ReconstructionWindowTaskV1 + + +class _InjectedWorkerTermination(RuntimeError): + """Fault injection after validation and before the commit handler.""" + + +class _StopBeforeCommitExecutor: + def __init__(self) -> None: + self.delegate = RegisteredReconstructionStageExecutor() + + async def execute( + self, invocation: ReconstructionStageInvocationV1 + ) -> ReconstructionStageOutcomeV1: + if ( + invocation.command.stage + is ReconstructionStage.ATOMIC_PARTITION_COMMIT + ): + raise _InjectedWorkerTermination("worker terminated before receipt") + return await self.delegate.execute(invocation) + + +def test_real_triangle_is_deterministic_and_recovers_post_rename_crash( + tmp_path: Path, +) -> None: + """All seven real handlers commit deterministic queryable Parquet.""" + first = _case(tmp_path / "concurrency-1", max_parallel_windows=1) + register_first_party_reconstruction_handlers() + first_store = ReconstructionCheckpointStore( + first.request.manifest_store_root + ) + with pytest.raises(_InjectedWorkerTermination): + asyncio.run( + run_reconstruction_window( + first.request, + first.task, + checkpoint_store=first_store, + stage_executor=_StopBeforeCommitExecutor(), + ) + ) + validated = first_store.load(first.task.window) + assert validated is not None + assert validated.checkpoint.phase is ReconstructionCommitPhase.VALIDATED + assert len(validated.outcomes) == 6 + + commit_invocation = ReconstructionStageInvocationV1( + run=first.request.run, + task=first.task, + command=first.task.commands[-1], + prior_outcomes=validated.outcomes, + ) + promoted_without_receipt = atomic_commit_handler(commit_invocation) + assert ( + promoted_without_receipt.status is ReconstructionStageStatus.COMPLETED + ) + assert not Path(first.task.commands[-1].receipt_path).exists() + + recovered = asyncio.run( + run_reconstruction_window( + first.request, + first.task, + checkpoint_store=first_store, + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + assert recovered.checkpoint.phase is ReconstructionCommitPhase.COMMITTED + assert recovered.outcomes[-1].output_refs[0].metadata["idempotent_retry"] + first_manifest = _committed_manifest(recovered) + first_streams = read_reconstruction_streams( + recovered.committed_manifest_ref.path # type: ignore[union-attr] + ) + assert first_manifest.observed_event_count > 0 + assert first_manifest.synthetic_event_count > 0 + assert sum(len(stream.events) for stream in first_streams) == ( + first_manifest.event_count + ) + for outcome in recovered.outcomes: + telemetry = outcome.output_refs[0].metadata + assert telemetry["runtime_seconds"] >= 0.0 + assert telemetry["peak_rss_bytes"] > 0 + assert telemetry["scratch_bytes"] >= 0 + assert telemetry["output_bytes"] > 0 + assert "candidate_amplification" in telemetry + carving_outcome = next( + outcome + for outcome in recovered.outcomes + if outcome.stage is ReconstructionStage.CARVING + ) + carving_ref = carving_outcome.output_refs[0] + carving_manifest = json.loads( + Path(carving_ref.path).read_text(encoding="utf-8") + ) + assert carving_ref.size_bytes is not None + assert carving_ref.size_bytes < 1_000_000 + assert carving_manifest["carved_batches_inline"] is False + ledger_ref = ArtifactRef.from_dict( + carving_manifest["carved_batch_ledger_ref"] + ) + assert ledger_ref.kind == "reconstruction_carved_batch_ledger_v1" + assert ( + ledger_ref.metadata["batch_count"] + == carving_manifest["carved_batch_count"] + ) + with Path(ledger_ref.path).open("rb") as stream: + assert sum(1 for _ in stream) == carving_manifest["carved_batch_count"] + + second = _case(tmp_path / "concurrency-2", max_parallel_windows=2) + second_state = asyncio.run( + run_reconstruction_window( + second.request, + second.task, + checkpoint_store=ReconstructionCheckpointStore( + second.request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + second_manifest = _committed_manifest(second_state) + assert second_manifest.replay.logical_content_sha256 == ( + first_manifest.replay.logical_content_sha256 + ) + assert second_manifest.publication_id == first_manifest.publication_id + assert second_manifest.replay.partition_byte_sha256 == ( + first_manifest.replay.partition_byte_sha256 + ) + + +def test_real_validation_refusal_prevents_atomic_commit(tmp_path: Path) -> None: + """A schema-valid but failing qualification cannot become discoverable.""" + case = _case( + tmp_path / "negative-validation", + max_parallel_windows=1, + failing_qualification=True, + ) + register_first_party_reconstruction_handlers() + state = asyncio.run( + run_reconstruction_window( + case.request, + case.task, + checkpoint_store=ReconstructionCheckpointStore( + case.request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + assert state.checkpoint.phase is ReconstructionCommitPhase.FAILED + assert "final_validation_failed" in state.checkpoint.interruption_reason + assert len(state.outcomes) == 5 + assert discover_reconstruction_manifests(case.execution.output_root) == () + + +def test_real_cancellation_removes_partial_window_scratch_for_every_handler( + tmp_path: Path, +) -> None: + """Every handler cancels before work and removes disposable artifacts.""" + case = _case(tmp_path / "cancellation", max_parallel_windows=1) + scratch = Path(case.task.scratch_directory) + invocation = ReconstructionStageInvocationV1( + run=case.request.run, + task=case.task, + command=case.task.commands[0], + prior_outcomes=(), + cancellation_check=lambda: True, + ) + handlers = ( + source_enrichment_handler, + proposal_handler, + carving_handler, + cross_series_reconciliation_handler, + delivery_projection_handler, + validation_handler, + atomic_commit_handler, + ) + for handler in handlers: + scratch.mkdir(parents=True) + (scratch / "partial.parquet").write_bytes(b"not publishable") + with pytest.raises(asyncio.CancelledError): + handler(invocation) + assert not scratch.exists() + assert ( + discover_reconstruction_manifests(case.execution.output_root) == () + ) + + +def test_real_public_cli_and_api_execute_same_one_window_product( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Installed CLI and typed API meet at one real committed logical output.""" + case = _case(tmp_path / "public-cli-api", max_parallel_windows=1) + estimate = case.task.resource_estimate + resources = ReconstructionPlanResourceSummaryV1( + source_event_count=estimate.input_event_count, + source_size_bytes=sum( + ref.size_bytes or 0 + for ref in case.execution.artifacts.values() + if ref.kind == "histdata_ascii_tick_arrow" + ), + planned_window_count=1, + executable_window_count=1, + refused_window_count=0, + ensemble_member_count=1, + retained_member_count=1, + workflow_request_count=1, + estimated_input_event_count=estimate.input_event_count, + estimated_candidate_event_count=estimate.candidate_event_count, + estimated_candidate_bytes=estimate.estimated_scratch_bytes, + estimated_peak_memory_bytes=estimate.estimated_memory_bytes, + estimated_peak_scratch_bytes=estimate.estimated_scratch_bytes, + estimated_output_bytes=estimate.estimated_output_bytes, + estimated_partition_count=3, + ) + execution_ref = case.task.commands[0].configuration_refs[0] + public_plan = SyntheticInfillPlanV1( + run=case.plan.run, + configuration_id=case.plan.configuration_id, + execution_manifest_id=case.execution.manifest_id, + information_mode=case.plan.information_mode, + delivery_mode=case.plan.delivery_mode, + requested_start_ns=case.task.window.core_start_ns, + requested_end_ns=case.task.window.core_end_ns, + workflow_requests=(case.request,), + artifact_graph={ + "execution_manifest": execution_ref, + **case.execution.artifacts, + }, + resources=resources, + ) + plan_ref = write_synthetic_infill_plan( + public_plan, tmp_path / "public-plan" + ) + client = ReconstructionClient() + execution_request = client.create_request( + plan_ref.path, + information_mode=case.plan.information_mode, + acknowledge_scientific_nonclaim=True, + ) + request_path = write_execution_request( + execution_request, tmp_path / "execution-request.json" + ) + cli_receipt_path = tmp_path / "cli-receipt.json" + + assert ( + reconstruction_cli.main( + [ + "--json", + "run", + "--request", + str(request_path), + "--local", + "--window-id", + case.task.window.window_id, + "--receipt", + str(cli_receipt_path), + ] + ) + == 0 + ) + cli_run = json.loads(capsys.readouterr().out) + cli_receipt = read_operation_receipt(cli_receipt_path) + api_receipt = client.execute_local( + execution_request, window_id=case.task.window.window_id + ) + manifest_path = cli_receipt.reports[0].committed_manifest_refs[0].path + manifest = load_reconstruction_manifest(manifest_path) + assert isinstance(manifest, ReconstructionProductManifestV2) + + assert cli_run["status"] == "committed" + assert cli_receipt.reports == api_receipt.reports + assert ( + cli_receipt.reports[0].observed_event_count + == manifest.observed_event_count + ) + assert ( + cli_receipt.reports[0].synthetic_event_count + == manifest.synthetic_event_count + ) + + api_preview = client.preview(manifest_path, limit=100) + assert ( + reconstruction_cli.main( + [ + "--json", + "preview", + "--manifest", + manifest_path, + "--limit", + "100", + ] + ) + == 0 + ) + cli_preview = json.loads(capsys.readouterr().out) + api_replay = client.replay(manifest_path) + assert ( + reconstruction_cli.main( + ["--json", "replay", "--manifest", manifest_path] + ) + == 0 + ) + cli_replay = json.loads(capsys.readouterr().out) + + assert cli_preview == api_preview + assert cli_replay == api_replay + assert {row["origin"] for row in api_preview["rows"]} == { + "observed", + "synthetic", + } + synthetic = next( + row for row in api_preview["rows"] if row["origin"] == "synthetic" + ) + assert synthetic["generation"]["generator_id"] + assert synthetic["generation"]["confidence"] is None + assert synthetic["constraint_decision"]["decision"] == "accepted" + assert synthetic["constraint_decision"]["constraint_set_id"] + assert synthetic["lineage"]["left_anchor_event_id"] + assert synthetic["lineage"]["right_anchor_event_id"] + assert api_replay["event_count"] == manifest.event_count + assert api_replay["replay_verified"] + + +def _case( + root: Path, + *, + max_parallel_windows: int, + failing_qualification: bool = False, +) -> _Case: + plan = _real_plan() + start = _real_start_ns() + retention = ReconstructionRetentionPlanV1.from_dict( + json.loads( + Path(plan.artifact_graph["retention_plan"].path).read_text( + encoding="utf-8" + ) + ) + ) + primary = retention.primary_member_id + original = next( + task + for request in plan.workflow_requests + for task in request.tasks + if task.window.ensemble_member_id == primary + and task.window.reads_event_time(start) + ) + old_execution = read_reconstruction_plan_execution_manifest( + original.commands[0].configuration_refs[0].path + ) + artifacts = dict(old_execution.artifacts) + replacement_qualification: ArtifactRef | None = None + if failing_qualification: + replacement_qualification = _failing_qualification( + root / "artifacts", artifacts["motif_qualification"] + ) + artifacts["motif_qualification"] = replacement_qualification + execution = replace( + old_execution, + artifacts=artifacts, + output_root=str(root / "output"), + checkpoint_root=str(root / "checkpoints"), + scratch_root=str(root / "scratch"), + manifest_id="", + ) + execution_ref = _write_execution_manifest(root / "artifacts", execution) + window = ReconstructionWindowV1( + run_id=plan.run.run_id, + ensemble_member_id=primary, + symbols=plan.run.symbols, + core_start_ns=start, + core_end_ns=start + 5 * 60 * 1_000_000_000, + left_halo_ns=original.window.left_halo_ns, + right_lookahead_ns=original.window.right_lookahead_ns, + ) + scratch = root / "scratch" / window.window_id + commands = [] + for ordinal, command in enumerate(original.commands): + input_refs = command.input_manifest_refs + if replacement_qualification is not None: + input_refs = tuple( + ( + replacement_qualification + if ref.kind == replacement_qualification.kind + else ref + ) + for ref in input_refs + ) + commands.append( + ReconstructionStageCommandV1( + stage=command.stage, + handler_name=command.handler_name, + receipt_path=str( + scratch + / "receipts" + / f"{ordinal:02d}-{command.stage.value}.json" + ), + input_manifest_refs=input_refs, + configuration_refs=(execution_ref,), + ) + ) + integration_input_event_estimate = 5_000 + task = ReconstructionWindowTaskV1( + window=window, + resource_estimate=ReconstructionResourceEstimateV1( + input_event_count=integration_input_event_estimate, + candidate_event_count=int( + integration_input_event_estimate + * plan.run.storage_policy.max_candidate_amplification + ), + retained_ensemble_members=1, + inflight_batches=1, + peak_events_per_batch=1_000, + estimated_memory_bytes=16 * 1024**2, + estimated_scratch_bytes=32 * 1024**2, + estimated_output_bytes=32 * 1024**2, + estimated_batch_count=10, + ), + commands=tuple(commands), + scratch_directory=str(scratch), + ) + request = ReconstructionWorkflowRequestV1( + request_id=f"real-handler-{root.name}", + run=plan.run, + tasks=(task,), + manifest_store_root=execution.checkpoint_root, + report_root=str(root / "reports"), + task_queues={"reconstruction": "local"}, + max_parallel_windows=max_parallel_windows, + max_inflight_memory_bytes=64 * 1024**2, + ) + return _Case(plan=plan, execution=execution, request=request, task=task) + + +def _real_plan() -> SyntheticInfillPlanV1: + value = os.environ.get(_PLAN_ENV, "").strip() + if not value: + pytest.skip(f"set {_PLAN_ENV} to run qualified real-artifact gates") + path = Path(value).expanduser().resolve() + if not path.is_file(): + pytest.skip(f"qualified reconstruction plan is missing: {path}") + return read_synthetic_infill_plan(path) + + +def _real_start_ns() -> int: + value = os.environ.get(_START_ENV, "2011-01-13T00:00:00+00:00") + timestamp = datetime.fromisoformat(value) + if timestamp.tzinfo is None: + timestamp = timestamp.replace(tzinfo=timezone.utc) + return int(timestamp.timestamp() * 1_000_000_000) + + +def _write_execution_manifest( + root: Path, manifest: ReconstructionPlanExecutionManifestV1 +) -> ArtifactRef: + encoded = manifest.to_json().encode("utf-8") + b"\n" + digest = hashlib.sha256(encoded).hexdigest() + root.mkdir(parents=True, exist_ok=True) + path = root / f"reconstruction-plan-execution-{digest}.json" + path.write_bytes(encoded) + return ArtifactRef( + kind="reconstruction_plan_execution_manifest_v1", + path=str(path.resolve()), + size_bytes=len(encoded), + sha256=digest, + metadata={"manifest_id": manifest.manifest_id}, + ) + + +def _failing_qualification(root: Path, original: ArtifactRef) -> ArtifactRef: + payload: dict[str, Any] = json.loads( + Path(original.path).read_text(encoding="utf-8") + ) + payload["candidate_promotion_eligible"] = False + encoded = canonical_contract_json(payload).encode("utf-8") + b"\n" + digest = hashlib.sha256(encoded).hexdigest() + root.mkdir(parents=True, exist_ok=True) + path = root / f"modern-reference-motif-qualification-{digest}.json" + path.write_bytes(encoded) + return ArtifactRef( + kind=original.kind, + path=str(path.resolve()), + size_bytes=len(encoded), + sha256=digest, + metadata=dict(original.metadata), + ) + + +def _committed_manifest(state: Any) -> ReconstructionProductManifestV2: + assert state.committed_manifest_ref is not None + manifest = load_reconstruction_manifest(state.committed_manifest_ref.path) + assert isinstance(manifest, ReconstructionProductManifestV2) + return manifest diff --git a/tests/unit/scraper/test_scraper.py b/tests/unit/scraper/test_scraper.py index 217a6bc8..ff765bb3 100644 --- a/tests/unit/scraper/test_scraper.py +++ b/tests/unit/scraper/test_scraper.py @@ -2,6 +2,7 @@ from __future__ import annotations +import io import os import zipfile from pathlib import Path @@ -9,6 +10,7 @@ import pytest from histdatacom import config +from histdatacom.exceptions import ArchiveDownloadError from histdatacom.helper_args import helper_runtime_args from histdatacom.legacy_boundary import LegacyHelperSideEffectWarning from histdatacom.manifest_store import ManifestStatusStore @@ -129,7 +131,7 @@ def post(url, *, data, headers, timeout): # noqa:ANN001 captured.append(headers) return Response() - monkeypatch.setattr("histdatacom.scraper.scraper.requests.post", post) + monkeypatch.setattr("histdatacom.activity_stages.requests.post", post) record = Record( url=ASCII_TICK_URL, data_tk="token", @@ -138,6 +140,7 @@ def post(url, *, data, headers, timeout): # noqa:ANN001 data_format="ASCII", data_timeframe="T", data_fxpair="eurusd", + data_dir="/tmp/", ) Scraper._request_file(record, 1) @@ -147,6 +150,63 @@ def post(url, *, data, headers, timeout): # noqa:ANN001 assert captured[0]["Referer"] == ASCII_TICK_URL +def test_request_file_rejects_unsupported_raw_dimensions(monkeypatch) -> None: + """The legacy private request seam must fail before network access.""" + monkeypatch.setattr( + "histdatacom.activity_stages.requests.post", + lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("unsupported input reached network") + ), + ) + record = Record( + url=ASCII_TICK_URL, + data_tk="token", + data_date="2022", + data_datemonth="202201", + data_format="METATRADER", + data_timeframe="T", + data_fxpair="eurusd", + data_dir="/tmp", + ) + + with pytest.raises(ArchiveDownloadError) as exc_info: + Scraper._request_file(record, 1) + + assert exc_info.value.code == "UNSUPPORTED_RAW_INPUT" + + +def test_write_file_rejects_retired_archive_member(tmp_path: Path) -> None: + """The legacy private writer must not persist platform raw payloads.""" + record = Record( + url=ASCII_TICK_URL, + data_dir=f"{tmp_path}{os.sep}", + zip_filename="opaque.zip", + data_format="ASCII", + data_timeframe="T", + ) + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("DAT_NT_EURUSD_T_LAST_2022.csv", "rows") + + with pytest.raises(ArchiveDownloadError) as exc_info: + Scraper._write_file(record, stream.getvalue()) + + assert exc_info.value.code == "UNSUPPORTED_RAW_INPUT" + assert not (tmp_path / record.zip_filename).exists() + + record.data_format = "METATRADER" + record.zip_filename = "DAT_ASCII_EURUSD_T_2022.zip" + supported_stream = io.BytesIO() + with zipfile.ZipFile(supported_stream, "w") as archive: + archive.writestr("DAT_ASCII_EURUSD_T_2022.csv", "rows") + + with pytest.raises(ArchiveDownloadError) as exc_info: + Scraper._write_file(record, supported_stream.getvalue()) + + assert exc_info.value.code == "UNSUPPORTED_RAW_INPUT" + assert not (tmp_path / record.zip_filename).exists() + + def test_download_zip_transitions_valid_record_to_csv_zip( tmp_path: Path, ) -> None: @@ -159,6 +219,8 @@ def test_download_zip_transitions_valid_record_to_csv_zip( status=WorkStatus.URL_VALID.value, data_dir=f"{data_dir}{os.sep}", zip_filename="DAT_ASCII_EURUSD_T_202201.zip", + data_format="ASCII", + data_timeframe="T", ) with zipfile.ZipFile(data_dir / record.zip_filename, "w") as archive: archive.writestr("DAT_ASCII_EURUSD_T_202201.csv", "rows") diff --git a/tests/unit/scraper/test_urls.py b/tests/unit/scraper/test_urls.py index 6eb106d7..df33bd17 100644 --- a/tests/unit/scraper/test_urls.py +++ b/tests/unit/scraper/test_urls.py @@ -26,6 +26,24 @@ def test_generate_form_urls_rejects_removed_m1_timeframe() -> None: raise AssertionError("M1 URL generation should fail") +def test_generate_form_urls_rejects_removed_platform_format() -> None: + """Platform-specific formats must not reach private URL generation.""" + try: + list( + Urls().generate_form_urls( + "202201", + "202203", + {"metatrader"}, + {"eurusd"}, + {"T"}, + ) + ) + except ValueError as exc: + assert "metatrader" in str(exc) + else: + raise AssertionError("platform URL generation should fail") + + def test_generate_form_urls_preserves_tick_month_units() -> None: """Tick data ranges should generate one URL per month.""" urls = list( diff --git a/tests/unit/test_activity_stages.py b/tests/unit/test_activity_stages.py index 969dea14..da2b267d 100644 --- a/tests/unit/test_activity_stages.py +++ b/tests/unit/test_activity_stages.py @@ -13,15 +13,22 @@ import pytest import requests -from histdatacom.exceptions import HistDataNoDataError, UrlValidationError +from histdatacom.exceptions import ( + CacheBuildError, + HistDataNoDataError, + UrlValidationError, +) from histdatacom.activity_stages import ( UrlPageData, build_cache_work_item, + create_cache_file, dataset_plan_stage, download_archive_work_item, + emit_influx_cache_batches, extract_csv_work_item, fetch_histdata_page_data, import_to_influx_work_item, + merge_cache_items, merge_cache_work_items, parse_histdata_form_metadata, read_repository_data_file, @@ -30,16 +37,26 @@ validate_url_work_item, write_repository_data_file, ) +from histdatacom.data_quality.training_features import ( + required_training_feature_columns, +) from histdatacom.histdata_ascii import ( CACHE_FILENAME, LEGACY_CACHE_ERROR, convert_polars_datetime_to_utc_ms, + read_polars_cache, read_ascii_file_to_polars, write_polars_cache, ) from histdatacom.manifest_store import ManifestStatusStore from histdatacom.records import Record from histdatacom.runtime_contracts import WorkItem, WorkStatus, derive_work_id +from histdatacom.random_windows import ( + RANDOM_WINDOW_SELECTION_METADATA_KEY, + RandomWindowEmptySelectionError, + RandomWindowSelectionV1, + random_window_planning_yearmonths, +) FIXTURES = Path(__file__).parents[1] / "fixtures" / "histdata_ascii" ASCII_TICK_URL = ( @@ -97,7 +114,25 @@ def _zip_bytes(filename: str = "DAT_ASCII_EURUSD_T_202201.csv") -> bytes: return stream.getvalue() -def _form_html(*, token: str = "token") -> str: +def _live_zip_bytes() -> bytes: + """Return the CSV plus same-stem status report used by live HistData.""" + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("DAT_ASCII_EURUSD_T_202201.csv", "rows") + archive.writestr( + "DAT_ASCII_EURUSD_T_202201.txt", + "HistData.com status report", + ) + return stream.getvalue() + + +def _form_html( + *, + token: str = "token", + platform: str = "ASCII", + timeframe: str = "T", + pair: str = "eurusd", +) -> str: """Return a minimal HistData download form.""" return f""" @@ -105,9 +140,9 @@ def _form_html(*, token: str = "token") -> str: """ @@ -173,6 +208,91 @@ def test_validate_url_work_item_parses_form_metadata( assert output.result.metrics["encoding"] == "gzip" +@pytest.mark.parametrize( + ("platform", "timeframe"), + (("NINJATRADER", "T"), ("ASCII", "M1")), +) +def test_validate_url_work_item_rejects_unsupported_form_dimensions( + tmp_path: Path, + platform: str, + timeframe: str, +) -> None: + """Server form metadata must not reopen retired raw dimensions.""" + record = Record(url=ASCII_TICK_URL, status=WorkStatus.URL_NEW.value) + + output = validate_url_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + fetch_page_data=lambda url, timeout: UrlPageData( + html=_form_html(platform=platform, timeframe=timeframe), + encoding="gzip", + bytes_length="123", + headers={}, + ), + ) + + assert not output.forward + assert output.work_item.status is WorkStatus.FAILED + assert output.result.failure is not None + assert output.result.failure.code == "UNSUPPORTED_RAW_INPUT" + assert output.result.failure.detail["data_format"] == platform + assert output.result.failure.detail["timeframe"] == timeframe + assert ManifestStatusStore(tmp_path).list_work_items() == () + + +def test_validate_url_work_item_rejects_form_dimension_mismatch( + tmp_path: Path, +) -> None: + """Returned form axes must still identify the planned raw target.""" + record = Record( + url=ASCII_TICK_URL, + status=WorkStatus.URL_NEW.value, + data_format="ASCII", + data_timeframe="T", + data_fxpair="eurusd", + ) + + output = validate_url_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + fetch_page_data=lambda url, timeout: UrlPageData( + html=_form_html(pair="gbpusd"), + encoding="gzip", + bytes_length="123", + headers={}, + ), + ) + + assert not output.forward + assert output.result.failure is not None + assert output.result.failure.code == "FORM_DIMENSION_MISMATCH" + assert output.result.failure.detail["mismatched_fields"] == ["fxpair"] + + +def test_validate_url_work_item_rejects_unsupported_legacy_scrape( + tmp_path: Path, +) -> None: + """Injected legacy scrapers must pass the same raw-input gate.""" + record = Record(url=ASCII_TICK_URL, status=WorkStatus.URL_NEW.value) + + def scrape(record_: Record) -> Record: + record_.data_tk = "token" + record_.data_format = "METATRADER" + record_.data_timeframe = "T" + return record_ + + output = validate_url_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + scrape_record_info=scrape, + check_for_valid_download=lambda record_: None, + ) + + assert not output.forward + assert output.result.failure is not None + assert output.result.failure.code == "UNSUPPORTED_RAW_INPUT" + + def test_parse_form_metadata_ignores_inputs_outside_download_form() -> None: """Only form#file_down values should be used for archive metadata.""" metadata = parse_histdata_form_metadata( @@ -349,6 +469,8 @@ def test_download_archive_work_item_returns_zip_artifact( url=ASCII_TICK_URL, status=WorkStatus.URL_VALID.value, data_dir=f"{data_dir}{os.sep}", + data_format="ascii", + data_timeframe="T", ) def download(record_: Record) -> None: @@ -422,6 +544,41 @@ def post(url: str, *, data, headers, timeout): # noqa:ANN001 assert output.result.metrics["decision"] == "downloaded" +def test_download_archive_work_item_rejects_retired_response_filename( + tmp_path: Path, +) -> None: + """A retired platform filename must fail before the ZIP is persisted.""" + record = Record( + url=ASCII_TICK_URL, + status=WorkStatus.URL_VALID.value, + data_dir=f"{tmp_path}{os.sep}", + data_tk="token", + data_date="2022", + data_datemonth="2022", + data_format="ASCII", + data_timeframe="T", + data_fxpair="eurusd", + ) + filename = "HISTDATA_COM_NT_EURUSD_T_LAST2022.zip" + + output = download_archive_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + post_archive=lambda *args, **kwargs: _FakeResponse( + headers={ + "Content-Disposition": f'attachment; filename="{filename}"', + }, + content=_zip_bytes("DAT_NT_EURUSD_T_LAST_2022.csv"), + ), + ) + + assert not output.forward + assert output.result.failure is not None + assert output.result.failure.code == "UNSUPPORTED_RAW_INPUT" + assert output.result.failure.detail["filename"] == filename + assert not (tmp_path / filename).exists() + + def test_download_archive_work_item_reuses_existing_zip( tmp_path: Path, ) -> None: @@ -435,6 +592,8 @@ def test_download_archive_work_item_reuses_existing_zip( status=WorkStatus.URL_VALID.value, data_dir=f"{data_dir}{os.sep}", zip_filename=zip_path.name, + data_format="ascii", + data_timeframe="T", ) output = download_archive_work_item( @@ -464,6 +623,8 @@ def test_download_archive_work_item_reuses_existing_csv( status=WorkStatus.URL_VALID.value, data_dir=f"{data_dir}{os.sep}", csv_filename=csv_path.name, + data_format="ascii", + data_timeframe="T", ) output = download_archive_work_item( @@ -490,6 +651,8 @@ def test_download_archive_work_item_reuses_existing_cache( status=WorkStatus.URL_VALID.value, data_dir=f"{data_dir}{os.sep}", cache_filename=cache_path.name, + data_format="ascii", + data_timeframe="T", ) output = download_archive_work_item( @@ -503,6 +666,35 @@ def test_download_archive_work_item_reuses_existing_cache( assert output.result.artifacts[0].kind == "cache" +def test_download_archive_work_item_rejects_unsupported_before_reuse( + tmp_path: Path, +) -> None: + """Retired dimensions must not reuse artifacts or invoke downloaders.""" + zip_path = tmp_path / "retired.zip" + zip_path.write_bytes(_zip_bytes()) + record = Record( + url=ASCII_TICK_URL, + status=WorkStatus.URL_VALID.value, + data_dir=f"{tmp_path}{os.sep}", + zip_filename=zip_path.name, + data_format="METATRADER", + data_timeframe="T", + ) + + output = download_archive_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + download_file=lambda record_: (_ for _ in ()).throw( + AssertionError("unsupported input reached downloader") + ), + ) + + assert not output.forward + assert output.result.failure is not None + assert output.result.failure.code == "UNSUPPORTED_RAW_INPUT" + assert zip_path.exists() + + def test_download_archive_work_item_network_failure_is_retried( tmp_path: Path, ) -> None: @@ -643,6 +835,8 @@ def test_extract_csv_work_item_extracts_data_member(tmp_path: Path) -> None: data_dir=f"{tmp_path}{os.sep}", zip_filename=archive_path.name, status=WorkStatus.CSV_ZIP.value, + data_format="ascii", + data_timeframe="T", ) output = extract_csv_work_item( @@ -665,6 +859,71 @@ def test_extract_csv_work_item_extracts_data_member(tmp_path: Path) -> None: assert record.status is WorkStatus.CSV_ZIP +def test_download_archive_allows_live_status_report_member( + tmp_path: Path, +) -> None: + """The vendor's same-stem TXT report is metadata, not a retired axis.""" + data_dir = tmp_path / "ASCII" / "T" / "eurusd" / "2022" / "1" + record = Record( + url=ASCII_TICK_URL, + status=WorkStatus.URL_VALID.value, + data_dir=f"{data_dir}{os.sep}", + data_tk="token", + data_date="2022", + data_datemonth="202201", + data_format="ASCII", + data_timeframe="T", + data_fxpair="eurusd", + ) + + output = download_archive_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + post_archive=lambda *args, **kwargs: _FakeResponse( + headers={ + "Content-Disposition": ( + "attachment; " + "filename=HISTDATA_COM_ASCII_EURUSD_T202201.zip" + ), + }, + content=_live_zip_bytes(), + ), + ) + + assert output.forward + assert output.work_item.status is WorkStatus.CSV_ZIP + archive_path = data_dir / output.work_item.zip_filename + assert archive_path.exists() + with zipfile.ZipFile(archive_path) as archive: + assert sorted(archive.namelist()) == [ + "DAT_ASCII_EURUSD_T_202201.csv", + "DAT_ASCII_EURUSD_T_202201.txt", + ] + + +def test_extract_csv_ignores_live_status_report_member(tmp_path: Path) -> None: + """Extraction selects the CSV while leaving vendor metadata in the ZIP.""" + archive_path = tmp_path / "live.zip" + archive_path.write_bytes(_live_zip_bytes()) + record = Record( + data_dir=f"{tmp_path}{os.sep}", + zip_filename=archive_path.name, + status=WorkStatus.CSV_ZIP.value, + data_format="ascii", + data_timeframe="T", + ) + + output = extract_csv_work_item( + WorkItem.from_record(record), + args={**_args(tmp_path), "zip_persist": True}, + ) + + assert output.forward + assert output.work_item.csv_filename == "DAT_ASCII_EURUSD_T_202201.csv" + assert (tmp_path / output.work_item.csv_filename).read_bytes() == b"rows" + assert archive_path.exists() + + def test_extract_csv_work_item_preserves_zip_when_configured( tmp_path: Path, ) -> None: @@ -676,6 +935,8 @@ def test_extract_csv_work_item_preserves_zip_when_configured( data_dir=f"{tmp_path}{os.sep}", zip_filename=archive_path.name, status=WorkStatus.CSV_ZIP.value, + data_format="ascii", + data_timeframe="T", ) output = extract_csv_work_item( @@ -702,6 +963,8 @@ def test_extract_csv_work_item_reuses_existing_csv( zip_filename=archive_path.name, csv_filename=csv_path.name, status=WorkStatus.CSV_ZIP.value, + data_format="ascii", + data_timeframe="T", ) output = extract_csv_work_item( @@ -728,6 +991,8 @@ def test_extract_csv_work_item_malformed_archive_is_failed( data_dir=f"{tmp_path}{os.sep}", zip_filename=archive_path.name, status=WorkStatus.CSV_ZIP.value, + data_format="ascii", + data_timeframe="T", ) output = extract_csv_work_item( @@ -752,6 +1017,8 @@ def test_extract_csv_work_item_bad_zip_is_failed(tmp_path: Path) -> None: data_dir=f"{tmp_path}{os.sep}", zip_filename=archive_path.name, status=WorkStatus.CSV_ZIP.value, + data_format="ascii", + data_timeframe="T", ) output = extract_csv_work_item( @@ -765,6 +1032,61 @@ def test_extract_csv_work_item_bad_zip_is_failed(tmp_path: Path) -> None: assert output.result.failure.code == "INVALID_ARCHIVE_PAYLOAD" +def test_extract_csv_work_item_rejects_unsupported_before_writes( + tmp_path: Path, +) -> None: + """Direct extraction must fail before retired raw data reaches disk.""" + archive_path = tmp_path / "retired.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("DAT_ASCII_EURUSD_T_2022.csv", b"rows") + record = Record( + data_dir=f"{tmp_path}{os.sep}", + zip_filename=archive_path.name, + status=WorkStatus.CSV_ZIP.value, + data_format="NINJATRADER", + data_timeframe="T", + ) + + output = extract_csv_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + ) + + assert not output.forward + assert output.result.failure is not None + assert output.result.failure.code == "UNSUPPORTED_RAW_INPUT" + assert not (tmp_path / "DAT_ASCII_EURUSD_T_2022.csv").exists() + assert archive_path.exists() + + +def test_extract_csv_work_item_rejects_retired_archive_member( + tmp_path: Path, +) -> None: + """Supported record metadata must not mask a retired ZIP member.""" + archive_path = tmp_path / "opaque.zip" + member = "DAT_NT_EURUSD_T_LAST_2022.csv" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr(member, b"rows") + record = Record( + data_dir=f"{tmp_path}{os.sep}", + zip_filename=archive_path.name, + status=WorkStatus.CSV_ZIP.value, + data_format="ascii", + data_timeframe="T", + ) + + output = extract_csv_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + ) + + assert not output.forward + assert output.result.failure is not None + assert output.result.failure.code == "UNSUPPORTED_RAW_INPUT" + assert output.result.failure.detail["filename"] == member + assert not (tmp_path / member).exists() + + def test_build_cache_work_item_writes_cache_from_csv( tmp_path: Path, ) -> None: @@ -804,12 +1126,14 @@ def test_build_cache_work_item_writes_cache_from_csv( assert output.result.metrics["start"] == str(EXPECTED_TICK_DATETIMES[0]) assert output.result.metrics["end"] == str(EXPECTED_TICK_DATETIMES[-1]) assert output.result.metrics["timeframe"] == "T" - assert output.result.metrics["schema"] == { + schema = output.result.metrics["schema"] + assert { "datetime": "Int64", "bid": "Float64", "ask": "Float64", "vol": "Int32", - } + }.items() <= schema.items() + assert set(required_training_feature_columns()).issubset(schema) assert output.result.artifacts[0].path == str(tmp_path / CACHE_FILENAME) assert output.result.artifacts[0].sha256 @@ -922,6 +1246,75 @@ def test_build_cache_work_item_invalid_source_is_failed( assert not (tmp_path / CACHE_FILENAME).exists() +def test_build_cache_work_item_noops_unsupported_raw_dimensions( + tmp_path: Path, +) -> None: + """Cache planning should clearly no-op retired raw dimensions.""" + source = tmp_path / "retired.csv" + source.write_text("rows", encoding="utf-8") + record = Record( + data_dir=f"{tmp_path}{os.sep}", + csv_filename=source.name, + data_format="ascii", + data_timeframe="M1", + status=WorkStatus.CSV_FILE.value, + ) + + output = build_cache_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + ) + + assert output.forward + assert output.result.status is WorkStatus.SKIPPED + assert output.result.metrics["decision"] == "skipped_unsupported" + assert not (tmp_path / CACHE_FILENAME).exists() + + +def test_create_cache_file_rejects_unsupported_raw_dimensions( + tmp_path: Path, +) -> None: + """The private cache writer must enforce the raw-input contract too.""" + filename = "DAT_ASCII_EURUSD_T_201202.csv" + shutil.copyfile(FIXTURES / filename, tmp_path / filename) + record = Record( + data_dir=f"{tmp_path}{os.sep}", + csv_filename=filename, + data_format="metatrader", + data_timeframe="T", + ) + + with pytest.raises(CacheBuildError) as exc_info: + create_cache_file(record, _args(tmp_path)) + + assert exc_info.value.code == "UNSUPPORTED_RAW_INPUT" + assert not (tmp_path / CACHE_FILENAME).exists() + + +def test_create_cache_file_rejects_retired_source_filename( + tmp_path: Path, +) -> None: + """Supported record metadata must not mask a retired cache source.""" + filename = "DAT_NT_EURUSD_T_LAST_201202.csv" + shutil.copyfile( + FIXTURES / "DAT_ASCII_EURUSD_T_201202.csv", + tmp_path / filename, + ) + record = Record( + data_dir=f"{tmp_path}{os.sep}", + csv_filename=filename, + data_format="ascii", + data_timeframe="T", + ) + + with pytest.raises(CacheBuildError) as exc_info: + create_cache_file(record, _args(tmp_path)) + + assert exc_info.value.code == "UNSUPPORTED_RAW_INPUT" + assert exc_info.value.detail["filename"] == filename + assert not (tmp_path / CACHE_FILENAME).exists() + + def test_merge_cache_work_items_uses_explicit_inputs(tmp_path: Path) -> None: """Cache merge should not read queue globals.""" frame = _tick_frame() @@ -939,6 +1332,7 @@ def test_merge_cache_work_items_uses_explicit_inputs(tmp_path: Path) -> None: cache_start=str(EXPECTED_TICK_DATETIMES[0]), cache_end=str(EXPECTED_TICK_DATETIMES[0]), data_fxpair="eurusd", + data_format="ascii", data_timeframe="T", ) ) @@ -950,6 +1344,7 @@ def test_merge_cache_work_items_uses_explicit_inputs(tmp_path: Path) -> None: cache_start=str(EXPECTED_TICK_DATETIMES[1]), cache_end=str(EXPECTED_TICK_DATETIMES[2]), data_fxpair="eurusd", + data_format="ascii", data_timeframe="T", ) ) @@ -988,6 +1383,7 @@ def test_merge_cache_work_items_can_skip_materialization( cache_start=str(EXPECTED_TICK_DATETIMES[0]), cache_end=str(EXPECTED_TICK_DATETIMES[0]), data_fxpair="eurusd", + data_format="ascii", data_timeframe="T", ) ) @@ -999,6 +1395,7 @@ def test_merge_cache_work_items_can_skip_materialization( cache_start=str(EXPECTED_TICK_DATETIMES[1]), cache_end=str(EXPECTED_TICK_DATETIMES[2]), data_fxpair="eurusd", + data_format="ascii", data_timeframe="T", ) ) @@ -1017,6 +1414,126 @@ def test_merge_cache_work_items_can_skip_materialization( assert "data" not in payload +def test_merge_cache_items_filters_projection_without_mutating_cache( + tmp_path: Path, +) -> None: + """API projection should honor [start,end) and leave monthly evidence intact.""" + import polars as pl + + start = 1_704_096_000_000 + end = start + 90 * 60_000 + frame = pl.DataFrame( + { + "datetime": (start - 1, start, end - 1, end), + "bid": (1.1, 1.2, 1.3, 1.4), + "ask": (1.2, 1.3, 1.4, 1.5), + "volume": (0.0, 0.0, 0.0, 0.0), + } + ) + cache_path = tmp_path / CACHE_FILENAME + write_polars_cache(frame, cache_path) + selection = RandomWindowSelectionV1( + expression="90m", + mode="random", + support_start_utc_ms=start - 1, + support_end_utc_ms=end + 1, + seed=12, + selected_start_utc_ms=start, + selected_end_utc_ms=end, + ) + item = WorkItem.from_record( + Record( + data_dir=f"{tmp_path}{os.sep}", + cache_filename=CACHE_FILENAME, + data_format="ascii", + data_timeframe="T", + data_fxpair="eurusd", + ) + ) + + merged = merge_cache_items( + [item], + return_type="polars", + random_selection=selection, + ) + + assert merged["datetime"].to_list() == [start, end - 1] + assert read_polars_cache(cache_path).height == 4 + + +def test_merge_cache_items_fails_on_empty_selected_projection( + tmp_path: Path, +) -> None: + """A resolved interval without ticks should fail instead of substituting data.""" + write_polars_cache(_tick_frame(), tmp_path / CACHE_FILENAME) + selection = RandomWindowSelectionV1( + expression="1m", + mode="random", + support_start_utc_ms=1_704_096_000_000, + support_end_utc_ms=1_704_096_120_000, + seed=9, + selected_start_utc_ms=1_704_096_000_000, + selected_end_utc_ms=1_704_096_060_000, + ) + item = WorkItem.from_record( + Record( + data_dir=f"{tmp_path}{os.sep}", + cache_filename=CACHE_FILENAME, + data_format="ascii", + data_timeframe="T", + data_fxpair="eurusd", + ) + ) + + with pytest.raises(RandomWindowEmptySelectionError, match="no cache rows"): + merge_cache_items( + [item], + return_type="polars", + random_selection=selection, + ) + + +def test_merge_cache_work_items_noops_unsupported_raw_dimensions( + tmp_path: Path, +) -> None: + """Existing retired cache metadata must not be materialized by the API.""" + write_polars_cache(_tick_frame(), tmp_path / CACHE_FILENAME) + retired = WorkItem.from_record( + Record( + data_dir=f"{tmp_path}{os.sep}", + cache_filename=CACHE_FILENAME, + data_format="ninjatrader", + data_timeframe="T", + data_fxpair="eurusd", + ) + ) + + output = merge_cache_work_items([retired], return_type="polars") + + assert output.result.status is WorkStatus.SKIPPED + assert output.result.metrics["record_count"] == 0 + assert output.result.metrics["unsupported_count"] == 1 + assert output.data == [] + + +def test_merge_cache_items_rejects_unsupported_private_input( + tmp_path: Path, +) -> None: + """The private materializer must not bypass the merge-stage filter.""" + write_polars_cache(_tick_frame(), tmp_path / CACHE_FILENAME) + retired = WorkItem.from_record( + Record( + data_dir=f"{tmp_path}{os.sep}", + cache_filename=CACHE_FILENAME, + data_format="ascii", + data_timeframe="M1", + ) + ) + + with pytest.raises(ValueError, match="ASCII tick"): + merge_cache_items([retired], return_type="polars") + + def test_import_to_influx_work_item_emits_batches_without_writer( tmp_path: Path, ) -> None: @@ -1042,7 +1559,117 @@ def test_import_to_influx_work_item_emits_batches_without_writer( assert output.result.status is WorkStatus.INFLUX_UPLOAD assert output.result.metrics == {"batch_count": 2, "line_count": 3} assert [len(batch) for batch in emitted] == [2, 1] - assert emitted[0][0] == EXPECTED_TICK_LINE + first_line = emitted[0][0] + assert "row_id=1" in first_line.split(" ", maxsplit=1)[0] + assert "bidquote=1.3066" in first_line + assert "askquote=1.30677" in first_line + assert "quality_status_code=0i" in first_line + assert "training_usable=true" in first_line + assert first_line.endswith(" 1328072403660") + + +def test_influx_projection_filters_before_augmentation_and_preserves_metadata( + tmp_path: Path, +) -> None: + """Influx should emit only selected ticks and carry selection through status.""" + import polars as pl + + start = 1_704_096_000_000 + end = start + 60_000 + frame = pl.DataFrame( + { + "datetime": (start - 1, start, end), + "bid": (1.1, 1.2, 1.3), + "ask": (1.2, 1.3, 1.4), + "volume": (0.0, 0.0, 0.0), + } + ) + write_polars_cache(frame, tmp_path / CACHE_FILENAME) + selection = RandomWindowSelectionV1( + expression="1m", + mode="random", + support_start_utc_ms=start - 1, + support_end_utc_ms=end + 1, + seed=2, + selected_start_utc_ms=start, + selected_end_utc_ms=end, + ) + item = WorkItem.from_record( + Record( + data_dir=f"{tmp_path}{os.sep}", + cache_filename=CACHE_FILENAME, + data_format="ascii", + data_timeframe="T", + data_fxpair="eurusd", + status=WorkStatus.CACHE_READY.value, + ) + ) + item = WorkItem.from_dict( + { + **item.to_dict(), + "metadata": { + RANDOM_WINDOW_SELECTION_METADATA_KEY: selection.to_dict() + }, + } + ) + emitted: list[list[str]] = [] + + output = import_to_influx_work_item( + item, + args=_args(tmp_path), + emit_lines=emitted.append, + ) + + assert output.result.metrics["line_count"] == 1 + assert len(emitted) == 1 + assert emitted[0][0].endswith(f" {start}") + assert output.work_item.metadata == item.metadata + assert read_polars_cache(tmp_path / CACHE_FILENAME).height == 3 + + +def test_import_to_influx_work_item_noops_unsupported_raw_dimensions( + tmp_path: Path, +) -> None: + """Influx projection must not mark retired raw dimensions uploaded.""" + emitted: list[list[str]] = [] + record = Record( + data_dir=f"{tmp_path}{os.sep}", + data_format="metatrader", + data_timeframe="T", + data_fxpair="eurusd", + status=WorkStatus.CACHE_READY.value, + ) + + output = import_to_influx_work_item( + WorkItem.from_record(record), + args=_args(tmp_path), + emit_lines=emitted.append, + ) + + assert output.result.status is WorkStatus.SKIPPED + assert output.work_item.status is WorkStatus.CACHE_READY + assert output.result.metrics["decision"] == "skipped_unsupported" + assert emitted == [] + + +def test_emit_influx_cache_batches_rejects_unsupported_private_input( + tmp_path: Path, +) -> None: + """The private Influx batch emitter must enforce the same contract.""" + retired = WorkItem.from_record( + Record( + data_dir=f"{tmp_path}{os.sep}", + data_format="ninjatrader", + data_timeframe="T", + ) + ) + + with pytest.raises(ValueError, match="ASCII tick"): + emit_influx_cache_batches( + retired, + args=_args(tmp_path), + emit_lines=lambda lines: None, + ) def test_dataset_plan_stage_emits_stable_historical_tick_work_items( @@ -1169,6 +1796,70 @@ def test_dataset_plan_stage_uses_repository_ranges_for_full_scope() -> None: } +def test_dataset_plan_stage_plans_only_seeded_window_months_with_metadata() -> ( + None +): + """Random selection should use common support and only intersecting months.""" + output = dataset_plan_stage( + start_yearmonth="", + end_yearmonth="", + formats=("ascii",), + pairs=("eurusd", "gbpusd"), + timeframes=("T",), + current_yearmonth="202606", + repository_ranges={ + "eurusd": {"start": "201001", "end": "202412"}, + "gbpusd": {"start": "201501", "end": "202311"}, + }, + random_window="40d", + random_seed=8675309, + ) + selection = RandomWindowSelectionV1.from_dict( + output.result.metrics["random_window_selection"] # type: ignore[arg-type] + ) + start, end = random_window_planning_yearmonths(selection) + + assert selection.support_start_utc_ms == 1_420_070_400_000 + assert selection.support_end_utc_ms == 1_701_388_800_000 + planned_periods = sorted( + {item.data_datemonth for item in output.work_items} + ) + assert planned_periods[0] == start + assert planned_periods[-1] == end + assert len(output.work_items) == len(planned_periods) * 2 + assert {item.data_fxpair for item in output.work_items} == { + "eurusd", + "gbpusd", + } + assert all( + item.metadata[RANDOM_WINDOW_SELECTION_METADATA_KEY] + == selection.to_dict() + for item in output.work_items + ) + + +def test_dataset_plan_stage_bounded_sessions_plan_all_occurrences() -> None: + """A doubly bounded session expression should plan its whole support month.""" + output = dataset_plan_stage( + start_yearmonth="202401", + end_yearmonth="202401", + formats=("ascii",), + pairs=("eurusd",), + timeframes=("T",), + current_yearmonth="202606", + repository_ranges={"eurusd": {"start": "201001", "end": "202412"}}, + random_window="ldn", + random_seed=None, + ) + selection = RandomWindowSelectionV1.from_dict( + output.work_items[0].metadata[RANDOM_WINDOW_SELECTION_METADATA_KEY] # type: ignore[arg-type] + ) + + assert [item.data_datemonth for item in output.work_items] == ["202401"] + assert selection.mode == "all_sessions" + assert selection.occurrence_count == 23 + + def test_dataset_plan_stage_explicit_range_overrides_repository_ranges() -> ( None ): diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index abdda79a..b92c4dc1 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -39,6 +39,7 @@ def _write_cache_record( cache_filename=CACHE_FILENAME, cache_start=str(start), data_fxpair=pair, + data_format="ascii", data_timeframe=timeframe, ) @@ -98,6 +99,20 @@ def test_import_frame_with_headers_rejects_unsupported_timeframes() -> None: ) +def test_import_frame_with_headers_rejects_retired_platform_filename( + tmp_path: Path, +) -> None: + """The private API import seam must reject platform raw filenames.""" + from histdatacom.api import Api + + source = FIXTURES / "DAT_ASCII_EURUSD_T_201202.csv" + retired = tmp_path / "DAT_NT_EURUSD_T_LAST_201202.csv" + shutil.copyfile(source, retired) + + with pytest.raises(ValueError, match="ASCII tick"): + Api._import_frame_with_headers("T", retired) + + def test_import_file_to_polars_wraps_raw_ingest_for_records() -> None: """Record-based ingest should normalize raw CSV timestamps in Polars.""" import polars as pl @@ -304,6 +319,30 @@ def test_merge_records_accepts_explicit_records_without_queue( ) +def test_merge_records_declared_random_window_requires_resolved_selection( + tmp_path: Path, +) -> None: + """The API helper must not fall back to a complete cache after metadata loss.""" + from histdatacom.api import Api + from histdatacom.random_windows import RandomWindowError + + source = Api._import_file_to_polars( + SimpleNamespace(data_timeframe="T"), + FIXTURES / "DAT_ASCII_EURUSD_T_201202.csv", + ) + record = _write_cache_record( + tmp_path, + "missing-random-selection", + source, + pair="eurusd", + timeframe="T", + start=EXPECTED_TICK_DATETIMES[0], + ) + + with pytest.raises(RandomWindowError, match="resolved selection"): + Api(args={"random_window": "1m"}).merge_records([record]) + + def test_merge_records_uses_explicit_return_type_without_parser_globals( tmp_path: Path, ) -> None: @@ -401,6 +440,86 @@ def test_merge_caches_converts_single_pair_timeframe_return_types( assert values == EXPECTED_TICK_DATETIMES +@pytest.mark.parametrize("api_return_type", ("polars", "pandas", "arrow")) +def test_merge_caches_appends_timezone_aware_output_projection( + tmp_path: Path, + api_return_type: str, +) -> None: + """Every API backend should preserve UTC millis and expose local wall time.""" + from datetime import datetime, timezone + + import pandas as pd + import polars as pl + import pyarrow as pa + + from histdatacom.api import Api + + utc_millis = [ + int( + datetime(2024, 3, 10, hour, 30, tzinfo=timezone.utc).timestamp() + * 1000 + ) + for hour in (6, 7) + ] + source = pl.DataFrame( + { + "datetime": utc_millis, + "bid": [1.1, 1.2], + "ask": [1.1001, 1.2001], + "vol": [0, 0], + "timestamp_utc_ms": utc_millis, + } + ) + record = _write_cache_record( + tmp_path, + "dst", + source, + pair="eurusd", + timeframe="T", + start=utc_millis[0], + ) + + result = Api( + args={"output_timezone": "America/New_York"}, + return_type=api_return_type, + ).merge_records([record]) + + expected_local = [ + "2024-03-10T01:30:00-05:00", + "2024-03-10T03:30:00-04:00", + ] + if api_return_type == "polars": + assert isinstance(result, pl.DataFrame) + assert result.schema["datetime_local"] == pl.Datetime( + "ms", "America/New_York" + ) + canonical = result["datetime"].to_list() + timestamp_utc_ms = result["timestamp_utc_ms"].to_list() + localized = result["datetime_local"].to_list() + elif api_return_type == "pandas": + assert isinstance(result, pd.DataFrame) + assert str(result["datetime_local"].dt.tz) == "America/New_York" + canonical = result["datetime"].tolist() + timestamp_utc_ms = result["timestamp_utc_ms"].tolist() + localized = result["datetime_local"].tolist() + else: + assert isinstance(result, pa.Table) + assert result.schema.field("datetime_local").type == pa.timestamp( + "ms", tz="America/New_York" + ) + canonical = result.column("datetime").to_pylist() + timestamp_utc_ms = result.column("timestamp_utc_ms").to_pylist() + localized = result.column("datetime_local").to_pylist() + + assert canonical == utc_millis + assert timestamp_utc_ms == utc_millis + assert [value.isoformat() for value in localized] == expected_local + + persisted = Api.import_cache_data(str(tmp_path / "dst" / CACHE_FILENAME)) + assert "datetime_local" not in persisted.columns + assert persisted["datetime"].to_list() == utc_millis + + def test_merge_caches_returns_only_observed_pair_timeframe_sets( tmp_path: Path, ) -> None: @@ -449,6 +568,43 @@ def test_merge_caches_returns_only_observed_pair_timeframe_sets( assert all(isinstance(item["data"], pl.DataFrame) for item in result) +def test_merge_caches_projects_every_multi_series_result( + tmp_path: Path, +) -> None: + """Multi-series result dictionaries should all receive the local projection.""" + from histdatacom.api import Api + + source = Api._import_file_to_polars( + SimpleNamespace(data_timeframe="T"), + FIXTURES / "DAT_ASCII_EURUSD_T_201202.csv", + ) + records = [ + _write_cache_record( + tmp_path, + pair, + source, + pair=pair, + timeframe="T", + start=EXPECTED_TICK_DATETIMES[0], + ) + for pair in ("eurusd", "gbpusd") + ] + + result = Api( + args={"output_timezone": "Asia/Tokyo"}, + return_type="polars", + ).merge_records(records) + + assert isinstance(result, list) + assert [item["pair"] for item in result] == ["eurusd", "gbpusd"] + assert all("datetime_local" in item["data"].columns for item in result) + assert all( + str(item["data"].schema["datetime_local"]) + == "Datetime(time_unit='ms', time_zone='Asia/Tokyo')" + for item in result + ) + + def test_merge_caches_returns_empty_list_when_no_cache_records( tmp_path: Path, ) -> None: diff --git a/tests/unit/test_benchmark_corpus.py b/tests/unit/test_benchmark_corpus.py new file mode 100644 index 00000000..aebb44f6 --- /dev/null +++ b/tests/unit/test_benchmark_corpus.py @@ -0,0 +1,338 @@ +"""Contracts and failure modes for the real reverse-degradation corpus.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from histdatacom.runtime_contracts import ArtifactRef +from histdatacom.synthetic.benchmark import BenchmarkEventV1 +from histdatacom.synthetic.benchmark_corpus import ( + PREDECLARED_GATE_COMMIT, + BenchmarkWindowPartitionV1, + ReverseDegradationBenchmarkCorpusV1, + ReverseDegradationCorpusProfileV1, + audit_holdout_neighbor_leakage, + read_reverse_degradation_benchmark_corpus, +) +from histdatacom.synthetic.benchmark_gates import ( + load_default_benchmark_promotion_gate_policy, +) +from histdatacom.synthetic.contracts import canonical_contract_json +import histdatacom.synthetic.benchmark_corpus as corpus_module + + +def _corpus() -> ReverseDegradationBenchmarkCorpusV1: + profile = ReverseDegradationCorpusProfileV1() + sources = [] + by_axis = {} + for period in profile.split_periods.values(): + for symbol in profile.symbols: + source = corpus_module.BenchmarkSourcePartitionV1( + symbol=symbol, + period=period, + relative_path=f"{symbol.lower()}/{period}/.data", + size_bytes=1024, + row_count=1000, + sha256=hashlib.sha256( + f"{symbol}:{period}".encode() + ).hexdigest(), + ) + sources.append(source) + by_axis[(period, symbol)] = source + windows = [] + split_offsets = { + "calibration": 1_200_000_000_000_000_000, + "validation": 1_600_000_000_000_000_000, + "final_holdout": 1_700_000_000_000_000_000, + } + for split, period in profile.split_periods.items(): + for index in range(profile.synchronized_windows_per_split): + start = split_offsets[split] + index * 86_400_000_000_000 + windows.append( + BenchmarkWindowPartitionV1( + split_kind=split, + period=period, + session=("asia", "london", "new_york")[index % 3], + start_ns=start, + end_ns=start + 600_000_000_000, + epoch_label="technology_epoch_03", + source_partition_ids=tuple( + by_axis[(period, symbol)].partition_id + for symbol in profile.symbols + ), + symbol_event_counts=dict.fromkeys(profile.symbols, 64), + symbol_partition_sha256={ + symbol: hashlib.sha256( + f"{split}:{index}:{symbol}".encode() + ).hexdigest() + for symbol in profile.symbols + }, + event_state_counts={"update_joint": 192}, + context_state="market_context:none:no_matching_event", + positioning_state="cftc_positioning:weekly:2020-01-01", + context_supported=True, + ) + ) + policy = load_default_benchmark_promotion_gate_policy() + dependencies = { + name: ArtifactRef( + kind=name, + path=f"{name}.json", + size_bytes=10, + sha256=hashlib.sha256(name.encode()).hexdigest(), + ) + for name in ( + "feed_epochs", + "observation_campaign", + "market_context", + "cftc_positioning", + "gate_policy", + ) + } + return ReverseDegradationBenchmarkCorpusV1( + profile=profile, + sources=tuple(sources), + windows=tuple(windows), + split_hashes=corpus_module._split_hashes(windows), + degradation_configs=corpus_module._degradation_configs("operator-1"), + metric_registry=corpus_module._required_metric_names(), + dependency_artifacts=dependencies, + feed_epoch_definition_id="feed-epochs-1", + observation_operator_id="operator-1", + market_context_corpus_id="context-1", + cftc_positioning_corpus_id="positioning-1", + gate_policy_id=policy.policy_id, + gate_policy_commit=PREDECLARED_GATE_COMMIT, + neighbor_leakage_count=0, + ) + + +def test_profile_and_corpus_round_trip_are_content_addressed() -> None: + corpus = _corpus() + + assert ( + ReverseDegradationBenchmarkCorpusV1.from_json(corpus.to_json()) + == corpus + ) + assert corpus.corpus_id.startswith("reverse-degradation-corpus:sha256:") + assert len(corpus.windows) == 18 + assert set(corpus.split_hashes) == { + "calibration", + "validation", + "final_holdout", + } + assert corpus.to_dict()["dense_and_holdout_rows_persisted"] is False + + with pytest.raises(ValueError, match="chronological"): + ReverseDegradationCorpusProfileV1( + split_periods={ + "calibration": "202501", + "validation": "202401", + "final_holdout": "202601", + } + ) + + +def test_manifest_reader_rejects_tamper(tmp_path: Path) -> None: + corpus = _corpus() + payload = { + "schema_version": "histdatacom.reverse-degradation-manifest.v1", + "corpus": corpus.to_dict(), + "artifact_contract": { + "content_addressed": True, + "dense_rows_embedded": False, + "holdout_rows_embedded": False, + "replay_required": True, + }, + } + encoded = canonical_contract_json(payload).encode() + b"\n" + digest = hashlib.sha256(encoded).hexdigest() + path = tmp_path / f"reverse-degradation-manifest-{digest}.json" + path.write_bytes(encoded) + + assert read_reverse_degradation_benchmark_corpus(path) == corpus + + bad = tmp_path / f"reverse-degradation-manifest-{'0' * 64}.json" + bad.write_bytes(encoded) + with pytest.raises(ValueError, match="content hash differs"): + read_reverse_degradation_benchmark_corpus(bad) + + +def test_neighbor_leakage_detects_cross_split_overlap() -> None: + corpus = _corpus() + calibration = next( + item for item in corpus.windows if item.split_kind == "calibration" + ) + holdout = next( + item for item in corpus.windows if item.split_kind == "final_holdout" + ) + overlapping = BenchmarkWindowPartitionV1( + split_kind=holdout.split_kind, + period=holdout.period, + session=holdout.session, + start_ns=calibration.start_ns + 1, + end_ns=calibration.end_ns + 1, + epoch_label=holdout.epoch_label, + source_partition_ids=holdout.source_partition_ids, + symbol_event_counts=holdout.symbol_event_counts, + symbol_partition_sha256=holdout.symbol_partition_sha256, + event_state_counts=holdout.event_state_counts, + context_state=holdout.context_state, + positioning_state=holdout.positioning_state, + context_supported=True, + ) + + assert ( + audit_holdout_neighbor_leakage( + (calibration, overlapping), guard_seconds=1800 + ) + == 1 + ) + + +def test_dense_identity_passes_and_anchor_drop_fails() -> None: + partition = _corpus().windows[0] + reference = tuple( + BenchmarkEventV1( + source_event_id=f"{symbol}-{index}", + symbol=symbol, + event_time_ns=partition.start_ns + index * 1_000_000_000, + event_sequence=index, + bid=1.0 + index / 10_000, + ask=1.0002 + index / 10_000, + epoch_id=partition.epoch_label, + session=partition.session, + event_state="update_joint", + sparsity="dense-reference", + anchor_id=f"anchor-{symbol}-{index}" if index in {0, 2} else None, + ) + for symbol in ("EURGBP", "EURUSD", "GBPUSD") + for index in range(3) + ) + dense = corpus_module._compare_streams(reference, reference, partition) + negative = corpus_module._compare_streams( + reference, corpus_module._drop_first_anchor(reference), partition + ) + + dense_accumulator = corpus_module._CandidateAccumulator() + dense_accumulator.consume(dense) + dense_report = corpus_module._candidate_report( + subject_id="dense-subject", + method_name="dense_identity", + role="baseline", + accumulator=dense_accumulator, + policy=load_default_benchmark_promotion_gate_policy(), + ensemble_member_count=1, + evaluated_window_count=1, + provisional=False, + ) + negative_accumulator = corpus_module._CandidateAccumulator() + negative_accumulator.consume(negative) + negative_report = corpus_module._candidate_report( + subject_id="negative-subject", + method_name="negative_anchor_drop", + role="negative_control", + accumulator=negative_accumulator, + policy=load_default_benchmark_promotion_gate_policy(), + ensemble_member_count=1, + evaluated_window_count=1, + provisional=False, + ) + + assert dense_report.gate_decision.promotion_eligible + assert dense_report.metrics["immutable_anchor_violation_count"] == 0 + assert not negative_report.gate_decision.promotion_eligible + assert negative_report.metrics["immutable_anchor_violation_count"] == 1 + + +@pytest.mark.parametrize( + ("name", "parameter", "quantum"), + ( + ("timestamp_quantization", "quantum_ns", 1_000_000_000), + ("batching", "batch_width_ns", 2_000_000_000), + ), +) +def test_time_degradations_preserve_protected_anchor_timestamps( + name: str, parameter: str, quantum: int +) -> None: + corpus = _corpus() + partition = corpus.windows[0] + anchor = BenchmarkEventV1( + source_event_id="EURUSD-anchor", + symbol="EURUSD", + event_time_ns=partition.start_ns + 123_456_789, + event_sequence=0, + bid=1.1, + ask=1.1002, + epoch_id=partition.epoch_label, + session=partition.session, + event_state="update_joint", + sparsity="dense-reference", + anchor_id="anchor-EURUSD-first", + ) + ordinary = BenchmarkEventV1( + source_event_id="EURUSD-ordinary", + symbol="EURUSD", + event_time_ns=partition.start_ns + 1_234_567_890, + event_sequence=1, + bid=1.1001, + ask=1.1003, + epoch_id=partition.epoch_label, + session=partition.session, + event_state="update_joint", + sparsity="dense-reference", + ) + + degraded = corpus_module._apply_degradation( + (anchor, ordinary), + config={"name": name, parameter: quantum}, + corpus=corpus, + partition=partition, + operator=None, # unused by deterministic time degradations + run_id="run-anchor-regression", + ) + by_source = {item.source_event_id: item for item in degraded} + + assert ( + by_source[anchor.source_event_id].event_time_ns == anchor.event_time_ns + ) + assert by_source[anchor.source_event_id].anchor_id == anchor.anchor_id + assert ( + by_source[ordinary.source_event_id].event_time_ns + == (ordinary.event_time_ns // quantum) * quantum + ) + assert ( + corpus_module._compare_streams((anchor, ordinary), degraded, partition)[ + "immutable_anchor_violation_count" + ] + == 0 + ) + + +def test_cli_exposes_installed_real_corpus_command() -> None: + from histdatacom.data_analytics.cli import build_parser + + args = build_parser().parse_args( + [ + "reverse-degradation-benchmark-corpus", + "--source-root", + "ticks", + "--definition", + "epochs.json", + "--observation-campaign", + "operator.json", + "--market-context-corpus", + "context.json", + "--cftc-positioning-corpus", + "positioning.json", + "--artifact-dir", + "artifacts", + ] + ) + + assert args.analytics_command == "reverse-degradation-benchmark-corpus" + assert args.gate_policy_commit == PREDECLARED_GATE_COMMIT + assert args.windows_per_split == 6 diff --git a/tests/unit/test_benchmark_gates.py b/tests/unit/test_benchmark_gates.py new file mode 100644 index 00000000..5caab081 --- /dev/null +++ b/tests/unit/test_benchmark_gates.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from histdatacom.synthetic.benchmark_gates import ( + BenchmarkGateComparator, + BenchmarkGateObservationV1, + BenchmarkGateRequirementV1, + BenchmarkGateScope, + BenchmarkGateSeverity, + BenchmarkGateStatus, + BenchmarkPromotionDecisionV1, + BenchmarkPromotionGatePolicyV1, + evaluate_benchmark_promotion_gates, + load_default_benchmark_promotion_gate_policy, +) + + +def _requirement( + requirement_id: str, + scope: BenchmarkGateScope, + severity: BenchmarkGateSeverity, + metric_name: str, + comparator: BenchmarkGateComparator, + threshold: int | float | bool, +) -> BenchmarkGateRequirementV1: + return BenchmarkGateRequirementV1( + requirement_id=requirement_id, + scope=scope, + severity=severity, + metric_name=metric_name, + comparator=comparator, + threshold=threshold, + description=f"Predeclared requirement for {metric_name}.", + ) + + +def _policy() -> BenchmarkPromotionGatePolicyV1: + return BenchmarkPromotionGatePolicyV1( + policy_name="fixture-policy", + policy_version="v1", + issue_number=463, + frozen_before_candidate_results=True, + requirements=( + _requirement( + "campaign-hard", + BenchmarkGateScope.CAMPAIGN, + BenchmarkGateSeverity.HARD, + "campaign_failures", + BenchmarkGateComparator.ZERO, + 0, + ), + _requirement( + "campaign-advisory", + BenchmarkGateScope.CAMPAIGN, + BenchmarkGateSeverity.ADVISORY, + "campaign_windows", + BenchmarkGateComparator.GREATER_OR_EQUAL, + 2, + ), + _requirement( + "candidate-hard", + BenchmarkGateScope.CANDIDATE, + BenchmarkGateSeverity.HARD, + "anchor_violations", + BenchmarkGateComparator.ZERO, + 0, + ), + _requirement( + "candidate-advisory", + BenchmarkGateScope.CANDIDATE, + BenchmarkGateSeverity.ADVISORY, + "uncertainty_reported", + BenchmarkGateComparator.TRUE, + True, + ), + ), + ) + + +def _observation( + scope: BenchmarkGateScope, + subject_id: str, + metric_name: str, + value: int | float | bool, +) -> BenchmarkGateObservationV1: + return BenchmarkGateObservationV1( + scope=scope, + subject_id=subject_id, + metric_name=metric_name, + value=value, + evidence_ids=("artifact:fixture",), + ) + + +def test_packaged_policy_is_predeclared_complete_and_content_addressed() -> ( + None +): + policy = load_default_benchmark_promotion_gate_policy() + + assert policy.issue_number == 463 + assert policy.frozen_before_candidate_results is True + assert policy.policy_id.startswith("benchmark-promotion-gates:sha256:") + assert len(policy.requirements) == 23 + assert {(item.scope, item.severity) for item in policy.requirements} == { + (BenchmarkGateScope.CAMPAIGN, BenchmarkGateSeverity.HARD), + (BenchmarkGateScope.CAMPAIGN, BenchmarkGateSeverity.ADVISORY), + (BenchmarkGateScope.CANDIDATE, BenchmarkGateSeverity.HARD), + (BenchmarkGateScope.CANDIDATE, BenchmarkGateSeverity.ADVISORY), + } + assert BenchmarkPromotionGatePolicyV1.from_json(policy.to_json()) == policy + + +def test_policy_identity_is_order_stable_and_tamper_evident() -> None: + policy = _policy() + reordered = replace( + policy, + requirements=tuple(reversed(policy.requirements)), + policy_id="", + ) + + assert reordered.policy_id == policy.policy_id + with pytest.raises(ValueError, match="policy_id differs"): + replace(policy, policy_version="v2") + with pytest.raises(ValueError, match="frozen before results"): + replace(policy, frozen_before_candidate_results=False, policy_id="") + + +def test_missing_hard_evidence_fails_closed_and_advisory_does_not_block() -> ( + None +): + policy = _policy() + decision = evaluate_benchmark_promotion_gates( + policy, + (), + scope=BenchmarkGateScope.CANDIDATE, + subject_id="candidate:fixture", + ) + + assert decision.promotion_eligible is False + assert decision.automatic_winner is False + by_requirement = {item.requirement_id: item for item in decision.checks} + hard = by_requirement["candidate-hard"] + advisory = by_requirement["candidate-advisory"] + assert hard.status is BenchmarkGateStatus.MISSING + assert hard.blocking is True + assert advisory.status is BenchmarkGateStatus.MISSING + assert advisory.blocking is False + + +def test_advisory_failure_is_visible_without_becoming_a_hidden_hard_gate() -> ( + None +): + policy = _policy() + observations = ( + _observation( + BenchmarkGateScope.CANDIDATE, + "candidate:fixture", + "anchor_violations", + 0, + ), + _observation( + BenchmarkGateScope.CANDIDATE, + "candidate:fixture", + "uncertainty_reported", + False, + ), + ) + decision = evaluate_benchmark_promotion_gates( + policy, + observations, + scope=BenchmarkGateScope.CANDIDATE, + subject_id="candidate:fixture", + ) + + assert decision.promotion_eligible is True + assert [item.status for item in decision.checks] == [ + BenchmarkGateStatus.FAILED, + BenchmarkGateStatus.PASSED, + ] + assert not any(item.blocking for item in decision.checks) + assert ( + BenchmarkPromotionDecisionV1.from_dict(decision.to_dict()) == decision + ) + + +def test_hard_failure_blocks_and_duplicate_metric_evidence_is_rejected() -> ( + None +): + policy = _policy() + failed = _observation( + BenchmarkGateScope.CANDIDATE, + "candidate:fixture", + "anchor_violations", + 1, + ) + advisory = _observation( + BenchmarkGateScope.CANDIDATE, + "candidate:fixture", + "uncertainty_reported", + True, + ) + decision = evaluate_benchmark_promotion_gates( + policy, + (failed, advisory), + scope=BenchmarkGateScope.CANDIDATE, + subject_id="candidate:fixture", + ) + + assert decision.promotion_eligible is False + assert any(item.blocking for item in decision.checks) + with pytest.raises(ValueError, match="duplicate.*metric"): + evaluate_benchmark_promotion_gates( + policy, + (failed, failed), + scope=BenchmarkGateScope.CANDIDATE, + subject_id="candidate:fixture", + ) + + +def test_comparator_types_and_policy_shape_fail_closed() -> None: + with pytest.raises(ValueError, match="boolean.*threshold"): + _requirement( + "bad-boolean", + BenchmarkGateScope.CANDIDATE, + BenchmarkGateSeverity.HARD, + "bad_boolean", + BenchmarkGateComparator.TRUE, + 1, + ) + with pytest.raises(ValueError, match="hard and advisory"): + BenchmarkPromotionGatePolicyV1( + policy_name="incomplete", + policy_version="v1", + issue_number=463, + frozen_before_candidate_results=True, + requirements=(_policy().requirements[0],), + ) diff --git a/tests/unit/test_broker_capture.py b/tests/unit/test_broker_capture.py new file mode 100644 index 00000000..d55eaf4a --- /dev/null +++ b/tests/unit/test_broker_capture.py @@ -0,0 +1,630 @@ +"""Trust and failure tests for append-only broker delivery capture.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +from histdatacom.broker_capture import ( + AppendOnlyBrokerCaptureWriterV1, + BrokerAdapterMessageV1, + BrokerCaptureActivitySemantics, + BrokerCaptureAdapterV1, + BrokerCaptureBackpressureError, + BrokerCaptureEventKind, + BrokerCaptureEventV1, + BrokerCaptureExistingSessionError, + BrokerCaptureIntegrityError, + BrokerCapturePriceTextSemantics, + BrokerCaptureQuotaError, + BrokerCaptureReplaySummaryV1, + BrokerCaptureRetentionError, + BrokerCaptureSessionManifestV1, + BrokerCaptureSessionState, + BrokerCaptureSessionV1, + BrokerCaptureSizeSemantics, + BrokerCaptureStoragePolicyV1, + BrokerCaptureSourceTimestampSemantics, + LiveBrokerCaptureSourceV1, + SequenceBrokerCaptureAdapterV1, + SequenceBrokerCaptureClockV1, + consume_broker_capture_source, + discover_broker_capture_session_manifests, + inspect_broker_capture_session, + load_broker_capture_session_manifest, + logical_capture_content_sha256, + replay_broker_capture_session, + verify_broker_capture_partition_manifests, +) + +_WALL_BASE = 1_700_000_000_000_000_000 +_MONOTONIC_BASE = 2_000_000_000 + + +@dataclass(slots=True) +class _CollectingConsumer: + events: list[BrokerCaptureEventV1] = field(default_factory=list) + + def on_event(self, event: BrokerCaptureEventV1) -> None: + self.events.append(event) + + +class _PrivateConfigurationAdapter: + adapter_id = "synthetic.fixture" + adapter_version = "1.0.0" + + def __init__( + self, credential: str, messages: tuple[BrokerAdapterMessageV1, ...] + ) -> None: + self._credential = credential + self._messages = messages + + def iter_messages(self) -> tuple[BrokerAdapterMessageV1, ...]: + return self._messages + + +def test_capture_contracts_are_versioned_roundtrippable_and_secret_free() -> ( + None +): + session = _session(1) + message = BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.QUOTE, + source_event_time_ns=_WALL_BASE, + source_timestamp_semantics=( + BrokerCaptureSourceTimestampSemantics.BROKER_EVENT + ), + source_timestamp_precision_ns=1_000_000, + source_sequence=7, + source_message_id="quote-7", + source_batch_id="batch-1", + symbol="eurusd", + bid=1.1, + ask=1.1002, + bid_text="1.1000", + ask_text="1.1002", + price_text_semantics=BrokerCapturePriceTextSemantics.SOURCE_LEXEME, + bid_size=2.0, + ask_size=3.0, + size_semantics=BrokerCaptureSizeSemantics.QUOTED_SIZE, + activity_value=5.0, + activity_semantics=BrokerCaptureActivitySemantics.BROKER_ACTIVITY, + raw_message_sha256=hashlib.sha256(b"public-message").hexdigest(), + public_metadata={"channel": "prices", "venue_class": "retail"}, + ) + event = BrokerCaptureEventV1( + session_id=session.session_id, + capture_sequence=0, + receive_time_utc_ns=_WALL_BASE + 1, + receive_time_monotonic_ns=_MONOTONIC_BASE + 1, + message=message, + ) + policy = _policy() + + assert BrokerCaptureSessionV1.from_json(session.to_json()) == session + assert BrokerAdapterMessageV1.from_json(message.to_json()) == message + assert BrokerCaptureEventV1.from_json(event.to_json()) == event + assert BrokerCaptureStoragePolicyV1.from_json(policy.to_json()) == policy + assert message.symbol == "EURUSD" + assert message.bid_text == "1.1000" + assert message.source_timestamp_precision_ns == 1_000_000 + assert message.message_id.startswith("broker-message-") + assert event.event_id.startswith("broker-capture-event-") + + secret = "do-not-persist-this-token" + with pytest.raises(ValueError, match="sensitive metadata key") as key_error: + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.HEARTBEAT, + public_metadata={"access_token": secret}, + ) + assert secret not in str(key_error.value) + with pytest.raises( + ValueError, match="sensitive metadata value" + ) as value_error: + BrokerCaptureSessionV1( + **{ + **_session_kwargs(2), + "public_metadata": {"header": f"Bearer {secret}"}, + } + ) + assert secret not in str(value_error.value) + with pytest.raises(ValueError, match="explicit size semantics"): + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.QUOTE, + symbol="EURUSD", + bid=1.0, + ask=1.1, + bid_size=1.0, + ) + + +def test_live_source_surfaces_fixture_health_ordering_and_clock_drift() -> None: + session = _session(3) + source, fixture_messages = _fixture_source(session) + events = tuple(source.iter_events()) + + assert [event.capture_sequence for event in events] == list( + range(len(events)) + ) + assert all(event.receive_time_utc_ns > 0 for event in events) + assert all(event.receive_time_monotonic_ns > 0 for event in events) + assert all( + current.receive_time_monotonic_ns >= previous.receive_time_monotonic_ns + for previous, current in zip(events, events[1:]) + ) + assert len(events) == len(fixture_messages) + 1 + + kinds = [event.kind for event in events] + for required in ( + BrokerCaptureEventKind.PROCESS_START, + BrokerCaptureEventKind.PROCESS_STOP, + BrokerCaptureEventKind.PROCESS_RESTART, + BrokerCaptureEventKind.CONNECTION_OPEN, + BrokerCaptureEventKind.CONNECTION_CLOSE, + BrokerCaptureEventKind.RECONNECT, + BrokerCaptureEventKind.SUBSCRIPTION_ADD, + BrokerCaptureEventKind.SUBSCRIPTION_REMOVE, + BrokerCaptureEventKind.HEARTBEAT, + BrokerCaptureEventKind.GAP, + BrokerCaptureEventKind.OUTAGE_START, + BrokerCaptureEventKind.OUTAGE_END, + BrokerCaptureEventKind.CLOCK_CORRECTION, + ): + assert required in kinds + + correction = next( + event + for event in events + if event.kind is BrokerCaptureEventKind.CLOCK_CORRECTION + ) + assert correction.clock_offset_change_ns == 100_000_000 + assert correction.message.reason_code == "wall_monotonic_divergence" + + quotes = [ + event for event in events if event.kind is BrokerCaptureEventKind.QUOTE + ] + assert quotes[0].message.message_id == quotes[1].message.message_id + assert quotes[0].event_id != quotes[1].event_id + assert quotes[0].message.bid == quotes[2].message.bid + assert quotes[0].message.ask == quotes[2].message.ask + assert ( + quotes[1].receive_time_monotonic_ns + - quotes[0].receive_time_monotonic_ns + < 1_000_000 + ) + gap = next( + event for event in events if event.kind is BrokerCaptureEventKind.GAP + ) + gap_index = events.index(gap) + assert gap.message.gap_duration_ns == 5_000_000_000 + assert ( + gap.receive_time_monotonic_ns + - events[gap_index - 1].receive_time_monotonic_ns + > 4_000_000_000 + ) + + +def test_append_only_rotation_replay_and_live_consumer_parity( + tmp_path: Path, +) -> None: + session = _session(4) + source, _messages = _fixture_source(session) + policy = _policy(max_partition_events=4) + writer = AppendOnlyBrokerCaptureWriterV1( + tmp_path, session=session, storage_policy=policy + ) + live_consumer = _CollectingConsumer() + + live_result = consume_broker_capture_source( + source, + sink=writer, + consumers=(live_consumer,), + ) + manifest = writer.close() + + assert manifest.state is BrokerCaptureSessionState.COMPLETED + assert manifest.complete + assert manifest.event_count == live_result.event_count == 17 + assert len(manifest.partitions) == 5 + assert all(partition.completed for partition in manifest.partitions) + assert manifest.partial_artifact_count == 0 + assert sum(manifest.event_kind_counts.values()) == 17 + assert inspect_broker_capture_session(tmp_path, session.session_id).clean + + manifest_path = tmp_path / session.session_id / "session.manifest.json" + restored = load_broker_capture_session_manifest(manifest_path) + assert restored == manifest + assert ( + BrokerCaptureSessionManifestV1.from_json(manifest.to_json()) == manifest + ) + assert discover_broker_capture_session_manifests(tmp_path) == (manifest,) + assert ( + verify_broker_capture_partition_manifests(tmp_path, manifest) + == manifest + ) + + replay_consumer = _CollectingConsumer() + replay_summary = replay_broker_capture_session( + tmp_path, manifest, consumers=(replay_consumer,) + ) + assert replay_consumer.events == live_consumer.events + assert replay_summary.event_kind_counts == live_result.event_kind_counts + assert replay_summary.logical_content_sha256 == ( + logical_capture_content_sha256(live_consumer.events) + ) + assert ( + BrokerCaptureReplaySummaryV1.from_json(replay_summary.to_json()) + == replay_summary + ) + + +def test_private_adapter_configuration_never_enters_capture_artifacts( + tmp_path: Path, +) -> None: + credential = "super-secret-broker-credential" + session = _session(5) + _source, messages = _fixture_source(session) + adapter = _PrivateConfigurationAdapter(credential, messages[:3]) + assert isinstance(adapter, BrokerCaptureAdapterV1) + source = LiveBrokerCaptureSourceV1( + session=session, + adapter=adapter, + clock=SequenceBrokerCaptureClockV1(_clock_samples(3)), + ) + writer = AppendOnlyBrokerCaptureWriterV1( + tmp_path, session=session, storage_policy=_policy() + ) + consume_broker_capture_source(source, sink=writer) + writer.close() + + artifact_bytes = b"".join( + path.read_bytes() + for path in sorted(tmp_path.rglob("*")) + if path.is_file() + ) + assert credential.encode("utf-8") not in artifact_bytes + + +def test_partial_and_orphan_artifacts_are_detected_but_not_advertised( + tmp_path: Path, +) -> None: + session = _session(6) + source, _messages = _fixture_source(session) + writer = AppendOnlyBrokerCaptureWriterV1( + tmp_path, session=session, storage_policy=_policy() + ) + consume_broker_capture_source(source, sink=writer) + manifest = writer.close() + session_dir = tmp_path / session.session_id + + partial = session_dir / "partition-999998.jsonl.partial" + orphan = session_dir / "partition-999999.jsonl" + partial.write_text('{"partial":', encoding="utf-8") + orphan.write_text("{}\n", encoding="utf-8") + + inspection = inspect_broker_capture_session(tmp_path, session.session_id) + assert inspection.partial_artifacts == (partial.name,) + assert inspection.orphan_data_artifacts == (orphan.name,) + assert not inspection.clean + discovered = discover_broker_capture_session_manifests(tmp_path) + assert discovered == (manifest,) + assert all( + partition.data_artifact.path.endswith(".jsonl") + for partition in discovered[0].partitions + ) + + +def test_quota_backpressure_and_retention_refuse_predictably( + tmp_path: Path, +) -> None: + retention_session = _session(7) + retention_source, _messages = _fixture_source(retention_session) + retention_events = tuple(retention_source.iter_events()) + retention_writer = AppendOnlyBrokerCaptureWriterV1( + tmp_path / "retention", + session=retention_session, + storage_policy=_policy( + max_partition_events=1, + max_retained_partitions=1, + ), + ) + retention_writer.append(retention_events[0]) + with pytest.raises(BrokerCaptureRetentionError, match="retention ceiling"): + retention_writer.append(retention_events[1]) + failed_manifest = retention_writer.close( + completed=False, limitations=("retention_refusal",) + ) + assert failed_manifest.state is BrokerCaptureSessionState.FAILED + assert failed_manifest.limitations == ("retention_refusal",) + assert failed_manifest.event_kind_counts == {"process_start": 1} + + quota_session = _session(8) + quota_event = _large_event(quota_session) + line_size = len((quota_event.to_json() + "\n").encode("utf-8")) + reserve = 64 * 1024 + quota_writer = AppendOnlyBrokerCaptureWriterV1( + tmp_path / "quota", + session=quota_session, + storage_policy=_policy( + max_partition_bytes=line_size + 100, + manifest_reserve_bytes=reserve, + max_session_bytes=line_size + reserve + 100, + high_watermark_bytes=line_size + reserve + 100, + ), + ) + with pytest.raises(BrokerCaptureQuotaError, match="disk quota"): + quota_writer.append(quota_event) + quota_writer.close(completed=False, limitations=("quota_refusal",)) + + pressure_session = _session(9) + pressure_event = _large_event(pressure_session) + pressure_line_size = len((pressure_event.to_json() + "\n").encode("utf-8")) + pressure_writer = AppendOnlyBrokerCaptureWriterV1( + tmp_path / "pressure", + session=pressure_session, + storage_policy=_policy( + max_partition_bytes=pressure_line_size + 100, + manifest_reserve_bytes=reserve, + max_session_bytes=pressure_line_size + reserve + 100_000, + high_watermark_bytes=pressure_line_size + reserve, + ), + ) + with pytest.raises(BrokerCaptureBackpressureError, match="high watermark"): + pressure_writer.append(pressure_event) + pressure_writer.close( + completed=False, limitations=("backpressure_refusal",) + ) + + +def test_existing_session_and_corrupt_replay_fail_closed( + tmp_path: Path, +) -> None: + session = _session(10) + source, _messages = _fixture_source(session) + policy = _policy() + writer = AppendOnlyBrokerCaptureWriterV1( + tmp_path, session=session, storage_policy=policy + ) + consume_broker_capture_source(source, sink=writer) + manifest = writer.close() + + with pytest.raises(BrokerCaptureExistingSessionError): + AppendOnlyBrokerCaptureWriterV1( + tmp_path, session=session, storage_policy=policy + ) + + first_sidecar = ( + tmp_path / session.session_id / "partition-000000.manifest.json" + ) + original_sidecar = first_sidecar.read_text(encoding="utf-8") + first_sidecar.write_text("{}\n", encoding="utf-8") + with pytest.raises( + BrokerCaptureIntegrityError, match="manifest is invalid" + ): + replay_broker_capture_session(tmp_path, manifest) + first_sidecar.write_text(original_sidecar, encoding="utf-8") + + first_path = tmp_path / manifest.partitions[0].data_artifact.path + first_path.write_bytes(first_path.read_bytes() + b"{}\n") + with pytest.raises(BrokerCaptureIntegrityError, match="size"): + replay_broker_capture_session(tmp_path, manifest) + + +def _session(seed: int) -> BrokerCaptureSessionV1: + return BrokerCaptureSessionV1(**_session_kwargs(seed)) + + +def _session_kwargs(seed: int) -> dict[str, object]: + return { + "adapter_id": "synthetic.fixture", + "adapter_version": "1.0.0", + "adapter_config_sha256": hashlib.sha256( + f"public-config-{seed}".encode("utf-8") + ).hexdigest(), + "protocol": "fixture-stream", + "environment_id": "paper", + "server_id": "fixture-server", + "started_at_utc_ns": _WALL_BASE + seed, + "started_at_monotonic_ns": _MONOTONIC_BASE + seed, + "account_id_sha256": hashlib.sha256( + f"account-{seed}".encode("utf-8") + ).hexdigest(), + "host_id_sha256": hashlib.sha256( + f"host-{seed}".encode("utf-8") + ).hexdigest(), + "public_metadata": {"fixture": True, "seed": seed}, + } + + +def _fixture_source( + session: BrokerCaptureSessionV1, +) -> tuple[LiveBrokerCaptureSourceV1, tuple[BrokerAdapterMessageV1, ...]]: + quote = BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.QUOTE, + source_event_time_ns=_WALL_BASE + 3_000_000, + source_timestamp_semantics=( + BrokerCaptureSourceTimestampSemantics.BROKER_EVENT + ), + source_timestamp_precision_ns=100_000, + source_batch_id="batch-burst-1", + symbol="EURUSD", + bid=1.1000, + ask=1.1002, + bid_text="1.1000", + ask_text="1.1002", + price_text_semantics=BrokerCapturePriceTextSemantics.SOURCE_LEXEME, + bid_size=2.0, + ask_size=3.0, + size_semantics=BrokerCaptureSizeSemantics.QUOTED_SIZE, + activity_value=1.0, + activity_semantics=BrokerCaptureActivitySemantics.MESSAGE_COUNT, + raw_message_sha256=hashlib.sha256(b"quote-a").hexdigest(), + ) + messages = ( + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.PROCESS_START, + reason_code="collector_started", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.CONNECTION_OPEN, + connection_id="connection-1", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.SUBSCRIPTION_ADD, + connection_id="connection-1", + subscription_id="subscription-eurusd", + symbol="EURUSD", + ), + quote, + BrokerAdapterMessageV1.from_json(quote.to_json()), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.QUOTE, + source_event_time_ns=_WALL_BASE + 3_100_000, + source_timestamp_semantics=( + BrokerCaptureSourceTimestampSemantics.BROKER_EVENT + ), + source_timestamp_precision_ns=100_000, + source_batch_id="batch-burst-1", + symbol="EURUSD", + bid=1.1000, + ask=1.1002, + bid_text="1.1000", + ask_text="1.1002", + price_text_semantics=( + BrokerCapturePriceTextSemantics.SOURCE_LEXEME + ), + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.QUOTE, + source_event_time_ns=_WALL_BASE + 3_200_000, + source_timestamp_semantics=( + BrokerCaptureSourceTimestampSemantics.BROKER_EVENT + ), + source_timestamp_precision_ns=100_000, + source_batch_id="batch-burst-1", + symbol="EURUSD", + bid=1.1001, + ask=1.1003, + bid_text="1.1001", + ask_text="1.1003", + price_text_semantics=( + BrokerCapturePriceTextSemantics.SOURCE_LEXEME + ), + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.HEARTBEAT, + connection_id="connection-1", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.GAP, + gap_duration_ns=5_000_000_000, + reason_code="known_quiet_gap", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.OUTAGE_START, + reason_code="fixture_outage", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.OUTAGE_END, + gap_duration_ns=1_000_000_000, + reason_code="fixture_outage_recovered", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.CONNECTION_CLOSE, + connection_id="connection-1", + reason_code="fixture_disconnect", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.RECONNECT, + connection_id="connection-2", + reason_code="fixture_reconnect", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.PROCESS_RESTART, + reason_code="fixture_process_restart", + public_metadata={"previous_process": "fixture-process-1"}, + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.SUBSCRIPTION_REMOVE, + connection_id="connection-2", + subscription_id="subscription-eurusd", + symbol="EURUSD", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.PROCESS_STOP, + reason_code="collector_stopped", + ), + ) + adapter = SequenceBrokerCaptureAdapterV1( + adapter_id=session.adapter_id, + adapter_version=session.adapter_version, + messages=messages, + ) + source = LiveBrokerCaptureSourceV1( + session=session, + adapter=adapter, + clock=SequenceBrokerCaptureClockV1(_clock_samples(len(messages))), + clock_correction_threshold_ns=5_000_000, + ) + return source, messages + + +def _clock_samples(count: int) -> tuple[tuple[int, int], ...]: + increments = ( + 0, + 1_000_000, + 2_000_000, + 3_000_000, + 3_100_000, + 3_200_000, + 3_300_000, + 4_000_000, + 5_004_000_000, + 5_005_000_000, + 5_006_000_000, + 5_007_000_000, + 5_008_000_000, + 5_009_000_000, + 5_010_000_000, + 5_011_000_000, + ) + if count > len(increments): + raise ValueError("fixture requests too many clock samples") + return tuple( + ( + _WALL_BASE + delta + (100_000_000 if index >= 10 else 0), + _MONOTONIC_BASE + delta, + ) + for index, delta in enumerate(increments[:count]) + ) + + +def _policy(**overrides: object) -> BrokerCaptureStoragePolicyV1: + values: dict[str, object] = { + "max_partition_events": 100, + "max_partition_bytes": 2 * 1024**2, + "max_partition_duration_ns": 60 * 1_000_000_000, + "max_session_bytes": 32 * 1024**2, + "high_watermark_bytes": 24 * 1024**2, + "max_retained_partitions": 100, + "manifest_reserve_bytes": 64 * 1024, + "fsync_each_event": True, + } + values.update(overrides) + return BrokerCaptureStoragePolicyV1(**values) + + +def _large_event(session: BrokerCaptureSessionV1) -> BrokerCaptureEventV1: + message = BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.HEARTBEAT, + public_metadata={"bounded_fixture_padding": "x" * 18_000}, + ) + return BrokerCaptureEventV1( + session_id=session.session_id, + capture_sequence=0, + receive_time_utc_ns=_WALL_BASE, + receive_time_monotonic_ns=_MONOTONIC_BASE, + message=message, + ) diff --git a/tests/unit/test_broker_delivery_fingerprints.py b/tests/unit/test_broker_delivery_fingerprints.py new file mode 100644 index 00000000..1d0d136d --- /dev/null +++ b/tests/unit/test_broker_delivery_fingerprints.py @@ -0,0 +1,891 @@ +"""Trust, replay, support, and drift tests for broker fingerprints.""" + +from __future__ import annotations + +import hashlib +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from histdatacom.broker_capture import ( + AppendOnlyBrokerCaptureWriterV1, + BrokerAdapterMessageV1, + BrokerCaptureEligibilityStatus, + BrokerCaptureEventKind, + BrokerCaptureEventV1, + BrokerCapturePriceTextSemantics, + BrokerCaptureSessionManifestV1, + BrokerCaptureSessionV1, + BrokerCaptureStoragePolicyV1, + BrokerCaptureSourceTimestampSemantics, + BrokerDeliveryDriftConfigV1, + BrokerDeliveryDriftStatus, + BrokerDeliveryCellV1, + BrokerDeliveryFingerprintArtifactError, + BrokerDeliveryFingerprintComparisonV1, + BrokerDeliveryFingerprintIdentityError, + BrokerDeliveryFingerprintV1, + BrokerDeliveryFitConfigV1, + BrokerDeliveryIneligibleCaptureError, + BrokerDeliveryResourceLimitError, + BrokerDeliverySupportStatus, + assess_broker_capture_eligibility, + compare_broker_delivery_fingerprints, + fit_broker_delivery_fingerprint, + load_broker_delivery_fingerprint, + write_broker_delivery_fingerprint, +) +from histdatacom.market_context import ( + MarketContextEventV1, + MarketContextKind, + MarketContextPrecision, + MarketContextSourceV1, + MarketContextTimelineV1, +) +from histdatacom.synthetic import ( + BrokerConditionedProposalV1, + BrokerTransferConfigV1, + BrokerTransferStatus, + InformationMode, + ReferenceMotifConditionV1, + ReferenceMotifQueryV1, + SyntheticEventStreamV1, + SyntheticEventV1, + condition_broker_proposal, + select_broker_profile, +) + +SECOND_NS = 1_000_000_000 +BASE_WALL_NS = int( + datetime(2023, 12, 25, 12, tzinfo=timezone.utc).timestamp() * SECOND_NS +) +BASE_MONOTONIC_NS = 10_000_000_000 +CONFIG_SHA256 = hashlib.sha256(b"stable-public-adapter-config").hexdigest() +ACCOUNT_SHA256 = hashlib.sha256(b"stable-account-identity").hexdigest() + + +def test_fit_is_deterministic_bounded_and_recovers_conditioned_characteristics( + tmp_path: Path, +) -> None: + first = _capture(tmp_path, seed=1, wall_start_ns=BASE_WALL_NS) + second = _capture( + tmp_path, + seed=2, + wall_start_ns=BASE_WALL_NS + 24 * 60 * 60 * SECOND_NS, + ) + config = BrokerDeliveryFitConfigV1( + min_cell_support=4, + max_samples_per_metric=5, + ) + + forward = fit_broker_delivery_fingerprint( + tmp_path, + (first, second), + config=config, + market_context_timeline=_timeline(), + ) + reversed_input = fit_broker_delivery_fingerprint( + tmp_path, + (second, first), + config=config, + market_context_timeline=_timeline(), + ) + + assert forward == reversed_input + assert BrokerDeliveryFingerprintV1.from_json(forward.to_json()) == forward + assert len(forward.capture_evidence) == 2 + assert all(item.logical_content_sha256 for item in forward.capture_evidence) + assert all( + item.partition_hashes_sha256 for item in forward.capture_evidence + ) + assert all( + item.status is BrokerCaptureEligibilityStatus.ELIGIBLE + for item in forward.eligibility_decisions + ) + assert any( + cell.condition.key == "holiday=major_holiday:christmas_day" + for cell in forward.cells + ) + assert any( + cell.condition.key == "event=market_macro_release" + for cell in forward.cells + ) + assert any( + cell.condition.key.startswith("session=") for cell in forward.cells + ) + global_cell = _cell(forward, "global") + metrics = {item.name: item for item in global_cell.metrics} + assert metrics["quote_interarrival_ns"].support_count > 5 + assert metrics["quote_interarrival_ns"].sample_count == 5 + assert metrics["quote_interarrival_ns"].limitations == ( + "deterministic_bottom_hash_sample", + ) + assert metrics["spread"].estimate == pytest.approx(0.0002) + assert metrics["source_timestamp_precision_ns"].estimate == 100_000 + assert metrics["price_decimal_places"].estimate == 4 + assert metrics["exact_duplicate_rate"].lower is not None + assert metrics["exact_duplicate_rate"].upper is not None + assert metrics["burst_interval_run_length"].support_count > 0 + assert metrics["quiet_interval_run_length"].estimate == 1 + assert metrics["stale_quote_run_length"].estimate == 1 + assert metrics["event_intensity_hz"].support_count == 2 + assert metrics["quote_intensity_hz"].support_count == 2 + assert metrics["event_kind.reconnect_rate"].estimate is not None + assert metrics["outage_or_gap_duration_ns"].estimate == 2 * SECOND_NS + assert "global_similarity_score" not in forward.to_dict() + + +def test_sparse_cells_back_off_and_unqualified_capture_fails_closed( + tmp_path: Path, +) -> None: + manifest = _capture(tmp_path, seed=3, wall_start_ns=BASE_WALL_NS) + config = BrokerDeliveryFitConfigV1( + min_cell_support=6, + post_lifecycle_quote_count=2, + ) + fingerprint = fit_broker_delivery_fingerprint( + tmp_path, (manifest,), config=config + ) + lifecycle = next( + cell + for cell in fingerprint.cells + if cell.condition.key == "lifecycle=post_reconnect" + ) + + assert lifecycle.support_count == 2 + assert lifecycle.support_status is BrokerDeliverySupportStatus.BACKED_OFF + assert ( + lifecycle.effective_condition_id + == _cell(fingerprint, "global").condition.condition_id + ) + assert lifecycle.backoff_condition_ids == ( + _cell(fingerprint, "global").condition.condition_id, + ) + + failed_root = tmp_path / "failed" + failed = _capture( + failed_root, + seed=4, + wall_start_ns=BASE_WALL_NS, + completed=False, + ) + eligibility = assess_broker_capture_eligibility( + failed_root, failed, config=config + ) + assert eligibility.status is BrokerCaptureEligibilityStatus.INELIGIBLE + assert "capture_not_completed" in eligibility.reason_codes + with pytest.raises(BrokerDeliveryIneligibleCaptureError) as error: + fit_broker_delivery_fingerprint(failed_root, (failed,), config=config) + assert error.value.eligibility == eligibility + + +def test_integrity_clock_and_resource_health_gates_are_explicit( + tmp_path: Path, +) -> None: + corrupt_root = tmp_path / "corrupt" + corrupt = _capture(corrupt_root, seed=5, wall_start_ns=BASE_WALL_NS) + data_path = corrupt_root / corrupt.partitions[0].data_artifact.path + data_path.write_bytes(data_path.read_bytes() + b"{}\n") + eligibility = assess_broker_capture_eligibility(corrupt_root, corrupt) + assert eligibility.status is BrokerCaptureEligibilityStatus.INELIGIBLE + assert "integrity_verification_failed" in eligibility.reason_codes + + correction_root = tmp_path / "correction" + correction = _capture( + correction_root, + seed=6, + wall_start_ns=BASE_WALL_NS, + clock_correction_ns=25_000_000, + ) + limited = assess_broker_capture_eligibility(correction_root, correction) + assert limited.status is BrokerCaptureEligibilityStatus.LIMITED + assert limited.max_abs_clock_correction_ns == 25_000_000 + strict = assess_broker_capture_eligibility( + correction_root, + correction, + config=BrokerDeliveryFitConfigV1(max_abs_clock_correction_ns=1), + ) + assert strict.status is BrokerCaptureEligibilityStatus.INELIGIBLE + assert "excessive_clock_correction_magnitude" in strict.reason_codes + + with pytest.raises( + BrokerDeliveryResourceLimitError, match="max_input_events" + ): + fit_broker_delivery_fingerprint( + correction_root, + (correction,), + config=BrokerDeliveryFitConfigV1(max_input_events=8), + ) + with pytest.raises(BrokerDeliveryResourceLimitError, match="max_cells"): + fit_broker_delivery_fingerprint( + correction_root, + (correction,), + config=BrokerDeliveryFitConfigV1(max_cells=1), + ) + context_root = tmp_path / "context-bound" + context_manifest = _capture( + context_root, seed=16, wall_start_ns=BASE_WALL_NS + ) + with pytest.raises( + BrokerDeliveryResourceLimitError, + match="max_market_matches_per_quote", + ): + fit_broker_delivery_fingerprint( + context_root, + (context_manifest,), + config=BrokerDeliveryFitConfigV1(max_market_matches_per_quote=1), + market_context_timeline=_timeline(), + ) + + +def test_drift_is_stratified_support_aware_and_has_no_winner_score( + tmp_path: Path, +) -> None: + reference_manifest = _capture( + tmp_path / "reference", seed=7, wall_start_ns=BASE_WALL_NS + ) + stable_manifest = _capture( + tmp_path / "stable", + seed=8, + wall_start_ns=BASE_WALL_NS + 24 * 60 * 60 * SECOND_NS, + ) + drift_manifest = _capture( + tmp_path / "drift", + seed=9, + wall_start_ns=BASE_WALL_NS + 48 * 60 * 60 * SECOND_NS, + cadence_ns=SECOND_NS, + spread=0.001, + precision_ns=1, + decimal_places=5, + ) + reference = fit_broker_delivery_fingerprint( + tmp_path / "reference", (reference_manifest,) + ) + stable = fit_broker_delivery_fingerprint( + tmp_path / "stable", (stable_manifest,) + ) + drift = fit_broker_delivery_fingerprint( + tmp_path / "drift", (drift_manifest,) + ) + + stable_comparison = compare_broker_delivery_fingerprints(reference, stable) + assert any( + item.status is BrokerDeliveryDriftStatus.STABLE + for item in stable_comparison.comparisons + ) + comparison = compare_broker_delivery_fingerprints(reference, drift) + material_names = { + item.metric_name + for item in comparison.comparisons + if item.status is BrokerDeliveryDriftStatus.MATERIAL_DRIFT + and item.condition_key == "global" + } + assert { + "active_quote_interarrival_ns", + "spread", + "source_timestamp_precision_ns", + "price_decimal_places", + }.issubset(material_names) + assert comparison.material_drift_count > 0 + assert comparison.to_dict()["global_similarity_score"] is None + selection = select_broker_profile( + drift, + requested_condition={}, + selected_at_utc_ns=drift.effective_start_utc_ns, + drift_comparison=comparison, + ) + assert selection.status is BrokerTransferStatus.APPLIED + assert selection.drift_comparison_id == comparison.comparison_id + assert selection.material_drift_count == comparison.material_drift_count + assert ( + BrokerDeliveryFingerprintComparisonV1.from_json(comparison.to_json()) + == comparison + ) + + bounded = compare_broker_delivery_fingerprints( + reference, + drift, + config=BrokerDeliveryDriftConfigV1(max_comparisons=3), + ) + assert len(bounded.comparisons) == 3 + assert bounded.truncated + assert bounded.comparison_candidate_count > 3 + + mismatched = select_broker_profile( + stable, + requested_condition={}, + selected_at_utc_ns=stable.effective_start_utc_ns, + drift_comparison=comparison, + ) + assert mismatched.status is BrokerTransferStatus.REFUSED + assert ( + "drift_comparison_does_not_include_profile" in mismatched.reason_codes + ) + + +def test_successor_is_versioned_without_mutating_prior_synthetic_lineage( + tmp_path: Path, +) -> None: + first_manifest = _capture( + tmp_path / "first", seed=10, wall_start_ns=BASE_WALL_NS + ) + second_manifest = _capture( + tmp_path / "second", + seed=11, + wall_start_ns=BASE_WALL_NS + 24 * 60 * 60 * SECOND_NS, + ) + first = fit_broker_delivery_fingerprint( + tmp_path / "first", (first_manifest,) + ) + stream = _synthetic_stream(first.fingerprint_id) + prior_bytes = stream.to_json() + + successor = fit_broker_delivery_fingerprint( + tmp_path / "second", + (second_manifest,), + supersedes=first, + effective_start_utc_ns=first.effective_start_utc_ns + SECOND_NS, + ) + + assert successor.supersedes_fingerprint_id == first.fingerprint_id + assert successor.fingerprint_id != first.fingerprint_id + assert stream.to_json() == prior_bytes + generated = next( + event for event in stream.events if event.broker_profile_id + ) + assert generated.broker_profile_id == first.fingerprint_id + with pytest.raises(BrokerDeliveryFingerprintIdentityError): + fit_broker_delivery_fingerprint( + tmp_path / "second", + (second_manifest,), + supersedes=first, + effective_start_utc_ns=first.effective_start_utc_ns, + ) + + +def test_fingerprint_artifacts_are_atomic_immutable_and_verified( + tmp_path: Path, +) -> None: + first_manifest = _capture( + tmp_path / "first", seed=12, wall_start_ns=BASE_WALL_NS + ) + second_manifest = _capture( + tmp_path / "second", + seed=13, + wall_start_ns=BASE_WALL_NS + 24 * 60 * 60 * SECOND_NS, + spread=0.0004, + ) + first = fit_broker_delivery_fingerprint( + tmp_path / "first", (first_manifest,) + ) + second = fit_broker_delivery_fingerprint( + tmp_path / "second", (second_manifest,) + ) + target = tmp_path / "profiles" / "broker-fingerprint.json" + + artifact = write_broker_delivery_fingerprint(target, first) + assert artifact.sha256 == hashlib.sha256(target.read_bytes()).hexdigest() + assert artifact.metadata["fingerprint_id"] == first.fingerprint_id + assert load_broker_delivery_fingerprint(target) == first + assert write_broker_delivery_fingerprint(target, first) == artifact + with pytest.raises( + BrokerDeliveryFingerprintArtifactError, match="other content" + ): + write_broker_delivery_fingerprint(target, second) + target.write_text("{}\n", encoding="utf-8") + with pytest.raises(BrokerDeliveryFingerprintArtifactError, match="invalid"): + load_broker_delivery_fingerprint(target) + + +def test_capture_identity_mixing_is_refused(tmp_path: Path) -> None: + first = _capture(tmp_path / "first", seed=14, wall_start_ns=BASE_WALL_NS) + second = _capture( + tmp_path / "second", + seed=15, + wall_start_ns=BASE_WALL_NS + SECOND_NS, + adapter_config_sha256=hashlib.sha256(b"other-config").hexdigest(), + ) + with pytest.raises(BrokerDeliveryFingerprintIdentityError): + fit_broker_delivery_fingerprint(tmp_path, (first, second)) + + +def test_proposal_conditioning_uses_cadence_burst_quiet_and_outage( + tmp_path: Path, +) -> None: + """Delivery topology changes cadence before motif retrieval/generation.""" + manifest = _capture(tmp_path, seed=16, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + query = _motif_query( + tick_intensity=2.0, + interarrival_ns=500_000_000.0, + timestamp_precision_ns=1.0, + ) + + proposal = condition_broker_proposal( + query, + fingerprint, + requested_condition={"symbol": "EURUSD"}, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=BrokerTransferConfigV1(strength=0.5), + ) + + assert proposal.status is BrokerTransferStatus.APPLIED + assert proposal.conditioned_query is not None + assert proposal.metrics_after["interarrival_ns"] != 500_000_000.0 + assert proposal.metrics_after["tick_intensity"] != 2.0 + assert { + "burst_interval_rate", + "quiet_interval_rate", + "outage_or_gap_duration_ns", + }.issubset(proposal.selection.metrics) + assert proposal.conditioned_query.query_id != query.query_id + assert BrokerConditionedProposalV1.from_json(proposal.to_json()) == proposal + + +def test_proposal_conditioning_transfers_timestamp_and_price_precision( + tmp_path: Path, +) -> None: + """Timestamp and source-lexeme precision have separate query evidence.""" + manifest = _capture( + tmp_path, + seed=17, + wall_start_ns=BASE_WALL_NS, + precision_ns=10_000_000, + decimal_places=3, + ) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + proposal = condition_broker_proposal( + _motif_query(timestamp_precision_ns=1.0), + fingerprint, + requested_condition={"symbol": "EURUSD"}, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=BrokerTransferConfigV1(strength=1.0), + ) + + assert proposal.metrics_after["timestamp_precision_ns"] == 10_000_000.0 + assert proposal.metrics_after["price_precision_digits"] == 3.0 + + +def test_zero_strength_proposal_preserves_the_original_query( + tmp_path: Path, +) -> None: + """The lower endpoint of the versioned blend is a true no-op.""" + manifest = _capture(tmp_path, seed=171, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + query = _motif_query() + + proposal = condition_broker_proposal( + query, + fingerprint, + requested_condition={"symbol": "EURUSD"}, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=BrokerTransferConfigV1(strength=0.0), + ) + + assert proposal.status is BrokerTransferStatus.APPLIED + assert proposal.conditioned_query == query + assert proposal.metrics_after == proposal.metrics_before + assert BrokerConditionedProposalV1.from_json(proposal.to_json()) == proposal + + +def test_proposal_selection_retains_stale_evidence( + tmp_path: Path, +) -> None: + """Stale-quote behavior remains available to the rendering stage.""" + manifest = _capture(tmp_path, seed=18, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + selection = select_broker_profile( + fingerprint, + requested_condition={"symbol": "EURUSD"}, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + ) + + assert selection.metrics["stale_quote_rate"] > 0.0 + + +def test_proposal_selection_retains_batching_evidence(tmp_path: Path) -> None: + """Source-message batching remains distinct from stale-quote evidence.""" + manifest = _capture(tmp_path, seed=181, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + selection = select_broker_profile( + fingerprint, + requested_condition={"symbol": "EURUSD"}, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + ) + + assert selection.metrics["source_batch_quote_count"] > 1.0 + assert selection.metric_condition_ids["source_batch_quote_count"] == ( + _cell(fingerprint, "global").condition.condition_id + ) + + +def test_reconnect_cell_uses_only_recorded_backoff(tmp_path: Path) -> None: + """Sparse reconnect evidence backs off explicitly; absent cells refuse.""" + manifest = _capture(tmp_path, seed=19, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + reconnect = select_broker_profile( + fingerprint, + requested_condition={ + "symbol": "EURUSD", + "lifecycle": "post_reconnect", + }, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + ) + absent = select_broker_profile( + fingerprint, + requested_condition={"symbol": "GBPUSD"}, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + ) + + assert reconnect.status is BrokerTransferStatus.BACKED_OFF + assert reconnect.requested_condition_id != reconnect.effective_condition_id + assert "requested_condition_backed_off" in reconnect.reason_codes + assert absent.status is BrokerTransferStatus.REFUSED + assert absent.effective_condition_id is None + assert "requested_condition_absent" in absent.reason_codes + + +def test_proposal_refuses_when_requested_condition_has_no_profile_cell( + tmp_path: Path, +) -> None: + """Unsupported proposal cells produce no query for retrieval or generation.""" + manifest = _capture(tmp_path, seed=191, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + + proposal = condition_broker_proposal( + _motif_query(), + fingerprint, + requested_condition={"symbol": "GBPUSD"}, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + ) + + assert proposal.status is BrokerTransferStatus.REFUSED + assert proposal.conditioned_query is None + assert proposal.metrics_after == proposal.metrics_before + + +def test_profile_effective_period_is_enforced(tmp_path: Path) -> None: + """A profile cannot be selected before or after its effective period.""" + manifest = _capture(tmp_path, seed=20, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint( + tmp_path, + (manifest,), + effective_end_utc_ns=BASE_WALL_NS + 100 * SECOND_NS, + ) + selection = select_broker_profile( + fingerprint, + requested_condition={}, + selected_at_utc_ns=BASE_WALL_NS + 101 * SECOND_NS, + ) + + assert selection.status is BrokerTransferStatus.REFUSED + assert "profile_not_effective_at_selection_time" in selection.reason_codes + + +def _motif_query(**metrics: float) -> ReferenceMotifQueryV1: + values = { + "tick_intensity": 2.0, + "interarrival_ns": 500_000_000.0, + "timestamp_precision_ns": 1.0, + **metrics, + } + return ReferenceMotifQueryV1( + condition=ReferenceMotifConditionV1( + symbol="EURUSD", + feed_epoch_id="epoch:fixture", + session_state="open", + metrics=values, + ), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + used_at_ns=BASE_WALL_NS, + ) + + +def _capture( + root: Path, + *, + seed: int, + wall_start_ns: int, + cadence_ns: int = 100_000_000, + spread: float = 0.0002, + precision_ns: int = 100_000, + decimal_places: int = 4, + completed: bool = True, + clock_correction_ns: int | None = None, + adapter_config_sha256: str = CONFIG_SHA256, +) -> BrokerCaptureSessionManifestV1: + session = BrokerCaptureSessionV1( + adapter_id="fixture.broker", + adapter_version="1.0.0", + adapter_config_sha256=adapter_config_sha256, + protocol="fixture-stream", + environment_id="paper", + server_id="fixture-server", + account_id_sha256=ACCOUNT_SHA256, + host_id_sha256=hashlib.sha256(f"host-{seed}".encode()).hexdigest(), + started_at_utc_ns=wall_start_ns + seed, + started_at_monotonic_ns=BASE_MONOTONIC_NS + seed, + public_metadata={"fixture_seed": seed}, + ) + writer = AppendOnlyBrokerCaptureWriterV1( + root, + session=session, + storage_policy=_storage_policy(), + ) + messages: list[BrokerAdapterMessageV1] = [ + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.PROCESS_START, + reason_code="collector_started", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.CONNECTION_OPEN, + connection_id="connection-1", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.SUBSCRIPTION_ADD, + connection_id="connection-1", + subscription_id="subscription-eurusd", + symbol="EURUSD", + ), + ] + first_quote = _quote( + index=0, + wall_ns=wall_start_ns, + spread=spread, + precision_ns=precision_ns, + decimal_places=decimal_places, + source_batch_id="batch-1", + ) + messages.extend( + (first_quote, BrokerAdapterMessageV1.from_json(first_quote.to_json())) + ) + messages.extend( + _quote( + index=index, + wall_ns=wall_start_ns + index * cadence_ns, + spread=spread, + precision_ns=precision_ns, + decimal_places=decimal_places, + source_batch_id=f"batch-{1 + index // 4}", + ) + for index in range(2, 12) + ) + messages.extend( + ( + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.OUTAGE_START, + reason_code="fixture_outage", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.OUTAGE_END, + gap_duration_ns=2 * SECOND_NS, + reason_code="fixture_outage_recovered", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.RECONNECT, + connection_id="connection-2", + reason_code="fixture_reconnect", + ), + _quote( + index=12, + wall_ns=wall_start_ns + 12 * cadence_ns, + spread=spread, + precision_ns=precision_ns, + decimal_places=decimal_places, + source_batch_id="batch-4", + ), + _quote( + index=13, + wall_ns=wall_start_ns + 13 * cadence_ns, + spread=spread, + precision_ns=precision_ns, + decimal_places=decimal_places, + source_batch_id="batch-4", + ), + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.PROCESS_STOP, + reason_code="collector_stopped", + ), + ) + ) + if clock_correction_ns is not None: + messages.insert( + 3, + BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.CLOCK_CORRECTION, + reason_code="wall_monotonic_divergence", + ), + ) + monotonic_ns = BASE_MONOTONIC_NS + receive_wall_ns = wall_start_ns + for sequence, message in enumerate(messages): + gap = ( + 6 * SECOND_NS + if message.kind is BrokerCaptureEventKind.OUTAGE_END + else cadence_ns + ) + monotonic_ns += gap + receive_wall_ns += gap + writer.append( + BrokerCaptureEventV1( + session_id=session.session_id, + capture_sequence=sequence, + receive_time_utc_ns=receive_wall_ns, + receive_time_monotonic_ns=monotonic_ns, + message=message, + clock_offset_change_ns=( + clock_correction_ns + if message.kind is BrokerCaptureEventKind.CLOCK_CORRECTION + else None + ), + ) + ) + return writer.close( + completed=completed, + limitations=(() if completed else ("collector_failure:fixture",)), + ) + + +def _quote( + *, + index: int, + wall_ns: int, + spread: float, + precision_ns: int, + decimal_places: int, + source_batch_id: str, +) -> BrokerAdapterMessageV1: + bid_text = f"{1.1 + index * 0.0001:.{decimal_places}f}" + bid = float(bid_text) + ask_text = f"{bid + spread:.{decimal_places}f}" + ask = float(ask_text) + return BrokerAdapterMessageV1( + kind=BrokerCaptureEventKind.QUOTE, + source_event_time_ns=wall_ns, + source_timestamp_semantics=BrokerCaptureSourceTimestampSemantics.BROKER_EVENT, + source_timestamp_precision_ns=precision_ns, + source_sequence=index, + source_message_id=f"quote-{index}", + source_batch_id=source_batch_id, + symbol="EURUSD", + bid=bid, + ask=ask, + bid_text=bid_text, + ask_text=ask_text, + price_text_semantics=BrokerCapturePriceTextSemantics.SOURCE_LEXEME, + ) + + +def _storage_policy() -> BrokerCaptureStoragePolicyV1: + return BrokerCaptureStoragePolicyV1( + max_partition_events=7, + max_partition_bytes=2 * 1024**2, + max_partition_duration_ns=60 * SECOND_NS, + max_session_bytes=32 * 1024**2, + high_watermark_bytes=24 * 1024**2, + max_retained_partitions=100, + manifest_reserve_bytes=64 * 1024, + fsync_each_event=False, + ) + + +def _timeline() -> MarketContextTimelineV1: + source = MarketContextSourceV1( + name="Fixture macro schedule", + source_version="2023-12-25-v1", + retrieved_at_ns=BASE_WALL_NS - SECOND_NS, + content_sha256="a" * 64, + adapter_name="fixture-macro", + adapter_version="1.0", + license_name="Fixture-only", + redistribution_allowed=False, + redistribution_constraints=("Fixture only.",), + limitations=("Synthetic test event.",), + ) + event = MarketContextEventV1( + canonical_key="us.fixture.release.2023-12-25", + kind=MarketContextKind.MACRO_RELEASE, + title="Fixture release", + source=source, + source_event_time="2023-12-25T12:00:00+00:00", + source_timezone="UTC", + event_time_ns=BASE_WALL_NS, + first_known_at_ns=BASE_WALL_NS - 2 * SECOND_NS, + available_at_ns=BASE_WALL_NS - 2 * SECOND_NS, + pre_event_ns=SECOND_NS, + post_event_ns=60 * SECOND_NS, + affected_currencies=("USD",), + affected_symbols=("EURUSD",), + confidence=1.0, + precision=MarketContextPrecision.EXACT, + limitations=("Synthetic test event.",), + vintage_id="fixture-v1", + tags=("scheduled",), + ) + return MarketContextTimelineV1( + timeline_version="fixture-2023-12-25-v1", + coverage_start_ns=BASE_WALL_NS - 60 * SECOND_NS, + coverage_end_ns=BASE_WALL_NS + 4 * 24 * 60 * 60 * SECOND_NS, + complete=True, + events=(event,), + limitations=("Fixture timeline only.",), + ) + + +def _synthetic_stream(profile_id: str) -> SyntheticEventStreamV1: + left = SyntheticEventV1.observed( + symbol="EURUSD", + event_time_ns=BASE_WALL_NS, + event_sequence=0, + bid=1.1, + ask=1.1002, + run_id="run-1", + ensemble_member_id="member-1", + source_version_id="source-v1", + source_series_id="series-1", + source_period="202312", + source_row_id=1, + ) + right = SyntheticEventV1.observed( + symbol="EURUSD", + event_time_ns=BASE_WALL_NS + 2 * SECOND_NS, + event_sequence=0, + bid=1.1001, + ask=1.1003, + run_id="run-1", + ensemble_member_id="member-1", + source_version_id="source-v1", + source_series_id="series-1", + source_period="202312", + source_row_id=2, + ) + generated = SyntheticEventV1.generated( + symbol="EURUSD", + event_time_ns=BASE_WALL_NS + SECOND_NS, + event_sequence=0, + bid=1.10005, + ask=1.10025, + run_id="run-1", + ensemble_member_id="member-1", + source_version_id="source-v1", + left_anchor_event_id=left.event_id, + right_anchor_event_id=right.event_id, + generator_id="fixture-generator", + generator_version="1.0.0", + generator_config_id="generator-config-v1", + broker_profile_id=profile_id, + constraint_set_id="constraint-set-v1", + ) + return SyntheticEventStreamV1( + run_id="run-1", + ensemble_member_id="member-1", + symbol="EURUSD", + events=(left, generated, right), + ) + + +def _cell( + fingerprint: BrokerDeliveryFingerprintV1, key: str +) -> BrokerDeliveryCellV1: + return next(item for item in fingerprint.cells if item.condition.key == key) diff --git a/tests/unit/test_cftc_positioning.py b/tests/unit/test_cftc_positioning.py new file mode 100644 index 00000000..9b2e5190 --- /dev/null +++ b/tests/unit/test_cftc_positioning.py @@ -0,0 +1,857 @@ +from __future__ import annotations + +import hashlib +import io +import json +import zipfile +from dataclasses import replace +from datetime import date, datetime, time, timezone +from pathlib import Path +from types import SimpleNamespace +from zoneinfo import ZoneInfo + +import pytest + +from histdatacom.data_analytics.cli import build_parser +from histdatacom.market_context.positioning import ( + CFTC_2025_BACKLOG_URI, + CFTC_HISTORICAL_ARCHIVE_TEMPLATE, + CFTC_HISTORICAL_ARCHIVES, + CFTC_HISTORICAL_COMPRESSED_URI, + CFTC_LEGACY_DATASET_ID, + CFTC_POSITIONING_ADAPTER_VERSION, + CFTC_PRE_METADATA_TEMPLATE, + CFTC_PRE_RESOURCE_TEMPLATE, + CFTC_RELEASE_SCHEDULE_URI, + CFTC_SPECIAL_ANNOUNCEMENTS_URI, + CFTC_TFF_DATASET_ID, + CFTC_WEB_POLICY_URI, + CME_EURGBP_RULE_URI, + CME_FX_QUOTE_URI, + CftcAvailabilityConfidence, + CftcMappingKind, + CftcPositioningConsumer, + CftcPositioningConsumerBindingV1, + CftcPositioningCorpusBuildV1, + CftcPositioningFetchProfileV1, + CftcPositioningBenchmarkSmokeV1, + CftcPositioningPreflightV1, + CftcPositioningQueryStatus, + CftcPositioningQueryV1, + CftcPositioningRawSourceV1, + CftcReportFamily, + CftcReportScope, + CftcRestatementStatus, + apply_cftc_positioning_to_benchmark_events, + apply_cftc_positioning_to_motif_condition, + bind_cftc_positioning_query, + build_cftc_positioning_benchmark_smoke, + build_cftc_positioning_corpus_from_sources, + cftc_positioning_information_inputs, + compare_cftc_positioning_corpora, + preflight_cftc_positioning_corpus, + query_cftc_positioning_corpus, + read_cftc_positioning_corpus, + replay_cftc_positioning_corpus, + require_cftc_positioning_corpus, + validate_cftc_positioning_consumer_binding, + write_cftc_positioning_benchmark_smoke, + write_cftc_positioning_consumer_binding, + write_cftc_positioning_corpus, +) +from histdatacom.synthetic import ( + InformationSplitKind, + ReconstructionInformationManifestV1, + ReconstructionInformationPolicyV1, + ReconstructionInformationSplitV1, + ReconstructionRunV1, + ReconstructionWindowV1, + audit_reconstruction_information, + reconstruction_information_window_plan_id, +) +from histdatacom.synthetic.benchmark import BenchmarkEventV1 +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.motifs import ReferenceMotifConditionV1 + +RETRIEVED_AT_NS = 1_767_225_600_000_000_000 + + +def _profile() -> CftcPositioningFetchProfileV1: + return CftcPositioningFetchProfileV1( + start_date="2005-01-01", + end_date="2025-12-23", + page_size=1000, + max_pages_per_dataset=2, + max_staleness_days=14, + ) + + +def _source( + key: str, + kind: str, + uri: str, + content: bytes, + *, + dataset_id: str | None = None, + family: CftcReportFamily | None = None, + scope: CftcReportScope | None = None, + parameters: dict[str, str] | None = None, + redistribution_allowed: bool = True, +) -> CftcPositioningRawSourceV1: + return CftcPositioningRawSourceV1( + source_key=key, + source_kind=kind, + source_uri=uri, + retrieved_at_ns=RETRIEVED_AT_NS, + content=content, + content_type=( + "application/zip" + if kind == "historical_archive" + else ( + "application/json" + if kind in {"pre_data", "pre_metadata"} + else "text/html" + ) + ), + query_parameters=parameters or {}, + dataset_id=dataset_id, + report_family=family, + report_scope=scope, + redistribution_allowed=redistribution_allowed, + limitations=("fixture source",), + ) + + +def _oi( + family: CftcReportFamily, + scope: CftcReportScope, + code: str, + report_date: str, +) -> int: + return ( + (1000 if family is CftcReportFamily.LEGACY else 2000) + + (100 if scope is CftcReportScope.COMBINED else 0) + + int(code[-2:]) + + int(report_date[-2:]) + ) + + +def _row( + family: CftcReportFamily, + scope: CftcReportScope, + code: str, + report_date: str, +) -> dict[str, str]: + row = { + "id": f"{family.value}-{scope.value}-{code}-{report_date}", + "report_date_as_yyyy_mm_dd": f"{report_date}T00:00:00.000", + "cftc_contract_market_code": code, + "futonly_or_combined": ( + "FutOnly" if scope is CftcReportScope.FUTURES_ONLY else "Combined" + ), + "contract_market_name": f"CONTRACT {code}", + "market_and_exchange_names": f"MARKET {code}", + "open_interest_all": str(_oi(family, scope, code, report_date)), + } + if family is CftcReportFamily.LEGACY: + row.update( + { + "noncomm_positions_long_all": str(500 + int(report_date[-2:])), + "noncomm_positions_short_all": "300", + "comm_positions_long_all": "250", + "comm_positions_short_all": "350", + } + ) + else: + row.update( + { + "lev_money_positions_long_all": str( + 600 + int(report_date[-2:]) + ), + "lev_money_positions_short_all": "275", + "dealer_positions_long_all": "200", + "dealer_positions_short_all": "325", + } + ) + return row + + +def _pre_parameters(profile: CftcPositioningFetchProfileV1) -> dict[str, str]: + quoted_codes = ",".join(f"'{item}'" for item in profile.contract_codes) + return { + "$where": ( + f"cftc_contract_market_code in ({quoted_codes}) " + "AND report_date_as_yyyy_mm_dd between " + f"'{profile.start_date}T00:00:00.000' and " + f"'{profile.end_date}T23:59:59.999'" + ), + "$order": ( + "report_date_as_yyyy_mm_dd,cftc_contract_market_code,futonly_or_combined,id" + ), + "$limit": str(profile.page_size), + "$offset": "0", + } + + +def _archive_content( + family: CftcReportFamily, + scope: CftcReportScope, + *, + market_name: str = "MARKET 099741", +) -> bytes: + report_date = "2014-06-10" + csv_content = ( + "CFTC_Contract_Market_Code,As_of_Date_In_Form_YYYYMMDD," + "Open_Interest_All,Market_and_Exchange_Names\n" + f"099741,{report_date},{_oi(family, scope, '099741', report_date)}," + f'"{market_name}"\n' + ).encode() + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("history.txt", csv_content) + return buffer.getvalue() + + +def _sources( + *, + conflicting_duplicate: bool = False, + identical_duplicate: bool = False, +) -> tuple[CftcPositioningRawSourceV1, ...]: + profile = _profile() + values: list[CftcPositioningRawSourceV1] = [] + report_dates = ( + "2005-01-04", + "2010-05-18", + "2013-12-31", + "2014-06-10", + "2015-06-16", + "2015-06-30", + "2025-09-30", + "2025-10-07", + ) + for dataset_id, family in ( + (CFTC_LEGACY_DATASET_ID, CftcReportFamily.LEGACY), + (CFTC_TFF_DATASET_ID, CftcReportFamily.TFF), + ): + values.append( + _source( + f"pre.{dataset_id}.metadata", + "pre_metadata", + CFTC_PRE_METADATA_TEMPLATE.format(dataset_id=dataset_id), + json.dumps({"id": dataset_id}).encode(), + dataset_id=dataset_id, + family=family, + ) + ) + rows = [ + _row(family, scope, code, report_date) + for report_date in report_dates + for scope in ( + CftcReportScope.FUTURES_ONLY, + CftcReportScope.COMBINED, + ) + for code in ("096742", "099741", "299741") + if code != "299741" or report_date >= "2014-06-10" + if family is not CftcReportFamily.TFF or report_date >= "2006-06-13" + ] + if family is CftcReportFamily.LEGACY and ( + conflicting_duplicate or identical_duplicate + ): + duplicate = dict(rows[0]) + duplicate["id"] = "duplicate-row" + if conflicting_duplicate: + duplicate["open_interest_all"] = "999999" + rows.append(duplicate) + values.append( + _source( + f"pre.{dataset_id}.page-0000", + "pre_data", + CFTC_PRE_RESOURCE_TEMPLATE.format(dataset_id=dataset_id), + json.dumps(rows, sort_keys=True).encode(), + dataset_id=dataset_id, + family=family, + parameters=_pre_parameters(profile), + ) + ) + official = ( + ( + "official.release-schedule", + "release_schedule", + CFTC_RELEASE_SCHEDULE_URI, + b"Commitments of Traders release schedule", + ), + ( + "official.special-announcements", + "special_announcements", + CFTC_SPECIAL_ANNOUNCEMENTS_URI, + b"Commitments of Traders historical special announcements", + ), + ( + "official.historical-compressed-index", + "historical_index", + CFTC_HISTORICAL_COMPRESSED_URI, + b"CFTC historical compressed files", + ), + ( + "official.2025-backlog", + "backlog_schedule", + CFTC_2025_BACKLOG_URI, + b"2025 CFTC Commitments of Traders publication dates", + ), + ( + "official.web-policy", + "license_policy", + CFTC_WEB_POLICY_URI, + b"CFTC web policy public domain", + ), + ( + "cme.fx-quote-conventions", + "quote_orientation", + CME_FX_QUOTE_URI, + b"CME FX quote conventions", + ), + ( + "cme.eurgbp-rule-301", + "quote_orientation", + CME_EURGBP_RULE_URI, + b"CME EURGBP Rule 301", + ), + ) + for key, kind, uri, content in official: + values.append( + _source( + key, + kind, + uri, + content, + redistribution_allowed=not key.startswith("cme."), + ) + ) + for family_text, scope_text, archive_name in CFTC_HISTORICAL_ARCHIVES: + family = CftcReportFamily(family_text) + scope = CftcReportScope(scope_text) + values.append( + _source( + f"official.archive.{archive_name.lower()}", + "historical_archive", + CFTC_HISTORICAL_ARCHIVE_TEMPLATE.format( + archive_name=archive_name + ), + _archive_content(family, scope), + family=family, + scope=scope, + ) + ) + return tuple(values) + + +@pytest.fixture # type: ignore[untyped-decorator] +def corpus_build() -> CftcPositioningCorpusBuildV1: + return build_cftc_positioning_corpus_from_sources( + _sources(), profile=_profile() + ) + + +def _window_ns(value: str) -> int: + return int( + datetime.combine( + date.fromisoformat(value), time(), tzinfo=timezone.utc + ).timestamp() + * 1_000_000_000 + ) + + +def test_build_keeps_family_scope_and_archive_evidence_separate( + corpus_build: CftcPositioningCorpusBuildV1, +) -> None: + corpus = corpus_build.corpus + assert len(corpus.snapshots) == 80 + assert len(corpus.archive_consistency) == 4 + assert all( + item.selected_row_count == 1 for item in corpus.archive_consistency + ) + assert all( + item.matched_pre_rows == 1 for item in corpus.archive_consistency + ) + assert all( + item.open_interest_mismatch_count == 0 + for item in corpus.archive_consistency + ) + assert any(item.missing_week_count > 0 for item in corpus.coverage) + assert all( + item["adapter_version"] == CFTC_POSITIONING_ADAPTER_VERSION + for item in corpus.sources + ) + mappings = { + (item.symbol, item.mapping_kind.value): item + for item in corpus.symbol_mappings + } + assert mappings[("EURUSD", "direct")].quote_convention == "USD per EUR" + assert mappings[("GBPUSD", "direct")].quote_convention == "USD per GBP" + assert mappings[("EURGBP", "direct")].quote_convention == "GBP per EUR" + keys = {item.logical_key for item in corpus.snapshots} + assert "legacy:futures_only:099741:2025-10-07" in keys + assert "tff:combined:099741:2025-10-07" in keys + + +def test_duplicate_rows_are_counted_and_conflicts_fail_closed() -> None: + build = build_cftc_positioning_corpus_from_sources( + _sources(identical_duplicate=True), profile=_profile() + ) + assert build.corpus.duplicate_key_count == 1 + with pytest.raises(ValueError, match="conflicting CFTC rows"): + build_cftc_positioning_corpus_from_sources( + _sources(conflicting_duplicate=True), profile=_profile() + ) + + +def test_archive_name_changes_and_family_transition_remain_explicit() -> None: + sources = list(_sources()) + for index, source in enumerate(sources): + if ( + source.source_kind == "historical_archive" + and source.report_family is CftcReportFamily.LEGACY + and source.report_scope is CftcReportScope.FUTURES_ONLY + ): + sources[index] = replace( + source, + content=_archive_content( + CftcReportFamily.LEGACY, + CftcReportScope.FUTURES_ONLY, + market_name="HISTORICAL MARKET NAME", + ), + ) + break + corpus = build_cftc_positioning_corpus_from_sources( + sources, profile=_profile() + ).corpus + evidence = next( + item + for item in corpus.archive_consistency + if item.report_family is CftcReportFamily.LEGACY + and item.report_scope is CftcReportScope.FUTURES_ONLY + ) + assert evidence.contract_name_change_count == 1 + start = _window_ns("2005-01-05") + transition = query_cftc_positioning_corpus( + corpus, + start_ns=start, + end_ns=start + 3_600_000_000_000, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + symbols=("EURUSD",), + report_families=(CftcReportFamily.LEGACY, CftcReportFamily.TFF), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + assert transition.status is CftcPositioningQueryStatus.MISSING + assert "tff:not-yet-published" in transition.reason + + +def test_release_evidence_distinguishes_delayed_nominal_and_corrected( + corpus_build: CftcPositioningCorpusBuildV1, +) -> None: + by_date = { + item.report_date: item + for item in corpus_build.corpus.snapshots + if item.report_family is CftcReportFamily.LEGACY + and item.report_scope is CftcReportScope.FUTURES_ONLY + and item.contract_code == "099741" + } + delayed = by_date["2025-10-07"].release_evidence + assert delayed.confidence is CftcAvailabilityConfidence.VERIFIED + expected = datetime( + 2025, 11, 21, 15, 30, tzinfo=ZoneInfo("America/New_York") + ) + assert delayed.publication_at_ns == int( + expected.timestamp() * 1_000_000_000 + ) + nominal = by_date["2014-06-10"].release_evidence + assert nominal.confidence is CftcAvailabilityConfidence.NOMINAL + expected_nominal = datetime( + 2014, 6, 13, 15, 30, tzinfo=ZoneInfo("America/New_York") + ) + assert nominal.publication_at_ns == int( + expected_nominal.timestamp() * 1_000_000_000 + ) + corrected = by_date["2010-05-18"] + assert ( + corrected.release_evidence.confidence + is CftcAvailabilityConfidence.RESTATEMENT_QUALIFIED + ) + assert ( + corrected.restatement_status + is CftcRestatementStatus.RESTATED_CURRENT_STATE + ) + premature = by_date["2015-06-16"].release_evidence + assert ( + premature.confidence is CftcAvailabilityConfidence.CORRECTION_QUALIFIED + ) + holiday = by_date["2015-06-30"].release_evidence + assert holiday.confidence is CftcAvailabilityConfidence.CORRECTION_QUALIFIED + assert holiday.restatement_detected_at_ns == int( + datetime( + 2015, 7, 6, 15, 30, tzinfo=ZoneInfo("America/New_York") + ).timestamp() + * 1_000_000_000 + ) + + +def test_ex_post_query_and_pre_2014_eurgbp_leg_mapping( + corpus_build: CftcPositioningCorpusBuildV1, +) -> None: + corpus = corpus_build.corpus + start = _window_ns("2025-10-08") + query = query_cftc_positioning_corpus( + corpus, + start_ns=start, + end_ns=start + 3_600_000_000_000, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + ) + assert query.status is CftcPositioningQueryStatus.READY + assert len(query.snapshots) == 12 + assert query.mapping_kinds["EURGBP"] == CftcMappingKind.DIRECT.value + assert query.derived_values + assert any(key.endswith(".net_change") for key in query.derived_values) + + historical = query_cftc_positioning_corpus( + corpus, + start_ns=_window_ns("2014-01-02"), + end_ns=_window_ns("2014-01-03"), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + symbols=("EURGBP",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + assert historical.status is CftcPositioningQueryStatus.READY + assert ( + historical.mapping_kinds["EURGBP"] + == CftcMappingKind.DERIVED_TWO_LEG.value + ) + assert {item.contract_code for item in historical.snapshots} == { + "096742", + "099741", + } + + stale_start = _window_ns("2025-11-01") + stale = query_cftc_positioning_corpus( + corpus, + start_ns=stale_start, + end_ns=stale_start + 3_600_000_000_000, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + symbols=("EURUSD",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + assert stale.status is CftcPositioningQueryStatus.STALE + assert len(stale.snapshots) == 1 + assert next(iter(stale.age_seconds.values())) > 14 * 86_400 + assert "age_seconds=" in stale.reason + + +def test_strict_ex_ante_fails_on_time_and_current_state_then_accepts_vintage( + corpus_build: CftcPositioningCorpusBuildV1, +) -> None: + corpus = corpus_build.corpus + start = _window_ns("2025-11-24") + before_release = query_cftc_positioning_corpus( + corpus, + start_ns=start, + end_ns=start + 3_600_000_000_000, + as_of_ns=_window_ns("2025-11-18"), + information_mode=InformationMode.EX_ANTE_SIMULATION, + symbols=("EURUSD",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + max_staleness_days=60, + ) + assert before_release.status is CftcPositioningQueryStatus.NOT_AVAILABLE + + current_state = query_cftc_positioning_corpus( + corpus, + start_ns=start, + end_ns=start + 3_600_000_000_000, + as_of_ns=_window_ns("2025-11-22"), + information_mode=InformationMode.EX_ANTE_SIMULATION, + symbols=("EURUSD",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + max_staleness_days=60, + ) + assert ( + current_state.status + is CftcPositioningQueryStatus.RESTATEMENT_INCOMPLETE + ) + + snapshots = tuple( + ( + replace( + item, + restatement_status=CftcRestatementStatus.ORIGINAL_VERIFIED, + snapshot_id="", + ) + if item.report_date == "2025-10-07" + else item + ) + for item in corpus.snapshots + ) + vintage_corpus = replace(corpus, snapshots=snapshots, corpus_id="") + vintage = query_cftc_positioning_corpus( + vintage_corpus, + start_ns=start, + end_ns=start + 3_600_000_000_000, + as_of_ns=_window_ns("2025-11-22"), + information_mode=InformationMode.EX_ANTE_SIMULATION, + symbols=("EURUSD",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + max_staleness_days=60, + ) + assert vintage.status is CftcPositioningQueryStatus.READY + assert all( + value == 0.0 + for key, value in vintage.derived_values.items() + if key.endswith((".net_change", ".rolling_52w_zscore")) + ) + + nominal_start = _window_ns("2014-06-14") + nominal_query = query_cftc_positioning_corpus( + corpus, + start_ns=nominal_start, + end_ns=nominal_start + 3_600_000_000_000, + as_of_ns=nominal_start, + information_mode=InformationMode.EX_ANTE_SIMULATION, + symbols=("EURUSD",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + assert nominal_query.status is CftcPositioningQueryStatus.NOT_AVAILABLE + + +def test_content_addressed_write_read_replay_and_diff( + corpus_build: CftcPositioningCorpusBuildV1, tmp_path: Path +) -> None: + artifacts = write_cftc_positioning_corpus(corpus_build, tmp_path) + corpus_path = Path(artifacts["corpus"].path) + loaded = read_cftc_positioning_corpus(corpus_path) + replayed = replay_cftc_positioning_corpus(corpus_path) + assert loaded.corpus_id == corpus_build.corpus.corpus_id + assert replayed.corpus.corpus_id == corpus_build.corpus.corpus_id + diff = compare_cftc_positioning_corpora(loaded, replayed.corpus) + assert not diff.added_keys + assert not diff.removed_keys + assert not diff.changed_keys + assert not list(tmp_path.rglob("*.tmp-*")) + + bad_path = tmp_path / f"cftc-positioning-corpus-{'0' * 64}.json" + bad_path.write_bytes(corpus_path.read_bytes()) + with pytest.raises(ValueError, match="hash differs"): + read_cftc_positioning_corpus(bad_path) + + +def test_information_and_all_consumer_seams( + corpus_build: CftcPositioningCorpusBuildV1, tmp_path: Path +) -> None: + start = _window_ns("2025-10-08") + query = query_cftc_positioning_corpus( + corpus_build.corpus, + start_ns=start, + end_ns=start + 3_600_000_000_000, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + symbols=("EURUSD",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + provisional_inputs = cftc_positioning_information_inputs( + query, + run_id="run-468", + used_at_ns=start, + split_kind=InformationSplitKind.VALIDATION, + ) + assert len(provisional_inputs) == 1 + assert provisional_inputs[0].allowed_lookahead_ns > 0 + policy = ReconstructionInformationPolicyV1( + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + max_allowed_lookahead_ns=provisional_inputs[0].allowed_lookahead_ns, + ) + run = ReconstructionRunV1( + symbols=("EURUSD",), + source_version_ids=("source:held-out",), + configuration_ids=(policy.policy_id, "cftc:fixture"), + ensemble_member_ids=("member-000",), + base_seed=468, + ) + window = ReconstructionWindowV1( + run_id=run.run_id, + ensemble_member_id="member-000", + symbols=run.symbols, + core_start_ns=start, + core_end_ns=start + 3_600_000_000_000, + ) + inputs = cftc_positioning_information_inputs( + query, + run_id=run.run_id, + used_at_ns=start, + split_kind=InformationSplitKind.VALIDATION, + ) + report_ns = inputs[0].event_time_ns + day_ns = 86_400_000_000_000 + splits = ( + ReconstructionInformationSplitV1( + InformationSplitKind.TRAIN, + report_ns - 30 * day_ns, + report_ns - 20 * day_ns, + ), + ReconstructionInformationSplitV1( + InformationSplitKind.CALIBRATION, + report_ns - 20 * day_ns, + report_ns - 10 * day_ns, + ), + ReconstructionInformationSplitV1( + InformationSplitKind.VALIDATION, + report_ns - 10 * day_ns, + start + 3_600_000_000_000, + ), + ) + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id((window,)), + inputs=inputs, + splits=splits, + ) + audit = audit_reconstruction_information( + manifest, policy, run=run, windows=(window,) + ) + assert audit.accepted, [item.to_dict() for item in audit.findings] + + benchmark_binding = bind_cftc_positioning_query( + query, + consumer=CftcPositioningConsumer.BENCHMARK, + consumer_artifact_id="benchmark-468", + run_id=run.run_id, + window_id=window.window_id, + information_inputs=inputs, + ) + assert CftcPositioningQueryV1.from_dict(query.to_dict()) == query + assert ( + CftcPositioningConsumerBindingV1.from_dict(benchmark_binding.to_dict()) + == benchmark_binding + ) + assert Path( + write_cftc_positioning_consumer_binding( + benchmark_binding, tmp_path + ).path + ).exists() + event = BenchmarkEventV1( + source_event_id="source-1", + symbol="EURUSD", + event_time_ns=start, + event_sequence=0, + bid=1.1, + ask=1.1002, + epoch_id="epoch-1", + session="london", + event_state="ordinary", + sparsity="dense", + ) + projected = apply_cftc_positioning_to_benchmark_events( + (event,), benchmark_binding + ) + assert "cftc_positioning:weekly" in projected[0].event_state + assert projected[0].benchmark_event_id != event.benchmark_event_id + + smoke = build_cftc_positioning_benchmark_smoke( + query, + benchmark_binding, + (event,), + source_artifact_id="held-out-month", + source_sha256=hashlib.sha256(b"held-out").hexdigest(), + reload_events=(event,), + ) + assert smoke.deterministic_reload + assert CftcPositioningBenchmarkSmokeV1.from_dict(smoke.to_dict()) == smoke + assert Path( + write_cftc_positioning_benchmark_smoke(smoke, tmp_path).path + ).exists() + + motif_binding = bind_cftc_positioning_query( + query, + consumer=CftcPositioningConsumer.MOTIF_SELECTION, + consumer_artifact_id="motif-468", + run_id=run.run_id, + window_id=window.window_id, + information_inputs=inputs, + ) + motif = ReferenceMotifConditionV1( + symbol="EURUSD", + feed_epoch_id="epoch-1", + session_state="london", + ) + projected_motif = apply_cftc_positioning_to_motif_condition( + motif, motif_binding + ) + assert motif_binding.state_label in projected_motif.event_tags + assert projected_motif.metrics == motif.metrics + assert motif_binding.metrics + + for consumer in ( + CftcPositioningConsumer.PLANNING, + CftcPositioningConsumer.CARVING, + ): + binding = bind_cftc_positioning_query( + query, + consumer=consumer, + consumer_artifact_id=f"{consumer.value}-468", + run_id=run.run_id, + window_id=window.window_id, + information_inputs=inputs, + ) + artifact = SimpleNamespace( + batch_id=f"{consumer.value}-468", + run_id=run.run_id, + window_id=window.window_id, + ) + validate_cftc_positioning_consumer_binding(artifact, binding) + + +def test_preflight_and_cli_surface( + corpus_build: CftcPositioningCorpusBuildV1, +) -> None: + start = _window_ns("2025-10-08") + decision = require_cftc_positioning_corpus( + corpus_build.corpus, + start_ns=start, + end_ns=start + 3_600_000_000_000, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + symbols=("EURUSD",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + assert decision.ready + assert CftcPositioningPreflightV1.from_dict(decision.to_dict()) == decision + unsupported = preflight_cftc_positioning_corpus( + corpus_build.corpus, + start_ns=start, + end_ns=start + 3_600_000_000_000, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + symbols=("USDJPY",), + report_families=(CftcReportFamily.LEGACY,), + report_scopes=(CftcReportScope.FUTURES_ONLY,), + ) + assert not unsupported.ready + assert "unsupported positioning symbols: USDJPY" in unsupported.reasons + args = build_parser().parse_args( + [ + "cftc-positioning-corpus", + "--artifact-dir", + "artifacts", + "--start-date", + "2010-01-01", + "--end-date", + "2025-12-31", + "--previous-corpus", + "prior.json", + ] + ) + assert args.analytics_command == "cftc-positioning-corpus" + assert args.previous_corpus == "prior.json" diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 51204244..1e601b63 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -209,6 +209,7 @@ def test_help_advertises_quality_preflight_mode() -> None: assert "--quality-preflight-evidence-max-age-seconds" in help_text assert "--quality-preflight-evidence-stale-ok" in help_text assert "--quality-preflight-validation-report" in help_text + assert "--quality-preflight-validation-evidence" in help_text assert "latest" in help_text assert "--quality-preflight-run-validation" in help_text assert "--quality-preflight-sample-size" in help_text @@ -230,6 +231,17 @@ def test_help_advertises_request_json_export() -> None: assert "without submitting work" in help_text +def test_help_advertises_deterministic_random_windows() -> None: + """The public help should expose expression, seed, and bounded-session modes.""" + parser = ArgParser(Options()) + parser._set_args() + help_text = parser.format_help() + + assert "-r, --random-window EXPRESSION" in help_text + assert "--random-seed INTEGER" in help_text + assert "all matching occurrences" in " ".join(help_text.split()) + + def test_help_advertises_pair_groups() -> None: """Named instrument groups should be visible from the main help.""" parser = ArgParser(Options()) @@ -538,6 +550,7 @@ def test_config_file_applies_recurrent_run_defaults( data_directory: {data_dir} cpu_utilization: low batch_size: 123 + timezone: America/New_York keep_runtime: true no_overlap: true orchestration_start: false @@ -571,6 +584,7 @@ def test_config_file_applies_recurrent_run_defaults( assert options.data_directory == str(data_dir) assert options.cpu_utilization == "low" assert options.batch_size == 123 + assert options.output_timezone == "America/New_York" assert options.validate_urls assert options.download_data_archives assert not options.extract_csvs @@ -584,6 +598,58 @@ def test_config_file_applies_recurrent_run_defaults( assert options.verbosity == 2 +def test_cli_timezone_flag_uses_output_projection_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The short and long public flag should populate output_timezone.""" + monkeypatch.setattr( + sys, + "argv", + ["histdatacom", "-z", "Europe/London", "--request-json-out", "-"], + ) + + options = ArgParser(Options())() + + assert options.output_timezone == "Europe/London" + + +def test_api_options_preserve_output_timezone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Programmatic API options should survive the argparse compatibility seam.""" + monkeypatch.setattr(sys, "argv", ["histdatacom"]) + options = Options() + options.from_api = True + options.pairs = {"eurusd"} + options.formats = {"ascii"} + options.timeframes = {"T"} + options.start_yearmonth = "2022-12" + options.output_timezone = "Asia/Tokyo" + + parsed = ArgParser(options)() + + assert parsed.output_timezone == "Asia/Tokyo" + + +def test_api_options_preserve_random_window_contract( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Programmatic expression/seed inputs should survive argparse compatibility.""" + monkeypatch.setattr(sys, "argv", ["histdatacom"]) + options = Options() + options.from_api = True + options.pairs = {"eurusd"} + options.formats = {"ascii"} + options.timeframes = {"T"} + options.random_window = "90m" + options.random_seed = 20260714 + + parsed = ArgParser(options)() + + assert parsed.random_window == "90m" + assert parsed.random_seed == 20260714 + + def test_config_file_keeps_explicit_cli_overrides( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -666,6 +732,105 @@ def test_config_file_applies_pair_groups( assert options.pair_groups == ["majors"] +def test_config_file_applies_random_window_and_seed( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Recurrent YAML requests should round-trip deterministic window inputs.""" + config_path = tmp_path / "random-window.yaml" + config_path.write_text( + """ +histdatacom: + validate_urls: true + pairs: [eurusd] + formats: [ascii] + timeframes: [tick-data-quotes] + random_window: 2d + random_seed: 1729 +""", + encoding="utf-8", + ) + monkeypatch.setattr( + sys, + "argv", + ["histdatacom", "--config", str(config_path)], + ) + + options = ArgParser(Options())() + + assert options.random_window == "2d" + assert options.random_seed == 1729 + + +def test_bounded_session_window_accepts_same_month_without_seed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Both bounds mean all session occurrences, including within one month.""" + monkeypatch.setattr( + sys, + "argv", + [ + "histdatacom", + "-V", + "-p", + "eurusd", + "-f", + "ascii", + "-t", + "tick-data-quotes", + "-s", + "2024-01", + "-e", + "2024-01", + "-r", + "ldn", + ], + ) + + options = ArgParser(Options())() + + assert options.start_yearmonth == "202401" + assert options.end_yearmonth == "202401" + assert options.random_window == "ldn" + assert options.random_seed is None + + +@pytest.mark.parametrize( + "tail", + ( + ("--random-window", "2d"), + ("--random-seed", "4"), + ("--random-window", "ldn--ny", "--random-seed", "4"), + ("-A", "--random-window", "2d", "--random-seed", "4"), + ), +) +def test_random_window_cli_fails_closed( + monkeypatch: pytest.MonkeyPatch, + tail: tuple[str, ...], +) -> None: + """Missing seeds, orphan seeds, malformed forms, and repo mode must fail.""" + monkeypatch.setattr( + sys, + "argv", + [ + "histdatacom", + "-V", + "-p", + "eurusd", + "-f", + "ascii", + "-t", + "tick-data-quotes", + *tail, + ], + ) + + with pytest.raises(SystemExit) as err: + ArgParser(Options())() + + assert err.value.code == 1 + + def test_config_file_applies_major_triangle_pair_group( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -778,6 +943,16 @@ def test_config_file_applies_quality_command_defaults( assert options.quality_profile["reporting"] == { "remediation_catalog_audit": {"enabled": True} } + sources = { + item["path"]: item + for item in options.quality_profile_resolution[ + "effective_value_sources" + ] + } + assert ( + sources["/reporting/remediation_catalog_audit/enabled"]["source"] + == "yaml_config" + ) assert options.quality_fail_on == "never" assert options.quality_max_errors == 2 assert options.quality_max_warnings == 5 @@ -792,6 +967,7 @@ def test_config_file_applies_quality_preflight_defaults( config_path = tmp_path / "quality-preflight.yaml" report_path = tmp_path / "reports" / "preflight.json" preview_path = tmp_path / "reports" / "preflight-profile.md" + validation_path = tmp_path / "reports" / "validation.json" config_path.write_text( f""" histdatacom: @@ -802,6 +978,7 @@ def test_config_file_applies_quality_preflight_defaults( quality_preflight_profile_preview_output: {preview_path} quality_preflight_profile_preview_format: markdown quality_preflight_validation_report: latest + quality_preflight_validation_evidence: {validation_path} quality_preflight_run_validation: true quality_preflight_sample_size: 2 pair_groups: [majors] @@ -827,6 +1004,9 @@ def test_config_file_applies_quality_preflight_defaults( ) assert options.quality_preflight_profile_preview_format == "markdown" assert options.quality_preflight_validation_report_path == "latest" + assert options.quality_preflight_validation_evidence_path == str( + validation_path + ) assert options.quality_preflight_run_validation assert options.quality_preflight_sample_size == 2 assert options.pair_groups == ["majors"] @@ -1041,6 +1221,19 @@ def test_data_quality_cli_loads_quality_profile_file( assert options.quality_profile["name"] == "cli-profile" assert options.quality_profile["source"] == "file" assert options.quality_profile["source_path"] == str(profile_path) + resolution = options.quality_profile_resolution + assert [channel["kind"] for channel in resolution["input_channels"]] == [ + "built_in_default", + "named_profile", + "profile_file", + ] + sources = { + item["path"]: item for item in resolution["effective_value_sources"] + } + assert ( + sources["/rules/ingestion.ascii.row_count/min_row_count"]["source"] + == "profile_file" + ) def test_data_quality_cli_enables_remediation_catalog_audit_profile( @@ -1067,6 +1260,22 @@ def test_data_quality_cli_enables_remediation_catalog_audit_profile( assert options.quality_profile["reporting"] == { "remediation_catalog_audit": {"enabled": True} } + sources = { + item["path"]: item + for item in options.quality_profile_resolution[ + "effective_value_sources" + ] + } + assert sources["/reporting/remediation_catalog_audit/enabled"] == { + "path": "/reporting/remediation_catalog_audit/enabled", + "value": True, + "source": "cli_override", + "profile_name": "operator", + "override": True, + "previous_source": "built_in_default", + "overridden_source": "built_in_default", + "previous_value": False, + } def test_data_quality_cli_merges_remediation_catalog_audit_with_profile( @@ -1119,6 +1328,16 @@ def test_data_quality_cli_merges_remediation_catalog_audit_with_profile( assert options.quality_profile["reporting"] == { "remediation_catalog_audit": {"enabled": True} } + sources = { + item["path"]: item + for item in options.quality_profile_resolution[ + "effective_value_sources" + ] + } + assert ( + sources["/reporting/remediation_catalog_audit/enabled"]["source"] + == "cli_override" + ) def test_quality_profile_preview_uses_normal_profile_validation( @@ -1277,6 +1496,11 @@ def test_repo_quality_columns_are_display_only_for_repo_table( ["histdatacom", "--quality-preflight-evidence", "preflight.json"], ["histdatacom", "--quality-preflight-evidence-max-age-seconds", "60"], ["histdatacom", "--quality-preflight-evidence-stale-ok"], + [ + "histdatacom", + "--quality-preflight-validation-evidence", + "validation.json", + ], ["histdatacom", "--quality", "--quality-preflight-evidence-stale-ok"], ["histdatacom", "--quality", "-D"], ["histdatacom", "--repo-quality", "-A"], @@ -1382,6 +1606,18 @@ def test_api_quality_options_accept_inline_profile( assert parsed.quality_profile["reporting"] == { "remediation_catalog_audit": {"enabled": True} } + assert [ + channel["kind"] + for channel in parsed.quality_profile_resolution["input_channels"] + ] == ["built_in_default", "named_profile", "api_options"] + sources = { + item["path"]: item + for item in parsed.quality_profile_resolution["effective_value_sources"] + } + assert ( + sources["/reporting/remediation_catalog_audit/enabled"]["source"] + == "api_options" + ) def test_argparser_bare_construction_uses_fresh_option_namespace( diff --git a/tests/unit/test_closure_readiness.py b/tests/unit/test_closure_readiness.py index 9d39f80a..e9b5e648 100644 --- a/tests/unit/test_closure_readiness.py +++ b/tests/unit/test_closure_readiness.py @@ -51,6 +51,7 @@ def __init__( precommit_file_mutation_sequence: ( Sequence[Mapping[str, str]] | None ) = None, + pytest_returncode: int = 0, release_returncode: int = 0, ps_outputs: Sequence[str] = ("",), issue_state: str = "OPEN", @@ -81,6 +82,7 @@ def __init__( self.precommit_file_mutation_sequence = tuple( dict(item) for item in precommit_file_mutation_sequence or () ) + self.pytest_returncode = pytest_returncode self.release_returncode = release_returncode self.ps_outputs = tuple(ps_outputs) self.issue_state = issue_state @@ -140,7 +142,7 @@ def __call__( ) if args == ("git", "status", "--porcelain=v1", "--untracked-files=all"): self.status_calls += 1 - if self.precommit_changes and self.status_calls >= 11: + if self.precommit_changes and self.precommit_calls: return _completed(args, stdout=self.precommit_changes) return _completed(args, stdout=self.status_stdout) if args[:3] == ("git", "add", "--"): @@ -270,7 +272,14 @@ def __call__( if args[:3] == (sys.executable, "-m", "histdatacom"): return _completed(args, stdout="usage: histdatacom\n") if args[:3] == (sys.executable, "-m", "pytest"): - return _completed(args, stdout="983 passed\n") + return _completed( + args, + returncode=self.pytest_returncode, + stdout=("983 passed\n" if self.pytest_returncode == 0 else ""), + stderr=( + "pytest failed\n" if self.pytest_returncode != 0 else "" + ), + ) if args[:3] == (sys.executable, "-m", "pre_commit"): index = self.precommit_calls self.precommit_calls += 1 @@ -527,12 +536,6 @@ def test_standalone_gate_run_without_mutation_opt_in_does_not_rerun( runner=runner, ) payload = json.loads(capsys.readouterr().out) - pytest_calls = [ - call - for call in runner.calls - if call[:3] == (sys.executable, "-m", "pytest") - ] - assert exit_code == 0 assert payload["readiness"]["state"] == "ready" assert payload["gates"]["changed_paths_after"] == [] @@ -549,7 +552,10 @@ def test_standalone_gate_run_without_mutation_opt_in_does_not_rerun( "formatter/tool-only mutation rerun is not required" ) assert runner.precommit_calls == 1 - assert len(pytest_calls) == 1 + assert payload["gates"]["final_coverage"]["state"] == "not-applicable" + assert any( + call == (sys.executable, "-m", "pytest") for call in runner.calls + ) def test_standalone_gate_run_non_formatter_mutation_blocks_rerun( @@ -593,6 +599,46 @@ def test_standalone_gate_run_non_formatter_mutation_blocks_rerun( assert report["gates"]["rerun"]["eligible"] is False +def test_full_tests_are_skipped_after_an_earlier_gate_failure( + tmp_path: Path, +) -> None: + """A cheap failure should block before the full plain suite starts.""" + module = _module() + runner = FakeRunner(precommit_returncode=1) + + gates = module.collect_gate_summary( + tmp_path, + run_gates=True, + runner=runner, + ) + + assert gates["state"] == "fail" + assert gates["final_coverage"]["state"] == "not-applicable" + assert [result["name"] for result in gates["results"]][-1] == "pre-commit" + assert not any( + result["name"] == "full-tests" for result in gates["results"] + ) + + +def test_full_plain_test_failure_blocks_closure_readiness( + tmp_path: Path, +) -> None: + """A failing release-independent test gate remains a closure blocker.""" + module = _module() + runner = FakeRunner(pytest_returncode=1) + + gates = module.collect_gate_summary( + tmp_path, + run_gates=True, + runner=runner, + ) + + assert gates["state"] == "fail" + assert gates["final_coverage"]["state"] == "not-applicable" + assert gates["results"][-1]["name"] == "full-tests" + assert gates["results"][-1]["status"] == "fail" + + def test_standalone_gate_run_formatter_rerun_success_clears_gate_blocker( tmp_path: Path, ) -> None: @@ -628,12 +674,17 @@ def test_standalone_gate_run_formatter_rerun_success_clears_gate_blocker( assert report["gates"]["state"] == "pass" assert report["gates"]["required_rerun"]["state"] == "passed" assert report["gates"]["rerun"]["state"] == "pass" + assert report["gates"]["final_coverage"]["state"] == "not-applicable" assert "closure-gates-changed-files" not in ( report["readiness"]["blocking_checks"] ) assert "gate:pre-commit" not in report["readiness"]["blocking_checks"] assert "dirty-worktree" in report["readiness"]["blocking_checks"] assert runner.precommit_calls == 2 + assert any( + result["name"] == "full-tests" + for result in report["gates"]["rerun"]["gates"]["results"] + ) def test_standalone_gate_run_formatter_rerun_failure_blocks_readiness( @@ -1616,7 +1667,8 @@ def test_closure_verification_infers_scope_runs_gates_without_mutation( for call in runner.calls if call[:3] == (sys.executable, "-m", "pytest") ] - assert len(pytest_calls) >= 2 + assert len(pytest_calls) == 2 + assert (sys.executable, "-m", "pytest") in pytest_calls assert not any(call[:3] == ("git", "add", "--") for call in runner.calls) assert not any(call[:3] == ("git", "commit", "-m") for call in runner.calls) assert not any(call[:2] == ("git", "push") for call in runner.calls) @@ -1957,7 +2009,7 @@ def test_execute_workflow_runs_ready_sequence_and_closes_issue( assert any(call[:3] == ("git", "commit", "-m") for call in runner.calls) assert any(call[:2] == ("git", "push") for call in runner.calls) assert any( - call[:3] == (sys.executable, "-m", "pytest") for call in runner.calls + call == (sys.executable, "-m", "pytest") for call in runner.calls ) assert any(call[:3] == ("gh", "issue", "close") for call in runner.calls) @@ -2362,10 +2414,10 @@ def test_execute_workflow_pre_mutation_gates_run_before_git_mutation( runner=runner, ) payload = json.loads(capsys.readouterr().out) - first_pytest = next( + first_full_tests = next( index for index, call in enumerate(runner.calls) - if call[:3] == (sys.executable, "-m", "pytest") + if call == (sys.executable, "-m", "pytest") ) first_precommit = next( index @@ -2384,8 +2436,7 @@ def test_execute_workflow_pre_mutation_gates_run_before_git_mutation( assert payload["pre_mutation_gates"]["enabled"] is True assert payload["pre_mutation_gates"]["state"] == "pass" assert payload["pre_mutation_gates"]["gates"]["state"] == "pass" - assert first_pytest < first_add - assert first_precommit < first_add + assert first_precommit < first_full_tests < first_add assert "## Pre-Mutation Gates" in markdown assert "- State: pass" in markdown assert any(call[:2] == ("git", "push") for call in runner.calls) @@ -2646,6 +2697,9 @@ def test_execute_workflow_pre_mutation_formatter_rerun_success_allows_commit( ) assert payload["pre_mutation_gates"]["rerun"]["changed_paths_after"] == [] assert runner.precommit_calls >= 2 + assert any( + call == (sys.executable, "-m", "pytest") for call in runner.calls + ) assert any(call[:3] == ("git", "add", "--") for call in runner.calls) assert any(call[:3] == ("git", "commit", "-m") for call in runner.calls) assert any(call[:2] == ("git", "push") for call in runner.calls) @@ -3390,7 +3444,7 @@ def test_guided_workflow_runs_gates_writes_reports_and_closes( ).exists() assert len(close_calls) == 1 assert any( - call[:3] == (sys.executable, "-m", "pytest") for call in runner.calls + call == (sys.executable, "-m", "pytest") for call in runner.calls ) assert any( call[:3] == (sys.executable, "-m", "pre_commit") diff --git a/tests/unit/test_container_distribution.py b/tests/unit/test_container_distribution.py new file mode 100644 index 00000000..226c9e54 --- /dev/null +++ b/tests/unit/test_container_distribution.py @@ -0,0 +1,158 @@ +"""Contract tests for the application container distribution surface.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] + + +def _text(relative_path: str) -> str: + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def _workflow() -> dict[str, object]: + loaded = yaml.safe_load(_text(".github/workflows/container.yml")) + assert isinstance(loaded, dict) + return loaded + + +def _triggers(workflow: dict[str, object]) -> dict[str, object]: + triggers = workflow.get("on", workflow.get(True)) + assert isinstance(triggers, dict) + return triggers + + +def test_docker_build_context_is_a_strict_package_allow_list() -> None: + """Large repository data and local state must never enter the context.""" + assert _text(".dockerignore").splitlines() == [ + "**", + "!Dockerfile", + "!pyproject.toml", + "!README.md", + "!LICENSE", + "!container", + "!container/constraints.txt", + "!src", + "!src/**", + ] + assert "!container/constraints.txt" in _text(".gitignore").splitlines() + + +def test_dockerfile_preserves_rootless_one_shot_runtime_contract() -> None: + """The image should be pinned, rootless, writable, and CLI-oriented.""" + dockerfile = _text("Dockerfile") + assert ( + "python:3.13-slim-bookworm@sha256:" + "9d7f287598e1a5a978c015ee176d8216435aaf335ed69ac3c38dd1bbb10e8d64" + in dockerfile + ) + assert dockerfile.count("FROM ${PYTHON_BASE}") == 2 + assert "--constraint container/constraints.txt" in dockerfile + assert "--no-build-isolation" in dockerfile + assert "--wheel-dir /wheels" in dockerfile + assert "--no-index" in dockerfile + assert "tini=0.19.0-1+b3" in dockerfile + assert "WORKDIR /workspace" in dockerfile + assert "USER 10001:10001" in dockerfile + assert 'ENTRYPOINT ["/usr/bin/tini", "--", "histdatacom"]' in dockerfile + assert 'CMD ["--help"]' in dockerfile + for name, path in { + "HISTDATACOM_RUNTIME_HOME": "/workspace/runtime", + "HISTDATACOM_RUNTIME_WORKSPACE": "/workspace", + "HISTDATACOM_TEMPORAL_CACHE_DIR": ("/workspace/cache/temporal-cli"), + }.items(): + assert f"{name}={path}" in dockerfile + assert not re.search( + r"^\s*(HEALTHCHECK|VOLUME)\b", dockerfile, re.MULTILINE + ) + assert dockerfile.count("org.opencontainers.image.") >= 8 + + +def test_container_dependency_graph_is_exactly_constrained() -> None: + """Container rebuilds should not resolve mutable Python version ranges.""" + constraints = [ + line + for line in _text("container/constraints.txt").splitlines() + if line and not line.startswith("#") + ] + assert len(constraints) >= 20 + assert all( + re.fullmatch(r"[A-Za-z0-9_.-]+==[^=\s]+", line) for line in constraints + ) + for required in ( + "polars==1.42.1", + "packaging==26.2", + "temporalio==1.30.0", + "setuptools==83.0.0", + "wheel==0.47.0", + ): + assert required in constraints + + +def test_container_workflow_avoids_dev_pushes_and_publishes_tags_only() -> None: + """Image credits and package writes should be confined to their policy.""" + workflow = _workflow() + triggers = _triggers(workflow) + push = triggers["push"] + assert isinstance(push, dict) + assert push == {"tags": ["v*"]} + pull_request = triggers["pull_request"] + assert isinstance(pull_request, dict) + paths = pull_request["paths"] + assert isinstance(paths, list) + assert "Dockerfile" in paths + assert ".dockerignore" in paths + assert "container/constraints.txt" in paths + assert "src/**" not in paths + assert "pyproject.toml" not in paths + + assert workflow["permissions"] == {"contents": "read"} + jobs = workflow["jobs"] + assert isinstance(jobs, dict) + validate = jobs["validate"] + publish = jobs["publish"] + assert isinstance(validate, dict) + assert isinstance(publish, dict) + assert "permissions" not in validate + assert publish["if"] == ( + "github.event_name == 'push' && " + "startsWith(github.ref, 'refs/tags/v')" + ) + assert publish["permissions"] == { + "contents": "read", + "id-token": "write", + "packages": "write", + } + + workflow_text = _text(".github/workflows/container.yml") + assert "coverage" not in workflow_text.lower() + assert "linux/amd64,linux/arm64" in workflow_text + assert "provenance: mode=max" in workflow_text + assert "sbom: true" in workflow_text + assert "docker/setup-qemu-action@v4.2.0" in workflow_text + assert "docker/setup-buildx-action@v4.2.0" in workflow_text + assert "docker/login-action@v4.4.0" in workflow_text + assert "docker/metadata-action@v6.2.0" in workflow_text + assert "docker/build-push-action@v7.3.0" in workflow_text + + +def test_container_documentation_covers_storage_and_process_lifecycle() -> None: + """Operators should see the persistence boundary and one-shot semantics.""" + guide = _text("docs/container.md") + readme = _text("README.md") + index = _text("docs/index.rst") + for phrase in ( + "one-shot command-line tool", + "histdatacom-workspace", + "10001:10001", + "checksum-verified Temporal binary", + "--deep-runtime", + "ordinary `dev` pushes", + ): + assert phrase in guide + assert "[container guide](docs/container.md)" in readme + assert " container" in index diff --git a/tests/unit/test_data_analytics_feed_epochs.py b/tests/unit/test_data_analytics_feed_epochs.py new file mode 100644 index 00000000..465c4abe --- /dev/null +++ b/tests/unit/test_data_analytics_feed_epochs.py @@ -0,0 +1,533 @@ +"""Tests for uncertainty-aware technological feed-epoch contracts.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from histdatacom.data_analytics import ( + DEFAULT_FEED_EPOCH_FEATURES, + FEED_EPOCH_DEFINITION_SCHEMA_VERSION, + FEED_EPOCH_EVIDENCE_SCHEMA_VERSION, + FEED_EPOCH_FIT_CONFIG_SCHEMA_VERSION, + FeedEpochAssignmentV1, + FeedEpochDefinitionV1, + FeedEpochEvidenceV1, + FeedEpochFitConfigV1, + feed_epoch_definition_to_json, + fit_feed_epochs, + write_feed_epoch_definition, +) +from histdatacom.data_quality import TIME_SERIES_FINGERPRINT_SCHEMA_VERSION + + +def _sha256(value: str) -> str: + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _fingerprint( + index: int, + *, + modern: bool, + symbol: str = "EURUSD", + feature_mode: str = "full", +) -> dict[str, object]: + period = f"2020{index + 1:02d}" + start = index * 10_000 + row_count = 1_000 if modern else 100 + median_gap = 100.0 if modern else 1_000.0 + p95_gap = 250.0 if modern else 2_500.0 + spread = 0.0001 if modern else 0.0008 + precision = 6 if modern else 4 + payload: dict[str, object] = { + "schema_version": TIME_SERIES_FINGERPRINT_SCHEMA_VERSION, + "fingerprint_id": _sha256(f"{symbol}:{period}:{modern}:{feature_mode}"), + "target_axis": { + "data_format": "ascii", + "timeframe": "T", + "symbol": symbol, + "period": period, + "kind": "cache", + }, + "coverage": { + "row_count": row_count, + "parsed_row_count": row_count, + "start_timestamp_utc_ms": start, + "end_timestamp_utc_ms": start + 9_000, + "duration_ms": 3_600_000, + }, + "temporal_topology": { + "row_count": row_count, + "parsed_row_count": row_count, + "interval_count": row_count - 1, + "min_interval_ms": 10 if modern else 1_000, + "median_interval_ms": median_gap, + "duplicate_timestamp_count": 0 if modern else 10, + "suspicious_gap_count": 0 if modern else 5, + "sampling_basis": "observed_sequence", + "computed_from": "direct_cache", + "cache_source": "direct", + }, + "tick_distribution": { + "spread": { + "min": spread / 2, + "median": spread, + "mean": spread, + "max": spread * 2, + }, + "precision": { + "column_decimal_place_counts": { + "bid": {str(precision): row_count}, + "ask": {str(precision): row_count}, + } + }, + }, + "microstructure_dynamics": { + "basis": "observed_sequence", + "computed_from": "direct_cache", + "cache_source": "direct", + "sequence_status": "ok", + "limitations": [], + "interarrival_ms": { + "median": median_gap, + "max": p95_gap * 2, + "quantiles": {"0.95": p95_gap}, + }, + "absolute_spread_change": {"median": spread / (4 if modern else 2)}, + "stale_quote": { + "repeat_rate": 0.02 if modern else 0.45, + "repeat_count": 20 if modern else 45, + "run_count": 1 if modern else 8, + "affected_row_count": 20 if modern else 60, + }, + "burst": {"burst_rate": 0.8 if modern else 0.1}, + }, + "calendar_regimes": { + "schema_version": "histdatacom.time-series-fingerprint-calendar-regimes.v1", + "status": "ok", + "calendar_profile_name": "fixture", + "calendar_profile_version": "1", + "calendar_profile_complete": True, + "session_state_counts": {"market_open": row_count}, + "active_session_counts": { + "london": row_count // 2, + "new_york": row_count - row_count // 2, + }, + "special_tag_counts": {"daily_rollover": 1}, + "holiday_tag_counts": {}, + "event_tag_counts": {"fixture_release": 1}, + }, + "conditional_distributions": { + "schema_version": "histdatacom.time-series-fingerprint-conditional-distributions.v1", + "status": "ok", + "by_active_session": { + "london": {"spread": {"median": spread}}, + "new_york": {"spread": {"median": spread * 1.1}}, + }, + }, + "fingerprint_audit": { + "schema_version": "histdatacom.time-series-fingerprint-audit.v1", + "section_statuses": { + "coverage": "valid", + "temporal_topology": "valid", + "calendar_regimes": "valid", + "tick_distribution": "valid", + "conditional_distributions": "valid", + "microstructure_dynamics": "valid", + }, + "sections_skipped": {}, + "source_status": {"kind": "cache", "readable": True}, + }, + "source": { + "kind": "cache", + "cache_source": "direct", + "path": f"fixture/{symbol}/{period}/.data", + }, + } + if feature_mode == "single_change": + payload["tick_distribution"] = { + "spread": { + "min": 0.0001, + "median": spread, + "mean": spread, + "max": 0.0008, + } + } + payload["microstructure_dynamics"] = { + "basis": "observed_sequence", + "computed_from": "direct_cache", + "cache_source": "direct", + "sequence_status": "ok", + "limitations": [], + "interarrival_ms": { + "median": 100.0, + "quantiles": {"0.95": 250.0}, + }, + "absolute_spread_change": {"median": 0.00001}, + "stale_quote": {"repeat_rate": 0.1}, + "burst": {"burst_rate": 0.2}, + } + payload["temporal_topology"] = { + "row_count": 100, + "parsed_row_count": 100, + "interval_count": 99, + "min_interval_ms": 10, + "duplicate_timestamp_count": 0, + "suspicious_gap_count": 0, + "sampling_basis": "observed_sequence", + "computed_from": "direct_cache", + } + return payload + + +def _evidence_series( + *, feature_mode: str = "full" +) -> tuple[FeedEpochEvidenceV1, ...]: + return tuple( + FeedEpochEvidenceV1.from_fingerprint( + _fingerprint( + index, + modern=index >= 6, + feature_mode=feature_mode, + ) + ) + for index in range(12) + ) + + +def _annual_evidence_series(count: int = 24) -> tuple[FeedEpochEvidenceV1, ...]: + result: list[FeedEpochEvidenceV1] = [] + for index in range(count): + base = FeedEpochEvidenceV1.from_fingerprint( + _fingerprint(index % 12, modern=index >= count // 2) + ) + source_hash = _sha256(f"annual-source:{index}") + result.append( + replace( + base, + period=str(2000 + index), + start_timestamp_utc_ms=index * 10_000, + end_timestamp_utc_ms=index * 10_000 + 9_000, + fingerprint_id=_sha256(f"annual-fingerprint:{index}"), + source_artifact_sha256=source_hash, + evidence_id="", + ) + ) + return tuple(result) + + +def test_canonical_fingerprint_evidence_preserves_conditioning_and_quality() -> ( + None +): + """Epoch evidence should retain hashes, feature provenance, and controls.""" + fingerprint = _fingerprint(0, modern=False) + + evidence = FeedEpochEvidenceV1.from_fingerprint(fingerprint) + payload = evidence.to_dict() + + assert payload["schema_version"] == FEED_EPOCH_EVIDENCE_SCHEMA_VERSION + assert payload["fingerprint_id"] == fingerprint["fingerprint_id"] + assert payload["source_artifact_sha256"] == fingerprint["fingerprint_id"] + assert payload["source_hash_basis"] == "canonical_fingerprint_id" + assert payload["source_kind"] == "cache" + assert payload["conditioning"]["calendar_status"] == "ok" + assert payload["conditioning"]["event_tag_counts"] == {"fixture_release": 1} + assert payload["quality"]["sequence_status"] == "ok" + assert payload["feature_provenance"]["spread_median"] == [ + "tick_distribution.spread.median" + ] + assert FeedEpochEvidenceV1.from_dict(payload) == evidence + + +def test_synthetic_regime_changes_recover_stable_uncertain_boundary() -> None: + """Known cadence, precision, spread, and stale changes should be recovered.""" + definition = fit_feed_epochs(_evidence_series()) + + assert definition.schema_version == FEED_EPOCH_DEFINITION_SCHEMA_VERSION + assert ( + definition.config.schema_version == FEED_EPOCH_FIT_CONFIG_SCHEMA_VERSION + ) + assert definition.valid_for_observation_models is True + assert definition.stability.status == "pass" + assert definition.stability.usable_run_counts == { + "feature_removal": len(DEFAULT_FEED_EPOCH_FEATURES), + "missing_periods": 10, + "sampling": 2, + } + assert len(definition.boundaries) == 1 + boundary = definition.boundaries[0] + assert boundary.left_period == "202006" + assert boundary.right_period == "202007" + assert boundary.support == 1.0 + assert boundary.confidence > 0.0 + assert boundary.uncertainty_start_utc_ms < boundary.central_timestamp_utc_ms + assert boundary.central_timestamp_utc_ms < boundary.uncertainty_end_utc_ms + assert len(definition.epochs) == 2 + assert definition.epochs[0].period_end == "202006" + assert definition.epochs[1].period_start == "202007" + + +def test_assignment_exposes_transition_and_refuses_unstable_artifacts() -> None: + """Assignments must preserve uncertainty and enforce the stability gate.""" + stable = fit_feed_epochs(_evidence_series()) + boundary = stable.boundaries[0] + + transition = stable.assign( + symbol="EURUSD", + timestamp_utc_ms=boundary.central_timestamp_utc_ms, + ) + early = stable.assign(symbol="EURUSD", timestamp_utc_ms=1_000) + late = stable.assign(symbol="EURUSD", timestamp_utc_ms=118_000) + + assert transition.assignment_kind == "transition" + assert transition.boundary_id == boundary.boundary_id + assert FeedEpochAssignmentV1.from_dict(transition.to_dict()) == transition + assert early.epoch_id == "epoch-001" + assert late.epoch_id == "epoch-002" + assert ( + stable.assign(symbol="GBPUSD", timestamp_utc_ms=1_000).assignment_kind + == "out_of_scope" + ) + + unstable_config = FeedEpochFitConfigV1( + feature_names=( + "spread_median", + "log_median_interarrival_ms", + "burst_rate", + ), + min_boundary_support=0.8, + ) + unstable = fit_feed_epochs( + _evidence_series(feature_mode="single_change"), + config=unstable_config, + ) + assert unstable.stability.status == "fail" + with pytest.raises(ValueError, match="stability"): + unstable.assign(symbol="EURUSD", timestamp_utc_ms=1_000) + assert ( + unstable.assign( + symbol="EURUSD", + timestamp_utc_ms=1_000, + require_stable=False, + ).assignment_kind + == "epoch" + ) + + +def test_definition_is_order_independent_serializable_and_lineage_complete() -> ( + None +): + """Source order must not change identity and every source hash must survive.""" + evidence = _evidence_series() + first = fit_feed_epochs(evidence) + second = fit_feed_epochs(reversed(evidence)) + + assert first.definition_id == second.definition_id + assert first.to_dict() == second.to_dict() + restored = FeedEpochDefinitionV1.from_json(first.to_json()) + assert restored == first + assert json.loads(feed_epoch_definition_to_json(first))[ + "definition_id" + ] == (first.definition_id) + sources = first.lineage["sources"] + assert isinstance(sources, list) + assert len(sources) == len(evidence) + assert {source["fingerprint_id"] for source in sources} == { + item.fingerprint_id for item in evidence + } + assert first.lineage["config_id"] == first.config.config_id + assert str(first.lineage["lineage_sha256"]).startswith("sha256:") + + +def test_stability_quantifies_all_required_perturbation_families() -> None: + """Sampling, missing-period, and feature-removal sensitivity stay explicit.""" + definition = fit_feed_epochs(_evidence_series()) + stability = definition.stability.to_dict() + + assert stability["run_counts"]["sampling"] == 2 + assert stability["run_counts"]["missing_periods"] == 10 + assert stability["run_counts"]["feature_removal"] > 0 + assert definition.boundaries[0].support_by_analysis == { + "feature_removal": 1.0, + "missing_periods": 1.0, + "sampling": 1.0, + } + + +def test_bounded_sensitivity_budget_retains_all_families_across_long_history() -> ( + None +): + """Long histories must not spend the run budget before sampling is tested.""" + config = FeedEpochFitConfigV1(max_sensitivity_runs=20) + + definition = fit_feed_epochs(_annual_evidence_series(), config=config) + + assert sum(definition.stability.run_counts.values()) == 20 + assert definition.stability.run_counts["sampling"] == 2 + assert definition.stability.run_counts["feature_removal"] == len( + DEFAULT_FEED_EPOCH_FEATURES + ) + assert definition.stability.run_counts["missing_periods"] == 5 + assert all(definition.stability.usable_run_counts.values()) + assert definition.stability.status == "pass" + + +def test_no_change_produces_one_stable_epoch_without_false_boundaries() -> None: + """A stable observation process should not be forced into date buckets.""" + evidence = tuple( + FeedEpochEvidenceV1.from_fingerprint(_fingerprint(index, modern=True)) + for index in range(12) + ) + + definition = fit_feed_epochs(evidence) + + assert definition.boundaries == () + assert len(definition.epochs) == 1 + assert definition.epochs[0].period_start == "202001" + assert definition.epochs[0].period_end == "202012" + assert definition.stability.status == "pass" + + +def test_panel_membership_change_does_not_create_a_false_epoch() -> None: + """A later stable symbol must not masquerade as a feed technology change.""" + eurusd = tuple( + FeedEpochEvidenceV1.from_fingerprint(_fingerprint(index, modern=True)) + for index in range(12) + ) + gbpusd: list[FeedEpochEvidenceV1] = [] + for index in range(6, 12): + base = FeedEpochEvidenceV1.from_fingerprint( + _fingerprint(index, modern=True, symbol="GBPUSD") + ) + shifted = { + name: value + (10.0 if "spread" not in name else 0.01) + for name, value in base.feature_values.items() + } + gbpusd.append(replace(base, feature_values=shifted, evidence_id="")) + + definition = fit_feed_epochs((*eurusd, *gbpusd)) + + assert definition.symbols == ("EURUSD", "GBPUSD") + assert definition.boundaries == () + assert definition.stability.status == "pass" + assert definition.lineage["panel_normalization"] == ( + "within_symbol_robust_then_period_median" + ) + + +def test_config_and_contract_identity_drift_fail_closed() -> None: + """Config and artifact identities must reject semantic or serialized drift.""" + config = FeedEpochFitConfigV1() + with pytest.raises(ValueError, match="config_id"): + FeedEpochFitConfigV1(config_id="feed-epoch-config:sha256:" + "0" * 64) + + definition = fit_feed_epochs(_evidence_series(), config=config) + payload = definition.to_dict() + payload["period_count"] = 11 + with pytest.raises(ValueError, match="definition_id"): + FeedEpochDefinitionV1.from_dict(payload) + + evidence_payload = _evidence_series()[0].to_dict() + evidence_payload["schema_version"] = "histdatacom.feed-epoch-evidence.v2" + with pytest.raises(ValueError, match="schema"): + FeedEpochEvidenceV1.from_dict(evidence_payload) + + +def test_definition_rejects_forged_stability_lineage_and_epoch_alignment() -> ( + None +): + """Strict artifact readers must reconcile trust evidence before identity.""" + definition = fit_feed_epochs(_evidence_series()) + + stability_payload = definition.to_dict() + stability_payload["stability"]["run_counts"]["sampling"] = 0 + stability_payload["stability"]["usable_run_counts"]["sampling"] = 0 + with pytest.raises(ValueError, match="stability status"): + FeedEpochDefinitionV1.from_dict(stability_payload) + + lineage_payload = definition.to_dict() + lineage_payload["lineage"]["panel_normalization"] = "untrusted" + with pytest.raises(ValueError, match="lineage hash"): + FeedEpochDefinitionV1.from_dict(lineage_payload) + + alignment_payload = definition.to_dict() + alignment_payload["boundaries"][0]["right_period"] = "202008" + with pytest.raises(ValueError, match="align with adjacent epochs"): + FeedEpochDefinitionV1.from_dict(alignment_payload) + + +def test_invalid_or_non_tick_fingerprint_is_rejected() -> None: + """Only readable canonical ASCII tick fingerprints may become evidence.""" + payload = _fingerprint(0, modern=False) + payload["schema_version"] = "histdatacom.time-series-fingerprint.v2" + with pytest.raises(ValueError, match="canonical"): + FeedEpochEvidenceV1.from_fingerprint(payload) + + payload = _fingerprint(0, modern=False) + payload["target_axis"]["timeframe"] = "M1" + with pytest.raises(ValueError, match="tick"): + FeedEpochEvidenceV1.from_fingerprint(payload) + + payload = _fingerprint(0, modern=False) + payload["fingerprint_id"] = "not-a-hash" + with pytest.raises(ValueError, match="sha256"): + FeedEpochEvidenceV1.from_fingerprint(payload) + + payload = _fingerprint(0, modern=False) + payload["target_axis"]["period"] = "202099" + with pytest.raises(ValueError, match="period"): + FeedEpochEvidenceV1.from_fingerprint(payload) + + +def test_duplicate_axes_insufficient_periods_and_resource_limits_fail_closed() -> ( + None +): + """Malformed or unbounded evidence sets should fail before fitting.""" + evidence = _evidence_series() + with pytest.raises(ValueError, match="duplicate feed epoch evidence_id"): + fit_feed_epochs((*evidence, evidence[0])) + duplicate_axis = replace( + evidence[0], + fingerprint_id=_sha256("duplicate-axis"), + source_artifact_sha256=_sha256("duplicate-axis"), + evidence_id="", + ) + with pytest.raises(ValueError, match="duplicates a symbol-period"): + fit_feed_epochs((*evidence, duplicate_axis)) + with pytest.raises(ValueError, match="at least 6"): + fit_feed_epochs(evidence[:5]) + with pytest.raises(ValueError, match="configured maximum"): + fit_feed_epochs( + evidence, + config=FeedEpochFitConfigV1(max_evidence=6), + ) + + +def test_definition_artifact_reference_is_compact_and_replayable( + tmp_path: Path, +) -> None: + """Durable output should be the compact definition, not analytical rows.""" + definition = fit_feed_epochs(_evidence_series()) + path = tmp_path / "epochs" / "definition.json" + + artifact = write_feed_epoch_definition(definition, path) + + assert artifact.kind == "feed-epoch-definition" + assert artifact.sha256 == hashlib.sha256(path.read_bytes()).hexdigest() + assert artifact.metadata["definition_id"] == definition.definition_id + assert artifact.metadata["valid_for_observation_models"] is True + assert FeedEpochDefinitionV1.from_json(path.read_text()) == definition + + +def test_feature_and_sensitivity_limits_are_bounded() -> None: + """Fit configuration should reject unbounded diagnostic policies.""" + with pytest.raises(ValueError, match="feature_names"): + FeedEpochFitConfigV1(feature_names=()) + with pytest.raises(ValueError, match="max_sensitivity_runs"): + FeedEpochFitConfigV1(max_sensitivity_runs=10_000) + with pytest.raises(ValueError, match="unsupported feed epoch feature"): + FeedEpochFitConfigV1(feature_names=("calendar_year",)) diff --git a/tests/unit/test_data_analytics_feed_epochs_v2.py b/tests/unit/test_data_analytics_feed_epochs_v2.py new file mode 100644 index 00000000..e6ccb778 --- /dev/null +++ b/tests/unit/test_data_analytics_feed_epochs_v2.py @@ -0,0 +1,472 @@ +"""Tests for active-time multivariate technological feed epochs.""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path + +import polars as pl +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from histdatacom.data_analytics import ( + FEED_EPOCH_DEFINITION_V2_SCHEMA_VERSION, + FeedEpochEvidenceV2, + FeedEpochFitConfigV2, + analyze_active_time_feed_epochs, + fit_active_time_feed_epochs, + read_active_time_feed_epoch_definition, + scan_active_time_evidence, + write_active_time_feed_epoch_campaign, +) +from histdatacom.data_analytics.cli import main as analytics_main +from histdatacom.data_analytics.feed_epochs_v2 import _pelt_boundaries +from histdatacom.synthetic.observation import ObservationFitEvidenceV1 + + +def _timestamp(period: str, day: int = 6) -> int: + return int( + datetime( + int(period[:4]), + int(period[4:]), + day, + tzinfo=timezone.utc, + ).timestamp() + * 1000 + ) + + +def _cache( + root: Path, + symbol: str, + period: str, + *, + dense: bool, +) -> Path: + start = _timestamp(period) + step = 100 if dense else 2_000 + rows = 120 if dense else 24 + path = ( + root + / "ASCII" + / "T" + / symbol.lower() + / period[:4] + / str(int(period[4:])) + / ".data" + ) + path.parent.mkdir(parents=True, exist_ok=True) + bids = [1.1 + (index % 7) * 0.00001 for index in range(rows)] + frame = pl.DataFrame( + { + "datetime": [start + index * step for index in range(rows)], + "bid": bids, + "ask": [value + (0.0001 if dense else 0.0005) for value in bids], + "vol": [0] * rows, + } + ).with_columns(pl.col("vol").cast(pl.Int32)) + frame.write_ipc(path) + return path + + +def _evidence( + symbol: str, + index: int, + *, + value: float, + omit_duplicate: bool = False, +) -> FeedEpochEvidenceV2: + year = 2020 + index // 12 + month = index % 12 + 1 + period = f"{year}{month:02d}" + features = { + "log_calendar_tick_rate_per_hour": value, + "duplicate_timestamp_rate": value / 100.0, + } + if omit_duplicate: + del features["duplicate_timestamp_rate"] + return FeedEpochEvidenceV2( + symbol=symbol, + period=period, + source_path=f"fixture/{symbol}/{period}/.data", + source_artifact_sha256="sha256:" + + hashlib.sha256(f"{symbol}:{period}".encode()).hexdigest(), + source_size_bytes=100, + start_timestamp_utc_ms=_timestamp(period), + end_timestamp_utc_ms=_timestamp(period) + 86_400_000, + row_count=100, + denominators_ms={ + "calendar_duration_ms": 1_000, + "market_open_duration_ms": 800, + "active_window_duration_ms": 500, + }, + counts={"transition_count": 99}, + feature_values=features, + feature_provenance={name: ("fixture",) for name in features}, + activity_bin_counts={str(index): 100, str(index + 1): 50}, + calendar_policy={"active_time_policy_version": "fixture.v1"}, + ) + + +def _fit_config(**changes) -> FeedEpochFitConfigV2: + values = { + "feature_names": ( + "log_calendar_tick_rate_per_hour", + "duplicate_timestamp_rate", + ), + "min_evidence_periods": 12, + "min_segment_periods": 4, + "min_feature_coverage": 0.75, + "penalty_multiplier": 0.5, + "min_boundary_support": 0.5, + } + values.update(changes) + return FeedEpochFitConfigV2(**values) + + +def test_cache_scan_uses_explicit_denominators_and_real_source_hash( + tmp_path: Path, +) -> None: + """Rates must name their calendar, open, and observed-active bases.""" + path = _cache(tmp_path, "EURUSD", "202001", dense=True) + + evidence = scan_active_time_evidence( + path, + symbol="EURUSD", + period="202001", + config=_fit_config(), + ) + + assert evidence.row_count == 120 + assert set(evidence.denominators_ms) == { + "calendar_duration_ms", + "market_open_duration_ms", + "active_window_duration_ms", + } + assert ( + evidence.denominators_ms["calendar_duration_ms"] == 31 * 24 * 3_600_000 + ) + assert ( + evidence.denominators_ms["market_open_duration_ms"] + < evidence.denominators_ms["calendar_duration_ms"] + ) + assert evidence.feature_values[ + "log_calendar_tick_rate_per_hour" + ] == pytest.approx( + math.log1p(120 * 3_600_000 / (31 * 24 * 3_600_000)), + abs=1e-10, + ) + assert evidence.counts["market_open_row_count"] == 120 + assert evidence.counts["active_window_interval_count"] == 119 + assert evidence.feature_values[ + "log_market_open_tick_rate_per_hour" + ] == pytest.approx( + math.log1p( + evidence.counts["market_open_row_count"] + * 3_600_000 + / evidence.denominators_ms["market_open_duration_ms"] + ), + abs=1e-10, + ) + assert evidence.feature_values[ + "log_active_window_tick_rate_per_hour" + ] == pytest.approx( + math.log1p( + evidence.counts["active_window_interval_count"] + * 3_600_000 + / evidence.denominators_ms["active_window_duration_ms"] + ), + abs=1e-10, + ) + payload = evidence.to_dict() + del payload["counts"]["active_window_interval_count"] + payload["evidence_id"] = "" + with pytest.raises(ValueError, match="missing rate numerator"): + FeedEpochEvidenceV2.from_dict(payload) + assert evidence.source_artifact_sha256 == ( + "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + ) + assert evidence.calendar_policy["active_time_policy_version"] + + +def test_feature_extraction_is_bounded_and_deterministic( + tmp_path: Path, +) -> None: + """A repeated scan must preserve compact identities and feature values.""" + path = _cache(tmp_path, "EURUSD", "202001", dense=True) + + left = scan_active_time_evidence( + path, symbol="EURUSD", period="202001", config=_fit_config() + ) + right = scan_active_time_evidence( + path, symbol="EURUSD", period="202001", config=_fit_config() + ) + + assert left == right + assert len(left.activity_bin_counts) <= 800 + assert left.feature_values["price_precision_digits"] == 5 + assert left.feature_values["joint_move_rate"] > 0.9 + assert 0 <= left.feature_values["timestamp_last_digit_entropy"] <= 1 + assert ( + sum( + count + for name, count in left.counts.items() + if name.startswith("timestamp_last_digit_") + ) + == left.row_count + ) + assert left.feature_values["session_activity_share_asia"] == 1.0 + + +def test_stale_run_summary_crosses_arrow_record_batch_boundaries( + tmp_path: Path, +) -> None: + """Unchanged-run lengths must survive cache record-batch boundaries.""" + path = _cache(tmp_path, "EURUSD", "202001", dense=True) + start = _timestamp("202001") + prices = [1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 3.0] + chunks = [] + for offset, values in ((0, prices[:2]), (2, prices[2:])): + chunks.append( + pl.DataFrame( + { + "datetime": [ + start + (offset + index) * 100 + for index in range(len(values)) + ], + "bid": values, + "ask": [value + 0.0001 for value in values], + "vol": [0] * len(values), + } + ).with_columns(pl.col("vol").cast(pl.Int32)) + ) + pl.concat(chunks, rechunk=False).write_ipc(path) + + evidence = scan_active_time_evidence( + path, symbol="EURUSD", period="202001", config=_fit_config() + ) + + assert evidence.counts["stale_run_count"] == 3 + assert evidence.counts["stale_run_p95"] == 3 + assert evidence.counts["stale_run_max"] == 3 + assert evidence.feature_values["log_stale_run_p95"] == pytest.approx( + math.log1p(3), abs=1e-10 + ) + + +@given( + boundary=st.integers(min_value=4, max_value=20), + left=st.floats(min_value=-5, max_value=-1, allow_nan=False), + right=st.floats(min_value=1, max_value=5, allow_nan=False), +) +@settings(max_examples=30, deadline=None) +def test_pelt_recovers_controlled_single_boundary( + boundary: int, left: float, right: float +) -> None: + """The exact PELT objective should recover a strong supported shift.""" + count = boundary + 8 + rows = tuple( + (left, left / 2) if index < boundary else (right, right / 2) + for index in range(count) + ) + + detected, gain = _pelt_boundaries(rows, min_segment=4, penalty=0.5) + + assert detected == (boundary,) + assert gain > 0 + + +def test_pelt_respects_minimum_segment_and_missing_features() -> None: + """Short excursions and absent cells must not create illegal segments.""" + rows = ( + *((0.0, None),) * 8, + *((9.0, None),) * 2, + *((0.0, None),) * 8, + *((8.0, 1.0),) * 8, + ) + + detected, _gain = _pelt_boundaries(rows, min_segment=6, penalty=1.0) + + assert all( + right - left >= 6 + for left, right in zip((0, *detected), (*detected, len(rows))) + ) + assert 8 not in detected and 10 not in detected + + +def test_panel_fit_recovers_shared_epochs_and_symbol_deviation() -> None: + """Shared boundaries and a later one-symbol shift stay separate.""" + evidence = [] + for symbol in ("EURUSD", "GBPUSD", "EURGBP"): + for index in range(36): + value = 1.0 if index < 12 else (9.0 if index < 24 else 15.0) + if symbol == "GBPUSD" and index >= 30: + value += 10.0 + evidence.append(_evidence(symbol, index, value=value)) + + definition = fit_active_time_feed_epochs(evidence, config=_fit_config()) + + assert [item.right_period for item in definition.boundaries] == [ + "202101", + "202201", + ] + assert definition.stability.status == "pass" + assert definition.valid_for_observation_models + assert any( + item.symbol == "GBPUSD" and item.right_period == "202207" + for item in definition.symbol_deviations + ) + assert definition.lineage["algorithm"] == "robust_multivariate_pelt" + assert definition.schema_version == FEED_EPOCH_DEFINITION_V2_SCHEMA_VERSION + projected = ObservationFitEvidenceV1.from_feed_epoch_evidence( + evidence[0], definition + ) + assert projected.evidence_kind == "active_time_feed_epoch" + + +def test_duplicate_only_candidate_is_rejected_by_family_holdout() -> None: + """A boundary that vanishes without duplicates cannot enter the definition.""" + evidence = [] + for symbol in ("EURUSD", "GBPUSD", "EURGBP"): + for index in range(24): + item = _evidence(symbol, index, value=1.0) + features = dict(item.feature_values) + features["duplicate_timestamp_rate"] = 0.0 if index < 12 else 0.8 + evidence.append( + replace( + item, + feature_values=features, + evidence_id="", + ) + ) + + definition = fit_active_time_feed_epochs(evidence, config=_fit_config()) + + assert not definition.boundaries + assert definition.stability.status == "fail" + rejected = definition.stability.rejected_candidates["202101"] + assert rejected["support_by_family"]["duplicate_policy"] == 0 + + +def test_common_period_and_feature_coverage_fail_closed() -> None: + """Missing axes reduce common support and unsupported features are omitted.""" + evidence = [ + _evidence( + symbol, + index, + value=1.0 if index < 6 else 8.0, + omit_duplicate=index % 3 == 0, + ) + for symbol in ("EURUSD", "GBPUSD", "EURGBP") + for index in range(12) + if not (symbol == "EURGBP" and index == 3) + ] + config = _fit_config( + min_evidence_periods=12, + min_segment_periods=4, + min_feature_coverage=0.9, + ) + + definition = fit_active_time_feed_epochs(evidence, config=config) + + assert definition.period_count == 11 + assert definition.feature_names == ("log_calendar_tick_rate_per_hour",) + assert definition.stability.status == "fail" + assert "insufficient_common_period_support" in definition.stability.reasons + + +def test_observation_projection_rejects_unstable_v2_definition() -> None: + """Downstream fitting must not treat failed v2 definitions as valid.""" + evidence = [ + _evidence(symbol, index, value=1.0) + for symbol in ("EURUSD", "GBPUSD", "EURGBP") + for index in range(12) + ] + definition = fit_active_time_feed_epochs(evidence, config=_fit_config()) + assert not definition.valid_for_observation_models + + with pytest.raises(ValueError, match="has not passed stability"): + ObservationFitEvidenceV1.from_feed_epoch_evidence( + evidence[0], definition + ) + + +def test_campaign_scans_all_discovered_caches_and_writes_hashed_artifacts( + tmp_path: Path, +) -> None: + """The real execution path must publish definition/evidence/run sidecars.""" + periods = ("202001", "202002", "202003", "202004") + for symbol in ("EURUSD", "GBPUSD", "EURGBP"): + for index, period in enumerate(periods): + _cache(tmp_path, symbol, period, dense=index >= 2) + config = _fit_config( + min_evidence_periods=4, + min_segment_periods=2, + min_boundary_support=0.0, + penalty_multiplier=0.05, + ) + + campaign = analyze_active_time_feed_epochs([tmp_path], config=config) + artifacts = write_active_time_feed_epoch_campaign( + campaign, tmp_path / "artifacts" + ) + + assert campaign.source_count == 12 + assert campaign.definition.symbols == ("EURGBP", "EURUSD", "GBPUSD") + assert set(artifacts) == {"campaign", "definition", "evidence"} + for artifact in artifacts.values(): + path = Path(artifact.path) + assert hashlib.sha256(path.read_bytes()).hexdigest() == artifact.sha256 + definition = json.loads(Path(artifacts["definition"].path).read_text()) + assert definition["definition_id"] == campaign.definition.definition_id + assert ( + read_active_time_feed_epoch_definition(artifacts["definition"].path) + == campaign.definition + ) + definition["period_count"] += 1 + tampered = tmp_path / "tampered-definition.json" + tampered.write_text(json.dumps(definition), encoding="utf-8") + with pytest.raises(ValueError, match="definition_id"): + read_active_time_feed_epoch_definition(tampered) + + +def test_v2_cli_runs_the_cache_campaign(tmp_path: Path, capsys) -> None: + """The existing analytics CLI must expose the complete v2 execution path.""" + periods = ("202001", "202002", "202003", "202004") + for symbol in ("EURUSD", "GBPUSD", "EURGBP"): + for index, period in enumerate(periods): + _cache(tmp_path, symbol, period, dense=index >= 2) + output = tmp_path / "artifacts" + + exit_code = analytics_main( + [ + "feed-epochs-v2", + "--target", + str(tmp_path / "ASCII" / "T"), + "--artifact-dir", + str(output), + "--features", + "log_calendar_tick_rate_per_hour", + "duplicate_timestamp_rate", + "--min-evidence-periods", + "4", + "--min-segment-periods", + "2", + "--min-boundary-support", + "0", + "--penalty-multiplier", + "0.05", + "--json", + ] + ) + + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["source_count"] == 12 + assert payload["definition"]["schema_version"].endswith(".v2") + assert Path(payload["artifacts"]["definition"]["path"]).exists() diff --git a/tests/unit/test_data_analytics_feed_regimes.py b/tests/unit/test_data_analytics_feed_regimes.py index 478d1dea..2b671376 100644 --- a/tests/unit/test_data_analytics_feed_regimes.py +++ b/tests/unit/test_data_analytics_feed_regimes.py @@ -58,10 +58,10 @@ def _sampled_long_history_dataset(root: Path) -> tuple[Path, Path]: return sparse, dense -def test_discovery_is_separate_from_data_quality_targets( +def test_discovery_projects_canonical_data_quality_targets( tmp_path: Path, ) -> None: - """Analytics discovery should expose its own target contract.""" + """Analytics discovery should reuse the canonical target scanner.""" sparse, dense = _sampled_long_history_dataset(tmp_path) discovery = discover_analytics_targets((tmp_path,)) @@ -74,6 +74,9 @@ def test_discovery_is_separate_from_data_quality_targets( assert discovery.metadata["quality_semantics"] == ( "analytics-only; no pass/fail status" ) + assert discovery.metadata["discovery_basis"] == ( + "canonical_quality_discovery" + ) def test_feed_regime_analysis_segments_sparse_and_dense_periods( @@ -92,9 +95,15 @@ def test_feed_regime_analysis_segments_sparse_and_dense_periods( assert payload["schema_version"] == ANALYTICS_REPORT_SCHEMA_VERSION assert payload["operation"] == "feed-regime-detection" assert payload["summary"]["symbols"] == ["EURUSD"] - assert {"sparse", "dense"} <= labels + assert labels == {"epoch-001", "epoch-002"} + assert payload["epoch_definition"]["boundaries"][0]["right_period"] == ( + "202201" + ) + assert payload["metadata"]["fitting_basis"] == ( + "canonical_time_series_fingerprint" + ) assert profile_by_period["200101"].quiet_gap_count == 2 - assert profile_by_period["200101"].zero_change_run_count == 1 + assert profile_by_period["200101"].zero_change_run_count == 0 assert profile_by_period["202201"].tick_rate_per_hour > ( profile_by_period["200101"].tick_rate_per_hour ) @@ -110,8 +119,84 @@ def test_feed_regime_report_console_summary(tmp_path: Path) -> None: assert "Feed regime analytics" in summary assert "regimes: 2" in summary - assert "EURUSD 200101-200101 sparse" in summary - assert "EURUSD 202201-202201 dense" in summary + assert "EURUSD 200101-200101 epoch-001" in summary + assert "EURUSD 202201-202201 epoch-002" in summary + assert "Uncertain transitions" in summary + + +def test_year_bucket_aggregates_canonical_months_without_losing_lineage( + tmp_path: Path, +) -> None: + """Annual fitting should coarsen fingerprints, not rescan or relabel rows.""" + _sampled_long_history_dataset(tmp_path) + + report = analyze_feed_regimes((tmp_path,), bucket="year") + definition = report.epoch_definition + + assert definition is not None + assert {profile.period for profile in report.period_profiles} == { + "2001", + "2022", + } + assert report.metadata["requested_bucket"] == "year" + preparation = report.metadata["evidence_preparation"] + assert preparation["effective_bucket"] == "year" + assert preparation["mixed_granularity_policy"] == ( + "coarsen_to_year_never_disaggregate" + ) + assert definition.lineage["canonical_source_count"] == 2 + assert { + source["period"] for source in definition.lineage["canonical_sources"] + } == { + "200101", + "202201", + } + assert { + source["source_hash_basis"] for source in definition.lineage["sources"] + } == {"canonical_fingerprint_aggregate_id"} + + +def test_mixed_annual_monthly_evidence_uses_lossless_annual_grid( + tmp_path: Path, +) -> None: + """Annual evidence must force safe coarsening and resolve overlapping axes.""" + _write_tick_csv( + tmp_path, + "2001", + ( + ("20010102 000000000", 1.00000, 1.00020, 0), + ("20010102 001000000", 1.00010, 1.00030, 0), + ), + ) + _write_tick_csv( + tmp_path, + "2022", + ( + ("20220103 000000000", 1.10000, 1.10020, 0), + ("20220103 000001000", 1.10010, 1.10030, 0), + ), + ) + _write_tick_csv( + tmp_path, + "202201", + ( + ("20220103 000000000", 1.10000, 1.10020, 0), + ("20220103 000001000", 1.10010, 1.10030, 0), + ), + ) + + report = analyze_feed_regimes((tmp_path,), bucket="month") + preparation = report.metadata["evidence_preparation"] + + assert preparation["effective_bucket"] == "year" + assert preparation["effective_bucket_reason"] == ( + "annual_evidence_cannot_be_safely_disaggregated" + ) + assert preparation["annual_overlap_skip_count"] == 1 + assert {profile.period for profile in report.period_profiles} == { + "2001", + "2022", + } def test_feed_regime_cli_writes_machine_readable_report( @@ -121,6 +206,7 @@ def test_feed_regime_cli_writes_machine_readable_report( """The analytics subcommand should write structured report JSON.""" _sampled_long_history_dataset(tmp_path) report_path = tmp_path / "reports" / "feed-regimes.json" + epoch_path = tmp_path / "reports" / "feed-epochs.json" exit_code = analytics_main( [ @@ -129,6 +215,15 @@ def test_feed_regime_cli_writes_machine_readable_report( str(tmp_path), "--report", str(report_path), + "--epoch-artifact", + str(epoch_path), + "--features", + "log_tick_rate_per_hour", + "log_median_interarrival_ms", + "--min-evidence-periods", + "2", + "--min-segment-periods", + "1", ] ) @@ -139,6 +234,11 @@ def test_feed_regime_cli_writes_machine_readable_report( payload = json.loads(report_path.read_text(encoding="utf-8")) assert payload["operation"] == "feed-regime-detection" assert payload["summary"]["regime_count"] == 2 + epoch_payload = json.loads(epoch_path.read_text(encoding="utf-8")) + assert ( + epoch_payload["definition_id"] + == payload["summary"]["epoch_definition_id"] + ) def test_feed_regime_cli_reads_yaml_config( @@ -149,6 +249,7 @@ def test_feed_regime_cli_reads_yaml_config( _sampled_long_history_dataset(tmp_path) config_path = tmp_path / "histdatacom.yaml" report_path = tmp_path / "reports" / "feed-regimes.json" + epoch_path = tmp_path / "reports" / "feed-epochs.json" config_path.write_text( f""" histdatacom: @@ -156,6 +257,13 @@ def test_feed_regime_cli_reads_yaml_config( command: feed-regimes target: {tmp_path} report: {report_path} + epoch_artifact: {epoch_path} + features: + - log_tick_rate_per_hour + - log_median_interarrival_ms + min_evidence_periods: 2 + min_segment_periods: 1 + max_sensitivity_runs: 3 json: true """, encoding="utf-8", @@ -166,7 +274,12 @@ def test_feed_regime_cli_reads_yaml_config( assert exit_code == 0 payload = json.loads(capsys.readouterr().out) assert payload["schema_version"] == ANALYTICS_REPORT_SCHEMA_VERSION + assert payload["epoch_definition"]["config"]["feature_names"] == [ + "log_tick_rate_per_hour", + "log_median_interarrival_ms", + ] assert report_path.exists() + assert epoch_path.exists() def test_top_level_main_dispatches_analytics_subcommand( diff --git a/tests/unit/test_data_quality_autoregressive.py b/tests/unit/test_data_quality_autoregressive.py new file mode 100644 index 00000000..02079a09 --- /dev/null +++ b/tests/unit/test_data_quality_autoregressive.py @@ -0,0 +1,657 @@ +"""Tests for explicit-order AR, ARMA, and ARIMA diagnostics.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping, cast + +import polars as pl +import pytest + +import histdatacom.data_quality.autoregressive as ar_module +from histdatacom.data_quality.autoregressive import ( + AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY, + AUTOREGRESSIVE_COLUMNS, + AUTOREGRESSIVE_SCHEMA_VERSION, + AUTOREGRESSIVE_SUMMARY_METADATA_KEY, + AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION, + AutoregressiveProfile, + AutoregressiveSpecification, + autoregressive_from_training_frame, + autoregressive_summary, + project_autoregressive_onto_training_frame, +) +from histdatacom.data_quality.classical_model_contracts import ( + ClassicalModelInputProfile, + ClassicalModelResourcePolicy, +) +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprints import ( + FINGERPRINT_AUDIT_SECTIONS, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.profiles import ( + QUALITY_PROFILE_SCHEMA_VERSION, + quality_profile_from_mapping, +) +from histdatacom.data_quality.reporting import ( + QualityExitPolicy, + bounded_quality_payload, + format_quality_console_summary, + quality_report_payload, +) +from histdatacom.data_quality.training_features import ( + ensure_tick_training_features, + training_feature_definitions, +) +from histdatacom.histdata_ascii import ( + CACHE_FILENAME, + TICK, + format_influx_line, + read_polars_cache, + write_polars_cache, +) + + +def test_full_family_is_explicit_deterministic_and_leakage_safe() -> None: + """All three named families should refit independently on shared folds.""" + first = _run(_process_frame(260)) + second = _run(_process_frame(260)) + + assert first.diagnostics == second.diagnostics + assert first.diagnostics["schema_version"] == AUTOREGRESSIVE_SCHEMA_VERSION + evaluation = _mapping(first.diagnostics["evaluation"]) + models = _mapping_rows(evaluation["models"]) + assert {model["family"] for model in models} == {"ar", "arma", "arima"} + assert { + tuple(_mapping(model["configuration"])["order"]) for model in models + } == { + (2, 0, 0), + (1, 0, 1), + (1, 1, 1), + } + rows = [ + row + for model in models + for row in _mapping_rows(_mapping(model)["fold_results"]) + ] + assert rows + assert all(row["future_values_visible"] is False for row in rows) + assert all( + row["residual_state_reused_across_origins"] is False for row in rows + ) + assert all(row["original_scale"] is True for row in rows) + assert evaluation["automatic_winner"] is False + assert evaluation["comparison_semantics"] == "descriptive_shared_folds_only" + assert _mapping(first.diagnostics["backend"])["provider"] == "statsmodels" + + +def test_orders_trends_estimators_and_fixed_parameters_validate_before_fit() -> ( + None +): + """Invalid or ambiguous configurations must not reach the backend.""" + with pytest.raises(ValueError, match="AR requires"): + AutoregressiveSpecification("bad-ar", "ar", 0) + with pytest.raises(ValueError, match="ARMA requires"): + AutoregressiveSpecification("bad-arma", "arma", 1) + with pytest.raises(ValueError, match="ARIMA requires"): + AutoregressiveSpecification("bad-arima", "arima", 1) + with pytest.raises(ValueError, match="constant trend"): + AutoregressiveSpecification("bad-trend", "arima", 1, d=1, trend="c") + with pytest.raises(ValueError, match="AR-only"): + AutoregressiveSpecification( + "bad-estimator", "arma", 1, q=1, estimation_method="burg" + ) + with pytest.raises(ValueError, match="fixed parameters require"): + AutoregressiveSpecification( + "bad-fixed", + "ar", + 1, + estimation_method="burg", + fixed_parameters=(("ar.L1", 0.5),), + ) + + +def test_profile_parser_preserves_first_class_family_configuration() -> None: + """Profile JSON should retain explicit families, orders, and constraints.""" + profile = ( + quality_profile_from_mapping( + { + "schema_version": QUALITY_PROFILE_SCHEMA_VERSION, + "name": "ar-family", + "rules": { + "fingerprint.series": { + "autoregressive": { + "enabled": True, + "projection_specification_ids": ["ar-fixed"], + "specifications": [ + { + "specification_id": "ar-fixed", + "family": "ar", + "p": 2, + "trend": "c", + "initialization_method": "stationary", + "estimation_method": "statespace", + "fixed_parameters": {"ar.L1": 0.4}, + "max_iterations": 17, + } + ], + } + } + }, + } + ) + .fingerprint_profile() + .autoregressive + ) + + assert profile.enabled is True + assert profile.projection_specification_ids == ("ar-fixed",) + assert profile.specifications[0].family == "ar" + assert profile.specifications[0].fixed_parameters == (("ar.L1", 0.4),) + assert profile.specifications[0].max_iterations == 17 + + +def test_short_zero_variance_missing_and_transformed_inputs_fail_safely() -> ( + None +): + """Structural limitations should remain bounded and never trigger filling.""" + short = _run(_process_frame(25)) + assert short.diagnostics["status"] in {"limited", "unavailable"} + constant = _run( + _process_frame(180).with_columns( + pl.lit(1.0).alias("bid"), pl.lit(1.0002).alias("ask") + ) + ) + reasons = _mapping( + _mapping(constant.diagnostics["fit_summary"])["reason_counts"] + ) + assert int(reasons["zero_variance"]) > 0 + + timestamps = [1_000 + index * 100 for index in range(220)] + timestamps = timestamps[:100] + [ + value + 2_000 for value in timestamps[100:] + ] + missing = _run(_process_frame(220, timestamps=timestamps)) + assert missing.diagnostics["forward_fill_policy"] == "never" + assert all( + sample["training_segment_policy"] == "trailing_contiguous_after_missing" + for sample in _mapping_rows( + _mapping(missing.diagnostics["fit_summary"])["fit_samples"] + ) + ) + + transformed = _run( + _process_frame(260), + input_profile=_input_profile( + transform="log_level", differencing_order=1 + ), + ) + assert transformed.diagnostics["original_scale_forecasts"] is True + assert ( + transformed.diagnostics[ + "model_differencing_is_separate_from_input_differencing" + ] + is True + ) + assert ( + _mapping(transformed.diagnostics["input_transform_policy"])["transform"] + == "log_level" + ) + + +def test_dependency_absence_and_one_model_failure_are_isolated( + monkeypatch: Any, +) -> None: + """Optional-backend absence and fold failure should be advisory.""" + monkeypatch.setattr(ar_module, "_load_backend", lambda: None) + unavailable = _run(_process_frame(180)) + assert unavailable.diagnostics["status"] == "unavailable" + assert unavailable.diagnostics["reason"] == "dependency_unavailable" + + monkeypatch.undo() + original = ar_module._fit_specification + + def fail_ar(*args: Any, **kwargs: Any) -> Any: + specification = cast(AutoregressiveSpecification, args[0]) + if specification.family == "ar": + return ar_module._empty_fit("failed", "singularity", 20) + return original(*args, **kwargs) + + monkeypatch.setattr(ar_module, "_fit_specification", fail_ar) + isolated = _run(_process_frame(220)) + models = _mapping_rows( + _mapping(isolated.diagnostics["evaluation"])["models"] + ) + statuses = {model["family"]: model["status"] for model in models} + assert statuses["ar"] == "limited" + assert statuses["arma"] == "ready" + assert statuses["arima"] == "ready" + assert isolated.diagnostics["hard_fail_quality_gate"] is False + + +def test_candidate_fit_memory_and_diagnostic_limits_are_enforced() -> None: + """The inherited #421 policy must bound model work and retained artifacts.""" + resources = ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=64, + max_horizons=4, + max_candidate_orders=1, + max_fit_attempts=1, + max_wall_time_seconds=30, + max_memory_bytes=10_000_000, + max_retained_diagnostics=1, + ) + result = _run( + _process_frame(260), + input_profile=_input_profile(resources=resources), + ) + usage = _mapping(result.diagnostics["resource_usage"]) + evaluation = _mapping(result.diagnostics["evaluation"]) + assert usage["fit_attempt_count"] == 1 + assert evaluation["model_count"] == 1 + assert result.diagnostics["status"] == "limited" + assert result.diagnostics["reason"] == "resource_limit" + assert ( + len( + _mapping_rows( + _mapping(result.diagnostics["fit_summary"])["fit_samples"] + ) + ) + == 1 + ) + + +def test_backend_options_fixed_parameters_and_ets_references_are_recorded() -> ( + None +): + """Backend controls and #422 comparison data should remain explicit.""" + specification = AutoregressiveSpecification( + "ar-fixed", + "ar", + 1, + trend="n", + initialization_method="stationary", + estimation_method="statespace", + fixed_parameters=(("ar.L1", 0.5),), + max_iterations=25, + ) + profile = AutoregressiveProfile( + enabled=True, + specifications=(specification,), + projection_specification_ids=("ar-fixed",), + ) + ets = { + "evaluation": { + "models": [ + { + "status": "ready", + "family": "ses", + "specification_id": "ses", + "model_id": "sha256:ets", + "horizon_metrics": [{"horizon": 1, "mae": 0.1}], + } + ] + } + } + result = autoregressive_from_training_frame( + _process_frame(220), + _fingerprint("EURUSD"), + input_profile=_input_profile(), + profile=profile, + exponential_smoothing=ets, + target=_target("DAT_ASCII_EURUSD_T_201202.csv"), + ) + fit_sample = _mapping_rows( + _mapping(result.diagnostics["fit_summary"])["fit_samples"] + )[0] + references = _mapping_rows( + _mapping(result.diagnostics["evaluation"])[ + "reference_exponential_smoothing" + ] + ) + + assert _mapping(fit_sample["parameters"])["ar.L1"] == pytest.approx(0.5) + assert references[0]["model_id"] == "sha256:ets" + assert references[0]["calculation_basis"] == "shared_regular_grid_folds" + assert ( + _mapping(result.diagnostics["prerequisite_readiness"])[ + "recommendations_applied_automatically" + ] + is False + ) + + +def test_projection_is_flat_identity_safe_available_and_serializable( + tmp_path: Path, +) -> None: + """All family projections should survive duplicate timestamps and IPC.""" + raw = _process_frame(260, duplicate_timestamp=True) + observed = raw.select("datetime", "bid", "ask").to_dicts() + target = _target("DAT_ASCII_EURUSD_T_201202.csv") + result = _run(raw, target=target) + projected = project_autoregressive_onto_training_frame( + raw, result, target=target + ) + + assert projected.select("datetime", "bid", "ask").to_dicts() == observed + assert projected.get_column("row_id").n_unique() == raw.height + assert set(AUTOREGRESSIVE_COLUMNS) <= set(projected.columns) + for family in ("ar", "arma", "arima"): + available = projected.filter(pl.col(f"cm_{family}_forecast_available")) + assert available.height > 0 + assert available.get_column(f"cm_{family}_training_eligible").all() + assert ( + available.get_column(f"cm_{family}_actual").null_count() + == available.height + ) + diagnostic = projected.filter(pl.col("cm_ar_diagnostic_available")) + assert diagnostic.height > 0 + assert diagnostic.get_column("cm_ar_diagnostic_only").all() + + line = format_influx_line( + "eurusd", + "ascii", + TICK, + projected.filter(pl.col("cm_ar_forecast_available")).row(0), + columns=projected.columns, + ) + assert "cm_ar_forecast_available=true" in line + cache = tmp_path / CACHE_FILENAME + write_polars_cache(projected, cache) + restored = read_polars_cache(cache) + assert ( + restored.select(AUTOREGRESSIVE_COLUMNS).to_dicts() + == projected.select(AUTOREGRESSIVE_COLUMNS).to_dicts() + ) + + +def test_projection_survives_masked_timestamps_and_legacy_enrichment() -> None: + """Projection joins by durable identity and enriches legacy raw rows in memory.""" + raw = _process_frame(220) + target = _target("legacy.data", QualityTargetKind.CACHE) + enriched = ensure_tick_training_features(raw, target=target) + result = _run(enriched, target=target) + masked = enriched.with_columns( + pl.when(pl.col("row_id") == 7) + .then(None) + .otherwise(pl.col("timestamp_utc_ms")) + .alias("timestamp_utc_ms") + ) + projected = project_autoregressive_onto_training_frame(masked, result) + dropped = project_autoregressive_onto_training_frame( + enriched.drop("timestamp_utc_ms", "datetime"), result + ) + + assert projected.get_column("row_id").to_list() == list(range(1, 221)) + assert dropped.get_column("cm_ar_forecast_available").any() + legacy = _run(raw, target=target) + assert ( + _mapping(legacy.input_result.contract["source"])[ + "legacy_cache_enriched_on_read" + ] + is True + ) + + +def test_columns_discovery_fingerprint_and_report_surfaces_are_complete() -> ( + None +): + """Schema, opt-in rule, full, bounded, and CLI surfaces must agree.""" + definitions = {row.name: row for row in training_feature_definitions()} + assert set(AUTOREGRESSIVE_COLUMNS) <= set(definitions) + assert all(definitions[name].nullable for name in AUTOREGRESSIVE_COLUMNS) + assert all( + definitions[name].dtype in {"Utf8", "Int64", "Float64", "Boolean"} + for name in AUTOREGRESSIVE_COLUMNS + ) + assert "autoregressive" in FINGERPRINT_AUDIT_SECTIONS + + diagnostics = _run(_process_frame(220)).diagnostics + finding = _finding(_target("DAT_ASCII_EURUSD_T_201202.csv"), diagnostics) + report = QualityReport( + targets=(finding.target,), + rule_results=( + QualityRuleResult( + rule_id=finding.rule_id, + target=finding.target, + findings=(finding,), + ), + ), + ) + full = quality_report_payload(report) + bounded = bounded_quality_payload( + report=report, + operation="data-quality", + check_groups=("fingerprint",), + discovery={"targets": []}, + artifact=None, + decision=QualityExitPolicy().evaluate(report.summary()), + ) + assert AUTOREGRESSIVE_SUMMARY_METADATA_KEY in _mapping(full["metadata"]) + assert AUTOREGRESSIVE_BOUNDED_PAYLOAD_KEY in bounded + assert "Autoregressive models" in format_quality_console_summary(report) + + +def test_fingerprint_rule_is_opt_in_and_summary_matches_golden() -> None: + """The rule should omit defaults and bounded run summaries should be stable.""" + assert not AutoregressiveProfile().enabled + assert "autoregressive" not in _mapping( + HistDataSeriesFingerprintRule() + .evaluate(_target("missing.csv"))[0] + .metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + + findings = [] + for symbol in ("AUDUSD", "EURUSD", "GBPUSD"): + diagnostics = _run(_process_frame(220), symbol=symbol).diagnostics + findings.append( + _finding(_target(f"DAT_ASCII_{symbol}_T_201202.csv"), diagnostics) + ) + summary = autoregressive_summary(findings, target_limit=1) + expected = json.loads( + ( + Path(__file__).parents[1] + / "fixtures" + / "data_quality_reports" + / "autoregressive_summary.json" + ).read_text(encoding="utf-8") + ) + assert summary == expected + assert ( + _mapping(summary)["schema_version"] + == AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION + ) + + +def test_fingerprint_rule_emits_family_from_the_supported_csv_path( + tmp_path: Path, +) -> None: + """The ordinary fingerprint lifecycle should run the opt-in family.""" + source = tmp_path / "DAT_ASCII_EURUSD_T_201202.csv" + source.write_text("\n".join(_tick_lines(220)) + "\n", encoding="ascii") + profile = HistDataFingerprintProfile( + classical_model_input=_input_profile(frequency_ms=1_000), + autoregressive=_profile(), + ) + finding = HistDataSeriesFingerprintRule(profile=profile).evaluate( + _target(str(source)) + )[0] + fingerprint = _mapping( + finding.metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + audit = _mapping(fingerprint["fingerprint_audit"]) + + assert _mapping(fingerprint["autoregressive"])["status"] in { + "ready", + "limited", + } + assert "autoregressive" in audit["sections_expected"] + assert "autoregressive" in audit["sections_emitted"] + assert _mapping(audit["section_statuses"])["autoregressive"] in { + "valid", + "limited", + } + + +def _run( + frame: pl.DataFrame, + *, + input_profile: ClassicalModelInputProfile | None = None, + target: QualityTarget | None = None, + symbol: str = "EURUSD", +) -> Any: + return autoregressive_from_training_frame( + frame, + _fingerprint(symbol), + input_profile=input_profile or _input_profile(), + profile=_profile(), + target=target or _target(f"DAT_ASCII_{symbol}_T_201202.csv"), + ) + + +def _profile() -> AutoregressiveProfile: + return AutoregressiveProfile( + enabled=True, + specifications=( + AutoregressiveSpecification("ar-2", "ar", 2, trend="c"), + AutoregressiveSpecification("arma-1-1", "arma", 1, q=1, trend="c"), + AutoregressiveSpecification("arima-1-1-1", "arima", 1, d=1, q=1), + ), + projection_specification_ids=("ar-2", "arma-1-1", "arima-1-1-1"), + ) + + +def _input_profile(**overrides: Any) -> ClassicalModelInputProfile: + values: dict[str, Any] = { + "enabled": True, + "frequency_ms": 500, + "minimum_training_observations": 20, + "minimum_evaluation_observations": 1, + "step_size": 100, + "horizons": (1, 2), + "resources": ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=64, + max_horizons=4, + max_candidate_orders=16, + max_fit_attempts=32, + max_wall_time_seconds=30, + max_retained_diagnostics=64, + ), + } + values.update(overrides) + return ClassicalModelInputProfile(**values) + + +def _process_frame( + count: int, + *, + timestamps: list[int] | None = None, + duplicate_timestamp: bool = False, +) -> pl.DataFrame: + times = timestamps or [1_000 + index * 100 for index in range(count)] + if duplicate_timestamp and count > 1: + times[1] = times[0] + innovations = [((index * 17) % 13 - 6) * 0.00003 for index in range(count)] + prices = [1.0, 1.0001] + for index in range(2, count): + delta = 0.65 * (prices[-1] - prices[-2]) + innovations[index] + prices.append(prices[-1] + 0.00005 + delta) + return pl.DataFrame( + { + "datetime": times, + "bid": prices, + "ask": [value + 0.0002 for value in prices], + "vol": [0] * count, + } + ) + + +def _fingerprint(symbol: str) -> dict[str, Any]: + return { + "fingerprint_id": f"fingerprint-{symbol.lower()}", + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": symbol, + "period": "201202", + "kind": "cache", + }, + "fingerprint_audit": { + "section_statuses": { + "dependence": "valid", + "stationarity_diagnostics": "limited", + } + }, + "stationarity_diagnostics": {"recommended_transforms": ["log_return"]}, + } + + +def _tick_lines(count: int) -> tuple[str, ...]: + return tuple( + ( + f"20120201 00{index // 60:02d}{index % 60:02d}000," + f"{1.0 + index * 0.001:.6f}," + f"{1.0002 + index * 0.001:.6f},0" + ) + for index in range(count) + ) + + +def _target( + path: str, kind: QualityTargetKind = QualityTargetKind.CSV +) -> QualityTarget: + symbol = next( + ( + item + for item in ("AUDUSD", "EURUSD", "GBPUSD") + if item in path.upper() + ), + "EURUSD", + ) + return QualityTarget( + path=path, + kind=kind, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period="201202", + ) + + +def _finding( + target: QualityTarget, diagnostics: Mapping[str, Any] +) -> QualityFinding: + return QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id="fingerprint.series", + target=target, + metadata={ + TIME_SERIES_FINGERPRINT_METADATA_KEY: { + "autoregressive": dict(diagnostics) + } + }, + ) + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(cast(Mapping[str, Any], value)) + + +def _mapping_rows(value: Any) -> list[dict[str, Any]]: + return [_mapping(row) for row in cast(list[Any], value)] diff --git a/tests/unit/test_data_quality_bounded_payload_contracts.py b/tests/unit/test_data_quality_bounded_payload_contracts.py index 506f7945..20955069 100644 --- a/tests/unit/test_data_quality_bounded_payload_contracts.py +++ b/tests/unit/test_data_quality_bounded_payload_contracts.py @@ -24,7 +24,7 @@ def test_bounded_payload_contract_audit_passes_representative_payload() -> None: assert payload["status"] == "pass" assert payload["finding_count"] == 0 assert payload["findings"] == [] - assert payload["checked_surfaces"]["sequence_contract_count"] == 13 + assert payload["checked_surfaces"]["sequence_contract_count"] == 19 assert "does not read local market data" in payload["non_goals"] @@ -98,6 +98,68 @@ def test_bounded_payload_contract_audit_detects_truncation_mismatch() -> None: assert finding["path"] == "payload_limits.target_summaries.truncated" +def test_bounded_payload_contract_audit_checks_quality_skip_events() -> None: + """Engine skip count metadata should match its bounded event sequence.""" + payload = _representative_payload_copy() + skip_events = payload["quality_engine"]["skip_events"] + assert isinstance(skip_events, dict) + events = skip_events["events"] + assert isinstance(events, list) + events.clear() + + audit = bounded_payload_contract_audit(payload) + + assert audit["status"] == "fail" + finding = _first_finding(audit, "bounded_payload_count_mismatch") + assert finding["path"] == ( + "quality_engine.skip_events.limit_metadata.events.included_count" + ) + + +def test_bounded_payload_contract_audit_checks_remediation_plan_limits() -> ( + None +): + """Remediation-plan item counts should remain independently bounded.""" + payload = _representative_payload_copy() + plan = payload["remediation_catalog_audit"]["remediation_plan"] + assert isinstance(plan, dict) + items = plan["items"] + assert isinstance(items, list) + items.clear() + + audit = bounded_payload_contract_audit(payload) + + assert audit["status"] == "fail" + finding = _first_finding(audit, "bounded_payload_count_mismatch") + assert finding["path"] == ( + "remediation_catalog_audit.payload_limits.remediation_plan." + "included_count" + ) + + +def test_bounded_payload_contract_audit_checks_topology_inspection_limits() -> ( + None +): + """Nested inspection samples should share bounded count invariants.""" + payload = _representative_payload_copy() + targets = payload["fingerprint_topology_attention"]["target_summaries"] + inspected = next( + target for target in targets if "inspection_context" in target + ) + sample_limits = inspected["inspection_context"]["duplicate_timestamps"][ + "limit_metadata" + ]["samples"] + sample_limits["included_count"] = 7 + + audit = bounded_payload_contract_audit(payload) + + assert audit["status"] == "fail" + finding = _first_finding(audit, "bounded_payload_count_mismatch") + assert "inspection_context.duplicate_timestamps.limit_metadata.samples" in ( + finding["path"] + ) + + def test_format_bounded_payload_contract_audit_renders_human_summary() -> None: """Human output should expose pass/fail audit state.""" output = format_bounded_payload_contract_audit( diff --git a/tests/unit/test_data_quality_calendar.py b/tests/unit/test_data_quality_calendar.py index cc45c9d4..3b10e719 100644 --- a/tests/unit/test_data_quality_calendar.py +++ b/tests/unit/test_data_quality_calendar.py @@ -4,6 +4,8 @@ from pathlib import Path +import pytest + from histdatacom.data_quality import ( DOMAIN_CALENDAR_SESSION_RULE_ID, QUALITY_PROFILE_SCHEMA_VERSION, @@ -55,6 +57,31 @@ def test_calendar_policy_documents_optional_static_holiday_scope() -> None: assert policy["month_end_policy"] == "source_calendar_date" assert policy["calendar_profile"]["static_advisory"] is True assert policy["calendar_profile"]["complete"] is False + assert policy["weekend_activity_policy"] == "advisory" + assert policy["expected_session_closure_policy"] == "expected" + + +def test_calendar_profile_validates_explicit_session_policies() -> None: + """Operator profiles should expose explicit weekend and closure policy.""" + profile = calendar_profile_from_mapping( + { + "weekend_activity_policy": "allowed", + "expected_session_closure_policy": "unexpected", + } + ) + + assert profile.weekend_activity_policy == "allowed" + assert profile.expected_session_closure_policy == "unexpected" + assert profile.to_metadata()["weekend_activity_policy"] == "allowed" + + +def test_calendar_profile_rejects_unknown_weekend_policy() -> None: + """Policy values should fail before ambiguous remediation is emitted.""" + with pytest.raises( + ValueError, + match="weekend_activity_policy must be one of", + ): + calendar_profile_from_mapping({"weekend_activity_policy": "maybe"}) def test_domain_calendar_rule_reports_session_and_overlap_counts( diff --git a/tests/unit/test_data_quality_classical_baselines.py b/tests/unit/test_data_quality_classical_baselines.py new file mode 100644 index 00000000..47a18ad6 --- /dev/null +++ b/tests/unit/test_data_quality_classical_baselines.py @@ -0,0 +1,548 @@ +"""Tests for optional classical baseline diagnostics.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping, cast + +import polars as pl + +from histdatacom.data_quality.classical_baselines import ( + CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY, + CLASSICAL_BASELINE_SCHEMA_VERSION, + CLASSICAL_BASELINE_SUMMARY_METADATA_KEY, + CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION, + CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION, + ClassicalBaselineProfile, + classical_baseline_diagnostics_from_training_frame, + classical_baseline_summary, + project_classical_baseline_onto_training_frame, +) +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprints import ( + FINGERPRINT_AUDIT_SECTIONS, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.reporting import ( + QualityExitPolicy, + bounded_quality_payload, + format_quality_console_summary, + quality_report_payload, +) +from histdatacom.data_quality.training_features import ( + ensure_tick_training_features, +) +from histdatacom.histdata_ascii import TICK, format_influx_line +from histdatacom.runtime_contracts import JSONValue + + +def test_classical_baselines_are_deterministic_and_use_explicit_split() -> None: + """Low-dependency models should be repeatable and leakage-resistant.""" + frame = _enriched_frame(30, session_states=(1, 2, 3)) + profile = _profile() + fingerprint = _fingerprint() + + first = classical_baseline_diagnostics_from_training_frame( + frame, fingerprint, profile=profile + ) + second = classical_baseline_diagnostics_from_training_frame( + frame, fingerprint, profile=profile + ) + + assert first == second + assert first["schema_version"] == CLASSICAL_BASELINE_SCHEMA_VERSION + assert first["status"] == "ready" + split = _mapping(first["split_policy"]) + assert split == { + "kind": "chronological_holdout", + "order_by": ["series_id", "period", "row_id"], + "timestamp_required": False, + "shuffle": False, + "walk_forward": True, + "future_values_visible": False, + "evaluation_fraction": 0.2, + "row_count": 30, + "training_row_count": 24, + "evaluation_row_count": 6, + "split_index": 24, + "split_row_id": 25, + "metrics_emitted": True, + } + evaluation = _mapping(first["evaluation"]) + assert evaluation["status"] == "evaluated" + assert evaluation["transforms_applied"] == [] + assert evaluation["guard_codes"] == [] + models = _mapping_rows(evaluation["models"]) + assert [model["model"] for model in models] == [ + "naive_random_walk", + "rolling_mean", + "rolling_median", + "session_seasonal_naive", + ] + assert all(model["status"] == "evaluated" for model in models) + assert _mapping(evaluation["best_model"])["model"] == ("naive_random_walk") + + +def test_classical_baselines_do_not_require_timestamp_for_identity() -> None: + """Timestamp masking must not break deterministic baseline identity.""" + frame = ( + _enriched_frame(31) + .with_columns( + pl.when(pl.col("row_id") == 2) + .then(False) + .otherwise(pl.col("training_usable")) + .alias("training_usable") + ) + .drop("datetime", "timestamp_utc_ms") + ) + + diagnostics = classical_baseline_diagnostics_from_training_frame( + frame, _fingerprint(), profile=_profile() + ) + projected = project_classical_baseline_onto_training_frame( + frame, diagnostics + ) + + assert diagnostics["status"] == "ready" + assert projected.get_column("row_id").to_list() == list(range(1, 32)) + assert projected.get_column("series_id").n_unique() == 1 + assert projected.get_column("baseline_training_ready").all() + assert projected.get_column("baseline_split_row_id").unique().item() == 26 + + +def test_classical_baseline_projection_preserves_observed_rows_and_influx() -> ( + None +): + """Projection should preserve quotes and share the enriched Influx point.""" + frame = _enriched_frame(30, duplicate_timestamp=True) + observed = frame.select("row_id", "datetime", "bid", "ask").to_dicts() + diagnostics = classical_baseline_diagnostics_from_training_frame( + frame, _fingerprint(), profile=_profile() + ) + + projected = project_classical_baseline_onto_training_frame( + frame, diagnostics + ) + + assert projected.select("row_id", "datetime", "bid", "ask").to_dicts() == ( + observed + ) + assert projected.get_column("row_id").n_unique() == 30 + lines = [ + format_influx_line( + "eurusd", + "ascii", + TICK, + row, + columns=projected.columns, + ) + for row in projected.head(2).iter_rows() + ] + assert lines[0].endswith(" 1000") + assert lines[1].endswith(" 1000") + assert "row_id=1" in lines[0].split(" ", maxsplit=1)[0] + assert "row_id=2" in lines[1].split(" ", maxsplit=1)[0] + assert "baseline_status_code=3i" in lines[0] + assert "baseline_training_ready=true" in lines[0] + assert "bidquote=1.0" in lines[0] + + +def test_classical_baselines_enrich_legacy_raw_tick_frame() -> None: + """Legacy raw caches should be enriched before baseline evaluation.""" + raw = _raw_frame(30) + target = _target(".data", QualityTargetKind.CACHE) + + diagnostics = classical_baseline_diagnostics_from_training_frame( + raw, + _fingerprint(), + profile=_profile(), + target=target, + ) + + substrate = _mapping(diagnostics["training_substrate"]) + assert diagnostics["status"] == "ready" + assert substrate["legacy_cache_enriched_on_read"] is True + assert substrate["timestamp_required"] is False + + +def test_classical_baselines_report_insufficient_data_without_metrics() -> None: + """Accuracy metrics require a valid chronological split.""" + diagnostics = classical_baseline_diagnostics_from_training_frame( + _enriched_frame(8), _fingerprint(), profile=_profile() + ) + + assert diagnostics["status"] == "unavailable" + assert diagnostics["reason"] == "insufficient_training_rows" + evaluation = _mapping(diagnostics["evaluation"]) + assert evaluation["status"] == "not_evaluated" + assert evaluation["models"] == [] + projection = _mapping( + _mapping(diagnostics["training_projection"])["values"] + ) + assert _mapping(diagnostics["training_projection"])["schema_version"] == ( + CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert projection["baseline_training_ready"] is False + assert projection["baseline_exclusion_reason_code"] == 3 + + +def test_stationarity_guards_are_advisory_and_do_not_hide_metrics() -> None: + """Limited/unavailable stationarity should change readiness, not severity.""" + for stationarity_status, guard in ( + ("limited", "stationarity_limited"), + ("unavailable", "stationarity_unavailable"), + ): + fingerprint = _fingerprint(stationarity_status=stationarity_status) + diagnostics = classical_baseline_diagnostics_from_training_frame( + _enriched_frame(30), fingerprint, profile=_profile() + ) + + assert diagnostics["status"] == "limited" + prerequisite = _mapping(diagnostics["prerequisite_readiness"]) + assert prerequisite["stationarity_status"] == stationarity_status + evaluation = _mapping(diagnostics["evaluation"]) + assert guard in _strings(evaluation["guard_codes"]) + assert evaluation["evaluated_model_count"] > 0 + assert _mapping(evaluation["best_model"])["mae"] is not None + + +def test_transform_recommendations_are_reported_but_not_applied() -> None: + """Fingerprint transform advice should remain explicit and non-mutating.""" + fingerprint = _fingerprint( + recommended_transforms=( + "log_return", + "differencing", + "session_conditioning", + ) + ) + diagnostics = classical_baseline_diagnostics_from_training_frame( + _enriched_frame(30), fingerprint, profile=_profile() + ) + + evaluation = _mapping(diagnostics["evaluation"]) + assert diagnostics["status"] == "limited" + assert evaluation["transforms_applied"] == [] + assert evaluation["recommended_transforms"] == [ + "log_return", + "differencing", + "session_conditioning", + ] + projection = _mapping( + _mapping(diagnostics["training_projection"])["values"] + ) + assert projection["baseline_transform_advisory_code"] == 7 + + +def test_fingerprint_profile_opt_in_emits_baselines_and_audit_status( + tmp_path: Path, +) -> None: + """The ordinary fingerprint path should emit baselines only when enabled.""" + source = tmp_path / "DAT_ASCII_EURUSD_T_201202.csv" + source.write_text("\n".join(_tick_lines(30)) + "\n", encoding="utf-8") + target = _target(str(source), QualityTargetKind.CSV) + baseline_profile = _profile() + fingerprint_profile = HistDataFingerprintProfile( + rolling_windows=(2, 3), + classical_baselines=baseline_profile, + ) + + finding = HistDataSeriesFingerprintRule( + profile=fingerprint_profile + ).evaluate(target)[0] + fingerprint = _mapping( + finding.metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + default_fingerprint = _mapping( + HistDataSeriesFingerprintRule() + .evaluate(target)[0] + .metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + + assert "classical_baselines" in fingerprint + assert "classical_baselines" not in default_fingerprint + audit = _mapping(fingerprint["fingerprint_audit"]) + assert "classical_baselines" in audit["sections_expected"] + assert "classical_baselines" in audit["sections_emitted"] + assert _mapping(audit["section_statuses"])["classical_baselines"] in { + "valid", + "limited", + } + assert "classical_baselines" in FINGERPRINT_AUDIT_SECTIONS + assert finding.severity.value == "info" + + +def test_classical_baseline_report_bounded_and_console_surfaces( + tmp_path: Path, +) -> None: + """Opt-in diagnostics should not be trapped in nested findings.""" + finding, target = _baseline_finding(tmp_path) + report = QualityReport( + targets=(target,), + rule_results=( + QualityRuleResult( + rule_id=finding.rule_id, + target=target, + findings=(finding,), + ), + ), + metadata={"check_groups": ["fingerprint"]}, + ) + + report_payload = quality_report_payload(report) + summary = _mapping( + _mapping(report_payload["metadata"])[ + CLASSICAL_BASELINE_SUMMARY_METADATA_KEY + ] + ) + bounded = bounded_quality_payload( + operation="data-quality", + check_groups=("fingerprint",), + discovery={"targets": []}, + report=report, + decision=QualityExitPolicy().evaluate(report.summary()), + artifact=None, + ) + console = format_quality_console_summary( + report, check_groups=("fingerprint",) + ) + + assert ( + summary["schema_version"] == CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION + ) + assert summary["target_count"] == 1 + assert CLASSICAL_BASELINE_BOUNDED_PAYLOAD_KEY in bounded + assert "Classical fingerprint baselines" in console + assert "best=naive_random_walk" in console + + +def test_classical_baseline_summary_is_bounded() -> None: + """Run summaries should retain full counts while bounding target details.""" + findings = [] + for symbol in ("AUDUSD", "EURUSD", "GBPUSD"): + target = _target(f"DAT_ASCII_{symbol}_T_201202.csv") + diagnostics = classical_baseline_diagnostics_from_training_frame( + _enriched_frame(30, symbol=symbol), + _fingerprint(symbol=symbol), + profile=_profile(), + ) + findings.append(_finding(target, diagnostics)) + + summary = cast( + Mapping[str, JSONValue], + classical_baseline_summary(findings, target_limit=1), + ) + + assert summary["target_count"] == 3 + assert summary["included_target_count"] == 1 + assert summary["omitted_target_count"] == 2 + assert summary["truncated"] is True + assert len(cast(list[JSONValue], summary["target_summaries"])) == 1 + + +def test_classical_baseline_golden_fixture() -> None: + """The public diagnostic contract should remain golden-testable.""" + payload = classical_baseline_diagnostics_from_training_frame( + _enriched_frame(30, session_states=(1, 2, 3)), + _fingerprint(), + profile=_profile(), + ) + expected = json.loads( + ( + Path(__file__).parents[1] + / "fixtures" + / "data_quality_reports" + / "classical_baseline.json" + ).read_text(encoding="utf-8") + ) + + assert payload == expected + + +def _profile() -> ClassicalBaselineProfile: + return ClassicalBaselineProfile( + enabled=True, + evaluation_fraction=0.2, + minimum_training_rows=10, + minimum_evaluation_rows=5, + rolling_windows=(3,), + session_seasonal_enabled=True, + rounding_digits=6, + ) + + +def _raw_frame( + count: int, + *, + duplicate_timestamp: bool = False, +) -> pl.DataFrame: + timestamps = [1000 + index * 100 for index in range(count)] + if duplicate_timestamp and count > 1: + timestamps[1] = timestamps[0] + bids = [1.0 + index * 0.01 for index in range(count)] + return pl.DataFrame( + { + "datetime": timestamps, + "bid": bids, + "ask": [value + 0.0002 for value in bids], + "vol": [0] * count, + }, + schema={ + "datetime": pl.Int64, + "bid": pl.Float64, + "ask": pl.Float64, + "vol": pl.Int32, + }, + ) + + +def _enriched_frame( + count: int, + *, + symbol: str = "EURUSD", + session_states: tuple[int, ...] = (0,), + duplicate_timestamp: bool = False, +) -> pl.DataFrame: + frame = ensure_tick_training_features( + _raw_frame(count, duplicate_timestamp=duplicate_timestamp), + symbol=symbol, + data_format="ascii", + timeframe=TICK, + period="201202", + ) + states = [ + session_states[index % len(session_states)] for index in range(count) + ] + return frame.with_columns( + pl.Series("class_session_state_code", states, dtype=pl.Int32) + ) + + +def _fingerprint( + *, + symbol: str = "EURUSD", + stationarity_status: str = "valid", + recommended_transforms: tuple[str, ...] = (), +) -> dict[str, JSONValue]: + raw_status = { + "valid": "ok", + "limited": "limited", + "unavailable": "unavailable", + }[stationarity_status] + section_statuses = { + "coverage": "valid", + "temporal_topology": "valid", + "calendar_regimes": "valid", + "tick_distribution": "valid", + "microstructure_dynamics": "valid", + "dependence": "valid", + "stationarity_diagnostics": stationarity_status, + "decomposition": "valid", + } + return { + "fingerprint_id": f"fingerprint-{symbol.lower()}", + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": symbol, + "period": "201202", + "kind": "cache", + }, + "calendar_regimes": {"status": "ok"}, + "stationarity_diagnostics": { + "stationarity_status": raw_status, + "reason": ( + None + if stationarity_status == "valid" + else f"stationarity_{stationarity_status}" + ), + "rolling_windows": {"3": {"status": "computed"}}, + "computed_window_count": 1, + "skipped_window_count": 0, + "zero_variance_metrics": [], + "first_middle_last_distribution_shift": {"status": "computed"}, + "recommended_transforms": list(recommended_transforms), + }, + "fingerprint_audit": {"section_statuses": section_statuses}, + } + + +def _target( + path: str, + kind: QualityTargetKind = QualityTargetKind.CSV, +) -> QualityTarget: + name = Path(path).name.upper() + symbol = "EURUSD" + for candidate in ("AUDUSD", "EURUSD", "GBPUSD"): + if candidate in name: + symbol = candidate + break + return QualityTarget( + path=path, + kind=kind, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period="201202", + ) + + +def _finding( + target: QualityTarget, diagnostics: Mapping[str, JSONValue] +) -> QualityFinding: + return QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id="fingerprint.series", + target=target, + metadata={ + TIME_SERIES_FINGERPRINT_METADATA_KEY: { + "classical_baselines": dict(diagnostics) + } + }, + ) + + +def _baseline_finding(tmp_path: Path) -> tuple[QualityFinding, QualityTarget]: + target = _target(str(tmp_path / "DAT_ASCII_EURUSD_T_201202.csv")) + diagnostics = classical_baseline_diagnostics_from_training_frame( + _enriched_frame(30), _fingerprint(), profile=_profile() + ) + return _finding(target, diagnostics), target + + +def _tick_lines(count: int) -> tuple[str, ...]: + return tuple( + ( + f"20120201 00{index // 60:02d}{index % 60:02d}000," + f"{1.0 + index * 0.01:.6f}," + f"{1.0002 + index * 0.01:.6f},0" + ) + for index in range(count) + ) + + +def _mapping(value: Any) -> Mapping[str, Any]: + assert isinstance(value, Mapping) + return value + + +def _mapping_rows(value: Any) -> list[Mapping[str, Any]]: + assert isinstance(value, list) + assert all(isinstance(item, Mapping) for item in value) + return cast(list[Mapping[str, Any]], value) + + +def _strings(value: Any) -> list[str]: + assert isinstance(value, list) + return [str(item) for item in value] diff --git a/tests/unit/test_data_quality_classical_model_comparison.py b/tests/unit/test_data_quality_classical_model_comparison.py new file mode 100644 index 00000000..cd60d604 --- /dev/null +++ b/tests/unit/test_data_quality_classical_model_comparison.py @@ -0,0 +1,700 @@ +"""Tests for family-neutral classical-model comparisons and projections.""" + +from __future__ import annotations + +from collections.abc import Mapping +import json +from pathlib import Path +from typing import Any, cast + +import polars as pl +import pytest + +from histdatacom.data_quality import ( + QualityExitPolicy, + bounded_quality_payload, + quality_report_payload, +) +from histdatacom.data_quality.classical_model_comparison import ( + CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY, + CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY, + ClassicalModelComparisonProfile, + classical_model_comparison_from_saved_results, + classical_model_comparison_summary, + format_classical_model_comparison_summary_lines, + project_classical_model_comparison_onto_training_frame, +) +from histdatacom.data_quality.contracts import ( + QualityTarget, + QualityTargetKind, + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, +) +from histdatacom.data_quality.profiles import ( + QUALITY_PROFILE_SCHEMA_VERSION, + quality_profile_from_mapping, +) +from histdatacom.data_quality.reporting import format_quality_console_summary +from histdatacom.data_quality.fingerprints import ( + FINGERPRINT_AUDIT_SECTIONS, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.training_features import ( + CLASSICAL_MODEL_COMPARISON_COLUMNS, + required_training_feature_columns, + training_feature_definitions, +) +from histdatacom.histdata_ascii import TICK, format_influx_line + + +def test_comparison_is_deterministic_multihorizon_and_has_no_winner() -> None: + payload = _mean_family_payload( + model_metrics={1: (2.0, 3.0), 2: (6.0, 7.0)}, + baseline_metrics={1: (4.0, 5.0), 2: (4.0, 5.0)}, + ) + + result = _compare(exponential_smoothing=payload) + repeated = _compare(exponential_smoothing=payload) + + assert result.diagnostics == repeated.diagnostics + assert result.diagnostics["schema_version"] == ( + CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION + ) + assert result.diagnostics["horizons"] == [1, 2] + records = _records(result.diagnostics, model_id="ets:ses") + skills = { + (record["horizon"], record["metric"]): _mapping(record["skill"]) + for record in records + } + assert skills[(1, "mae")]["value"] == 0.5 + assert skills[(2, "mae")]["value"] == -0.5 + assert skills[(2, "mae")]["negative"] is True + assert all(record["metric_value"] is not None for record in records) + assert result.diagnostics["selection_policy"] == "none" + _assert_no_key(result.diagnostics, {"winner", "best_model"}) + + +def test_saved_artifact_identity_mismatch_is_ineligible() -> None: + baseline_family = _mean_family_payload(models=()) + mismatched_family = _mean_family_payload( + input_derivation_id="different-input", + include_baselines=False, + ) + + result = _compare( + exponential_smoothing=baseline_family, + autoregressive=mismatched_family, + ) + + [record] = [ + row + for row in _records(result.diagnostics, model_id="ets:ses") + if row["metric"] == "mae" + ] + assert record["eligible"] is False + assert "regularization_contract_mismatch" in record["eligibility_reasons"] + assert _mapping(record["skill"])["status"] == "unavailable" + + +def test_missing_near_zero_and_truncated_baselines_are_explicit() -> None: + near_zero = _mean_family_payload( + baseline_metrics={1: (0.0, 0.0)}, + ) + missing = _mean_family_payload(include_baselines=False) + truncated = _mean_family_payload(fold_results_truncated=True) + + zero_record = _record(_compare(exponential_smoothing=near_zero), "mae") + missing_record = _record(_compare(exponential_smoothing=missing), "mae") + truncated_record = _record(_compare(exponential_smoothing=truncated), "mae") + + assert _mapping(zero_record["skill"])["reason"] == "baseline_near_zero" + assert ( + "reference_baseline_unavailable" + in missing_record["eligibility_reasons"] + ) + assert truncated_record["eligible"] is False + assert "fold_evidence_truncated" in truncated_record["eligibility_reasons"] + + +def test_mean_variance_and_volatility_metrics_are_never_interchanged() -> None: + result = _compare( + exponential_smoothing=_mean_family_payload(), + volatility=_volatility_payload(), + ) + records = _records(result.diagnostics) + groups = {(row["target_metric"], row["scale"]) for row in records} + + assert ("mid_level", "original_mid") in groups + assert ("return_mean", "unscaled_return") in groups + assert ("conditional_variance", "unscaled_return_squared") in groups + assert ( + "absolute_return_volatility", + "absolute_unscaled_return", + ) in groups + assert all( + not ( + row["target_metric"] == "conditional_variance" + and row["reference_baseline"] == "naive_random_walk" + ) + for row in records + ) + + +def test_fit_accounting_preserves_failures_and_resource_reasons() -> None: + payload = _volatility_payload( + fit_samples=( + {"status": "converged", "reason": None}, + {"status": "failed", "reason": "singular_covariance"}, + {"status": "timed_out", "reason": "wall_time_timeout"}, + {"status": "unavailable", "reason": "dependency_unavailable"}, + ) + ) + result = _compare(volatility=payload) + accounting = _mapping(result.diagnostics["fit_accounting"]) + totals = _mapping(accounting["totals"]) + + assert totals["attempted"] == 4 + assert totals["converged"] == 1 + assert totals["failed"] == 2 + assert totals["timed_out"] >= 1 + assert totals["resource_limited"] >= 1 + assert totals["numerically_invalid"] >= 1 + assert totals["dependency_unavailable"] >= 1 + assert accounting["failed_models_preserved_in_denominator"] is True + specifications = cast( + list[dict[str, Any]], + accounting["by_specification_horizon_period"], + ) + garch = next(row for row in specifications if row["family"] == "garch") + assert _mapping(garch["counts"])["attempted"] == 4 + + +@pytest.mark.parametrize( + ("errors", "parameter_stability", "expected"), + ( + ((1.0, 1.0, 1.0), {}, "stable"), + ((1.0, 1.2, 3.0), {}, "persistent_degradation"), + ( + (1.0, 1.0, 1.0), + {"parameters": {"level": {"min": 1.0, "median": 1.0, "max": 2.0}}}, + "structural_shift", + ), + ), +) +def test_stability_states_are_distinct( + errors: tuple[float, ...], + parameter_stability: Mapping[str, Any], + expected: str, +) -> None: + payload = _mean_family_payload( + fold_errors=errors, + parameter_stability=parameter_stability, + ) + record = _record(_compare(exponential_smoothing=payload), "mae") + + assert _mapping(record["stability"])["status"] == expected + + +def test_projection_preserves_duplicate_rows_cache_scalars_and_influx() -> None: + frame = _raw_frame(duplicate_timestamp=True) + observed = frame.select("datetime", "bid", "ask").to_dicts() + result = _compare(frame=frame, exponential_smoothing=_mean_family_payload()) + projected = project_classical_model_comparison_onto_training_frame( + frame, result, target=_target() + ) + + assert projected.select("datetime", "bid", "ask").to_dicts() == observed + assert projected.height == frame.height + assert projected.get_column("row_id").n_unique() == frame.height + assert set(CLASSICAL_MODEL_COMPARISON_COLUMNS).issubset(projected.columns) + assert projected.get_column( + "cm_comparison_training_eligible" + ).drop_nulls().to_list() == [False] + assert not any(name.startswith("winner") for name in projected.columns) + annotated = projected.filter(pl.col("cm_comparison_diagnostic_available")) + assert annotated.height == 1 + line = format_influx_line( + "EURUSD", + "ascii", + TICK, + annotated.row(0), + columns=projected.columns, + ) + assert "cm_comparison_eligible=" in line + assert "cm_skill_value=" in line + assert "cm_stability_status_code=" in line + + +def test_profile_registry_bounds_and_publication_safety() -> None: + parsed = quality_profile_from_mapping( + { + "schema_version": QUALITY_PROFILE_SCHEMA_VERSION, + "name": "comparison", + "rules": { + "fingerprint.series": { + "classical_model_comparison": { + "enabled": True, + "mean_reference_baseline": "rolling_mean", + "max_models": 2, + "max_horizons": 1, + "max_comparisons": 3, + } + } + }, + } + ).fingerprint_profile() + assert parsed.classical_model_comparison.mean_reference_baseline == ( + "rolling_mean" + ) + assert parsed.classical_model_comparison.max_comparisons == 3 + with pytest.raises(ValueError, match="max_models"): + ClassicalModelComparisonProfile(max_models=0) + + result = _compare( + exponential_smoothing=_mean_family_payload( + model_metrics={1: (2.0, 3.0), 2: (2.0, 3.0)}, + baseline_metrics={1: (4.0, 5.0), 2: (4.0, 5.0)}, + ), + profile=ClassicalModelComparisonProfile( + enabled=True, + max_models=2, + max_horizons=1, + max_comparisons=3, + ), + ) + assert ( + _mapping(_mapping(result.diagnostics["bounds"])["horizons"])[ + "included_count" + ] + == 1 + ) + text = str(result.diagnostics).lower() + assert "/users/" not in text + assert "traceback" not in text + _assert_no_key( + result.diagnostics, {"exception", "residuals", "fitted_object"} + ) + + definitions = training_feature_definitions() + registered = {row.name for row in definitions} + assert set(CLASSICAL_MODEL_COMPARISON_COLUMNS).issubset(registered) + assert set(CLASSICAL_MODEL_COMPARISON_COLUMNS).issubset( + required_training_feature_columns() + ) + assert len(CLASSICAL_MODEL_COMPARISON_COLUMNS) == 43 + + +def test_fingerprint_rule_emits_opt_in_saved_artifact_comparison( + tmp_path: Path, +) -> None: + source = tmp_path / "DAT_ASCII_EURUSD_T_201202.csv" + source.write_text( + "\n".join( + f"20120201 0000{index:02d}000,1.{index:06d},1.{index + 200:06d},0" + for index in range(20) + ) + + "\n", + encoding="ascii", + ) + target = QualityTarget( + path=source, + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + finding = HistDataSeriesFingerprintRule( + profile=HistDataFingerprintProfile( + classical_model_comparison=ClassicalModelComparisonProfile( + enabled=True + ) + ) + ).evaluate(target)[0] + fingerprint = _mapping( + finding.metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + comparison = _mapping(fingerprint["classical_model_comparison"]) + audit = _mapping(fingerprint["fingerprint_audit"]) + + assert comparison["status"] == "unavailable" + assert comparison["reason"] == "no_model_results" + assert ( + _mapping(comparison["source_contracts"])["model_fits_triggered"] + is False + ) + assert "classical_model_comparison" in FINGERPRINT_AUDIT_SECTIONS + assert "classical_model_comparison" in audit["sections_expected"] + assert "classical_model_comparison" in audit["sections_emitted"] + + +def test_bounded_summary_and_cli_output_match_goldens() -> None: + diagnostics = _compare( + exponential_smoothing=_mean_family_payload() + ).diagnostics + finding = QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id="fingerprint.series", + target=_target(), + metadata={ + "time_series_fingerprint": { + "classical_model_comparison": diagnostics + } + }, + ) + summary = classical_model_comparison_summary([finding], target_limit=1) + fixture_root = ( + Path(__file__).parents[1] / "fixtures" / "data_quality_reports" + ) + expected = json.loads( + (fixture_root / "classical_model_comparison_summary.json").read_text( + encoding="utf-8" + ) + ) + cli_expected = ( + fixture_root / "classical_model_comparison_cli.golden" + ).read_text(encoding="utf-8") + + assert summary == expected + assert ( + "\n".join(format_classical_model_comparison_summary_lines(summary)) + + "\n" + == cli_expected + ) + + report = QualityReport( + targets=(finding.target,), + rule_results=( + QualityRuleResult( + rule_id=finding.rule_id, + target=finding.target, + findings=(finding,), + ), + ), + ) + full = quality_report_payload(report) + bounded = bounded_quality_payload( + report=report, + operation="data-quality", + check_groups=("fingerprint",), + discovery={"targets": []}, + artifact=None, + decision=QualityExitPolicy().evaluate(report.summary()), + ) + surfaces = { + "full_report_metadata": _mapping(full["metadata"])[ + CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY + ], + "bounded_payload": bounded[ + CLASSICAL_MODEL_COMPARISON_BOUNDED_PAYLOAD_KEY + ], + "cli_contains_summary": "Classical model comparison" + in format_quality_console_summary(report), + } + expected_surfaces = json.loads( + (fixture_root / "classical_model_comparison_surfaces.json").read_text( + encoding="utf-8" + ) + ) + assert surfaces == expected_surfaces + + +def _compare( + *, + frame: pl.DataFrame | None = None, + exponential_smoothing: Mapping[str, Any] | None = None, + autoregressive: Mapping[str, Any] | None = None, + volatility: Mapping[str, Any] | None = None, + profile: ClassicalModelComparisonProfile | None = None, +) -> Any: + return classical_model_comparison_from_saved_results( + frame, + _fingerprint(), + model_input=_model_input(), + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + volatility=volatility, + profile=profile or ClassicalModelComparisonProfile(enabled=True), + target=_target(), + ) + + +def _mean_family_payload( + *, + input_derivation_id: str = "input-1", + model_metrics: Mapping[int, tuple[float, float]] | None = None, + baseline_metrics: Mapping[int, tuple[float, float]] | None = None, + models: tuple[str, ...] = ("ets:ses",), + include_baselines: bool = True, + fold_errors: tuple[float, ...] = (1.0, 1.0, 1.0), + fold_results_truncated: bool = False, + parameter_stability: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + model_values = model_metrics or {1: (2.0, 3.0)} + baseline_values = baseline_metrics or {1: (4.0, 5.0)} + model_rows = [] + for model_id in models: + model_rows.append( + { + "model_id": model_id, + "specification_id": "ses", + "specification_code": 1, + "family": "ses", + "status": "ready", + "fit_status_counts": {"converged": len(fold_errors)}, + "evaluation_status_counts": {"evaluated": len(fold_errors)}, + "reason_counts": {}, + "horizon_metrics": [ + { + "horizon": horizon, + "evaluation_count": len(fold_errors), + "mae": values[0], + "rmse": values[1], + "bias": values[0] / 2, + } + for horizon, values in sorted(model_values.items()) + ], + "fold_results": [ + { + "fold_id": index + 1, + "horizon": 1, + "target_row_id": index + 10, + "status": "evaluated", + "error": error, + } + for index, error in enumerate(fold_errors) + ], + "fold_results_truncated": fold_results_truncated, + "parameter_stability": dict(parameter_stability or {}), + } + ) + baselines = ( + [ + { + "model": "naive_random_walk", + "horizon_metrics": [ + { + "horizon": horizon, + "evaluation_count": len(fold_errors), + "mae": values[0], + "rmse": values[1], + "bias": values[0] / 2, + } + for horizon, values in sorted(baseline_values.items()) + ], + } + ] + if include_baselines + else [] + ) + return { + "schema_version": "histdatacom.exponential-smoothing.v1", + "input_derivation_id": input_derivation_id, + "input_transform_policy": {"transform": "level"}, + "target_axis": _fingerprint()["target_axis"], + "fit_summary": { + "fit_attempt_count": len(fold_errors), + "status_counts": {"converged": len(fold_errors)}, + "reason_counts": {}, + "warning_counts": {}, + "failed_fit_count": 0, + }, + "evaluation": { + "models": model_rows, + "reference_baselines": baselines, + }, + } + + +def _volatility_payload( + *, fit_samples: tuple[Mapping[str, Any], ...] | None = None +) -> dict[str, Any]: + samples = fit_samples or ({"status": "converged", "reason": None},) + model = { + "model_id": "garch:garch-1-1", + "specification_id": "garch-1-1", + "specification_code": 1, + "family": "garch", + "status": "evaluated", + "fit_samples": list(samples), + "horizon_metrics": { + "1": { + "mean_metrics": { + "count": 3, + "mae": 0.2, + "rmse": 0.3, + "bias": 0.1, + }, + "variance_metrics": { + "count": 3, + "mae": 0.4, + "rmse": 0.5, + "mean_qlike": 0.6, + }, + "volatility_metrics": { + "count": 3, + "mae": 0.3, + "rmse": 0.4, + "bias": 0.1, + }, + } + }, + } + statuses: dict[str, int] = {} + reasons: dict[str, int] = {} + for sample in samples: + status = str(sample.get("status") or "") + reason = str(sample.get("reason") or "") + if status: + statuses[status] = statuses.get(status, 0) + 1 + if reason: + reasons[reason] = reasons.get(reason, 0) + 1 + return { + "schema_version": "histdatacom.volatility.v1", + "input_derivation_id": "input-1", + "input_transform_policy": {"transform": "level"}, + "target_axis": _fingerprint()["target_axis"], + "fit_summary": { + "fit_attempt_count": len(samples), + "status_counts": statuses, + "reason_counts": reasons, + "warning_counts": {}, + "failed_fit_count": sum( + value + for key, value in statuses.items() + if key in {"failed", "unavailable"} + ), + }, + "evaluation": { + "models": [model], + "reference_variance_baselines": [ + { + "name": "ewma_variance_0.94", + "horizon_metrics": { + "1": { + "count": 3, + "mae": 0.8, + "rmse": 0.9, + "mean_qlike": 1.0, + } + }, + } + ], + }, + } + + +def _model_input() -> dict[str, Any]: + return { + "schema_version": "histdatacom.classical-model-input.v1", + "derivation_id": "input-1", + "regularization": { + "frequency_ms": 100, + "expected_closure_policy": "retain_missing_bin", + "unexpected_missing_policy": "retain_missing_bin", + "empty_bin_value_policy": "null", + "forward_fill_policy": "never", + }, + "transform_policy": {"transform": "level"}, + "fold_policy": { + "kind": "rolling_origin_expanding", + "fold_count": 1, + "horizons": [1], + "minimum_training_observations": 2, + "rolling_window_observations": None, + "embargo_observations": 0, + "fold_samples": [ + { + "series_id": "ascii:T:EURUSD:histdata.com", + "period": "201202", + "fold_id": 1, + "horizon": 1, + "origin_row_id": 2, + "target_row_id": 3, + "target_bin_end_utc_ms": 1200, + } + ], + }, + } + + +def _fingerprint() -> dict[str, Any]: + return { + "fingerprint_id": "fingerprint-eurusd", + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": "EURUSD", + "period": "201202", + "kind": "cache", + }, + "stationarity_diagnostics": {"status": "valid"}, + "decomposition": {"status": "valid"}, + "calendar_regimes": {"status": "valid"}, + } + + +def _target() -> QualityTarget: + return QualityTarget( + path=Path("DAT_ASCII_EURUSD_T_201202.data"), + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + + +def _raw_frame(*, duplicate_timestamp: bool = False) -> pl.DataFrame: + timestamps = [1000, 1100, 1200, 1300] + if duplicate_timestamp: + timestamps[1] = timestamps[0] + return pl.DataFrame( + { + "datetime": timestamps, + "bid": [1.0, 1.1, 1.2, 1.3], + "ask": [1.01, 1.11, 1.21, 1.31], + "vol": [0, 0, 0, 0], + } + ) + + +def _records( + diagnostics: Mapping[str, Any], *, model_id: str | None = None +) -> list[dict[str, Any]]: + rows = [ + _mapping(row) + for row in cast(list[Any], diagnostics["comparison_records"]) + ] + return [ + row for row in rows if model_id is None or row["model_id"] == model_id + ] + + +def _record(result: Any, metric: str) -> dict[str, Any]: + return next( + row + for row in _records(result.diagnostics, model_id="ets:ses") + if row["metric"] == metric + ) + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(cast(Mapping[str, Any], value)) + + +def _assert_no_key(value: Any, forbidden: set[str]) -> None: + if isinstance(value, Mapping): + assert not (set(value) & forbidden) + for child in value.values(): + _assert_no_key(child, forbidden) + elif isinstance(value, (list, tuple)): + for child in value: + _assert_no_key(child, forbidden) diff --git a/tests/unit/test_data_quality_classical_model_contracts.py b/tests/unit/test_data_quality_classical_model_contracts.py new file mode 100644 index 00000000..63bee79f --- /dev/null +++ b/tests/unit/test_data_quality_classical_model_contracts.py @@ -0,0 +1,674 @@ +"""Tests for classical model input and evaluation contracts.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import json +import os +from pathlib import Path +from typing import Any, Mapping, cast + +import polars as pl +import pytest + +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION, + CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION, + CLASSICAL_MODEL_FOLD_SCHEMA_VERSION, + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY, + CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY, + ClassicalModelInputProfile, + ClassicalModelResourcePolicy, + build_classical_model_input, + classical_model_dependency_status, + classical_model_evaluation_result, + classical_model_fit_result, + classical_model_input_summary, + format_classical_model_input_summary_lines, + project_classical_model_input_onto_training_frame, +) +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.reporting import ( + QualityExitPolicy, + bounded_quality_payload, + format_quality_console_summary, + quality_report_payload, +) +from histdatacom.data_quality.fingerprints import ( + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.training_features import ( + CLASSICAL_MODEL_CONTRACT_COLUMNS, + ensure_tick_training_features, + training_feature_definitions, +) +from histdatacom.histdata_ascii import ( + CACHE_FILENAME, + TICK, + format_influx_line, + parse_ascii_lines, + to_polars_frame, + write_polars_cache, +) + + +def test_model_input_regularizes_ticks_deterministically() -> None: + """UTC bins, aggregation, OHLC, and IDs should be deterministic.""" + profile = _profile() + first = build_classical_model_input( + _raw_frame(50), _fingerprint(), profile=profile + ) + second = build_classical_model_input( + _raw_frame(50), _fingerprint(), profile=profile + ) + + assert first.contract == second.contract + assert ( + first.regularized_frame.to_dicts() + == second.regularized_frame.to_dicts() + ) + assert ( + first.contract["schema_version"] == CLASSICAL_MODEL_INPUT_SCHEMA_VERSION + ) + assert first.contract["status"] == "ready" + assert str(first.contract["derivation_id"]).startswith("sha256:") + regularization = _mapping(first.contract["regularization"]) + assert regularization["frequency_ms"] == 500 + assert regularization["bin_interval"] == "[start,end)" + assert regularization["forward_fill_policy"] == "never" + assert regularization["row_mapping_policy"] == ( + "availability_safe_repetition_after_bin_close" + ) + assert regularization["regularized_observation_count"] == 10 + row = first.regularized_frame.row(0, named=True) + assert row["cm_input_observation_count"] == 5 + assert row["cm_input_mid_open"] == pytest.approx(1.0001) + assert row["cm_input_mid_close"] == pytest.approx(1.0401) + assert row["cm_input_mid_high"] == pytest.approx(1.0401) + assert row["cm_input_mid_low"] == pytest.approx(1.0001) + + +def test_model_input_marks_closures_and_missing_without_fill() -> None: + """Weekend closures and unexpected gaps must remain different null bins.""" + timestamps = [ + _utc_ms(2022, 1, 7, 21), + _utc_ms(2022, 1, 10, 12), + ] + result = build_classical_model_input( + _raw_at(timestamps), + _fingerprint(), + profile=_profile( + frequency_ms=6 * 60 * 60 * 1000, + minimum_training_observations=1, + minimum_evaluation_observations=1, + step_size=1, + horizons=(1,), + ), + ) + + regularization = _mapping(result.contract["regularization"]) + assert int(regularization["expected_closure_count"]) > 0 + assert int(regularization["unexpected_missing_count"]) > 0 + missing = result.regularized_frame.filter( + pl.col("cm_input_observation_count") == 0 + ) + assert missing.get_column("cm_input_value").null_count() == missing.height + assert missing.get_column("cm_input_spread").null_count() == missing.height + + +def test_closure_omit_policy_preserves_grid_horizons_but_omits_fold_targets() -> ( + None +): + """Omitting closures must not collapse elapsed regular-grid horizons.""" + timestamps = [ + _utc_ms(2022, 1, 7, 21), + _utc_ms(2022, 1, 10, 12), + ] + common = { + "frequency_ms": 6 * 60 * 60 * 1000, + "minimum_training_observations": 1, + "minimum_evaluation_observations": 1, + "step_size": 1, + "horizons": (1, 11), + } + marked = build_classical_model_input( + _raw_at(timestamps), + _fingerprint(), + profile=_profile(expected_closure_policy="mark", **common), + ) + omitted = build_classical_model_input( + _raw_at(timestamps), + _fingerprint(), + profile=_profile(expected_closure_policy="omit", **common), + ) + + assert omitted.regularized_frame.height == marked.regularized_frame.height + omitted_regularization = _mapping(omitted.contract["regularization"]) + assert omitted_regularization["expected_closure_grid_rows_retained"] is True + assert ( + omitted_regularization["expected_closure_model_observations_omitted"] + is True + ) + assert any( + fold["status"] == "skipped" and fold["reason"] == "target_unavailable" + for fold in marked.folds + ) + assert all(fold["status"] == "valid" for fold in omitted.folds) + + +def test_model_transforms_and_differences_are_explicit() -> None: + """Configured transforms should report warm-up and inverse behavior.""" + profile = _profile( + transform="log_return", + differencing_order=1, + seasonal_differencing_order=1, + seasonal_period=2, + ) + result = build_classical_model_input( + _raw_frame(80), _fingerprint(), profile=profile + ) + + policy = _mapping(result.contract["transform_policy"]) + assert policy["transform"] == "log_return" + assert policy["differencing_order"] == 1 + assert policy["seasonal_differencing_order"] == 1 + assert policy["seasonal_period"] == 2 + assert policy["inverse_transform"] == "exp_and_compound_from_last_level" + assert policy["applied_explicitly"] is True + assert int(policy["warmup_loss"]) > 0 + + +def test_model_folds_are_chronological_and_leakage_safe() -> None: + """Every fold should keep target rows after its training origin.""" + result = build_classical_model_input( + _raw_frame(80), _fingerprint(), profile=_profile(horizons=(1, 2, 3)) + ) + + assert result.folds + assert all( + fold["schema_version"] == CLASSICAL_MODEL_FOLD_SCHEMA_VERSION + for fold in result.folds + ) + assert all(fold["shuffle"] is False for fold in result.folds) + assert all(fold["future_values_visible"] is False for fold in result.folds) + assert all( + int(fold["target_index"]) > int(fold["training_end_index"]) + for fold in result.folds + ) + assert all( + int(fold["target_bin_end_utc_ms"]) > int(fold["origin_bin_end_utc_ms"]) + for fold in result.folds + ) + + +def test_projection_augments_same_rows_only_after_bin_close() -> None: + """Completed-bin values should appear no earlier than their close.""" + raw = _raw_frame(50, duplicate_timestamp=True) + observed = raw.select("datetime", "bid", "ask").to_dicts() + result = build_classical_model_input( + raw, _fingerprint(), profile=_profile() + ) + projected = project_classical_model_input_onto_training_frame(raw, result) + + assert projected.select("datetime", "bid", "ask").to_dicts() == observed + assert projected.get_column("row_id").n_unique() == 50 + assert projected.row(0, named=True)["cm_input_value"] is None + available = projected.filter(pl.col("cm_input_available")) + assert available.height > 0 + assert ( + available.get_column("timestamp_utc_ms") + >= available.get_column("cm_input_available_at_utc_ms") + ).all() + lines = [ + format_influx_line( + "eurusd", "ascii", TICK, row, columns=projected.columns + ) + for row in projected.filter(pl.col("cm_input_available")) + .head(2) + .iter_rows() + ] + assert all("cm_input_status_code=3i" in line for line in lines) + assert all("cm_input_available=true" in line for line in lines) + assert all("bidquote=" in line for line in lines) + + +def test_projection_preserves_identity_when_timestamp_is_masked_or_dropped() -> ( + None +): + """Timestamp is a projection feature, never the durable row key.""" + enriched = ensure_tick_training_features( + _raw_frame(50), + symbol="EURUSD", + data_format="ascii", + timeframe=TICK, + period="201202", + ) + result = build_classical_model_input( + enriched, _fingerprint(), profile=_profile() + ) + masked = enriched.with_columns( + pl.when(pl.col("row_id") == 7) + .then(None) + .otherwise(pl.col("timestamp_utc_ms")) + .alias("timestamp_utc_ms") + ) + projected = project_classical_model_input_onto_training_frame( + masked, result + ) + dropped = project_classical_model_input_onto_training_frame( + enriched.drop("timestamp_utc_ms", "datetime"), result + ) + + assert projected.get_column("row_id").to_list() == list(range(1, 51)) + assert ( + projected.filter(pl.col("row_id") == 7).item(0, "cm_input_value") + is None + ) + assert dropped.get_column("row_id").to_list() == list(range(1, 51)) + assert dropped.get_column("cm_input_value").null_count() == 50 + + +def test_resource_limits_are_explicit_and_deterministic() -> None: + """Source and grid bounds should limit rather than silently consume all rows.""" + profile = _profile( + resources=ClassicalModelResourcePolicy( + max_source_rows=30, + max_regularized_observations=5, + max_folds=2, + max_horizons=2, + max_candidate_orders=2, + max_fit_attempts=2, + max_wall_time_seconds=1, + max_memory_bytes=1024, + max_retained_diagnostics=2, + ) + ) + result = build_classical_model_input( + _raw_frame(100), _fingerprint(), profile=profile + ) + + assert result.contract["status"] == "limited" + assert result.contract["reason"] == "source_row_limit" + assert _mapping(result.contract["source"])["source_rows_truncated"] is True + assert _mapping(result.contract["regularization"])["truncated"] is True + assert result.regularized_frame.height == 5 + assert len(result.folds) <= 2 + + +def test_legacy_raw_input_is_enriched_and_insufficient_data_is_advisory() -> ( + None +): + """Raw caches should enrich on read and short data should not hard fail.""" + result = build_classical_model_input( + _raw_frame(8), _fingerprint(), profile=_profile() + ) + + assert result.contract["status"] == "limited" + assert result.contract["reason"] == "insufficient_regularized_observations" + assert ( + _mapping(result.contract["source"])["legacy_cache_enriched_on_read"] + is True + ) + assert result.contract["hard_fail_quality_gate"] is False + projected = project_classical_model_input_onto_training_frame( + _raw_frame(8), result + ) + assert not projected.get_column("cm_input_ready").any() + assert _mapping(result.contract["training_projection"])["ready"] is False + + +def test_fit_and_evaluation_result_contracts_are_bounded() -> None: + """Future families should share stable fit/evaluation status contracts.""" + fit = classical_model_fit_result( + model_id="ets-1", + family="ets", + status="dependency_unavailable", + reason="dependency_unavailable", + warning_codes=("z", "a", "a"), + ) + evaluation = classical_model_evaluation_result( + model_id="ets-1", + status="contract_ready", + fold_count=4, + horizon_count=2, + metric_scale="original_mid", + ) + + assert fit["schema_version"] == CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION + assert fit["warning_codes"] == ["a", "z"] + assert fit["backend_exception_text_included"] is False + assert ( + evaluation["schema_version"] + == CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION + ) + assert evaluation["automatic_winner"] is False + assert evaluation["full_forecasts_included"] is False + with pytest.raises(ValueError, match="unsupported fit status"): + classical_model_fit_result(model_id="x", family="x", status="bad") + + +def test_optional_model_dependencies_are_not_core( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The contract should work when rich numerical providers are absent.""" + monkeypatch.setattr( + "histdatacom.data_quality.classical_model_contracts.importlib.util.find_spec", + lambda _name: None, + ) + + status = classical_model_dependency_status(probe=True) + + assert status["core_dependency_added"] is False + assert status["future_install_extra"] == "models" + assert status["contract_available_without_optional_dependencies"] is True + assert status["rich_model_fitting_available"] is False + assert status["availability_basis"] == "runtime_probe" + assert all( + dependency["available"] is False + for dependency in cast(list[dict[str, Any]], status["dependencies"]) + ) + + +def test_model_contract_columns_are_registered_flat_scalars() -> None: + """All Phase II foundation columns should belong to the row registry.""" + definitions = { + definition.name: definition + for definition in training_feature_definitions() + } + + assert set(CLASSICAL_MODEL_CONTRACT_COLUMNS).issubset(definitions) + assert all( + definitions[name].grain == "row" + for name in CLASSICAL_MODEL_CONTRACT_COLUMNS + ) + assert all( + definitions[name].nullable for name in CLASSICAL_MODEL_CONTRACT_COLUMNS + ) + assert all( + definitions[name].source == "classical_model_contracts" + for name in CLASSICAL_MODEL_CONTRACT_COLUMNS + ) + + +def test_model_input_summary_is_bounded_and_human_readable() -> None: + """Contract status should have ordinary bounded report surfaces.""" + findings = [] + for symbol in ("AUDUSD", "EURUSD"): + result = build_classical_model_input( + _raw_frame(50), _fingerprint(symbol), profile=_profile() + ) + findings.append(_finding(symbol, result.contract)) + + summary = classical_model_input_summary(findings, target_limit=1) + lines = format_classical_model_input_summary_lines(summary) + + assert summary is not None + assert summary["target_count"] == 2 + assert summary["included_target_count"] == 1 + assert summary["truncated"] is True + assert lines[1] == "Classical model input contracts" + assert "targets: 2" in lines[2] + + +def test_model_input_has_full_bounded_and_console_report_surfaces() -> None: + """The ordinary quality report path should expose the model input contract.""" + result = build_classical_model_input( + _raw_frame(50), _fingerprint(), profile=_profile() + ) + finding = _finding("EURUSD", result.contract) + report = QualityReport( + targets=(finding.target,), + rule_results=( + QualityRuleResult( + rule_id="fingerprint.series", + target=finding.target, + findings=(finding,), + ), + ), + ) + + full = quality_report_payload(report) + bounded = bounded_quality_payload( + report=report, + operation="quality", + check_groups=("fingerprint",), + discovery={"targets": []}, + artifact=None, + decision=QualityExitPolicy().evaluate(report.summary()), + ) + console = format_quality_console_summary(report) + + assert CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY in full["metadata"] + assert CLASSICAL_MODEL_INPUT_BOUNDED_PAYLOAD_KEY in bounded + assert "Classical model input contracts" in console + + +def test_fingerprint_profile_opt_in_emits_model_input_and_audit( + tmp_path: Path, +) -> None: + """The ordinary fingerprint rule should own opt-in contract emission.""" + source = tmp_path / "DAT_ASCII_EURUSD_T_201202.csv" + source.write_text("\n".join(_tick_lines(50)) + "\n", encoding="ascii") + target = QualityTarget( + path=str(source), + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + finding = HistDataSeriesFingerprintRule( + profile=HistDataFingerprintProfile(classical_model_input=_profile()) + ).evaluate(target)[0] + fingerprint = _mapping(finding.metadata["time_series_fingerprint"]) + model_input = _mapping(fingerprint["classical_model_input"]) + audit = _mapping(fingerprint["fingerprint_audit"]) + statuses = _mapping(audit["section_statuses"]) + + assert model_input["schema_version"] == CLASSICAL_MODEL_INPUT_SCHEMA_VERSION + assert model_input["status"] == "ready" + assert str(model_input["reference_fingerprint_id"]).startswith("sha256:") + assert model_input["reference_fingerprint_basis"] == ( + "canonical_pre_contract_snapshot" + ) + assert str(model_input["derivation_id"]).startswith("sha256:") + assert statuses["classical_model_input"] == "valid" + assert "classical_model_input" in cast(list[str], audit["sections_emitted"]) + + +def test_model_input_is_available_from_direct_and_fresh_sibling_caches( + tmp_path: Path, +) -> None: + """Canonical cache paths must feed the same model-input annotation engine.""" + source = tmp_path / "DAT_ASCII_EURUSD_T_201202.csv" + rows = _tick_lines(50) + source.write_text("\n".join(rows) + "\n", encoding="ascii") + cache = tmp_path / CACHE_FILENAME + write_polars_cache(to_polars_frame(parse_ascii_lines(TICK, rows)), cache) + csv_mtime_ns = source.stat().st_mtime_ns + os.utime( + cache, + ns=(csv_mtime_ns + 1_000_000, csv_mtime_ns + 1_000_000), + ) + profile = HistDataFingerprintProfile(classical_model_input=_profile()) + direct_target = QualityTarget( + path=str(cache), + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + sibling_target = QualityTarget( + path=str(source), + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + + direct = _model_input_from_target(direct_target, profile) + sibling = _model_input_from_target(sibling_target, profile) + + assert direct["status"] == sibling["status"] == "ready" + assert direct["regularization"] == sibling["regularization"] + assert direct["transform_policy"] == sibling["transform_policy"] + assert _mapping(direct["source"])["usable_row_count"] == 50 + assert _mapping(sibling["source"])["usable_row_count"] == 50 + + +def test_classical_model_input_golden_fixture() -> None: + """The representative model-input contract should not drift silently.""" + contract = build_classical_model_input( + _raw_frame(50), _fingerprint(), profile=_profile() + ).contract + expected = ( + json.dumps( + contract, + indent=2, + sort_keys=True, + ensure_ascii=True, + ) + + "\n" + ) + fixture = ( + Path(__file__).parents[1] + / "fixtures" + / "data_quality_reports" + / "classical_model_input.json" + ) + if os.environ.get("UPDATE_GOLDEN_FIXTURES") == "1": + fixture.write_text(expected, encoding="utf-8") + assert fixture.read_text(encoding="utf-8") == expected + + +def _profile(**overrides: Any) -> ClassicalModelInputProfile: + values: dict[str, Any] = { + "enabled": True, + "frequency_ms": 500, + "minimum_training_observations": 4, + "minimum_evaluation_observations": 2, + "step_size": 1, + "horizons": (1, 2), + "resources": ClassicalModelResourcePolicy( + max_source_rows=1_000, + max_regularized_observations=1_000, + max_folds=64, + ), + } + values.update(overrides) + return ClassicalModelInputProfile(**values) + + +def _raw_frame( + count: int, *, duplicate_timestamp: bool = False +) -> pl.DataFrame: + timestamps = [1_000 + index * 100 for index in range(count)] + if duplicate_timestamp and count > 1: + timestamps[1] = timestamps[0] + return pl.DataFrame( + { + "datetime": timestamps, + "bid": [1.0 + index * 0.01 for index in range(count)], + "ask": [1.0002 + index * 0.01 for index in range(count)], + "vol": [0] * count, + }, + schema={ + "datetime": pl.Int64, + "bid": pl.Float64, + "ask": pl.Float64, + "vol": pl.Int32, + }, + ) + + +def _raw_at(timestamps: list[int]) -> pl.DataFrame: + return pl.DataFrame( + { + "datetime": timestamps, + "bid": [1.0 + index * 0.01 for index in range(len(timestamps))], + "ask": [1.0002 + index * 0.01 for index in range(len(timestamps))], + "vol": [0] * len(timestamps), + } + ) + + +def _utc_ms(year: int, month: int, day: int, hour: int) -> int: + return int( + datetime(year, month, day, hour, tzinfo=timezone.utc).timestamp() * 1000 + ) + + +def _tick_lines(count: int) -> tuple[str, ...]: + rows = [] + for index in range(count): + total_ms = index * 100 + second, millisecond = divmod(total_ms, 1_000) + rows.append( + f"20120201 0000{second:02d}{millisecond:03d}," + f"{1.0 + index * 0.01:.6f}," + f"{1.0002 + index * 0.01:.6f},0" + ) + return tuple(rows) + + +def _fingerprint(symbol: str = "EURUSD") -> dict[str, Any]: + return { + "fingerprint_id": f"fingerprint-{symbol.lower()}", + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": symbol, + "period": "201202", + "kind": "cache", + }, + "stationarity_diagnostics": { + "stationarity_status": "ok", + "recommended_transforms": [], + }, + "decomposition": {"status": "ok"}, + } + + +def _finding(symbol: str, contract: Mapping[str, Any]) -> QualityFinding: + target = QualityTarget( + path=f"/tmp/DAT_ASCII_{symbol}_T_201202.csv", + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period="201202", + ) + return QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id="fingerprint.series", + target=target, + metadata={ + "time_series_fingerprint": {"classical_model_input": dict(contract)} + }, + ) + + +def _model_input_from_target( + target: QualityTarget, profile: HistDataFingerprintProfile +) -> Mapping[str, Any]: + finding = HistDataSeriesFingerprintRule(profile=profile).evaluate(target)[0] + fingerprint = _mapping(finding.metadata["time_series_fingerprint"]) + return _mapping(fingerprint["classical_model_input"]) + + +def _mapping(value: object) -> Mapping[str, Any]: + assert isinstance(value, Mapping) + return value diff --git a/tests/unit/test_data_quality_engine.py b/tests/unit/test_data_quality_engine.py index 8dbb0e02..86b0d143 100644 --- a/tests/unit/test_data_quality_engine.py +++ b/tests/unit/test_data_quality_engine.py @@ -3,17 +3,25 @@ from __future__ import annotations from dataclasses import dataclass +import json from pathlib import Path import pytest from histdatacom.data_quality import ( + DEFAULT_QUALITY_SKIP_COUNT_LIMIT, + DEFAULT_QUALITY_SKIP_EVENT_LIMIT, + QUALITY_ENGINE_METADATA_KEY, + QUALITY_ENGINE_SCHEMA_VERSION, + QUALITY_SKIP_EVENTS_SCHEMA_VERSION, + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV, QualityFinding, QualityLocation, QualityReport, QualityRuleResult, QualityRunSummary, QualitySeverity, + QualitySkipEvent, QualityStatus, QualityTarget, QualityTargetKind, @@ -92,6 +100,38 @@ def test_quality_target_and_finding_round_trip_preserves_context( assert restored_finding.location.metadata["source_timezone"] == "EST-no-DST" +def test_quality_skip_event_round_trip_is_normalized_and_path_free() -> None: + """Skip events should retain stable target identity without local paths.""" + target = QualityTarget( + path="/Users/alice/private/DAT_ASCII_eurusd_t_201202.zip", + kind=QualityTargetKind.ZIP, + data_format="ASCII", + timeframe="t", + symbol="eurusd", + period="201202", + ) + event = QualitySkipEvent.from_target( + reason_code="duplicate-archive preferred csv", + rule_id="time.ascii.gaps", + target=target, + ) + + payload = event.to_dict() + restored = QualitySkipEvent.from_dict(payload) + + assert event.reason_code == "duplicate_archive_preferred_csv" + assert restored == event + assert payload["target_kind"] == "zip" + assert payload["target_axis"] == { + "data_format": "ascii", + "timeframe": "T", + "symbol": "EURUSD", + "period": "201202", + "kind": "zip", + } + assert "/Users/" not in json.dumps(payload, sort_keys=True) + + def test_quality_engine_runs_multiple_rules_and_aggregates_status( tmp_path: Path, ) -> None: @@ -234,7 +274,22 @@ def test_quality_engine_skips_duplicate_archive_semantic_scans() -> None: ("time.ascii.gaps", QualityTargetKind.CSV), ] assert report.status is QualityStatus.CLEAN - assert report.metadata["quality_engine"] == { + engine = report.metadata[QUALITY_ENGINE_METADATA_KEY] + assert isinstance(engine, dict) + assert engine["schema_version"] == QUALITY_ENGINE_SCHEMA_VERSION + assert engine["planned_target_rule_evaluation_count"] == 4 + assert engine["target_rule_evaluation_count"] == 3 + assert engine["skipped_rule_evaluation_count"] == 1 + assert { + key: engine[key] + for key in ( + "target_count", + "rule_count", + "target_rule_evaluation_count", + "skipped_duplicate_archive_rule_evaluation_count", + "duplicate_archive_scan_policy", + ) + } == { "target_count": 2, "rule_count": 2, "target_rule_evaluation_count": 3, @@ -243,6 +298,161 @@ def test_quality_engine_skips_duplicate_archive_semantic_scans() -> None: "prefer_extracted_csv_for_non_inventory_rules" ), } + skips = engine["skip_events"] + assert isinstance(skips, dict) + assert skips["schema_version"] == QUALITY_SKIP_EVENTS_SCHEMA_VERSION + assert skips["event_count"] == 1 + assert skips["included_event_count"] == 1 + assert skips["omitted_event_count"] == 0 + assert skips["truncated"] is False + assert skips["reason_counts"] == { + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV: 1, + } + assert skips["rule_id_counts"] == {"time.ascii.gaps": 1} + assert skips["target_kind_counts"] == {"zip": 1} + assert skips["events"] == [ + { + "reason_code": ( + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV + ), + "rule_id": "time.ascii.gaps", + "target_kind": "zip", + "target_axis": { + "data_format": "ascii", + "timeframe": "T", + "symbol": "EURUSD", + "period": "201202", + "kind": "zip", + }, + } + ] + assert "/tmp/" not in json.dumps(skips, sort_keys=True) + + +def test_quality_engine_skip_events_are_bounded_and_order_independent() -> None: + """Large skip sets should retain complete counts and bounded evidence.""" + targets: list[QualityTarget] = [] + for index in range(DEFAULT_QUALITY_SKIP_EVENT_LIMIT + 1): + period = f"{index:06d}" + metadata = {"case": "clean"} + targets.extend( + ( + QualityTarget( + path=f"/tmp/DAT_ASCII_EURUSD_T_{period}.zip", + kind=QualityTargetKind.ZIP, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period=period, + metadata=metadata, + ), + QualityTarget( + path=f"/tmp/DAT_ASCII_EURUSD_T_{period}.csv", + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period=period, + metadata=metadata, + ), + ) + ) + rule = _StaticRule( + rule_id="time.ascii.gaps", + description="semantic scans prefer extracted CSVs", + severity_by_case={}, + ) + + first = run_quality_assessment(targets=targets, rules=(rule,)) + second = run_quality_assessment( + targets=tuple(reversed(targets)), + rules=(rule,), + ) + first_engine = first.metadata[QUALITY_ENGINE_METADATA_KEY] + second_engine = second.metadata[QUALITY_ENGINE_METADATA_KEY] + assert isinstance(first_engine, dict) + assert isinstance(second_engine, dict) + first_skips = first_engine["skip_events"] + second_skips = second_engine["skip_events"] + assert isinstance(first_skips, dict) + assert isinstance(second_skips, dict) + + assert first_skips == second_skips + assert first_skips["event_count"] == DEFAULT_QUALITY_SKIP_EVENT_LIMIT + 1 + assert ( + first_skips["included_event_count"] == DEFAULT_QUALITY_SKIP_EVENT_LIMIT + ) + assert first_skips["omitted_event_count"] == 1 + assert first_skips["truncated"] is True + assert len(first_skips["events"]) == DEFAULT_QUALITY_SKIP_EVENT_LIMIT + assert first_skips["reason_counts"] == { + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV: ( + DEFAULT_QUALITY_SKIP_EVENT_LIMIT + 1 + ) + } + assert first_engine["planned_target_rule_evaluation_count"] == len(targets) + assert first_engine["target_rule_evaluation_count"] == ( + DEFAULT_QUALITY_SKIP_EVENT_LIMIT + 1 + ) + assert first_engine["skipped_rule_evaluation_count"] == ( + DEFAULT_QUALITY_SKIP_EVENT_LIMIT + 1 + ) + assert "/tmp/" not in json.dumps(first_skips, sort_keys=True) + + +def test_quality_engine_skip_aggregate_dimensions_are_bounded() -> None: + """High-cardinality rule aggregates should expose explicit omission.""" + archive = QualityTarget( + path="/tmp/DAT_ASCII_EURUSD_T_201202.zip", + kind=QualityTargetKind.ZIP, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + metadata={"case": "clean"}, + ) + csv = QualityTarget( + path="/tmp/DAT_ASCII_EURUSD_T_201202.csv", + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + metadata={"case": "clean"}, + ) + rules = tuple( + _StaticRule( + rule_id=f"time.test.rule-{index:03d}", + description="bounded aggregate test rule", + severity_by_case={}, + ) + for index in range(DEFAULT_QUALITY_SKIP_COUNT_LIMIT + 1) + ) + + report = run_quality_assessment( + targets=(archive, csv), + rules=rules, + ) + engine = report.metadata[QUALITY_ENGINE_METADATA_KEY] + assert isinstance(engine, dict) + skips = engine["skip_events"] + assert isinstance(skips, dict) + rule_counts = skips["rule_id_counts"] + assert isinstance(rule_counts, dict) + limits = skips["limit_metadata"] + assert isinstance(limits, dict) + rule_limits = limits["rules"] + assert isinstance(rule_limits, dict) + + assert len(rule_counts) == DEFAULT_QUALITY_SKIP_COUNT_LIMIT + assert tuple(rule_counts) == tuple( + f"time.test.rule-{index:03d}" + for index in range(DEFAULT_QUALITY_SKIP_COUNT_LIMIT) + ) + assert rule_limits["total_count"] == DEFAULT_QUALITY_SKIP_COUNT_LIMIT + 1 + assert rule_limits["included_count"] == DEFAULT_QUALITY_SKIP_COUNT_LIMIT + assert rule_limits["omitted_count"] == 1 + assert rule_limits["truncated"] is True def test_quality_engine_reports_bounded_progress(tmp_path: Path) -> None: diff --git a/tests/unit/test_data_quality_exponential_smoothing.py b/tests/unit/test_data_quality_exponential_smoothing.py new file mode 100644 index 00000000..0b07e669 --- /dev/null +++ b/tests/unit/test_data_quality_exponential_smoothing.py @@ -0,0 +1,869 @@ +"""Tests for optional exponential-smoothing model diagnostics.""" + +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, cast +import warnings + +import polars as pl +import pytest +from statsmodels.tools.sm_exceptions import ConvergenceWarning + +import histdatacom.data_quality.exponential_smoothing as ets_module +from histdatacom.data_quality.classical_model_contracts import ( + ClassicalModelInputProfile, + ClassicalModelResourcePolicy, +) +from histdatacom.data_quality.exponential_smoothing import ( + EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY, + EXPONENTIAL_SMOOTHING_COLUMNS, + EXPONENTIAL_SMOOTHING_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY, + EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION, + ExponentialSmoothingProfile, + ExponentialSmoothingSpecification, + exponential_smoothing_from_training_frame, + exponential_smoothing_summary, + project_exponential_smoothing_onto_training_frame, +) +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprints import ( + FINGERPRINT_AUDIT_SECTIONS, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.reporting import ( + QualityExitPolicy, + bounded_quality_payload, + format_quality_console_summary, + quality_report_payload, +) +from histdatacom.data_quality.training_features import ( + ensure_tick_training_features, + training_feature_definitions, +) +from histdatacom.histdata_ascii import ( + CACHE_FILENAME, + TICK, + format_influx_line, + parse_ascii_lines, + read_polars_cache, + to_polars_frame, + write_polars_cache, +) + + +def test_full_exponential_smoothing_family_is_explicit_and_deterministic() -> ( + None +): + """SES, Holt, damped Holt, Holt-Winters, and ETS should share folds.""" + profile = _family_profile() + first = exponential_smoothing_from_training_frame( + _raw_frame(240), + _fingerprint(), + input_profile=_input_profile(), + profile=profile, + ) + second = exponential_smoothing_from_training_frame( + _raw_frame(240), + _fingerprint(), + input_profile=_input_profile(), + profile=profile, + ) + + assert first.diagnostics == second.diagnostics + assert ( + first.diagnostics["schema_version"] + == EXPONENTIAL_SMOOTHING_SCHEMA_VERSION + ) + assert first.diagnostics["status"] in {"ready", "limited"} + evaluation = _mapping(first.diagnostics["evaluation"]) + models = _mapping_rows(evaluation["models"]) + assert {model["family"] for model in models} == { + "ses", + "holt", + "holt_winters", + "ets", + } + assert {model["specification_id"] for model in models} == { + "ses", + "holt-add", + "holt-damped", + "hw-add", + "hw-mul", + "ets-aaa", + } + assert all(_mapping(model)["automatic_winner"] is False for model in models) + fold_results = [ + row + for model in models + for row in _mapping_rows(_mapping(model)["fold_results"]) + ] + assert fold_results + assert all( + any( + row["status"] == "evaluated" + for row in _mapping_rows(model["fold_results"]) + ) + for model in models + ) + assert all(row["future_values_visible"] is False for row in fold_results) + assert all( + row["target_bin_end_utc_ms"] > row["origin_bin_end_utc_ms"] + for row in fold_results + ) + assert all( + row["target_bin_end_utc_ms"] - row["origin_bin_end_utc_ms"] + == row["horizon"] * _input_profile().frequency_ms + for row in fold_results + ) + assert all(row["original_scale"] is True for row in fold_results) + assert int(evaluation["evaluated_fold_count"]) > 0 + assert evaluation["automatic_winner"] is False + assert { + row["model"] for row in _mapping_rows(evaluation["reference_baselines"]) + } == { + "naive_random_walk", + "rolling_mean", + "rolling_median", + "session_seasonal_naive", + } + assert first.diagnostics["fit_duration_included"] is False + + +def test_ses_constant_series_forecasts_on_original_scale() -> None: + """The simplest family member should preserve a constant level.""" + raw = _raw_frame(160).with_columns( + pl.lit(1.25).alias("bid"), pl.lit(1.2502).alias("ask") + ) + result = exponential_smoothing_from_training_frame( + raw, + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + ) + model = _mapping_rows(_mapping(result.diagnostics["evaluation"])["models"])[ + 0 + ] + evaluated = [ + row + for row in _mapping_rows(_mapping(model)["fold_results"]) + if row["status"] == "evaluated" + ] + + assert evaluated + assert all(row["forecast"] == pytest.approx(1.2501) for row in evaluated) + + +def test_multiplicative_models_require_positive_training_values() -> None: + """Multiplicative components must fail safely on non-positive values.""" + raw = _raw_frame(160).with_columns( + (1.0 + (pl.int_range(0, pl.len()) % 10).cast(pl.Float64) * 0.01).alias( + "bid" + ), + ( + 1.0002 + (pl.int_range(0, pl.len()) % 10).cast(pl.Float64) * 0.01 + ).alias("ask"), + ) + profile = ExponentialSmoothingProfile( + enabled=True, + specifications=( + ExponentialSmoothingSpecification( + specification_id="mul", + family="holt_winters", + trend="mul", + seasonal="mul", + seasonal_periods=4, + ), + ), + projection_specification_id="mul", + ) + result = exponential_smoothing_from_training_frame( + raw, + _fingerprint(), + input_profile=_input_profile( + transform="return", minimum_training_observations=10 + ), + profile=profile, + ) + + fit = _mapping(result.diagnostics["fit_summary"]) + assert _mapping(fit["reason_counts"])["invalid_multiplicative_domain"] > 0 + assert result.diagnostics["status"] == "limited" + + +def test_missing_bins_reset_training_segment_without_forward_fill() -> None: + """A grid gap should bound the fit segment instead of being filled.""" + raw = _raw_frame(200) + raw = pl.concat([raw.head(100), raw.tail(80)], how="vertical") + result = exponential_smoothing_from_training_frame( + raw, + _fingerprint(), + input_profile=_input_profile(step_size=1), + profile=_ses_profile(), + ) + + assert result.diagnostics["forward_fill_policy"] == "never" + assert result.diagnostics["missing_observation_policy"] == ( + "reset_to_trailing_contiguous_segment" + ) + samples = _mapping_rows( + _mapping(result.diagnostics["fit_summary"])["fit_samples"] + ) + assert samples + assert all( + sample["training_segment_policy"] == "trailing_contiguous_after_missing" + for sample in samples + ) + + +def test_expected_closures_remain_distinct_from_unexpected_missing_bins() -> ( + None +): + """The fitted-family payload must preserve #421 closure semantics.""" + raw = _raw_at( + ( + _utc_ms(2022, 1, 7, 21), + _utc_ms(2022, 1, 10, 12), + ) + ) + profile = _input_profile( + frequency_ms=6 * 60 * 60 * 1000, + minimum_training_observations=1, + minimum_evaluation_observations=1, + step_size=1, + horizons=(1,), + ) + result = exponential_smoothing_from_training_frame( + raw, + _fingerprint(), + input_profile=profile, + profile=_ses_profile(), + ) + policy = _mapping(result.diagnostics["input_missingness_policy"]) + + assert int(policy["expected_closure_count"]) > 0 + assert int(policy["unexpected_missing_count"]) > 0 + assert policy["expected_closure_grid_rows_retained"] is True + assert result.diagnostics["forward_fill_policy"] == "never" + + +def test_explicit_transform_and_differencing_are_inverted_for_metrics() -> None: + """Configured transformations should remain explicit and invertible.""" + result = exponential_smoothing_from_training_frame( + _raw_frame(240), + _fingerprint(), + input_profile=_input_profile( + transform="log_return", + differencing_order=1, + minimum_training_observations=20, + ), + profile=_ses_profile(), + ) + policy = _mapping(result.diagnostics["input_transform_policy"]) + evaluation = _mapping(result.diagnostics["evaluation"]) + + assert policy["transform"] == "log_return" + assert policy["differencing_order"] == 1 + assert int(policy["warmup_loss"]) > 0 + assert policy["original_scale_metrics_required"] is True + assert int(evaluation["evaluated_fold_count"]) > 0 + assert evaluation["original_scale"] is True + + +def test_insufficient_seasonal_cycles_are_bounded_advisory_failures() -> None: + """Seasonal initialization should not fit without two complete cycles.""" + profile = ExponentialSmoothingProfile( + enabled=True, + specifications=( + ExponentialSmoothingSpecification( + specification_id="seasonal", + family="holt_winters", + trend="add", + seasonal="add", + seasonal_periods=20, + ), + ), + projection_specification_id="seasonal", + ) + result = exponential_smoothing_from_training_frame( + _raw_frame(120), + _fingerprint(), + input_profile=_input_profile(minimum_training_observations=10), + profile=profile, + ) + + reasons = _mapping( + _mapping(result.diagnostics["fit_summary"])["reason_counts"] + ) + assert reasons["insufficient_seasonal_cycles"] > 0 + assert result.diagnostics["hard_fail_quality_gate"] is False + + +def test_dependency_absence_keeps_contract_available( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Core users should receive a stable unavailable payload without Statsmodels.""" + monkeypatch.setattr(ets_module, "_load_backend", lambda: None) + result = exponential_smoothing_from_training_frame( + _raw_frame(160), + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + ) + + assert result.diagnostics["status"] == "unavailable" + assert result.diagnostics["reason"] == "dependency_unavailable" + assert _mapping(result.diagnostics["backend"])["available"] is False + + +def test_memory_and_fit_attempt_limits_are_enforced() -> None: + """Large-period work must stop predictably at shared #421 limits.""" + input_profile = _input_profile( + resources=ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=64, + max_horizons=4, + max_candidate_orders=16, + max_fit_attempts=1, + max_wall_time_seconds=30, + max_memory_bytes=1, + max_retained_diagnostics=64, + ) + ) + result = exponential_smoothing_from_training_frame( + _raw_frame(240), + _fingerprint(), + input_profile=input_profile, + profile=_family_profile(), + ) + resources = _mapping(result.diagnostics["resource_usage"]) + + assert result.diagnostics["status"] == "limited" + assert result.diagnostics["reason"] == "resource_limit" + assert resources["memory_limit_exceeded"] is True + assert resources["fit_attempt_count"] == 0 + + +def test_convergence_warning_is_limited_but_forecast_remains_advisory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A usable non-converged fit should remain visible with a stable warning.""" + + class FakeFit: + params = {"smoothing_level": 0.5} + mle_retvals = {"success": False} + + @staticmethod + def forecast(horizon: int) -> list[float]: + return [1.5] * horizon + + def fake_fit(*args: object, **kwargs: object) -> FakeFit: + del args, kwargs + warnings.warn("did not converge", ConvergenceWarning, stacklevel=2) + return FakeFit() + + monkeypatch.setattr(ets_module, "_statsmodels_fit", fake_fit) + result = exponential_smoothing_from_training_frame( + _raw_frame(160), + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + ) + fit = _mapping(result.diagnostics["fit_summary"]) + + assert _mapping(fit["status_counts"])["limited"] > 0 + assert _mapping(fit["warning_counts"])["convergence_warning"] > 0 + + +def test_projection_uses_row_identity_and_separates_forecast_from_diagnostic() -> ( + None +): + """Forecast and realized error must appear only at their availability rows.""" + raw = _raw_frame(200, duplicate_timestamp=True) + observed = raw.select("datetime", "bid", "ask").to_dicts() + result = exponential_smoothing_from_training_frame( + raw, + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + ) + projected = project_exponential_smoothing_onto_training_frame(raw, result) + + assert projected.select("datetime", "bid", "ask").to_dicts() == observed + assert projected.get_column("row_id").n_unique() == raw.height + forecast_rows = projected.filter(pl.col("cm_ets_forecast_available")) + diagnostic_rows = projected.filter(pl.col("cm_ets_diagnostic_available")) + assert forecast_rows.height > 0 + assert diagnostic_rows.height > 0 + assert ( + forecast_rows.get_column("cm_ets_actual").null_count() + == forecast_rows.height + ) + assert forecast_rows.get_column("cm_ets_training_eligible").all() + assert diagnostic_rows.get_column("cm_ets_diagnostic_only").all() + assert ( + forecast_rows.get_column("timestamp_utc_ms") + >= forecast_rows.get_column("cm_ets_forecast_available_at_utc_ms") + ).all() + line = format_influx_line( + "eurusd", + "ascii", + TICK, + forecast_rows.row(0), + columns=forecast_rows.columns, + ) + assert "cm_ets_forecast_available=true" in line + assert "bidquote=" in line + + +def test_projection_survives_masked_or_dropped_timestamps() -> None: + """Build-time availability should project later through durable row IDs.""" + raw = _raw_frame(200) + enriched = ensure_tick_training_features( + raw, + symbol="EURUSD", + data_format="ascii", + timeframe=TICK, + period="201202", + ) + result = exponential_smoothing_from_training_frame( + enriched, + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + ) + masked = enriched.with_columns( + pl.when(pl.col("row_id") == 7) + .then(None) + .otherwise(pl.col("timestamp_utc_ms")) + .alias("timestamp_utc_ms") + ) + projected = project_exponential_smoothing_onto_training_frame( + masked, result + ) + dropped = project_exponential_smoothing_onto_training_frame( + enriched.drop("timestamp_utc_ms", "datetime"), result + ) + + assert projected.get_column("row_id").to_list() == list(range(1, 201)) + assert dropped.get_column("row_id").to_list() == list(range(1, 201)) + assert dropped.get_column("cm_ets_forecast_available").any() + + +def test_projection_round_trips_through_the_polars_cache( + tmp_path: Path, +) -> None: + """All registered ETS scalars should remain Arrow IPC serializable.""" + raw = _raw_frame(200) + result = exponential_smoothing_from_training_frame( + raw, + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + ) + projected = project_exponential_smoothing_onto_training_frame(raw, result) + cache = tmp_path / CACHE_FILENAME + + write_polars_cache(projected, cache) + restored = read_polars_cache(cache) + + assert restored.select(EXPONENTIAL_SMOOTHING_COLUMNS).to_dicts() == ( + projected.select(EXPONENTIAL_SMOOTHING_COLUMNS).to_dicts() + ) + + +def test_ets_columns_are_registered_flat_nullable_scalars() -> None: + """Every augmented ETS column should be discoverable and cache-safe.""" + definitions = { + definition.name: definition + for definition in training_feature_definitions() + } + assert set(EXPONENTIAL_SMOOTHING_COLUMNS) <= set(definitions) + assert all( + definitions[name].nullable for name in EXPONENTIAL_SMOOTHING_COLUMNS + ) + assert all( + definitions[name].grain == "row" + for name in EXPONENTIAL_SMOOTHING_COLUMNS + ) + assert all( + definitions[name].dtype in {"Utf8", "Int64", "Float64", "Boolean"} + for name in EXPONENTIAL_SMOOTHING_COLUMNS + ) + + +def test_exponential_smoothing_profile_rejects_ambiguous_specifications() -> ( + None +): + """Invalid family/component combinations should fail before fitting.""" + with pytest.raises(ValueError, match="SES cannot"): + ExponentialSmoothingSpecification(family="ses", trend="add") + with pytest.raises(ValueError, match="Holt-Winters requires"): + ExponentialSmoothingSpecification(family="holt_winters") + with pytest.raises(ValueError, match="add or mul"): + ExponentialSmoothingSpecification(error="none") + + +def test_legacy_raw_and_enriched_frames_share_the_annotation_engine() -> None: + """Legacy caches should enrich in memory without changing ETS semantics.""" + raw = _raw_frame(200) + enriched = ensure_tick_training_features( + raw, + symbol="EURUSD", + data_format="ascii", + timeframe=TICK, + period="201202", + ) + legacy = exponential_smoothing_from_training_frame( + raw, + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + target=_target("legacy.data", QualityTargetKind.CACHE), + ) + current = exponential_smoothing_from_training_frame( + enriched, + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + ) + + legacy_model = _mapping_rows( + _mapping(legacy.diagnostics["evaluation"])["models"] + )[0] + current_model = _mapping_rows( + _mapping(current.diagnostics["evaluation"])["models"] + )[0] + assert ( + _mapping(legacy_model)["horizon_metrics"] + == _mapping(current_model)["horizon_metrics"] + ) + assert ( + _mapping(legacy.input_result.contract["source"])[ + "legacy_cache_enriched_on_read" + ] + is True + ) + + +def test_fingerprint_opt_in_works_for_csv_direct_and_fresh_sibling_cache( + tmp_path: Path, +) -> None: + """The ordinary rule must expose the family across supported cache paths.""" + source = tmp_path / "DAT_ASCII_EURUSD_T_201202.csv" + rows = _tick_lines(240) + source.write_text("\n".join(rows) + "\n", encoding="ascii") + cache = tmp_path / CACHE_FILENAME + write_polars_cache(to_polars_frame(parse_ascii_lines(TICK, rows)), cache) + csv_mtime_ns = source.stat().st_mtime_ns + os.utime(cache, ns=(csv_mtime_ns + 1_000_000, csv_mtime_ns + 1_000_000)) + profile = HistDataFingerprintProfile( + classical_model_input=_input_profile(), + exponential_smoothing=_ses_profile(), + ) + direct = _ets_from_target( + _target(str(cache), QualityTargetKind.CACHE), profile + ) + sibling = _ets_from_target( + _target(str(source), QualityTargetKind.CSV), profile + ) + default = _mapping( + HistDataSeriesFingerprintRule() + .evaluate(_target(str(source), QualityTargetKind.CSV))[0] + .metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + + assert direct["status"] == sibling["status"] + assert direct["input_transform_policy"] == sibling["input_transform_policy"] + direct_model = _mapping_rows(_mapping(direct["evaluation"])["models"])[0] + sibling_model = _mapping_rows(_mapping(sibling["evaluation"])["models"])[0] + assert ( + _mapping(direct_model)["horizon_metrics"] + == _mapping(sibling_model)["horizon_metrics"] + ) + assert "exponential_smoothing" not in default + assert "exponential_smoothing" in FINGERPRINT_AUDIT_SECTIONS + + +def test_exponential_smoothing_has_full_bounded_and_console_surfaces() -> None: + """Users should not need to parse a nested finding for model status.""" + diagnostics = exponential_smoothing_from_training_frame( + _raw_frame(200), + _fingerprint(), + input_profile=_input_profile(), + profile=_ses_profile(), + ).diagnostics + finding = _finding(_target("DAT_ASCII_EURUSD_T_201202.csv"), diagnostics) + report = QualityReport( + targets=(finding.target,), + rule_results=( + QualityRuleResult( + rule_id=finding.rule_id, + target=finding.target, + findings=(finding,), + ), + ), + ) + full = quality_report_payload(report) + bounded = bounded_quality_payload( + report=report, + operation="data-quality", + check_groups=("fingerprint",), + discovery={"targets": []}, + artifact=None, + decision=QualityExitPolicy().evaluate(report.summary()), + ) + console = format_quality_console_summary(report) + + summary = _mapping( + _mapping(full["metadata"])[EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY] + ) + assert ( + summary["schema_version"] + == EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION + ) + assert EXPONENTIAL_SMOOTHING_BOUNDED_PAYLOAD_KEY in bounded + assert "Exponential-smoothing models" in console + + +def test_exponential_smoothing_summary_is_bounded_and_golden() -> None: + """Run-level fitted-family summaries should be bounded and stable.""" + findings = [] + for symbol in ("AUDUSD", "EURUSD", "GBPUSD"): + fingerprint = _fingerprint(symbol=symbol) + diagnostics = exponential_smoothing_from_training_frame( + _raw_frame(200), + fingerprint, + input_profile=_input_profile(), + profile=_ses_profile(), + ).diagnostics + findings.append( + _finding(_target(f"DAT_ASCII_{symbol}_T_201202.csv"), diagnostics) + ) + summary = exponential_smoothing_summary(findings, target_limit=1) + expected = json.loads( + ( + Path(__file__).parents[1] + / "fixtures" + / "data_quality_reports" + / "exponential_smoothing_summary.json" + ).read_text(encoding="utf-8") + ) + + assert summary == expected + + +def _family_profile() -> ExponentialSmoothingProfile: + specifications = ( + ExponentialSmoothingSpecification( + parameter_bounds=(("smoothing_level", 0.01, 0.99),) + ), + ExponentialSmoothingSpecification( + specification_id="holt-add", family="holt", trend="add" + ), + ExponentialSmoothingSpecification( + specification_id="holt-damped", + family="holt", + trend="add", + damped_trend=True, + ), + ExponentialSmoothingSpecification( + specification_id="hw-add", + family="holt_winters", + trend="add", + seasonal="add", + seasonal_periods=4, + ), + ExponentialSmoothingSpecification( + specification_id="hw-mul", + family="holt_winters", + trend="add", + seasonal="mul", + seasonal_periods=4, + ), + ExponentialSmoothingSpecification( + specification_id="ets-aaa", + family="ets", + error="add", + trend="add", + seasonal="add", + seasonal_periods=4, + ), + ) + return ExponentialSmoothingProfile( + enabled=True, + specifications=specifications, + projection_specification_id="ses", + projection_horizon=1, + ) + + +def _ses_profile() -> ExponentialSmoothingProfile: + return ExponentialSmoothingProfile(enabled=True) + + +def _input_profile(**overrides: Any) -> ClassicalModelInputProfile: + values: dict[str, Any] = { + "enabled": True, + "frequency_ms": 500, + "minimum_training_observations": 20, + "minimum_evaluation_observations": 1, + "step_size": 100, + "horizons": (1, 2), + "resources": ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=64, + max_horizons=4, + max_candidate_orders=16, + max_fit_attempts=32, + max_wall_time_seconds=30, + max_retained_diagnostics=64, + ), + } + values.update(overrides) + return ClassicalModelInputProfile(**values) + + +def _raw_frame( + count: int, *, duplicate_timestamp: bool = False +) -> pl.DataFrame: + timestamps = [1_000 + index * 100 for index in range(count)] + if duplicate_timestamp and count > 1: + timestamps[1] = timestamps[0] + prices = [ + 1.0 + index * 0.001 + (index % 20) * 0.0002 for index in range(count) + ] + return pl.DataFrame( + { + "datetime": timestamps, + "bid": prices, + "ask": [value + 0.0002 for value in prices], + "vol": [0] * count, + } + ) + + +def _raw_at(timestamps: tuple[int, ...]) -> pl.DataFrame: + return pl.DataFrame( + { + "datetime": list(timestamps), + "bid": [1.0 + index * 0.01 for index in range(len(timestamps))], + "ask": [1.0002 + index * 0.01 for index in range(len(timestamps))], + "vol": [0] * len(timestamps), + } + ) + + +def _utc_ms(year: int, month: int, day: int, hour: int) -> int: + return int( + datetime(year, month, day, hour, tzinfo=timezone.utc).timestamp() * 1000 + ) + + +def _fingerprint(*, symbol: str = "EURUSD") -> dict[str, Any]: + return { + "fingerprint_id": f"fingerprint-{symbol.lower()}", + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": symbol, + "period": "201202", + "kind": "cache", + }, + } + + +def _target( + path: str, + kind: QualityTargetKind = QualityTargetKind.CSV, +) -> QualityTarget: + name = Path(path).name.upper() + symbol = next( + ( + candidate + for candidate in ("AUDUSD", "EURUSD", "GBPUSD") + if candidate in name + ), + "EURUSD", + ) + return QualityTarget( + path=path, + kind=kind, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period="201202", + ) + + +def _finding( + target: QualityTarget, + diagnostics: Mapping[str, Any], +) -> QualityFinding: + return QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id="fingerprint.series", + target=target, + metadata={ + TIME_SERIES_FINGERPRINT_METADATA_KEY: { + "exponential_smoothing": dict(diagnostics) + } + }, + ) + + +def _ets_from_target( + target: QualityTarget, + profile: HistDataFingerprintProfile, +) -> Mapping[str, Any]: + finding = HistDataSeriesFingerprintRule(profile=profile).evaluate(target)[0] + fingerprint = _mapping( + finding.metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + audit = _mapping(fingerprint["fingerprint_audit"]) + assert "exponential_smoothing" in audit["sections_expected"] + assert "exponential_smoothing" in audit["sections_emitted"] + assert _mapping(audit["section_statuses"])["exponential_smoothing"] in { + "valid", + "limited", + } + return _mapping(fingerprint["exponential_smoothing"]) + + +def _tick_lines(count: int) -> tuple[str, ...]: + return tuple( + ( + f"20120201 00{index // 60:02d}{index % 60:02d}000," + f"{1.0 + index * 0.001:.6f}," + f"{1.0002 + index * 0.001:.6f},0" + ) + for index in range(count) + ) + + +def _mapping(value: object) -> Mapping[str, Any]: + assert isinstance(value, Mapping) + return value + + +def _mapping_rows(value: object) -> list[Mapping[str, Any]]: + assert isinstance(value, list) + assert all(isinstance(row, Mapping) for row in value) + return cast(list[Mapping[str, Any]], value) diff --git a/tests/unit/test_data_quality_fingerprint_discovery.py b/tests/unit/test_data_quality_fingerprint_discovery.py index 61e916bd..69fb4760 100644 --- a/tests/unit/test_data_quality_fingerprint_discovery.py +++ b/tests/unit/test_data_quality_fingerprint_discovery.py @@ -6,6 +6,17 @@ import json from pathlib import Path +from histdatacom.data_quality.autoregressive import ( + AUTOREGRESSIVE_COLUMNS, + AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION, + AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION, + AUTOREGRESSIVE_FIT_SCHEMA_VERSION, + AUTOREGRESSIVE_FORECAST_SCHEMA_VERSION, + AUTOREGRESSIVE_SCHEMA_VERSION, + AUTOREGRESSIVE_SUMMARY_METADATA_KEY, + AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION, + AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION, +) from histdatacom.data_quality.fingerprint_discovery import ( TIME_SERIES_FINGERPRINT_CONTRACT_AUDIT_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_REPORT_SURFACE_EVIDENCE_SCHEMA_VERSION, @@ -17,24 +28,95 @@ format_fingerprint_contract_audit, format_fingerprint_schema_discovery, ) +from histdatacom.data_quality.classical_baselines import ( + CLASSICAL_BASELINE_SCHEMA_VERSION, + CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION, + CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.classical_model_contracts import ( + CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION, + CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION, + CLASSICAL_MODEL_FOLD_SCHEMA_VERSION, + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION, + CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY, + CLASSICAL_MODEL_INPUT_SUMMARY_SCHEMA_VERSION, + CLASSICAL_MODEL_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.classical_model_comparison import ( + CLASSICAL_MODEL_COMPARISON_COLUMNS, + CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY, + CLASSICAL_MODEL_COMPARISON_SUMMARY_SCHEMA_VERSION, + CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.exponential_smoothing import ( + EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_FORECAST_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY, + EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION, + EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.seasonal_exogenous import ( + SEASONAL_EXOGENOUS_CONFIGURATION_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_FORECAST_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_REGRESSOR_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY, + SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.state_space import ( + STATE_SPACE_CONFIGURATION_SCHEMA_VERSION, + STATE_SPACE_EVALUATION_SCHEMA_VERSION, + STATE_SPACE_FIT_SCHEMA_VERSION, + STATE_SPACE_FORECAST_SCHEMA_VERSION, + STATE_SPACE_SCHEMA_VERSION, + STATE_SPACE_STATE_RESULT_SCHEMA_VERSION, + STATE_SPACE_SUMMARY_METADATA_KEY, + STATE_SPACE_SUMMARY_SCHEMA_VERSION, + STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION, +) +from histdatacom.data_quality.volatility import ( + VOLATILITY_CONFIGURATION_SCHEMA_VERSION, + VOLATILITY_EVALUATION_SCHEMA_VERSION, + VOLATILITY_FIT_SCHEMA_VERSION, + VOLATILITY_FORECAST_SCHEMA_VERSION, + VOLATILITY_SCHEMA_VERSION, + VOLATILITY_SUMMARY_METADATA_KEY, + VOLATILITY_SUMMARY_SCHEMA_VERSION, + VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION, +) from histdatacom.data_quality.fingerprint_contracts import ( FINGERPRINT_DISTRIBUTION_ATTENTION_DEFAULTS, FINGERPRINT_REPORT_SURFACE_CONTRACTS, FINGERPRINT_SCHEMA_CONTRACTS, FINGERPRINT_SECTION_LIMIT_DEFAULTS, FingerprintReportSurfaceContract, + IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS, IMPLEMENTED_FINGERPRINT_TARGET_SECTION_CONTRACTS, PLANNED_FINGERPRINT_RUN_SECTION_CONTRACTS, PLANNED_FINGERPRINT_TARGET_SECTION_CONTRACTS, ) from histdatacom.data_quality.fingerprints import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, + CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, FINGERPRINT_AUDIT_SECTIONS, FINGERPRINT_DYNAMICS_SECTIONS, SERIES_FINGERPRINT_RULE_ID, HistDataFingerprintProfile, TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DEPENDENCE_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY, @@ -46,6 +128,24 @@ QUALITY_PROFILE_SCHEMA_VERSION, load_quality_profile_file, ) +from histdatacom.data_quality.synthetic_constraints import ( + SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION, + SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION, + SYNTHETIC_VALIDATION_SCHEMA_VERSION, +) +from histdatacom.data_quality.synthetic_generation import ( + SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION, + SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION, + SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION, +) +from histdatacom.data_quality.training_features import ( + EXPONENTIAL_SMOOTHING_COLUMNS, + SEASONAL_EXOGENOUS_COLUMNS, + KALMAN_COLUMNS, + STATE_SPACE_COLUMNS, + VOLATILITY_COLUMNS, + training_feature_definitions, +) from histdatacom.runtime_contracts import JSONValue @@ -76,15 +176,238 @@ def test_fingerprint_schema_discovery_reports_contract_surface() -> None: schemas["fingerprint_stationarity_diagnostics"]["schema_version"] == TIME_SERIES_FINGERPRINT_STATIONARITY_SCHEMA_VERSION ) + assert schemas["fingerprint_decomposition"]["schema_version"] == ( + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION + ) + assert schemas["fingerprint_classical_baselines"]["schema_version"] == ( + CLASSICAL_BASELINE_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_classical_baseline_training_projection"][ + "schema_version" + ] + == CLASSICAL_BASELINE_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_classical_baseline_summary"]["schema_version"] + == CLASSICAL_BASELINE_SUMMARY_SCHEMA_VERSION + ) + assert schemas["fingerprint_classical_model_input"]["schema_version"] == ( + CLASSICAL_MODEL_INPUT_SCHEMA_VERSION + ) + assert schemas["fingerprint_classical_model_fold"]["schema_version"] == ( + CLASSICAL_MODEL_FOLD_SCHEMA_VERSION + ) + assert schemas["fingerprint_classical_model_fit_result"][ + "schema_version" + ] == (CLASSICAL_MODEL_FIT_RESULT_SCHEMA_VERSION) + assert schemas["fingerprint_classical_model_evaluation_result"][ + "schema_version" + ] == (CLASSICAL_MODEL_EVALUATION_RESULT_SCHEMA_VERSION) + assert schemas["fingerprint_classical_model_training_projection"][ + "schema_version" + ] == (CLASSICAL_MODEL_TRAINING_PROJECTION_SCHEMA_VERSION) + assert schemas["fingerprint_classical_model_input_summary"][ + "schema_version" + ] == (CLASSICAL_MODEL_INPUT_SUMMARY_SCHEMA_VERSION) + assert schemas["fingerprint_exponential_smoothing"]["schema_version"] == ( + EXPONENTIAL_SMOOTHING_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_exponential_smoothing_configuration"][ + "schema_version" + ] + == EXPONENTIAL_SMOOTHING_CONFIGURATION_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_exponential_smoothing_fit"]["schema_version"] + == EXPONENTIAL_SMOOTHING_FIT_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_exponential_smoothing_forecast"]["schema_version"] + == EXPONENTIAL_SMOOTHING_FORECAST_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_exponential_smoothing_evaluation"][ + "schema_version" + ] + == EXPONENTIAL_SMOOTHING_EVALUATION_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_exponential_smoothing_training_projection"][ + "schema_version" + ] + == EXPONENTIAL_SMOOTHING_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_exponential_smoothing_summary"]["schema_version"] + == EXPONENTIAL_SMOOTHING_SUMMARY_SCHEMA_VERSION + ) + assert schemas["fingerprint_autoregressive"]["schema_version"] == ( + AUTOREGRESSIVE_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_autoregressive_configuration"]["schema_version"] + == AUTOREGRESSIVE_CONFIGURATION_SCHEMA_VERSION + ) + assert schemas["fingerprint_autoregressive_fit"]["schema_version"] == ( + AUTOREGRESSIVE_FIT_SCHEMA_VERSION + ) + assert schemas["fingerprint_autoregressive_forecast"]["schema_version"] == ( + AUTOREGRESSIVE_FORECAST_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_autoregressive_evaluation"]["schema_version"] + == AUTOREGRESSIVE_EVALUATION_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_autoregressive_training_projection"][ + "schema_version" + ] + == AUTOREGRESSIVE_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert schemas["fingerprint_autoregressive_summary"]["schema_version"] == ( + AUTOREGRESSIVE_SUMMARY_SCHEMA_VERSION + ) + assert schemas["fingerprint_seasonal_exogenous"]["schema_version"] == ( + SEASONAL_EXOGENOUS_SCHEMA_VERSION + ) + assert schemas["fingerprint_seasonal_exogenous_configuration"][ + "schema_version" + ] == (SEASONAL_EXOGENOUS_CONFIGURATION_SCHEMA_VERSION) + assert schemas["fingerprint_seasonal_exogenous_regressors"][ + "schema_version" + ] == (SEASONAL_EXOGENOUS_REGRESSOR_SCHEMA_VERSION) + assert schemas["fingerprint_seasonal_exogenous_fit"]["schema_version"] == ( + SEASONAL_EXOGENOUS_FIT_SCHEMA_VERSION + ) + assert schemas["fingerprint_seasonal_exogenous_forecast"][ + "schema_version" + ] == (SEASONAL_EXOGENOUS_FORECAST_SCHEMA_VERSION) + assert schemas["fingerprint_seasonal_exogenous_evaluation"][ + "schema_version" + ] == (SEASONAL_EXOGENOUS_EVALUATION_SCHEMA_VERSION) + assert schemas["fingerprint_seasonal_exogenous_training_projection"][ + "schema_version" + ] == (SEASONAL_EXOGENOUS_TRAINING_PROJECTION_SCHEMA_VERSION) + assert schemas["fingerprint_seasonal_exogenous_summary"][ + "schema_version" + ] == (SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION) + assert schemas["fingerprint_state_space"]["schema_version"] == ( + STATE_SPACE_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_state_space_configuration"]["schema_version"] + == STATE_SPACE_CONFIGURATION_SCHEMA_VERSION + ) + assert schemas["fingerprint_state_space_fit"]["schema_version"] == ( + STATE_SPACE_FIT_SCHEMA_VERSION + ) + assert schemas["fingerprint_kalman_state_result"]["schema_version"] == ( + STATE_SPACE_STATE_RESULT_SCHEMA_VERSION + ) + assert schemas["fingerprint_state_space_forecast"]["schema_version"] == ( + STATE_SPACE_FORECAST_SCHEMA_VERSION + ) + assert schemas["fingerprint_state_space_evaluation"]["schema_version"] == ( + STATE_SPACE_EVALUATION_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_state_space_training_projection"]["schema_version"] + == STATE_SPACE_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert schemas["fingerprint_state_space_summary"]["schema_version"] == ( + STATE_SPACE_SUMMARY_SCHEMA_VERSION + ) + assert schemas["fingerprint_volatility"]["schema_version"] == ( + VOLATILITY_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_volatility_configuration"]["schema_version"] + == VOLATILITY_CONFIGURATION_SCHEMA_VERSION + ) + assert schemas["fingerprint_volatility_fit"]["schema_version"] == ( + VOLATILITY_FIT_SCHEMA_VERSION + ) + assert schemas["fingerprint_volatility_forecast"]["schema_version"] == ( + VOLATILITY_FORECAST_SCHEMA_VERSION + ) + assert schemas["fingerprint_volatility_evaluation"]["schema_version"] == ( + VOLATILITY_EVALUATION_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_volatility_training_projection"]["schema_version"] + == VOLATILITY_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert schemas["fingerprint_volatility_summary"]["schema_version"] == ( + VOLATILITY_SUMMARY_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_classical_model_comparison"]["schema_version"] + == CLASSICAL_MODEL_COMPARISON_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_classical_model_comparison_training_projection"][ + "schema_version" + ] + == CLASSICAL_MODEL_COMPARISON_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_classical_model_comparison_summary"][ + "schema_version" + ] + == CLASSICAL_MODEL_COMPARISON_SUMMARY_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_decomposition_training_projection"][ + "schema_version" + ] + == TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION + ) assert schemas["fingerprint_readiness_summary"]["schema_version"] == ( TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_SCHEMA_VERSION ) assert schemas["fingerprint_readiness_risk"]["schema_version"] == ( TIME_SERIES_FINGERPRINT_READINESS_RISK_SCHEMA_VERSION ) - assert schemas["cross_series_fingerprint"]["status"] == "planned" + assert schemas["fingerprint_cache_source_parity"]["schema_version"] == ( + TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_cache_source_parity_summary"]["schema_version"] + == TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION + ) + assert schemas["fingerprint_synthetic_constraints"]["schema_version"] == ( + SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION + ) + assert ( + schemas["fingerprint_synthetic_constraint_summary"]["schema_version"] + == SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION + ) + assert schemas["fingerprint_synthetic_validation"]["schema_version"] == ( + SYNTHETIC_VALIDATION_SCHEMA_VERSION + ) + assert schemas["synthetic_tick_generation"]["schema_version"] == ( + SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION + ) + assert ( + schemas["synthetic_tick_generation_configuration"]["schema_version"] + == SYNTHETIC_TICK_GENERATION_CONFIGURATION_SCHEMA_VERSION + ) + assert ( + schemas["synthetic_tick_generation_validation"]["schema_version"] + == SYNTHETIC_TICK_GENERATION_VALIDATION_SCHEMA_VERSION + ) + assert schemas["cross_series_fingerprint"] == { + "schema_version": CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, + "status": "implemented", + "rule_id": "fingerprint.cross_series", + "metadata_key": CROSS_SERIES_FINGERPRINT_METADATA_KEY, + "bounded_payload_key": "fingerprint_cross_series", + } assert payload["metadata_keys"]["finding_metadata"] == { - "series_fingerprint": TIME_SERIES_FINGERPRINT_METADATA_KEY + "series_fingerprint": TIME_SERIES_FINGERPRINT_METADATA_KEY, + "cross_series_fingerprint": CROSS_SERIES_FINGERPRINT_METADATA_KEY, } assert payload["metadata_keys"]["report_metadata"]["readiness_summary"] == ( TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY @@ -92,6 +415,35 @@ def test_fingerprint_schema_discovery_reports_contract_surface() -> None: assert payload["metadata_keys"]["report_metadata"]["readiness_risk"] == ( TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY ) + assert ( + payload["metadata_keys"]["report_metadata"]["cache_source_parity"] + == TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY + ) + assert payload["metadata_keys"]["report_metadata"][ + "classical_model_input" + ] == (CLASSICAL_MODEL_INPUT_SUMMARY_METADATA_KEY) + assert ( + payload["metadata_keys"]["report_metadata"]["exponential_smoothing"] + == EXPONENTIAL_SMOOTHING_SUMMARY_METADATA_KEY + ) + assert payload["metadata_keys"]["report_metadata"]["autoregressive"] == ( + AUTOREGRESSIVE_SUMMARY_METADATA_KEY + ) + assert payload["metadata_keys"]["report_metadata"][ + "seasonal_exogenous" + ] == (SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY) + assert payload["metadata_keys"]["report_metadata"]["state_space"] == ( + STATE_SPACE_SUMMARY_METADATA_KEY + ) + assert payload["metadata_keys"]["report_metadata"]["volatility"] == ( + VOLATILITY_SUMMARY_METADATA_KEY + ) + assert ( + payload["metadata_keys"]["report_metadata"][ + "classical_model_comparison" + ] + == CLASSICAL_MODEL_COMPARISON_SUMMARY_METADATA_KEY + ) implemented = payload["sections"]["implemented"]["target_sections"] assert [section["name"] for section in implemented] == [ @@ -103,16 +455,165 @@ def test_fingerprint_schema_discovery_reports_contract_surface() -> None: "microstructure_dynamics", "dependence", "stationarity_diagnostics", + "decomposition", + "cache_source_parity", + "classical_baselines", + "classical_model_input", + "exponential_smoothing", + "autoregressive", + "seasonal_exogenous", + "state_space", + "volatility", + "classical_model_comparison", + "synthetic_constraints", "fingerprint_audit", ] + exponential = next( + section + for section in implemented + if section["name"] == "exponential_smoothing" + ) + definitions = { + definition.name: definition + for definition in training_feature_definitions() + } + synthetic = next( + section + for section in implemented + if section["name"] == "synthetic_constraints" + ) + assert synthetic["generation_in_scope"] is True + assert synthetic["generation_method"] == "empirical_block_bootstrap" + assert synthetic["issue"] == "#81" + assert exponential["augmented_columns"] == [ + { + "name": name, + "dtype": definitions[name].dtype, + "nullable": definitions[name].nullable, + "grain": definitions[name].grain, + "source": definitions[name].source, + } + for name in EXPONENTIAL_SMOOTHING_COLUMNS + ] + autoregressive = next( + section + for section in implemented + if section["name"] == "autoregressive" + ) + assert autoregressive["augmented_columns"] == [ + { + "name": name, + "dtype": definitions[name].dtype, + "nullable": definitions[name].nullable, + "grain": definitions[name].grain, + "source": definitions[name].source, + } + for name in AUTOREGRESSIVE_COLUMNS + ] + seasonal_exogenous = next( + section + for section in implemented + if section["name"] == "seasonal_exogenous" + ) + assert seasonal_exogenous["model_families"] == [ + "sarima", + "arimax", + "sarimax", + ] + assert seasonal_exogenous["augmented_column_prefixes"] == [ + "cm_sarima_", + "cm_arimax_", + "cm_sarimax_", + ] + assert seasonal_exogenous["augmented_columns"] == [ + { + "name": name, + "dtype": definitions[name].dtype, + "nullable": definitions[name].nullable, + "grain": definitions[name].grain, + "source": definitions[name].source, + } + for name in SEASONAL_EXOGENOUS_COLUMNS + ] + state_space = next( + section for section in implemented if section["name"] == "state_space" + ) + assert state_space["model_families"] == [ + "local_level", + "local_linear_trend", + "structural", + ] + assert state_space["augmented_column_prefixes"] == [ + "cm_state_space_", + "cm_kalman_", + ] + assert state_space["augmented_columns"] == [ + { + "name": name, + "dtype": definitions[name].dtype, + "nullable": definitions[name].nullable, + "grain": definitions[name].grain, + "source": definitions[name].source, + } + for name in (*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS) + ] + volatility = next( + section for section in implemented if section["name"] == "volatility" + ) + assert volatility["model_families"] == ["arch", "garch"] + assert volatility["augmented_column_prefixes"] == [ + "cm_arch_", + "cm_garch_", + ] + assert volatility["augmented_columns"] == [ + { + "name": name, + "dtype": definitions[name].dtype, + "nullable": definitions[name].nullable, + "grain": definitions[name].grain, + "source": definitions[name].source, + } + for name in VOLATILITY_COLUMNS + ] + comparison = next( + section + for section in implemented + if section["name"] == "classical_model_comparison" + ) + assert comparison["augmented_column_prefixes"] == [ + "cm_comparison_", + "cm_skill_", + "cm_stability_", + ] + assert comparison["augmented_columns"] == [ + { + "name": name, + "dtype": definitions[name].dtype, + "nullable": definitions[name].nullable, + "grain": definitions[name].grain, + "source": definitions[name].source, + } + for name in CLASSICAL_MODEL_COMPARISON_COLUMNS + ] planned = payload["sections"]["planned"]["target_sections"] - assert [section["name"] for section in planned] == [ - "decomposition", - "synthetic_constraints", + assert planned == [] + assert payload["sections"]["planned"]["run_sections"] == [] + assert payload["sections"]["implemented"]["run_sections"] == [ + { + "name": "cross_series_fingerprint", + "status": "implemented", + "rule_id": "fingerprint.cross_series", + "issue": "#331", + } ] + assert payload["target_capabilities"]["run_rule_status"]["status"] == ( + "implemented" + ) assert "observed_sequence" in payload["calculation_bases"]["basis"] assert "source_text_order" in payload["calculation_bases"]["row_order"] assert "not_emitted" in payload["vocabularies"]["skip_and_reason_codes"] + assert len(IMPLEMENTED_FINGERPRINT_RUN_SECTION_CONTRACTS) == 1 + assert PLANNED_FINGERPRINT_RUN_SECTION_CONTRACTS == () def test_fingerprint_schema_discovery_uses_contract_registry() -> None: @@ -412,6 +913,7 @@ def test_fingerprint_schema_discovery_reflects_profile_overrides() -> None: "histogram_bins": 12, "max_rows": 250, "rounding_digits": 6, + "topology_inspection_sample_limit": 2, "distribution_attention": { "zero_spread_min_rate": 0.25, "negative_spread_min_count": 2, @@ -431,6 +933,7 @@ def test_fingerprint_schema_discovery_reflects_profile_overrides() -> None: assert effective["histogram_bins"] == 12 assert effective["max_rows"] == 250 assert effective["rounding_digits"] == 6 + assert effective["topology_inspection_sample_limit"] == 2 assert effective["distribution_attention"]["zero_spread_min_rate"] == 0.25 assert effective["distribution_attention"]["negative_spread_min_count"] == 2 @@ -477,6 +980,7 @@ def test_format_fingerprint_schema_discovery_renders_human_summary() -> None: assert "- microstructure_dynamics: implemented; timeframes=[T]" in output assert "- dependence: implemented; timeframes=[T]" in output assert "- stationarity_diagnostics: implemented; timeframes=[T]" in output + assert "- decomposition: implemented; timeframes=[T]" in output assert "without reading source or running data quality checks" in output diff --git a/tests/unit/test_data_quality_fingerprint_next_work.py b/tests/unit/test_data_quality_fingerprint_next_work.py new file mode 100644 index 00000000..4aed4bd2 --- /dev/null +++ b/tests/unit/test_data_quality_fingerprint_next_work.py @@ -0,0 +1,399 @@ +"""Tests for saved-report next fingerprint work recommendations.""" + +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +from histdatacom.data_quality.contracts import ( + QualityReport, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprint_discovery import ( + fingerprint_schema_discovery, +) +from histdatacom.data_quality.fingerprint_next_work import ( + FINGERPRINT_NEXT_WORK_SCHEMA_VERSION, + fingerprint_next_work_recommendation, + format_fingerprint_next_work, +) +from histdatacom.data_quality.fingerprints import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY, +) +from histdatacom.data_quality.remediation_audit import load_quality_report +from histdatacom.data_quality.reporting import quality_report_to_json +from histdatacom.data_quality.training_features import TRAINING_SCHEMA_VERSION +from histdatacom.runtime_contracts import JSONValue + + +def test_next_work_ranks_widespread_missing_section_first() -> None: + """Missing evidence across more targets should outrank smaller gaps.""" + report = _report( + targets=( + _target("EURUSD"), + _target("GBPUSD"), + _target("USDJPY"), + ), + target_risks=( + _target_risk( + "EURUSD", + "dependence", + status="missing", + score=35, + reason="not_emitted", + ), + _target_risk( + "GBPUSD", + "dependence", + status="missing", + score=35, + reason="not_emitted", + ), + _target_risk( + "USDJPY", + "stationarity_diagnostics", + status="missing", + score=50, + reason="insufficient_samples", + ), + ), + ) + + payload = fingerprint_next_work_recommendation([("quality.json", report)]) + + assert payload["schema_version"] == FINGERPRINT_NEXT_WORK_SCHEMA_VERSION + assert payload["status"] == "recommended" + assert payload["recommendation"]["capability"] == "dependence" + assert payload["recommendation"]["affected_target_count"] == 2 + assert payload["recommendation"]["reason_codes"] == ["not_emitted"] + + +def test_next_work_handles_no_fingerprint_and_low_risk_reports() -> None: + """No evidence and clean evidence without roadmap gaps should be no-work.""" + no_fingerprint = QualityReport(targets=(_target("EURUSD"),)) + no_evidence = fingerprint_next_work_recommendation( + [("plain.json", no_fingerprint)] + ) + assert no_evidence["status"] == "no_work" + assert no_evidence["recommendation"] is None + + discovery = deepcopy(fingerprint_schema_discovery()) + discovery["sections"]["planned"]["target_sections"] = [] + discovery["sections"]["planned"]["run_sections"] = [] + clean = _report( + targets=(_target("EURUSD"),), + target_risks=(), + section_status_counts={"stationarity_diagnostics": {"valid": 1}}, + ) + low_risk = fingerprint_next_work_recommendation( + [("clean.json", clean)], + discovery=discovery, + ) + assert low_risk["status"] == "no_work" + assert low_risk["recommendation_count"] == 0 + + +def test_next_work_ignores_legacy_m1_as_a_base_grain() -> None: + """M1 risk must not displace the supported ascii/T recommendation.""" + report = _report( + targets=( + _target("EURUSD", timeframe="M1"), + _target("GBPUSD"), + ), + target_risks=( + _target_risk( + "EURUSD", + "decomposition", + status="missing", + score=500, + reason="not_emitted", + timeframe="M1", + ), + _target_risk( + "GBPUSD", + "dependence", + status="limited", + score=10, + reason="insufficient_samples", + ), + ), + ) + + payload = fingerprint_next_work_recommendation([("mixed.json", report)]) + + assert payload["recommendation"]["capability"] == "dependence" + assert payload["basis"]["ignored_non_base_target_count"] == 1 + assert payload["basis"]["base_grain"] == { + "data_format": "ascii", + "timeframe": "T", + } + + +def test_next_work_uses_current_discovery_and_historical_decomposition_case() -> ( + None +): + """A planned decomposition is recommended only with stationarity evidence.""" + discovery = deepcopy(fingerprint_schema_discovery()) + implemented = discovery["sections"]["implemented"]["target_sections"] + discovery["sections"]["implemented"]["target_sections"] = [ + row for row in implemented if row["name"] != "decomposition" + ] + discovery["sections"]["planned"]["target_sections"] = [ + { + "name": "decomposition", + "status": "planned", + "schema_version": None, + "issue": "#330", + } + ] + report = _report( + targets=(_target("EURUSD"),), + target_risks=(), + section_status_counts={"stationarity_diagnostics": {"valid": 1}}, + ) + + historical = fingerprint_next_work_recommendation( + [("stationary.json", report)], + discovery=discovery, + ) + current = fingerprint_next_work_recommendation( + [("stationary.json", report)] + ) + + assert historical["recommendation"]["capability"] == "decomposition" + assert historical["recommendation"]["issue_reference"] == "#330" + assert historical["recommendation"]["prerequisite_evidence"][0] == { + "section": "stationarity_diagnostics", + "implemented": True, + "observed_valid_target_count": 1, + "ready": True, + "basis": "readiness_section_status_counts", + } + assert current["status"] == "no_work" + assert current["recommendation"] is None + + +def test_next_work_bounds_alternates_axes_and_breaks_ties() -> None: + """Candidate and axis ordering should be deterministic and explicitly bounded.""" + report = _report( + targets=tuple(_target(symbol) for symbol in ("EURUSD", "GBPUSD")), + target_risks=tuple( + _target_risk( + symbol, + section, + status="missing", + score=35, + reason="not_emitted", + ) + for section in ("stationarity_diagnostics", "dependence") + for symbol in ("EURUSD", "GBPUSD") + ), + ) + + payload = fingerprint_next_work_recommendation( + [("bounded.json", report)], + alternate_limit=0, + target_axis_limit=1, + ) + + assert payload["recommendation"]["capability"] == "dependence" + assert payload["alternates"] == [] + assert payload["truncated"] is True + assert payload["limit_metadata"]["alternates"]["truncated"] is True + axes_limit = payload["recommendation"]["target_axis_limit_metadata"] + assert axes_limit["included_count"] == 1 + assert axes_limit["omitted_count"] == 1 + assert axes_limit["truncated"] is True + + +def test_next_work_aggregates_multiple_reports_and_target_kinds() -> None: + """Multiple saved reports should contribute to one product recommendation.""" + csv_report = _report( + targets=(_target("EURUSD"),), + target_risks=( + _target_risk( + "EURUSD", + "dependence", + status="missing", + score=35, + reason="not_emitted", + ), + ), + ) + cache_report = _report( + targets=(_target("GBPUSD", kind=QualityTargetKind.CACHE),), + target_risks=( + _target_risk( + "GBPUSD", + "dependence", + status="limited", + score=20, + reason="insufficient_samples", + kind="cache", + ), + ), + ) + + payload = fingerprint_next_work_recommendation( + [("csv.json", csv_report), ("cache.json", cache_report)] + ) + + assert payload["input_report_count"] == 2 + assert payload["recommendation"]["capability"] == "dependence" + assert payload["recommendation"]["affected_target_count"] == 2 + assert payload["recommendation"]["reason_codes"] == [ + "insufficient_samples", + "not_emitted", + ] + assert { + axis["kind"] + for axis in payload["recommendation"]["representative_target_axes"] + } == {"csv", "cache"} + + +def test_next_work_reports_surface_and_training_cross_series_evidence() -> None: + """Recommendation basis should retain bounded training/cross-series facts.""" + report = _report( + targets=(_target("EURUSD", kind=QualityTargetKind.CACHE),), + target_risks=(), + report_surface_evidence={ + "report_metadata_state_counts": {"present": 8, "missing": 1}, + }, + cross_series={ + "status": "limited", + "group_count": 1, + "incomplete_group_count": 0, + "cache_source_counts": {"direct": 1}, + "row_identity": { + "columns": [ + "series_id", + "period", + "row_id", + "source_row_number", + "event_seq", + ], + "training_schema_version": TRAINING_SCHEMA_VERSION, + "duplicate_timestamp_row_count": 2, + }, + "groups": [{"coverage_ranges": {"unequal_ranges": True}}], + "triangular_consistency": { + "candidate_count": 1, + "compared_timestamp_count": 3, + }, + }, + ) + + payload = fingerprint_next_work_recommendation([("cache.json", report)]) + training = payload["basis"]["training_substrate"] + cross = payload["basis"]["cross_series"] + + assert payload["recommendation"]["capability"] == ( + "fingerprint_report_surfaces" + ) + assert training["training_facing_columns_status"] == "confirmed" + assert training["single_row_training_surface"] is True + assert training["observed_enriched_cache_projection"] is True + assert cross["duplicate_timestamp_row_count"] == 2 + assert cross["unequal_range_group_count"] == 1 + assert cross["triangle_candidate_count"] == 1 + + +def test_next_work_is_publish_safe_and_does_not_mutate_report_golden( + tmp_path: Path, +) -> None: + """Report identity should be stable and the source report unchanged.""" + fixture = Path( + "tests/fixtures/data_quality_reports/fingerprint_report.json" + ) + report = load_quality_report(fixture) + before = quality_report_to_json(report) + + first = fingerprint_next_work_recommendation( + [(str(tmp_path / "fingerprint-report.json"), report)] + ) + second = fingerprint_next_work_recommendation( + [(str(tmp_path / "fingerprint-report.json"), report)] + ) + + assert first == second + assert first["input_reports"][0]["report_name"] == ( + "fingerprint-report.json" + ) + assert len(first["input_reports"][0]["content_sha256"]) == 64 + assert str(tmp_path) not in str(first) + assert quality_report_to_json(report) == before + assert "Next fingerprint work" in format_fingerprint_next_work(first) + + +def _report( + *, + targets: tuple[QualityTarget, ...], + target_risks: tuple[dict[str, JSONValue], ...], + section_status_counts: dict[str, JSONValue] | None = None, + report_surface_evidence: dict[str, JSONValue] | None = None, + cross_series: dict[str, JSONValue] | None = None, +) -> QualityReport: + risk: dict[str, JSONValue] = { + "schema_version": "histdatacom.time-series-fingerprint-readiness-risk.v1", + "target_count": len(targets), + "risk_target_count": len(target_risks), + "included_target_count": len(target_risks), + "truncated": False, + "section_status_counts": section_status_counts or {}, + "report_surface_evidence": report_surface_evidence or {}, + "target_risks": list(target_risks), + } + metadata: dict[str, JSONValue] = { + TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY: risk, + } + if cross_series is not None: + metadata[CROSS_SERIES_FINGERPRINT_METADATA_KEY] = cross_series + return QualityReport(targets=targets, metadata=metadata) + + +def _target( + symbol: str, + *, + timeframe: str = "T", + kind: QualityTargetKind = QualityTargetKind.CSV, +) -> QualityTarget: + return QualityTarget( + path=f"data/{symbol}-{timeframe}.csv", + kind=kind, + data_format="ascii", + timeframe=timeframe, + symbol=symbol, + period="201202", + ) + + +def _target_risk( + symbol: str, + section: str, + *, + status: str, + score: int, + reason: str, + timeframe: str = "T", + kind: str = "csv", +) -> dict[str, JSONValue]: + return { + "target_axis": { + "data_format": "ascii", + "kind": kind, + "period": "201202", + "symbol": symbol, + "timeframe": timeframe, + }, + "risk_score": score, + "section_risks": [ + { + "section": section, + "status": status, + "score": score, + "reasons": [reason], + } + ], + } diff --git a/tests/unit/test_data_quality_fingerprints.py b/tests/unit/test_data_quality_fingerprints.py index 877061ea..ed6dc532 100644 --- a/tests/unit/test_data_quality_fingerprints.py +++ b/tests/unit/test_data_quality_fingerprints.py @@ -7,7 +7,14 @@ from pathlib import Path from typing import Any +import histdatacom.data_quality.symbols as symbols_module +import histdatacom.data_quality.fingerprints as fingerprints_module +import pytest + from histdatacom.data_quality import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, + CROSS_SERIES_FINGERPRINT_RULE_ID, + CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, DEFAULT_FINGERPRINT_HISTOGRAM_BINS, DEFAULT_FINGERPRINT_LAGS, DEFAULT_FINGERPRINT_MAX_ROWS, @@ -15,12 +22,17 @@ DEFAULT_FINGERPRINT_ROLLING_WINDOWS, DEFAULT_FINGERPRINT_ROUNDING_DIGITS, QUALITY_PROFILE_SCHEMA_VERSION, + QUALITY_ENGINE_METADATA_KEY, + QUALITY_SKIP_EVENTS_SCHEMA_VERSION, SERIES_FINGERPRINT_RULE_ID, + SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_CALENDAR_REGIMES_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_CONDITIONAL_DISTRIBUTIONS_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY, TIME_SERIES_FINGERPRINT_COVERAGE_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DEPENDENCE_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DISTRIBUTION_ATTENTION_METADATA_KEY, TIME_SERIES_FINGERPRINT_DISTRIBUTION_ATTENTION_SCHEMA_VERSION, @@ -28,6 +40,7 @@ TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_DYNAMICS_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_STATIONARITY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_TOPOLOGY_ATTENTION_METADATA_KEY, @@ -35,14 +48,19 @@ TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_SCHEMA_VERSION, HistDataFingerprintDistributionAttentionProfile, + HistDataFingerprintParityProfile, HistDataFingerprintProfile, + HistDataCrossSeriesFingerprintRule, HistDataSeriesFingerprintRule, QualityFinding, + QualityReport, QualitySeverity, QualityTarget, QualityTargetKind, discover_quality_targets, quality_rules_for_groups, + quality_next_actions_summary, + quality_report_to_json, quality_run_rules_for_groups, quality_target_from_path, run_quality_assessment, @@ -52,6 +70,8 @@ series_fingerprint_topology_attention_summary, series_fingerprint_topology_summary, ) +from histdatacom.data_quality.reporting import quality_report_payload +from histdatacom.data_quality.training_features import TRAINING_SCHEMA_VERSION from histdatacom.histdata_ascii import ( CACHE_FILENAME, TICK, @@ -77,6 +97,8 @@ "microstructure_dynamics", "dependence", "stationarity_diagnostics", + "decomposition", + "synthetic_constraints", ] @@ -89,7 +111,418 @@ def test_fingerprint_group_registers_series_rule_surface() -> None: assert SERIES_FINGERPRINT_RULE_ID in { rule.rule_id for rule in quality_rules_for_groups(("all",)) } - assert quality_run_rules_for_groups(("fingerprint",)) == () + run_rules = quality_run_rules_for_groups(("fingerprint",)) + assert [rule.rule_id for rule in run_rules] == [ + CROSS_SERIES_FINGERPRINT_RULE_ID + ] + assert isinstance(run_rules[0], HistDataCrossSeriesFingerprintRule) + all_run_rules = quality_run_rules_for_groups(("all",)) + assert CROSS_SERIES_FINGERPRINT_RULE_ID in { + rule.rule_id for rule in all_run_rules + } + shared_domain_rule = next( + rule + for rule in all_run_rules + if rule.rule_id == "domain.cross_instrument_consistency" + ) + shared_fingerprint_rule = next( + rule + for rule in all_run_rules + if rule.rule_id == CROSS_SERIES_FINGERPRINT_RULE_ID + ) + assert shared_domain_rule.scan_provider is not None + assert shared_fingerprint_rule.scan_provider is ( + shared_domain_rule.scan_provider + ) + + +def test_cross_series_fingerprint_profiles_unequal_triangle_and_identity( + tmp_path: Path, +) -> None: + """Triangle fingerprints should preserve identity and limiting ranges.""" + cases = ( + _cross_series_case( + "EURUSD", + ( + "20120201 000000000,1.200000,1.200200,0", + "20120201 000001000,1.210000,1.210200,0", + "20120201 000001000,1.211000,1.211200,0", + "20120201 000002000,1.220000,1.220200,0", + "20120201 000003000,1.240000,1.240200,0", + ), + ), + _cross_series_case( + "GBPUSD", + ( + "20120201 000000000,1.500000,1.500200,0", + "20120201 000001000,1.510000,1.510200,0", + "20120201 000002000,1.525000,1.525200,0", + "20120201 000003000,1.540000,1.540200,0", + ), + ), + _cross_series_case( + "EURGBP", + ( + "20120201 000001000,0.801000,0.801200,0", + "20120201 000002000,0.900000,0.900200,0", + "20120201 000003000,0.805000,0.805200,0", + ), + ), + ) + targets = tuple( + _discovered_target(write_ascii_case(tmp_path / case.name, case)) + for case in cases + ) + + report = run_quality_assessment( + targets, + quality_rules_for_groups(("fingerprint",)), + run_rules=quality_run_rules_for_groups(("fingerprint",)), + metadata={"roots": [str(tmp_path)]}, + ) + payload = _mapping(report.metadata[CROSS_SERIES_FINGERPRINT_METADATA_KEY]) + group = _mapping(_list(payload["groups"])[0]) + grid = _mapping(group["timestamp_grid"]) + ranges = _mapping(group["coverage_ranges"]) + series_by_symbol = { + str(item["symbol"]): item + for item in (_mapping(value) for value in _list(group["series"])) + } + + assert payload["schema_version"] == CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION + assert payload["rule_id"] == CROSS_SERIES_FINGERPRINT_RULE_ID + assert payload["group_count"] == 1 + triangular = _mapping(payload["triangular_consistency"]) + assert triangular["candidate_count"] == 1 + assert grid["union_timestamp_count"] == 4 + assert grid["common_timestamp_count"] == 3 + assert grid["common_timestamp_ratio"] == 0.75 + assert ranges["unequal_ranges"] is True + assert ranges["limiting_start_symbols"] == ["EURGBP"] + assert series_by_symbol["EURUSD"]["row_count"] == 5 + assert series_by_symbol["EURUSD"]["unique_timestamp_count"] == 4 + assert series_by_symbol["EURUSD"]["duplicate_timestamp_row_count"] == 2 + assert series_by_symbol["EURUSD"]["identity_columns"] == [ + "series_id", + "period", + "row_id", + "source_row_number", + "event_seq", + ] + topology = _mapping(group["topology"]) + assert topology["target_count"] == 3 + assert topology["duplicate_timestamp_row_count"] == 2 + correlation = _mapping(group["return_correlation"]) + assert correlation["pair_count"] == 3 + assert any( + _mapping(pair)["status"] == "valid" + for pair in _list(correlation["pairs"]) + ) + triangle_samples = [ + *_list(triangular["warning_samples"]), + *_list(triangular["error_samples"]), + ] + assert triangle_samples + direct_identity = _mapping( + _mapping(_mapping(triangle_samples[0])["row_identity"])["direct"] + ) + assert direct_identity["series_id"] == "ascii:T:EURGBP:histdata.com" + assert direct_identity["period"] == "201202" + assert int(direct_identity["row_id"]) > 0 + cross_finding = next( + finding + for finding in report.findings + if finding.rule_id == CROSS_SERIES_FINGERPRINT_RULE_ID + ) + assert cross_finding.severity is QualitySeverity.INFO + assert str(tmp_path) not in str(payload) + rerun = quality_run_rules_for_groups(("fingerprint",))[0].evaluate_run( + targets, + metadata={ + TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY: ( + report.metadata[ + TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_METADATA_KEY + ] + ) + }, + ) + assert rerun.metadata[CROSS_SERIES_FINGERPRINT_METADATA_KEY] == payload + + +def test_all_group_shares_cross_instrument_scan_between_rule_surfaces( + tmp_path: Path, + monkeypatch: Any, +) -> None: + """The all group should not read the same panel twice.""" + targets = tuple( + _discovered_target( + write_ascii_case( + tmp_path / symbol.lower(), + _cross_series_case( + symbol, + ( + "20120201 000000000,1.200000,1.200200,0", + "20120201 000001000,1.210000,1.210200,0", + ), + ), + ) + ) + for symbol in ("EURGBP", "EURUSD", "GBPUSD") + ) + calls = 0 + original_scan = symbols_module._scan_cross_instrument_consistency + + def counted_scan(*args: Any, **kwargs: Any) -> Any: + nonlocal calls + calls += 1 + return original_scan(*args, **kwargs) + + monkeypatch.setattr( + symbols_module, + "_scan_cross_instrument_consistency", + counted_scan, + ) + cross_rules = tuple( + rule + for rule in quality_run_rules_for_groups(("all",)) + if rule.rule_id + in { + "domain.cross_instrument_consistency", + CROSS_SERIES_FINGERPRINT_RULE_ID, + } + ) + + report = run_quality_assessment(targets, (), run_rules=cross_rules) + + assert calls == 1 + assert "cross_instrument_consistency" in report.metadata + assert CROSS_SERIES_FINGERPRINT_METADATA_KEY in report.metadata + + +def test_cross_series_fingerprint_reports_inverse_sparse_and_stale_risk( + tmp_path: Path, +) -> None: + """Inverse and sparse panels should retain descriptive risk summaries.""" + cases = ( + _cross_series_case( + "EURUSD", + ( + "20120201 000000000,1.250000,1.250000,0", + "20120201 000001000,1.260000,1.260000,0", + "20120201 000002000,1.270000,1.270000,0", + "20120201 000003000,1.280000,1.280000,0", + ), + ), + _cross_series_case( + "USDEUR", + ("20120201 000000000,0.800000,0.800000,0",), + ), + ) + targets = tuple( + _discovered_target(write_ascii_case(tmp_path / case.name, case)) + for case in cases + ) + + report = run_quality_assessment( + targets, + (), + run_rules=quality_run_rules_for_groups(("fingerprint",)), + ) + payload = _mapping(report.metadata[CROSS_SERIES_FINGERPRINT_METADATA_KEY]) + group = _mapping(_list(payload["groups"])[0]) + pair = _mapping(_list(_mapping(group["return_correlation"])["pairs"])[0]) + + assert _mapping(payload["inverse_consistency"])["candidate_count"] == 1 + assert _mapping(payload["stale_join_risk"])["risk_count"] == 1 + assert _mapping(group["timestamp_grid"])["common_timestamp_ratio"] == 0.25 + assert pair == { + "left_symbol": "EURUSD", + "right_symbol": "USDEUR", + "overlap_return_count": 0, + "status": "unavailable", + "reason": "insufficient_overlap", + } + assert payload["status"] == "limited" + + +def test_cross_series_fingerprint_reports_limiting_triangle_period_range( + tmp_path: Path, +) -> None: + """A later-starting triangle leg should limit common panel coverage.""" + cases = ( + _cross_series_case( + "EURUSD", + ("20000501 000000000,1.100000,1.100200,0",), + period="200005", + ), + _cross_series_case( + "GBPUSD", + ("20000501 000000000,1.500000,1.500200,0",), + period="200005", + ), + _cross_series_case( + "EURUSD", + ("20020301 000000000,1.200000,1.200200,0",), + period="200203", + ), + _cross_series_case( + "GBPUSD", + ("20020301 000000000,1.500000,1.500200,0",), + period="200203", + ), + _cross_series_case( + "EURGBP", + ("20020301 000000000,0.800000,0.800200,0",), + period="200203", + ), + ) + targets = tuple( + _discovered_target( + write_ascii_case(tmp_path / f"{case.name}-{index}", case) + ) + for index, case in enumerate(cases) + ) + + report = run_quality_assessment( + targets, + (), + run_rules=quality_run_rules_for_groups(("fingerprint",)), + ) + payload = _mapping(report.metadata[CROSS_SERIES_FINGERPRINT_METADATA_KEY]) + panel = _mapping(_list(payload["panel_coverage"])[0]) + groups = { + str(_mapping(group)["group_id"]): _mapping(group) + for group in _list(payload["groups"]) + } + + assert panel["union_period_count"] == 2 + assert panel["common_period_count"] == 1 + assert panel["common_first_period"] == "200203" + assert panel["unequal_period_ranges"] is True + assert panel["limiting_start_symbols"] == ["EURGBP"] + assert _mapping(panel["missing_period_count_by_symbol"])["EURGBP"] == 1 + assert groups["ascii:T:200005"]["complete"] is False + assert groups["ascii:T:200005"]["missing_symbols"] == ["EURGBP"] + assert groups["ascii:T:200203"]["complete"] is True + assert payload["incomplete_group_count"] == 1 + assert payload["status"] == "limited" + + +def test_cross_series_fingerprint_enriches_legacy_raw_cache_for_report( + tmp_path: Path, +) -> None: + """Legacy raw caches should be enriched in memory before projection.""" + targets: list[QualityTarget] = [] + for symbol, prices in { + "EURUSD": ("1.200000", "1.210000", "1.220000"), + "GBPUSD": ("1.500000", "1.510000", "1.520000"), + }.items(): + rows = tuple( + f"20120201 00000{index}000,{price},{price},0" + for index, price in enumerate(prices) + ) + batch = parse_ascii_lines(TICK, rows) + cache_path = tmp_path / symbol.lower() / CACHE_FILENAME + cache_path.parent.mkdir(parents=True) + write_polars_cache(to_polars_frame(batch), cache_path) + targets.append( + QualityTarget( + path=str(cache_path), + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period="201202", + ) + ) + + report = run_quality_assessment( + targets, + (), + run_rules=quality_run_rules_for_groups(("fingerprint",)), + ) + report_payload = quality_report_payload(report) + payload = _mapping( + _mapping(report_payload["metadata"])[ + CROSS_SERIES_FINGERPRINT_METADATA_KEY + ] + ) + series = [ + _mapping(item) + for item in _list(_mapping(_list(payload["groups"])[0])["series"]) + ] + + assert {item["computed_from"] for item in series} == {"direct_cache"} + assert {item["cache_source"] for item in series} == {"direct"} + assert {item["training_schema_version"] for item in series} == { + TRAINING_SCHEMA_VERSION + } + assert all(str(item["series_id"]).startswith("ascii:T:") for item in series) + assert str(tmp_path) not in str(report_payload) + + +def test_cross_series_fingerprint_reports_mixed_cache_provenance( + tmp_path: Path, +) -> None: + """Group topology should expose direct, sibling, and text scan bases.""" + rows = ( + "20120201 000000000,1.200000,1.200200,0", + "20120201 000001000,1.210000,1.210200,0", + "20120201 000002000,1.220000,1.220200,0", + ) + batch = parse_ascii_lines(TICK, rows) + + direct_path = tmp_path / "eurusd" / CACHE_FILENAME + direct_path.parent.mkdir(parents=True) + write_polars_cache(to_polars_frame(batch), direct_path) + direct_target = QualityTarget( + path=str(direct_path), + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + + sibling_case = _cross_series_case("GBPUSD", rows) + sibling_csv = write_ascii_case(tmp_path / "gbpusd", sibling_case) + sibling_cache = sibling_csv.with_name(CACHE_FILENAME) + write_polars_cache(to_polars_frame(batch), sibling_cache) + csv_mtime_ns = sibling_csv.stat().st_mtime_ns + os.utime( + sibling_cache, + ns=(csv_mtime_ns + 1_000_000, csv_mtime_ns + 1_000_000), + ) + sibling_target = _discovered_target(sibling_csv) + + text_target = _discovered_target( + write_ascii_case( + tmp_path / "eurgbp", + _cross_series_case("EURGBP", rows), + ) + ) + + report = run_quality_assessment( + (direct_target, sibling_target, text_target), + quality_rules_for_groups(("fingerprint",)), + run_rules=quality_run_rules_for_groups(("fingerprint",)), + ) + payload = _mapping(report.metadata[CROSS_SERIES_FINGERPRINT_METADATA_KEY]) + topology = _mapping(_mapping(_list(payload["groups"])[0])["topology"]) + + assert topology["computed_from_counts"] == { + "direct_cache": 1, + "fresh_sibling_cache": 1, + "text_scan": 1, + } + assert topology["cache_source_counts"] == {"direct": 1, "sibling": 1} + assert topology["topology_computed_from_counts"] == { + "direct_cache": 1, + "fresh_sibling_cache": 1, + "text_scan": 1, + } + assert topology["mixed_computation_basis"] is True + assert topology["mixed_cache_source"] is True def test_fingerprint_rule_emits_tick_csv_payload(tmp_path: Path) -> None: @@ -153,6 +586,36 @@ def test_fingerprint_rule_emits_tick_csv_payload(tmp_path: Path) -> None: assert stationarity["metric"] == "mid_price" assert _mapping(stationarity["sample_counts"]) == {"level": 3, "return": 2} + decomposition = _mapping(payload["decomposition"]) + assert decomposition["schema_version"] == ( + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION + ) + assert decomposition["metric"] == "mid_price" + assert _mapping(decomposition["sample_counts"]) == { + "level": 3, + "return": 2, + } + projection = _mapping(decomposition["training_projection"]) + assert projection["schema_version"] == ( + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert projection["grain"] == "period" + assert _list(projection["identity_fields"]) == [ + "series_id", + "period", + "row_id", + ] + + constraints = _mapping(payload["synthetic_constraints"]) + assert constraints["schema_version"] == SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION + assert constraints["status"] == "ready" + assert ( + _mapping(constraints["training_substrate"])[ + "source_rows_enriched_in_memory" + ] + is True + ) + audit = _mapping(payload["fingerprint_audit"]) assert ( audit["schema_version"] == TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION @@ -166,6 +629,10 @@ def test_fingerprint_rule_emits_tick_csv_payload(tmp_path: Path) -> None: assert statuses["microstructure_dynamics"] == "valid" assert statuses["dependence"] == "limited" assert statuses["stationarity_diagnostics"] == "limited" + assert statuses["decomposition"] == "limited" + assert statuses["synthetic_constraints"] == "valid" + assert _mapping(audit["decomposition_readiness"])["status"] == "limited" + assert _retired_bar_schema_keys(payload) == set() def test_fingerprint_tick_microstructure_dynamics_describe_sequence( @@ -273,6 +740,149 @@ def test_fingerprint_stationarity_diagnostics_describe_stable_tick_series( assert level_mean_drift["absolute_change"] == 0.0 +def test_fingerprint_decomposition_handles_flat_and_trending_ticks( + tmp_path: Path, +) -> None: + """Decomposition proxies should distinguish flat and linear tick levels.""" + profile = HistDataFingerprintProfile( + rolling_windows=(2, 3), rounding_digits=6 + ) + flat = _mapping( + _payload_for_case( + tmp_path / "flat", + _tick_case_from_mid_prices("tick-flat", (1.0,) * 9), + profile, + )["decomposition"] + ) + trending = _mapping( + _payload_for_case( + tmp_path / "trend", + _tick_case_from_mid_prices( + "tick-trend", + (1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8), + ), + profile, + )["decomposition"] + ) + + assert flat["decomposition_status"] == "limited" + assert "zero_variance" in _list(flat["limitations"]) + assert _mapping(flat["trend_proxy"])["direction"] == "flat" + assert ( + _mapping(flat["residual_proxy"])["residual_to_level_variance_ratio"] + is None + ) + assert _mapping(flat["stationarity_basis"])["status"] == "valid" + assert _mapping(flat["stationarity_basis"])["zero_variance_metrics"] + + trend = _mapping(trending["trend_proxy"]) + assert trend["direction"] == "increasing" + assert trend["slope_per_observation"] == 0.1 + assert trend["trend_strength"] == 1.0 + assert trending["computed_window_count"] == 2 + assert _mapping(trending["stationarity_basis"])["status"] == "valid" + assert _retired_bar_schema_keys(trending) == set() + + +def test_fingerprint_decomposition_seasonality_is_calendar_based_and_bounded( + tmp_path: Path, +) -> None: + """Seasonality buckets should reuse source-calendar sessions and limits.""" + case = HistDataAsciiCase( + name="tick-decomposition-seasonality", + timeframe=TICK, + filename="DAT_ASCII_EURUSD_T_201202.csv", + rows=( + "20120102 010000000,1.000000,1.000000,0", + "20120102 090000000,1.100000,1.100000,0", + "20120103 170000000,1.200000,1.200000,0", + "20120104 230000000,1.300000,1.300000,0", + ), + ) + profile = HistDataFingerprintProfile( + rolling_windows=(2,), histogram_bins=2, rounding_digits=6 + ) + decomposition = _mapping( + _payload_for_case(tmp_path, case, profile)["decomposition"] + ) + seasonality = _mapping(decomposition["seasonality_proxy"]) + by_hour = _mapping(seasonality["by_source_hour"]) + by_weekday = _mapping(seasonality["by_source_weekday"]) + by_session = _mapping(seasonality["by_active_session"]) + + assert seasonality["grouped_by"] == [ + "source_hour", + "source_weekday", + "active_session", + ] + assert by_hour["bucket_count"] == 4 + assert by_hour["included_bucket_count"] == 2 + assert by_hour["truncated"] is True + assert by_weekday["bucket_count"] == 3 + assert by_session["bucket_count"] >= 3 + assert all( + len(_mapping(group)["buckets"]) <= 2 + for group in ( + by_hour, + by_weekday, + by_session, + ) + ) + + +def test_fingerprint_decomposition_insufficient_series_is_unavailable( + tmp_path: Path, +) -> None: + """A one-row series should report deterministic unavailable proxies.""" + decomposition = _mapping( + _payload_for_case( + tmp_path, + _tick_case_from_mid_prices("tick-insufficient", (1.0,)), + HistDataFingerprintProfile(rolling_windows=(2, 3)), + )["decomposition"] + ) + + assert decomposition["decomposition_status"] == "unavailable" + assert decomposition["reason"] == "insufficient_sequence_rows" + assert "insufficient_sample_count" in _list(decomposition["limitations"]) + assert decomposition["computed_window_count"] == 0 + assert decomposition["skipped_window_count"] == 2 + assert _mapping(decomposition["structural_break_proxy"])["status"] == ( + "skipped" + ) + assert _mapping(decomposition["stationarity_basis"])["status"] == ( + "unavailable" + ) + + +def test_fingerprint_decomposition_structural_break_proxy_is_deterministic( + tmp_path: Path, +) -> None: + """Structural candidates should rank a fixed step change identically.""" + case = _tick_case_from_mid_prices( + "tick-structural-break", + (1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0), + ) + profile = HistDataFingerprintProfile( + rolling_windows=(2,), histogram_bins=3, rounding_digits=6 + ) + first = _mapping( + _payload_for_case(tmp_path / "first", case, profile)["decomposition"] + ) + second = _mapping( + _payload_for_case(tmp_path / "second", case, profile)["decomposition"] + ) + first_structural = _mapping(first["structural_break_proxy"]) + second_structural = _mapping(second["structural_break_proxy"]) + + assert first_structural == second_structural + assert first_structural["status"] == "computed" + assert first_structural["candidate_count"] == 5 + assert first_structural["included_candidate_count"] == 3 + assert first_structural["truncated"] is True + assert _mapping(first_structural["strongest_candidate"])["split_index"] == 3 + + def test_fingerprint_calendar_regimes_and_conditioning_are_tick_based( tmp_path: Path, ) -> None: @@ -484,6 +1094,40 @@ def test_fingerprint_rule_prefers_direct_cache_payload(tmp_path: Path) -> None: assert dynamics["row_order"] == "cache_order" assert dynamics["computed_from"] == "direct_cache" assert dynamics["cache_source"] == "direct" + training = _mapping( + _mapping(payload["synthetic_constraints"])["training_substrate"] + ) + assert training["legacy_cache_enriched_on_read"] is True + assert training["training_schema_version"] == TRAINING_SCHEMA_VERSION + + +def test_direct_cache_topology_inspection_counts_duplicate_timestamps( + tmp_path: Path, +) -> None: + """Polars-backed topology should retain bounded duplicate evidence.""" + cache_path = tmp_path / CACHE_FILENAME + rows = (CLEAN_TICK_ROWS[0], CLEAN_TICK_ROWS[0], CLEAN_TICK_ROWS[1]) + batch = parse_ascii_lines(TICK, rows) + write_polars_cache(to_polars_frame(batch), cache_path) + target = QualityTarget( + path=str(cache_path), + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + + payload = _fingerprint_payload(_fingerprint_finding(target)) + topology = _mapping(payload["temporal_topology"]) + duplicate = _mapping( + _mapping(topology["inspection_context"])["duplicate_timestamps"] + ) + + assert topology["computed_from"] == "direct_cache" + assert topology["duplicate_timestamp_count"] == 1 + assert duplicate["duplicate_row_count"] == 1 + assert _mapping(_list(duplicate["samples"])[0])["occurrence_count"] == 2 def test_fingerprint_rule_prefers_fresh_sibling_cache(tmp_path: Path) -> None: @@ -511,6 +1155,208 @@ def test_fingerprint_rule_prefers_fresh_sibling_cache(tmp_path: Path) -> None: assert topology["cache_source"] == "sibling" +def test_fingerprint_parity_is_opt_in_and_matches_fresh_sibling_cache( + tmp_path: Path, +) -> None: + """Opt-in parity should compare source, cache, and training projections.""" + csv_path = write_ascii_case(tmp_path, CLEAN_TICK_CASE) + cache_path = csv_path.with_name(CACHE_FILENAME) + batch = parse_ascii_lines(TICK, CLEAN_TICK_ROWS) + write_polars_cache(to_polars_frame(batch), cache_path) + csv_mtime_ns = csv_path.stat().st_mtime_ns + os.utime( + cache_path, + ns=(csv_mtime_ns + 1_000_000, csv_mtime_ns + 1_000_000), + ) + target = _discovered_target(csv_path) + + default_payload = _fingerprint_payload(_fingerprint_finding(target)) + parity_payload = _fingerprint_payload( + _fingerprint_finding(target, _parity_profile()) + ) + parity = _mapping(parity_payload["cache_source_parity"]) + + assert "cache_source_parity" not in default_payload + assert ( + parity["schema_version"] + == TIME_SERIES_FINGERPRINT_PARITY_SCHEMA_VERSION + ) + assert parity["status"] == "match" + assert parity["mismatch_codes"] == [] + assert _mapping(_mapping(parity["bases"])["raw_cache"])["freshness"] == ( + "fresh" + ) + enriched = _mapping(_mapping(parity["bases"])["enriched_cache"]) + assert enriched["legacy_cache_enriched_on_read"] is True + assert enriched["training_schema_version"] == TRAINING_SCHEMA_VERSION + assert {item["section"] for item in _list(parity["comparisons"])} >= { + "coverage", + "temporal_topology", + "calendar_regimes", + "conditional_distributions", + "training_columns", + "row_identity", + "duplicate_timestamps", + "quality_report_projection", + "influx_projection", + } + audit = _mapping(parity_payload["fingerprint_audit"]) + assert "cache_source_parity" in _list(audit["sections_expected"]) + assert "cache_source_parity" in _list(audit["sections_emitted"]) + assert _mapping(audit["section_statuses"])["cache_source_parity"] == ( + "valid" + ) + assert str(tmp_path) not in str(parity) + + +def test_fingerprint_parity_reports_divergent_and_stale_cache( + tmp_path: Path, +) -> None: + """Stale and divergent cache evidence should produce stable reason codes.""" + csv_path = write_ascii_case(tmp_path, CLEAN_TICK_CASE) + cache_path = csv_path.with_name(CACHE_FILENAME) + divergent = parse_ascii_lines(TICK, CLEAN_TICK_ROWS[:2]) + write_polars_cache(to_polars_frame(divergent), cache_path) + csv_mtime_ns = csv_path.stat().st_mtime_ns + os.utime( + cache_path, + ns=(csv_mtime_ns - 1_000_000, csv_mtime_ns - 1_000_000), + ) + + payload = _fingerprint_payload( + _fingerprint_finding(_discovered_target(csv_path), _parity_profile()) + ) + parity = _mapping(payload["cache_source_parity"]) + codes = set(_list(parity["mismatch_codes"])) + + assert parity["status"] == "mismatch" + assert "fingerprint_cache_source_stale_cache" in codes + assert "fingerprint_cache_source_row_count_mismatch" in codes + assert "fingerprint_cache_source_topology_mismatch" in codes + assert "fingerprint_cache_source_calendar_mismatch" in codes + assert "fingerprint_cache_source_conditioned_spread_mismatch" in codes + + +def test_fingerprint_parity_detects_same_cardinality_regime_and_spread_drift( + tmp_path: Path, +) -> None: + """Calendar and spread drift should not depend on row-count divergence.""" + csv_path = write_ascii_case(tmp_path, CLEAN_TICK_CASE) + cache_path = csv_path.with_name(CACHE_FILENAME) + divergent_rows = ( + CLEAN_TICK_ROWS[0] + .replace("000003660", "120003660") + .replace("1.306770", "1.307770"), + CLEAN_TICK_ROWS[1].replace("000003973", "120003973"), + CLEAN_TICK_ROWS[2].replace("000014990", "120014990"), + ) + write_polars_cache( + to_polars_frame(parse_ascii_lines(TICK, divergent_rows)), + cache_path, + ) + csv_mtime_ns = csv_path.stat().st_mtime_ns + os.utime( + cache_path, + ns=(csv_mtime_ns + 1_000_000, csv_mtime_ns + 1_000_000), + ) + + payload = _fingerprint_payload( + _fingerprint_finding(_discovered_target(csv_path), _parity_profile()) + ) + codes = set( + _list(_mapping(payload["cache_source_parity"])["mismatch_codes"]) + ) + + assert "fingerprint_cache_source_row_count_mismatch" not in codes + assert "fingerprint_cache_source_calendar_mismatch" in codes + assert "fingerprint_cache_source_conditioned_spread_mismatch" in codes + + +def test_fingerprint_parity_handles_missing_cache_source_and_zip_member( + tmp_path: Path, +) -> None: + """Missing bases should skip safely while ZIP sibling caches compare.""" + csv_path = write_ascii_case(tmp_path / "csv", CLEAN_TICK_CASE) + missing_cache = _fingerprint_payload( + _fingerprint_finding(_discovered_target(csv_path), _parity_profile()) + ) + missing = _mapping(missing_cache["cache_source_parity"]) + assert missing["status"] == "not_compared" + assert missing["skipped_reasons"] == ["cache_unavailable"] + + direct_target = _cache_target(tmp_path / "direct") + missing_source = _fingerprint_payload( + _fingerprint_finding(direct_target, _parity_profile()) + ) + direct = _mapping(missing_source["cache_source_parity"]) + assert direct["status"] == "not_compared" + assert direct["skipped_reasons"] == ["source_target_unavailable"] + assert ( + _mapping(_mapping(direct["bases"])["raw_cache"])["cache_source"] + == "direct" + ) + + archive = write_zip_case( + tmp_path / "zip", + CLEAN_TICK_CASE, + zip_filename="HISTDATA_COM_ASCII_EURUSD_T201202.zip", + ) + zip_cache = archive.with_name(CACHE_FILENAME) + batch = parse_ascii_lines(TICK, CLEAN_TICK_ROWS) + write_polars_cache(to_polars_frame(batch), zip_cache) + archive_mtime_ns = archive.stat().st_mtime_ns + os.utime( + zip_cache, + ns=(archive_mtime_ns + 1_000_000, archive_mtime_ns + 1_000_000), + ) + zip_payload = _fingerprint_payload( + _fingerprint_finding(_discovered_target(archive), _parity_profile()) + ) + zip_parity = _mapping(zip_payload["cache_source_parity"]) + raw_source = _mapping(_mapping(zip_parity["bases"])["raw_source"]) + assert zip_parity["status"] == "match" + assert raw_source["kind"] == "zip_member" + assert raw_source["member"] == CLEAN_TICK_CASE.filename + + +def test_fingerprint_parity_detects_market_only_projection_regressions( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Report and Influx bases should reject market-only projections.""" + csv_path = write_ascii_case(tmp_path, CLEAN_TICK_CASE) + cache_path = csv_path.with_name(CACHE_FILENAME) + batch = parse_ascii_lines(TICK, CLEAN_TICK_ROWS) + write_polars_cache(to_polars_frame(batch), cache_path) + csv_mtime_ns = csv_path.stat().st_mtime_ns + os.utime( + cache_path, + ns=(csv_mtime_ns + 1_000_000, csv_mtime_ns + 1_000_000), + ) + monkeypatch.setattr( + fingerprints_module, + "quality_report_from_training_features", + lambda *_args, **_kwargs: QualityReport(), + ) + monkeypatch.setattr( + fingerprints_module, + "format_influx_line", + lambda *_args, **_kwargs: "EURUSD,source=histdata.com bidquote=1.0,askquote=1.1 1", + ) + + payload = _fingerprint_payload( + _fingerprint_finding(_discovered_target(csv_path), _parity_profile()) + ) + codes = set( + _list(_mapping(payload["cache_source_parity"])["mismatch_codes"]) + ) + + assert ( + "fingerprint_cache_source_quality_report_projection_mismatch" in codes + ) + assert "fingerprint_cache_source_influx_projection_mismatch" in codes + + def test_fingerprint_cache_distribution_counts_full_rows_and_fills_sample( tmp_path: Path, ) -> None: @@ -742,6 +1588,19 @@ def test_fingerprint_coverage_summary_reports_duplicate_archive_skip( "duplicate_archive_preferred_csv": 1, } assert summary["source_kind_counts"] == {"csv_text": 1} + engine = _mapping(report.metadata[QUALITY_ENGINE_METADATA_KEY]) + skips = _mapping(engine["skip_events"]) + assert skips["schema_version"] == QUALITY_SKIP_EVENTS_SCHEMA_VERSION + assert skips["event_count"] == summary["skipped_fingerprint_target_count"] + assert skips["reason_counts"] == summary["skipped_reason_counts"] + assert skips["rule_id_counts"] == {SERIES_FINGERPRINT_RULE_ID: 1} + assert _mapping(_list(skips["events"])[0])["target_axis"] == { + "data_format": "ascii", + "timeframe": "T", + "symbol": "EURUSD", + "period": "201202", + "kind": "zip", + } def test_series_fingerprint_distribution_summary_counts_tick_payloads( @@ -1025,6 +1884,73 @@ def test_series_fingerprint_topology_attention_orders_mixed_remediation_hints( assert "expected_session_closures" in _list(target_summary["flags"]) +def test_topology_inspection_context_links_bounded_evidence_to_next_actions( + tmp_path: Path, +) -> None: + """Attention evidence should link to stable run-level action identities.""" + case = HistDataAsciiCase( + name="tick_topology_inspection", + timeframe=TICK, + filename="DAT_ASCII_EURUSD_T_201202_INSPECTION.csv", + rows=( + "20120203 165900000,1.306600,1.306770,0", + "bad-timestamp,1.306600,1.306770,0", + "20120205 170100000,1.306570,1.306740,17", + "20120205 172000000,1.306580,1.306750,18", + "20120205 172000000,1.306580,1.306750,18", + "20120205 171000000,1.306590,1.306760,19", + ), + ) + target = _discovered_target(write_ascii_case(tmp_path, case)) + report = run_quality_assessment( + (target,), + quality_rules_for_groups( + ("fingerprint",), + profile={ + "schema_version": QUALITY_PROFILE_SCHEMA_VERSION, + "name": "inspection-limit", + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "topology_inspection_sample_limit": 1, + } + }, + }, + ), + ) + attention = _mapping( + report.metadata[TIME_SERIES_FINGERPRINT_TOPOLOGY_ATTENTION_METADATA_KEY] + ) + target_summary = _mapping(_list(attention["target_summaries"])[0]) + context = _mapping(target_summary["inspection_context"]) + next_actions = _mapping(quality_next_actions_summary(report)) + action_codes = { + _mapping(action)["code"] for action in _list(next_actions["actions"]) + } + + expected_links = { + "invalid_timestamps": "inspect_invalid_timestamp_rows", + "non_monotonic_timestamps": "repair_timestamp_order", + "duplicate_timestamps": "inspect_duplicate_timestamp_rows", + "suspicious_gaps": "inspect_gap_boundaries", + } + for section_name, action_code in expected_links.items(): + section = _mapping(context[section_name]) + next_action = _mapping(section["next_action"]) + assert section["actionable"] is True + assert next_action["code"] == action_code + assert next_action["rule_id"] == SERIES_FINGERPRINT_RULE_ID + assert next_action["flag"] == section_name + assert _mapping(section["target_axis"])["symbol"] == "EURUSD" + assert action_code in action_codes + assert section["included_count"] <= 1 + closure = _mapping(context["expected_session_closures"]) + assert closure["actionable"] is False + assert closure["contextual_for"] == "suspicious_gaps" + assert "next_action" not in closure + assert str(tmp_path) not in str(context) + assert "duplicate_row_values" not in str(context) + + def test_series_fingerprint_topology_attention_ignores_context_only_targets( tmp_path: Path, ) -> None: @@ -1051,6 +1977,104 @@ def test_series_fingerprint_topology_attention_ignores_context_only_targets( assert attention["target_summaries"] == [] +def test_weekend_topology_guidance_follows_active_calendar_policy( + tmp_path: Path, +) -> None: + """Default, strict, and allowed profiles should produce distinct advice.""" + cases = ( + ("default", None, "advisory", "verify", "session", True), + ( + "strict", + _calendar_policy_profile(weekend="strict"), + "strict", + "inspect", + "session", + True, + ), + ( + "allowed", + _calendar_policy_profile(weekend="allowed"), + "allowed", + "context", + "contextual", + False, + ), + ) + for name, profile, policy, action_kind, level, actionable in cases: + target = _discovered_target( + write_ascii_case(tmp_path / name, _weekend_activity_case()) + ) + report = run_quality_assessment( + (target,), + quality_rules_for_groups(("fingerprint",), profile=profile), + ) + attention = _mapping( + report.metadata[ + TIME_SERIES_FINGERPRINT_TOPOLOGY_ATTENTION_METADATA_KEY + ] + ) + target_summary = _mapping(_list(attention["target_summaries"])[0]) + hint = _mapping(_list(target_summary["remediation_hints"])[0]) + context = _mapping(hint["policy_context"]) + inspection = _mapping(target_summary["inspection_context"]) + weekend = _mapping(inspection["weekend_activity"]) + + assert target_summary["attention_level"] == level + assert hint["code"] == "verify_weekend_session_policy" + assert hint["action_kind"] == action_kind + assert context["weekend_activity_policy"] == policy + assert context["actionable"] is actionable + assert weekend["actionable"] is actionable + if actionable: + actions = _mapping(quality_next_actions_summary(report)) + action = _mapping(_list(actions["actions"])[0]) + assert ( + _mapping(action["policy_context"])["weekend_activity_policy"] + == policy + ) + assert action["action_kind"] == action_kind + assert "next_action" in weekend + else: + assert quality_next_actions_summary(report) is None + assert "policy_note" in weekend + assert "next_action" not in weekend + assert quality_report_to_json(report) == quality_report_to_json(report) + + +def test_expected_closure_becomes_actionable_only_when_profile_marks_unexpected( + tmp_path: Path, +) -> None: + """An explicit unexpected-closure policy should create bounded guidance.""" + target = _discovered_target( + write_ascii_case(tmp_path, _expected_weekend_closure_case()) + ) + report = run_quality_assessment( + (target,), + quality_rules_for_groups( + ("fingerprint",), + profile=_calendar_policy_profile(closures="unexpected"), + ), + ) + attention = _mapping( + report.metadata[TIME_SERIES_FINGERPRINT_TOPOLOGY_ATTENTION_METADATA_KEY] + ) + target_summary = _mapping(_list(attention["target_summaries"])[0]) + hint = _mapping(_list(target_summary["remediation_hints"])[0]) + closure = _mapping( + _mapping(target_summary["inspection_context"])[ + "expected_session_closures" + ] + ) + + assert target_summary["attention_flags"] == ["expected_session_closures"] + assert target_summary["attention_level"] == "session" + assert hint["code"] == "inspect_unexpected_session_closure" + assert closure["actionable"] is True + assert _mapping(closure["next_action"])["code"] == ( + "inspect_unexpected_session_closure" + ) + + def test_fingerprint_id_excludes_source_path_volatility(tmp_path: Path) -> None: """Identical content and target axis should hash the same across paths.""" first = write_ascii_case( @@ -1145,6 +2169,14 @@ def test_fingerprint_constants_are_stable() -> None: TIME_SERIES_FINGERPRINT_STATIONARITY_SCHEMA_VERSION == "histdatacom.time-series-fingerprint-stationarity.v1" ) + assert ( + TIME_SERIES_FINGERPRINT_DECOMPOSITION_SCHEMA_VERSION + == "histdatacom.time-series-fingerprint-decomposition.v1" + ) + assert ( + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION + == "histdatacom.time-series-fingerprint-decomposition-training-projection.v1" + ) assert ( TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION == "histdatacom.time-series-fingerprint-audit.v1" @@ -1174,6 +2206,20 @@ def _payload_for_case( return _fingerprint_payload(_fingerprint_finding(target, profile)) +def _cross_series_case( + symbol: str, + rows: tuple[str, ...], + *, + period: str = "201202", +) -> HistDataAsciiCase: + return HistDataAsciiCase( + name=f"cross_series_{symbol.lower()}", + timeframe=TICK, + filename=f"DAT_ASCII_{symbol}_T_{period}.csv", + rows=rows, + ) + + def _discovered_target(path: Path) -> QualityTarget: target = quality_target_from_path(path) assert target is not None @@ -1211,6 +2257,24 @@ def _list(value: Any) -> list[Any]: return value +def _retired_bar_schema_keys(value: Any) -> set[str]: + matches: set[str] = set() + if isinstance(value, Mapping): + for key, nested in value.items(): + normalized = str(key).lower() + if ( + "m1" in normalized + or "ohlc" in normalized + or normalized.startswith("bar_") + ): + matches.add(str(key)) + matches.update(_retired_bar_schema_keys(nested)) + elif isinstance(value, list): + for nested in value: + matches.update(_retired_bar_schema_keys(nested)) + return matches + + def _tick_case_from_mid_prices( name: str, mid_prices: tuple[float, ...], @@ -1340,6 +2404,41 @@ def _weekend_activity_case() -> HistDataAsciiCase: ) +def _calendar_policy_profile( + *, + weekend: str = "advisory", + closures: str = "expected", +) -> dict[str, Any]: + return { + "schema_version": QUALITY_PROFILE_SCHEMA_VERSION, + "name": f"calendar-{weekend}-{closures}", + "rules": { + "domain.calendar_sessions": { + "calendar_profile": { + "name": f"calendar-{weekend}-{closures}", + "source": "operator-config", + "version": "2026.07", + "complete": True, + "weekend_activity_policy": weekend, + "expected_session_closure_policy": closures, + } + } + }, + } + + +def _parity_profile( + *, + mismatch_limit: int = 16, +) -> HistDataFingerprintProfile: + return HistDataFingerprintProfile( + cache_source_parity=HistDataFingerprintParityProfile( + enabled=True, + mismatch_limit=mismatch_limit, + ) + ) + + def _cache_target(directory: Path) -> QualityTarget: cache_path = directory / CACHE_FILENAME cache_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/unit/test_data_quality_ingestion.py b/tests/unit/test_data_quality_ingestion.py index 378d2fba..c7730444 100644 --- a/tests/unit/test_data_quality_ingestion.py +++ b/tests/unit/test_data_quality_ingestion.py @@ -10,6 +10,8 @@ ASCII_TEXT_INGESTION_RULE_ID, QualitySeverity, QualityStatus, + QualityTarget, + QualityTargetKind, discover_quality_targets, quality_rules_for_groups, run_quality_assessment, @@ -42,6 +44,22 @@ def test_ingestion_group_registers_text_rule() -> None: ] +def test_ingestion_rules_noop_unsupported_raw_dimensions() -> None: + """Private ingestion rules must not reopen retired raw dimensions.""" + rules = quality_rules_for_groups(("ingestion",)) + for data_format, timeframe in ( + ("ascii", "M1"), + ("ninjatrader", "T"), + ): + target = QualityTarget( + path="/missing/retired.csv", + kind=QualityTargetKind.CSV, + data_format=data_format, + timeframe=timeframe, + ) + assert all(rule.evaluate(target) == () for rule in rules) + + def test_clean_ascii_file_passes_ingestion_text_checks( tmp_path: Path, ) -> None: @@ -164,7 +182,10 @@ def test_cache_target_reports_row_count_size_schema_and_bounds( tmp_path: Path, ) -> None: """Readable Polars cache files should expose equivalent metadata.""" - cache_path = tmp_path / CACHE_FILENAME + cache_path = ( + tmp_path / "ASCII" / "T" / "eurusd" / "2012" / "02" / CACHE_FILENAME + ) + cache_path.parent.mkdir(parents=True) batch = parse_ascii_lines(TICK, CLEAN_TICK_ROWS) write_polars_cache(to_polars_frame(batch), cache_path) diff --git a/tests/unit/test_data_quality_preflight.py b/tests/unit/test_data_quality_preflight.py index 2ddedd52..206daadd 100644 --- a/tests/unit/test_data_quality_preflight.py +++ b/tests/unit/test_data_quality_preflight.py @@ -26,6 +26,7 @@ from histdatacom.data_quality.preflight import ( DEFAULT_QUALITY_PREFLIGHT_VALIDATION_REPORT_DIR, QUALITY_PREFLIGHT_SCHEMA_VERSION, + QUALITY_PREFLIGHT_VALIDATION_EVIDENCE_SCHEMA_VERSION, QUALITY_PREFLIGHT_VALIDATION_REPORT_LATEST, discover_latest_quality_preflight_validation_report, format_quality_preflight_evidence_inspection, @@ -509,8 +510,15 @@ def test_quality_preflight_embeds_enabled_remediation_catalog_audit( assert audit["summary"]["report_count"] == 1 assert audit["report_coverage"][0]["source"] == "current-report" + assert audit["remediation_plan"]["items"] assert "sample remediation audit:" in console + assert "sample remediation plan:" in console + assert "actionable_warning_error_gaps=" in console + assert "intentional_boundaries=" in console assert "### Sample Remediation Catalog Audit" in markdown + assert "#### Top Remediation Plan Items" in markdown + assert "Actionable warning/error gaps" in markdown + assert "Intentional warning/error boundaries" in markdown assert str(tmp_path) not in encoded assert str(tmp_path) not in markdown @@ -583,6 +591,50 @@ def test_quality_preflight_imports_validation_report_status( assert str(tmp_path) not in markdown +def test_quality_preflight_imports_full_plain_pytest_result( + tmp_path: Path, +) -> None: + """The release-independent closure gate should satisfy full pytest.""" + data_dir = tmp_path / "data" + _write_tick_cache(data_dir, symbol="eurusd", row_multiplier=1) + validation_path = _write_validation_report( + tmp_path / "closure.json", + generated_at_utc="2026-07-10T12:00:00Z", + results=[ + { + "name": "full-tests", + "command": "python -m pytest", + "status": "pass", + "returncode": 0, + "duration_seconds": 0.04, + "stdout_tail": "1370 passed", + "log_path": str(tmp_path / "logs" / "pytest.log"), + } + ], + ) + + payload = run_cache_quality_preflight( + data_dir, + pairs=("eurusd",), + formats=("ascii",), + timeframes=("T",), + quality_check_groups=("inventory",), + sample_size=1, + validation_report_path=validation_path, + ) + rows = { + str(row["name"]): row + for row in payload["evidence"]["validation_commands"] + } + + assert rows["full-pytest"]["status"] == "pass" + assert rows["full-pytest"]["command"] == "python -m pytest" + assert rows["full-pytest"]["duration_seconds"] == 0.04 + assert rows["full-pytest"]["output_artifact_path"] == "pytest.log" + assert "1370 passed" in str(rows["full-pytest"]["summary"]) + assert str(tmp_path) not in json.dumps(payload, sort_keys=True) + + def test_quality_preflight_discovers_latest_validation_report( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -738,6 +790,8 @@ def runner(command: Sequence[str]) -> subprocess.CompletedProcess[str]: calls.append(args) return subprocess.CompletedProcess(args, 0, stdout="ok\n", stderr="") + validation_ticks = iter((1.0, 1.2, 2.0, 2.3, 3.0, 3.4)) + payload = run_cache_quality_preflight( data_dir, pairs=("eurusd",), @@ -747,16 +801,26 @@ def runner(command: Sequence[str]) -> subprocess.CompletedProcess[str]: sample_size=1, run_validation=True, validation_runner=runner, + validation_clock=lambda: next(validation_ticks), ) statuses = _validation_statuses(payload) assert payload["evidence"]["validation_source"]["state"] == "generated" assert statuses["focused-quality-preflight-tests"] == "pass" assert statuses["git-diff-check"] == "pass" + assert statuses["readme-help-sync"] == "pass" assert statuses["full-pytest"] == "skipped" assert statuses["full-pre-commit"] == "skipped" assert any(call[-3:] == ("git", "diff", "--check") for call in calls) + assert any("sync_readme_cli_help.py" in " ".join(call) for call in calls) assert not any("pre_commit" in " ".join(call) for call in calls) + durations = { + str(row["name"]): row.get("duration_seconds") + for row in payload["evidence"]["validation_commands"] + } + assert durations["focused-quality-preflight-tests"] == 0.2 + assert durations["readme-help-sync"] == 0.3 + assert durations["git-diff-check"] == 0.4 def test_quality_preflight_flags_budget_failures(tmp_path: Path) -> None: @@ -1364,6 +1428,8 @@ def test_cli_accepts_quality_preflight_without_temporal_mode( "markdown", "--quality-preflight-validation-report", "latest", + "--quality-preflight-validation-evidence", + str(tmp_path / "validation-evidence.json"), "--quality-preflight-run-validation", "--pair-groups", "majors", @@ -1389,6 +1455,9 @@ def test_cli_accepts_quality_preflight_without_temporal_mode( ) assert options.quality_preflight_profile_preview_format == "markdown" assert options.quality_preflight_validation_report_path == "latest" + assert options.quality_preflight_validation_evidence_path == str( + tmp_path / "validation-evidence.json" + ) assert options.quality_preflight_run_validation assert options.pair_groups == ["majors"] @@ -1530,6 +1599,124 @@ def fail_submit(*args: object, **kwargs: object) -> None: assert "Fingerprint Contract Audit" in markdown +def test_api_quality_preflight_writes_dry_validation_evidence_artifact( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Artifact output alone should inspect the plan without running gates.""" + import histdatacom.histdata_com as histdata_com + + data_dir = tmp_path / "data" + artifact_path = tmp_path / "reports" / "validation.json" + _write_tick_cache(data_dir, symbol="eurusd", row_multiplier=1) + + def fail_run(*args: object, **kwargs: object) -> None: + raise AssertionError("dry validation evidence must not run commands") + + monkeypatch.setattr(preflight_module.subprocess, "run", fail_run) + options = Options() + options.quality_preflight = True + options.quality_paths = (str(data_dir),) + options.quality_check_groups = {"inventory"} + options.quality_preflight_sample_size = 1 + options.quality_preflight_validation_evidence_path = str(artifact_path) + options.pairs = {"eurusd"} + options.formats = {"ascii"} + options.timeframes = {"T"} + + payload = histdata_com.main(options) + artifact_bytes = artifact_path.read_bytes() + validation = json.loads(artifact_bytes) + artifact = payload["evidence"]["artifacts"]["validation_evidence"] + markdown = quality_preflight_to_markdown(payload) + console = format_quality_preflight_console_summary(payload) + + assert validation["schema_version"] == ( + QUALITY_PREFLIGHT_VALIDATION_EVIDENCE_SCHEMA_VERSION + ) + assert validation["state"] == "planned" + assert validation["status_counts"]["not-run"] == 5 + assert all(row["exit_code"] is None for row in validation["commands"]) + assert all( + row["duration_seconds"] is None for row in validation["commands"] + ) + assert artifact["kind"] == "quality-preflight-validation-evidence" + assert artifact["path"] == "reports/validation.json" + assert artifact["format"] == "json" + assert artifact["schema_version"] == ( + QUALITY_PREFLIGHT_VALIDATION_EVIDENCE_SCHEMA_VERSION + ) + assert artifact["generated_at_utc"] == validation["generated_at_utc"] + assert artifact["validation_state"] == "planned" + assert artifact["size_bytes"] == len(artifact_bytes) + assert artifact["sha256"] == hashlib.sha256(artifact_bytes).hexdigest() + assert "Validation evidence" in markdown + assert "validation evidence: written: reports/validation.json (json)" in ( + console + ) + assert str(tmp_path) not in json.dumps(payload, sort_keys=True) + assert str(tmp_path) not in artifact_bytes.decode("utf-8") + + +def test_api_quality_preflight_validation_artifact_preserves_failure_bounded( + tmp_path: Path, +) -> None: + """Imported failures should remain failed, bounded, and publish-safe.""" + import histdatacom.histdata_com as histdata_com + + data_dir = tmp_path / "data" + artifact_path = tmp_path / "reports" / "validation-fail.json" + validation_path = _write_validation_report( + tmp_path / "closure-fail.json", + generated_at_utc="2026-07-10T12:00:00Z", + results=[ + { + "name": "pre-commit", + "command": "python -m pre_commit run --all-files", + "status": "fail", + "returncode": 1, + "duration_seconds": 2.75, + "stderr_tail": f"{tmp_path / 'secret.py'} " + "x" * 4000, + "log_path": str(tmp_path / "logs" / "pre-commit.log"), + }, + { + "name": "readme-help-sync", + "command": "python scripts/sync_readme_cli_help.py --check", + "status": "pass", + "returncode": 0, + "duration_seconds": 0.25, + "stdout_tail": "README help is synchronized", + }, + ], + ) + _write_tick_cache(data_dir, symbol="eurusd", row_multiplier=1) + options = Options() + options.quality_preflight = True + options.quality_paths = (str(data_dir),) + options.quality_check_groups = {"inventory"} + options.quality_preflight_sample_size = 1 + options.quality_preflight_validation_report_path = str(validation_path) + options.quality_preflight_validation_evidence_path = str(artifact_path) + options.pairs = {"eurusd"} + options.formats = {"ascii"} + options.timeframes = {"T"} + + payload = histdata_com.main(options) + validation = json.loads(artifact_path.read_text(encoding="utf-8")) + rows = {row["name"]: row for row in validation["commands"]} + + assert validation["state"] == "fail" + assert rows["full-pre-commit"]["status"] == "fail" + assert rows["full-pre-commit"]["exit_code"] == 1 + assert rows["full-pre-commit"]["duration_seconds"] == 2.75 + assert len(rows["full-pre-commit"]["summary"]) <= 1200 + assert rows["full-pre-commit"]["output_artifact_path"] == ("pre-commit.log") + assert rows["readme-help-sync"]["status"] == "pass" + encoded = json.dumps(payload, sort_keys=True) + assert str(tmp_path) not in encoded + assert str(tmp_path) not in artifact_path.read_text(encoding="utf-8") + + @pytest.mark.parametrize( ("preview_format", "filename", "expected_prefix"), ( diff --git a/tests/unit/test_data_quality_profiles.py b/tests/unit/test_data_quality_profiles.py index 401dcc0e..b53d732c 100644 --- a/tests/unit/test_data_quality_profiles.py +++ b/tests/unit/test_data_quality_profiles.py @@ -12,15 +12,25 @@ QUALITY_PROFILE_SCHEMA_VERSION, QUALITY_REPORTING_METADATA_KEY, SERIES_FINGERPRINT_RULE_ID, + AutoregressiveProfile, + ClassicalModelComparisonProfile, + ClassicalModelInputProfile, + ExponentialSmoothingProfile, HistDataSeriesFingerprintRule, QualityFinding, QualityProfileError, QualityReport, QualityStatus, + SeasonalExogenousProfile, + StateSpaceProfile, + VolatilityProfile, + apply_quality_profile_overrides, discover_quality_targets, load_quality_profile_file, + load_quality_profile_file_resolution, quality_profile_report_metadata, quality_rules_for_groups, + resolve_quality_profile, run_quality_assessment, ) from histdatacom.histdata_ascii import TICK @@ -169,6 +179,7 @@ def test_profile_fingerprint_knobs_flow_to_rule_surface() -> None: "histogram_bins": 16, "max_rows": 1000, "rounding_digits": 8, + "topology_inspection_sample_limit": 2, "distribution_attention": { "invalid_row_min_count": 2, "invalid_row_min_rate": 0.5, @@ -179,6 +190,19 @@ def test_profile_fingerprint_knobs_flow_to_rule_surface() -> None: "flag_truncated_distribution": False, "flag_cache_float_precision": False, }, + "cache_source_parity": { + "enabled": True, + "mismatch_limit": 7, + }, + "classical_baselines": { + "enabled": True, + "evaluation_fraction": 0.25, + "minimum_training_rows": 12, + "minimum_evaluation_rows": 4, + "rolling_windows": [3, 9], + "session_seasonal_enabled": False, + "rounding_digits": 6, + }, } }, }, @@ -193,6 +217,7 @@ def test_profile_fingerprint_knobs_flow_to_rule_surface() -> None: "histogram_bins": 16, "max_rows": 1000, "rounding_digits": 8, + "topology_inspection_sample_limit": 2, "distribution_attention": { "invalid_row_min_count": 2, "invalid_row_min_rate": 0.5, @@ -203,9 +228,115 @@ def test_profile_fingerprint_knobs_flow_to_rule_surface() -> None: "flag_truncated_distribution": False, "flag_cache_float_precision": False, }, + "cache_source_parity": { + "enabled": True, + "mismatch_limit": 7, + }, + "classical_baselines": { + "enabled": True, + "evaluation_fraction": 0.25, + "minimum_training_rows": 12, + "minimum_evaluation_rows": 4, + "rolling_windows": [3, 9], + "session_seasonal_enabled": False, + "rounding_digits": 6, + }, + "classical_model_input": ClassicalModelInputProfile().to_metadata(), + "exponential_smoothing": ExponentialSmoothingProfile().to_metadata(), + "autoregressive": AutoregressiveProfile().to_metadata(), + "seasonal_exogenous": SeasonalExogenousProfile().to_metadata(), + "state_space": StateSpaceProfile().to_metadata(), + "volatility": VolatilityProfile().to_metadata(), + "classical_model_comparison": ( + ClassicalModelComparisonProfile().to_metadata() + ), } +def test_classical_model_input_profile_flows_to_rule_surface() -> None: + """Regularization, fold, transform, and resource controls should parse.""" + rules = quality_rules_for_groups( + ("fingerprint",), + profile={ + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "classical_model_input": { + "enabled": True, + "frequency_ms": 1_000, + "midpoint_aggregation": "median", + "spread_aggregation": "mean", + "transform": "log_return", + "horizons": [1, 3], + "fold_kind": "rolling", + "minimum_training_observations": 10, + "minimum_evaluation_observations": 2, + "rolling_window": 12, + "resources": {"max_folds": 8}, + } + } + } + }, + ) + + model_input = rules[0].profile.classical_model_input + assert model_input.enabled is True + assert model_input.frequency_ms == 1_000 + assert model_input.midpoint_aggregation == "median" + assert model_input.spread_aggregation == "mean" + assert model_input.transform == "log_return" + assert model_input.horizons == (1, 3) + assert model_input.fold_kind == "rolling" + assert model_input.rolling_window == 12 + assert model_input.resources.max_folds == 8 + + +def test_exponential_smoothing_profile_flows_to_rule_surface() -> None: + """Explicit fitted-family configurations should parse without search.""" + rules = quality_rules_for_groups( + ("fingerprint",), + profile={ + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "exponential_smoothing": { + "enabled": True, + "projection_specification_id": "hw", + "projection_horizon": 3, + "baseline_rolling_windows": [3, 9], + "specifications": [ + { + "specification_id": "hw", + "family": "holt_winters", + "trend": "add", + "seasonal": "mul", + "seasonal_periods": 12, + "initialization_method": "estimated", + "parameter_bounds": [ + { + "parameter": "smoothing_level", + "lower": 0.01, + "upper": 0.99, + } + ], + } + ], + } + } + } + }, + ) + + profile = rules[0].profile.exponential_smoothing + assert profile.enabled is True + assert profile.projection_specification_id == "hw" + assert profile.projection_horizon == 3 + assert profile.baseline_rolling_windows == (3, 9) + assert profile.specifications[0].family == "holt_winters" + assert profile.specifications[0].seasonal == "mul" + assert profile.specifications[0].parameter_bounds == ( + ("smoothing_level", 0.01, 0.99), + ) + + @pytest.mark.parametrize( "profile", ( @@ -225,6 +356,13 @@ def test_profile_fingerprint_knobs_flow_to_rule_surface() -> None: {"rules": {SERIES_FINGERPRINT_RULE_ID: {"quantiles": [0.5, 0.1]}}}, {"rules": {SERIES_FINGERPRINT_RULE_ID: {"lags": [1, 1]}}}, {"rules": {SERIES_FINGERPRINT_RULE_ID: {"histogram_bins": 0}}}, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "topology_inspection_sample_limit": 6 + } + } + }, { "rules": { SERIES_FINGERPRINT_RULE_ID: {"distribution_attention": "loose"} @@ -253,6 +391,72 @@ def test_profile_fingerprint_knobs_flow_to_rule_surface() -> None: } } }, + {"rules": {SERIES_FINGERPRINT_RULE_ID: {"cache_source_parity": True}}}, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "cache_source_parity": {"mismatch_limit": -1} + } + } + }, + {"rules": {SERIES_FINGERPRINT_RULE_ID: {"classical_baselines": True}}}, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "classical_baselines": {"evaluation_fraction": 1.0} + } + } + }, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "classical_baselines": {"minimum_training_rows": 0} + } + } + }, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: {"classical_model_input": True} + } + }, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "classical_model_input": {"horizons": [3, 1]} + } + } + }, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "classical_model_input": { + "fold_kind": "rolling", + "rolling_window": 1, + } + } + } + }, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "exponential_smoothing": {"specifications": []} + } + } + }, + { + "rules": { + SERIES_FINGERPRINT_RULE_ID: { + "exponential_smoothing": { + "specifications": [ + { + "specification_id": "bad", + "family": "holt_winters", + } + ] + } + } + } + }, {"reporting": "enabled"}, {"reporting": {"unknown": {}}}, {"reporting": {"remediation_catalog_audit": True}}, @@ -293,6 +497,102 @@ def test_quality_profile_file_loads_json_payload(tmp_path: Path) -> None: assert profile.row_count_profile().min_row_count == 10 +def test_default_profile_resolution_attributes_every_value_to_builtin() -> None: + """Default resolution should expose deterministic built-in provenance.""" + resolution = resolve_quality_profile() + payload = resolution.to_payload() + + assert resolution.profile.is_default + assert [channel["kind"] for channel in payload["input_channels"]] == [ + "built_in_default" + ] + assert {item["source"] for item in payload["effective_value_sources"]} == { + "built_in_default" + } + + +def test_profile_file_resolution_preserves_nested_and_yaml_selection_sources( + tmp_path: Path, +) -> None: + """Nested thresholds should retain file, name, and YAML selection facts.""" + profile_path = tmp_path / "quality-profile.json" + config_path = tmp_path / "histdatacom.yaml" + profile_path.write_text( + json.dumps( + { + "name": "nested-file-profile", + "rules": { + "ingestion.ascii.row_count": { + "min_row_count": 25, + } + }, + } + ), + encoding="utf-8", + ) + + resolution = load_quality_profile_file_resolution( + profile_path, + config_path=str(config_path), + selected_by="yaml_config", + ) + payload = resolution.to_payload() + sources = { + item["path"]: item for item in payload["effective_value_sources"] + } + + assert [channel["kind"] for channel in payload["input_channels"]] == [ + "built_in_default", + "yaml_config", + "named_profile", + "profile_file", + ] + assert sources["/name"]["source"] == "named_profile" + assert sources["/rules/ingestion.ascii.row_count/min_row_count"] == { + "path": "/rules/ingestion.ascii.row_count/min_row_count", + "value": 25, + "source": "profile_file", + "profile_name": "nested-file-profile", + "source_path": str(profile_path), + "selected_by": "yaml_config", + } + + +def test_profile_resolution_override_records_previous_source_and_value() -> ( + None +): + """Overrides should preserve what source and value they replaced.""" + resolution = resolve_quality_profile( + { + "name": "override-profile", + "reporting": {"remediation_catalog_audit": {"enabled": False}}, + }, + source="file", + source_path="quality-profile.json", + ) + + overridden = apply_quality_profile_overrides( + resolution, + {"reporting.remediation_catalog_audit.enabled": True}, + source="cli_override", + ) + sources = { + item["path"]: item + for item in overridden.to_payload()["effective_value_sources"] + } + + assert sources["/reporting/remediation_catalog_audit/enabled"] == { + "path": "/reporting/remediation_catalog_audit/enabled", + "value": True, + "source": "cli_override", + "profile_name": "override-profile", + "override": True, + "previous_source": "profile_file", + "overridden_source": "profile_file", + "previous_value": False, + } + + def _report_for_path( path: Path, *, diff --git a/tests/unit/test_data_quality_remediation.py b/tests/unit/test_data_quality_remediation.py index cf929dd6..c54c60af 100644 --- a/tests/unit/test_data_quality_remediation.py +++ b/tests/unit/test_data_quality_remediation.py @@ -2,10 +2,15 @@ from __future__ import annotations +import pytest + from histdatacom.data_quality import ( + CALENDAR_POLICY_REMEDIATION_CONTEXT_SCHEMA_VERSION, QualityFinding, QualitySeverity, QualityTarget, + RemediationActionability, + classify_remediation_actionability, remediation_hint_payloads_for_finding, remediation_hint_payloads_for_flags, remediation_hints_for_finding_code, @@ -15,6 +20,138 @@ TARGET = QualityTarget(path="DAT_ASCII_EURUSD_T_201202.csv") +def _calendar_policy( + *, + weekend: str = "advisory", + closures: str = "expected", +) -> dict: + return { + "source_timezone": "EST-no-DST", + "canonical_timezone": "UTC", + "holiday_calendar_complete": weekend != "advisory", + "holiday_calendar_static_advisory": weekend == "advisory", + "weekend_activity_policy": weekend, + "expected_session_closure_policy": closures, + "calendar_profile": { + "name": "policy-profile", + "source": "operator-config", + "version": "2026.07", + }, + } + + +def test_remediation_actionability_classifies_supported_boundaries() -> None: + """Every public actionability status should have deterministic evidence.""" + cases = ( + ( + { + "rule_id": "inventory.zip.integrity", + "finding_code": "ZIP_CORRUPT", + "severity": "error", + "mapped": True, + }, + RemediationActionability.REMEDIABLE_DEFECT, + "mapped_remediation_hint", + ), + ( + { + "rule_id": "modeling.readiness", + "finding_code": "MODELING_CALENDAR_REGIME_POLICY_MISSING", + "severity": "warning", + "mapped": False, + }, + RemediationActionability.POLICY_OR_PROFILE_DECISION, + "policy_or_profile_context_required", + ), + ( + { + "rule_id": "inventory.format_support", + "finding_code": "HISTDATA_FORMAT_UNSUPPORTED", + "severity": "error", + "mapped": False, + }, + RemediationActionability.UNSUPPORTED_FORMAT_OR_CAPABILITY, + "unsupported_format_rule", + ), + ( + { + "rule_id": "provenance.manifest.lineage", + "finding_code": "PROVENANCE_MANIFEST_UNAVAILABLE", + "severity": "warning", + "mapped": False, + }, + RemediationActionability.EXPECTED_ARTIFACT_OR_CONTEXT, + "expected_artifact_or_context", + ), + ( + { + "rule_id": "time.unresolved", + "finding_code": "CUSTOM_SHARED_FAILURE", + "severity": "error", + "mapped": False, + "attribution_status": "unresolved", + }, + RemediationActionability.NEEDS_RULE_ATTRIBUTION, + "unresolved_rule_attribution", + ), + ( + { + "rule_id": "custom.diagnostics", + "finding_code": "DIAGNOSTIC_CONTEXT_MISSING", + "severity": "warning", + "mapped": False, + }, + RemediationActionability.NEEDS_DIAGNOSTIC_CONTEXT, + "missing_diagnostic_context", + ), + ( + { + "rule_id": "custom.repair", + "finding_code": "DESTRUCTIVE_REPAIR_REQUIRED", + "severity": "error", + "mapped": False, + }, + RemediationActionability.UNSAFE_TO_AUTOMATE, + "unsafe_automatic_repair", + ), + ( + { + "rule_id": "fingerprint.series", + "finding_code": "FINGERPRINT_SERIES_SUMMARY", + "severity": "info", + "mapped": False, + }, + RemediationActionability.INFORMATIONAL_ONLY, + "informational_severity", + ), + ) + + for arguments, expected_status, expected_reason in cases: + decision = classify_remediation_actionability(**arguments) + + assert decision.actionability is expected_status + assert decision.reason == expected_reason + assert decision.to_payload() == { + "actionability": expected_status.value, + "actionability_reason": expected_reason, + } + + +def test_remediation_actionability_defaults_warning_errors_to_actionable() -> ( + None +): + """Unknown warning/error gaps must not be hidden as boundaries.""" + decision = classify_remediation_actionability( + rule_id="custom.rule", + finding_code="CUSTOM_FAILURE", + severity="error", + mapped=False, + ) + + assert decision.actionability is RemediationActionability.REMEDIABLE_DEFECT + assert decision.reason == "unmapped_warning_or_error" + + def test_remediation_catalog_reproduces_topology_hint_codes_and_messages() -> ( None ): @@ -104,6 +241,97 @@ def test_remediation_catalog_preserves_input_order_and_ignores_unknowns() -> ( ] +@pytest.mark.parametrize( + ("policy", "message", "action_kind", "actionable"), + ( + ( + "strict", + "inspect weekend activity against strict no-weekend policy", + "inspect", + True, + ), + ( + "advisory", + "verify weekend-session profile assumptions", + "verify", + True, + ), + ( + "allowed", + "weekend activity is allowed by the active profile", + "context", + False, + ), + ), +) +def test_weekend_remediation_hint_is_calendar_policy_aware( + policy: str, + message: str, + action_kind: str, + actionable: bool, +) -> None: + """Weekend guidance should preserve its code while policy changes advice.""" + payload = remediation_hint_payloads_for_flags( + ("weekend_activity",), + calendar_policy=_calendar_policy(weekend=policy), + )[0] + context = payload["policy_context"] + + assert payload["code"] == "verify_weekend_session_policy" + assert payload["message"] == message + assert payload["action_kind"] == action_kind + assert context == { + "schema_version": CALENDAR_POLICY_REMEDIATION_CONTEXT_SCHEMA_VERSION, + "flag": "weekend_activity", + "actionable": actionable, + "profile_name": "policy-profile", + "profile_source": "operator-config", + "profile_version": "2026.07", + "source_timezone": "EST-no-DST", + "canonical_timezone": "UTC", + "calendar_complete": policy != "advisory", + "calendar_static_advisory": policy == "advisory", + "weekend_activity_policy": policy, + "expected_session_closure_policy": "expected", + } + + +def test_expected_closure_hint_requires_explicit_unexpected_policy() -> None: + """Expected closures remain contextual unless the profile says otherwise.""" + assert ( + remediation_hint_payloads_for_flags( + ("expected_session_closures",), + calendar_policy=_calendar_policy(closures="expected"), + ) + == [] + ) + + payload = remediation_hint_payloads_for_flags( + ("expected_session_closures",), + calendar_policy=_calendar_policy(closures="unexpected"), + )[0] + + assert payload["code"] == "inspect_unexpected_session_closure" + assert payload["action_kind"] == "inspect" + assert ( + payload["policy_context"]["expected_session_closure_policy"] + == "unexpected" + ) + + +def test_calendar_policy_context_bounds_profile_text() -> None: + """Policy-aware hints should never echo unbounded profile metadata.""" + policy = _calendar_policy(weekend="strict") + policy["calendar_profile"]["name"] = "x" * 500 + + payload = remediation_hint_payloads_for_flags( + ("weekend_activity",), + calendar_policy=policy, + )[0] + + assert payload["policy_context"]["profile_name"] == "x" * 128 + + def test_remediation_catalog_maps_representative_time_finding() -> None: finding = QualityFinding( severity=QualitySeverity.WARNING, diff --git a/tests/unit/test_data_quality_remediation_audit.py b/tests/unit/test_data_quality_remediation_audit.py index 2297a2bc..e7a395e9 100644 --- a/tests/unit/test_data_quality_remediation_audit.py +++ b/tests/unit/test_data_quality_remediation_audit.py @@ -3,9 +3,11 @@ from __future__ import annotations import json +import os from pathlib import Path from histdatacom.data_quality import ( + QUALITY_REMEDIATION_PLAN_SCHEMA_VERSION, KnownQualityFindingCode, QualityFinding, QualityReport, @@ -119,7 +121,19 @@ def test_remediation_catalog_audit_maps_inventory_archive_batch() -> None: assert payload["summary"]["unmapped_warning_error_gap_count"] == 1 assert payload["known_unmapped_codes"] == [ { + "actionability": "unsupported_format_or_capability", + "actionability_reason": "unsupported_format_rule", + "attribution_reason": "provided_rule_id", + "attribution_reason_counts": {"provided_rule_id": 1}, + "attribution_status": "exact", + "attribution_status_counts": {"exact": 1}, "finding_code": "HISTDATA_FORMAT_UNSUPPORTED", + "finding_code_prefix_counts": [ + { + "count": 1, + "finding_code_prefix": "HISTDATA_FORMAT", + }, + ], "included_source_count": 1, "mapped": False, "max_severity": "error", @@ -132,6 +146,7 @@ def test_remediation_catalog_audit_maps_inventory_archive_batch() -> None: "source_family_counts": [ {"count": 1, "source_family": "inventory"}, ], + "source_helper_counts": [], "sources": [ { "count": 1, @@ -152,6 +167,158 @@ def test_remediation_catalog_audit_maps_inventory_archive_batch() -> None: assert encoded == remediation_catalog_audit_to_json(payload) +def test_remediation_catalog_audit_ranks_actionable_gaps_before_boundaries() -> ( + None +): + """Actionable defects should outrank more frequent support boundaries.""" + payload = audit_remediation_catalog( + known_findings=( + _known( + "inventory.format_support", + "HISTDATA_FORMAT_UNSUPPORTED", + QualitySeverity.ERROR, + source_family="inventory", + ), + _known( + "inventory.format_support", + "HISTDATA_FORMAT_UNSUPPORTED", + QualitySeverity.ERROR, + source="data_quality/inventory.py:2", + source_family="inventory", + ), + _known( + "inventory.format_support", + "HISTDATA_FORMAT_UNSUPPORTED", + QualitySeverity.ERROR, + source="data_quality/inventory.py:3", + source_family="inventory", + ), + _known( + "custom.rule", + "CUSTOM_REPAIRABLE_FAILURE", + QualitySeverity.WARNING, + ), + KnownQualityFindingCode( + rule_id="time.unresolved", + finding_code="CUSTOM_SHARED_FAILURE", + severity=QualitySeverity.ERROR, + source="data_quality/time.py:1", + source_family="time", + attribution_status="unresolved", + attribution_reason="ambiguous_helper_rules", + ), + _known( + "custom.diagnostics", + "DIAGNOSTIC_CONTEXT_MISSING", + QualitySeverity.WARNING, + ), + _known( + "modeling.readiness", + "MODELING_CALENDAR_REGIME_POLICY_MISSING", + QualitySeverity.WARNING, + ), + _known( + "custom.repair", + "DESTRUCTIVE_REPAIR_REQUIRED", + QualitySeverity.ERROR, + ), + ) + ) + + ranked = payload["ranked_gaps"] + summary = payload["summary"] + + assert ranked[0]["finding_code"] == "CUSTOM_REPAIRABLE_FAILURE" + assert ranked[0]["actionability"] == "remediable_defect" + assert ranked[1]["actionability"] == "needs_diagnostic_context" + assert ranked[2]["actionability"] == "needs_rule_attribution" + assert ranked[-1]["actionability"] == ("unsupported_format_or_capability") + assert "actionability=remediable_defect" in ranked[0]["rank_reasons"] + assert summary["unmapped_actionable_warning_error_code_count"] == 1 + assert summary["blocked_by_attribution_warning_error_code_count"] == 1 + assert ( + summary["blocked_by_missing_diagnostics_warning_error_code_count"] == 1 + ) + assert summary["intentionally_unremediable_warning_error_code_count"] == 3 + + +def test_remediation_catalog_audit_emits_fixability_ranked_plan() -> None: + """Plan items should turn exact actionable gaps into catalog-edit inputs.""" + payload = audit_remediation_catalog( + known_findings=( + _known( + "custom.rule", + "CUSTOM_INVALID_ROW", + QualitySeverity.ERROR, + ), + _known( + "inventory.format_support", + "HISTDATA_FORMAT_UNSUPPORTED", + QualitySeverity.ERROR, + ), + ) + ) + + plan = payload["remediation_plan"] + first = plan["items"][0] + + assert plan["schema_version"] == QUALITY_REMEDIATION_PLAN_SCHEMA_VERSION + assert plan["plan_item_count"] == 2 + assert plan["included_plan_item_count"] == 2 + assert plan["truncated"] is False + assert first["rank"] == 1 + assert first["catalog_gap_rank"] == 1 + assert first["finding_code"] == "CUSTOM_INVALID_ROW" + assert first["suggested_selector"] == { + "shape": "exact_rule_and_finding", + "rule_id": "custom.rule", + "finding_code": "CUSTOM_INVALID_ROW", + "finding_code_prefix": "CUSTOM_INVALID_ROW", + "confidence": "high", + "basis": "exact_rule_attribution", + } + assert first["draft_hint_code"] == "repair_custom_invalid_row" + assert first["suggested_action"] == { + "action_kind": "repair", + "confidence": "high", + "basis": "finding_code_marker=invalid", + "concrete": True, + } + assert first["fixability"]["level"] == "high" + assert first["fixability"]["score"] == 96 + assert first["missing_fields"] == ["message"] + assert plan["items"][1]["fixability"]["level"] == "low" + assert "Remediation plan" in format_remediation_catalog_audit(payload) + + +def test_remediation_plan_marks_unresolved_attribution_blocked() -> None: + """An unresolved rule must not become an apparently exact catalog plan.""" + payload = audit_remediation_catalog( + known_findings=( + KnownQualityFindingCode( + rule_id="time.unresolved", + finding_code="CUSTOM_INVALID_ROW", + severity=QualitySeverity.ERROR, + source="data_quality/time.py:1", + source_family="time", + attribution_status="unresolved", + attribution_reason="ambiguous_helper_rules", + ), + ) + ) + item = payload["remediation_plan"]["items"][0] + + assert item["suggested_selector"]["shape"] == "finding_family" + assert item["fixability"]["level"] == "blocked" + assert item["fixability"]["score"] == 24 + assert item["missing_fields"] == [ + "message", + "exact_rule_id", + "action_kind_confirmation", + "blocking_evidence", + ] + + def test_remediation_catalog_audit_keeps_info_only_gaps_advisory() -> None: """INFO-only missing guidance should be visible without failing the audit.""" payload = audit_remediation_catalog( @@ -211,6 +378,21 @@ def test_remediation_catalog_audit_truncates_deterministically() -> None: default_limit=16, ) assert len(payload["known_code_counts"]["rule_id_counts"]) == 1 + plan = payload["remediation_plan"] + assert plan["plan_item_count"] == 3 + assert plan["included_plan_item_count"] == 2 + assert plan["omitted_plan_item_count"] == 1 + assert plan["truncated"] is True + _assert_count_limit_metadata( + payload["payload_limits"]["remediation_plan"], + limit=2, + total_count=3, + included_count=2, + omitted_count=1, + truncated=True, + requested_limit=2, + default_limit=16, + ) def test_remediation_catalog_audit_ranks_report_observed_gaps( @@ -314,6 +496,19 @@ def test_remediation_catalog_audit_ranks_report_only_gaps( } ] assert "report_occurrences=2" in ranked[0]["rank_reasons"] + plan_item = payload["remediation_plan"]["items"][0] + assert plan_item["catalog_gap_rank"] == 1 + assert plan_item["suggested_selector"]["shape"] == ( + "exact_rule_and_finding" + ) + assert plan_item["suggested_selector"]["basis"] == ( + "reported_rule_and_finding" + ) + assert plan_item["evidence"]["known_source_occurrence_count"] == 0 + assert plan_item["evidence"]["report_occurrence_count"] == 2 + assert plan_item["evidence"]["reports"] == [ + {"count": 1, "source": "reports/quality.json"} + ] def test_discover_known_quality_findings_resolves_source_attribution( @@ -377,14 +572,186 @@ def source_error(): "ticks.ascii.spread" ) assert findings["ASCII_TICK_BID_ASK_INVALID"].rule_id == ( - "ticks.unresolved" + "ticks.ascii.spread" + ) + assert ( + findings["ASCII_TICK_BID_ASK_INVALID"].attribution_reason + == "finding_code_prefix" ) assert findings["ASCII_TICK_CACHE_SCHEMA_UNSUPPORTED"].rule_id == ( - "ticks.unresolved" + "ticks.ascii.spread" + ) + assert ( + findings["ASCII_TICK_CACHE_SCHEMA_UNSUPPORTED"].attribution_reason + == "unique_module_rule" ) assert {item.source_family for item in findings.values()} == {"ticks"} +def test_discovery_infers_constructor_assignments_and_helper_callers( + tmp_path: Path, +) -> None: + """Local rule objects and single-rule helper chains should be inferable.""" + source = tmp_path / "custom.py" + source.write_text( + """ +CUSTOM_RULE_ID = "custom.rule" + + +class CustomRule: + rule_id: str = CUSTOM_RULE_ID + + def evaluate(self, target): + return source_error(target) + + +def source_error(target): + return _finding(target, code="CUSTOM_SOURCE_UNREADABLE") + + +def local_rule(target): + rule = CustomRule() + return _finding( + target, + code="CUSTOM_LOCAL_RULE", + rule_id=rule.rule_id, + ) + + +def typed_rule(target, rule: CustomRule): + return _finding( + target, + code="CUSTOM_TYPED_RULE", + rule_id=rule.rule_id, + ) + + +def default_rule(target, rule_id: str = CUSTOM_RULE_ID): + return _finding( + target, + code="CUSTOM_DEFAULT_RULE", + rule_id=rule_id, + ) +""", + encoding="utf-8", + ) + + findings = { + item.finding_code: item + for item in discover_known_quality_findings(tmp_path) + } + + assert findings["CUSTOM_SOURCE_UNREADABLE"].rule_id == "custom.rule" + assert ( + findings["CUSTOM_SOURCE_UNREADABLE"].attribution_reason + == "unique_helper_rule" + ) + assert findings["CUSTOM_LOCAL_RULE"].rule_id == "custom.rule" + assert ( + findings["CUSTOM_LOCAL_RULE"].attribution_reason == "local_rule_object" + ) + assert findings["CUSTOM_TYPED_RULE"].rule_id == "custom.rule" + assert ( + findings["CUSTOM_TYPED_RULE"].attribution_reason == "local_rule_object" + ) + assert findings["CUSTOM_DEFAULT_RULE"].rule_id == "custom.rule" + assert ( + findings["CUSTOM_DEFAULT_RULE"].attribution_reason + == "unique_helper_rule" + ) + + +def test_discovery_preserves_ambiguous_family_fallback_with_reason( + tmp_path: Path, +) -> None: + """Helpers shared by multiple rules should remain explicitly unresolved.""" + source = tmp_path / "ticks.py" + source.write_text( + """ +FIRST_RULE_ID = "ticks.first" +SECOND_RULE_ID = "ticks.second" + + +class FirstRule: + rule_id: str = FIRST_RULE_ID + + def evaluate(self, target): + return shared_error(target) + + +class SecondRule: + rule_id: str = SECOND_RULE_ID + + def evaluate(self, target): + return shared_error(target) + + +def shared_error(target): + return _finding(target, code="CUSTOM_SHARED_FAILURE") +""", + encoding="utf-8", + ) + + finding = discover_known_quality_findings(tmp_path)[0] + payload = audit_remediation_catalog(known_findings=(finding,)) + gap = payload["ranked_gaps"][0] + + assert finding.rule_id == "ticks.unresolved" + assert finding.attribution_status == "unresolved" + assert finding.attribution_reason == "ambiguous_helper_rules" + assert gap["attribution_status"] == "unresolved" + assert gap["attribution_reason"] == "ambiguous_helper_rules" + assert gap["source_helper_counts"] == [ + {"count": 1, "source_helper": "shared_error"}, + ] + assert payload["summary"]["unresolved_attribution_occurrence_count"] == 1 + assert payload["known_code_counts"][ + "unresolved_finding_code_prefix_counts" + ] == [{"count": 1, "finding_code_prefix": "CUSTOM_SHARED_FAILURE"}] + for key in ( + "attribution_reason_counts", + "unresolved_source_helper_counts", + "unresolved_finding_code_prefix_counts", + ): + _assert_count_limit_metadata( + payload["payload_limits"][key], + limit=16, + total_count=1, + included_count=1, + omitted_count=0, + truncated=False, + requested_limit=16, + default_limit=16, + ) + + +def test_current_top_gaps_have_specific_rule_attribution() -> None: + """Current high-priority families should not regress to broad rule IDs.""" + findings = discover_known_quality_findings() + by_code = {item.finding_code: item for item in findings} + + assert by_code["ASCII_TICK_SPREAD_CACHE_SCHEMA_UNSUPPORTED"].rule_id == ( + "ticks.ascii.spread" + ) + assert by_code["DOMAIN_CALENDAR_SOURCE_UNREADABLE"].rule_id == ( + "domain.calendar_sessions" + ) + assert by_code["PROVENANCE_CACHE_METADATA_MISMATCH"].rule_id == ( + "provenance.manifest.lineage" + ) + unresolved = [ + item for item in findings if item.attribution_status == "unresolved" + ] + assert {item.source_family for item in unresolved} <= { + "ingestion", + "time", + } + assert all( + item.attribution_reason == "ambiguous_helper_rules" + for item in unresolved + ) + + def test_remediation_catalog_audit_uses_report_coverage_and_sanitizes_paths( tmp_path: Path, ) -> None: @@ -451,10 +818,11 @@ def test_remediation_catalog_audit_json_matches_golden_fixture() -> None: fixture = Path( "tests/fixtures/data_quality_reports/remediation_catalog_audit.json" ) + encoded = remediation_catalog_audit_to_json(payload) + if os.environ.get("HISTDATACOM_UPDATE_QUALITY_GOLDENS") == "1": + fixture.write_text(encoded, encoding="utf-8") - assert remediation_catalog_audit_to_json(payload) == fixture.read_text( - encoding="utf-8" - ) + assert encoded == fixture.read_text(encoding="utf-8") def _known( diff --git a/tests/unit/test_data_quality_repair_plan.py b/tests/unit/test_data_quality_repair_plan.py new file mode 100644 index 00000000..2472921f --- /dev/null +++ b/tests/unit/test_data_quality_repair_plan.py @@ -0,0 +1,403 @@ +"""Tests for non-mutating quality repair plans.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from histdatacom.data_quality import ( + QUALITY_REPAIR_PLAN_SCHEMA_VERSION, + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, + format_quality_repair_plan, + quality_repair_plan, + quality_repair_plan_to_json, +) +from histdatacom.runtime_contracts import JSONValue + +_ZIP_OPERATION_CASES: tuple[tuple[str, str], ...] = ( + ("ZIP_UNREADABLE", "restore_read_access"), + ("ZIP_CORRUPT", "redownload_archive"), + ("ZIP_CRC_ERROR", "redownload_archive"), + ("HISTDATA_ZIP_FILENAME_INVALID", "rename_archive"), + ("ZIP_MEMBER_MISSING", "restore_archive_member"), + ("ZIP_MEMBER_UNEXPECTED", "rebuild_archive_members"), + ("HISTDATA_ZIP_MEMBER_FILENAME_INVALID", "rename_archive_member"), + ("ZIP_EXTRA_MEMBER", "inspect_archive_members"), +) + +_FINDING_METADATA: dict[str, dict[str, JSONValue]] = { + "ZIP_UNREADABLE": {"error_type": "PermissionError"}, + "ZIP_CORRUPT": {"error_type": "BadZipFile"}, + "ZIP_CRC_ERROR": {"bad_member": "DAT_ASCII_EURUSD_T_201202.csv"}, + "HISTDATA_ZIP_FILENAME_INVALID": { + "observed_filename": "DAT_ASCII_EURUSD_T_201202_DIRTY.zip", + "expected_filename": "DAT_ASCII_EURUSD_T_201202.zip", + "accepted_filenames": [ + "DAT_ASCII_EURUSD_T_201202.zip", + "HISTDATA_COM_ASCII_EURUSD_T201202.zip", + ], + }, + "ZIP_MEMBER_MISSING": { + "expected_member": "DAT_ASCII_EURUSD_T_201202.csv", + "observed_members": [], + }, + "ZIP_MEMBER_UNEXPECTED": { + "expected_member": "DAT_ASCII_EURUSD_T_201202.csv", + "observed_members": ["DAT_ASCII_EURUSD_T_201201.csv"], + }, + "HISTDATA_ZIP_MEMBER_FILENAME_INVALID": { + "observed_member": "EURUSD.csv", + "expected_member": "DAT_ASCII_EURUSD_T_201202.csv", + }, + "ZIP_EXTRA_MEMBER": { + "expected_member": "DAT_ASCII_EURUSD_T_201202.csv", + "extra_members": ["README.txt"], + }, +} + + +def test_repair_plan_supports_initial_archive_operations() -> None: + """Every #350 archive hint should produce a concrete manual operation.""" + for finding_code, category in _ZIP_OPERATION_CASES: + payload = quality_repair_plan(_report(_finding(finding_code))) + + assert payload["schema_version"] == QUALITY_REPAIR_PLAN_SCHEMA_VERSION + assert payload["mode"] == "non_mutating" + assert payload["apply_supported"] is False + assert payload["mutating_operations_performed"] is False + item = payload["items"][0] + assert item["finding_code"] == finding_code + assert item["rule_id"] == "inventory.zip.integrity" + assert item["remediation_hint_code"] + assert item["action_kind"] + assert item["operation"]["category"] == category + assert item["operation"]["proposal_status"] == "proposed" + assert item["operation"]["specificity"] == "exact" + assert item["operation"]["automation_status"] == "manual_only" + assert item["preconditions"] + assert item["evidence_needed"] + assert item["confidence"]["level"] == "high" + + +def test_repair_plan_keeps_missing_context_visible() -> None: + """A supported operation should not pretend incomplete evidence is exact.""" + finding = _finding( + "HISTDATA_ZIP_MEMBER_FILENAME_INVALID", + metadata={"observed_member": "EURUSD.csv"}, + ) + + item = quality_repair_plan(_report(finding))["items"][0] + + assert item["operation"]["category"] == "rename_archive_member" + assert item["operation"]["proposal_status"] == "needs_context" + assert item["operation"]["specificity"] == "contextual" + assert item["operation"]["automation_status"] == "manual_only" + assert item["missing_evidence"] == ["expected_member"] + assert item["confidence"]["level"] == "medium" + + +def test_repair_plan_marks_unmapped_and_unsupported_actions() -> None: + """Unknown findings and out-of-scope hints should be explicit boundaries.""" + payload = quality_repair_plan( + _report( + _finding("CUSTOM_UNKNOWN_FINDING", rule_id="custom.rule"), + _finding( + "ASCII_TICK_DUPLICATE_ROW", + rule_id="time.ascii.sequence", + ), + ) + ) + + items = {item["finding_code"]: item for item in payload["items"]} + unmapped = items["CUSTOM_UNKNOWN_FINDING"] + unsupported = items["ASCII_TICK_DUPLICATE_ROW"] + assert unmapped["remediation_hint_code"] == "" + assert unmapped["operation"]["proposal_status"] == "unsupported" + assert unmapped["operation"]["reason"] == "unmapped_finding" + assert unsupported["remediation_hint_code"] == ( + "inspect_duplicate_tick_rows" + ) + assert unsupported["operation"]["reason"] == "unsupported_action" + assert payload["proposal_status_counts"] == {"unsupported": 2} + + +def test_repair_plan_is_bounded_and_reports_truncation() -> None: + """Item and per-item evidence limits should expose complete count metadata.""" + findings = tuple( + _finding(code, path=f"quality-fixtures/{index}.zip") + for index, (code, _category) in enumerate(_ZIP_OPERATION_CASES) + ) + + payload = quality_repair_plan( + _report(*findings), + item_limit=3, + evidence_limit=1, + ) + + assert payload["plan_item_count"] == 8 + assert payload["included_plan_item_count"] == 3 + assert payload["omitted_plan_item_count"] == 5 + assert payload["truncated"] is True + assert len(payload["items"]) == 3 + assert payload["payload_limits"]["items"] == { + "default_limit": 16, + "effective_limit": 3, + "included_count": 3, + "limit": 3, + "maximum_limit": 64, + "minimum_limit": 0, + "omitted_count": 5, + "requested_limit": 3, + "total_count": 8, + "truncated": True, + "unbounded": False, + } + assert all( + item["evidence"]["included_count"] <= 1 for item in payload["items"] + ) + + +def test_repair_plan_is_publish_safe_and_excludes_raw_diagnostics() -> None: + """Plans should sanitize local paths and omit arbitrary finding metadata.""" + finding = _finding( + "ZIP_CORRUPT", + path="/Users/alice/private/market/DAT_ASCII_EURUSD_T_201202.zip", + metadata={ + "error_type": "BadZipFile", + "error": ( + "token=secret at /Users/alice/private/market/" + "DAT_ASCII_EURUSD_T_201202.zip" + ), + "raw_rows": ["sensitive row"], + }, + ) + + payload = quality_repair_plan( + _report(finding), + report_path="/Users/alice/private/reports/quality.json", + ) + encoded = quality_repair_plan_to_json(payload) + + assert "/Users/alice" not in encoded + assert "token=secret" not in encoded + assert "sensitive row" not in encoded + assert payload["input_report"]["path"] == "reports/quality.json" + assert payload["items"][0]["target"]["path"] == ( + "DAT_ASCII_EURUSD_T_201202.zip" + ) + + +def test_repair_plan_preserves_bounded_topology_context_without_raw_samples() -> ( + None +): + """Existing #343 context should remain useful without copying row samples.""" + finding = _finding( + "CUSTOM_TOPOLOGY_FINDING", + rule_id="fingerprint.series", + metadata={ + "time_series_fingerprint": { + "temporal_topology": { + "inspection_context": { + "schema_version": ( + "histdatacom.timestamp-topology-inspection.v1" + ), + "duplicate_timestamps": { + "total_count": 4, + "included_count": 1, + "omitted_count": 3, + "truncated": True, + "samples": [ + { + "row_number": 42, + "timestamp_source": "private raw value", + } + ], + }, + } + } + } + }, + ) + + item = quality_repair_plan(_report(finding))["items"][0] + evidence = item["evidence"] + + assert evidence["items"] == [ + { + "kind": "inspection_context.duplicate_timestamps", + "value": { + "included_count": 1, + "omitted_count": 3, + "total_count": 4, + "truncated": True, + }, + } + ] + assert "private raw value" not in json.dumps(item) + + +def test_repair_plan_order_and_json_are_deterministic() -> None: + """Input finding order should not change ranks or serialized JSON.""" + findings = tuple(_finding(code) for code, _category in _ZIP_OPERATION_CASES) + forward = quality_repair_plan(_report(*findings)) + reverse = quality_repair_plan(_report(*reversed(findings))) + + assert forward == reverse + assert quality_repair_plan_to_json(forward) == ( + quality_repair_plan_to_json(reverse) + ) + assert [item["finding_code"] for item in forward["items"]] == [ + "ZIP_UNREADABLE", + "ZIP_CORRUPT", + "ZIP_CRC_ERROR", + "HISTDATA_ZIP_FILENAME_INVALID", + "ZIP_MEMBER_MISSING", + "ZIP_MEMBER_UNEXPECTED", + "HISTDATA_ZIP_MEMBER_FILENAME_INVALID", + "ZIP_EXTRA_MEMBER", + ] + + first_duplicate = _finding( + "HISTDATA_ZIP_FILENAME_INVALID", + metadata={ + "observed_filename": "z.zip", + "expected_filename": "expected.zip", + }, + ) + second_duplicate = _finding( + "HISTDATA_ZIP_FILENAME_INVALID", + metadata={ + "observed_filename": "a.zip", + "expected_filename": "expected.zip", + }, + ) + duplicate_forward = quality_repair_plan( + _report(first_duplicate, second_duplicate) + ) + duplicate_reverse = quality_repair_plan( + _report(second_duplicate, first_duplicate) + ) + assert duplicate_forward == duplicate_reverse + + +def test_repair_plan_generation_does_not_mutate_target_or_report( + tmp_path: Path, +) -> None: + """Pure planning must leave the report object and target bytes unchanged.""" + archive = tmp_path / "DAT_ASCII_EURUSD_T_201202.zip" + archive.write_bytes(b"not a zip") + finding = _finding("ZIP_CORRUPT", path=str(archive)) + report = _report(finding) + report_before = report.to_dict() + bytes_before = archive.read_bytes() + + payload = quality_repair_plan(report) + + assert payload["safety"] == { + "advisory_only": True, + "automatic_execution": "unsupported", + "filesystem_mutation_performed": False, + "network_access_performed": False, + "requires_user_verification_before_action": True, + } + assert report.to_dict() == report_before + assert archive.read_bytes() == bytes_before + + +def test_repair_plan_human_output_is_concise() -> None: + """Human output should show actions without dumping evidence lists.""" + payload = quality_repair_plan( + _report(_finding("ZIP_CORRUPT")), + report_path="reports/quality.json", + ) + + rendered = format_quality_repair_plan(payload) + + assert "Quality repair plan" in rendered + assert "mode: non_mutating" in rendered + assert "ZIP_CORRUPT" in rendered + assert "redownload_archive" in rendered + assert "advisory only" in rendered + assert "error_type" not in rendered + + omitted = format_quality_repair_plan( + quality_repair_plan( + _report(_finding("ZIP_CORRUPT")), + item_limit=0, + ) + ) + assert "all 1 plan items omitted by limit" in omitted + + +def test_repair_plan_schema_golden() -> None: + """The representative repair-plan JSON should remain golden-testable.""" + report_path = Path( + "tests/fixtures/data_quality_reports/corrupt_zip_report.json" + ) + report_payload = json.loads(report_path.read_text(encoding="utf-8")) + report = QualityReport.from_dict(report_payload) + payload = quality_repair_plan(report, report_path=str(report_path)) + golden_path = Path( + "tests/fixtures/data_quality_reports/corrupt_zip_repair_plan.json" + ) + + assert quality_repair_plan_to_json(payload) == golden_path.read_text( + encoding="utf-8" + ) + + +def _finding( + code: str, + *, + rule_id: str = "inventory.zip.integrity", + path: str = "quality-fixtures/DAT_ASCII_EURUSD_T_201202.zip", + metadata: dict[str, JSONValue] | None = None, +) -> QualityFinding: + target = QualityTarget( + path=path, + kind=QualityTargetKind.ZIP, + data_format="ascii", + timeframe="T", + symbol="EURUSD", + period="201202", + metadata={"filename": Path(path).name}, + ) + severity = ( + QualitySeverity.WARNING + if code == "ZIP_EXTRA_MEMBER" + else QualitySeverity.ERROR + ) + return QualityFinding( + severity=severity, + code=code, + message=f"Finding {code}", + rule_id=rule_id, + target=target, + metadata=dict( + _FINDING_METADATA.get(code, {}) if metadata is None else metadata + ), + ) + + +def _report(*findings: QualityFinding) -> QualityReport: + target_items: list[QualityTarget] = [] + for finding in findings: + if finding.target not in target_items: + target_items.append(finding.target) + targets = tuple(target_items) + return QualityReport( + targets=targets, + rule_results=tuple( + QualityRuleResult( + rule_id=finding.rule_id, + target=finding.target, + findings=(finding,), + ) + for finding in findings + ), + ) diff --git a/tests/unit/test_data_quality_report_goldens.py b/tests/unit/test_data_quality_report_goldens.py index b2ee4f76..50373288 100644 --- a/tests/unit/test_data_quality_report_goldens.py +++ b/tests/unit/test_data_quality_report_goldens.py @@ -11,11 +11,18 @@ import pytest from histdatacom.data_quality import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, + CROSS_SERIES_FINGERPRINT_RULE_ID, + CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, + QUALITY_ENGINE_METADATA_KEY, + QUALITY_ENGINE_SCHEMA_VERSION, QUALITY_NEXT_ACTIONS_METADATA_KEY, QUALITY_NEXT_ACTIONS_SCHEMA_VERSION, QUALITY_REMEDIATION_COVERAGE_METADATA_KEY, QUALITY_REMEDIATION_COVERAGE_SCHEMA_VERSION, QUALITY_REPORT_SCHEMA_VERSION, + QUALITY_SKIP_EVENTS_SCHEMA_VERSION, + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV, QualityExitPolicy, QualityFinding, QualityLocation, @@ -47,6 +54,7 @@ TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_SCHEMA_VERSION, bounded_quality_payload, quality_report_payload, + run_quality_assessment, ) from histdatacom.runtime_contracts import ArtifactRef, JSONValue @@ -68,6 +76,16 @@ "_coverage_manifest_failure_report_payload", ), ("cache_target_report", "report", "_cache_target_report_payload"), + ( + "quality_engine_skip_report", + "report", + "_quality_engine_skip_report_payload", + ), + ( + "quality_engine_skip_bounded_payload", + "bounded", + "_quality_engine_skip_bounded_payload", + ), ("fingerprint_report", "report", "_fingerprint_report_payload"), ( "fingerprint_bounded_payload", @@ -358,6 +376,69 @@ def _cache_target_report_payload() -> dict[str, JSONValue]: ) +class _GoldenQualityEngineSkipRule: + rule_id = "time.ascii.gaps" + description = "semantic scans prefer extracted CSVs" + + def evaluate(self, target: QualityTarget) -> tuple[QualityFinding, ...]: + del target + return () + + +def _quality_engine_skip_report() -> QualityReport: + archive = _target( + path="/quality-fixtures/DAT_ASCII_EURUSD_T_201202.zip", + kind=QualityTargetKind.ZIP, + ) + csv = _target( + path="/quality-fixtures/DAT_ASCII_EURUSD_T_201202.csv", + kind=QualityTargetKind.CSV, + ) + return run_quality_assessment( + targets=(archive, csv), + rules=(_GoldenQualityEngineSkipRule(),), + metadata={ + "operation": "data-quality", + "check_groups": ["time"], + }, + ) + + +def _quality_engine_skip_report_payload() -> dict[str, JSONValue]: + return quality_report_payload(_quality_engine_skip_report()) + + +def _quality_engine_skip_bounded_payload() -> dict[str, JSONValue]: + report = _quality_engine_skip_report() + artifact = ArtifactRef( + kind="quality-report", + path=("/quality-fixtures/reports/quality-engine-skip-report.json"), + size_bytes=2048, + sha256="2" * 64, + metadata={ + "schema_version": QUALITY_REPORT_SCHEMA_VERSION, + "status": report.status.value, + "max_severity": report.max_severity.value, + "target_count": report.summary().target_count, + "finding_count": report.summary().finding_count, + "warning_count": report.summary().warning_count, + "error_count": report.summary().error_count, + }, + ) + return bounded_quality_payload( + operation="data-quality", + check_groups=("time",), + discovery={ + "roots": ["/quality-fixtures"], + "target_count": 2, + "metadata": {"supported_kinds": ["zip", "csv", "cache"]}, + }, + report=report, + decision=QualityExitPolicy.from_values().evaluate(report.summary()), + artifact=artifact, + ) + + def _fingerprint_report_payload() -> dict[str, JSONValue]: return quality_report_payload(_fingerprint_report()) @@ -438,11 +519,11 @@ def _fingerprint_report() -> QualityReport: "parsed_row_count": 3, "invalid_timestamp_count": 0, "non_monotonic_count": 0, - "duplicate_timestamp_count": 0, + "duplicate_timestamp_count": 1, "duplicate_timestamp_source_counts": { - "tick_duplicate_row": 0, + "tick_duplicate_row": 1, }, - "tick_duplicate_row_count": 0, + "tick_duplicate_row_count": 1, "min_interval_ms": 60000, "median_interval_ms": 60000, "interval_count": 2, @@ -477,6 +558,44 @@ def _fingerprint_report() -> QualityReport: "dynamic_window_growth_factor": 2.0, "dynamic_window_shrink_factor": 0.5, }, + "inspection_context": { + "schema_version": ( + "histdatacom.timestamp-topology-inspection.v1" + ), + "duplicate_timestamps": { + "total_count": 1, + "included_count": 1, + "omitted_count": 0, + "truncated": False, + "limit_metadata": { + "samples": { + "limit": 5, + "effective_limit": 5, + "requested_limit": 5, + "default_limit": 5, + "minimum_limit": 0, + "maximum_limit": 5, + "unbounded": False, + "total_count": 1, + "included_count": 1, + "omitted_count": 0, + "truncated": False, + } + }, + "duplicate_row_count": 1, + "samples": [ + { + "row_number": 2, + "timestamp_source": "20120201 000000000", + "timestamp_source_truncated": False, + "timestamp_utc_ms": 1328072400000, + "utc_timestamp": "2012-02-01T05:00:00Z", + "occurrence_count": 2, + "exact_row_group_count": 1, + } + ], + }, + }, }, "fingerprint_audit": { "schema_version": TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION, @@ -570,10 +689,188 @@ def _fingerprint_report() -> QualityReport: metadata={ "operation": "data-quality", "check_groups": ["fingerprint"], + CROSS_SERIES_FINGERPRINT_METADATA_KEY: ( + _cross_series_golden_payload() + ), }, ) +def _cross_series_golden_payload() -> dict[str, JSONValue]: + limit = { + "limit": 32, + "effective_limit": 32, + "requested_limit": 32, + "default_limit": 32, + "minimum_limit": 0, + "maximum_limit": None, + "unbounded": False, + } + return { + "schema_version": CROSS_SERIES_FINGERPRINT_SCHEMA_VERSION, + "rule_id": CROSS_SERIES_FINGERPRINT_RULE_ID, + "status": "limited", + "calculation_basis": "shared_cross_instrument_scan", + "data_format": "ascii", + "timeframe": "T", + "target_count": 3, + "ascii_tick_target_count": 3, + "fx_series_count": 3, + "group_count": 1, + "incomplete_group_count": 0, + "included_group_count": 1, + "omitted_group_count": 0, + "truncated": False, + "limit_metadata": { + "groups": limit, + "correlations_per_group": limit, + }, + "source_basis_counts": {"text_scan": 3}, + "cache_source_counts": {}, + "row_identity": { + "columns": [ + "series_id", + "period", + "row_id", + "source_row_number", + "event_seq", + ], + "timestamp_is_durable_identity": False, + "duplicate_timestamp_row_count": 2, + "legacy_cache_enrichment_required": True, + "training_schema_version": "histdatacom.tick-training-row.v1", + }, + "topology_basis": { + "schema_version": TIME_SERIES_FINGERPRINT_TOPOLOGY_SUMMARY_SCHEMA_VERSION, + "target_count": 3, + "included_target_count": 3, + "truncated": False, + }, + "return_correlation_status_counts": { + "unavailable": 1, + "valid": 2, + }, + "triangular_consistency": { + "candidate_count": 1, + "compared_timestamp_count": 3, + "warning_count": 0, + "error_count": 0, + "warning_samples": [], + "error_samples": [], + }, + "inverse_consistency": { + "candidate_count": 0, + "compared_timestamp_count": 0, + "warning_count": 0, + "error_count": 0, + "warning_samples": [], + "error_samples": [], + }, + "stale_join_risk": {"risk_count": 0, "samples": []}, + "unavailable": {"count": 1, "samples": []}, + "panel_coverage": [ + { + "timeframe": "T", + "symbols": ["EURGBP", "EURUSD", "GBPUSD"], + "union_period_count": 1, + "common_period_count": 1, + "common_first_period": "201202", + "common_last_period": "201202", + "unequal_period_ranges": False, + "limiting_start_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD", + ], + "limiting_end_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD", + ], + "first_period_by_symbol": { + "EURGBP": "201202", + "EURUSD": "201202", + "GBPUSD": "201202", + }, + "last_period_by_symbol": { + "EURGBP": "201202", + "EURUSD": "201202", + "GBPUSD": "201202", + }, + "missing_period_count_by_symbol": { + "EURGBP": 0, + "EURUSD": 0, + "GBPUSD": 0, + }, + } + ], + "tolerance": { + "triangular_warning_relative_tolerance": 0.005, + "triangular_error_relative_tolerance": 0.02, + "inverse_warning_relative_tolerance": 0.005, + "inverse_error_relative_tolerance": 0.02, + "minimum_common_timestamp_ratio": 0.8, + "stale_forward_fill_min_run": 2, + }, + "groups": [ + { + "group_id": "ascii:T:201202", + "target_axis": { + "data_format": "ascii", + "timeframe": "T", + "period": "201202", + }, + "symbols": ["EURGBP", "EURUSD", "GBPUSD"], + "expected_symbols": ["EURGBP", "EURUSD", "GBPUSD"], + "missing_symbols": [], + "complete": True, + "series_count": 3, + "series": [], + "timestamp_grid": { + "union_timestamp_count": 4, + "common_timestamp_count": 3, + "common_timestamp_ratio": 0.75, + "missing_by_symbol": { + "EURGBP": 1, + "EURUSD": 0, + "GBPUSD": 0, + }, + }, + "coverage_ranges": { + "common_start_timestamp_utc_ms": 1328072401000, + "common_end_timestamp_utc_ms": 1328072403000, + "unequal_ranges": True, + "limiting_start_symbols": ["EURGBP"], + "limiting_end_symbols": [ + "EURGBP", + "EURUSD", + "GBPUSD", + ], + }, + "topology": { + "target_count": 3, + "source_series_count": 3, + "duplicate_timestamp_row_count": 2, + "computed_from_counts": {"text_scan": 3}, + "cache_source_counts": {}, + "topology_computed_from_counts": {"text_scan": 3}, + "topology_cache_source_counts": {}, + "mixed_computation_basis": False, + "mixed_cache_source": False, + }, + "return_correlation": { + "pair_count": 3, + "included_pair_count": 3, + "omitted_pair_count": 0, + "truncated": False, + "limit_metadata": {"pairs": limit}, + "pairs": [], + }, + } + ], + } + + def _run_scoped_report_payload() -> dict[str, JSONValue]: return quality_report_payload(_run_scoped_report()) @@ -778,6 +1075,8 @@ def _assert_report_contract(payload: dict[str, JSONValue]) -> None: _assert_quality_remediation_coverage( _mapping(metadata[QUALITY_REMEDIATION_COVERAGE_METADATA_KEY]) ) + if QUALITY_ENGINE_METADATA_KEY in metadata: + _assert_quality_engine(_mapping(metadata[QUALITY_ENGINE_METADATA_KEY])) summary = _mapping(payload["summary"]) _assert_summary(summary) @@ -813,6 +1112,7 @@ def _assert_bounded_payload_contract(payload: dict[str, JSONValue]) -> None: } optional_keys = { "fingerprint_coverage", + "fingerprint_cross_series", "fingerprint_distribution", "fingerprint_distribution_attention", "fingerprint_readiness", @@ -821,6 +1121,7 @@ def _assert_bounded_payload_contract(payload: dict[str, JSONValue]) -> None: "fingerprint_topology", "fingerprint_topology_attention", "next_actions", + "quality_engine", "remediation_coverage", } assert expected_keys <= set(payload) @@ -865,6 +1166,8 @@ def _assert_bounded_payload_contract(payload: dict[str, JSONValue]) -> None: _assert_quality_remediation_coverage( _mapping(payload["remediation_coverage"]) ) + if QUALITY_ENGINE_METADATA_KEY in payload: + _assert_quality_engine(_mapping(payload[QUALITY_ENGINE_METADATA_KEY])) for target_summary in _list(payload["target_summaries"]): _assert_target_summary(_mapping(target_summary)) @@ -885,6 +1188,7 @@ def _assert_bounded_payload_contract(payload: dict[str, JSONValue]) -> None: assert artifact["kind"] == "quality-report" assert artifact["path"] in { "quality-fixtures/reports/fingerprint-report.json", + "quality-fixtures/reports/quality-engine-skip-report.json", "quality-fixtures/reports/run-scoped-report.json", } assert len(str(artifact["sha256"])) == 64 @@ -898,6 +1202,104 @@ def _assert_bounded_payload_contract(payload: dict[str, JSONValue]) -> None: assert set(policy) == {"fail_on", "max_errors", "max_warnings"} +def _assert_quality_engine(payload: dict[str, JSONValue]) -> None: + assert set(payload) == { + "duplicate_archive_scan_policy", + "planned_target_rule_evaluation_count", + "rule_count", + "run_rule_count", + "schema_version", + "skip_events", + "skipped_duplicate_archive_rule_evaluation_count", + "skipped_rule_evaluation_count", + "target_count", + "target_rule_evaluation_count", + } + assert payload["schema_version"] == QUALITY_ENGINE_SCHEMA_VERSION + for key in ( + "planned_target_rule_evaluation_count", + "rule_count", + "run_rule_count", + "skipped_duplicate_archive_rule_evaluation_count", + "skipped_rule_evaluation_count", + "target_count", + "target_rule_evaluation_count", + ): + assert isinstance(payload[key], int) + assert payload[key] >= 0 + assert payload["planned_target_rule_evaluation_count"] == ( + payload["target_rule_evaluation_count"] + + payload["skipped_rule_evaluation_count"] + ) + assert payload["duplicate_archive_scan_policy"] == ( + "prefer_extracted_csv_for_non_inventory_rules" + ) + + skips = _mapping(payload["skip_events"]) + assert set(skips) == { + "event_count", + "events", + "included_event_count", + "limit_metadata", + "omitted_event_count", + "reason_counts", + "rule_id_counts", + "schema_version", + "target_kind_counts", + "truncated", + } + assert skips["schema_version"] == QUALITY_SKIP_EVENTS_SCHEMA_VERSION + for key in ("event_count", "included_event_count", "omitted_event_count"): + assert isinstance(skips[key], int) + assert skips[key] >= 0 + assert skips["event_count"] == payload["skipped_rule_evaluation_count"] + assert skips["included_event_count"] + skips["omitted_event_count"] == ( + skips["event_count"] + ) + assert skips["truncated"] is (skips["omitted_event_count"] > 0) + _assert_limit_metadata_map( + skips["limit_metadata"], + keys=("events", "reasons", "rules", "target_kinds"), + ) + for count_key in ("reason_counts", "rule_id_counts", "target_kind_counts"): + counts = _mapping(skips[count_key]) + for name, count in counts.items(): + assert name + assert isinstance(count, int) + assert count > 0 + assert ( + _mapping(skips["reason_counts"])[ + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV + ] + == payload["skipped_duplicate_archive_rule_evaluation_count"] + ) + + events = _list(skips["events"]) + assert len(events) == skips["included_event_count"] + for event_value in events: + event = _mapping(event_value) + assert set(event) == { + "reason_code", + "rule_id", + "target_axis", + "target_kind", + } + assert isinstance(event["reason_code"], str) + assert event["reason_code"] + assert isinstance(event["rule_id"], str) + assert event["rule_id"] + assert event["target_kind"] in TARGET_KIND_VALUES + axis = _mapping(event["target_axis"]) + assert set(axis) == { + "data_format", + "kind", + "period", + "symbol", + "timeframe", + } + assert axis["kind"] == event["target_kind"] + + def _assert_fingerprint_coverage(payload: dict[str, JSONValue]) -> None: assert set(payload) == { "cache_source_counts", @@ -1367,7 +1769,7 @@ def _assert_fingerprint_topology(payload: dict[str, JSONValue]) -> None: def _assert_fingerprint_topology_target( payload: dict[str, JSONValue], ) -> None: - assert set(payload) == { + expected_keys = { "cache_source", "computed_from", "duplicate_timestamp_count", @@ -1385,6 +1787,11 @@ def _assert_fingerprint_topology_target( "target_axis", "weekend_activity_count", } + assert expected_keys <= set(payload) + assert set(payload) <= expected_keys | { + "calendar_policy", + "inspection_context", + } axis = _mapping(payload["target_axis"]) assert set(axis) == { "data_format", @@ -1405,6 +1812,10 @@ def _assert_fingerprint_topology_target( "weekend_activity_count", ): assert isinstance(payload[key], int) + if "inspection_context" in payload: + _assert_fingerprint_topology_inspection_context( + _mapping(payload["inspection_context"]) + ) def _assert_fingerprint_topology_attention( @@ -1446,7 +1857,7 @@ def _assert_fingerprint_topology_attention( def _assert_fingerprint_topology_attention_target( payload: dict[str, JSONValue], ) -> None: - assert set(payload) == { + expected_keys = { "attention_flags", "attention_level", "cache_source", @@ -1463,6 +1874,8 @@ def _assert_fingerprint_topology_attention_target( "target_axis", "weekend_activity_count", } + assert expected_keys <= set(payload) + assert set(payload) <= expected_keys | {"inspection_context"} axis = _mapping(payload["target_axis"]) assert set(axis) == { "data_format", @@ -1476,11 +1889,17 @@ def _assert_fingerprint_topology_attention_target( "structural", "sequence", "session", + "contextual", } assert isinstance(payload["attention_flags"], list) assert isinstance(payload["flags"], list) for hint in _list(payload["remediation_hints"]): _assert_fingerprint_remediation_hint(_mapping(hint)) + if "inspection_context" in payload: + _assert_fingerprint_topology_inspection_context( + _mapping(payload["inspection_context"]), + action_linked=True, + ) assert payload["status"] in {"regular", "irregular", "unavailable"} for key in ( "duplicate_timestamp_count", @@ -1497,6 +1916,56 @@ def _assert_fingerprint_topology_attention_target( ) +def _assert_fingerprint_topology_inspection_context( + payload: dict[str, JSONValue], + *, + action_linked: bool = False, +) -> None: + assert payload["schema_version"] == ( + "histdatacom.timestamp-topology-inspection.v1" + ) + section_names = set(payload) - {"schema_version"} + assert section_names + assert section_names <= { + "invalid_timestamps", + "non_monotonic_timestamps", + "duplicate_timestamps", + "suspicious_gaps", + "expected_session_closures", + "weekend_activity", + } + for section_name in section_names: + section = _mapping(payload[section_name]) + for key in ( + "total_count", + "included_count", + "omitted_count", + ): + assert isinstance(section[key], int) + assert isinstance(section["truncated"], bool) + assert isinstance(section["samples"], list) + _assert_limit_metadata_map( + section["limit_metadata"], + keys=("samples",), + ) + if action_linked and ( + section_name != "expected_session_closures" + or section.get("actionable") is True + ): + assert isinstance(section["actionable"], bool) + assert set(_mapping(section["target_axis"])) == { + "data_format", + "kind", + "period", + "symbol", + "timeframe", + } + action_key = ( + "next_action" if section["actionable"] else "policy_note" + ) + _assert_fingerprint_remediation_hint(_mapping(section[action_key])) + + def _assert_fingerprint_readiness(payload: dict[str, JSONValue]) -> None: assert set(payload) == { "applicable_dynamics_status_counts", @@ -1509,6 +1978,16 @@ def _assert_fingerprint_readiness(payload: dict[str, JSONValue]) -> None: "dependence_skipped_lag_count", "dependence_skipped_lag_reason_counts", "dependence_status_counts", + "decomposition_basis_counts", + "decomposition_computed_window_count", + "decomposition_limitation_counts", + "decomposition_reason_counts", + "decomposition_skipped_window_count", + "decomposition_skipped_window_reason_counts", + "decomposition_stationarity_status_counts", + "decomposition_status_counts", + "decomposition_structural_break_candidate_count", + "decomposition_structural_break_status_counts", "dynamics_limitation_counts", "dynamics_reason_counts", "dynamics_status_counts", @@ -1548,6 +2027,9 @@ def _assert_fingerprint_readiness(payload: dict[str, JSONValue]) -> None: "dependence_skipped_lag_count", "stationarity_computed_window_count", "stationarity_skipped_window_count", + "decomposition_computed_window_count", + "decomposition_skipped_window_count", + "decomposition_structural_break_candidate_count", ): assert isinstance(payload[key], int) assert payload[key] >= 0 @@ -1561,6 +2043,13 @@ def _assert_fingerprint_readiness(payload: dict[str, JSONValue]) -> None: "dependence_reason_counts", "dependence_skipped_lag_reason_counts", "dependence_status_counts", + "decomposition_basis_counts", + "decomposition_limitation_counts", + "decomposition_reason_counts", + "decomposition_skipped_window_reason_counts", + "decomposition_stationarity_status_counts", + "decomposition_status_counts", + "decomposition_structural_break_status_counts", "dynamics_limitation_counts", "dynamics_reason_counts", "dynamics_status_counts", @@ -1732,6 +2221,7 @@ def _assert_fingerprint_readiness_target( "applicable_dynamics_section", "applicable_dynamics_status", "dependence", + "decomposition", "microstructure_dynamics", "profile_completeness", "section_skip_reasons", @@ -1788,6 +2278,9 @@ def _assert_fingerprint_readiness_target( _assert_fingerprint_readiness_stationarity( _mapping(payload["stationarity_diagnostics"]) ) + _assert_fingerprint_readiness_decomposition( + _mapping(payload["decomposition"]) + ) def _assert_fingerprint_readiness_topology( @@ -2043,6 +2536,63 @@ def _assert_fingerprint_readiness_stationarity_window( assert isinstance(payload["required_sample_count"], int) +def _assert_fingerprint_readiness_decomposition( + payload: dict[str, JSONValue], +) -> None: + assert set(payload) == { + "basis", + "cache_source", + "calculation_basis", + "computed_from", + "computed_window_count", + "invalid_row_count", + "level_sample_count", + "limitations", + "metric", + "partial_row_count", + "reason", + "regular_grid", + "return_sample_count", + "rounding_digits", + "row_count", + "row_order", + "sampled_row_count", + "skipped_window_count", + "skipped_window_reason_counts", + "stationarity", + "status", + "structural_break", + "training_projection", + "trend", + "truncated", + "usable_row_count", + "windows", + } + assert payload["status"] in {"limited", "skipped", "unavailable", "valid"} + assert isinstance(payload["limitations"], list) + assert isinstance(payload["regular_grid"], bool) + assert isinstance(payload["truncated"], bool) + assert isinstance(payload["windows"], list) + assert isinstance(payload["skipped_window_reason_counts"], dict) + assert isinstance(payload["stationarity"], dict) + assert isinstance(payload["structural_break"], dict) + assert isinstance(payload["training_projection"], dict) + assert isinstance(payload["trend"], dict) + for key in ( + "computed_window_count", + "invalid_row_count", + "level_sample_count", + "partial_row_count", + "return_sample_count", + "rounding_digits", + "row_count", + "sampled_row_count", + "skipped_window_count", + "usable_row_count", + ): + assert isinstance(payload[key], int) + + def _assert_fingerprint_readiness_stationarity_change( payload: dict[str, JSONValue], ) -> None: @@ -2086,7 +2636,15 @@ def _assert_compact_numeric_summary(payload: dict[str, JSONValue]) -> None: def _assert_fingerprint_remediation_hint( payload: dict[str, JSONValue], ) -> None: - assert set(payload) == {"action_kind", "code", "flag", "message", "rule_id"} + assert {"action_kind", "code", "flag", "message", "rule_id"} <= set(payload) + assert set(payload) <= { + "action_kind", + "code", + "flag", + "message", + "policy_context", + "rule_id", + } assert isinstance(payload["action_kind"], str) assert payload["action_kind"] in { "configure", @@ -2094,6 +2652,7 @@ def _assert_fingerprint_remediation_hint( "rebuild", "repair", "verify", + "context", } assert isinstance(payload["code"], str) assert payload["code"] @@ -2103,6 +2662,12 @@ def _assert_fingerprint_remediation_hint( assert payload["message"] assert isinstance(payload["rule_id"], str) assert payload["rule_id"] + if "policy_context" in payload: + context = _mapping(payload["policy_context"]) + assert context["schema_version"] == ( + "histdatacom.calendar-policy-remediation-context.v1" + ) + assert isinstance(context["actionable"], bool) def _assert_quality_next_actions(payload: dict[str, JSONValue]) -> None: @@ -2131,7 +2696,7 @@ def _assert_quality_next_actions(payload: dict[str, JSONValue]) -> None: def _assert_quality_next_action(payload: dict[str, JSONValue]) -> None: - assert set(payload) == { + expected_keys = { "action_kind", "affected_target_count", "attention_level_counts", @@ -2153,6 +2718,8 @@ def _assert_quality_next_action(payload: dict[str, JSONValue]) -> None: "target_axis_truncated", "urgency", } + assert expected_keys <= set(payload) + assert set(payload) <= expected_keys | {"policy_context"} assert payload["urgency"] in {"high", "medium", "low"} _assert_limit_metadata_map(payload["limit_metadata"], keys=("target_axes",)) assert payload["action_kind"] in { @@ -2162,6 +2729,8 @@ def _assert_quality_next_action(payload: dict[str, JSONValue]) -> None: "repair", "verify", } + if "policy_context" in payload: + assert isinstance(payload["policy_context"], dict) assert isinstance(payload["code"], str) assert payload["code"] assert isinstance(payload["message"], str) @@ -2213,22 +2782,30 @@ def _assert_quality_remediation_coverage( payload: dict[str, JSONValue], ) -> None: assert set(payload) == { + "actionability_counts", + "blocked_by_attribution_warning_error_finding_count", + "blocked_by_missing_diagnostics_warning_error_finding_count", "count_limits", "finding_code_counts", "finding_count", "included_unmapped_group_count", + "included_unmapped_actionable_warning_error_group_count", "included_unmapped_warning_error_group_count", "limit_metadata", "mapped_finding_code_counts", "mapped_finding_count", "mapped_rule_id_counts", "mapped_severity_counts", + "intentionally_unremediable_warning_error_finding_count", "omitted_unmapped_group_count", + "omitted_unmapped_actionable_warning_error_group_count", "omitted_unmapped_warning_error_group_count", "rule_id_counts", "schema_version", "severity_counts", "unmapped_finding_code_counts", + "unmapped_actionable_warning_error_finding_count", + "unmapped_actionable_warning_error_group_count", "unmapped_finding_count", "unmapped_group_count", "unmapped_groups", @@ -2246,13 +2823,20 @@ def _assert_quality_remediation_coverage( keys=("groups", "target_axes"), ) for key in ( + "blocked_by_attribution_warning_error_finding_count", + "blocked_by_missing_diagnostics_warning_error_finding_count", "finding_count", + "included_unmapped_actionable_warning_error_group_count", "included_unmapped_group_count", "included_unmapped_warning_error_group_count", "mapped_finding_count", + "intentionally_unremediable_warning_error_finding_count", + "omitted_unmapped_actionable_warning_error_group_count", "omitted_unmapped_group_count", "omitted_unmapped_warning_error_group_count", "unmapped_finding_count", + "unmapped_actionable_warning_error_finding_count", + "unmapped_actionable_warning_error_group_count", "unmapped_group_count", "unmapped_warning_error_finding_count", "unmapped_warning_error_group_count", @@ -2261,6 +2845,7 @@ def _assert_quality_remediation_coverage( assert payload[key] >= 0 assert isinstance(payload["unmapped_truncated"], bool) for key in ( + "actionability_counts", "mapped_severity_counts", "severity_counts", "unmapped_severity_counts", @@ -2298,6 +2883,8 @@ def _assert_quality_remediation_coverage_group( payload: dict[str, JSONValue], ) -> None: assert set(payload) == { + "actionability", + "actionability_reason", "finding_code", "included_target_axis_count", "limit_metadata", @@ -2312,6 +2899,18 @@ def _assert_quality_remediation_coverage_group( "target_axis_truncated", } assert payload["mapped"] is False + assert payload["actionability"] in { + "remediable_defect", + "policy_or_profile_decision", + "unsupported_format_or_capability", + "expected_artifact_or_context", + "needs_rule_attribution", + "needs_diagnostic_context", + "unsafe_to_automate", + "informational_only", + } + assert isinstance(payload["actionability_reason"], str) + assert payload["actionability_reason"] _assert_limit_metadata_map(payload["limit_metadata"], keys=("target_axes",)) assert payload["max_severity"] in SEVERITY_VALUES assert isinstance(payload["finding_code"], str) diff --git a/tests/unit/test_data_quality_reporting.py b/tests/unit/test_data_quality_reporting.py index e533e30b..4a555699 100644 --- a/tests/unit/test_data_quality_reporting.py +++ b/tests/unit/test_data_quality_reporting.py @@ -6,6 +6,9 @@ from pathlib import Path from histdatacom.data_quality import ( + CROSS_SERIES_FINGERPRINT_METADATA_KEY, + QUALITY_ENGINE_METADATA_KEY, + QUALITY_ENGINE_SCHEMA_VERSION, QUALITY_NEXT_ACTIONS_METADATA_KEY, QUALITY_NEXT_ACTIONS_SCHEMA_VERSION, QUALITY_REMEDIATION_CATALOG_AUDIT_METADATA_KEY, @@ -13,10 +16,17 @@ QUALITY_REMEDIATION_COVERAGE_SCHEMA_VERSION, QUALITY_REPORTING_METADATA_KEY, QUALITY_REPORT_SCHEMA_VERSION, + QUALITY_SKIP_EVENTS_SCHEMA_VERSION, + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV, SERIES_FINGERPRINT_RULE_ID, + SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY, + SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY, + SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_COVERAGE_METADATA_KEY, TIME_SERIES_FINGERPRINT_DISTRIBUTION_ATTENTION_METADATA_KEY, TIME_SERIES_FINGERPRINT_DISTRIBUTION_SUMMARY_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY, + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_METADATA_KEY, TIME_SERIES_FINGERPRINT_READINESS_SUMMARY_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_READINESS_RISK_METADATA_KEY, @@ -34,6 +44,9 @@ QualityTarget, QualityTargetKind, bounded_quality_payload, + format_cross_series_fingerprint_lines, + format_fingerprint_parity_summary_lines, + format_fingerprint_topology_attention_lines, format_quality_console_summary, format_quality_remediation_catalog_audit_lines, format_quality_remediation_coverage_lines, @@ -44,11 +57,17 @@ quality_remediation_coverage_summary, quality_report_payload, quality_report_to_json, + run_quality_assessment, series_fingerprint_readiness_summary, series_fingerprint_readiness_risk_summary, + series_fingerprint_parity_summary, series_fingerprint_regime_summary, write_quality_report, ) +from histdatacom.data_quality.bounded_payload_contracts import ( + representative_bounded_quality_payload, + representative_quality_report, +) from histdatacom.runtime_contracts import ArtifactRef @@ -84,6 +103,84 @@ def test_quality_json_report_is_deterministic_and_investigable( assert str(tmp_path) not in first +def test_cross_series_fingerprint_has_report_bounded_and_console_surfaces() -> ( + None +): + """Cross-series metadata should remain visible on every report surface.""" + report = representative_quality_report() + full = quality_report_payload(report) + bounded = representative_bounded_quality_payload() + summary = full["metadata"][CROSS_SERIES_FINGERPRINT_METADATA_KEY] + + assert ( + summary["schema_version"] == "histdatacom.cross-series-fingerprint.v1" + ) + assert bounded["fingerprint_cross_series"] == summary + lines = format_cross_series_fingerprint_lines(summary) + assert lines[1] == "Cross-series fingerprint" + assert "groups=1" in lines[2] + console = format_quality_console_summary( + report, + check_groups=("fingerprint",), + ) + assert "Cross-series fingerprint" in console + assert "ascii:T:201202" in console + + +def test_fingerprint_parity_has_report_bounded_and_console_surfaces() -> None: + """Opt-in parity should be visible without reading nested findings.""" + report = representative_quality_report() + report_payload = quality_report_payload(report) + bounded = representative_bounded_quality_payload() + console = format_quality_console_summary( + report, + check_groups=("fingerprint",), + ) + summary = report_payload["metadata"][ + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_METADATA_KEY + ] + + assert summary["schema_version"] == ( + TIME_SERIES_FINGERPRINT_PARITY_SUMMARY_SCHEMA_VERSION + ) + assert summary["target_count"] == 3 + assert summary["matching_target_count"] == 3 + assert summary["computed_from_counts"] + assert summary["cache_source_counts"] == {"sibling": 3} + assert summary["freshness_counts"] == {"fresh": 3} + assert bounded["fingerprint_parity"] == summary + assert "Fingerprint cache/source parity" in console + assert format_fingerprint_parity_summary_lines(summary)[1] == ( + "Fingerprint cache/source parity" + ) + direct = series_fingerprint_parity_summary(report.findings, target_limit=1) + assert direct is not None + assert direct["included_target_count"] == 1 + assert direct["omitted_target_count"] == 2 + assert direct["truncated"] is True + + +def test_synthetic_constraints_have_report_bounded_and_console_surfaces() -> ( + None +): + """Generator constraints should be visible without nested findings.""" + report = representative_quality_report() + full = quality_report_payload(report) + bounded = representative_bounded_quality_payload() + console = format_quality_console_summary( + report, + check_groups=("fingerprint",), + ) + summary = full["metadata"][SYNTHETIC_CONSTRAINT_SUMMARY_METADATA_KEY] + + assert summary["schema_version"] == ( + SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION + ) + assert summary["target_count"] == 3 + assert bounded[SYNTHETIC_CONSTRAINT_BOUNDED_PAYLOAD_KEY] == summary + assert "Synthetic fingerprint constraints" in console + + def test_quality_report_payload_is_publish_safe_by_default( tmp_path: Path, ) -> None: @@ -112,6 +209,33 @@ def test_quality_report_payload_can_preserve_raw_local_paths( assert payload["targets"][0]["path"] == str(tmp_path / "clean.csv") +def test_quality_report_payload_exposes_publish_safe_skip_events( + tmp_path: Path, +) -> None: + """Full report consumers should receive structured path-free skips.""" + report = _quality_engine_skip_report(tmp_path) + payload = quality_report_payload(report) + metadata = payload["metadata"] + assert isinstance(metadata, dict) + engine = metadata[QUALITY_ENGINE_METADATA_KEY] + assert isinstance(engine, dict) + skips = engine["skip_events"] + assert isinstance(skips, dict) + + assert engine["schema_version"] == QUALITY_ENGINE_SCHEMA_VERSION + assert engine["planned_target_rule_evaluation_count"] == 2 + assert engine["target_rule_evaluation_count"] == 1 + assert engine["skipped_rule_evaluation_count"] == 1 + assert skips["schema_version"] == QUALITY_SKIP_EVENTS_SCHEMA_VERSION + assert skips["reason_counts"] == { + QUALITY_SKIP_REASON_DUPLICATE_ARCHIVE_PREFERRED_CSV: 1 + } + assert skips["rule_id_counts"] == {"time.ascii.gaps": 1} + encoded = json.dumps(payload, sort_keys=True) + assert str(tmp_path) not in encoded + assert "/Users/" not in encoded + + def test_publish_safe_json_value_sanitizes_nested_path_metadata() -> None: """Metadata path fields and embedded local paths should be publishable.""" payload = { @@ -296,6 +420,22 @@ def test_quality_console_summary_separates_target_statuses( assert "Failed files\n- csv:" in output +def test_quality_console_summary_reconciles_skipped_evaluations( + tmp_path: Path, +) -> None: + """Human output should explain intentional engine-level rule skips.""" + output = format_quality_console_summary( + _quality_engine_skip_report(tmp_path), + check_groups=("time",), + ) + + assert "Quality engine skips" in output + assert "- evaluations: planned=2 executed=1 skipped=1" in output + assert "- events: included=1 omitted=0" in output + assert "- reasons: duplicate_archive_preferred_csv=1" in output + assert "- rules: time.ascii.gaps=1" in output + + def test_quality_report_payload_adds_fingerprint_coverage_metadata( tmp_path: Path, ) -> None: @@ -411,6 +551,128 @@ def test_quality_report_payload_adds_fingerprint_topology_attention_metadata( assert target_summaries[0]["computed_from"] == "unavailable" +def test_topology_attention_cli_lines_render_bounded_inspection_context() -> ( + None +): + """CLI drill-down should render compact rows, counts, and action links.""" + lines = format_fingerprint_topology_attention_lines( + { + "attention_target_count": 1, + "included_attention_target_count": 1, + "omitted_attention_target_count": 0, + "target_summaries": [ + { + "target_axis": { + "data_format": "ascii", + "timeframe": "T", + "symbol": "EURUSD", + "period": "201202", + "kind": "csv", + }, + "attention_level": "structural", + "attention_flags": ["invalid_timestamps"], + "remediation_hints": [], + "invalid_timestamp_count": 2, + "duplicate_timestamp_count": 0, + "non_monotonic_count": 0, + "suspicious_gap_count": 0, + "weekend_activity_count": 0, + "max_gap_ms": 60_000, + "computed_from": "text_scan", + "inspection_context": { + "invalid_timestamps": { + "total_count": 2, + "included_count": 1, + "omitted_count": 1, + "samples": [ + { + "row_number": 7, + "timestamp_source": "bad-timestamp", + } + ], + "next_action": { + "code": "inspect_invalid_timestamp_rows", + }, + } + }, + } + ], + } + ) + + assert lines[-1] == ( + " - context invalid_timestamps: samples=1/2 omitted=1 " + "action=inspect_invalid_timestamp_rows row 7=bad-timestamp" + ) + + +def test_topology_attention_cli_renders_compact_allowed_weekend_policy() -> ( + None +): + """Allowed weekend activity should render as context, not urgent action.""" + policy_context = { + "actionable": False, + "weekend_activity_policy": "allowed", + "profile_name": "weekend-market", + } + note = { + "code": "verify_weekend_session_policy", + "message": "weekend activity is allowed by the active profile", + "action_kind": "context", + "rule_id": SERIES_FINGERPRINT_RULE_ID, + "flag": "weekend_activity", + "policy_context": policy_context, + } + lines = format_fingerprint_topology_attention_lines( + { + "attention_target_count": 1, + "included_attention_target_count": 1, + "omitted_attention_target_count": 0, + "target_summaries": [ + { + "target_axis": { + "data_format": "ascii", + "timeframe": "T", + "symbol": "EURUSD", + "period": "201202", + "kind": "csv", + }, + "attention_level": "contextual", + "attention_flags": ["weekend_activity"], + "remediation_hints": [note], + "calendar_policy": { + "weekend_activity_policy": "allowed", + "expected_session_closure_policy": "expected", + "calendar_profile": {"name": "weekend-market"}, + }, + "invalid_timestamp_count": 0, + "duplicate_timestamp_count": 0, + "non_monotonic_count": 0, + "suspicious_gap_count": 0, + "weekend_activity_count": 1, + "max_gap_ms": None, + "computed_from": "text_scan", + "inspection_context": { + "weekend_activity": { + "included_count": 1, + "total_count": 1, + "omitted_count": 0, + "samples": [], + "policy_note": note, + } + }, + } + ], + } + ) + output = "\n".join(lines) + + assert "contextual" in output + assert "policy weekend=allowed profile=weekend-market" in output + assert "policy-note=verify_weekend_session_policy" in output + assert "action=verify_weekend_session_policy" not in output + + def test_quality_report_payload_adds_fingerprint_distribution_metadata( tmp_path: Path, ) -> None: @@ -641,8 +903,8 @@ def test_fingerprint_readiness_risk_summary_handles_missing_sections( assert summary is not None assert summary["target_count"] == 2 assert summary["risk_target_count"] == 2 - assert summary["reason_counts"]["not_emitted"] == 4 - assert summary["reason_counts"]["unsupported_target_kind"] == 3 + assert summary["reason_counts"]["not_emitted"] == 7 + assert summary["reason_counts"]["unsupported_target_kind"] == 4 assert summary["target_risks"][0]["target_axis"]["kind"] == "unknown" assert ( "unsupported_target_kind" in summary["target_risks"][0]["reason_codes"] @@ -671,6 +933,11 @@ def test_fingerprint_console_summary_reports_readiness_lines( "insufficient_sample_count=3 " "acf_basis: observed_sequence=3 computed_lags=15 skipped_lags=3" ) in output + assert ( + "- decomposition: skipped=3 reasons: not_emitted=3 " + "stationarity: unknown=3 structural-breaks: unknown=3 " + "computed_windows=0 skipped_windows=0 structural_candidates=0" + ) in output assert ( "- topology limitations: duplicate_timestamps=1, " "expected_session_closures=1, invalid_timestamps_skipped=1, " @@ -697,6 +964,12 @@ def test_fingerprint_console_summary_reports_readiness_lines( "skipped_lags=3 skipped_reasons=insufficient_sample_count=3" ) in output assert "spread_acf:samples=2/computed=1/skipped=1" in output + assert ( + "decomposition=skipped reason=not_emitted basis=unknown metric=unknown " + "samples=0/0 windows=[] computed_windows=0 skipped_windows=0 " + "stationarity=unknown trend=unknown structural=unknown/0 " + "projection=unknown" + ) in output assert ( "- ascii EURUSD T 201202 csv: microstructure_dynamics valid" ) in output @@ -840,6 +1113,15 @@ def test_quality_report_payload_adds_remediation_coverage_metadata( assert coverage["unmapped_warning_error_group_count"] == 2 assert coverage["included_unmapped_warning_error_group_count"] == 2 assert coverage["omitted_unmapped_warning_error_group_count"] == 0 + assert coverage["actionability_counts"] == { + "informational_only": 1, + "remediable_defect": 4, + } + assert coverage["unmapped_actionable_warning_error_finding_count"] == 3 + assert coverage["unmapped_actionable_warning_error_group_count"] == 2 + assert ( + coverage["intentionally_unremediable_warning_error_finding_count"] == 0 + ) groups = coverage["unmapped_groups"] assert [group["max_severity"] for group in groups] == [ @@ -851,6 +1133,64 @@ def test_quality_report_payload_adds_remediation_coverage_metadata( assert groups[0]["finding_code"] == "FILE_MISSING" assert groups[0]["occurrence_count"] == 2 assert groups[0]["target_axis_count"] == 2 + assert groups[0]["actionability"] == "remediable_defect" + assert groups[0]["actionability_reason"] == "unmapped_warning_or_error" + + +def test_quality_remediation_coverage_separates_actionable_boundaries( + tmp_path: Path, +) -> None: + """Published report coverage should distinguish support boundaries.""" + target = _target(tmp_path / "unsupported.csv") + report = QualityReport( + targets=(target,), + rule_results=( + QualityRuleResult( + rule_id="custom.rule", + target=target, + findings=( + QualityFinding( + severity=QualitySeverity.WARNING, + code="CUSTOM_REPAIRABLE_FAILURE", + message="repairable failure", + rule_id="custom.rule", + target=target, + ), + ), + ), + QualityRuleResult( + rule_id="inventory.format_support", + target=target, + findings=( + QualityFinding( + severity=QualitySeverity.ERROR, + code="HISTDATA_FORMAT_UNSUPPORTED", + message="unsupported format", + rule_id="inventory.format_support", + target=target, + ), + ), + ), + ), + ) + + coverage = quality_remediation_coverage_summary(report) + assert coverage is not None + groups = coverage["unmapped_groups"] + + assert coverage["unmapped_warning_error_group_count"] == 2 + assert coverage["unmapped_actionable_warning_error_group_count"] == 1 + assert coverage["unmapped_actionable_warning_error_finding_count"] == 1 + assert ( + coverage["intentionally_unremediable_warning_error_finding_count"] == 1 + ) + assert groups[0]["finding_code"] == "CUSTOM_REPAIRABLE_FAILURE" + assert groups[0]["actionability"] == "remediable_defect" + assert groups[1]["finding_code"] == "HISTDATA_FORMAT_UNSUPPORTED" + assert groups[1]["actionability"] == "unsupported_format_or_capability" + output = "\n".join(format_quality_remediation_coverage_lines(coverage)) + assert "actionability: actionable=1" in output + assert "intentional_boundary=1" in output def test_quality_report_payload_adds_all_mapped_remediation_coverage( @@ -959,11 +1299,23 @@ def test_quality_report_embeds_enabled_remediation_catalog_audit( assert audit["summary"]["report_count"] == 1 assert audit["summary"]["report_finding_count"] == 2 assert audit["summary"]["report_unmapped_warning_error_group_count"] == 1 + assert ( + audit["summary"]["report_unmapped_actionable_warning_error_group_count"] + == 1 + ) observed_groups = audit["report_coverage"][0]["remediation_coverage"][ "unmapped_groups" ] assert observed_groups[0]["finding_code"] == ("CUSTOM_REPORT_ONLY_GAP") assert observed_groups[0]["occurrence_count"] == 1 + assert observed_groups[0]["actionability"] == "remediable_defect" + assert audit["remediation_plan"]["schema_version"] == ( + "histdatacom.quality-remediation-plan.v1" + ) + assert any( + item["finding_code"] == "CUSTOM_REPORT_ONLY_GAP" + for item in audit["remediation_plan"]["items"] + ) assert str(tmp_path) not in encoded @@ -980,7 +1332,16 @@ def test_quality_console_summary_renders_remediation_catalog_audit( assert audit is not None assert "Remediation catalog audit" in output assert "- observed report: reports=1 findings=2" in output + assert "- attribution: exact=" in output + assert " inferred=" in output + assert " unresolved=" in output + assert "- actionability: actionable=" in output + assert "- plan: candidates=" in output + assert "- plan rank=" in output assert "- observed error custom.report:CUSTOM_REPORT_ONLY_GAP" in output + assert ( + "actionability=remediable_defect(unmapped_warning_or_error)" in output + ) assert lines[0] == "" assert "CUSTOM_REPORT_ONLY_GAP" in "\n".join(lines) @@ -1230,6 +1591,38 @@ def test_bounded_quality_payload_includes_fingerprint_coverage( } +def test_bounded_quality_payload_includes_structured_skip_events( + tmp_path: Path, +) -> None: + """Bounded orchestration consumers should reconcile engine skips.""" + report = _quality_engine_skip_report(tmp_path) + payload = bounded_quality_payload( + operation="data-quality", + check_groups=("time",), + discovery={"roots": [str(tmp_path)], "target_count": 2}, + report=report, + decision=QualityExitPolicy.from_values().evaluate(report.summary()), + artifact=None, + ) + engine = payload[QUALITY_ENGINE_METADATA_KEY] + assert isinstance(engine, dict) + skips = engine["skip_events"] + assert isinstance(skips, dict) + + assert engine["planned_target_rule_evaluation_count"] == 2 + assert engine["target_rule_evaluation_count"] == 1 + assert engine["skipped_rule_evaluation_count"] == 1 + assert skips["event_count"] == 1 + assert skips["events"][0]["target_axis"] == { + "data_format": "ascii", + "timeframe": "T", + "symbol": "EURUSD", + "period": "201202", + "kind": "zip", + } + assert str(tmp_path) not in json.dumps(payload, sort_keys=True) + + def test_bounded_quality_payload_includes_fingerprint_distribution( tmp_path: Path, ) -> None: @@ -1319,6 +1712,77 @@ def test_bounded_quality_payload_includes_fingerprint_topology_attention( ] +def test_bounded_quality_payload_preserves_topology_inspection_context( + tmp_path: Path, +) -> None: + """Orchestration payloads should retain bounded, action-linked evidence.""" + report = _fingerprint_report(tmp_path) + fingerprint = ( + report.rule_results[0].findings[0].metadata["time_series_fingerprint"] + ) + topology = fingerprint["temporal_topology"] + topology["duplicate_timestamp_count"] = 1 + topology["inspection_context"] = { + "schema_version": "histdatacom.timestamp-topology-inspection.v1", + "duplicate_timestamps": { + "total_count": 1, + "included_count": 1, + "omitted_count": 0, + "truncated": False, + "limit_metadata": { + "samples": { + "limit": 5, + "effective_limit": 5, + "requested_limit": 5, + "default_limit": 5, + "minimum_limit": 0, + "maximum_limit": 5, + "unbounded": False, + "total_count": 1, + "included_count": 1, + "omitted_count": 0, + "truncated": False, + } + }, + "duplicate_row_count": 1, + "samples": [ + { + "row_number": 2, + "timestamp_source": "20120201 000000000", + "timestamp_source_truncated": False, + "timestamp_utc_ms": 1328072400000, + "utc_timestamp": "2012-02-01T05:00:00Z", + "occurrence_count": 2, + "exact_row_group_count": 1, + } + ], + }, + } + + payload = bounded_quality_payload( + operation="data-quality", + check_groups=("fingerprint",), + discovery={"roots": [str(tmp_path)], "target_count": 2}, + report=report, + decision=QualityExitPolicy.from_values().evaluate(report.summary()), + artifact=None, + ) + targets = payload["fingerprint_topology_attention"]["target_summaries"] + duplicate_target = next( + target + for target in targets + if "duplicate_timestamps" in target["attention_flags"] + ) + context = duplicate_target["inspection_context"]["duplicate_timestamps"] + + assert context["samples"][0]["occurrence_count"] == 2 + assert context["next_action"]["code"] == ( + "inspect_duplicate_timestamp_rows" + ) + assert str(tmp_path) not in json.dumps(payload, sort_keys=True) + assert "duplicate_row_values" not in json.dumps(context, sort_keys=True) + + def test_bounded_quality_payload_includes_fingerprint_readiness( tmp_path: Path, ) -> None: @@ -1536,6 +2000,11 @@ def test_fingerprint_readiness_summary_is_bounded_and_issue_first( assert summary["target_summaries"][0]["applicable_dynamics_status"] == ( "limited" ) + assert summary["decomposition_status_counts"] == {"skipped": 3} + assert summary["decomposition_reason_counts"] == {"not_emitted": 3} + assert summary["target_summaries"][0]["decomposition"]["status"] == ( + "skipped" + ) def test_fingerprint_readiness_summary_orders_limited_dependence_first( @@ -1757,6 +2226,12 @@ def test_bounded_quality_payload_includes_enabled_remediation_catalog_audit( assert audit["report_coverage"][0]["remediation_coverage"][ "unmapped_groups" ][0]["finding_code"] == ("CUSTOM_REPORT_ONLY_GAP") + assert audit["remediation_plan"]["plan_item_count"] > 0 + assert audit["remediation_plan"]["items"] + assert ( + audit["payload_limits"]["remediation_plan"]["included_count"] + == audit["remediation_plan"]["included_plan_item_count"] + ) assert ( payload["payload_limits"]["remediation_catalog_audit"][ "target_axis_limit" @@ -1795,6 +2270,38 @@ def test_quality_exit_policy_applies_error_warning_and_never_modes( ) +class _QualityEngineSkipRule: + rule_id = "time.ascii.gaps" + description = "semantic scans prefer extracted CSVs" + + def evaluate(self, target: QualityTarget) -> tuple[QualityFinding, ...]: + del target + return () + + +def _quality_engine_skip_report(tmp_path: Path) -> QualityReport: + archive = QualityTarget( + path=str(tmp_path / "DAT_ASCII_EURUSD_T_201202.zip"), + kind=QualityTargetKind.ZIP, + data_format="ascii", + timeframe="T", + symbol="EURUSD", + period="201202", + ) + csv = QualityTarget( + path=str(tmp_path / "DAT_ASCII_EURUSD_T_201202.csv"), + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe="T", + symbol="EURUSD", + period="201202", + ) + return run_quality_assessment( + targets=(archive, csv), + rules=(_QualityEngineSkipRule(),), + ) + + def _mixed_report(tmp_path: Path) -> QualityReport: clean = _target(tmp_path / "clean.csv") warning = _target(tmp_path / "warning.csv") diff --git a/tests/unit/test_data_quality_seasonal_exogenous.py b/tests/unit/test_data_quality_seasonal_exogenous.py new file mode 100644 index 00000000..1add71e5 --- /dev/null +++ b/tests/unit/test_data_quality_seasonal_exogenous.py @@ -0,0 +1,739 @@ +"""Tests for explicit SARIMA, ARIMAX, and SARIMAX diagnostics.""" + +from __future__ import annotations + +import json +import math +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Mapping, cast + +import polars as pl +import pytest + +import histdatacom.data_quality.seasonal_exogenous as seasonal_module +from histdatacom.data_quality.calendar_profiles import default_calendar_profile +from histdatacom.data_quality.classical_model_contracts import ( + ClassicalModelInputProfile, + ClassicalModelResourcePolicy, +) +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprints import ( + FINGERPRINT_AUDIT_SECTIONS, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.profiles import ( + QUALITY_PROFILE_SCHEMA_VERSION, + quality_profile_from_mapping, +) +from histdatacom.data_quality.reporting import ( + QualityExitPolicy, + bounded_quality_payload, + format_quality_console_summary, + quality_report_payload, +) +from histdatacom.data_quality.seasonal_exogenous import ( + SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY, + SEASONAL_EXOGENOUS_COLUMNS, + SEASONAL_EXOGENOUS_SCHEMA_VERSION, + SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY, + SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION, + CalendarRegressorProfile, + SeasonalExogenousProfile, + SeasonalExogenousSpecification, + project_seasonal_exogenous_onto_training_frame, + seasonal_exogenous_from_training_frame, + seasonal_exogenous_summary, +) +from histdatacom.data_quality.training_features import ( + ensure_tick_training_features, + training_feature_definitions, +) +from histdatacom.histdata_ascii import ( + CACHE_FILENAME, + TICK, + format_influx_line, + read_polars_cache, + write_polars_cache, +) + + +def test_full_family_is_explicit_deterministic_and_leakage_safe() -> None: + """All named families should use shared folds and known calendar values.""" + first = _run(_process_frame(420)) + second = _run(_process_frame(420)) + + assert first.diagnostics == second.diagnostics + assert ( + first.diagnostics["schema_version"] == SEASONAL_EXOGENOUS_SCHEMA_VERSION + ) + evaluation = _mapping(first.diagnostics["evaluation"]) + models = _mapping_rows(evaluation["models"]) + assert {model["family"] for model in models} == { + "sarima", + "arimax", + "sarimax", + } + assert { + tuple(_mapping(model["configuration"])["seasonal_order"]) + for model in models + } == {(1, 0, 0, 4), (0, 0, 0, 0)} + fold_rows = [ + row + for model in models + for row in _mapping_rows(_mapping(model)["fold_results"]) + ] + assert fold_rows + assert all(row["future_values_visible"] is False for row in fold_rows) + assert all(row["original_scale"] is True for row in fold_rows) + regressors = _mapping(first.diagnostics["regressors"]) + assert regressors["future_observed_market_values_allowed"] is False + assert ( + regressors["future_values_derived_without_market_observations"] is True + ) + assert evaluation["automatic_winner"] is False + assert evaluation["comparison_semantics"] == "descriptive_shared_folds_only" + + +def test_configuration_and_profile_parser_keep_each_family_first_class() -> ( + None +): + """Orders, seasonality, exogenous columns, and controls must stay explicit.""" + with pytest.raises(ValueError, match="SARIMA requires"): + SeasonalExogenousSpecification("bad", "sarima", 1) + with pytest.raises(ValueError, match="ARIMAX requires"): + SeasonalExogenousSpecification("bad", "arimax", 1) + with pytest.raises(ValueError, match="SARIMAX requires"): + SeasonalExogenousSpecification( + "bad", "sarimax", 1, regressor_names=("source_hour_sin",) + ) + with pytest.raises(ValueError, match="constant"): + SeasonalExogenousSpecification( + "bad", + "sarima", + 1, + d=1, + seasonal_p=1, + seasonal_period=4, + seasonal_cycle_ms=240_000, + trend="c", + ) + with pytest.raises(ValueError, match="unknown calendar regressor"): + SeasonalExogenousSpecification( + "bad", "arimax", 1, regressor_names=("future_price",) + ) + + parsed = ( + quality_profile_from_mapping( + { + "schema_version": QUALITY_PROFILE_SCHEMA_VERSION, + "name": "seasonal-family", + "rules": { + "fingerprint.series": { + "seasonal_exogenous": { + "enabled": True, + "projection_specification_ids": [ + "sarimax-explicit" + ], + "regressor_profile": { + "allow_partial_calendar": False, + "max_regressors": 3, + }, + "specifications": [ + { + "specification_id": "sarimax-explicit", + "family": "sarimax", + "p": 1, + "seasonal_p": 1, + "seasonal_period": 4, + "seasonal_cycle_ms": 240000, + "regressor_names": ["source_hour_sin"], + "optimizer": "powell", + "max_iterations": 17, + } + ], + } + } + }, + } + ) + .fingerprint_profile() + .seasonal_exogenous + ) + assert parsed.enabled is True + assert parsed.specifications[0].family == "sarimax" + assert parsed.specifications[0].seasonal_period == 4 + assert parsed.specifications[0].optimizer == "powell" + assert parsed.regressor_profile.allow_partial_calendar is False + + +def test_invalid_cycle_rank_guards_and_dependency_absence_are_advisory( + monkeypatch: Any, +) -> None: + """Runtime mismatches, collinearity, and missing extras isolate failures.""" + mismatch = _run( + _process_frame(360), + profile=_profile(seasonal_cycle_ms=120_000), + ) + reasons = _mapping( + _mapping(mismatch.diagnostics["fit_summary"])["reason_counts"] + ) + assert int(reasons["invalid_seasonality"]) > 0 + + collinear = SeasonalExogenousProfile( + enabled=True, + specifications=( + SeasonalExogenousSpecification( + "arimax-collinear", + "arimax", + 1, + regressor_names=("market_open", "session_london"), + ), + ), + projection_specification_ids=("arimax-collinear",), + ) + limited = _run(_process_frame(360), profile=collinear) + rank_reasons = _mapping( + _mapping(limited.diagnostics["fit_summary"])["reason_counts"] + ) + assert set(rank_reasons) & {"rank_deficient_regressors", "collinearity"} + + monkeypatch.setattr(seasonal_module, "_load_backend", lambda: None) + unavailable = _run(_process_frame(360)) + assert unavailable.diagnostics["status"] == "unavailable" + assert unavailable.diagnostics["reason"] == "dependency_unavailable" + + +def test_partial_calendar_and_resource_limits_are_bounded() -> None: + """Incomplete event calendars and inherited resource limits are explicit.""" + event_profile = SeasonalExogenousProfile( + enabled=True, + specifications=( + SeasonalExogenousSpecification( + "arimax-event", "arimax", 1, regressor_names=("event_any",) + ), + ), + projection_specification_ids=("arimax-event",), + regressor_profile=CalendarRegressorProfile( + allow_partial_calendar=False, + require_complete_calendar_for=("event_any",), + ), + ) + event = _run(_process_frame(360), profile=event_profile) + event_reasons = _mapping( + _mapping(event.diagnostics["fit_summary"])["reason_counts"] + ) + assert set(event_reasons) & { + "partial_calendar_unavailable", + "future_regressor_unavailable", + } + + resources = ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=64, + max_horizons=4, + max_candidate_orders=1, + max_fit_attempts=1, + max_wall_time_seconds=30, + max_memory_bytes=10_000_000, + max_retained_diagnostics=1, + ) + bounded = _run( + _process_frame(420), input_profile=_input_profile(resources=resources) + ) + usage = _mapping(bounded.diagnostics["resource_usage"]) + assert usage["fit_attempt_count"] == 1 + assert _mapping(bounded.diagnostics["evaluation"])["model_count"] == 1 + assert bounded.diagnostics["status"] == "limited" + + +def test_calendar_regressors_preserve_fixed_est_boundaries_and_tag_vocabulary() -> ( + None +): + """DST dates must not shift the documented fixed-EST source-clock basis.""" + profile = SeasonalExogenousProfile( + enabled=True, + specifications=( + SeasonalExogenousSpecification( + "arimax-calendar", + "arimax", + 1, + regressor_names=( + "source_hour_sin", + "tag:major_holiday:new_years_day", + ), + ), + ), + projection_specification_ids=("arimax-calendar",), + ) + timestamps = ( + _utc_ms(2012, 1, 1, 6, 30), + _utc_ms(2022, 3, 13, 6, 30), + _utc_ms(2022, 3, 13, 7, 30), + ) + rows = [{"cm_input_bin_start_utc_ms": value} for value in timestamps] + first, contract = seasonal_module._calendar_regressors( + rows, + profile, + default_calendar_profile(), + target=_target("calendar.csv"), + ) + second, _ = seasonal_module._calendar_regressors( + rows, + profile, + default_calendar_profile(), + target=_target("calendar.csv"), + ) + + assert [row.values for row in first] == [row.values for row in second] + assert contract["column_order"] == [ + "source_hour_sin", + "tag:major_holiday:new_years_day", + ] + assert first[0].values["tag:major_holiday:new_years_day"] == 1.0 + assert first[1].values["source_hour_sin"] == pytest.approx( + math.sin(2.0 * math.pi * 1.5 / 24.0) + ) + assert first[2].values["source_hour_sin"] == pytest.approx( + math.sin(2.0 * math.pi * 2.5 / 24.0) + ) + + +def test_expected_closures_and_true_missing_bins_remain_distinct() -> None: + """The family must inherit #421 closure/missing semantics without filling.""" + frame = pl.DataFrame( + { + "datetime": [ + _utc_ms(2022, 1, 7, 21), + _utc_ms(2022, 1, 10, 12), + ], + "bid": [1.1, 1.2], + "ask": [1.1002, 1.2002], + "vol": [0, 0], + } + ) + profile = SeasonalExogenousProfile( + enabled=True, + specifications=( + SeasonalExogenousSpecification( + "sarima-closure", + "sarima", + 0, + seasonal_p=1, + seasonal_period=2, + seasonal_cycle_ms=12 * 60 * 60 * 1000, + ), + ), + projection_specification_ids=("sarima-closure",), + ) + result = _run( + frame, + profile=profile, + input_profile=_input_profile( + frequency_ms=6 * 60 * 60 * 1000, + minimum_training_observations=1, + step_size=1, + horizons=(1,), + ), + ) + missingness = _mapping(result.diagnostics["input_missingness_policy"]) + assert int(missingness["expected_closure_count"]) > 0 + assert int(missingness["unexpected_missing_count"]) > 0 + assert missingness["forward_fill_policy"] == "never" + + +def test_projection_is_flat_identity_safe_serializable_and_no_fill( + tmp_path: Path, +) -> None: + """Augmented columns survive duplicate/masked timestamps, IPC, and Influx.""" + raw = _process_frame(420, duplicate_timestamp=True) + target = _target("DAT_ASCII_EURUSD_T_201202.csv") + result = _run(raw, target=target) + projected = project_seasonal_exogenous_onto_training_frame( + raw, result, target=target + ) + + assert ( + projected.select("datetime", "bid", "ask").to_dicts() + == raw.select("datetime", "bid", "ask").to_dicts() + ) + assert projected.get_column("row_id").n_unique() == raw.height + assert set(SEASONAL_EXOGENOUS_COLUMNS) <= set(projected.columns) + for family in ("sarima", "arimax", "sarimax"): + available = projected.filter(pl.col(f"cm_{family}_forecast_available")) + assert available.height > 0 + assert available.get_column(f"cm_{family}_training_eligible").all() + assert ( + available.get_column(f"cm_{family}_actual").null_count() + == available.height + ) + + enriched = ensure_tick_training_features(raw, target=target) + masked = enriched.with_columns( + pl.when(pl.col("row_id") == 7) + .then(None) + .otherwise(pl.col("timestamp_utc_ms")) + .alias("timestamp_utc_ms") + ) + masked_projection = project_seasonal_exogenous_onto_training_frame( + masked, result + ) + assert masked_projection.get_column("row_id").to_list() == list( + range(1, raw.height + 1) + ) + line = format_influx_line( + "eurusd", + "ascii", + TICK, + projected.filter(pl.col("cm_sarima_forecast_available")).row(0), + columns=projected.columns, + ) + assert "cm_sarima_forecast_available=true" in line + cache = tmp_path / CACHE_FILENAME + write_polars_cache(projected, cache) + restored = read_polars_cache(cache) + assert ( + restored.select(SEASONAL_EXOGENOUS_COLUMNS).to_dicts() + == projected.select(SEASONAL_EXOGENOUS_COLUMNS).to_dicts() + ) + assert result.diagnostics["forward_fill_policy"] == "never" + + +def test_columns_and_report_surfaces_are_complete() -> None: + """Registry, full JSON, bounded JSON, and CLI expose the same contract.""" + definitions = {row.name: row for row in training_feature_definitions()} + assert len(SEASONAL_EXOGENOUS_COLUMNS) == 123 + assert set(SEASONAL_EXOGENOUS_COLUMNS) <= set(definitions) + assert all( + definitions[name].nullable for name in SEASONAL_EXOGENOUS_COLUMNS + ) + assert "seasonal_exogenous" in FINGERPRINT_AUDIT_SECTIONS + + diagnostics = _run(_process_frame(360)).diagnostics + finding = _finding(_target("DAT_ASCII_EURUSD_T_201202.csv"), diagnostics) + report = QualityReport( + targets=(finding.target,), + rule_results=( + QualityRuleResult( + rule_id=finding.rule_id, + target=finding.target, + findings=(finding,), + ), + ), + ) + full = quality_report_payload(report) + bounded = bounded_quality_payload( + report=report, + operation="data-quality", + check_groups=("fingerprint",), + discovery={"targets": []}, + artifact=None, + decision=QualityExitPolicy().evaluate(report.summary()), + ) + assert SEASONAL_EXOGENOUS_SUMMARY_METADATA_KEY in _mapping(full["metadata"]) + assert SEASONAL_EXOGENOUS_BOUNDED_PAYLOAD_KEY in bounded + assert "Seasonal and exogenous models" in format_quality_console_summary( + report + ) + + +def test_comparison_references_are_descriptive_and_never_select_a_winner() -> ( + None +): + """Available #422/#423 results should be carried as shared-fold references.""" + reference = { + "evaluation": { + "models": [ + { + "status": "ready", + "family": "reference-family", + "specification_id": "reference-spec", + "model_id": "sha256:reference", + "horizon_metrics": [{"horizon": 1, "mae": 0.1}], + } + ] + } + } + result = _run( + _process_frame(360), + exponential_smoothing=reference, + autoregressive=reference, + ) + evaluation = _mapping(result.diagnostics["evaluation"]) + references = _mapping(evaluation["reference_models"]) + for family in ("exponential_smoothing", "autoregressive"): + rows = _mapping_rows(references[family]) + assert rows[0]["model_id"] == "sha256:reference" + assert rows[0]["calculation_basis"] == "shared_regular_grid_folds" + assert rows[0]["automatic_winner"] is False + assert evaluation["automatic_winner"] is False + + +def test_bounded_summary_matches_golden() -> None: + """Target sorting, status counts, and truncation should remain stable.""" + findings = [] + for symbol, status, failed in ( + ("GBPUSD", "limited", 1), + ("AUDUSD", "ready", 0), + ("EURUSD", "ready", 0), + ): + target = QualityTarget( + path=f"DAT_ASCII_{symbol}_T_201201.csv", + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period="201201", + ) + findings.append( + _finding( + target, + { + "status": status, + "reason": "optimizer_failure" if failed else None, + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": symbol, + "period": "201201", + "kind": "csv", + }, + "fit_summary": { + "fit_attempt_count": 6, + "failed_fit_count": failed, + }, + "evaluation": { + "model_count": 3, + "evaluated_fold_count": 5 if failed else 6, + }, + }, + ) + ) + summary = seasonal_exogenous_summary(findings, target_limit=1) + expected = json.loads( + ( + Path(__file__).parents[1] + / "fixtures" + / "data_quality_reports" + / "seasonal_exogenous_summary.json" + ).read_text(encoding="utf-8") + ) + assert summary == expected + assert _mapping(summary)["schema_version"] == ( + SEASONAL_EXOGENOUS_SUMMARY_SCHEMA_VERSION + ) + + +def test_fingerprint_rule_runs_the_opt_in_family_on_supported_csv( + tmp_path: Path, +) -> None: + """The ordinary fingerprint lifecycle should emit and audit the family.""" + source = tmp_path / "DAT_ASCII_EURUSD_T_201201.csv" + source.write_text("\n".join(_tick_lines(360)) + "\n", encoding="ascii") + profile = HistDataFingerprintProfile( + classical_model_input=_input_profile(), + seasonal_exogenous=_profile(), + ) + finding = HistDataSeriesFingerprintRule(profile=profile).evaluate( + _target(str(source)) + )[0] + fingerprint = _mapping( + finding.metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + audit = _mapping(fingerprint["fingerprint_audit"]) + + assert _mapping(fingerprint["seasonal_exogenous"])["status"] in { + "ready", + "limited", + } + assert "seasonal_exogenous" in audit["sections_expected"] + assert "seasonal_exogenous" in audit["sections_emitted"] + + +def _run( + frame: pl.DataFrame, + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: SeasonalExogenousProfile | None = None, + target: QualityTarget | None = None, + exponential_smoothing: Mapping[str, Any] | None = None, + autoregressive: Mapping[str, Any] | None = None, +) -> Any: + return seasonal_exogenous_from_training_frame( + frame, + _fingerprint(), + input_profile=input_profile or _input_profile(), + profile=profile or _profile(), + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + target=target or _target("DAT_ASCII_EURUSD_T_201202.csv"), + ) + + +def _profile(*, seasonal_cycle_ms: int = 240_000) -> SeasonalExogenousProfile: + return SeasonalExogenousProfile( + enabled=True, + specifications=( + SeasonalExogenousSpecification( + "sarima-1x1", + "sarima", + 1, + seasonal_p=1, + seasonal_period=4, + seasonal_cycle_ms=seasonal_cycle_ms, + ), + SeasonalExogenousSpecification( + "arimax-hour", + "arimax", + 1, + regressor_names=("source_hour_sin",), + ), + SeasonalExogenousSpecification( + "sarimax-1x1-hour", + "sarimax", + 1, + seasonal_p=1, + seasonal_period=4, + seasonal_cycle_ms=seasonal_cycle_ms, + regressor_names=("source_hour_sin",), + ), + ), + projection_specification_ids=( + "sarima-1x1", + "arimax-hour", + "sarimax-1x1-hour", + ), + ) + + +def _input_profile(**overrides: Any) -> ClassicalModelInputProfile: + values: dict[str, Any] = { + "enabled": True, + "frequency_ms": 60_000, + "minimum_training_observations": 24, + "minimum_evaluation_observations": 1, + "step_size": 16, + "horizons": (1, 2), + "resources": ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=64, + max_horizons=4, + max_candidate_orders=16, + max_fit_attempts=32, + max_wall_time_seconds=30, + max_retained_diagnostics=64, + ), + } + values.update(overrides) + return ClassicalModelInputProfile(**values) + + +def _process_frame( + count: int, *, duplicate_timestamp: bool = False +) -> pl.DataFrame: + base = 1_325_376_000_000 # 2012-01-01 UTC + times = [base + index * 10_000 for index in range(count)] + if duplicate_timestamp and count > 1: + times[1] = times[0] + prices = [ + 1.1 + + index * 0.000003 + + ((index // 6) % 4 - 1.5) * 0.00004 + + ((index * 17) % 11 - 5) * 0.000002 + for index in range(count) + ] + return pl.DataFrame( + { + "datetime": times, + "bid": prices, + "ask": [value + 0.0002 for value in prices], + "vol": [0] * count, + } + ) + + +def _utc_ms(year: int, month: int, day: int, hour: int, minute: int = 0) -> int: + return int( + datetime( + year, month, day, hour, minute, tzinfo=timezone.utc + ).timestamp() + * 1000 + ) + + +def _tick_lines(count: int) -> tuple[str, ...]: + start = datetime(2012, 1, 1) + return tuple( + ( + f"{(start + timedelta(seconds=index * 10)).strftime('%Y%m%d %H%M%S')}000," + f"{1.1 + index * 0.000003:.6f}," + f"{1.1002 + index * 0.000003:.6f},0" + ) + for index in range(count) + ) + + +def _fingerprint() -> dict[str, Any]: + return { + "fingerprint_id": "fingerprint-eurusd", + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": "EURUSD", + "period": "201201", + "kind": "cache", + }, + "fingerprint_audit": { + "section_statuses": { + "dependence": "valid", + "stationarity_diagnostics": "limited", + } + }, + } + + +def _target(path: str) -> QualityTarget: + return QualityTarget( + path=path, + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201201", + ) + + +def _finding( + target: QualityTarget, diagnostics: Mapping[str, Any] +) -> QualityFinding: + return QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id="fingerprint.series", + target=target, + metadata={ + TIME_SERIES_FINGERPRINT_METADATA_KEY: { + "seasonal_exogenous": dict(diagnostics) + } + }, + ) + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(cast(Mapping[str, Any], value)) + + +def _mapping_rows(value: Any) -> list[dict[str, Any]]: + return [_mapping(row) for row in cast(list[Any], value)] diff --git a/tests/unit/test_data_quality_state_space.py b/tests/unit/test_data_quality_state_space.py new file mode 100644 index 00000000..e84e3b11 --- /dev/null +++ b/tests/unit/test_data_quality_state_space.py @@ -0,0 +1,791 @@ +"""Tests for structural state-space and leakage-safe Kalman diagnostics.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, cast + +import polars as pl +import pytest + +import histdatacom.data_quality.state_space as state_module +from histdatacom.data_quality.classical_model_contracts import ( + ClassicalModelInputProfile, + ClassicalModelResourcePolicy, +) +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprint_contracts import ( + implemented_fingerprint_target_section_names, +) +from histdatacom.data_quality.fingerprints import ( + FINGERPRINT_AUDIT_SECTIONS, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.profiles import ( + QUALITY_PROFILE_SCHEMA_VERSION, + quality_profile_from_mapping, +) +from histdatacom.data_quality.reporting import ( + QualityExitPolicy, + bounded_quality_payload, + format_quality_console_summary, + quality_report_payload, +) +from histdatacom.data_quality.state_space import ( + STATE_SPACE_BOUNDED_PAYLOAD_KEY, + STATE_SPACE_SCHEMA_VERSION, + STATE_SPACE_SUMMARY_METADATA_KEY, + STATE_SPACE_SUMMARY_SCHEMA_VERSION, + StateSpaceProfile, + StateSpaceSpecification, + project_state_space_onto_training_frame, + state_space_from_training_frame, + state_space_summary, +) +from histdatacom.data_quality.training_features import ( + KALMAN_COLUMNS, + STATE_SPACE_COLUMNS, + training_feature_definitions, +) +from histdatacom.histdata_ascii import ( + CACHE_FILENAME, + TICK, + format_influx_line, + read_polars_cache, + write_polars_cache, +) + + +def test_full_family_is_explicit_and_filtered_smoothed_boundaries_are_safe() -> ( + None +): + """Local, trend, and structural models keep state availability explicit.""" + result = _run(_process_frame(420)) + repeated = _run(_process_frame(420)) + diagnostics = _mapping(result.diagnostics) + + assert result.diagnostics == repeated.diagnostics + assert diagnostics["schema_version"] == STATE_SPACE_SCHEMA_VERSION + models = _mapping_rows(_mapping(diagnostics["evaluation"])["models"]) + assert {model["family"] for model in models} == { + "local_level", + "local_linear_trend", + "structural", + } + assert _mapping(diagnostics["evaluation"])["automatic_winner"] is False + assert diagnostics["smoothed_state_used_for_forecast"] is False + assert diagnostics["full_series_smoothing_used"] is False + folds = [ + row + for model in models + for row in _mapping_rows(_mapping(model)["fold_results"]) + if row["status"] == "evaluated" + ] + assert folds + assert all(row["future_values_visible"] is False for row in folds) + assert all( + row["smoothed_state_used_for_forecast"] is False for row in folds + ) + assert all(row["prediction_only_transition_count"] >= 0 for row in folds) + + projected = project_state_space_onto_training_frame( + _process_frame(420), result, target=_target("projection.csv") + ) + filtered = projected.filter(pl.col("cm_kalman_filtered_available")) + smoothed = projected.filter(pl.col("cm_kalman_smoothed_available")) + assert filtered.height > 0 + assert filtered.get_column("cm_kalman_filtered_training_eligible").all() + assert smoothed.height > 0 + assert smoothed.get_column("cm_kalman_smoothed_retrospective").all() + assert smoothed.get_column("cm_kalman_smoothed_diagnostic_only").all() + assert not smoothed.get_column("cm_kalman_smoothed_training_eligible").any() + + +def test_configuration_and_profile_parser_keep_components_first_class() -> None: + """Structural choices, initialization, and limits remain explicit.""" + with pytest.raises(ValueError, match="local-level"): + StateSpaceSpecification("bad", "local_level", stochastic_trend=True) + with pytest.raises(ValueError, match="requires stochastic_trend"): + StateSpaceSpecification("bad", "local_linear_trend") + with pytest.raises(ValueError, match="explicit component"): + StateSpaceSpecification( + "bad", + "structural", + stochastic_level=False, + irregular=False, + ) + + parsed = ( + quality_profile_from_mapping( + { + "schema_version": QUALITY_PROFILE_SCHEMA_VERSION, + "name": "state-space-family", + "rules": { + "fingerprint.series": { + "state_space": { + "enabled": True, + "projection_specification_id": "structural-explicit", + "max_state_dimension": 24, + "max_prediction_only_gap": 12, + "specifications": [ + { + "specification_id": "structural-explicit", + "family": "structural", + "seasonal_period": 4, + "seasonal_cycle_ms": 240000, + "initialization_method": "exact_diffuse", + "optimizer": "powell", + "max_iterations": 17, + } + ], + } + } + }, + } + ) + .fingerprint_profile() + .state_space + ) + assert parsed.enabled is True + assert parsed.max_state_dimension == 24 + assert parsed.max_prediction_only_gap == 12 + assert parsed.specifications[0].seasonal_period == 4 + assert parsed.specifications[0].initialization_method == "exact_diffuse" + + +def test_missing_observations_use_prediction_only_transitions_without_fill() -> ( + None +): + """Expected closures and true missing bins stay regular-grid omissions.""" + timestamps = [1_641_571_200_000] + [ + 1_641_816_000_000 + index * 6 * 60 * 60 * 1000 for index in range(10) + ] + frame = pl.DataFrame( + { + "datetime": timestamps, + "bid": [1.1 + index * 0.001 for index in range(len(timestamps))], + "ask": [1.1002 + index * 0.001 for index in range(len(timestamps))], + "vol": [0] * len(timestamps), + } + ) + result = _run( + frame, + input_profile=_input_profile( + frequency_ms=6 * 60 * 60 * 1000, + minimum_training_observations=1, + step_size=1, + horizons=(1,), + ), + profile=StateSpaceProfile( + enabled=True, + specifications=(StateSpaceSpecification("local", "local_level"),), + projection_specification_id="local", + max_prediction_only_gap=20, + ), + ) + policy = _mapping(result.diagnostics["missing_observation_policy"]) + assert policy["fill_policy"] == "none" + assert policy["transition_policy"] == "prediction_only" + assert policy["regular_time_basis"] is True + input_policy = _mapping(result.diagnostics["input_missingness_policy"]) + assert int(input_policy["expected_closure_count"]) > 0 + assert int(input_policy["unexpected_missing_count"]) > 0 + samples = _mapping_rows( + _mapping(result.diagnostics["fit_summary"])["fit_samples"] + ) + assert any( + int(row["prediction_only_transition_count"]) > 0 for row in samples + ) + + +def test_dependency_configuration_resource_and_gap_failures_are_bounded( + monkeypatch: Any, +) -> None: + """Expected runtime failures are normalized without exception text.""" + mismatch = _run( + _process_frame(360), + profile=_profile(seasonal_cycle_ms=120_000), + ) + reasons = _mapping( + _mapping(mismatch.diagnostics["fit_summary"])["reason_counts"] + ) + assert int(reasons["invalid_time_basis"]) > 0 + + long_gap_frame = _process_frame(180).vstack( + _process_frame(180).with_columns( + (pl.col("datetime") + 60 * 60 * 1000).alias("datetime") + ) + ) + long_gap = _run( + long_gap_frame, + profile=StateSpaceProfile( + enabled=True, + specifications=(StateSpaceSpecification("local", "local_level"),), + projection_specification_id="local", + max_prediction_only_gap=1, + ), + ) + gap_reasons = _mapping( + _mapping(long_gap.diagnostics["fit_summary"])["reason_counts"] + ) + assert int(gap_reasons["long_missing_gap"]) > 0 + + assert ( + state_module._backend_failure_reason( + RuntimeError("singular covariance") + ) + == "singular_covariance" + ) + assert ( + state_module._backend_failure_reason(RuntimeError("NaN state")) + == "numerical_instability" + ) + + resources = ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=64, + max_horizons=4, + max_candidate_orders=1, + max_fit_attempts=1, + max_wall_time_seconds=30, + max_memory_bytes=10_000_000, + max_retained_diagnostics=1, + ) + bounded = _run( + _process_frame(420), + input_profile=_input_profile(resources=resources), + ) + assert ( + _mapping(bounded.diagnostics["resource_usage"])["fit_attempt_count"] + == 1 + ) + assert bounded.diagnostics["status"] == "limited" + + monkeypatch.setattr(state_module, "_load_backend", lambda: None) + unavailable = _run(_process_frame(360)) + assert unavailable.diagnostics["status"] == "unavailable" + assert unavailable.diagnostics["reason"] == "dependency_unavailable" + assert unavailable.diagnostics["backend_exception_text_included"] is False + + +def test_state_serialization_aligns_absent_partial_and_ready_vectors() -> None: + """State payloads publish only complete named entries without fabrication.""" + specification = StateSpaceSpecification("local", "local_level") + profile = StateSpaceProfile( + enabled=True, + specifications=(specification,), + projection_specification_id="local", + ) + fold = { + "series_id": "EURUSD", + "period": "201201", + "fold_id": 1, + "origin_row_id": 1, + "target_row_id": 2, + "origin_bin_end_utc_ms": 1_000, + "target_bin_end_utc_ms": 2_000, + "target_index": 0, + "horizon": 1, + "status": "valid", + } + failed = _outcome( + status="failed", + reason="non_positive_covariance", + state_names=("level", "trend"), + ) + failed_sample = state_module._fit_sample( + failed, + specification, + "sha256:failed", + fold, + (0,), + 1, + profile, + ) + assert failed_sample["status"] == "failed" + assert failed_sample["reason"] == "non_positive_covariance" + assert failed_sample["state_dimension"] == 2 + assert failed_sample["state_names"] == [] + assert failed_sample["states"] == [] + assert failed_sample["states_truncated"] is True + + failed_fold = state_module._fold_evaluation( + [{"cm_input_observed_value": 1.0}], + fold, + specification, + 1, + "sha256:failed", + failed, + (), + _input_profile(), + profile.rounding_digits, + profile.max_retained_states, + ) + assert failed_fold["status"] == "not_evaluated" + assert failed_fold["reason"] == "non_positive_covariance" + assert failed_fold["state_names"] == [] + assert failed_fold["filtered_state"] == [] + assert failed_fold["smoothed_state"] == [] + assert failed_fold["filtered_state_available_at_origin"] is False + assert failed_fold["smoothed_state_retrospective"] is False + + partial = _outcome( + state_names=("level", "trend", "seasonal"), + filtered_state=(1.0, 2.0), + filtered_variance=(0.1,), + smoothed_state=(1.1, 2.1), + smoothed_variance=(0.2, 0.3), + ) + partial_sample = state_module._fit_sample( + partial, + specification, + "sha256:partial", + fold, + (0,), + 1, + profile, + ) + assert partial_sample["state_names"] == ["level"] + assert partial_sample["states"] == [ + { + "name": "level", + "filtered": 1.0, + "filtered_variance": 0.1, + "smoothed": 1.1, + "smoothed_variance": 0.2, + } + ] + assert partial_sample["states_truncated"] is True + + ready = _outcome( + state_names=("level", "trend"), + filtered_state=(1.0, 2.0), + filtered_variance=(0.1, 0.2), + smoothed_state=(1.1, 2.1), + smoothed_variance=(0.3, 0.4), + ) + ready_fold = state_module._fold_evaluation( + [{"cm_input_observed_value": 1.0}], + fold, + specification, + 1, + "sha256:ready", + ready, + (1.0,), + _input_profile(), + profile.rounding_digits, + profile.max_retained_states, + ) + assert ready_fold["state_names"] == ["level", "trend"] + assert ready_fold["filtered_state"] == [1.0, 2.0] + assert ready_fold["smoothed_state"] == [1.1, 2.1] + assert ready_fold["states_truncated"] is False + assert ready_fold["filtered_state_available_at_origin"] is True + assert ready_fold["smoothed_state_retrospective"] is True + + +def test_projection_preserves_identity_and_serializes_cache_and_influx( + tmp_path: Path, +) -> None: + """The augmented row contract survives duplicates, IPC, and Influx.""" + raw = _process_frame(420, duplicate_timestamp=True) + target = _target("DAT_ASCII_EURUSD_T_201202.csv") + result = _run(raw, target=target) + projected = project_state_space_onto_training_frame( + raw, result, target=target + ) + + assert ( + projected.select("datetime", "bid", "ask").to_dicts() + == raw.select("datetime", "bid", "ask").to_dicts() + ) + assert projected.get_column("row_id").n_unique() == raw.height + assert set((*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS)) <= set( + projected.columns + ) + available = projected.filter(pl.col("cm_state_space_forecast_available")) + assert available.height > 0 + assert available.get_column("cm_state_space_training_eligible").all() + line = format_influx_line( + "eurusd", "ascii", TICK, available.row(0), columns=available.columns + ) + assert "cm_state_space_forecast_available=true" in line + + masked = projected.with_columns( + pl.when(pl.col("row_id") == 7) + .then(None) + .otherwise(pl.col("timestamp_utc_ms")) + .alias("timestamp_utc_ms") + ) + masked_projection = project_state_space_onto_training_frame(masked, result) + dropped_projection = project_state_space_onto_training_frame( + projected.drop("timestamp_utc_ms"), result + ) + expected_ids = list(range(1, raw.height + 1)) + assert masked_projection.get_column("row_id").to_list() == expected_ids + assert dropped_projection.get_column("row_id").to_list() == expected_ids + + cache = tmp_path / CACHE_FILENAME + write_polars_cache(projected, cache) + restored = read_polars_cache(cache) + columns = (*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS) + assert ( + restored.select(columns).to_dicts() + == projected.select(columns).to_dicts() + ) + + +def test_columns_discovery_reports_and_comparisons_cover_the_full_contract() -> ( + None +): + """Registry and every report surface expose the same implemented family.""" + definitions = {row.name: row for row in training_feature_definitions()} + assert len(STATE_SPACE_COLUMNS) == 32 + assert len(KALMAN_COLUMNS) == 20 + assert set((*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS)) <= set(definitions) + assert all( + definitions[name].nullable + for name in (*STATE_SPACE_COLUMNS, *KALMAN_COLUMNS) + ) + assert "state_space" in FINGERPRINT_AUDIT_SECTIONS + assert "state_space" in implemented_fingerprint_target_section_names() + + reference = { + "evaluation": { + "models": [ + { + "status": "ready", + "family": "reference", + "specification_id": "reference", + "model_id": "sha256:reference", + "horizon_metrics": [{"horizon": 1, "mae": 0.1}], + } + ] + } + } + result = _run( + _process_frame(360), + exponential_smoothing=reference, + autoregressive=reference, + seasonal_exogenous=reference, + ) + references = _mapping( + _mapping(result.diagnostics["evaluation"])["reference_models"] + ) + for family in ( + "exponential_smoothing", + "autoregressive", + "seasonal_exogenous", + ): + row = _mapping_rows(references[family])[0] + assert row["model_id"] == "sha256:reference" + assert row["automatic_winner"] is False + + finding = _finding( + _target("DAT_ASCII_EURUSD_T_201202.csv"), result.diagnostics + ) + report = QualityReport( + targets=(finding.target,), + rule_results=( + QualityRuleResult( + rule_id=finding.rule_id, + target=finding.target, + findings=(finding,), + ), + ), + ) + full = quality_report_payload(report) + bounded = bounded_quality_payload( + report=report, + operation="data-quality", + check_groups=("fingerprint",), + discovery={"targets": []}, + artifact=None, + decision=QualityExitPolicy().evaluate(report.summary()), + ) + assert STATE_SPACE_SUMMARY_METADATA_KEY in _mapping(full["metadata"]) + assert STATE_SPACE_BOUNDED_PAYLOAD_KEY in bounded + assert "State-space and Kalman models" in format_quality_console_summary( + report + ) + + +def test_bounded_summary_matches_golden() -> None: + """Target sorting, statuses, and truncation remain deterministic.""" + findings = [] + for symbol, status, failed in ( + ("GBPUSD", "limited", 1), + ("AUDUSD", "ready", 0), + ("EURUSD", "ready", 0), + ): + target = _target(f"DAT_ASCII_{symbol}_T_201201.csv", symbol=symbol) + findings.append( + _finding( + target, + { + "status": status, + "reason": "optimizer_failure" if failed else None, + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": symbol, + "period": "201201", + "kind": "csv", + }, + "fit_summary": { + "fit_attempt_count": 6, + "failed_fit_count": failed, + }, + "evaluation": { + "model_count": 3, + "evaluated_fold_count": 5 if failed else 6, + }, + }, + ) + ) + summary = state_space_summary(findings, target_limit=1) + expected = json.loads( + ( + Path(__file__).parents[1] + / "fixtures" + / "data_quality_reports" + / "state_space_summary.json" + ).read_text(encoding="utf-8") + ) + assert summary == expected + assert _mapping(cast(Mapping[str, Any], summary))["schema_version"] == ( + STATE_SPACE_SUMMARY_SCHEMA_VERSION + ) + + +def test_fingerprint_rule_runs_opt_in_state_space_on_supported_csv( + tmp_path: Path, +) -> None: + """The ordinary fingerprint lifecycle emits and audits the family.""" + source = tmp_path / "DAT_ASCII_EURUSD_T_201201.csv" + source.write_text("\n".join(_tick_lines(360)) + "\n", encoding="ascii") + profile = HistDataFingerprintProfile( + classical_model_input=_input_profile(), + state_space=_profile(), + ) + finding = HistDataSeriesFingerprintRule(profile=profile).evaluate( + _target(str(source)) + )[0] + fingerprint = _mapping( + finding.metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + audit = _mapping(fingerprint["fingerprint_audit"]) + assert _mapping(fingerprint["state_space"])["status"] in { + "ready", + "limited", + } + assert "state_space" in audit["sections_expected"] + assert "state_space" in audit["sections_emitted"] + + +def _run( + frame: pl.DataFrame, + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: StateSpaceProfile | None = None, + target: QualityTarget | None = None, + exponential_smoothing: Mapping[str, Any] | None = None, + autoregressive: Mapping[str, Any] | None = None, + seasonal_exogenous: Mapping[str, Any] | None = None, +) -> Any: + return state_space_from_training_frame( + frame, + _fingerprint(), + input_profile=input_profile or _input_profile(), + profile=profile or _profile(), + target=target or _target("DAT_ASCII_EURUSD_T_201202.csv"), + exponential_smoothing=exponential_smoothing, + autoregressive=autoregressive, + seasonal_exogenous=seasonal_exogenous, + ) + + +def _profile(*, seasonal_cycle_ms: int = 240_000) -> StateSpaceProfile: + return StateSpaceProfile( + enabled=True, + specifications=( + StateSpaceSpecification("local-level", "local_level"), + StateSpaceSpecification( + "local-linear-trend", + "local_linear_trend", + stochastic_trend=True, + ), + StateSpaceSpecification( + "structural-seasonal", + "structural", + seasonal_period=4, + seasonal_cycle_ms=seasonal_cycle_ms, + ), + ), + projection_specification_id="local-level", + ) + + +def _input_profile(**overrides: Any) -> ClassicalModelInputProfile: + values: dict[str, Any] = { + "enabled": True, + "frequency_ms": 60_000, + "minimum_training_observations": 24, + "minimum_evaluation_observations": 1, + "step_size": 16, + "horizons": (1, 2), + "resources": ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=64, + max_horizons=4, + max_candidate_orders=16, + max_fit_attempts=32, + max_wall_time_seconds=30, + max_retained_diagnostics=64, + ), + } + values.update(overrides) + return ClassicalModelInputProfile(**values) + + +def _process_frame( + count: int, *, duplicate_timestamp: bool = False +) -> pl.DataFrame: + base = 1_325_376_000_000 + times = [base + index * 10_000 for index in range(count)] + if duplicate_timestamp and count > 1: + times[1] = times[0] + prices = [ + 1.1 + + index * 0.000003 + + ((index // 6) % 4 - 1.5) * 0.00004 + + ((index * 17) % 11 - 5) * 0.000002 + for index in range(count) + ] + return pl.DataFrame( + { + "datetime": times, + "bid": prices, + "ask": [value + 0.0002 for value in prices], + "vol": [0] * count, + } + ) + + +def _tick_lines(count: int) -> tuple[str, ...]: + start = datetime(2012, 1, 1) + return tuple( + ( + f"{(start + timedelta(seconds=index * 10)).strftime('%Y%m%d %H%M%S')}000," + f"{1.1 + index * 0.000003:.6f}," + f"{1.1002 + index * 0.000003:.6f},0" + ) + for index in range(count) + ) + + +def _fingerprint() -> dict[str, Any]: + return { + "fingerprint_id": "fingerprint-eurusd", + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": "EURUSD", + "period": "201201", + "kind": "cache", + }, + "fingerprint_audit": { + "section_statuses": { + "dependence": "valid", + "stationarity_diagnostics": "limited", + } + }, + } + + +def _target(path: str, *, symbol: str = "EURUSD") -> QualityTarget: + return QualityTarget( + path=path, + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period="201201", + ) + + +def _finding( + target: QualityTarget, diagnostics: Mapping[str, Any] +) -> QualityFinding: + return QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id="fingerprint.series", + target=target, + metadata={ + TIME_SERIES_FINGERPRINT_METADATA_KEY: { + "state_space": dict(diagnostics) + } + }, + ) + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(cast(Mapping[str, Any], value)) + + +def _mapping_rows(value: Any) -> list[dict[str, Any]]: + return [_mapping(row) for row in cast(list[Any], value)] + + +def _outcome( + *, + status: str = "converged", + reason: str = "", + state_names: tuple[str, ...], + filtered_state: tuple[float | None, ...] = (), + filtered_variance: tuple[float | None, ...] = (), + smoothed_state: tuple[float | None, ...] = (), + smoothed_variance: tuple[float | None, ...] = (), +) -> Any: + return state_module._FitOutcome( + status=status, + reason=reason, + forecasts=(1.0,) if status == "converged" else (), + standard_errors=(0.1,) if status == "converged" else (), + lower_bounds=(0.8,) if status == "converged" else (), + upper_bounds=(1.2,) if status == "converged" else (), + parameters={}, + warning_codes=(), + converged=status == "converged", + state_dimension=len(state_names), + state_names=state_names, + filtered_state=filtered_state, + filtered_variance=filtered_variance, + smoothed_state=smoothed_state, + smoothed_variance=smoothed_variance, + effective_observation_count=24, + missing_observation_count=0, + prediction_only_transition_count=0, + max_prediction_only_gap=0, + log_likelihood=None, + aic=None, + bic=None, + covariance_condition_number=None, + innovation_summary={}, + ) diff --git a/tests/unit/test_data_quality_synthetic_constraints.py b/tests/unit/test_data_quality_synthetic_constraints.py new file mode 100644 index 00000000..5cd5123b --- /dev/null +++ b/tests/unit/test_data_quality_synthetic_constraints.py @@ -0,0 +1,465 @@ +"""Tests for generator-facing synthetic fingerprint constraints.""" + +from __future__ import annotations + +from copy import deepcopy +import json +import os +from pathlib import Path + +import polars as pl + +from histdatacom.data_quality import ( + SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION, + SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION, + SYNTHETIC_VALIDATION_SCHEMA_VERSION, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + QualityFinding, + QualityReport, + QualityRuleResult, + QualitySeverity, + QualityTarget, + QualityTargetKind, + synthetic_constraint_summary, + synthetic_constraints_from_fingerprint, + synthetic_constraints_from_training_frame, + validate_synthetic_constraint_reports, +) +from histdatacom.data_quality.training_features import ( + SYNTHETIC_PLACEHOLDER_COLUMNS, + TRAINING_SCHEMA_VERSION, + ensure_tick_training_features, +) +from histdatacom.histdata_ascii import ( + TICK, + format_influx_line, + parse_ascii_lines, + to_polars_frame, +) +from tests.fixtures.histdata_ascii.quality_cases import CLEAN_TICK_ROWS + + +def test_constraints_derive_categories_from_enriched_tick_rows() -> None: + """The protocol should expose bounded row-first generator contracts.""" + target = _target() + frame = ensure_tick_training_features(_clean_frame(), target=target) + fingerprint = _fingerprint_stub() + + first = synthetic_constraints_from_training_frame( + frame, + fingerprint=fingerprint, + target=target, + ) + second = synthetic_constraints_from_training_frame( + frame, + fingerprint=fingerprint, + target=target, + ) + + assert first["schema_version"] == SYNTHETIC_CONSTRAINTS_SCHEMA_VERSION + assert first["status"] == "ready" + assert first["constraint_id"] == second["constraint_id"] + training = _mapping(first["training_substrate"]) + assert training["training_schema_version"] == TRAINING_SCHEMA_VERSION + assert training["identity_columns_present"] is True + assert training["synthetic_output_columns_present"] is True + output = _mapping(first["output_contract"]) + assert output["observed_columns_preserved"] == ["bid", "ask"] + assert output["synthetic_output_columns"] == list( + SYNTHETIC_PLACEHOLDER_COLUMNS + ) + assert output["durable_identity_columns"] == [ + "series_id", + "period", + "row_id", + ] + assert output["timestamp_is_sole_identity"] is False + assert output["non_tick_input_constraints_supported"] is False + assert output["generation_in_scope"] is True + assert output["generation_issue"] == "#81" + assert output["generation_method"] == "empirical_block_bootstrap" + assert { + item["code"] for item in _mapping_rows(first["defects_to_avoid"]) + } >= { + "avoid_negative_spread", + "avoid_duplicate_timestamps", + "avoid_non_monotonic_timestamps", + "avoid_suspicious_non_session_gaps", + "avoid_invalid_rows", + "avoid_partial_rows", + "avoid_topology_unavailable", + "avoid_fingerprint_unready", + "avoid_unsupported_schema", + "avoid_structurally_invalid_timestamps", + } + assert { + item["code"] + for item in _mapping_rows(first["stylized_facts_to_preserve"]) + } >= { + "session_activity_mix", + "gap_bucket_shape", + "spread_distribution", + "precision_regime", + "stale_quote_runs", + "volatility_clustering_proxy", + "rolling_drift", + "stationarity_transform_policy", + } + assert "write_only_synth_columns" in first["advisory_hints"] + assert "preserve_row_identity" in first["advisory_hints"] + assert "raw_m1_ohlc_constraints_deferred" not in first["limitation_codes"] + assert "bidquote" not in json.dumps(first, sort_keys=True) + + +def test_constraints_use_explicit_row_issue_columns_for_defects() -> None: + """Row issue columns should drive generator defect observations.""" + rows = ( + "20120201 000000000,1.200000,1.100000,0", + "20120201 000000000,1.200000,1.100000,0", + "20120131 235959000,1.200000,1.200200,0", + ) + target = _target() + frame = ensure_tick_training_features( + to_polars_frame(parse_ascii_lines(TICK, rows)), + target=target, + ) + assert frame.get_column("row_id").to_list() == [1, 2, 3] + assert frame.get_column("timestamp_utc_ms").n_unique() == 2 + + payload = synthetic_constraints_from_fingerprint( + _fingerprint_stub(), + training_frame=frame, + target=target, + ) + defects = { + item["code"]: item + for item in _mapping_rows(payload["defects_to_avoid"]) + } + + assert defects["avoid_negative_spread"]["observed_count"] == 2 + assert defects["avoid_duplicate_timestamps"]["observed_count"] == 2 + assert defects["avoid_non_monotonic_timestamps"]["observed_count"] == 1 + assert all( + item["source"] == "training_feature_column" + for item in defects.values() + if item.get("issue_column") + ) + + +def test_constraints_are_bounded_and_publish_safe(tmp_path: Path) -> None: + """Constraint categories should truncate without publishing local paths.""" + target = _target(path=str(tmp_path / "secret" / ".data")) + payload = synthetic_constraints_from_fingerprint( + _fingerprint_stub(), + training_frame=_clean_frame(), + target=target, + category_limit=1, + hint_limit=1, + ) + + assert payload["included_defect_count"] == 1 + assert payload["included_stylized_fact_count"] == 1 + assert payload["included_source_artifact_count"] == 1 + assert payload["included_hint_count"] == 1 + assert payload["truncated"] is True + assert str(tmp_path) not in json.dumps(payload, sort_keys=True) + + +def test_validation_matches_and_reports_stable_drift_codes() -> None: + """Candidate fingerprint drift should yield deterministic mismatch codes.""" + reference_constraints = synthetic_constraints_from_fingerprint( + _fingerprint_stub(), + training_frame=_clean_frame(), + target=_target(), + ) + matching = _report(reference_constraints) + + matched = validate_synthetic_constraint_reports(matching, matching) + + assert matched["schema_version"] == SYNTHETIC_VALIDATION_SCHEMA_VERSION + assert matched["status"] == "match" + assert matched["matching_target_count"] == 1 + + candidate_constraints = deepcopy(reference_constraints) + defects = _mapping_rows(candidate_constraints["defects_to_avoid"]) + defects[0]["observed_count"] = 2 + facts = _mapping_rows(candidate_constraints["stylized_facts_to_preserve"]) + spread = next( + item for item in facts if item["code"] == "spread_distribution" + ) + _mapping(_mapping(spread["value"])["quantiles"])["0.5"] = 0.01 + mismatched = validate_synthetic_constraint_reports( + matching, + _report(candidate_constraints), + ) + codes = set( + _mapping_rows(mismatched["target_results"])[0]["mismatch_codes"] + ) + + assert mismatched["status"] == "mismatch" + assert "synthetic_candidate_avoid_negative_spread_present" in codes + assert "synthetic_candidate_spread_distribution_mismatch" in codes + + +def test_validation_reports_missing_candidate_and_bounds_targets() -> None: + """Missing candidate axes and target limits should remain explicit.""" + constraints = synthetic_constraints_from_fingerprint( + _fingerprint_stub(), + training_frame=_clean_frame(), + target=_target(), + ) + + payload = validate_synthetic_constraint_reports( + _report(constraints), + QualityReport(), + target_limit=0, + mismatch_limit=0, + ) + + assert payload["status"] == "not_compared" + assert payload["not_compared_target_count"] == 1 + assert payload["included_target_count"] == 0 + assert payload["omitted_target_count"] == 1 + assert payload["truncated"] is True + + +def test_constraint_summary_is_bounded_and_issue_visible() -> None: + """Reports should expose constraints without nested finding inspection.""" + constraints = synthetic_constraints_from_fingerprint( + _fingerprint_stub(), + training_frame=_clean_frame(), + target=_target(), + ) + defects = _mapping_rows(constraints["defects_to_avoid"]) + defects[0]["observed_count"] = 1 + + summary = synthetic_constraint_summary( + _report(constraints).findings, + target_limit=1, + ) + + assert summary is not None + assert summary["schema_version"] == ( + SYNTHETIC_CONSTRAINT_SUMMARY_SCHEMA_VERSION + ) + assert summary["target_count"] == 1 + assert summary["observed_defect_target_counts"] == { + "avoid_negative_spread": 1 + } + + +def test_populated_synthetic_columns_share_the_enriched_influx_point() -> None: + """Influx projection should not regress to market-only candidate points.""" + target = _target() + frame = ensure_tick_training_features(_clean_frame(), target=target) + frame = frame.with_columns( + [ + pl.col("bid").add(0.01).alias("synth_bid"), + pl.col("ask").add(0.01).alias("synth_ask"), + pl.col("spread").alias("synth_spread"), + pl.col("mid").add(0.01).alias("synth_mid"), + pl.lit(1).cast(pl.Int32).alias("synth_method_code"), + pl.lit(0.9).alias("synth_confidence"), + pl.lit(True).alias("synth_usable"), + ] + ) + line = format_influx_line( + "EURUSD", + "ascii", + TICK, + frame.row(0), + columns=frame.columns, + ) + + assert frame.get_column("bid")[0] == 1.3066 + assert frame.get_column("ask")[0] == 1.30677 + assert "bidquote=1.3066" in line + assert "askquote=1.30677" in line + assert "synth_bid=1.3166" in line + assert "synth_ask=1.31677" in line + assert "synth_method_code=1i" in line + assert "synth_usable=true" in line + assert line.count(" ") == 2 + + +def test_synthetic_validation_matches_golden_fixture() -> None: + """Topology and defect validation output should remain golden-testable.""" + reference_constraints = synthetic_constraints_from_fingerprint( + _fingerprint_stub(), + training_frame=_clean_frame(), + target=_target(), + ) + candidate_constraints = deepcopy(reference_constraints) + defects = _mapping_rows(candidate_constraints["defects_to_avoid"]) + next( + item for item in defects if item["code"] == "avoid_duplicate_timestamps" + )["observed_count"] = 2 + facts = _mapping_rows(candidate_constraints["stylized_facts_to_preserve"]) + gap_shape = next( + item for item in facts if item["code"] == "gap_bucket_shape" + ) + _mapping(gap_shape["value"])["gt_5m"] = 3 + payload = validate_synthetic_constraint_reports( + _report(reference_constraints), + _report(candidate_constraints), + ) + expected = json.dumps(payload, indent=2, sort_keys=True) + "\n" + fixture = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "data_quality_reports" + / "synthetic_validation.json" + ) + if os.environ.get("HISTDATACOM_UPDATE_QUALITY_GOLDENS") == "1": + fixture.write_text(expected, encoding="utf-8") + + assert fixture.read_text(encoding="utf-8") == expected + + +def _clean_frame() -> pl.DataFrame: + return to_polars_frame(parse_ascii_lines(TICK, CLEAN_TICK_ROWS)) + + +def _target(path: str = ".data") -> QualityTarget: + return QualityTarget( + path=path, + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + + +def _fingerprint_stub() -> dict: + return { + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": "EURUSD", + "period": "201202", + "kind": "cache", + }, + "source": {"kind": "cache", "cache_source": "direct", "path": ".data"}, + "coverage": { + "row_count": 3, + "parsed_row_count": 3, + "start_timestamp_utc_ms": 1, + "end_timestamp_utc_ms": 3, + }, + "temporal_topology": { + "computed_from": "direct_cache", + "cache_source": "direct", + "sampling_basis": "observed_sequence", + "invalid_timestamp_count": 0, + "duplicate_timestamp_count": 0, + "non_monotonic_count": 0, + "suspicious_gap_count": 0, + "expected_session_closure_count": 0, + "weekend_activity_count": 0, + "min_interval_ms": 313, + "median_interval_ms": 5665, + "max_gap_ms": 11017, + "gap_bucket_counts": {"gt_1m": 0, "gt_5m": 0}, + }, + "calendar_regimes": { + "session_state_counts": {"market_open": 3}, + "active_session_counts": {"asia": 3}, + "special_tag_counts": {}, + "calendar_policy": { + "calendar_profile": { + "name": "test-profile", + "complete": True, + } + }, + }, + "tick_distribution": { + "negative_spread_count": 0, + "invalid_row_count": 0, + "partial_row_count": 0, + "spread": { + "count": 3, + "min": 0.00017, + "max": 0.00017, + "mean": 0.00017, + "median": 0.00017, + "mad": 0.0, + "quantiles": {"0.5": 0.00017}, + }, + }, + "microstructure_dynamics": { + "spread_jump": {"count": 0, "rate": 0.0}, + "stale_quote": {"run_count": 0, "repeat_rate": 0.0}, + "burst": {"run_count": 0, "burst_rate": 0.0}, + "one_sided_movement": {"count": 0, "rate": 0.0}, + "limitations": [], + }, + "dependence": {"absolute_spread_change_acf": {"lag_acf": {"1": 0.2}}}, + "stationarity_diagnostics": { + "first_middle_last_distribution_shift": { + "return": {"status": "computed", "median_shift": 0.0} + }, + "rolling_windows": { + "2": {"status": "computed", "absolute_change": 0.0} + }, + "recommended_transforms": ["log_return"], + "limitations": [], + "zero_variance_metrics": [], + "skipped_window_reason_counts": {}, + }, + "decomposition": { + "structural_break_proxy": { + "status": "computed", + "candidate_count": 0, + } + }, + "fingerprint_audit": { + "section_statuses": { + "coverage": "valid", + "temporal_topology": "valid", + "calendar_regimes": "valid", + "tick_distribution": "valid", + "microstructure_dynamics": "valid", + "dependence": "valid", + "stationarity_diagnostics": "valid", + "decomposition": "valid", + } + }, + } + + +def _report(constraints: dict) -> QualityReport: + target = _target() + finding = QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Synthetic constraint test fingerprint.", + rule_id="fingerprint.series", + target=target, + metadata={ + TIME_SERIES_FINGERPRINT_METADATA_KEY: { + "target_axis": constraints["target_axis"], + "synthetic_constraints": constraints, + } + }, + ) + return QualityReport( + targets=(target,), + rule_results=( + QualityRuleResult( + rule_id="fingerprint.series", + target=target, + findings=(finding,), + ), + ), + ) + + +def _mapping(value: object) -> dict: + return value if isinstance(value, dict) else {} + + +def _mapping_rows(value: object) -> list[dict]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, dict)] diff --git a/tests/unit/test_data_quality_synthetic_generation.py b/tests/unit/test_data_quality_synthetic_generation.py new file mode 100644 index 00000000..dee0ea91 --- /dev/null +++ b/tests/unit/test_data_quality_synthetic_generation.py @@ -0,0 +1,356 @@ +"""Tests for deterministic reference-set synthetic tick generation.""" + +from __future__ import annotations + +from collections.abc import Mapping +import json +import os +from pathlib import Path +from typing import Any, cast + +import polars as pl +import pytest + +from histdatacom.data_quality.contracts import ( + QualityReport, + QualityRuleResult, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprints import ( + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.synthetic_generation import ( + SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION, + SyntheticTickGenerationProfile, + format_synthetic_tick_generation, + generate_synthetic_ticks_from_reference, + reference_fingerprint_from_report, +) +from histdatacom.data_quality.training_features import ( + SYNTHETIC_PLACEHOLDER_COLUMNS, +) +from histdatacom.histdata_ascii import ( + TICK, + format_influx_line, + read_polars_cache, + write_polars_cache, +) + + +def test_generation_is_deterministic_seeded_and_fingerprint_validated( + tmp_path: Path, +) -> None: + frame, fingerprint, target, _ = _reference(tmp_path, count=80) + profile = _profile(seed=17) + + first = generate_synthetic_ticks_from_reference( + frame, fingerprint, profile=profile, target=target + ) + second = generate_synthetic_ticks_from_reference( + frame, fingerprint, profile=profile, target=target + ) + alternate = generate_synthetic_ticks_from_reference( + frame, + fingerprint, + profile=_profile(seed=18), + target=target, + ) + + assert first.diagnostics == second.diagnostics + assert ( + first.frame.select(SYNTHETIC_PLACEHOLDER_COLUMNS).to_dicts() + == second.frame.select(SYNTHETIC_PLACEHOLDER_COLUMNS).to_dicts() + ) + assert ( + first.frame.get_column("synth_mid").to_list() + != alternate.frame.get_column("synth_mid").to_list() + ) + assert first.diagnostics["schema_version"] == ( + SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION + ) + assert first.diagnostics["status"] == "ready" + application = _mapping(first.diagnostics["constraint_application"]) + assert _mapping(application["defects_to_avoid"])["count"] > 0 + assert _mapping(application["stylized_facts_to_preserve"])["count"] > 0 + assert ( + _mapping(application["source_artifacts_to_parameterize"])["count"] > 0 + ) + validation = _mapping(first.diagnostics["validation"]) + assert validation["same_fingerprint_path_used"] is True + assert validation["candidate_fingerprint_id"] + assert validation["hard_quality_gate"] is False + influx = _mapping(validation["same_point_influx_projection"]) + assert influx["status"] == "valid" + assert influx["same_measurement"] is True + assert first.candidate_report is not None + + expected = json.dumps(first.diagnostics, indent=2, sort_keys=True) + "\n" + fixture = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "data_quality_reports" + / "synthetic_tick_generation.json" + ) + if os.environ.get("HISTDATACOM_UPDATE_QUALITY_GOLDENS") == "1": + fixture.write_text(expected, encoding="utf-8") + assert fixture.read_text(encoding="utf-8") == expected + + +def test_generation_preserves_rows_observed_values_cache_and_influx( + tmp_path: Path, +) -> None: + frame, fingerprint, target, _ = _reference( + tmp_path, count=40, duplicate_timestamp=True + ) + observed = frame.select("datetime", "bid", "ask").to_dicts() + result = generate_synthetic_ticks_from_reference( + frame, + fingerprint, + profile=_profile(), + target=target, + ) + generated = result.frame + + assert generated.select("datetime", "bid", "ask").to_dicts() == observed + assert generated.height == frame.height + assert generated.get_column("row_id").n_unique() == frame.height + assert generated.head(2).get_column("datetime").n_unique() == 1 + assert generated.get_column("synth_usable").all() + assert ( + generated.get_column("synth_ask") >= generated.get_column("synth_bid") + ).all() + assert ( + generated.get_column("synth_spread") + == generated.get_column("synth_ask") - generated.get_column("synth_bid") + ).all() + + output = tmp_path / "generated" / "DAT_ASCII_EURUSD_T_201202.data" + output.parent.mkdir() + write_polars_cache(generated, output) + restored = read_polars_cache(output) + assert restored.select(SYNTHETIC_PLACEHOLDER_COLUMNS).to_dicts() == ( + generated.select(SYNTHETIC_PLACEHOLDER_COLUMNS).to_dicts() + ) + line = format_influx_line( + "EURUSD", + "ascii", + TICK, + restored.row(0), + columns=restored.columns, + ) + assert "synth_bid=" in line + assert "synth_ask=" in line + assert "synth_method_code=1i" in line + assert "synth_usable=true" in line + assert "observed bid/ask preserved: yes" in ( + format_synthetic_tick_generation(result.diagnostics) + ) + + +def test_resource_bounds_existing_values_and_insufficient_evidence( + tmp_path: Path, +) -> None: + frame, fingerprint, target, _ = _reference(tmp_path, count=40) + limited = generate_synthetic_ticks_from_reference( + frame, + fingerprint, + profile=_profile(max_generated_rows=10), + target=target, + ) + generation = _mapping(limited.diagnostics["generation"]) + + assert limited.diagnostics["status"] == "limited" + assert limited.diagnostics["reason"] == "generation_row_limit" + assert generation["generated_row_count"] == 10 + assert generation["omitted_row_count"] == 30 + assert limited.frame.head(10).get_column("synth_usable").all() + assert not limited.frame.tail(30).get_column("synth_usable").any() + assert limited.frame.tail(30).get_column("synth_bid").null_count() == 30 + + with pytest.raises(ValueError, match="existing_synthetic_values"): + generate_synthetic_ticks_from_reference( + limited.frame, + fingerprint, + profile=_profile(), + target=target, + ) + overwritten = generate_synthetic_ticks_from_reference( + limited.frame, + fingerprint, + profile=_profile( + overwrite_existing=True, + ), + target=target, + ) + assert overwritten.diagnostics["status"] == "ready" + + insufficient = generate_synthetic_ticks_from_reference( + frame.head(7), + fingerprint, + profile=_profile( + minimum_reference_rows=10, + ), + target=target, + ) + assert insufficient.diagnostics["status"] == "unavailable" + assert insufficient.diagnostics["reason"] == ("insufficient_reference_rows") + + +def test_defective_reference_rows_are_filtered_not_reproduced( + tmp_path: Path, +) -> None: + frame = _raw_frame(60).with_columns( + pl.when(pl.int_range(pl.len()) == 10) + .then(pl.col("bid") - 0.01) + .otherwise(pl.col("ask")) + .alias("ask") + ) + frame, fingerprint, target, _ = _reference(tmp_path, frame=frame) + result = generate_synthetic_ticks_from_reference( + frame, + fingerprint, + profile=_profile(), + target=target, + ) + evidence = _mapping(result.diagnostics["reference_evidence"]) + + assert evidence["filtered_reference_row_count"] >= 1 + assert evidence["defective_rows_used"] is False + assert (result.frame.get_column("synth_spread") >= 0).all() + assert (result.frame.get_column("synth_bid") > 0).all() + + +def test_report_selection_profile_guards_and_axis_requirements( + tmp_path: Path, +) -> None: + frame, fingerprint, target, report = _reference(tmp_path, count=30) + assert reference_fingerprint_from_report(report, target=target) == ( + fingerprint + ) + with pytest.raises(ValueError, match="reference_fingerprint_required"): + reference_fingerprint_from_report(QualityReport(), target=target) + with pytest.raises(ValueError, match="block_size"): + SyntheticTickGenerationProfile(block_size=0) + with pytest.raises(ValueError, match="max_reference_rows"): + SyntheticTickGenerationProfile( + minimum_reference_rows=20, + max_reference_rows=10, + ) + with pytest.raises( + ValueError, match="unsupported_reference_fingerprint_schema" + ): + generate_synthetic_ticks_from_reference( + frame, + {**fingerprint, "schema_version": "future.v2"}, + profile=_profile(), + target=target, + ) + with pytest.raises(ValueError, match="reference_target_axis_mismatch"): + generate_synthetic_ticks_from_reference( + frame, + fingerprint, + profile=_profile(), + target=QualityTarget( + path=target.path, + kind=target.kind, + data_format=target.data_format, + timeframe=target.timeframe, + symbol="GBPUSD", + period=target.period, + ), + ) + with pytest.raises(ValueError, match="unsupported_base_grain"): + generate_synthetic_ticks_from_reference( + frame, + { + **fingerprint, + "target_axis": { + **_mapping(fingerprint["target_axis"]), + "timeframe": "M1", + }, + }, + profile=_profile(), + target=target, + ) + + +def _reference( + tmp_path: Path, + *, + count: int = 80, + duplicate_timestamp: bool = False, + frame: pl.DataFrame | None = None, +) -> tuple[pl.DataFrame, dict[str, Any], QualityTarget, QualityReport]: + raw = ( + frame + if frame is not None + else _raw_frame(count, duplicate_timestamp=duplicate_timestamp) + ) + cache = tmp_path / "reference" / "DAT_ASCII_EURUSD_T_201202.data" + cache.parent.mkdir(exist_ok=True) + write_polars_cache(raw, cache) + target = QualityTarget( + path=str(cache), + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe=TICK, + symbol="EURUSD", + period="201202", + ) + [finding] = HistDataSeriesFingerprintRule().evaluate(target) + fingerprint = _mapping( + finding.metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + report = QualityReport( + targets=(target,), + rule_results=( + QualityRuleResult( + rule_id=finding.rule_id, + target=target, + findings=(finding,), + ), + ), + ) + return raw, fingerprint, target, report + + +def _raw_frame( + count: int, + *, + duplicate_timestamp: bool = False, +) -> pl.DataFrame: + timestamps = [1_325_376_000_000 + index * 1_000 for index in range(count)] + if duplicate_timestamp and count > 1: + timestamps[1] = timestamps[0] + mids = [ + 1.2 + index * 0.00001 + ((index * 17) % 11 - 5) * 0.000004 + for index in range(count) + ] + spreads = [0.0001 + (index % 4) * 0.00001 for index in range(count)] + return pl.DataFrame( + { + "datetime": timestamps, + "bid": [mid - spread / 2 for mid, spread in zip(mids, spreads)], + "ask": [mid + spread / 2 for mid, spread in zip(mids, spreads)], + "vol": [0] * count, + } + ) + + +def _profile(**overrides: Any) -> SyntheticTickGenerationProfile: + values: dict[str, Any] = { + "block_size": 4, + "minimum_reference_rows": 8, + "max_reference_rows": 1_000, + "max_generated_rows": 1_000, + "rounding_digits": 8, + "diagnostic_sample_limit": 4, + } + values.update(overrides) + return SyntheticTickGenerationProfile(**values) + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(cast(Mapping[str, Any], value)) diff --git a/tests/unit/test_data_quality_time.py b/tests/unit/test_data_quality_time.py index 87e37b93..f7bd02bb 100644 --- a/tests/unit/test_data_quality_time.py +++ b/tests/unit/test_data_quality_time.py @@ -15,6 +15,8 @@ HistDataGapTolerance, QualitySeverity, QualityStatus, + QualityTarget, + QualityTargetKind, discover_quality_targets, quality_run_rules_for_groups, quality_rules_for_groups, @@ -50,6 +52,22 @@ def test_time_group_registers_continuity_run_rule() -> None: ] == [ASCII_TIMESTAMP_CONTINUITY_RULE_ID] +def test_time_rules_noop_unsupported_raw_dimensions() -> None: + """Timestamp rules must remain limited to ASCII tick targets.""" + rules = quality_rules_for_groups(("time",)) + for data_format, timeframe in ( + ("ascii", "M1"), + ("metatrader", "T"), + ): + target = QualityTarget( + path="/missing/retired.csv", + kind=QualityTargetKind.CSV, + data_format=data_format, + timeframe=timeframe, + ) + assert all(rule.evaluate(target) == () for rule in rules) + + def test_clean_ascii_file_reports_est_no_dst_conversion_summary( tmp_path: Path, ) -> None: @@ -491,6 +509,158 @@ def test_unexpected_weekend_activity_warns_without_hard_failure( assert summary.metadata["weekend_activity_count"] == 1 +def test_timestamp_topology_inspection_context_is_bounded_and_publish_safe( + tmp_path: Path, +) -> None: + """Topology evidence should expose row context without raw rows or paths.""" + path = tmp_path / "DAT_ASCII_EURUSD_T_201202_INSPECTION.csv" + path.write_text( + "\n".join( + ( + _tick_row("20120201 000000000"), + "not-a-timestamp,1.306600,1.306770,0", + _tick_row("20120201 000000000"), + _tick_row("20120201 001000000", volume=1), + _tick_row("20120201 000500000", volume=2), + _tick_row("20120204 120000000", volume=3), + ) + ) + + "\n", + encoding="utf-8", + ) + target = discover_quality_targets((path,)).targets[0] + + payload = time_quality.timestamp_topology_payload_for_target( + target, + inspection_sample_limit=1, + ) + context = payload["inspection_context"] + + invalid = context["invalid_timestamps"] + assert invalid["parse_failure_count"] == 1 + assert invalid["samples"] == [ + { + "row_number": 2, + "timestamp_source": "not-a-timestamp", + "timestamp_source_truncated": False, + "timestamp_utc_ms": None, + "utc_timestamp": "", + "metadata": {"reason_code": "timestamp_parse_failure"}, + } + ] + transition = context["non_monotonic_timestamps"]["samples"][0] + assert transition["row_number"] == 5 + assert transition["metadata"]["previous_row_number"] == 4 + duplicate = context["duplicate_timestamps"] + assert duplicate["duplicate_row_count"] == 1 + assert duplicate["samples"][0]["timestamp_source"] == ("20120201 000000000") + assert duplicate["samples"][0]["occurrence_count"] == 2 + assert ( + context["suspicious_gaps"]["samples"][0]["expected_session_related"] + is False + ) + weekend = context["weekend_activity"] + assert weekend["session_bucket_counts"] == {"weekend_closure": 1} + assert weekend["samples"][0]["session_state"] == "weekend_closure" + assert str(tmp_path) not in str(context) + assert "duplicate_row_values" not in str(context) + + +def test_timestamp_topology_inspection_ranks_largest_gaps_and_truncates( + tmp_path: Path, +) -> None: + """Gap drill-downs should rank largest boundaries with explicit limits.""" + path = tmp_path / "DAT_ASCII_EURUSD_T_201202_RANKED_GAPS.csv" + path.write_text( + "\n".join( + ( + _tick_row("20120201 000000000"), + _tick_row("20120201 001000000", volume=1), + _tick_row("20120201 003000000", volume=2), + _tick_row("20120201 004500000", volume=3), + ) + ) + + "\n", + encoding="utf-8", + ) + target = discover_quality_targets((path,)).targets[0] + + payload = time_quality.timestamp_topology_payload_for_target( + target, + inspection_sample_limit=2, + ) + gaps = payload["inspection_context"]["suspicious_gaps"] + + assert [sample["gap_ms"] for sample in gaps["samples"]] == [ + 1_200_000, + 900_000, + ] + assert gaps["total_count"] == 3 + assert gaps["included_count"] == 2 + assert gaps["omitted_count"] == 1 + assert gaps["truncated"] is True + assert gaps["limit_metadata"]["samples"] == { + "limit": 2, + "effective_limit": 2, + "requested_limit": 2, + "default_limit": 5, + "minimum_limit": 0, + "maximum_limit": 5, + "unbounded": False, + "total_count": 3, + "included_count": 2, + "omitted_count": 1, + "truncated": True, + } + + +def test_timestamp_topology_inspection_truncates_oversized_timestamp_text( + tmp_path: Path, +) -> None: + """Malformed timestamp cells must not become large raw-row excerpts.""" + oversized = "sensitive-looking-value-" + ("x" * 120) + path = tmp_path / "DAT_ASCII_EURUSD_T_201202_LONG_INVALID.csv" + path.write_text( + f"{oversized},1.306600,1.306770,0\n", + encoding="utf-8", + ) + target = discover_quality_targets((path,)).targets[0] + + payload = time_quality.timestamp_topology_payload_for_target(target) + sample = payload["inspection_context"]["invalid_timestamps"]["samples"][0] + + assert sample["timestamp_source_truncated"] is True + assert len(sample["timestamp_source"]) == 80 + assert oversized not in str(payload["inspection_context"]) + + +def test_timestamp_topology_expected_closure_context_is_non_actionable( + tmp_path: Path, +) -> None: + """Expected market closures should remain diagnostic context only.""" + path = tmp_path / "DAT_ASCII_EURUSD_T_201202_EXPECTED_CONTEXT.csv" + path.write_text( + "\n".join( + ( + _tick_row("20120203 165900000"), + _tick_row("20120205 170100000", volume=17), + ) + ) + + "\n", + encoding="utf-8", + ) + target = discover_quality_targets((path,)).targets[0] + + payload = time_quality.timestamp_topology_payload_for_target(target) + closure = payload["inspection_context"]["expected_session_closures"] + + assert closure["actionable"] is False + assert closure["samples"][0]["expected_session_related"] is True + assert closure["samples"][0]["classification"] == ( + "expected_session_closure" + ) + + def test_gap_tolerance_windows_are_adjustable( tmp_path: Path, ) -> None: diff --git a/tests/unit/test_data_quality_training_features.py b/tests/unit/test_data_quality_training_features.py new file mode 100644 index 00000000..24956277 --- /dev/null +++ b/tests/unit/test_data_quality_training_features.py @@ -0,0 +1,318 @@ +"""ASCII tick training feature substrate tests.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import polars as pl +import pytest + +from histdatacom.activity_stages import create_cache_file +from histdatacom.data_quality import ( + QualitySeverity, + QualityStatus, + QualityTarget, + QualityTargetKind, + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION, + decomposition_training_projection, + project_decomposition_onto_training_frame, +) +from histdatacom.data_quality.training_features import ( + DEFAULT_SUSPICIOUS_TICK_GAP_MS, + QUALITY_ISSUE_COLUMNS, + TRAINING_SCHEMA_VERSION, + enrich_tick_cache_with_training_features, + quality_report_from_training_features, + required_training_feature_columns, + training_feature_definitions, +) +from histdatacom.histdata_ascii import CACHE_FILENAME, read_polars_cache +from histdatacom.records import Record + +FIXTURES = Path(__file__).parents[1] / "fixtures" / "histdata_ascii" + + +def test_enriched_tick_frame_contains_flat_row_aligned_training_schema() -> ( + None +): + """Training rows should need no side joins or report parsing.""" + frame = _tick_frame( + [ + (1_000, 1.0000, 1.0002, 0), + (1_300, 1.1000, 1.1002, 1), + ] + ) + + enriched = enrich_tick_cache_with_training_features( + frame, + symbol="EURUSD", + data_format="ascii", + timeframe="T", + period="201202", + ) + rows = enriched.to_dicts() + + assert set(required_training_feature_columns()).issubset(enriched.columns) + assert enriched.height == frame.height + assert enriched.get_column("bid").to_list() == [1.0, 1.1] + assert enriched.get_column("ask").to_list() == [1.0002, 1.1002] + assert enriched.get_column("spread").to_list() == pytest.approx( + [0.0002, 0.0002] + ) + assert enriched.get_column("mid").to_list() == pytest.approx( + [1.0001, 1.1001] + ) + assert [row["row_id"] for row in rows] == [1, 2] + assert [row["source_row_number"] for row in rows] == [1, 2] + assert [row["event_seq"] for row in rows] == [1, 2] + assert {row["series_id"] for row in rows} == {"ascii:T:EURUSD:histdata.com"} + assert {row["period"] for row in rows} == {"201202"} + assert rows[0]["timestamp_utc_ms"] == 1_000 + assert rows[0]["training_schema_version"] == TRAINING_SCHEMA_VERSION + assert all(row["training_usable"] for row in rows) + assert all(row["class_training_action_code"] == 0 for row in rows) + assert all(row["synth_bid"] is None for row in rows) + assert all(row["synth_ask"] is None for row in rows) + + masked = enriched.with_columns( + pl.lit(None).cast(pl.Int64).alias("timestamp_utc_ms") + ) + assert masked.select(["series_id", "period", "row_id"]).to_dicts() == ( + enriched.select(["series_id", "period", "row_id"]).to_dicts() + ) + + +def test_decomposition_projection_preserves_enriched_tick_identity() -> None: + """Period decomposition scalars should project without a side join.""" + enriched = enrich_tick_cache_with_training_features( + _tick_frame( + [ + (1_000, 1.0, 1.1, 0), + (1_300, 1.1, 1.2, 1), + ] + ), + symbol="EURUSD", + data_format="ascii", + timeframe="T", + period="201202", + ) + decomposition = { + "decomposition_status": "ok", + "computed_window_count": 2, + "trend_proxy": { + "direction": "increasing", + "slope_per_observation": 0.1, + "trend_strength": 0.9, + }, + "residual_proxy": {"residual_to_level_variance_ratio": 0.1}, + "structural_break_proxy": { + "candidate_count": 3, + "strongest_candidate": {"split_index": 2, "score": 4.5}, + }, + "stationarity_basis": {"status": "valid"}, + } + + projection = decomposition_training_projection(decomposition) + projected = project_decomposition_onto_training_frame( + enriched, + decomposition, + ) + + assert projection["schema_version"] == ( + TIME_SERIES_FINGERPRINT_DECOMPOSITION_TRAINING_PROJECTION_SCHEMA_VERSION + ) + assert projection["grain"] == "period" + assert projected.height == enriched.height + assert projected.select(["series_id", "period", "row_id"]).equals( + enriched.select(["series_id", "period", "row_id"]) + ) + assert projected.get_column("decomposition_status_code").to_list() == [3, 3] + assert projected.get_column( + "decomposition_trend_direction_code" + ).to_list() == [ + 1, + 1, + ] + assert projected.get_column( + "decomposition_structural_break_score" + ).to_list() == [ + 4.5, + 4.5, + ] + + with pytest.raises(ValueError, match="identity columns: row_id"): + project_decomposition_onto_training_frame( + enriched.drop("row_id"), + decomposition, + ) + + +def test_training_feature_issues_drive_deterministic_classification() -> None: + """Issue columns should explain row state directly.""" + frame = _tick_frame( + [ + (1_000, 1.0, 1.2, 0), + (1_000, 1.3, 1.2, 1), + (900, 1.0, 1.1, 2), + (DEFAULT_SUSPICIOUS_TICK_GAP_MS + 1_000, 1.0, 1.1, 3), + ] + ) + + enriched = enrich_tick_cache_with_training_features( + frame, + symbol="EURUSD", + data_format="ascii", + timeframe="T", + period="201202", + ) + rows = enriched.to_dicts() + + assert rows[0]["dq_issue_duplicate_timestamp"] + assert rows[1]["dq_issue_duplicate_timestamp"] + assert rows[1]["dq_issue_negative_spread"] + assert rows[1]["class_spread_regime_code"] == 2 + assert rows[1]["training_usable"] is False + assert rows[1]["class_training_action_code"] == 4 + assert rows[2]["dq_issue_non_monotonic_timestamp"] + assert rows[2]["class_gap_state_code"] == 3 + assert rows[3]["dq_issue_suspicious_gap"] + assert rows[3]["dq_issue_gap_after_previous"] + assert rows[3]["class_gap_state_code"] == 2 + assert {row["period_negative_spread_rate"] for row in rows} == {0.25} + assert {row["period_duplicate_timestamp_count"] for row in rows} == {2} + assert {row["period_suspicious_gap_count"] for row in rows} == {1} + assert all(row["row_id"] for row in rows) + + +def test_training_report_derives_issue_counts_from_enriched_columns() -> None: + """Quality reports should be downstream audit artifacts.""" + target = QualityTarget( + path="/tmp/data/ASCII/T/eurusd/2012/02/.data", + kind=QualityTargetKind.CACHE, + data_format="ascii", + timeframe="T", + symbol="EURUSD", + period="201202", + ) + enriched = enrich_tick_cache_with_training_features( + _tick_frame([(1_000, 1.0, 1.2, 0), (1_300, 1.4, 1.3, 1)]), + target=target, + ) + + report = quality_report_from_training_features(enriched, target=target) + + assert report.status is QualityStatus.FAILED + [finding] = [ + item + for item in report.findings + if item.code == "DQ_ISSUE_NEGATIVE_SPREAD" + ] + assert finding.severity is QualitySeverity.ERROR + assert finding.metadata["issue_column"] == "dq_issue_negative_spread" + assert finding.metadata["row_count"] == 1 + assert report.metadata["issue_counts"]["dq_issue_negative_spread"] == 1 + + +def test_training_feature_registry_is_the_schema_catalog() -> None: + """Required training-facing columns should be defined in one registry.""" + definitions = training_feature_definitions() + names = [definition.name for definition in definitions] + + assert len(names) == len(set(names)) + assert set(required_training_feature_columns()).issubset(names) + assert set(QUALITY_ISSUE_COLUMNS).issubset(names) + + +def test_training_enrichment_rejects_non_tick_training_inputs() -> None: + """The initial training substrate is intentionally ASCII tick only.""" + frame = _tick_frame([(1_000, 1.0, 1.2, 0)]) + + with pytest.raises(ValueError, match="ASCII tick"): + enrich_tick_cache_with_training_features( + frame, + symbol="EURUSD", + data_format="ascii", + timeframe="M1", + period="201202", + ) + + with pytest.raises(ValueError, match="ASCII tick"): + enrich_tick_cache_with_training_features( + frame, + symbol="EURUSD", + data_format="ninjatrader", + timeframe="T", + period="201202", + ) + + +def test_training_report_rejects_unsupported_target_dimensions() -> None: + """Direct report derivation must not bypass training input validation.""" + enriched = enrich_tick_cache_with_training_features( + _tick_frame([(1_000, 1.0, 1.2, 0)]), + symbol="EURUSD", + data_format="ascii", + timeframe="T", + period="201202", + ) + target = QualityTarget( + path="/tmp/retired.csv", + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe="M1", + symbol="EURUSD", + period="201202", + ) + + with pytest.raises(ValueError, match="ASCII tick"): + quality_report_from_training_features(enriched, target=target) + + +def test_cache_build_writes_enriched_ascii_tick_data_as_canonical_cache( + tmp_path: Path, +) -> None: + """The application cache builder should write enriched training rows.""" + source = FIXTURES / "DAT_ASCII_EURUSD_T_201202.csv" + target = tmp_path / source.name + shutil.copyfile(source, target) + record = Record( + data_dir=str(tmp_path), + csv_filename=source.name, + data_format="ascii", + data_timeframe="T", + data_fxpair="eurusd", + data_datemonth="201202", + ) + + create_cache_file(record, {}) + + cache = read_polars_cache(tmp_path / CACHE_FILENAME) + assert set(required_training_feature_columns()).issubset(cache.columns) + assert cache.height == 3 + assert ( + cache.get_column("series_id").to_list() + == ["ascii:T:EURUSD:histdata.com"] * 3 + ) + assert cache.get_column("row_id").to_list() == [1, 2, 3] + assert ( + cache.get_column("training_schema_version").to_list() + == [TRAINING_SCHEMA_VERSION] * 3 + ) + + +def _tick_frame(rows: list[tuple[int, float, float, int]]) -> pl.DataFrame: + return pl.DataFrame( + { + "datetime": [row[0] for row in rows], + "bid": [row[1] for row in rows], + "ask": [row[2] for row in rows], + "vol": [row[3] for row in rows], + }, + schema={ + "datetime": pl.Int64, + "bid": pl.Float64, + "ask": pl.Float64, + "vol": pl.Int32, + }, + ) diff --git a/tests/unit/test_data_quality_volatility.py b/tests/unit/test_data_quality_volatility.py new file mode 100644 index 00000000..c431e009 --- /dev/null +++ b/tests/unit/test_data_quality_volatility.py @@ -0,0 +1,399 @@ +"""Tests for explicit ARCH/GARCH volatility contracts.""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, cast + +import polars as pl +import pytest + +import histdatacom.data_quality.volatility as volatility_module +from histdatacom.data_quality.classical_model_contracts import ( + ClassicalModelInputProfile, + ClassicalModelResourcePolicy, +) +from histdatacom.data_quality.contracts import ( + QualityFinding, + QualitySeverity, + QualityTarget, + QualityTargetKind, +) +from histdatacom.data_quality.fingerprint_contracts import ( + implemented_fingerprint_target_section_names, +) +from histdatacom.data_quality.fingerprints import ( + FINGERPRINT_AUDIT_SECTIONS, + TIME_SERIES_FINGERPRINT_METADATA_KEY, + HistDataFingerprintProfile, + HistDataSeriesFingerprintRule, +) +from histdatacom.data_quality.profiles import ( + QUALITY_PROFILE_SCHEMA_VERSION, + quality_profile_from_mapping, +) +from histdatacom.data_quality.training_features import ( + VOLATILITY_COLUMNS, + required_training_feature_columns, + training_feature_definitions, +) +from histdatacom.data_quality.volatility import ( + ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY, + VOLATILITY_SCHEMA_VERSION, + VOLATILITY_SUMMARY_SCHEMA_VERSION, + VolatilityProfile, + VolatilitySpecification, + project_volatility_onto_training_frame, + volatility_from_training_frame, + volatility_summary, +) +from histdatacom.histdata_ascii import ( + TICK, + format_influx_line, + read_polars_cache, + write_polars_cache, +) + + +def test_arch_and_garch_are_deterministic_and_keep_metrics_separate() -> None: + result = _run(_process_frame(1_000)) + repeated = _run(_process_frame(1_000)) + + assert result.diagnostics == repeated.diagnostics + assert result.diagnostics["schema_version"] == VOLATILITY_SCHEMA_VERSION + evaluation = _mapping(result.diagnostics["evaluation"]) + models = _mapping_rows(evaluation["models"]) + assert {model["family"] for model in models} == {"arch", "garch"} + assert evaluation["automatic_winner"] is False + assert evaluation["variance_scale"] == "unscaled_return_squared" + assert evaluation["mean_scale"] == "unscaled_return" + assert evaluation["realized_variance_proxy"] == "squared_return" + assert evaluation["reference_variance_baselines"] + assert evaluation["baseline_relative_skill"] + for model in models: + for metrics in _mapping(model["horizon_metrics"]).values(): + summary = _mapping(metrics) + assert "mean_metrics" in summary + assert "variance_metrics" in summary + assert summary["metrics_are_not_interchangeable"] is True + fit_samples = _mapping_rows(model["fit_samples"]) + assert all( + sample["covariance_condition_number"] is not None + for sample in fit_samples + ) + + +def test_projection_is_same_row_point_in_time_and_survives_cache_and_influx( + tmp_path: Path, +) -> None: + frame = _process_frame(1_000) + result = _run(frame) + projected = project_volatility_onto_training_frame( + frame, result, target=_target() + ) + + assert set(VOLATILITY_COLUMNS).issubset(projected.columns) + for family in ("arch", "garch"): + forecasts = projected.filter(pl.col(f"cm_{family}_forecast_available")) + diagnostics = projected.filter( + pl.col(f"cm_{family}_diagnostic_available") + ) + assert forecasts.height > 0 + assert diagnostics.height > 0 + assert forecasts.get_column(f"cm_{family}_training_eligible").all() + assert diagnostics.get_column(f"cm_{family}_diagnostic_only").all() + + cache_path = tmp_path / "volatility.data" + write_polars_cache(projected, cache_path) + restored = read_polars_cache(cache_path) + assert ( + restored.select(VOLATILITY_COLUMNS).to_dicts() + == projected.select(VOLATILITY_COLUMNS).to_dicts() + ) + available = restored.filter(pl.col("cm_garch_forecast_available")).row(0) + line = format_influx_line( + "EURUSD", "ascii", TICK, available, columns=restored.columns + ) + assert "cm_garch_variance_forecast=" in line + assert "cm_garch_training_eligible=true" in line + + +def test_legacy_masked_and_dropped_frames_receive_nullable_columns() -> None: + frame = _process_frame(800) + result = _run(frame) + enriched = project_volatility_onto_training_frame( + frame, result, target=_target() + ) + masked = enriched.with_columns( + pl.lit(None).cast(pl.Float64).alias("cm_garch_variance_forecast") + ) + dropped = enriched.drop(VOLATILITY_COLUMNS) + + masked_projection = project_volatility_onto_training_frame(masked, result) + dropped_projection = project_volatility_onto_training_frame(dropped, result) + assert masked_projection.select(VOLATILITY_COLUMNS).to_dicts() == ( + dropped_projection.select(VOLATILITY_COLUMNS).to_dicts() + ) + + +def test_configuration_guards_and_stable_failures_are_explicit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(ValueError, match="ARCH requires"): + VolatilitySpecification("bad", "arch", variance_order=1) + with pytest.raises(ValueError, match="reference ID"): + VolatilitySpecification( + "bad", "garch", input_definition="mean_model_residual" + ) + with pytest.raises(ValueError, match="zero mean"): + VolatilitySpecification( + "bad", + "garch", + input_definition="demeaned_return", + mean_model="constant", + ) + + level_input = _input_profile(transform="level") + invalid = _run(_process_frame(800), input_profile=level_input) + reasons = _mapping( + _mapping(invalid.diagnostics["fit_summary"])["reason_counts"] + ) + assert reasons == {"invalid_transform": 12} + + singular = _run( + _process_frame(800), + profile=VolatilityProfile( + enabled=True, maximum_covariance_condition_number=10.0 + ), + ) + singular_reasons = _mapping( + _mapping(singular.diagnostics["fit_summary"])["reason_counts"] + ) + assert singular_reasons == {"singular_covariance": 12} + + monkeypatch.setattr(volatility_module, "_load_backend", lambda: None) + unavailable = _run(_process_frame(800)) + assert unavailable.diagnostics["reason"] == "dependency_unavailable" + assert "traceback" not in str(unavailable.diagnostics).lower() + + +def test_profile_discovery_columns_and_asymmetric_registry_are_public() -> None: + parsed = quality_profile_from_mapping( + { + "schema_version": QUALITY_PROFILE_SCHEMA_VERSION, + "name": "volatility", + "rules": { + "fingerprint.series": { + "classical_model_input": { + "enabled": True, + "transform": "return", + }, + "volatility": { + "enabled": True, + "annualization_periods": 252, + "specifications": [ + { + "specification_id": "arch-t", + "family": "arch", + "distribution": "students_t", + "innovation_order": 3, + }, + { + "specification_id": "garch-normal", + "family": "garch", + "mean_model": "constant", + "innovation_order": 1, + "variance_order": 1, + }, + ], + }, + } + }, + } + ).fingerprint_profile() + assert parsed.volatility.enabled is True + assert parsed.volatility.annualization_periods == 252 + assert parsed.volatility.projection_specification_ids == ( + "arch-t", + "garch-normal", + ) + assert "volatility" in FINGERPRINT_AUDIT_SECTIONS + assert "volatility" in implemented_fingerprint_target_section_names() + assert set(VOLATILITY_COLUMNS).issubset(required_training_feature_columns()) + definitions = {item.name: item for item in training_feature_definitions()} + assert all( + definitions[name].source == "volatility" for name in VOLATILITY_COLUMNS + ) + assert { + item["family"] for item in ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY + } == { + "gjr_garch", + "egarch", + } + assert all( + item["status"] == "registered_not_enabled" + for item in ASYMMETRIC_VOLATILITY_EXTENSION_REGISTRY + ) + + +def test_summary_is_bounded_and_schema_versioned() -> None: + findings = [ + _finding( + _target(symbol="EURUSD"), _run(_process_frame(800)).diagnostics + ), + _finding( + _target(symbol="GBPUSD"), _run(_process_frame(800)).diagnostics + ), + ] + summary = volatility_summary(findings, target_limit=1) + assert summary is not None + assert summary["schema_version"] == VOLATILITY_SUMMARY_SCHEMA_VERSION + assert summary["target_count"] == 2 + assert summary["included_target_count"] == 1 + assert summary["truncated"] is True + + +def test_fingerprint_rule_runs_opt_in_volatility_on_supported_csv( + tmp_path: Path, +) -> None: + source = tmp_path / "DAT_ASCII_EURUSD_T_201201.csv" + source.write_text("\n".join(_tick_lines(1_000)) + "\n", encoding="ascii") + finding = HistDataSeriesFingerprintRule( + profile=HistDataFingerprintProfile( + classical_model_input=_input_profile(), + volatility=VolatilityProfile(enabled=True), + ) + ).evaluate(_target(path=str(source)))[0] + fingerprint = _mapping( + finding.metadata[TIME_SERIES_FINGERPRINT_METADATA_KEY] + ) + audit = _mapping(fingerprint["fingerprint_audit"]) + assert _mapping(fingerprint["volatility"])["status"] in { + "ready", + "limited", + } + assert "volatility" in audit["sections_expected"] + assert "volatility" in audit["sections_emitted"] + + +def _run( + frame: pl.DataFrame, + *, + input_profile: ClassicalModelInputProfile | None = None, + profile: VolatilityProfile | None = None, +) -> Any: + return volatility_from_training_frame( + frame, + _fingerprint(), + input_profile=input_profile or _input_profile(), + profile=profile or VolatilityProfile(enabled=True), + target=_target(), + ) + + +def _input_profile(**overrides: Any) -> ClassicalModelInputProfile: + values: dict[str, Any] = { + "enabled": True, + "frequency_ms": 60_000, + "transform": "return", + "minimum_training_observations": 32, + "minimum_evaluation_observations": 1, + "step_size": 20, + "horizons": (1, 2), + "resources": ClassicalModelResourcePolicy( + max_source_rows=10_000, + max_regularized_observations=10_000, + max_folds=12, + max_horizons=4, + max_candidate_orders=16, + max_fit_attempts=12, + max_wall_time_seconds=30, + max_retained_diagnostics=12, + ), + } + values.update(overrides) + return ClassicalModelInputProfile(**values) + + +def _process_frame(count: int) -> pl.DataFrame: + base = 1_325_376_000_000 + prices = [ + 1.1 + + index * 0.000001 + + ((index * 17) % 23 - 11) * 0.000003 + + ((index // 7) % 5 - 2) * 0.00001 + for index in range(count) + ] + return pl.DataFrame( + { + "datetime": [base + index * 10_000 for index in range(count)], + "bid": prices, + "ask": [value + 0.0002 for value in prices], + "vol": [0] * count, + } + ) + + +def _tick_lines(count: int) -> tuple[str, ...]: + start = datetime(2012, 1, 1) + return tuple( + ( + f"{(start + timedelta(seconds=index * 10)).strftime('%Y%m%d %H%M%S')}000," + f"{1.1 + index * 0.000001 + ((index * 17) % 23 - 11) * 0.000003:.6f}," + f"{1.1002 + index * 0.000001 + ((index * 17) % 23 - 11) * 0.000003:.6f},0" + ) + for index in range(count) + ) + + +def _fingerprint() -> dict[str, Any]: + return { + "fingerprint_id": "fingerprint-eurusd", + "target_axis": { + "data_format": "ascii", + "timeframe": TICK, + "symbol": "EURUSD", + "period": "201201", + "kind": "cache", + }, + } + + +def _target( + *, symbol: str = "EURUSD", path: str | None = None +) -> QualityTarget: + return QualityTarget( + path=path or f"DAT_ASCII_{symbol}_T_201201.csv", + kind=QualityTargetKind.CSV, + data_format="ascii", + timeframe=TICK, + symbol=symbol, + period="201201", + ) + + +def _finding( + target: QualityTarget, diagnostics: Mapping[str, Any] +) -> QualityFinding: + return QualityFinding( + severity=QualitySeverity.INFO, + code="FINGERPRINT_SERIES_SUMMARY", + message="Canonical target time-series fingerprint.", + rule_id="fingerprint.series", + target=target, + metadata={ + TIME_SERIES_FINGERPRINT_METADATA_KEY: { + "volatility": dict(diagnostics) + } + }, + ) + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(cast(Mapping[str, Any], value)) + + +def _mapping_rows(value: Any) -> list[dict[str, Any]]: + return [_mapping(row) for row in cast(list[Any], value)] diff --git a/tests/unit/test_dev_tool_hooks.py b/tests/unit/test_dev_tool_hooks.py index 047f4f1e..2575b17f 100644 --- a/tests/unit/test_dev_tool_hooks.py +++ b/tests/unit/test_dev_tool_hooks.py @@ -48,30 +48,67 @@ def test_run_dev_tool_remove_is_cross_platform(tmp_path: Path) -> None: assert not artifact.exists() -def test_local_pre_commit_hooks_use_dev_tool_launcher() -> None: - """Local system hooks should resolve tools through the repo launcher.""" +def test_local_pre_commit_hooks_do_not_run_coverage() -> None: + """Routine commits and pushes must not start coverage.""" config = yaml.safe_load((REPO_ROOT / ".pre-commit-config.yaml").read_text()) - [local_repo] = [ - repo for repo in config["repos"] if repo.get("repo") == "local" + ordered_hooks = [ + hook for repo in config["repos"] for hook in repo.get("hooks", []) ] - hooks = {hook["id"]: hook for hook in local_repo["hooks"]} + hooks = {hook["id"]: hook for hook in ordered_hooks} assert hooks["histdatacom"]["entry"] == ( "scripts/run_dev_tool.py histdatacom" ) - for hook_id in ("coverage-run", "coverage-combine", "coverage-report"): - assert hooks[hook_id]["entry"] == "scripts/run_dev_tool.py coverage" - assert hooks["coverage-rm"]["entry"] == "scripts/run_dev_tool.py" - assert hooks["coverage-rm"]["args"] == ["--remove", ".coverage"] + assert not { + "coverage-run", + "coverage-combine", + "coverage-report", + "coverage-rm", + }.intersection(hooks) + hook_ids = [hook["id"] for hook in ordered_hooks] + assert not any("coverage" in hook_id for hook_id in hook_ids) + + +def test_coverage_runs_only_for_dev_to_main_promotion() -> None: + """CI should instrument tests only at the production promotion boundary.""" + config = yaml.safe_load( + (REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text() + ) + jobs = config["jobs"] + production = jobs["production-coverage"] + + assert production["name"] == "Production coverage" + assert production["if"] == ( + "github.event_name == 'pull_request' && " + "github.base_ref == 'main' && github.head_ref == 'dev'" + ) + + coverage_steps = [] + for job_name, job in jobs.items(): + for step in job.get("steps", []): + command = str(step.get("run", "")) + if "--cov=" in command or "coverage report" in command: + coverage_steps.append((job_name, command)) + + assert [job_name for job_name, _ in coverage_steps] == [ + "production-coverage", + "production-coverage", + ] + assert all( + "--cov=" not in str(step.get("run", "")) + for step in jobs["test"]["steps"] + ) def test_architecture_diagram_generation_is_pre_commit_gated() -> None: """Architecture SVGs should be regenerated by the local pre-commit loop.""" config = yaml.safe_load((REPO_ROOT / ".pre-commit-config.yaml").read_text()) - [local_repo] = [ - repo for repo in config["repos"] if repo.get("repo") == "local" - ] - hooks = {hook["id"]: hook for hook in local_repo["hooks"]} + hooks = { + hook["id"]: hook + for repo in config["repos"] + if repo.get("repo") == "local" + for hook in repo["hooks"] + } diagram_hook = hooks["architecture-diagrams"] diff --git a/tests/unit/test_documentation.py b/tests/unit/test_documentation.py new file mode 100644 index 00000000..e7eddb22 --- /dev/null +++ b/tests/unit/test_documentation.py @@ -0,0 +1,74 @@ +"""Repository contracts for the maintained documentation build.""" + +import re +from importlib import metadata +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +DOCS_ROOT = REPO_ROOT / "docs" +TOCTREE_ENTRY = re.compile(r"^ ([a-z0-9][a-z0-9_./-]*)$", re.MULTILINE) + + +def test_read_the_docs_uses_the_versioned_sphinx_configuration() -> None: + config = yaml.safe_load( + (REPO_ROOT / ".readthedocs.yaml").read_text(encoding="utf-8") + ) + + assert config["version"] == 2 + assert config["sphinx"] == { + "configuration": "docs/conf.py", + "fail_on_warning": True, + } + assert config["python"]["install"] == [ + { + "method": "pip", + "path": ".", + "extra_requirements": ["docs"], + } + ] + + +def test_documentation_extra_is_python_310_compatible_and_pinned() -> None: + requirements = metadata.requires("histdatacom") or [] + docs_requirements = sorted( + requirement.partition(";")[0].strip() + for requirement in requirements + if 'extra == "docs"' in requirement + ) + + assert docs_requirements == sorted( + [ + "myst-parser==4.0.1", + "sphinx==8.1.3", + "sphinx-rtd-theme==3.1.0", + ] + ) + + +def test_ci_builds_documentation_without_coverage() -> None: + workflow = yaml.safe_load( + (REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text( + encoding="utf-8" + ) + ) + docs_job = workflow["jobs"]["docs"] + commands = "\n".join(step.get("run", "") for step in docs_job["steps"]) + + assert ( + "python -m sphinx -W --keep-going -b html docs docs/_build/html" + in commands + ) + assert "--cov" not in commands + + +def test_every_maintained_markdown_document_is_in_the_root_toctree() -> None: + index = (DOCS_ROOT / "index.rst").read_text(encoding="utf-8") + included_documents = set(TOCTREE_ENTRY.findall(index)) + maintained_documents = { + path.relative_to(DOCS_ROOT).with_suffix("").as_posix() + for path in DOCS_ROOT.rglob("*.md") + } + + assert included_documents == maintained_documents diff --git a/tests/unit/test_histdata_ascii.py b/tests/unit/test_histdata_ascii.py index 795138a6..57eda199 100644 --- a/tests/unit/test_histdata_ascii.py +++ b/tests/unit/test_histdata_ascii.py @@ -17,6 +17,7 @@ convert_polars_datetime_to_utc_ms, convert_batch_for_api, delimiter_for_timeframe, + filename_has_unsupported_raw_dimensions, format_influx_line, merge_batches, normalize_ascii_row, @@ -253,6 +254,48 @@ def test_polars_zip_ingest_requires_exactly_one_csv_member( read_ascii_file_to_polars(archive_path, "T") +def test_ascii_import_rejects_retired_platform_filename( + tmp_path: Path, +) -> None: + """Direct raw imports must not trust caller-supplied tick context.""" + source = FIXTURES / "DAT_ASCII_EURUSD_T_201202.csv" + retired = tmp_path / "DAT_NT_EURUSD_T_LAST_201202.csv" + retired.write_bytes(source.read_bytes()) + + with pytest.raises(ValueError, match="ASCII tick"): + read_ascii_file_to_polars(retired, "T") + with pytest.raises(ValueError, match="ASCII tick"): + read_ascii_file(retired, "T") + + +def test_ascii_status_report_filename_keeps_supported_dimensions() -> None: + """Live same-stem TXT reports retain the supported ASCII tick axes.""" + assert not filename_has_unsupported_raw_dimensions( + "DAT_ASCII_EURUSD_T_202201.txt" + ) + assert filename_has_unsupported_raw_dimensions( + "DAT_NT_EURUSD_T_LAST_202201.txt" + ) + + +def test_ascii_zip_import_rejects_retired_platform_member( + tmp_path: Path, +) -> None: + """Opaque ZIP wrappers must not hide a retired raw member filename.""" + archive_path = tmp_path / "opaque.zip" + source = FIXTURES / "DAT_ASCII_EURUSD_T_201202.csv" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr( + "DAT_NT_EURUSD_T_LAST_201202.csv", + source.read_bytes(), + ) + + with pytest.raises(ValueError, match="ASCII tick"): + read_ascii_file_to_polars(archive_path, "T") + with pytest.raises(ValueError, match="ASCII tick"): + read_ascii_file(archive_path, "T") + + @pytest.mark.parametrize( ("timeframe", "filename"), (("T", "DAT_ASCII_EURUSD_T_201202.csv"),), @@ -287,6 +330,58 @@ def test_polars_cache_rejects_legacy_cache_payloads( assert str(err.value) == LEGACY_CACHE_ERROR +@pytest.mark.parametrize( + ("data_format", "timeframe"), + (("metatrader", "T"), ("ascii", "M1")), +) +def test_polars_cache_rejects_enriched_retired_dimensions( + tmp_path: Path, + data_format: str, + timeframe: str, +) -> None: + """Canonical cache reads must not trust retired row metadata.""" + import polars as pl + + cache_path = tmp_path / CACHE_FILENAME + pl.DataFrame( + { + "datetime": [1], + "bid": [1.0], + "ask": [1.1], + "vol": [0], + "format": [data_format], + "timeframe": [timeframe], + } + ).write_ipc(cache_path) + + with pytest.raises(ValueError, match="ASCII tick"): + read_polars_cache(cache_path) + + +def test_polars_cache_writer_rejects_enriched_retired_dimensions( + tmp_path: Path, +) -> None: + """The low-level canonical cache writer must also fail closed.""" + import polars as pl + + cache_path = tmp_path / CACHE_FILENAME + frame = pl.DataFrame( + { + "datetime": [1], + "bid": [1.0], + "ask": [1.1], + "vol": [0], + "format": ["ascii"], + "timeframe": ["M1"], + } + ) + + with pytest.raises(ValueError, match="ASCII tick"): + write_polars_cache(frame, cache_path) + + assert not cache_path.exists() + + @pytest.mark.parametrize( ("timeframe", "filename", "expected_rows"), (("T", "DAT_ASCII_EURUSD_T_201202.csv", EXPECTED_TICK_ROWS),), @@ -532,3 +627,29 @@ def test_influx_line_protocol_rejects_removed_m1_timeframe() -> None: """M1 rows should not be accepted by the tick-only line protocol.""" with pytest.raises(ValueError, match="unsupported ASCII timeframe"): format_influx_line("eurusd", "ascii", "M1", EXPECTED_TICK_ROWS[1]) + + +def test_influx_line_protocol_rejects_removed_platform_format() -> None: + """Platform-specific formats must not leak through direct projection.""" + with pytest.raises(ValueError, match="ASCII tick"): + format_influx_line( + "eurusd", + "ninjatrader", + "T", + EXPECTED_TICK_ROWS[1], + ) + + +def test_enriched_influx_line_rejects_retired_row_dimensions() -> None: + """Enriched row metadata must not override a supported projection target.""" + columns = ("datetime", "bid", "ask", "vol", "format", "timeframe") + row = (*EXPECTED_TICK_ROWS[1], "metatrader", "T") + + with pytest.raises(ValueError, match="ASCII tick"): + format_influx_line( + "eurusd", + "ascii", + "T", + row, + columns=columns, + ) diff --git a/tests/unit/test_histdata_com.py b/tests/unit/test_histdata_com.py index 73726a94..b3c2f38c 100644 --- a/tests/unit/test_histdata_com.py +++ b/tests/unit/test_histdata_com.py @@ -18,6 +18,10 @@ ) from histdatacom.options import Options from histdatacom.runtime_contracts import RunRequest, WorkStatus +from histdatacom.random_windows import ( + RANDOM_WINDOW_SELECTION_METADATA_KEY, + RandomWindowSelectionV1, +) from histdatacom.orchestration.client import ( JobHandle, JobResult, @@ -132,6 +136,7 @@ def _orchestration_quality_result( error: str = "", check_groups: list[str] | None = None, next_actions: dict[str, object] | None = None, + quality_engine: dict[str, object] | None = None, remediation_coverage: dict[str, object] | None = None, fingerprint_distribution: dict[str, object] | None = None, fingerprint_distribution_attention: dict[str, object] | None = None, @@ -208,6 +213,8 @@ def _orchestration_quality_result( } if next_actions is not None: quality["next_actions"] = next_actions + if quality_engine is not None: + quality["quality_engine"] = quality_engine if remediation_coverage is not None: quality["remediation_coverage"] = remediation_coverage if fingerprint_distribution is not None: @@ -875,6 +882,63 @@ def fake_submit(request, **kwargs: object) -> JobResult: assert f"targets: {expected_count}" in output +def test_data_quality_cli_renders_quality_engine_skips( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> None: + """Quality CLI should reconcile skips from bounded engine metadata.""" + import histdatacom.histdata_com as histdata_com + + target = write_ascii_case(tmp_path, CLEAN_TICK_CASE) + + def fake_submit(request, **kwargs: object) -> JobResult: + return _orchestration_quality_result( + target_count=2, + check_groups=["time"], + quality_engine={ + "planned_target_rule_evaluation_count": 2, + "target_rule_evaluation_count": 1, + "skipped_rule_evaluation_count": 1, + "skip_events": { + "included_event_count": 1, + "omitted_event_count": 0, + "reason_counts": { + "duplicate_archive_preferred_csv": 1, + }, + "rule_id_counts": {"time.ascii.gaps": 1}, + }, + }, + ) + + monkeypatch.setattr( + histdata_com, + "submit_run_request_and_observe_sync", + fake_submit, + ) + monkeypatch.setattr( + sys, + "argv", + [ + "histdatacom", + "--quality", + "--quality-checks", + "time", + "--quality-target", + str(target), + ], + ) + + assert histdata_com.main() is None + + output = capsys.readouterr().out + assert "Quality engine skips" in output + assert "- evaluations: planned=2 executed=1 skipped=1" in output + assert "- events: included=1 omitted=0" in output + assert "- reasons: duplicate_archive_preferred_csv=1" in output + assert "- rules: time.ascii.gaps=1" in output + + def test_data_quality_cli_renders_fingerprint_topology_summary( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], @@ -1070,6 +1134,25 @@ def fake_submit(request, **kwargs: object) -> JobResult: "max_gap_ms": 60_000, "computed_from": "text_scan", "cache_source": None, + "inspection_context": { + "duplicate_timestamps": { + "total_count": 1, + "included_count": 1, + "omitted_count": 0, + "samples": [ + { + "row_number": 2, + "timestamp_source": ( + "20120201 000000000" + ), + "occurrence_count": 2, + } + ], + "next_action": { + "code": ("inspect_duplicate_timestamp_rows") + }, + } + }, } ], }, @@ -1120,6 +1203,11 @@ def fake_submit(request, **kwargs: object) -> JobResult: "max gap 60s, computed_from=text_scan, " "next=inspect duplicate timestamp rows" ) in output + assert ( + " - context duplicate_timestamps: samples=1/1 omitted=0 " + "action=inspect_duplicate_timestamp_rows " + "20120201 000000000 x2" + ) in output assert "Fingerprint topology" in output assert ( "- targets: 1 included: 1 regular: 0 irregular: 1 unavailable: 0" @@ -1694,6 +1782,7 @@ def fail_submit(*args: object, **kwargs: object) -> object: } assert [channel["kind"] for channel in explanation["input_channels"]] == [ "built_in_default", + "named_profile", "profile_file", "cli_override", ] @@ -1715,8 +1804,11 @@ def fail_submit(*args: object, **kwargs: object) -> object: "path": "/reporting/remediation_catalog_audit/enabled", "value": True, "source": "cli_override", + "profile_name": "preview-profile", + "previous_source": "profile_file", "overridden_source": "profile_file", "override": True, + "previous_value": False, } diff = explanation["effective_diff"] assert diff["schema_version"] == ( @@ -2058,9 +2150,10 @@ def fail_submit(*args: object, **kwargs: object) -> object: assert explanation["profile_source"]["kind"] == "api_options" assert [channel["kind"] for channel in explanation["input_channels"]] == [ "built_in_default", + "named_profile", "api_options", - "cli_override", ] + assert payload["cli_overrides"] == {} value_sources = { item["path"]: item for item in explanation["effective_value_sources"]["values"] @@ -2071,7 +2164,7 @@ def fail_submit(*args: object, **kwargs: object) -> object: ) assert ( value_sources["/reporting/remediation_catalog_audit/enabled"]["source"] - == "cli_override" + == "api_options" ) assert preview_path.read_text(encoding="utf-8").startswith( "Quality Profile Preview\n" @@ -2138,6 +2231,7 @@ def fail_submit(*args: object, **kwargs: object) -> object: assert [channel["kind"] for channel in explanation["input_channels"]] == [ "built_in_default", "yaml_config", + "named_profile", "profile_file", ] value_sources = { @@ -2150,6 +2244,81 @@ def fail_submit(*args: object, **kwargs: object) -> object: ) +def test_config_quality_profile_override_is_attributed_to_yaml( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> None: + """YAML reporting overrides should not be mislabeled as CLI input.""" + import histdatacom.histdata_com as histdata_com + + def fail_submit(*args: object, **kwargs: object) -> object: + raise AssertionError("quality profile preview must not submit") + + profile_path = tmp_path / "quality-profile.json" + profile_path.write_text( + json.dumps( + { + "name": "yaml-override-profile", + "reporting": {"remediation_catalog_audit": {"enabled": False}}, + } + ), + encoding="utf-8", + ) + config_path = tmp_path / "histdatacom.yaml" + config_path.write_text( + f""" +histdatacom: + quality: true + quality_target: {tmp_path} + quality_profile: {profile_path} + quality_profile_preview: true + remediation_catalog_audit: true +""", + encoding="utf-8", + ) + monkeypatch.setattr( + histdata_com, + "submit_run_request_and_observe_sync", + fail_submit, + ) + monkeypatch.setattr( + sys, + "argv", + ["histdatacom", "--config", str(config_path)], + ) + + assert histdata_com.main() is None + + payload = json.loads(capsys.readouterr().out) + assert payload["cli_overrides"] == {} + channels = { + channel["kind"]: channel + for channel in payload["profile_explanation"]["input_channels"] + } + assert channels["yaml_config"]["path"] == str(config_path) + assert channels["yaml_config"]["paths"] == [ + "/reporting/remediation_catalog_audit/enabled" + ] + sources = { + item["path"]: item + for item in payload["profile_explanation"]["effective_value_sources"][ + "values" + ] + } + assert sources["/reporting/remediation_catalog_audit/enabled"] == { + "path": "/reporting/remediation_catalog_audit/enabled", + "value": True, + "source": "yaml_config", + "profile_name": "yaml-override-profile", + "source_path": str(config_path), + "override": True, + "previous_source": "profile_file", + "overridden_source": "profile_file", + "previous_value": False, + } + + def test_config_quality_profile_preview_markdown_renders_tables( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], @@ -2777,6 +2946,180 @@ def fake_submit(request, **kwargs: object) -> JobResult: } +def test_api_orchestration_materialization_applies_exact_random_window( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Client materialization should recover selection metadata from the run.""" + import polars as pl + + import histdatacom.histdata_com as histdata_com + from histdatacom.histdata_ascii import CACHE_FILENAME, write_polars_cache + + start = 1_704_096_000_000 + end = start + 60_000 + cache_path = tmp_path / CACHE_FILENAME + source = pl.DataFrame( + { + "datetime": (start - 1, start, end), + "bid": (1.1, 1.2, 1.3), + "ask": (1.2, 1.3, 1.4), + "volume": (0.0, 0.0, 0.0), + } + ) + job_result = _orchestration_cache_result(tmp_path) + write_polars_cache(source, cache_path) + selection = RandomWindowSelectionV1( + expression="1m", + mode="random", + support_start_utc_ms=start - 1, + support_end_utc_ms=end + 1, + seed=7, + selected_start_utc_ms=start, + selected_end_utc_ms=end, + ) + stage_result = job_result.result["stage_results"][0] + stage_result["work_item"] = { + "metadata": {RANDOM_WINDOW_SELECTION_METADATA_KEY: selection.to_dict()} + } + + monkeypatch.setattr( + histdata_com, + "submit_run_request_and_observe_sync", + lambda request, **kwargs: job_result, + ) + options = _orchestration_options() + options.random_window = "1m" + options.random_seed = 7 + + result = histdata_com.main(options) + + assert isinstance(result, pl.DataFrame) + assert result["datetime"].to_list() == [start] + assert histdata_com.RunRequest.from_options(options).random_seed == 7 + assert pl.read_ipc(cache_path).height == 3 + + +def test_api_orchestration_random_window_fails_if_selection_is_missing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Completed cache artifacts cannot bypass a lost selection contract.""" + import histdatacom.histdata_com as histdata_com + from histdatacom.random_windows import RandomWindowError + + monkeypatch.setattr( + histdata_com, + "submit_run_request_and_observe_sync", + lambda request, **kwargs: _orchestration_cache_result(tmp_path), + ) + options = _orchestration_options() + options.random_window = "1m" + options.random_seed = 7 + + with pytest.raises(RandomWindowError, match="missing its resolved"): + histdata_com.main(options) + + +def test_api_orchestration_materialization_applies_output_timezone( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The client-side view should use the timezone carried by public Options.""" + import polars as pl + + import histdatacom.histdata_com as histdata_com + + captured: dict[str, object] = {} + + def fake_submit(request, **kwargs: object) -> JobResult: + captured["request"] = request + return _orchestration_cache_result(tmp_path) + + monkeypatch.setattr( + histdata_com, + "submit_run_request_and_observe_sync", + fake_submit, + ) + options = _orchestration_options() + options.output_timezone = "Europe/London" + + result = histdata_com.main(options) + + assert isinstance(result, pl.DataFrame) + assert result.schema["datetime_local"] == pl.Datetime("ms", "Europe/London") + request = captured["request"] + assert isinstance(request, RunRequest) + assert request.metadata["output_timezone"] == "Europe/London" + + +def test_api_rejects_unknown_output_timezone_before_submission( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Invalid IANA zones must fail before starting or submitting orchestration.""" + import histdatacom.histdata_com as histdata_com + + def fail_submit(*args: object, **kwargs: object) -> object: + raise AssertionError("invalid timezone should not submit") + + monkeypatch.setattr( + histdata_com, + "submit_run_request_and_observe_sync", + fail_submit, + ) + options = _orchestration_options() + options.output_timezone = "Mars/Olympus_Mons" + + with pytest.raises(ValueError, match="unsupported output timezone"): + histdata_com.main(options) + + +def test_cli_reports_unknown_output_timezone_without_traceback( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """CLI configuration failures should use the bounded report boundary.""" + import histdatacom.histdata_com as histdata_com + + def fail_submit(*args: object, **kwargs: object) -> object: + raise AssertionError("invalid timezone should not submit") + + monkeypatch.setattr( + histdata_com, + "submit_run_request_and_observe_sync", + fail_submit, + ) + monkeypatch.setattr( + sys, + "argv", + [ + "histdatacom", + "--timezone", + "Mars/Olympus_Mons", + "--request-json-out", + "-", + "-p", + "eurusd", + "-f", + "ascii", + "-t", + "tick-data-quotes", + "-s", + "2022-12", + ], + ) + + with pytest.raises(SystemExit) as err: + histdata_com.main() + + captured = capsys.readouterr() + assert err.value.code == 1 + assert "HistData configuration invalid" in captured.err + assert "CONFIGURATION_ERROR" in captured.err + assert "Mars/Olympus_Mons" in captured.err + assert "Traceback" not in captured.err + + @pytest.mark.parametrize( ("status", "expected"), ( diff --git a/tests/unit/test_histdatacom__init__.py b/tests/unit/test_histdatacom__init__.py index 7323f467..a74e7629 100644 --- a/tests/unit/test_histdatacom__init__.py +++ b/tests/unit/test_histdatacom__init__.py @@ -19,6 +19,15 @@ def test_exposed_classes() -> None: "Pairs", "Timeframe", "Format", + "ReconstructionClient", + "ReconstructionExecutionRequestV1", + "ReconstructionExitCode", + "ReconstructionOperationReceiptV1", + "ReconstructionPlanSetPreflightV1", + "ReconstructionPlanSetV1", + "ReconstructionPlanShardV1", + "ReconstructionPlanSpecV1", + "ReconstructionPreflightV1", } diff --git a/tests/unit/test_influx.py b/tests/unit/test_influx.py index 6f379f80..705ceff3 100644 --- a/tests/unit/test_influx.py +++ b/tests/unit/test_influx.py @@ -5,9 +5,13 @@ from pathlib import Path from types import SimpleNamespace +import polars as pl import pytest -from histdatacom.histdata_ascii import CACHE_FILENAME +from histdatacom.data_quality.training_features import ( + enrich_tick_cache_with_training_features, +) +from histdatacom.histdata_ascii import CACHE_FILENAME, format_influx_line FIXTURES = Path(__file__).parents[1] / "fixtures" / "histdata_ascii" @@ -124,7 +128,153 @@ def put(self, item: list[str]) -> None: ) assert [len(item) for item in sink.items] == [2, 1] - assert sink.items[0][0] == EXPECTED_TICK_LINE + first_line = sink.items[0][0] + assert "row_id=1" in first_line.split(" ", maxsplit=1)[0] + assert "bidquote=1.3066" in first_line + assert "askquote=1.30677" in first_line + assert "quality_status_code=0i" in first_line + assert "training_usable=true" in first_line + assert first_line.endswith(" 1328072403660") + + +def test_import_cache_enriches_legacy_raw_cache_before_projection( + tmp_path: Path, +) -> None: + """Existing raw .data caches should project enriched training fields.""" + from histdatacom.api import Api + from histdatacom.influx import Influx + + class FakeSink: + def __init__(self) -> None: + self.items: list[list[str]] = [] + + def put(self, item: list[str]) -> None: + self.items.append(item) + + source_record = SimpleNamespace(data_timeframe="T") + raw_frame = Api._import_file_to_polars( + source_record, + FIXTURES / "DAT_ASCII_EURUSD_T_201202.csv", + ) + Api._write_cache_data(raw_frame, str(tmp_path / CACHE_FILENAME)) + sink = FakeSink() + record = SimpleNamespace( + data_dir=str(tmp_path) + "/", + cache_filename=CACHE_FILENAME, + data_fxpair="eurusd", + data_format="ascii", + data_timeframe="T", + data_datemonth="201202", + ) + + Influx()._import_cache( + record, + {"batch_size": "2"}, + sink, # type: ignore[arg-type] + ) + + [first_batch, second_batch] = sink.items + first_line = first_batch[0] + assert [len(first_batch), len(second_batch)] == [2, 1] + assert "row_id=1" in first_line.split(" ", maxsplit=1)[0] + assert "period=201202" in first_line.split(" ", maxsplit=1)[0] + assert "bidquote=1.3066" in first_line + assert "askquote=1.30677" in first_line + assert "spread=" in first_line + assert "quality_status_code=0i" in first_line + assert "dq_issue_negative_spread=false" in first_line + assert "training_usable=true" in first_line + assert first_line.endswith(" 1328072403660") + + +def test_enriched_influx_line_projects_training_fields_on_tick_point() -> None: + """Influx should project enriched .data fields, not a joined report table.""" + frame = enrich_tick_cache_with_training_features( + pl.DataFrame( + { + "datetime": [1_000], + "bid": [1.0], + "ask": [1.0002], + "vol": [0], + }, + schema={ + "datetime": pl.Int64, + "bid": pl.Float64, + "ask": pl.Float64, + "vol": pl.Int32, + }, + ), + symbol="EURUSD", + data_format="ascii", + timeframe="T", + period="201202", + ) + row = next(frame.iter_rows()) + + line = format_influx_line( + "eurusd", + "ascii", + "T", + row, + columns=frame.columns, + ) + + assert "row_id=1" in line.split(" ", maxsplit=1)[0] + assert "period=201202" in line.split(" ", maxsplit=1)[0] + assert "bidquote=1.0" in line + assert "askquote=1.0002" in line + assert ",row_id=1i" not in line.split(" ", maxsplit=2)[1] + assert "spread=" in line + assert "mid=1.0001" in line + assert "quality_status_code=0i" in line + assert "dq_issue_negative_spread=false" in line + assert "training_usable=true" in line + assert "class_training_action_code=0i" in line + assert line.endswith(" 1000") + + +def test_enriched_influx_projection_preserves_duplicate_timestamp_rows() -> ( + None +): + """Duplicate tick timestamps need distinct Influx point identity.""" + frame = enrich_tick_cache_with_training_features( + pl.DataFrame( + { + "datetime": [1_000, 1_000], + "bid": [1.0, 1.1], + "ask": [1.2, 1.3], + "vol": [0, 1], + }, + schema={ + "datetime": pl.Int64, + "bid": pl.Float64, + "ask": pl.Float64, + "vol": pl.Int32, + }, + ), + symbol="EURUSD", + data_format="ascii", + timeframe="T", + period="201202", + ) + + lines = [ + format_influx_line( + "eurusd", + "ascii", + "T", + row, + columns=frame.columns, + ) + for row in frame.iter_rows() + ] + + assert lines[0].endswith(" 1000") + assert lines[1].endswith(" 1000") + assert "row_id=1" in lines[0].split(" ", maxsplit=1)[0] + assert "row_id=2" in lines[1].split(" ", maxsplit=1)[0] + assert "dq_issue_duplicate_timestamp=true" in lines[0] + assert "dq_issue_duplicate_timestamp=true" in lines[1] def test_influx_batch_writer_writes_direct_synchronous_batches( diff --git a/tests/unit/test_inspect_wheel.py b/tests/unit/test_inspect_wheel.py index c1e30fa2..2cb88254 100644 --- a/tests/unit/test_inspect_wheel.py +++ b/tests/unit/test_inspect_wheel.py @@ -99,6 +99,10 @@ def _write_common_assets( manifest: dict[str, object], ) -> None: """Write common runtime assets to a fake wheel.""" + wheel.writestr( + "histdatacom/market_context/assets/operator_shocks_v1.json", + '{"schema_version":"fixture"}', + ) wheel.writestr( "histdatacom/orchestration/assets/README.md", "runtime assets", @@ -146,8 +150,22 @@ def _write_common_assets( ) -def _write_dist_info(wheel: ZipFile, *, tag: str) -> None: +def _write_dist_info( + wheel: ZipFile, + *, + tag: str, + include_tzdata: bool = True, +) -> None: """Write minimal package metadata needed by the wheel inspector.""" + requirements = [ + 'Requires-Dist: statsmodels>=0.14.6,<0.15; extra == "models"', + 'Requires-Dist: statsmodels>=0.14.6,<0.15; extra == "all"', + "Requires-Dist: temporalio>=1.10,<1.29", + 'Requires-Dist: temporalio>=1.10,<1.29; extra == "temporal"', + 'Requires-Dist: temporalio>=1.10,<1.29; extra == "all"', + ] + if include_tzdata: + requirements.append("Requires-Dist: tzdata>=2026.3") wheel.writestr( "histdatacom-1.0.0.dist-info/METADATA", "\n".join( @@ -160,11 +178,10 @@ def _write_dist_info(wheel: ZipFile, *, tag: str) -> None: "Classifier: Operating System :: Microsoft :: Windows", "Classifier: Operating System :: POSIX", "Classifier: Operating System :: POSIX :: Linux", + "Provides-Extra: models", "Provides-Extra: temporal", "Provides-Extra: all", - "Requires-Dist: temporalio>=1.10,<1.29", - 'Requires-Dist: temporalio>=1.10,<1.29; extra == "temporal"', - 'Requires-Dist: temporalio>=1.10,<1.29; extra == "all"', + *requirements, "", ] ), @@ -247,6 +264,29 @@ def test_inspect_wheel_accepts_bundled_platform_executable( assert report["wheel_tags"] == ["py3-none-macosx_11_0_arm64"] +def test_inspect_wheel_requires_core_tzdata_dependency( + tmp_path: Path, +) -> None: + """Cross-platform wheels must carry the ZoneInfo fallback dependency.""" + module = _load_script() + manifest = _base_manifest() + wheel_path = tmp_path / "histdatacom-1.0.0-py3-none-any.whl" + + with ZipFile(wheel_path, "w") as wheel: + _write_common_assets(wheel, manifest=manifest) + _write_dist_info( + wheel, + tag="py3-none-any", + include_tzdata=False, + ) + + with pytest.raises( + SystemExit, + match="tzdata dependency missing from core metadata", + ): + module.inspect_wheel(wheel_path) + + def test_inspect_wheel_accepts_windows_exe_without_unix_execute_mode( tmp_path: Path, ) -> None: diff --git a/tests/unit/test_market_context.py b/tests/unit/test_market_context.py new file mode 100644 index 00000000..fbd24690 --- /dev/null +++ b/tests/unit/test_market_context.py @@ -0,0 +1,600 @@ +"""Tests for point-in-time market-context contracts and bounded joins.""" + +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from histdatacom.market_context import ( + MARKET_CONTEXT_EVENT_SCHEMA_VERSION, + MARKET_CONTEXT_SOURCE_SCHEMA_VERSION, + MARKET_CONTEXT_TIMELINE_SCHEMA_VERSION, + MarketContextEventV1, + MarketContextKind, + MarketContextMissingReason, + MarketContextPrecision, + MarketContextQueryLimitError, + MarketContextQueryStatus, + MarketContextQueryV1, + MarketContextSourceV1, + MarketContextTimelineV1, + MarketContextView, + StaticMarketContextSourceAdapterV1, + build_market_context_timeline, + market_context_calendar_state, + market_context_information_inputs, + normalize_market_context_datetime, + query_market_context, + query_market_context_window, +) +from histdatacom.synthetic import ( + InformationSplitKind, + InformationMode, + InformationScope, + ReconstructionInformationManifestV1, + ReconstructionInformationPolicyV1, + ReconstructionInformationSplitV1, + ReconstructionRunV1, + ReconstructionWindowV1, + audit_reconstruction_information, + reconstruction_information_window_plan_id, +) + +HOUR_NS = 3_600_000_000_000 +DAY_NS = 24 * HOUR_NS +EVENT_TIME_TEXT = "2022-07-13T08:30:00-04:00" +EVENT_TIME_NS = 1_657_715_400_000_000_000 + + +def _source( + *, + version: str = "2022-07-06-schedule", + retrieved_at_ns: int = EVENT_TIME_NS - 6 * DAY_NS, + digest: str = "a" * 64, + adapter_name: str = "fixture-macro", +) -> MarketContextSourceV1: + return MarketContextSourceV1( + name="Operator-approved macro fixture", + source_version=version, + retrieved_at_ns=retrieved_at_ns, + content_sha256=digest, + adapter_name=adapter_name, + adapter_version="1.0", + license_name="Fixture-only license", + redistribution_allowed=False, + redistribution_constraints=("Do not redistribute source text.",), + limitations=("Fixture values are not a licensed production feed.",), + source_uri="https://example.invalid/macro/fixture", + metadata={"retrieval_method": "operator_fixture"}, + ) + + +def _initial_event( + *, + canonical_key: str = "us.cpi.headline.2022-07", + source: MarketContextSourceV1 | None = None, +) -> MarketContextEventV1: + return MarketContextEventV1( + canonical_key=canonical_key, + kind=MarketContextKind.MACRO_RELEASE, + title="US CPI headline", + source=source or _source(), + source_event_time=EVENT_TIME_TEXT, + source_timezone="America/New_York", + event_time_ns=EVENT_TIME_NS, + first_known_at_ns=EVENT_TIME_NS - 7 * DAY_NS, + available_at_ns=EVENT_TIME_NS - 7 * DAY_NS, + pre_event_ns=30 * 60 * 1_000_000_000, + post_event_ns=60 * 60 * 1_000_000_000, + affected_currencies=("usd",), + affected_symbols=("eurusd", "gbpusd"), + confidence=1.0, + precision=MarketContextPrecision.EXACT, + limitations=("Schedule vintage contains no realized actual value.",), + vintage_id="schedule-v1", + expected_value=2.0, + value_unit="percent_yoy", + tags=("inflation", "scheduled"), + ) + + +def _revision_event(initial: MarketContextEventV1) -> MarketContextEventV1: + return MarketContextEventV1( + canonical_key=initial.canonical_key, + kind=initial.kind, + title=initial.title, + source=_source( + version="2022-07-13-initial-release", + retrieved_at_ns=EVENT_TIME_NS + HOUR_NS, + digest="b" * 64, + ), + source_event_time=initial.source_event_time, + source_timezone=initial.source_timezone, + event_time_ns=initial.event_time_ns, + first_known_at_ns=initial.first_known_at_ns, + available_at_ns=EVENT_TIME_NS, + pre_event_ns=initial.pre_event_ns, + post_event_ns=initial.post_event_ns, + affected_currencies=initial.affected_currencies, + affected_symbols=initial.affected_symbols, + confidence=1.0, + precision=initial.precision, + limitations=("Initial actual may be revised by the source.",), + vintage_id="initial-actual-v1", + revision_sequence=1, + supersedes_event_id=initial.event_id, + expected_value=2.0, + actual_value=3.0, + value_unit="percent_yoy", + tags=initial.tags, + ) + + +def _timeline( + *, + complete: bool = True, + events: tuple[MarketContextEventV1, ...] | None = None, +) -> MarketContextTimelineV1: + initial = _initial_event() + revision = _revision_event(initial) + return MarketContextTimelineV1( + timeline_version="fixture-2022-07-v1", + coverage_start_ns=EVENT_TIME_NS - 30 * DAY_NS, + coverage_end_ns=EVENT_TIME_NS + 30 * DAY_NS, + complete=complete, + events=events if events is not None else (revision, initial), + limitations=("Only operator-approved fixture events are included.",), + ) + + +def test_source_and_event_contracts_are_versioned_and_reproducible() -> None: + source = _source() + event = _initial_event(source=source) + + assert source.schema_version == MARKET_CONTEXT_SOURCE_SCHEMA_VERSION + assert event.schema_version == MARKET_CONTEXT_EVENT_SCHEMA_VERSION + assert event.source.content_sha256 == "a" * 64 + assert event.source.redistribution_allowed is False + assert event.source.redistribution_constraints + assert event.window_start_ns == EVENT_TIME_NS - event.pre_event_ns + assert event.window_end_ns == EVENT_TIME_NS + event.post_event_ns + assert MarketContextSourceV1.from_json(source.to_json()) == source + assert MarketContextEventV1.from_json(event.to_json()) == event + + +def test_timezone_normalization_rejects_ambiguous_and_nonexistent_times() -> ( + None +): + with pytest.raises(ValueError, match="ambiguous"): + normalize_market_context_datetime( + "2022-11-06T01:30:00", + "America/New_York", + ) + + first = normalize_market_context_datetime( + "2022-11-06T01:30:00", + "America/New_York", + fold=0, + ) + second = normalize_market_context_datetime( + "2022-11-06T01:30:00", + "America/New_York", + fold=1, + ) + assert second - first == HOUR_NS + + with pytest.raises(ValueError, match="nonexistent"): + normalize_market_context_datetime( + "2022-03-13T02:30:00", + "America/New_York", + ) + with pytest.raises(ValueError, match="does not match"): + normalize_market_context_datetime( + "2022-07-13T08:30:00-05:00", + "America/New_York", + ) + + +def test_event_time_must_match_source_timezone_normalization() -> None: + with pytest.raises(ValueError, match="does not match normalized"): + replace(_initial_event(), event_time_ns=EVENT_TIME_NS + 1) + + +def test_timeline_retains_initial_and_revision_without_overwrite() -> None: + timeline = _timeline() + + assert timeline.schema_version == MARKET_CONTEXT_TIMELINE_SCHEMA_VERSION + assert [item.revision_sequence for item in timeline.events] == [0, 1] + assert timeline.events[1].supersedes_event_id == timeline.events[0].event_id + assert timeline.events[0].actual_value is None + assert timeline.events[1].actual_value == 3.0 + assert timeline.events[1].surprise_value == 1.0 + assert MarketContextTimelineV1.from_json(timeline.to_json()) == timeline + + +def test_timeline_rejects_duplicate_and_invalid_revision_fixtures() -> None: + initial = _initial_event() + revision = _revision_event(initial) + + with pytest.raises(ValueError, match="duplicate market context event_id"): + _timeline(events=(initial, initial)) + + orphan = replace( + revision, + supersedes_event_id="market-context-event:sha256:" + "0" * 64, + event_id="", + ) + with pytest.raises(ValueError, match="predecessor is absent"): + _timeline(events=(initial, orphan)) + + competing_initial = replace( + initial, + source=_source(version="competing-source", digest="c" * 64), + vintage_id="competing-vintage", + event_id="", + ) + with pytest.raises(ValueError, match="duplicate logical"): + _timeline(events=(initial, competing_initial)) + + +def test_ex_ante_view_hides_unavailable_actual_and_revision() -> None: + timeline = _timeline() + before_release = query_market_context( + timeline, + start_ns=EVENT_TIME_NS - HOUR_NS, + end_ns=EVENT_TIME_NS + HOUR_NS, + view=MarketContextView.EX_ANTE, + as_of_ns=EVENT_TIME_NS - DAY_NS, + currencies=("USD",), + ) + after_release = query_market_context( + timeline, + start_ns=EVENT_TIME_NS - HOUR_NS, + end_ns=EVENT_TIME_NS + HOUR_NS, + view=MarketContextView.EX_ANTE, + as_of_ns=EVENT_TIME_NS + HOUR_NS, + currencies=("USD",), + ) + ex_post = query_market_context( + timeline, + start_ns=EVENT_TIME_NS - HOUR_NS, + end_ns=EVENT_TIME_NS + HOUR_NS, + view=MarketContextView.EX_POST, + currencies=("USD",), + ) + + assert [item.revision_sequence for item in before_release.events] == [0] + assert [item.revision_sequence for item in after_release.events] == [0, 1] + assert [item.revision_sequence for item in ex_post.events] == [0, 1] + assert ( + MarketContextQueryV1.from_json(after_release.to_json()) == after_release + ) + + +def test_ex_ante_view_reports_context_not_yet_available() -> None: + result = query_market_context( + _timeline(), + start_ns=EVENT_TIME_NS - HOUR_NS, + end_ns=EVENT_TIME_NS + HOUR_NS, + view=MarketContextView.EX_ANTE, + as_of_ns=EVENT_TIME_NS - 8 * DAY_NS, + ) + + assert result.status is MarketContextQueryStatus.MISSING + assert result.missing_reason is ( + MarketContextMissingReason.NOT_AVAILABLE_AS_OF + ) + + +def test_information_bridge_preserves_vintages_and_revision_lineage() -> None: + query = query_market_context( + _timeline(), + start_ns=EVENT_TIME_NS - HOUR_NS, + end_ns=EVENT_TIME_NS + HOUR_NS, + view=MarketContextView.EX_ANTE, + as_of_ns=EVENT_TIME_NS + HOUR_NS, + ) + + inputs = market_context_information_inputs( + query, + run_id="run-context-fixture", + used_at_ns=EVENT_TIME_NS + HOUR_NS, + ) + + assert len(inputs) == 2 + assert all( + item.information_mode is InformationMode.EX_ANTE_SIMULATION + for item in inputs + ) + assert inputs[0].scope is InformationScope.POINT_IN_TIME + assert inputs[1].scope is InformationScope.REVISION + assert inputs[1].supersedes_input_id == inputs[0].input_id + assert all(item.available_at_ns <= item.used_at_ns for item in inputs) + + +def test_known_schedule_binds_at_knowledge_time_not_future_release_time() -> ( + None +): + used_at_ns = EVENT_TIME_NS - DAY_NS + query = query_market_context( + _timeline(), + start_ns=EVENT_TIME_NS - HOUR_NS, + end_ns=EVENT_TIME_NS + HOUR_NS, + view=MarketContextView.EX_ANTE, + as_of_ns=used_at_ns, + ) + + inputs = market_context_information_inputs( + query, + run_id="run-scheduled-context-fixture", + used_at_ns=used_at_ns, + ) + + assert len(inputs) == 1 + assert inputs[0].event_time_ns == query.events[0].available_at_ns + assert inputs[0].event_time_ns < query.events[0].event_time_ns + assert inputs[0].event_time_ns <= inputs[0].used_at_ns + + +def test_point_in_time_context_binding_passes_existing_leakage_audit() -> None: + used_at_ns = EVENT_TIME_NS + HOUR_NS + policy = ReconstructionInformationPolicyV1( + information_mode=InformationMode.EX_ANTE_SIMULATION, + ) + run = ReconstructionRunV1( + symbols=("EURUSD", "GBPUSD"), + source_version_ids=("source:fixture",), + configuration_ids=(policy.policy_id, "context:fixture"), + ensemble_member_ids=("member-000",), + base_seed=437, + ) + window = ReconstructionWindowV1( + run_id=run.run_id, + ensemble_member_id="member-000", + symbols=run.symbols, + core_start_ns=EVENT_TIME_NS - HOUR_NS, + core_end_ns=EVENT_TIME_NS + 2 * HOUR_NS, + ) + query = query_market_context_window( + _timeline(), + window, + view=MarketContextView.EX_ANTE, + as_of_ns=used_at_ns, + ) + inputs = market_context_information_inputs( + query, + run_id=run.run_id, + used_at_ns=used_at_ns, + ) + splits = ( + ReconstructionInformationSplitV1( + kind=InformationSplitKind.TRAIN, + start_ns=EVENT_TIME_NS - 30 * DAY_NS, + end_ns=EVENT_TIME_NS - 20 * DAY_NS, + ), + ReconstructionInformationSplitV1( + kind=InformationSplitKind.CALIBRATION, + start_ns=EVENT_TIME_NS - 20 * DAY_NS, + end_ns=EVENT_TIME_NS - 10 * DAY_NS, + ), + ReconstructionInformationSplitV1( + kind=InformationSplitKind.VALIDATION, + start_ns=EVENT_TIME_NS - 10 * DAY_NS, + end_ns=EVENT_TIME_NS + 10 * DAY_NS, + ), + ) + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id((window,)), + inputs=inputs, + splits=splits, + ) + + report = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=(window,), + ) + + assert report.accepted is True + assert report.total_violation_count == 0 + + +def test_pre_and_post_event_windows_have_half_open_overlap_semantics() -> None: + event = _initial_event() + timeline = _timeline(events=(event,)) + + before = query_market_context( + timeline, + start_ns=event.window_start_ns - 2, + end_ns=event.window_start_ns, + view=MarketContextView.EX_POST, + include_calendar=False, + ) + boundary = query_market_context( + timeline, + start_ns=event.window_start_ns, + end_ns=event.window_start_ns + 1, + view=MarketContextView.EX_POST, + include_calendar=False, + ) + after = query_market_context( + timeline, + start_ns=event.window_end_ns, + end_ns=event.window_end_ns + 1, + view=MarketContextView.EX_POST, + include_calendar=False, + ) + + assert before.status is MarketContextQueryStatus.MISSING + assert [item.event_id for item in boundary.events] == [event.event_id] + assert after.status is MarketContextQueryStatus.MISSING + + +def test_unscheduled_context_cannot_silently_claim_exact_time() -> None: + initial = _initial_event() + + with pytest.raises(ValueError, match="requires non-exact precision"): + replace( + initial, + kind=MarketContextKind.UNSCHEDULED_SHOCK, + canonical_key="us.unscheduled.fixture", + event_id="", + ) + + shock = replace( + initial, + kind=MarketContextKind.UNSCHEDULED_SHOCK, + canonical_key="us.unscheduled.fixture", + precision=MarketContextPrecision.WINDOW_ONLY, + ambiguity_reason="Publication time is only known to the minute.", + confidence=0.6, + event_id="", + ) + assert shock.precision is MarketContextPrecision.WINDOW_ONLY + assert shock.ambiguity_reason + assert shock.confidence == 0.6 + + +def test_streaming_window_join_is_bounded_and_reuses_calendar_state() -> None: + window = ReconstructionWindowV1( + run_id="run-context-fixture", + ensemble_member_id="member-000", + symbols=("EURUSD", "GBPUSD"), + core_start_ns=EVENT_TIME_NS - HOUR_NS, + core_end_ns=EVENT_TIME_NS + HOUR_NS, + ) + + result = query_market_context_window( + _timeline(), + window, + view=MarketContextView.EX_ANTE, + as_of_ns=EVENT_TIME_NS + HOUR_NS, + currencies=("USD",), + ) + + assert result.window_id == window.window_id + assert len(result.events) == 2 + assert result.calendar_state is not None + assert result.calendar_state.session_state == "market_open" + assert "london" in result.calendar_state.clock_sessions + assert "rows" not in result.to_dict() + assert "dataframe" not in result.to_dict() + + with pytest.raises(MarketContextQueryLimitError): + query_market_context_window( + _timeline(), + window, + view=MarketContextView.EX_POST, + max_events=1, + ) + + +def test_streaming_window_matches_currency_only_context() -> None: + event = replace(_initial_event(), affected_symbols=(), event_id="") + window = ReconstructionWindowV1( + run_id="run-currency-context-fixture", + ensemble_member_id="member-000", + symbols=("EURUSD",), + core_start_ns=EVENT_TIME_NS - HOUR_NS, + core_end_ns=EVENT_TIME_NS + HOUR_NS, + ) + + result = query_market_context_window( + _timeline(events=(event,)), + window, + view=MarketContextView.EX_POST, + ) + + assert [item.event_id for item in result.events] == [event.event_id] + assert result.requested_currencies == ("EUR", "USD") + + +def test_missing_context_is_explicit_inside_outside_and_incomplete() -> None: + timeline = _timeline() + quiet_start = EVENT_TIME_NS + 10 * DAY_NS + + quiet = query_market_context( + timeline, + start_ns=quiet_start, + end_ns=quiet_start + HOUR_NS, + view=MarketContextView.EX_POST, + ) + outside = query_market_context( + timeline, + start_ns=timeline.coverage_end_ns, + end_ns=timeline.coverage_end_ns + HOUR_NS, + view=MarketContextView.EX_POST, + ) + incomplete = query_market_context( + _timeline(complete=False), + start_ns=quiet_start, + end_ns=quiet_start + HOUR_NS, + view=MarketContextView.EX_POST, + ) + + assert quiet.missing_reason is MarketContextMissingReason.NO_MATCHING_EVENT + assert outside.missing_reason is ( + MarketContextMissingReason.OUTSIDE_TIMELINE_COVERAGE + ) + assert incomplete.missing_reason is ( + MarketContextMissingReason.TIMELINE_INCOMPLETE + ) + assert quiet.calendar_state is not None + + +def test_adapter_seam_retains_source_identity_and_rejects_mismatch() -> None: + initial = _initial_event() + adapter = StaticMarketContextSourceAdapterV1( + adapter_name="fixture-macro", + adapter_version="1.0", + events=(initial,), + ) + + timeline = build_market_context_timeline( + (adapter,), + timeline_version="adapter-fixture-v1", + coverage_start_ns=EVENT_TIME_NS - DAY_NS, + coverage_end_ns=EVENT_TIME_NS + DAY_NS, + complete=True, + limitations=("Fixture adapter only.",), + ) + + assert timeline.events[0].source.source_id == initial.source.source_id + assert timeline.events[0].source.content_sha256 == "a" * 64 + + with pytest.raises(ValueError, match="adapter_name does not match"): + StaticMarketContextSourceAdapterV1( + adapter_name="wrong-adapter", + adapter_version="1.0", + events=(initial,), + ) + + +def test_source_requires_explicit_nonredistribution_constraints() -> None: + with pytest.raises(ValueError, match="require explicit constraints"): + replace( + _source(), + redistribution_constraints=(), + source_id="", + ) + + +def test_calendar_state_reuses_rollover_and_period_end_classifier() -> None: + timestamp_ns = normalize_market_context_datetime( + "2022-03-31T21:59:00Z", + "UTC", + ) + + state = market_context_calendar_state(timestamp_ns) + + assert "daily_rollover" in state.special_tags + assert "month_end" in state.special_tags + assert "quarter_end" in state.special_tags + assert state.profile_source == "static_month_day_major_holidays" + assert state.profile_complete is False + assert state.limitations diff --git a/tests/unit/test_market_context_corpus.py b/tests/unit/test_market_context_corpus.py new file mode 100644 index 00000000..cfc02f06 --- /dev/null +++ b/tests/unit/test_market_context_corpus.py @@ -0,0 +1,1006 @@ +"""Production-source and artifact tests for the market-context corpus.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from datetime import date, timedelta +from pathlib import Path + +import pytest + +from histdatacom.data_analytics.cli import main as analytics_main +from histdatacom.market_context import ( + MARKET_CONTEXT_CORPUS_SCHEMA_VERSION, + BankOfEnglandBankRateAdapterV1, + EcbPolicyRateAdapterV1, + FederalReserveFomcCalendarAdapterV1, + FederalReserveFomcHistoricalAdapterV1, + MarketContextCorpusBuildV1, + MarketContextCorpusPreflightError, + MarketContextFetchProfileV1, + MarketContextKind, + MarketContextSourceEvidenceV1, + MarketContextSourceSnapshotV1, + MarketContextTimelineV1, + MarketContextView, + OnsReleaseCalendarAdapterV1, + OperatorMarketContextCatalogAdapterV1, + build_live_market_context_corpus, + build_market_context_corpus_from_snapshots, + market_context_benchmark_event_state, + normalize_market_context_datetime, + packaged_operator_catalog_path, + preflight_market_context_corpus, + query_market_context, + query_market_context_corpus, + read_market_context_corpus, + replay_market_context_corpus, + require_market_context_corpus, + write_market_context_corpus, +) + +DAY_NS = 86_400_000_000_000 +RETRIEVED_NS = normalize_market_context_datetime( + "2030-01-01T00:00:00+00:00", "UTC" +) + + +def _snapshot( + *, + source_key: str, + content: bytes, + adapter_name: str, + content_type: str, +) -> MarketContextSourceSnapshotV1: + return MarketContextSourceSnapshotV1( + source_key=source_key, + source_name=f"Fixture {source_key}", + source_uri=f"https://example.invalid/{source_key}", + retrieved_at_ns=RETRIEVED_NS, + content=content, + content_type=content_type, + adapter_name=adapter_name, + adapter_version="1.0", + license_name="Official-source fixture terms", + redistribution_allowed=True, + redistribution_constraints=("Attribute the fixture provider.",), + limitations=("Parser fixture, not a production acquisition.",), + metadata={"fixture": True}, + ) + + +def _operator_snapshot( + payload: dict[str, object], + *, + source_key: str = "operator.shock-catalog", +) -> MarketContextSourceSnapshotV1: + return _snapshot( + source_key=source_key, + content=json.dumps(payload, sort_keys=True).encode("utf-8"), + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + content_type="application/json", + ) + + +def _ecb_daily_csv( + start: date, + end: date, + changes: dict[date, float], +) -> bytes: + rows = ["KEY,TIME_PERIOD,OBS_VALUE,TITLE"] + current: float | None = None + observed = start + while observed <= end: + if observed in changes: + current = changes[observed] + if current is None: + raise AssertionError("ECB fixture requires an initial level") + rows.append( + "FM.D.U2.EUR.4F.KR.MRR_RT.LEV," + f"{observed.isoformat()},{current},Main refinancing operations" + ) + observed += timedelta(days=1) + return ("\n".join(rows) + "\n").encode("utf-8") + + +def test_ons_adapter_is_point_in_time_conservative_and_filters_titles() -> None: + payload = { + "breakdown": {"total": 4}, + "releases": [ + { + "uri": "/releases/consumerpriceinflationukjanuary2025", + "date_changes": [ + { + "previous_date": "2025-02-18T07:00:00.000Z", + "change_notice": "Rescheduled.", + } + ], + "description": { + "title": "Consumer price inflation, UK: January 2025", + "release_date": "2025-02-19T07:00:00.000Z", + "cancelled": False, + }, + }, + { + "uri": "/releases/unrelated", + "date_changes": None, + "description": { + "title": "Population estimates", + "release_date": "2025-02-20T07:00:00.000Z", + "cancelled": False, + }, + }, + { + "uri": "/releases/retailsalescancelled", + "date_changes": None, + "description": { + "title": "Retail sales, Great Britain: January 2025", + "release_date": "2025-02-21T07:00:00.000Z", + "cancelled": True, + }, + }, + { + "uri": "/releases/labourmarketmissingdate", + "date_changes": None, + "description": { + "title": "Labour market overview, UK: February 2025", + "release_date": "", + "cancelled": False, + }, + }, + ], + } + adapter = OnsReleaseCalendarAdapterV1( + _snapshot( + source_key="ons.q00.p00", + content=json.dumps(payload).encode("utf-8"), + adapter_name=OnsReleaseCalendarAdapterV1.adapter_name, + content_type="application/json", + ) + ) + + events = tuple(adapter.load_events()) + + assert len(events) == 1 + event = events[0] + assert event.kind is MarketContextKind.MACRO_RELEASE + assert event.affected_currencies == ("GBP",) + assert event.first_known_at_ns == event.event_time_ns + assert event.available_at_ns == event.event_time_ns + assert "schedule_changed_without_change_timestamp" in event.tags + assert "cancelled_release:2" in adapter.diagnostics + assert "missing_release_field:3" in adapter.diagnostics + + +def test_ecb_and_boe_adapters_preserve_date_only_limitations() -> None: + ecb_csv = ( + "TIME_PERIOD,OBS_VALUE,TITLE,OBS_STATUS\n" + "2022-07-27,0.5,Main refinancing operations,A\n" + "2022-09-14,,Main refinancing operations,A\n" + ).encode("utf-8") + ecb = EcbPolicyRateAdapterV1( + _snapshot( + source_key="ecb.policy-rate", + content=ecb_csv, + adapter_name=EcbPolicyRateAdapterV1.adapter_name, + content_type="text/csv", + ) + ) + ecb_events = tuple(ecb.load_events()) + + boe_html = b""" + + +
Date ChangedRate
07 Nov 244.75
missing
+ """ + boe = BankOfEnglandBankRateAdapterV1( + _snapshot( + source_key="boe.bank-rate", + content=boe_html, + adapter_name=BankOfEnglandBankRateAdapterV1.adapter_name, + content_type="text/html", + ) + ) + boe_events = tuple(boe.load_events()) + + assert len(ecb_events) == 1 + assert ecb_events[0].actual_value == 0.5 + assert ecb_events[0].precision.value == "window_only" + assert "missing_rate_field:1" in ecb.diagnostics + assert len(boe_events) == 1 + assert boe_events[0].actual_value == 4.75 + assert boe_events[0].affected_symbols == ("EURGBP", "GBPUSD") + assert "invalid_bank_rate_row:1" in boe.diagnostics + + +def test_ecb_adapter_collapses_daily_state_across_variable_rate_era() -> None: + ecb = EcbPolicyRateAdapterV1( + _snapshot( + source_key="ecb.policy-rate", + content=_ecb_daily_csv( + date(2002, 12, 1), + date(2003, 3, 10), + { + date(2002, 12, 1): 3.25, + date(2002, 12, 6): 2.75, + date(2003, 3, 7): 2.5, + }, + ), + adapter_name=EcbPolicyRateAdapterV1.adapter_name, + content_type="text/csv", + ) + ) + + events = tuple(ecb.load_events()) + + assert [item.actual_value for item in events] == [3.25, 2.75, 2.5] + assert [item.previous_value for item in events] == [None, 3.25, 2.75] + assert events[1].source_event_time == "2002-12-06T00:00:00+00:00" + assert "effective_rate_change" in events[1].tags + assert ecb.coverage_complete is True + + +def test_ecb_adapter_marks_non_contiguous_daily_state_incomplete() -> None: + ecb = EcbPolicyRateAdapterV1( + _snapshot( + source_key="ecb.policy-rate", + content=( + b"KEY,TIME_PERIOD,OBS_VALUE,TITLE\n" + b"FM.D.U2.EUR.4F.KR.MRR_RT.LEV,2002-01-01,3.25,MRO\n" + b"FM.D.U2.EUR.4F.KR.MRR_RT.LEV,2002-01-03,3.25,MRO\n" + ), + adapter_name=EcbPolicyRateAdapterV1.adapter_name, + content_type="text/csv", + ) + ) + + tuple(ecb.load_events()) + + assert ecb.coverage_complete is False + assert ecb.diagnostics == ( + "non_contiguous_rate_date:2002-01-01:2002-01-03", + ) + + +def test_ecb_coverage_is_policy_change_not_decision_completeness() -> None: + snapshot = _snapshot( + source_key="ecb.policy-rate", + content=_ecb_daily_csv( + date(2023, 1, 1), + date(2023, 1, 31), + {date(2023, 1, 1): 2.5}, + ), + adapter_name=EcbPolicyRateAdapterV1.adapter_name, + content_type="text/csv", + ) + corpus = build_market_context_corpus_from_snapshots( + (snapshot,), + profile=MarketContextFetchProfileV1( + start_date="2023-01-01", + end_date="2023-01-31", + sources=("ecb",), + ), + ).corpus + + assert ( + corpus.timeline.events[0].kind is MarketContextKind.POLICY_RATE_CHANGE + ) + assert [item.kind for item in corpus.coverage] == [ + MarketContextKind.POLICY_RATE_CHANGE + ] + + +def test_federal_reserve_adapters_cover_current_cross_month_and_history() -> ( + None +): + current_html = b""" +

2024 FOMC Meetings

+
Apr/May
+
30-1
+
August
+
22 (notation vote)
+ """ + current = FederalReserveFomcCalendarAdapterV1( + _snapshot( + source_key="fed.fomc-calendar", + content=current_html, + adapter_name=FederalReserveFomcCalendarAdapterV1.adapter_name, + content_type="text/html", + ) + ) + current_events = tuple(current.load_events()) + + historical_html = b""" +
January 29-30 Meeting - 2008
+
March 10 Conference Call - 2008
+
April 29-30 (cancelled) Meeting - 2008
+
October 8 (unscheduled) Meeting - 2008
+ """ + historical = FederalReserveFomcHistoricalAdapterV1( + _snapshot( + source_key="fed.fomc-historical.2008", + content=historical_html, + adapter_name=FederalReserveFomcHistoricalAdapterV1.adapter_name, + content_type="text/html", + ) + ) + historical_events = tuple(historical.load_events()) + + assert [item.canonical_key for item in current_events] == [ + "federal-reserve.fomc.2024-05-01", + "federal-reserve.fomc.2024-08-22", + ] + assert current_events[0].precision.value == "exact" + assert current_events[1].precision.value == "window_only" + assert [item.canonical_key for item in historical_events] == [ + "federal-reserve.fomc.2008-01-30", + "federal-reserve.fomc.2008-10-08", + ] + assert ( + historical_events[0].available_at_ns + > historical_events[0].event_time_ns + ) + assert "unscheduled" in historical_events[1].tags + assert "cancelled_historical_meeting:2008-04-30" in historical.diagnostics + + +def test_operator_adapter_retains_revisions_and_late_publication() -> None: + common = { + "affected_currencies": ["USD"], + "affected_symbols": ["EURUSD", "GBPUSD"], + "canonical_key": "operator.macro.fixture.2022-07-13", + "confidence": 1.0, + "first_known_at": "2022-07-06T08:30:00-04:00", + "kind": "macro_release", + "limitations": ["Point-in-time adapter fixture."], + "post_event_ns": 3_600_000_000_000, + "pre_event_ns": 1_800_000_000_000, + "precision": "exact", + "source_event_time": "2022-07-13T08:30:00-04:00", + "source_timezone": "America/New_York", + "tags": ["fixture", "scheduled"], + "title": "US CPI fixture", + "value_unit": "percent_yoy", + } + payload = { + "schema_version": "histdatacom.operator-market-context-catalog.v1", + "events": [ + { + **common, + "available_at": "2022-07-06T08:30:00-04:00", + "expected_value": 2.0, + "revision_sequence": 0, + "vintage_id": "schedule-v1", + }, + { + **common, + "actual_value": 3.0, + "available_at": "2022-07-13T10:00:00-04:00", + "expected_value": 2.0, + "revision_sequence": 1, + "vintage_id": "late-actual-v1", + }, + ], + } + events = tuple( + OperatorMarketContextCatalogAdapterV1( + _operator_snapshot(payload) + ).load_events() + ) + event_time = events[0].event_time_ns + timeline = MarketContextTimelineV1( + timeline_version="operator-revision-test", + coverage_start_ns=event_time - DAY_NS, + coverage_end_ns=event_time + DAY_NS, + complete=True, + events=events, + limitations=("Fixture timeline.",), + ) + + before_late_actual = query_market_context( + timeline, + start_ns=event_time - 1, + end_ns=event_time + 1, + view=MarketContextView.EX_ANTE, + as_of_ns=normalize_market_context_datetime( + "2022-07-13T09:00:00-04:00", "America/New_York" + ), + currencies=("USD",), + include_calendar=False, + ) + after_late_actual = query_market_context( + timeline, + start_ns=event_time - 1, + end_ns=event_time + 1, + view=MarketContextView.EX_ANTE, + as_of_ns=normalize_market_context_datetime( + "2022-07-13T10:01:00-04:00", "America/New_York" + ), + currencies=("USD",), + include_calendar=False, + ) + + assert [item.revision_sequence for item in events] == [0, 1] + assert events[1].supersedes_event_id == events[0].event_id + assert events[1].surprise_value == 1.0 + assert [item.revision_sequence for item in before_late_actual.events] == [0] + assert [item.revision_sequence for item in after_late_actual.events] == [ + 0, + 1, + ] + + +def test_operator_adapter_fails_closed_on_dst_and_missing_values() -> None: + event = { + "affected_currencies": ["USD"], + "affected_symbols": ["EURUSD"], + "ambiguity_reason": "Window-only fixture.", + "available_at": "2022-11-06T01:30:00", + "canonical_key": "operator.shock.dst.2022-11-06", + "confidence": 1.0, + "first_known_at": "2022-11-06T01:30:00", + "kind": "unscheduled_shock", + "limitations": ["DST fixture."], + "post_event_ns": DAY_NS, + "pre_event_ns": 0, + "precision": "window_only", + "revision_sequence": 0, + "source_event_time": "2022-11-06T01:30:00", + "source_timezone": "America/New_York", + "tags": ["fixture"], + "title": "DST fixture", + "vintage_id": "dst-v1", + } + payload = { + "schema_version": "histdatacom.operator-market-context-catalog.v1", + "events": [event], + } + with pytest.raises(ValueError, match="ambiguous"): + tuple( + OperatorMarketContextCatalogAdapterV1( + _operator_snapshot(payload) + ).load_events() + ) + + payload["events"] = [{**event, "source_time_fold": 1}] + parsed = tuple( + OperatorMarketContextCatalogAdapterV1( + _operator_snapshot(payload) + ).load_events() + ) + assert parsed[0].source_time_fold == 1 + + payload["events"] = [ + { + **{key: value for key, value in event.items() if key != "title"}, + "source_time_fold": 1, + } + ] + with pytest.raises(ValueError, match="required text"): + tuple( + OperatorMarketContextCatalogAdapterV1( + _operator_snapshot(payload) + ).load_events() + ) + + +def test_corpus_artifacts_replay_and_do_not_rewrite_prior_content( + tmp_path: Path, +) -> None: + content = packaged_operator_catalog_path().read_bytes() + snapshot = _snapshot( + source_key="operator.shock-catalog", + content=content, + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + content_type="application/json", + ) + profile = MarketContextFetchProfileV1( + start_date="2001-01-01", + end_date="2023-12-31", + sources=("operator",), + ) + build = build_market_context_corpus_from_snapshots( + (snapshot,), profile=profile + ) + + artifacts = write_market_context_corpus(build, tmp_path) + loaded = read_market_context_corpus(artifacts["corpus"].path) + replayed = replay_market_context_corpus(artifacts["corpus"].path) + repeated = write_market_context_corpus(build, tmp_path) + + assert loaded.schema_version == MARKET_CONTEXT_CORPUS_SCHEMA_VERSION + assert replayed.corpus.corpus_id == loaded.corpus_id + assert repeated["corpus"].sha256 == artifacts["corpus"].sha256 + assert len(loaded.timeline.events) == 6 + assert Path(artifacts["timeline"].path).exists() + raw = next((tmp_path / "sources").glob("operator.shock-catalog-*.json")) + raw.write_bytes(b"different") + with pytest.raises(ValueError, match="different content"): + write_market_context_corpus(build, tmp_path) + corpus_path = Path(artifacts["corpus"].path) + corpus_path.write_bytes(corpus_path.read_bytes() + b" ") + with pytest.raises(ValueError, match="hash differs from name"): + read_market_context_corpus(corpus_path) + + +def test_corpus_rejects_internally_inconsistent_reports_and_provenance() -> ( + None +): + snapshot = _snapshot( + source_key="operator.shock-catalog", + content=packaged_operator_catalog_path().read_bytes(), + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + content_type="application/json", + ) + build = build_market_context_corpus_from_snapshots( + (snapshot,), + profile=MarketContextFetchProfileV1( + start_date="2001-01-01", + end_date="2023-12-31", + sources=("operator",), + ), + ) + counts = dict(build.corpus.counts_by_year_currency_kind) + first_count = next(iter(counts)) + counts[first_count] += 1 + with pytest.raises(ValueError, match="counts differ"): + replace( + build.corpus, + counts_by_year_currency_kind=counts, + corpus_id="", + ) + + coverage = list(build.corpus.coverage) + coverage[0] = replace(coverage[0], event_count=coverage[0].event_count + 1) + with pytest.raises(ValueError, match="coverage count differs"): + replace(build.corpus, coverage=tuple(coverage), corpus_id="") + + events = list(build.corpus.timeline.events) + changed_source = replace( + events[0].source, + license_name="Different source terms", + source_id="", + ) + events[0] = replace(events[0], source=changed_source, event_id="") + changed_timeline = replace( + build.corpus.timeline, events=tuple(events), timeline_id="" + ) + with pytest.raises(ValueError, match="provenance differs"): + replace( + build.corpus, + timeline=changed_timeline, + corpus_id="", + ) + + +def test_corpus_build_rejects_snapshot_evidence_mismatch() -> None: + snapshot = _snapshot( + source_key="operator.shock-catalog", + content=packaged_operator_catalog_path().read_bytes(), + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + content_type="application/json", + ) + build = build_market_context_corpus_from_snapshots( + (snapshot,), + profile=MarketContextFetchProfileV1( + start_date="2001-01-01", + end_date="2023-12-31", + sources=("operator",), + ), + ) + changed_snapshot = replace( + snapshot, source_uri="https://example.invalid/changed-source" + ) + with pytest.raises(ValueError, match="snapshot differs"): + MarketContextCorpusBuildV1( + corpus=build.corpus, snapshots=(changed_snapshot,) + ) + + +def test_source_evidence_enforces_reuse_and_limitation_invariants() -> None: + snapshot = _snapshot( + source_key="operator.shock-catalog", + content=packaged_operator_catalog_path().read_bytes(), + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + content_type="application/json", + ) + evidence = MarketContextSourceEvidenceV1.from_snapshot( + snapshot, event_count=0 + ) + + with pytest.raises(ValueError, match="requires limitations"): + replace(evidence, limitations=()) + with pytest.raises(ValueError, match="requires constraints"): + replace( + evidence, + redistribution_allowed=False, + redistribution_constraints=(), + ) + + +@pytest.mark.parametrize( + ("timeout_seconds", "max_runtime_seconds"), + ((float("inf"), 300.0), (30.0, float("inf"))), +) +def test_fetch_profile_rejects_infinite_runtime_bounds( + timeout_seconds: float, max_runtime_seconds: float +) -> None: + with pytest.raises(ValueError, match="finite and positive"): + MarketContextFetchProfileV1( + start_date="2023-01-01", + end_date="2023-01-02", + sources=("operator",), + timeout_seconds=timeout_seconds, + max_runtime_seconds=max_runtime_seconds, + ) + + +def test_corpus_preflight_distinguishes_support_from_no_matching_event() -> ( + None +): + operator = _snapshot( + source_key="operator.shock-catalog", + content=packaged_operator_catalog_path().read_bytes(), + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + content_type="application/json", + ) + ecb = _snapshot( + source_key="ecb.policy-rate", + content=_ecb_daily_csv( + date(2022, 1, 1), + date(2023, 12, 31), + { + date(2022, 1, 1): 0.0, + date(2022, 7, 27): 0.5, + }, + ), + adapter_name=EcbPolicyRateAdapterV1.adapter_name, + content_type="text/csv", + ) + profile = MarketContextFetchProfileV1( + start_date="2001-01-01", + end_date="2023-12-31", + sources=("ecb", "operator"), + ) + corpus = build_market_context_corpus_from_snapshots( + (operator, ecb), profile=profile + ).corpus + start = normalize_market_context_datetime( + "2023-01-10T00:00:00+00:00", "UTC" + ) + end = start + DAY_NS + + ready = preflight_market_context_corpus( + corpus, + start_ns=start, + end_ns=end, + currencies=("EUR",), + kinds=(MarketContextKind.POLICY_RATE_CHANGE,), + ) + query = query_market_context_corpus( + corpus, + start_ns=start, + end_ns=end, + view=MarketContextView.EX_POST, + currencies=("EUR",), + include_calendar=False, + ) + + assert ready.ready is True + assert query.missing_reason is not None + assert query.missing_reason.value == "no_matching_event" + assert market_context_benchmark_event_state(query).endswith( + "no_matching_event" + ) + + unsupported = preflight_market_context_corpus( + corpus, + start_ns=start, + end_ns=end, + currencies=("USD",), + kinds=(MarketContextKind.CENTRAL_BANK_DECISION,), + ) + assert unsupported.ready is False + with pytest.raises(MarketContextCorpusPreflightError): + require_market_context_corpus( + corpus, + start_ns=start, + end_ns=end, + currencies=("USD",), + kinds=(MarketContextKind.CENTRAL_BANK_DECISION,), + ) + + shocks = preflight_market_context_corpus( + corpus, + start_ns=start, + end_ns=end, + currencies=("EUR",), + kinds=(MarketContextKind.UNSCHEDULED_SHOCK,), + ) + assert shocks.ready is False + + with pytest.raises(MarketContextCorpusPreflightError): + query_market_context_corpus( + corpus, + start_ns=start, + end_ns=end, + view=MarketContextView.EX_POST, + currencies=("USD",), + kinds=(MarketContextKind.CENTRAL_BANK_DECISION,), + include_calendar=False, + ) + + +def test_corpus_deduplicates_query_specific_ons_highlights() -> None: + item = { + "uri": "/releases/consumerpriceinflationukjanuary2025", + "date_changes": None, + "description": { + "title": "Consumer price inflation, UK: January 2025", + "release_date": "2025-02-19T07:00:00.000Z", + "cancelled": False, + }, + "highlight": {"title": "Consumer price inflation"}, + } + other = json.loads(json.dumps(item)) + other["highlight"] = {"title": "Consumer price inflation"} + snapshots = tuple( + _snapshot( + source_key=f"ons.q{index:02d}.p00", + content=json.dumps( + {"breakdown": {"total": 1}, "releases": [release]} + ).encode("utf-8"), + adapter_name=OnsReleaseCalendarAdapterV1.adapter_name, + content_type="application/json", + ) + for index, release in enumerate((item, other)) + ) + + corpus = build_market_context_corpus_from_snapshots( + snapshots, + profile=MarketContextFetchProfileV1( + start_date="2025-01-01", + end_date="2025-12-31", + sources=("ons",), + ), + ).corpus + + assert len(corpus.timeline.events) == 1 + assert corpus.duplicate_event_count == 1 + + +def test_corpus_rejects_conflicting_duplicate_logical_events() -> None: + first = { + "breakdown": {"total": 1}, + "releases": [ + { + "uri": "/releases/retailsalesgreatbritainjanuary2025", + "date_changes": None, + "description": { + "title": "Retail sales, Great Britain: January 2025", + "release_date": "2025-02-21T07:00:00.000Z", + "cancelled": False, + }, + } + ], + } + second = json.loads(json.dumps(first)) + second["releases"][0]["description"][ + "release_date" + ] = "2025-02-21T08:00:00.000Z" + snapshots = ( + _snapshot( + source_key="ons.q00.p00", + content=json.dumps(first).encode("utf-8"), + adapter_name=OnsReleaseCalendarAdapterV1.adapter_name, + content_type="application/json", + ), + _snapshot( + source_key="ons.q01.p00", + content=json.dumps(second).encode("utf-8"), + adapter_name=OnsReleaseCalendarAdapterV1.adapter_name, + content_type="application/json", + ), + ) + profile = MarketContextFetchProfileV1( + start_date="2025-01-01", + end_date="2025-12-31", + sources=("ons",), + ) + + with pytest.raises(ValueError, match="conflicting duplicate"): + build_market_context_corpus_from_snapshots(snapshots, profile=profile) + + +def test_snapshot_builder_rejects_profile_source_drift() -> None: + operator = _snapshot( + source_key="operator.shock-catalog", + content=packaged_operator_catalog_path().read_bytes(), + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + content_type="application/json", + ) + + with pytest.raises(ValueError, match="families differ"): + build_market_context_corpus_from_snapshots( + (operator,), + profile=MarketContextFetchProfileV1( + start_date="2001-01-01", + end_date="2023-12-31", + sources=("operator", "ecb"), + ), + ) + + +def test_live_acquisition_enforces_running_total_byte_budget( + monkeypatch, +) -> None: + calls: list[str] = [] + closed: list[bool] = [] + + class _Response: + headers = { + "Content-Length": "6", + "Content-Type": "application/octet-stream", + } + + def raise_for_status(self) -> None: + return None + + def iter_content(self, *, chunk_size: int): + del chunk_size + yield b"123456" + + def close(self) -> None: + closed.append(True) + + def fake_get(uri, **kwargs): + del kwargs + calls.append(uri) + return _Response() + + import histdatacom.market_context.corpus as corpus_module + + monkeypatch.setattr(corpus_module.requests, "get", fake_get) + profile = MarketContextFetchProfileV1( + start_date="2023-01-01", + end_date="2023-01-02", + sources=("ecb", "boe"), + max_response_bytes=10, + max_total_source_bytes=10, + ) + + with pytest.raises(ValueError, match="declared byte limit"): + build_live_market_context_corpus(profile) + + assert len(calls) == 2 + assert len(closed) == 2 + + +def test_packaged_date_only_shock_is_not_visible_intraday_ex_ante() -> None: + snapshot = _snapshot( + source_key="operator.shock-catalog", + content=packaged_operator_catalog_path().read_bytes(), + adapter_name=OperatorMarketContextCatalogAdapterV1.adapter_name, + content_type="application/json", + ) + corpus = build_market_context_corpus_from_snapshots( + (snapshot,), + profile=MarketContextFetchProfileV1( + start_date="2001-09-11", + end_date="2001-09-11", + sources=("operator",), + ), + ).corpus + start = normalize_market_context_datetime( + "2001-09-11T00:00:00+00:00", "UTC" + ) + + hidden = query_market_context_corpus( + corpus, + start_ns=start, + end_ns=start + DAY_NS, + view=MarketContextView.EX_ANTE, + as_of_ns=start + 60 * 60 * 1_000_000_000, + currencies=("USD",), + include_calendar=False, + ) + visible = query_market_context_corpus( + corpus, + start_ns=start, + end_ns=start + DAY_NS, + view=MarketContextView.EX_ANTE, + as_of_ns=start + DAY_NS, + currencies=("USD",), + include_calendar=False, + ) + + assert hidden.missing_reason is not None + assert hidden.missing_reason.value == "not_available_as_of" + assert [item.title for item in visible.events] == ["September 11 attacks"] + + +def test_market_context_cli_writes_operator_only_corpus( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + artifact_dir = tmp_path / "context" + + exit_code = analytics_main( + [ + "market-context-corpus", + "--artifact-dir", + str(artifact_dir), + "--start-date", + "2001-01-01", + "--end-date", + "2023-12-31", + "--sources", + "operator", + "--json", + ] + ) + + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["source_count"] == 1 + assert payload["event_count"] == 6 + assert Path(payload["artifacts"]["corpus"]["path"]).exists() + assert Path(payload["artifacts"]["timeline"]["path"]).exists() + + +def test_market_context_cli_reads_yaml_config( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + artifact_dir = tmp_path / "configured-context" + config_path = tmp_path / "histdatacom.yaml" + config_path.write_text( + "\n".join( + ( + "histdatacom:", + " analytics:", + " command: market-context-corpus", + f" artifact_dir: {artifact_dir}", + " start_date: 2001-01-01", + " end_date: 2023-12-31", + " sources:", + " - operator", + " ons_queries:", + " - consumer price inflation", + " max_events: 32", + " max_runtime_seconds: 30", + " json: true", + ) + ), + encoding="utf-8", + ) + + exit_code = analytics_main(["--config", str(config_path)]) + + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["source_count"] == 1 + assert payload["event_count"] == 6 + + +def test_market_context_cli_reports_invalid_profile_cleanly( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + exit_code = analytics_main( + [ + "market-context-corpus", + "--artifact-dir", + str(tmp_path), + "--start-date", + "2024-01-02", + "--end-date", + "2024-01-01", + "--sources", + "operator", + ] + ) + + assert exit_code == 1 + assert "market-context corpus error:" in capsys.readouterr().err diff --git a/tests/unit/test_modern_motif_library.py b/tests/unit/test_modern_motif_library.py new file mode 100644 index 00000000..ec2a9751 --- /dev/null +++ b/tests/unit/test_modern_motif_library.py @@ -0,0 +1,116 @@ +"""Tests for the installed modern reference-motif library boundary.""" + +from __future__ import annotations + +from collections import Counter +import hashlib +import json +from pathlib import Path + +import pytest + +from histdatacom.data_analytics.cli import build_parser +from histdatacom.synthetic.motif_library import ( + MODERN_REFERENCE_MOTIF_COVERAGE_SCHEMA_VERSION, + ModernReferenceMotifProfileV1, + _coverage_axis_rates, + read_modern_reference_motif_artifact, +) + + +def test_profile_round_trip_is_strict_and_chronological() -> None: + profile = ModernReferenceMotifProfileV1() + + assert ModernReferenceMotifProfileV1.from_dict(profile.to_dict()) == profile + canonical_payload = json.loads( + json.dumps(profile.to_dict(), sort_keys=True, separators=(",", ":")) + ) + assert ModernReferenceMotifProfileV1.from_dict(canonical_payload) == profile + assert profile.split_periods["train"][0] == "201901" + assert profile.split_periods["final_holdout"] == ("202510",) + assert profile.max_fragments == 256 + assert profile.max_matches == 64 + + with pytest.raises(ValueError, match="overlap or regress"): + ModernReferenceMotifProfileV1( + split_periods={ + "train": ("202401",), + "calibration": ("202307",), + "validation": ("202501",), + "final_holdout": ("202510",), + } + ) + + +def test_content_addressed_companion_reader_rejects_tampering( + tmp_path: Path, +) -> None: + payload = { + "schema_version": MODERN_REFERENCE_MOTIF_COVERAGE_SCHEMA_VERSION, + "index_id": "reference-motif-index:sha256:" + "0" * 64, + } + content = ( + json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + digest = hashlib.sha256(content).hexdigest() + path = tmp_path / f"modern-reference-motif-coverage-{digest}.json" + path.write_bytes(content) + + assert ( + read_modern_reference_motif_artifact(path, kind="coverage") == payload + ) + path.write_bytes(content + b" ") + with pytest.raises(ValueError, match="content hash differs"): + read_modern_reference_motif_artifact(path, kind="coverage") + + +def test_coverage_axis_rates_keep_support_refusal_and_backoff_visible() -> None: + summary = _coverage_axis_rates( + Counter( + { + "query_count": 4, + "status:matched": 3, + "status:no_supported_cell": 1, + "backoff:exact": 1, + "backoff:symbol_epoch_session": 2, + } + ) + ) + + assert summary == { + "query_count": 4, + "status_counts": {"matched": 3, "no_supported_cell": 1}, + "backoff_counts": {"exact": 1, "symbol_epoch_session": 2}, + "backoff_rates": {"exact": 0.25, "symbol_epoch_session": 0.5}, + } + + +def test_cli_exposes_installed_modern_motif_library_command() -> None: + args = build_parser().parse_args( + [ + "modern-reference-motif-library", + "--source-root", + "ticks", + "--definition", + "epochs.json", + "--market-context-corpus", + "context.json", + "--cftc-positioning-corpus", + "positioning.json", + "--benchmark-manifest", + "benchmark.json", + "--artifact-dir", + "artifacts", + ] + ) + + assert args.analytics_command == "modern-reference-motif-library" + assert args.max_fragments == 256 + assert args.max_matches == 64 + assert args.train_periods == ( + "201901", + "202001", + "202101", + "202201", + "202301", + ) diff --git a/tests/unit/test_orchestration_activities.py b/tests/unit/test_orchestration_activities.py index 8112155d..f6c2c751 100644 --- a/tests/unit/test_orchestration_activities.py +++ b/tests/unit/test_orchestration_activities.py @@ -44,6 +44,10 @@ QualityTargetKind, ) from histdatacom.runtime_contracts import RunRequest, WorkItem, WorkStatus +from histdatacom.random_windows import ( + RANDOM_WINDOW_SELECTION_METADATA_KEY, + RandomWindowSelectionV1, +) from histdatacom.orchestration.control import OrchestrationJobSnapshot from histdatacom.orchestration.activities import ( build_cache_activity, @@ -54,6 +58,8 @@ extract_csv_activity, import_to_influx_activity, merge_cache_activity, + reconstruction_report_activity, + reconstruction_window_activity, repository_refresh_activity, validate_urls_activity, ) @@ -61,6 +67,7 @@ CLEAN_TICK_CASE, write_ascii_case, write_corrupt_zip, + write_zip_case, ) FIXTURES = Path(__file__).parents[1] / "fixtures" / "histdata_ascii" @@ -95,14 +102,14 @@ def write_lines(self, lines: list[str]) -> None: self.batches.append(list(lines)) -def _form_html(*, token: str = "token") -> str: +def _form_html(*, token: str = "token", datemonth: str = "2022") -> str: """Return a minimal HistData download form.""" return f"""
- + @@ -157,6 +164,8 @@ def _download_payload(tmp_path) -> dict: ), "data_dir": f"{tmp_path}/", "zip_filename": zip_path.name, + "data_format": "ASCII", + "data_timeframe": "T", }, } @@ -176,6 +185,8 @@ def _extraction_payload(tmp_path) -> dict: "status": WorkStatus.CSV_ZIP.value, "data_dir": f"{tmp_path}/", "zip_filename": zip_path.name, + "data_format": "ASCII", + "data_timeframe": "T", }, } @@ -408,6 +419,38 @@ def test_dataset_plan_activity_uses_repo_ranges_for_full_scope( assert result["result"]["metrics"]["repository_range_count"] == 3 +def test_dataset_plan_activity_random_window_intersects_repo_and_user_bounds( + tmp_path: Path, +) -> None: + """Random planning should load inventory even when user bounds are present.""" + write_repository_data_file( + {"eurusd": {"start": "202001", "end": "202012"}}, + tmp_path / ".repo", + ) + request = RunRequest( + request_id="run-plan-random-common-support", + pairs=("eurusd",), + formats=("ascii",), + timeframes=("T",), + start_yearmonth="201001", + end_yearmonth="202512", + random_window="1M", + random_seed=81, + data_directory=str(tmp_path), + ) + + result = dataset_plan_activity({"request": request.to_dict()}) + selection = RandomWindowSelectionV1.from_dict( + result["result"]["metrics"]["random_window_selection"] + ) + + assert result["result"]["metrics"]["repository_range_count"] == 1 + assert selection.support_start_utc_ms == 1_577_836_800_000 + assert selection.support_end_utc_ms == 1_609_459_200_000 + assert len(result["work_items"]) == 1 + assert result["work_items"][0]["data_datemonth"].startswith("2020") + + def test_dataset_plan_activity_spills_large_plan_to_manifest( tmp_path: Path, ) -> None: @@ -443,6 +486,76 @@ def test_dataset_plan_activity_spills_large_plan_to_manifest( ] +def test_random_window_metadata_survives_plan_spill_and_reload( + tmp_path: Path, +) -> None: + """SQLite plan batching should retain the exact compact selection contract.""" + write_repository_data_file( + {"eurusd": {"start": "202001", "end": "202412"}}, + tmp_path / ".repo", + ) + request = RunRequest( + request_id="run-plan-random-spill", + pairs=("eurusd",), + formats=("ascii",), + timeframes=("T",), + random_window="40d", + random_seed=42, + data_directory=str(tmp_path), + metadata={ + "temporal_plan_spill": {"inline_work_item_limit": 1}, + "temporal_batching": {"max_work_items_per_batch": 1}, + }, + ) + + result = dataset_plan_activity({"request": request.to_dict()}) + store = ManifestStatusStore(str(tmp_path)) + loaded = store.get_dataset_plan_work_items( + str(result[DATASET_PLAN_REF_KEY]["plan_id"]) + ) + selection_payload = result["result"]["metrics"]["random_window_selection"] + + assert "work_items" not in result + assert len(loaded) > 1 + assert all( + item.metadata[RANDOM_WINDOW_SELECTION_METADATA_KEY] == selection_payload + for item in loaded + ) + assert RandomWindowSelectionV1.from_dict(selection_payload).selection_id + + +def test_random_window_consumer_activity_fails_if_metadata_was_dropped( + tmp_path: Path, +) -> None: + """A declared selection may not degrade into unfiltered merge behavior.""" + request = RunRequest( + request_id="run-random-metadata-loss", + pairs=("eurusd",), + formats=("ascii",), + timeframes=("T",), + random_window="1d", + random_seed=42, + data_directory=str(tmp_path), + api_return_type="polars", + ) + item = WorkItem( + work_id="work-random-metadata-loss", + data_format="ascii", + data_timeframe="T", + data_fxpair="eurusd", + data_dir=str(tmp_path), + cache_filename=CACHE_FILENAME, + ) + + with pytest.raises(ValueError, match="missing its resolved"): + merge_cache_activity( + { + "request": request.to_dict(), + "work_items": [item.to_dict()], + } + ) + + def test_validate_urls_activity_loads_work_items_from_plan_ref( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -451,7 +564,7 @@ def test_validate_urls_activity_loads_work_items_from_plan_ref( monkeypatch.setattr( "histdatacom.activity_stages.fetch_histdata_page_data", lambda url, timeout: { - "html": _form_html(), + "html": _form_html(datemonth="202202"), "encoding": "gzip", "bytes_length": "123", }, @@ -499,6 +612,8 @@ def test_dataset_plan_activity_payload_survives_temporal_converter( timeframes=("T",), start_yearmonth="202201", end_yearmonth="202201", + random_window="1d", + random_seed=17, data_directory=str(tmp_path), metadata={ "requests_timeout": "30", @@ -518,6 +633,10 @@ def test_dataset_plan_activity_payload_survives_temporal_converter( assert result["result"]["metrics"]["work_item_count"] == 1 assert len(result["work_items"]) == 1 assert result["work_items"][0]["data_fxpair"] == "eurusd" + assert ( + RANDOM_WINDOW_SELECTION_METADATA_KEY + in result["work_items"][0]["metadata"] + ) def test_validate_urls_activity_returns_form_metadata( @@ -972,7 +1091,13 @@ def test_import_to_influx_activity_writes_tick_batches( [writer] = FakeInfluxWriter.instances assert writer.args["batch_size"] == "2" assert [len(batch) for batch in writer.batches] == [2, 1] - assert writer.batches[0][0] == EXPECTED_T_LINE + first_line = writer.batches[0][0] + assert "row_id=1" in first_line.split(" ", maxsplit=1)[0] + assert "bidquote=1.3066" in first_line + assert "askquote=1.30677" in first_line + assert "quality_status_code=0i" in first_line + assert "training_usable=true" in first_line + assert first_line.endswith(" 1328072403660") assert writer.closed assert result["work_item"]["status"] == WorkStatus.INFLUX_UPLOAD.value assert result["result"]["stage"] == "import_to_influx" @@ -1187,6 +1312,49 @@ def test_data_quality_activity_writes_report_and_bounded_metrics( ) +def test_data_quality_activity_projects_structured_skip_events( + tmp_path: Path, +) -> None: + """Activity metrics and detailed reports should agree on engine skips.""" + write_ascii_case(tmp_path, CLEAN_TICK_CASE) + write_zip_case( + tmp_path, + CLEAN_TICK_CASE, + zip_filename="DAT_ASCII_EURUSD_T_201202.zip", + ) + report_path = tmp_path / "reports" / "quality-skips.json" + request = RunRequest( + request_id="run-quality-skips", + data_directory=str(tmp_path), + data_quality=True, + quality_paths=(str(tmp_path),), + quality_check_groups=("time",), + quality_report_path=str(report_path), + quality_fail_on="never", + ) + + payload = data_quality_activity({"request": request.to_dict()}) + + quality = payload["result"]["metrics"]["quality"] + engine = quality["quality_engine"] + skips = engine["skip_events"] + detailed_report = json.loads(report_path.read_text(encoding="utf-8")) + + assert payload["result"]["status"] == WorkStatus.COMPLETED.value + assert engine == detailed_report["metadata"]["quality_engine"] + assert engine["planned_target_rule_evaluation_count"] == ( + engine["target_rule_evaluation_count"] + + engine["skipped_rule_evaluation_count"] + ) + assert skips["event_count"] == engine["skipped_rule_evaluation_count"] + assert skips["event_count"] > 0 + assert skips["reason_counts"] == { + "duplicate_archive_preferred_csv": skips["event_count"] + } + assert {event["target_kind"] for event in skips["events"]} == {"zip"} + assert str(tmp_path) not in json.dumps(engine, sort_keys=True) + + def test_data_quality_activity_deletes_default_scratch_report_on_success( tmp_path: Path, ) -> None: @@ -1624,4 +1792,6 @@ def test_default_activities_register_operation_activities() -> None: build_cache_activity, merge_cache_activity, import_to_influx_activity, + reconstruction_window_activity, + reconstruction_report_activity, ) diff --git a/tests/unit/test_orchestration_contracts.py b/tests/unit/test_orchestration_contracts.py index a9138d66..7a40e185 100644 --- a/tests/unit/test_orchestration_contracts.py +++ b/tests/unit/test_orchestration_contracts.py @@ -64,6 +64,8 @@ def test_orchestration_contracts_run_request_round_trip( timeframes=("tick-data-quotes",), start_yearmonth="202201", end_yearmonth="202202", + random_window="90m", + random_seed=1729, data_directory="data", api_return_type="polars", validate_urls=True, @@ -74,6 +76,8 @@ def test_orchestration_contracts_run_request_round_trip( assert restored == request assert restored.pairs == ("eurusd",) + assert restored.random_window == "90m" + assert restored.random_seed == 1729 assert restored.metadata == {"source": "test"} diff --git a/tests/unit/test_orchestration_influx_contract.py b/tests/unit/test_orchestration_influx_contract.py index 35e40540..43aab827 100644 --- a/tests/unit/test_orchestration_influx_contract.py +++ b/tests/unit/test_orchestration_influx_contract.py @@ -106,7 +106,13 @@ def test_import_workflow_contract_writes_batches_without_live_influx( assert writer.args["batch_size"] == "2" assert writer.args["delete_after_influx"] is True assert [len(batch) for batch in writer.batches] == [2, 1] - assert writer.batches[0][0] == EXPECTED_T_LINE + first_line = writer.batches[0][0] + assert "row_id=1" in first_line.split(" ", maxsplit=1)[0] + assert "bidquote=1.3066" in first_line + assert "askquote=1.30677" in first_line + assert "quality_status_code=0i" in first_line + assert "training_usable=true" in first_line + assert first_line.endswith(" 1328072403660") assert writer.closed assert summary["status"] == WorkStatus.INFLUX_UPLOAD.value assert summary["progress"]["completed_children"] == 1 diff --git a/tests/unit/test_orchestration_reconstruction.py b/tests/unit/test_orchestration_reconstruction.py new file mode 100644 index 00000000..b67f378b --- /dev/null +++ b/tests/unit/test_orchestration_reconstruction.py @@ -0,0 +1,1058 @@ +"""Tests for durable synthetic reconstruction orchestration.""" + +from __future__ import annotations + +import asyncio +import json +import shutil +import sys +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from histdatacom.runtime_contracts import ArtifactRef +from histdatacom.synthetic.streaming import ( + ReconstructionCommitPhase, + ReconstructionResourceEstimateV1, + ReconstructionRunV1, + ReconstructionStoragePolicyV1, + plan_reconstruction_windows, +) +from histdatacom.orchestration import activities, workflows +from histdatacom.orchestration.client import submit_reconstruction_request +from histdatacom.orchestration.queues import build_orchestration_worker_config +from histdatacom.orchestration.reconstruction import ( + RECONSTRUCTION_STAGE_ORDER, + ReconstructionArtifactError, + ReconstructionCheckpointConflict, + ReconstructionCheckpointStore, + ReconstructionReportMismatch, + ReconstructionStage, + ReconstructionStageCommandV1, + ReconstructionStageInvocationV1, + ReconstructionStageOutcomeV1, + ReconstructionStageStatus, + ReconstructionWindowTaskV1, + ReconstructionWorkflowRequestV1, + RegisteredReconstructionStageExecutor, + artifact_ref_for_file, + execute_reconstruction_stage, + plan_reconstruction_waves, + reconcile_reconstruction_report, + register_reconstruction_stage_handler, + run_reconstruction_window, + unregister_reconstruction_stage_handler, + verify_artifact_ref, +) + +RECONSTRUCTION_MODULE = sys.modules[reconcile_reconstruction_report.__module__] + + +def _run( + *, policy: ReconstructionStoragePolicyV1 | None = None +) -> ReconstructionRunV1: + return ReconstructionRunV1( + symbols=("eurusd", "eurgbp", "gbpusd"), + source_version_ids=("source:sha256:" + "1" * 64,), + configuration_ids=("config:sha256:" + "2" * 64,), + ensemble_member_ids=("member-0",), + base_seed=42, + storage_policy=policy or ReconstructionStoragePolicyV1(), + ) + + +def _estimate(*, memory_bytes: int = 1024) -> ReconstructionResourceEstimateV1: + return ReconstructionResourceEstimateV1( + input_event_count=10, + candidate_event_count=20, + retained_ensemble_members=1, + inflight_batches=1, + peak_events_per_batch=10, + estimated_memory_bytes=memory_bytes, + estimated_scratch_bytes=2048, + estimated_output_bytes=1024, + estimated_batch_count=1, + ) + + +def _task( + tmp_path: Path, + *, + run: ReconstructionRunV1 | None = None, + memory_bytes: int = 1024, + offset: int = 0, + handler_name: str = "test-handler", +) -> ReconstructionWindowTaskV1: + resolved_run = run or _run() + window = plan_reconstruction_windows( + resolved_run, + ensemble_member_id="member-0", + start_ns=offset, + end_ns=offset + 10_000, + window_size_ns=10_000, + )[0] + scratch = tmp_path / f"scratch-{offset}" + commands = tuple( + ReconstructionStageCommandV1( + stage=stage, + handler_name=handler_name, + receipt_path=str(scratch / "receipts" / f"{stage.value}.json"), + ) + for stage in RECONSTRUCTION_STAGE_ORDER + ) + return ReconstructionWindowTaskV1( + window=window, + resource_estimate=_estimate(memory_bytes=memory_bytes), + commands=commands, + scratch_directory=str(scratch), + ) + + +def _request( + tmp_path: Path, + *tasks: ReconstructionWindowTaskV1, + run: ReconstructionRunV1 | None = None, + max_parallel: int = 2, + max_memory: int = 10_000, +) -> ReconstructionWorkflowRequestV1: + resolved_tasks = tasks or (_task(tmp_path, run=run),) + return ReconstructionWorkflowRequestV1( + request_id="reconstruction-test", + run=run or _run(), + tasks=tuple(resolved_tasks), + manifest_store_root=str(tmp_path / "status"), + report_root=str(tmp_path / "reports"), + task_queues={ + "orchestration": "test.orchestration", + "cpu_file": "test.cpu-file", + }, + max_parallel_windows=max_parallel, + max_inflight_memory_bytes=max_memory, + ) + + +def _file_ref(path: Path, text: str, *, phase: str = "") -> ArtifactRef: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + metadata = {"commit_phase": phase} if phase else {} + return artifact_ref_for_file(path, kind="test-artifact", metadata=metadata) + + +def _stage_handler( + invocation: ReconstructionStageInvocationV1, +) -> ReconstructionStageOutcomeV1: + stage = invocation.command.stage + output_root = Path(invocation.task.scratch_directory) / "outputs" + if stage is ReconstructionStage.ATOMIC_PARTITION_COMMIT: + output_root = ( + Path(invocation.task.scratch_directory).parent + / "committed" + / invocation.task.window.window_id + ) + output = output_root / f"{stage.value}.json" + phase = "" + if stage is ReconstructionStage.VALIDATION: + phase = "staged" + text = "validated-manifest" + elif stage is ReconstructionStage.ATOMIC_PARTITION_COMMIT: + phase = "committed" + text = "validated-manifest" + else: + text = stage.value + ref = _file_ref(output, text, phase=phase) + ref = replace( + ref, + metadata={ + **ref.metadata, + "runtime_seconds": 1.0, + "peak_rss_bytes": 1_000, + "scratch_bytes": 100, + "output_bytes": ref.size_bytes, + "candidate_amplification": 2.0, + }, + ) + return invocation.completed( + output_refs=(ref,), + observed_event_count=10, + candidate_event_count=20, + accepted_event_count=15, + scratch_bytes=100, + output_bytes=ref.size_bytes or 0, + ) + + +@pytest.fixture(autouse=True) +def _clean_handler() -> None: + unregister_reconstruction_stage_handler("test-handler") + yield + unregister_reconstruction_stage_handler("test-handler") + + +def test_request_round_trip_contains_only_bounded_control_metadata( + tmp_path: Path, +) -> None: + request = _request(tmp_path) + + restored = ReconstructionWorkflowRequestV1.from_dict(request.to_dict()) + + assert restored == request + payload = json.dumps(request.to_dict(), sort_keys=True) + assert '"events"' not in payload + assert '"rows"' not in payload + assert len(payload.encode("utf-8")) < 1_048_576 + + +def test_request_rejects_inline_rows_hidden_in_artifact_metadata( + tmp_path: Path, +) -> None: + source = _file_ref(tmp_path / "source.json", "source") + source = ArtifactRef( + kind=source.kind, + path=source.path, + size_bytes=source.size_bytes, + sha256=source.sha256, + metadata={"rows": [1, 2, 3]}, + ) + task = _task(tmp_path) + command = task.commands[0] + commands = ( + ReconstructionStageCommandV1( + stage=command.stage, + handler_name=command.handler_name, + receipt_path=command.receipt_path, + input_manifest_refs=(source,), + ), + *task.commands[1:], + ) + task = ReconstructionWindowTaskV1( + window=task.window, + resource_estimate=task.resource_estimate, + commands=commands, + scratch_directory=task.scratch_directory, + ) + + with pytest.raises(ValueError, match="cannot contain.*rows"): + _request(tmp_path, task) + + +def test_stage_outcome_rejects_inline_events_hidden_in_artifact_metadata( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=task.commands[0], + prior_outcomes=(), + ) + output = _file_ref(tmp_path / "output.json", "output") + output = ArtifactRef( + kind=output.kind, + path=output.path, + size_bytes=output.size_bytes, + sha256=output.sha256, + metadata={"events": [{"timestamp": 1}]}, + ) + + with pytest.raises(ValueError, match="cannot contain.*events"): + invocation.completed(output_refs=(output,)) + + +def test_artifact_metadata_participates_in_retry_identity( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=task.commands[0], + prior_outcomes=(), + ) + output = _file_ref(tmp_path / "output.json", "output") + staged = ArtifactRef( + kind=output.kind, + path=output.path, + size_bytes=output.size_bytes, + sha256=output.sha256, + metadata={"commit_phase": "staged"}, + ) + committed = ArtifactRef( + kind=output.kind, + path=output.path, + size_bytes=output.size_bytes, + sha256=output.sha256, + metadata={"commit_phase": "committed"}, + ) + + assert invocation.completed(output_refs=(staged,)).outcome_id != ( + invocation.completed(output_refs=(committed,)).outcome_id + ) + + +def test_receipts_must_remain_inside_window_scratch(tmp_path: Path) -> None: + task = _task(tmp_path) + commands = list(task.commands) + commands[0] = ReconstructionStageCommandV1( + stage=commands[0].stage, + handler_name="test-handler", + receipt_path=str(tmp_path / "outside.json"), + ) + + with pytest.raises(ValueError, match="inside window scratch"): + ReconstructionWindowTaskV1( + window=task.window, + resource_estimate=task.resource_estimate, + commands=tuple(commands), + scratch_directory=task.scratch_directory, + ) + + +def test_request_rejects_overlapping_window_scratch_directories( + tmp_path: Path, +) -> None: + narrow = _task(tmp_path) + broad_source = _task(tmp_path, offset=10_000) + broad = ReconstructionWindowTaskV1( + window=broad_source.window, + resource_estimate=broad_source.resource_estimate, + commands=broad_source.commands, + scratch_directory=str(tmp_path), + ) + + with pytest.raises( + ValueError, match="scratch directories must be disjoint" + ): + _request(tmp_path, narrow, broad) + + +def test_request_rejects_scratch_overlapping_durable_roots( + tmp_path: Path, +) -> None: + source = _task(tmp_path) + unsafe = ReconstructionWindowTaskV1( + window=source.window, + resource_estimate=source.resource_estimate, + commands=source.commands, + scratch_directory=str(tmp_path), + ) + + with pytest.raises(ValueError, match="manifest or report storage"): + _request(tmp_path, unsafe) + + +def test_request_rejects_partial_cross_symbol_window(tmp_path: Path) -> None: + source = _task(tmp_path) + partial = ReconstructionWindowTaskV1( + window=replace( + source.window, + symbols=("eurusd",), + window_id="", + synchronization_unit_id="", + ), + resource_estimate=source.resource_estimate, + commands=source.commands, + scratch_directory=source.scratch_directory, + ) + + with pytest.raises( + ValueError, match="complete synchronized run symbol set" + ): + _request(tmp_path, partial) + + +def test_request_rejects_overlapping_core_windows(tmp_path: Path) -> None: + first = _task(tmp_path) + overlapping = _task(tmp_path, offset=5_000) + + with pytest.raises(ValueError, match="core intervals must not overlap"): + _request(tmp_path, first, overlapping) + + +def test_checkpoint_compare_and_swap_rejects_stale_worker( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + store = ReconstructionCheckpointStore(tmp_path / "status") + planned = store.initialize("request-a", task) + running = planned.running() + assert store.save(running, expected_state_id=planned.state_id) == running + failed_a = running.interrupted(ReconstructionCommitPhase.FAILED, "worker-a") + failed_b = running.interrupted(ReconstructionCommitPhase.FAILED, "worker-b") + stored = store.save(failed_a, expected_state_id=running.state_id) + + assert store.save(stored, expected_state_id=running.state_id) == stored + with pytest.raises(ReconstructionCheckpointConflict, match="stale"): + store.save(failed_b, expected_state_id=running.state_id) + + +def test_worker_loss_after_receipt_reuses_artifact_without_handler( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=task.commands[0], + prior_outcomes=(), + ) + register_reconstruction_stage_handler("test-handler", _stage_handler) + first = asyncio.run(execute_reconstruction_stage(invocation)) + unregister_reconstruction_stage_handler("test-handler") + + resumed = asyncio.run(execute_reconstruction_stage(invocation)) + + assert resumed.outcome_id == first.outcome_id + assert resumed.reused is True + + +def test_timeout_restarts_from_last_durable_stage(tmp_path: Path) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + calls: list[ReconstructionStage] = [] + + def timeout_handler( + invocation: ReconstructionStageInvocationV1, + ) -> ReconstructionStageOutcomeV1: + calls.append(invocation.command.stage) + if invocation.command.stage is ReconstructionStage.PROPOSAL: + raise TimeoutError("injected timeout") + return _stage_handler(invocation) + + register_reconstruction_stage_handler("test-handler", timeout_handler) + store = ReconstructionCheckpointStore(request.manifest_store_root) + with pytest.raises(TimeoutError, match="injected"): + asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=store, + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + durable = store.load(task.window) + assert durable is not None + assert tuple(item.stage for item in durable.outcomes) == ( + ReconstructionStage.SOURCE_ENRICHMENT, + ) + + register_reconstruction_stage_handler( + "test-handler", _stage_handler, replace_existing=True + ) + finished = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + assert finished.checkpoint.phase is ReconstructionCommitPhase.COMMITTED + assert calls.count(ReconstructionStage.SOURCE_ENRICHMENT) == 1 + + +def test_full_window_is_restart_safe_and_heartbeats_are_bounded( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + heartbeats = [] + register_reconstruction_stage_handler("test-handler", _stage_handler) + + state = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + heartbeat=heartbeats.append, + ) + ) + unregister_reconstruction_stage_handler("test-handler") + restarted = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + + assert state.checkpoint.phase is ReconstructionCommitPhase.COMMITTED + assert restarted.state_id == state.state_id + assert len(state.outcomes) == len(RECONSTRUCTION_STAGE_ORDER) + assert len(heartbeats) == len(RECONSTRUCTION_STAGE_ORDER) * 2 + assert ( + max(len(item.to_json().encode("utf-8")) for item in heartbeats) < 65_536 + ) + assert heartbeats[-1].accepted_event_count == 15 + + +def test_cancellation_persists_state_and_removes_only_window_scratch( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + scratch = Path(task.scratch_directory) + scratch.mkdir(parents=True) + (scratch / "partial.bin").write_bytes(b"partial") + sibling = tmp_path / "keep.bin" + sibling.write_bytes(b"keep") + + state = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + cancellation_requested=lambda: True, + ) + ) + + assert state.checkpoint.phase is ReconstructionCommitPhase.CANCELLED + assert not scratch.exists() + assert sibling.read_bytes() == b"keep" + + +def test_resource_preflight_records_refusal_without_running_handler( + tmp_path: Path, +) -> None: + policy = ReconstructionStoragePolicyV1(max_memory_bytes=100) + run = _run(policy=policy) + task = _task(tmp_path, run=run, memory_bytes=101) + request = _request(tmp_path, task, run=run, max_memory=1000) + + state = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + + assert state.checkpoint.phase is ReconstructionCommitPhase.FAILED + assert "estimated_memory_bytes" in state.checkpoint.interruption_reason + assert state.outcomes == () + + +def test_measured_peak_rss_above_policy_fails_before_checkpoint( + tmp_path: Path, +) -> None: + policy = ReconstructionStoragePolicyV1(max_memory_bytes=2_000) + run = _run(policy=policy) + task = _task(tmp_path, run=run, memory_bytes=1_024) + request = _request(tmp_path, task, run=run, max_memory=10_000) + + def over_memory_handler( + invocation: ReconstructionStageInvocationV1, + ) -> ReconstructionStageOutcomeV1: + output = _file_ref(tmp_path / "over-memory.json", "output") + measured = replace( + output, + metadata={"peak_rss_bytes": policy.max_memory_bytes + 1}, + ) + return invocation.completed( + output_refs=(measured,), + observed_event_count=10, + candidate_event_count=20, + accepted_event_count=15, + scratch_bytes=100, + output_bytes=measured.size_bytes or 0, + ) + + register_reconstruction_stage_handler("test-handler", over_memory_handler) + state = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + + assert state.checkpoint.phase is ReconstructionCommitPhase.FAILED + assert "peak_rss_bytes 2001 exceeds admitted limit 2000" in ( + state.checkpoint.interruption_reason + ) + assert state.outcomes == () + + +def test_many_stage_refusal_reasons_persist_as_bounded_summary( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + reasons = tuple( + f"infeasible_relationship_point:{index:02d}:" + "x" * 180 + for index in range(32) + ) + + def refusing_handler( + invocation: ReconstructionStageInvocationV1, + ) -> ReconstructionStageOutcomeV1: + return invocation.refused(*reasons, message="bounded refusal") + + register_reconstruction_stage_handler("test-handler", refusing_handler) + state = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + + summary = state.checkpoint.interruption_reason + assert state.checkpoint.phase is ReconstructionCommitPhase.FAILED + assert summary.startswith("infeasible_relationship_point:00:") + assert "more [sha256:" in summary + assert len(summary.encode("utf-8")) <= 2_048 + assert state.outcomes == () + + +def test_memory_weighted_waves_enforce_backpressure(tmp_path: Path) -> None: + run = _run() + tasks = tuple( + _task(tmp_path, run=run, memory_bytes=60, offset=index * 10_000) + for index in range(3) + ) + + waves = plan_reconstruction_waves( + tasks, + max_parallel_windows=3, + max_inflight_memory_bytes=100, + ) + + assert tuple(len(wave) for wave in waves) == (1, 1, 1) + + +def test_oversize_lane_task_gets_singleton_preflight_wave( + tmp_path: Path, +) -> None: + run = _run() + task = _task(tmp_path, run=run, memory_bytes=101) + + waves = plan_reconstruction_waves( + (task,), + max_parallel_windows=2, + max_inflight_memory_bytes=100, + ) + request = _request(tmp_path, task, run=run, max_memory=100) + state = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + repeated = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + + assert waves == ((task,),) + assert state.checkpoint.phase is ReconstructionCommitPhase.FAILED + assert repeated.state_id == state.state_id + assert "lane limit" in state.checkpoint.interruption_reason + + +def test_stage_handler_can_emit_bounded_progress_and_observe_cancel( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + heartbeats = [] + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=task.commands[0], + prior_outcomes=(), + heartbeat_callback=heartbeats.append, + cancellation_check=lambda: True, + ) + + invocation.heartbeat( + sequence=1, + completed_units=3, + total_units=10, + candidate_event_count=20, + scratch_bytes=100, + ) + + assert invocation.cancellation_requested is True + assert heartbeats[0].completed_units == 3 + assert heartbeats[0].cancellation_requested is True + assert len(heartbeats[0].to_json().encode("utf-8")) < 65_536 + + +def test_corrupt_receipt_and_stale_output_fail_closed(tmp_path: Path) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=task.commands[0], + prior_outcomes=(), + ) + receipt = Path(task.commands[0].receipt_path) + receipt.parent.mkdir(parents=True) + receipt.write_text("not-json", encoding="utf-8") + with pytest.raises( + ReconstructionArtifactError, match="invalid stage receipt" + ): + asyncio.run(execute_reconstruction_stage(invocation)) + + receipt.unlink() + register_reconstruction_stage_handler("test-handler", _stage_handler) + outcome = asyncio.run(execute_reconstruction_stage(invocation)) + Path(outcome.output_refs[0].path).write_text("x" * 17, encoding="utf-8") + with pytest.raises(ReconstructionArtifactError, match="sha256 differs"): + verify_artifact_ref(outcome.output_refs[0]) + + +def test_next_stage_rejects_corrupt_prior_stage_artifact( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + register_reconstruction_stage_handler("test-handler", _stage_handler) + first_invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=task.commands[0], + prior_outcomes=(), + ) + first = asyncio.run(execute_reconstruction_stage(first_invocation)) + Path(first.output_refs[0].path).write_text("corrupt", encoding="utf-8") + next_invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=task.commands[1], + prior_outcomes=(first,), + ) + + with pytest.raises( + ReconstructionArtifactError, match="artifact .* differs" + ): + asyncio.run(execute_reconstruction_stage(next_invocation)) + + +def test_duplicate_completion_resolves_to_newer_identical_prefix( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + store = ReconstructionCheckpointStore(request.manifest_store_root) + planned = store.initialize(request.request_id, task) + running = store.save(planned.running(), expected_state_id=planned.state_id) + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=task.commands[0], + prior_outcomes=(), + ) + register_reconstruction_stage_handler("test-handler", _stage_handler) + outcome = asyncio.run(execute_reconstruction_stage(invocation)) + completed = running.complete(outcome) + winner = store.save(completed, expected_state_id=running.state_id) + + duplicate = store.save(completed, expected_state_id=running.state_id) + + assert duplicate.state_id == winner.state_id + + +def test_report_reconciles_checkpoint_scope_and_storage_counts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + register_reconstruction_stage_handler("test-handler", _stage_handler) + state = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + fake_manifest = SimpleNamespace( + run_id=request.run.run_id, + window_id=task.window.window_id, + synchronization_unit_id=task.window.synchronization_unit_id, + symbols=task.window.symbols, + observed_event_count=10, + synthetic_event_count=15, + event_count=25, + manifest_id="manifest-1", + publication_id="publication-1", + ) + monkeypatch.setattr( + RECONSTRUCTION_MODULE, + "verify_reconstruction_publication", + lambda _path: fake_manifest, + ) + + report = reconcile_reconstruction_report(request, (state,)) + activity_result = activities.reconstruction_report_activity( + {"request": request.to_dict()} + ) + + assert report.status == "committed" + assert report.observed_event_count == 10 + assert report.synthetic_event_count == 15 + assert report.committed_window_count == 1 + resources = report.window_states[0]["resource_usage"] + assert isinstance(resources, dict) + assert resources["runtime_seconds"] == 7.0 + assert resources["peak_rss_bytes"] == 1_000 + assert resources["peak_scratch_bytes"] == 100 + assert resources["peak_candidate_amplification"] == 2.0 + assert resources["basis"] == "sum-stage-runtime-max-stage-resources-v1" + assert activity_result["report"]["report_id"] == report.report_id + + +def test_report_rejects_manifest_from_wrong_window( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + register_reconstruction_stage_handler("test-handler", _stage_handler) + state = asyncio.run( + run_reconstruction_window( + request, + task, + checkpoint_store=ReconstructionCheckpointStore( + request.manifest_store_root + ), + stage_executor=RegisteredReconstructionStageExecutor(), + ) + ) + monkeypatch.setattr( + RECONSTRUCTION_MODULE, + "verify_reconstruction_publication", + lambda _path: SimpleNamespace( + run_id=request.run.run_id, + window_id="wrong-window", + synchronization_unit_id=task.window.synchronization_unit_id, + symbols=task.window.symbols, + ), + ) + + with pytest.raises(ReconstructionReportMismatch, match="scope differs"): + reconcile_reconstruction_report(request, (state,)) + + +def test_default_worker_registration_includes_reconstruction() -> None: + assert workflows.ReconstructionRunWorkflow in workflows.DEFAULT_WORKFLOWS + assert workflows.ReconstructionWindowWorkflow in workflows.DEFAULT_WORKFLOWS + defaults = activities.default_activities() + assert activities.reconstruction_window_activity in defaults + assert activities.reconstruction_report_activity in defaults + assert ( + workflows.activity_execution_policy( + "reconstruction_window" + ).heartbeat_timeout_seconds + == 60 + ) + + +def test_client_submission_adds_workspace_queues_and_persists_snapshot( + tmp_path: Path, +) -> None: + request = _request(tmp_path) + config = build_orchestration_worker_config( + workspace=tmp_path, + runtime_home=tmp_path / "runtime", + ) + + class FakeClient: + def __init__(self) -> None: + self.calls = [] + + async def start_workflow(self, workflow, payload, **options): + self.calls.append((workflow, payload, options)) + return SimpleNamespace(id=options["id"], run_id="temporal-run") + + client = FakeClient() + handle = asyncio.run( + submit_reconstruction_request(request, config=config, client=client) + ) + submitted_request = client.calls[0][1]["request"] + snapshot = ReconstructionCheckpointStore( + request.manifest_store_root + ).store.get_job_snapshot(handle.workflow_id) + + assert client.calls[0][0] == "ReconstructionRunWorkflow" + assert client.calls[0][2]["task_queue"] == config.task_queues.orchestration + assert submitted_request["task_queues"]["cpu_file"] == ( + config.task_queues.cpu_file + ) + assert snapshot is not None + assert snapshot["metadata"]["window_count"] == 1 + + +def test_recovery_submission_uses_fresh_parent_and_child_identities( + tmp_path: Path, +) -> None: + """A resume attempt must not collide with earlier Temporal child IDs.""" + request = _request(tmp_path) + config = build_orchestration_worker_config( + workspace=tmp_path, + runtime_home=tmp_path / "runtime", + ) + + class FakeClient: + def __init__(self) -> None: + self.calls = [] + + async def start_workflow(self, workflow, payload, **options): + self.calls.append((workflow, payload, options)) + return SimpleNamespace(id=options["id"], run_id="resume-run") + + client = FakeClient() + handle = asyncio.run( + submit_reconstruction_request( + request, + config=config, + client=client, + workflow_id="reconstruction-resume-parent-001", + execution_attempt_id="resume-001", + ) + ) + payload = client.calls[0][1] + snapshot = ReconstructionCheckpointStore( + request.manifest_store_root + ).store.get_job_snapshot(handle.workflow_id) + initial_child_id = workflows._reconstruction_child_workflow_id( + request.request_id, request.tasks[0].window.window_id + ) + resumed_child_id = workflows._reconstruction_child_workflow_id( + request.request_id, + request.tasks[0].window.window_id, + execution_attempt_id="resume-001", + ) + + assert handle.workflow_id == "reconstruction-resume-parent-001" + assert payload["execution_attempt_id"] == "resume-001" + assert payload["request"]["request_id"] == request.request_id + assert initial_child_id != resumed_child_id + assert snapshot is not None + assert snapshot["metadata"]["execution_attempt_id"] == "resume-001" + + with pytest.raises(ValueError, match="unsupported characters"): + asyncio.run( + submit_reconstruction_request( + request, + config=config, + client=client, + execution_attempt_id="resume token must not enter history", + ) + ) + + +def test_stale_input_fingerprint_receipt_is_rejected(tmp_path: Path) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + command = task.commands[0] + outcome = ReconstructionStageOutcomeV1( + run_id=task.window.run_id, + window_id=task.window.window_id, + synchronization_unit_id=task.window.synchronization_unit_id, + stage=command.stage, + command_id=command.command_id, + input_fingerprint="f" * 64, + status=ReconstructionStageStatus.COMPLETED, + ) + receipt = Path(command.receipt_path) + receipt.parent.mkdir(parents=True) + receipt.write_text(outcome.to_json(), encoding="utf-8") + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=command, + prior_outcomes=(), + ) + + with pytest.raises(ReconstructionArtifactError, match="input fingerprint"): + asyncio.run(execute_reconstruction_stage(invocation)) + + +def test_state_store_survives_process_style_restart(tmp_path: Path) -> None: + task = _task(tmp_path) + first_store = ReconstructionCheckpointStore(tmp_path / "status") + state = first_store.initialize("request-a", task) + running = first_store.save( + state.running(), expected_state_id=state.state_id + ) + del first_store + + restored = ReconstructionCheckpointStore(tmp_path / "status").load( + task.window + ) + + assert restored == running + + +def test_cancelled_window_resume_discards_entire_disposable_stage_prefix( + tmp_path: Path, +) -> None: + task = _task(tmp_path) + request = _request(tmp_path, task) + register_reconstruction_stage_handler("test-handler", _stage_handler) + store = ReconstructionCheckpointStore(request.manifest_store_root) + state = store.initialize(request.request_id, task) + state = store.save(state.running(), expected_state_id=state.state_id) + for command in task.commands[:6]: + invocation = ReconstructionStageInvocationV1( + run=request.run, + task=task, + command=command, + prior_outcomes=state.outcomes, + ) + outcome = asyncio.run(execute_reconstruction_stage(invocation)) + next_state = state.complete(outcome) + state = store.save(next_state, expected_state_id=state.state_id) + if command.stage is ReconstructionStage.VALIDATION: + state = store.save( + state.validated(), expected_state_id=state.state_id + ) + cancelled = state.interrupted(ReconstructionCommitPhase.CANCELLED, "stop") + cancelled = store.save(cancelled, expected_state_id=state.state_id) + shutil.rmtree(task.scratch_directory) + + resumed = store.save( + cancelled.running(), expected_state_id=cancelled.state_id + ) + + assert resumed.checkpoint.phase is ReconstructionCommitPhase.RUNNING + assert resumed.outcomes == () diff --git a/tests/unit/test_orchestration_worker.py b/tests/unit/test_orchestration_worker.py index b05567cb..1e7951f5 100644 --- a/tests/unit/test_orchestration_worker.py +++ b/tests/unit/test_orchestration_worker.py @@ -11,12 +11,18 @@ from pathlib import Path from histdatacom.orchestration import worker +from histdatacom.orchestration.reconstruction import ( + registered_reconstruction_stage_handlers, +) from histdatacom.orchestration.queues import ( TaskQueueLane, build_orchestration_worker_config, ) from histdatacom.orchestration.readiness import read_worker_readiness from histdatacom.orchestration.runtime import build_orchestration_runtime_policy +from histdatacom.synthetic.reconstruction_plan import ( + FIRST_PARTY_RECONSTRUCTION_HANDLERS, +) class _FakeWorker: @@ -136,6 +142,28 @@ def test_build_temporal_worker_applies_configured_concurrency( ) +def test_default_worker_registers_first_party_reconstruction_handlers( + tmp_path: Path, +) -> None: + """Default startup makes every planned scientific adapter executable.""" + _FakeWorker.instances.clear() + built = worker.build_temporal_worker( + object(), + config=_config(tmp_path), + worker_class=_FakeWorker, + workflows=("workflow",), + ) + + registered = registered_reconstruction_stage_handlers() + assert set(FIRST_PARTY_RECONSTRUCTION_HANDLERS.values()).issubset( + registered + ) + assert any( + activity.__name__ == "reconstruction_window_activity" + for activity in built.activities + ) + + def test_build_temporal_worker_preserves_explicit_activity_executor( tmp_path: Path, ) -> None: @@ -204,6 +232,8 @@ def test_run_temporal_worker_accepts_fake_temporal_classes( "build_cache_activity", "merge_cache_activity", "import_to_influx_activity", + "reconstruction_window_activity", + "reconstruction_report_activity", } @@ -354,6 +384,8 @@ def test_default_workflows_include_topology_classes() -> None: "BuildCacheWorkflow", "MergeCacheWorkflow", "ImportWorkflow", + "ReconstructionRunWorkflow", + "ReconstructionWindowWorkflow", ] @@ -384,6 +416,8 @@ def test_default_activities_include_repository_refresh() -> None: "build_cache_activity", "merge_cache_activity", "import_to_influx_activity", + "reconstruction_window_activity", + "reconstruction_report_activity", ] diff --git a/tests/unit/test_orchestration_workflows.py b/tests/unit/test_orchestration_workflows.py index dd4aaf6a..f8a1ccbe 100644 --- a/tests/unit/test_orchestration_workflows.py +++ b/tests/unit/test_orchestration_workflows.py @@ -1072,6 +1072,8 @@ def test_workflow_topology_documents_expected_hierarchy() -> None: "BuildCacheWorkflow", "MergeCacheWorkflow", "ImportWorkflow", + "ReconstructionRunWorkflow", + "ReconstructionWindowWorkflow", } assert specs["HistDataRunWorkflow"]["children"] == [ "RepositoryRefreshWorkflow", @@ -1087,10 +1089,15 @@ def test_workflow_topology_documents_expected_hierarchy() -> None: "MergeCacheWorkflow", "ImportWorkflow", ] + assert specs["ReconstructionRunWorkflow"]["children"] == [ + "ReconstructionWindowWorkflow" + ] assert "rows" in str(document["history_policy"]) - assert set(activity_policies) == set( - workflows.OPERATION_ACTIVITIES.values() - ) + assert set(activity_policies) == { + *workflows.OPERATION_ACTIVITIES.values(), + "reconstruction_window", + "reconstruction_report", + } assert ( activity_policies["validate_urls"]["retry_policy"]["name"] == RetryPolicyName.NETWORK.value @@ -2195,7 +2202,7 @@ async def execute_activity( summary = asyncio.run(workflow.run(invocation.payload)) pending_metric = "activity" + "_pending" - assert not hasattr(workflows, "Pending" "ActivityExecutor") + assert not hasattr(workflows, "PendingActivityExecutor") assert captured["activity_name"] == "validate_urls" assert captured["options"]["task_queue"] == "queue-network" assert summary["status"] == WorkStatus.COMPLETED.value diff --git a/tests/unit/test_quality_cli.py b/tests/unit/test_quality_cli.py index 5aea590a..3e42a3c7 100644 --- a/tests/unit/test_quality_cli.py +++ b/tests/unit/test_quality_cli.py @@ -28,17 +28,27 @@ TIME_SERIES_FINGERPRINT_SCHEMA_DISCOVERY_SCHEMA_VERSION, ) from histdatacom.data_quality.fingerprints import ( + HistDataSeriesFingerprintRule, SERIES_FINGERPRINT_RULE_ID, TIME_SERIES_FINGERPRINT_AUDIT_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_METADATA_KEY, TIME_SERIES_FINGERPRINT_READINESS_RISK_SCHEMA_VERSION, TIME_SERIES_FINGERPRINT_SCHEMA_VERSION, ) +from histdatacom.data_quality.discovery import quality_target_from_path from histdatacom.data_quality.profiles import QUALITY_PROFILE_SCHEMA_VERSION +from histdatacom.data_quality.synthetic_constraints import ( + SYNTHETIC_VALIDATION_SCHEMA_VERSION, + synthetic_constraints_from_fingerprint, +) +from histdatacom.data_quality.synthetic_generation import ( + SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION, +) from histdatacom.histdata_ascii import ( CACHE_FILENAME, TICK, parse_ascii_lines, + read_polars_cache, to_polars_frame, write_polars_cache, ) @@ -63,6 +73,90 @@ def fake_quality_main(argv: list[str]) -> int: assert captured == ["evidence"] +def test_quality_repair_plan_cli_emits_bounded_non_mutating_json( + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> None: + """The repair-plan command should translate a report without changing data.""" + archive = tmp_path / "DAT_ASCII_EURUSD_T_201202.zip" + archive.write_bytes(b"not a zip") + target = QualityTarget( + path=str(archive), + kind=QualityTargetKind.ZIP, + data_format="ascii", + timeframe="T", + symbol="EURUSD", + period="201202", + ) + finding = QualityFinding( + severity=QualitySeverity.ERROR, + code="ZIP_CORRUPT", + message="ZIP archive could not be opened.", + rule_id="inventory.zip.integrity", + target=target, + metadata={"error_type": "BadZipFile"}, + ) + report = QualityReport( + targets=(target,), + rule_results=( + QualityRuleResult( + rule_id="inventory.zip.integrity", + target=target, + findings=(finding,), + ), + ), + ) + report_path = tmp_path / "quality.json" + write_quality_report(report, report_path) + archive_before = archive.read_bytes() + report_before = report_path.read_bytes() + + exit_code = main( + [ + "repair-plan", + "--report", + str(report_path), + "--item-limit", + "1", + "--evidence-limit", + "1", + "--json", + ] + ) + captured = capsys.readouterr() + payload = json.loads(captured.out) + + assert exit_code == 0 + assert payload["schema_version"] == "histdatacom.quality-repair-plan.v1" + assert payload["mode"] == "non_mutating" + assert payload["apply_supported"] is False + assert payload["items"][0]["operation"]["category"] == ( + "redownload_archive" + ) + assert str(tmp_path) not in captured.out + assert archive.read_bytes() == archive_before + assert report_path.read_bytes() == report_before + + +def test_quality_repair_plan_cli_human_output_is_concise( + capsys: pytest.CaptureFixture[str], +) -> None: + """The repair-plan command should explain missing plans without failing.""" + report_path = Path( + "tests/fixtures/data_quality_reports/corrupt_zip_report.json" + ) + + exit_code = main(["repair-plan", "--report", str(report_path)]) + captured = capsys.readouterr() + + assert exit_code == 0 + assert "Quality repair plan" in captured.out + assert "mode: non_mutating" in captured.out + assert "ZIP_CORRUPT" in captured.out + assert "redownload_archive" in captured.out + assert "error_type" not in captured.out + + def test_quality_evidence_cli_reports_human_accepted_status( capsys: pytest.CaptureFixture[str], tmp_path: Path, @@ -247,7 +341,13 @@ def fake_audit( assert exit_code == 1 assert "Ranked remediation gaps" in output + assert "Remediation plan" in output + assert "#1 high/90 CLI_GAP selector=exact_rule_and_finding" in output assert "#1 warning CLI_GAP family=time" in output + assert "attribution=inferred(unique_helper_rule)" in output + assert ( + "actionability=remediable_defect(unmapped_warning_or_error)" in output + ) assert "report_occurrences=3" in output @@ -426,6 +526,135 @@ def test_quality_bounded_payload_contract_cli_reports_human_output( assert "No bounded payload contract drift detected." in output +def test_quality_synthetic_validate_cli_compares_saved_reports( + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> None: + """Synthetic validation should use the saved quality-report CLI path.""" + report_path = _write_fingerprint_quality_report(tmp_path) + + exit_code = main( + [ + "synthetic-validate", + "--reference-report", + str(report_path), + "--candidate-report", + str(report_path), + "--json", + ] + ) + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["schema_version"] == SYNTHETIC_VALIDATION_SCHEMA_VERSION + assert payload["status"] == "mismatch" + assert payload["mismatched_target_count"] == 1 + assert "synthetic_candidate_avoid_duplicate_timestamps_present" in ( + payload["mismatch_code_counts"] + ) + + +def test_quality_synthetic_generate_cli_writes_enriched_validated_cache( + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> None: + """Generation should preserve observations and save validator evidence.""" + reference_path = _write_tick_cache( + tmp_path / "reference", + symbol="EURUSD", + row_multiplier=8, + ) + target = quality_target_from_path(reference_path) + assert target is not None + [finding] = HistDataSeriesFingerprintRule().evaluate(target) + reference_report = QualityReport( + targets=(target,), + rule_results=( + QualityRuleResult( + rule_id=finding.rule_id, + target=target, + findings=(finding,), + ), + ), + ) + reference_report_path = tmp_path / "reference-quality.json" + write_quality_report(reference_report, reference_report_path) + output_path = tmp_path / "generated" / ".data" + candidate_report_path = tmp_path / "generated-quality.json" + + exit_code = main( + [ + "synthetic-generate", + "--reference-cache", + str(reference_path), + "--reference-report", + str(reference_report_path), + "--output-cache", + str(output_path), + "--candidate-report", + str(candidate_report_path), + "--minimum-reference-rows", + "8", + "--block-size", + "4", + "--json", + ] + ) + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["schema_version"] == SYNTHETIC_TICK_GENERATION_SCHEMA_VERSION + assert payload["status"] == "ready" + assert payload["validation"]["same_fingerprint_path_used"] is True + assert output_path.is_file() + assert candidate_report_path.is_file() + reference = read_polars_cache(reference_path) + generated = read_polars_cache(output_path) + assert generated.select("datetime", "bid", "ask").to_dicts() == ( + reference.select("datetime", "bid", "ask").to_dicts() + ) + assert generated.get_column("synth_usable").all() + assert generated.get_column("synth_bid").null_count() == 0 + + configured_output = tmp_path / "configured-generated" / ".data" + config_path = tmp_path / "synthetic-generation.yaml" + config_path.write_text( + f""" +histdatacom: + quality: + command: synthetic_generate + reference_cache: {reference_path} + reference_report: {reference_report_path} + output_cache: {configured_output} + minimum_reference_rows: 8 + block_size: 4 + seed: 23 + json: true +""", + encoding="utf-8", + ) + assert main(["--config", str(config_path)]) == 0 + configured_payload = json.loads(capsys.readouterr().out) + assert configured_payload["configuration"]["seed"] == 23 + assert configured_output.is_file() + + assert ( + main( + [ + "synthetic-generate", + "--reference-cache", + str(reference_path), + "--reference-report", + str(reference_report_path), + "--output-cache", + str(output_path), + ] + ) + == 1 + ) + assert "output cache already exists" in capsys.readouterr().err + + def test_quality_fingerprint_schema_cli_applies_yaml_defaults( capsys: pytest.CaptureFixture[str], tmp_path: Path, @@ -556,6 +785,51 @@ def test_quality_fingerprint_readiness_cli_reports_human_output( assert str(tmp_path) not in output +def test_quality_fingerprint_readiness_cli_recommends_next_work( + capsys: pytest.CaptureFixture[str], +) -> None: + """The readiness command should optionally render bounded next work.""" + report_path = Path( + "tests/fixtures/data_quality_reports/fingerprint_report.json" + ) + + exit_code = main( + [ + "fingerprint-readiness", + "--report", + str(report_path), + "--next-work", + "--alternate-limit", + "1", + "--json", + ] + ) + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["next_work"]["schema_version"] == ( + "histdatacom.fingerprint-next-work.v1" + ) + assert payload["next_work"]["recommendation"]["rank"] == 1 + assert len(payload["next_work"]["alternates"]) == 1 + assert payload["next_work"]["basis"]["market_data_rescanned"] is False + + exit_code = main( + [ + "fingerprint-readiness", + "--report", + str(report_path), + "--next-work", + ] + ) + output = capsys.readouterr().out + + assert exit_code == 0 + assert "Next fingerprint work" in output + assert "suggested acceptance criteria:" in output + assert "does not" not in output + + def test_quality_help_advertises_fingerprint_schema_command() -> None: """The quality utility help should expose fingerprint schema discovery.""" help_text = quality_cli.build_parser().format_help() @@ -722,6 +996,9 @@ def _write_fingerprint_quality_report(tmp_path: Path) -> Path: }, }, } + payload["synthetic_constraints"] = synthetic_constraints_from_fingerprint( + payload + ) finding = QualityFinding( severity=QualitySeverity.INFO, code="FINGERPRINT_SERIES_SUMMARY", @@ -769,6 +1046,46 @@ def _catalog_payload( "report_occurrence_count": 3, "rule_id": "time.ascii.sequence", "source_family": "time", + "attribution_status": "inferred", + "attribution_reason": "unique_helper_rule", + "actionability": "remediable_defect", + "actionability_reason": "unmapped_warning_or_error", + } + ) + remediation_plan: dict[str, object] = { + "schema_version": "histdatacom.quality-remediation-plan.v1", + "plan_item_count": 0, + "included_plan_item_count": 0, + "omitted_plan_item_count": 0, + "truncated": False, + "actionability_counts": {}, + "fixability_counts": {}, + "items": [], + } + if ranked_gap: + remediation_plan.update( + { + "plan_item_count": 1, + "included_plan_item_count": 1, + "actionability_counts": {"remediable_defect": 1}, + "fixability_counts": {"high": 1}, + "items": [ + { + "rank": 1, + "finding_code": "CLI_GAP", + "rule_id": "time.ascii.sequence", + "suggested_selector": { + "shape": "exact_rule_and_finding" + }, + "suggested_action": {"action_kind": "inspect"}, + "fixability": { + "level": "high", + "score": 90, + "confidence": "high", + }, + "missing_fields": ["message"], + } + ], } ) return { @@ -788,10 +1105,14 @@ def _catalog_payload( "unmapped_known_code_count": 1 if gap_count else 0, "unmapped_warning_error_code_count": gap_count, "unmapped_warning_error_gap_count": gap_count, + "exact_attribution_occurrence_count": 0, + "inferred_attribution_occurrence_count": 1, + "unresolved_attribution_occurrence_count": 0, }, "known_code_counts": {}, "known_unmapped_codes": [], "ranked_gaps": ranked_gaps, + "remediation_plan": remediation_plan, "report_coverage": [], "payload_limits": {}, } diff --git a/tests/unit/test_random_windows.py b/tests/unit/test_random_windows.py new file mode 100644 index 00000000..d3d8b721 --- /dev/null +++ b/tests/unit/test_random_windows.py @@ -0,0 +1,313 @@ +"""Deterministic random/session tick-window contract tests.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import polars as pl +import pytest + +from histdatacom.random_windows import ( + RANDOM_WINDOW_MODE_ALL_SESSIONS, + RANDOM_WINDOW_SESSION_PROFILES, + RandomWindowSelectionV1, + RandomWindowSupportError, + RandomWindowSyntaxError, + filter_polars_frame_to_random_window, + parse_random_window_expression, + random_window_intervals_for_range, + random_window_planning_yearmonths, + random_window_requires_seed, + random_window_selection_from_metadata, + resolve_random_window_selection, +) + +DOCUMENTED_EXPRESSIONS = ( + "1y", + "1q", + "1M", + "2w", + "2d", + "6h", + "90m", + "ldn", + "ldn-ny", + "syd-syd", + "hk-3d", + "hk-3d-hk", + "45m-auk", + "1h-auk-1h", + "30m-ldn-1w-syd-1h", +) + + +def _repo(**ranges: tuple[str, str]) -> dict[str, dict[str, str]]: + return { + pair: {"start": start, "end": end} + for pair, (start, end) in ranges.items() + } + + +def _ms(value: str) -> int: + return int( + datetime.fromisoformat(value).replace(tzinfo=timezone.utc).timestamp() + * 1000 + ) + + +@pytest.mark.parametrize("expression", DOCUMENTED_EXPRESSIONS) +def test_parser_accepts_documented_expressions(expression: str) -> None: + """Every issue-contract example should parse without normalization.""" + parsed = parse_random_window_expression(expression) + + assert parsed.expression == expression + + +@pytest.mark.parametrize( + "expression", + ( + "", + " 1d", + "0d", + "-1d", + "1D", + "1d-2d", + "1d-ldn", + "ldn-1h-ny", + "ldn-ny-1d", + "ldn--ny", + "ldn-ny-hk", + "not-a-session", + ), +) +def test_parser_rejects_ambiguous_or_unsupported_forms( + expression: str, +) -> None: + """Unsupported mixtures should fail closed at the parser boundary.""" + with pytest.raises(RandomWindowSyntaxError): + parse_random_window_expression(expression) + + +def test_session_profiles_are_explicit_iana_sampling_windows() -> None: + """Legacy session codes should expose reproducible clock semantics.""" + assert set(RANDOM_WINDOW_SESSION_PROFILES) == { + "fra", + "ldn", + "ny", + "chi", + "la", + "auk", + "syd", + "tyo", + "hk", + } + for profile in RANDOM_WINDOW_SESSION_PROFILES.values(): + payload = profile.to_dict() + assert payload["dst_policy"] == "iana_zone_rules" + assert payload["semantics"] == "sampling_window_not_exchange_hours" + + +def test_seeded_selection_is_order_independent_and_round_trips() -> None: + """Pair input order and Temporal serialization must not change selection.""" + repository = _repo( + eurusd=("201001", "202412"), + gbpusd=("201101", "202311"), + ) + first = resolve_random_window_selection( + "2d", + seed=1729, + pairs=("eurusd", "gbpusd"), + repository_ranges=repository, + ) + second = resolve_random_window_selection( + "2d", + seed=1729, + pairs=("gbpusd", "eurusd"), + repository_ranges=repository, + ) + + assert first == second + assert RandomWindowSelectionV1.from_dict(first.to_dict()) == first + assert first.support_start_utc_ms == _ms("2011-01-01T00:00:00") + assert first.support_end_utc_ms == _ms("2023-12-01T00:00:00") + + +def test_multi_symbol_selection_refuses_empty_common_support() -> None: + """Disjoint instrument inventories must not silently select one symbol.""" + with pytest.raises(RandomWindowSupportError, match="no common"): + resolve_random_window_selection( + "1d", + seed=4, + pairs=("eurusd", "gbpusd"), + repository_ranges=_repo( + eurusd=("201001", "201012"), + gbpusd=("201101", "201112"), + ), + ) + + +def test_multi_symbol_selection_refuses_missing_inventory_pair() -> None: + """Present inventory must cover every requested instrument, even with bounds.""" + with pytest.raises(RandomWindowSupportError, match="gbpusd"): + resolve_random_window_selection( + "1d", + seed=4, + pairs=("eurusd", "gbpusd"), + repository_ranges=_repo(eurusd=("201001", "202412")), + start_yearmonth="202001", + end_yearmonth="202012", + ) + + +@pytest.mark.parametrize( + ("expression", "month_multiple"), + (("1M", 1), ("1q", 3), ("1y", 12)), +) +def test_calendar_durations_are_calendar_aligned( + expression: str, + month_multiple: int, +) -> None: + """Month/quarter/year selections should start at UTC calendar boundaries.""" + selection = resolve_random_window_selection( + expression, + seed=11, + pairs=("eurusd",), + repository_ranges=_repo(eurusd=("201001", "202412")), + ) + start = datetime.fromtimestamp( + selection.selected_start_utc_ms / 1000, # type: ignore[operator] + tz=timezone.utc, + ) + end = datetime.fromtimestamp( + selection.selected_end_utc_ms / 1000, # type: ignore[operator] + tz=timezone.utc, + ) + + assert (start.day, start.hour, start.minute) == (1, 0, 0) + assert (end.day, end.hour, end.minute) == (1, 0, 0) + assert (end.year * 12 + end.month) - (start.year * 12 + start.month) == ( + month_multiple + ) + + +def test_session_window_obeys_london_dst_and_padding() -> None: + """IANA rules should shift UTC boundaries while preserving local clocks.""" + winter = resolve_random_window_selection( + "1h-ldn-1h", + seed=1, + pairs=("eurusd",), + repository_ranges=_repo(eurusd=("202401", "202401")), + ) + summer = resolve_random_window_selection( + "1h-ldn-1h", + seed=1, + pairs=("eurusd",), + repository_ranges=_repo(eurusd=("202407", "202407")), + ) + winter_start = datetime.fromtimestamp( + winter.selected_start_utc_ms / 1000, # type: ignore[operator] + tz=timezone.utc, + ) + summer_start = datetime.fromtimestamp( + summer.selected_start_utc_ms / 1000, # type: ignore[operator] + tz=timezone.utc, + ) + + assert winter_start.hour == 7 + assert summer_start.hour == 6 + assert winter.selected_end_utc_ms - winter.selected_start_utc_ms == 11 * 3_600_000 # type: ignore[operator] + + +def test_ordered_and_same_session_windows_wrap_as_documented() -> None: + """Ordered spans use the end close; same sessions advance one local day.""" + repository = _repo(eurusd=("202401", "202401")) + ordered = resolve_random_window_selection( + "ldn-ny", + seed=3, + pairs=("eurusd",), + repository_ranges=repository, + ) + same = resolve_random_window_selection( + "syd-syd", + seed=3, + pairs=("eurusd",), + repository_ranges=repository, + ) + + assert ordered.selected_end_utc_ms - ordered.selected_start_utc_ms == 14 * 3_600_000 # type: ignore[operator] + assert same.selected_end_utc_ms - same.selected_start_utc_ms == 33 * 3_600_000 # type: ignore[operator] + + +def test_bounded_session_expression_selects_all_occurrences_without_seed() -> ( + None +): + """Both user bounds switch session expressions to compact all-session mode.""" + assert not random_window_requires_seed( + "ldn", + start_yearmonth="202401", + end_yearmonth="202401", + ) + selection = resolve_random_window_selection( + "ldn", + seed=None, + pairs=("eurusd",), + repository_ranges=_repo(eurusd=("202001", "202412")), + start_yearmonth="202401", + end_yearmonth="202401", + ) + + assert selection.mode == RANDOM_WINDOW_MODE_ALL_SESSIONS + assert selection.occurrence_count == 23 + assert selection.selected_start_utc_ms is None + assert random_window_planning_yearmonths(selection) == ("202401", "202401") + + +def test_exact_half_open_filtering_handles_interval_union() -> None: + """Projection should retain exact session rows and exclude end boundaries.""" + selection = resolve_random_window_selection( + "ldn", + seed=None, + pairs=("eurusd",), + repository_ranges=_repo(eurusd=("202401", "202401")), + start_yearmonth="202401", + end_yearmonth="202401", + ) + timestamps = [ + _ms("2024-01-02T07:59:00"), + _ms("2024-01-02T08:00:00"), + _ms("2024-01-02T16:59:00"), + _ms("2024-01-02T17:00:00"), + _ms("2024-01-06T10:00:00"), + ] + frame = pl.DataFrame({"datetime": timestamps, "bid": range(5)}) + + filtered = filter_polars_frame_to_random_window(frame, selection) + + assert filtered["datetime"].to_list() == timestamps[1:3] + intervals = random_window_intervals_for_range( + selection, + range_start_utc_ms=timestamps[0], + range_end_utc_ms=timestamps[-1] + 1, + ) + assert intervals + + +def test_duration_larger_than_support_fails_boundedly() -> None: + """An impossible requested duration should not substitute another window.""" + with pytest.raises(RandomWindowSupportError, match="exceeds common"): + resolve_random_window_selection( + "40d", + seed=1, + pairs=("eurusd",), + repository_ranges=_repo(eurusd=("202401", "202401")), + start_yearmonth="202401", + end_yearmonth="202401", + ) + + +def test_malformed_persisted_selection_fails_closed() -> None: + """Corrupt selection metadata must never be interpreted as full-cache mode.""" + with pytest.raises(ValueError, match="must be a mapping"): + random_window_selection_from_metadata( + {"random_window_selection": "not-a-contract"} + ) diff --git a/tests/unit/test_reconstruction_cli.py b/tests/unit/test_reconstruction_cli.py new file mode 100644 index 00000000..a576de0a --- /dev/null +++ b/tests/unit/test_reconstruction_cli.py @@ -0,0 +1,514 @@ +"""Tests for the installed reconstruction CLI and stable exit contract.""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys +from typing import Any + +import pytest + +from histdatacom import histdata_com +from histdatacom import reconstruction_cli +from histdatacom.cli_config import configured_reconstruction_argv +from histdatacom.reconstruction import ( + ReconstructionExecutionRequestV1, + ReconstructionExitCode, + ReconstructionOperationReceiptV1, + ReconstructionPlanSetPreflightV1, + ReconstructionPreflightV1, + ReconstructionRefusedError, + ReconstructionUnsupportedError, + read_operation_receipt, +) +from histdatacom.runtime_contracts import ArtifactRef +from histdatacom.synthetic.information import InformationMode +from histdatacom.synthetic.certification import CertificationState + + +def _request(tmp_path: Path) -> ReconstructionExecutionRequestV1: + return ReconstructionExecutionRequestV1( + plan_path=str(tmp_path / "plan.json"), + plan_id="synthetic-infill-plan:sha256:" + "1" * 64, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + scientific_nonclaim_acknowledged=True, + ) + + +def _receipt( + tmp_path: Path, *, status: str = "submitted" +) -> ReconstructionOperationReceiptV1: + return ReconstructionOperationReceiptV1( + operation="submit_only", + request=_request(tmp_path), + status=status, + ) + + +def test_installed_help_lists_complete_reconstruction_family( + capsys: pytest.CaptureFixture[str], +) -> None: + parser = reconstruction_cli.build_parser() + + with pytest.raises(SystemExit) as exc_info: + parser.parse_args(["--help"]) + + assert exc_info.value.code == 0 + output = capsys.readouterr().out + for command in ( + "plan", + "plan-set", + "preflight-set", + "request", + "preflight", + "run", + "status", + "cancel", + "resume", + "outputs", + "preview", + "replay", + "certify", + ): + assert command in output + assert "not recovered historical truth" in output + assert "M1/bar inputs" in output + + +def test_cli_constructs_and_preflights_public_plan_set( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """The full-range bounded planning surface is available to operators.""" + spec = object() + ref = ArtifactRef( + kind="reconstruction_plan_set_v1", + path=str(tmp_path / "plan-set.json"), + size_bytes=10, + sha256="1" * 64, + metadata={"plan_set_id": "reconstruction-plan-set:test"}, + ) + preflight = ReconstructionPlanSetPreflightV1( + plan_set_id="reconstruction-plan-set:test", + status="ready_with_refusals", + executable=True, + shard_count=25, + verified_shard_count=25, + refusal_count=3, + resource_summary={"planned_window_count": 8888}, + shard_preflights=(), + ) + calls: list[tuple[str, int | str]] = [] + + class FakeClient: + def construct_plan_set(self, supplied, *, periods_per_shard): + assert supplied is spec + calls.append(("plan-set", periods_per_shard)) + return ref + + def preflight_plan_set(self, supplied): + calls.append(("preflight-set", supplied)) + return preflight + + monkeypatch.setattr(reconstruction_cli, "_client", lambda _: FakeClient()) + monkeypatch.setattr(reconstruction_cli, "read_plan_spec", lambda _: spec) + + assert ( + reconstruction_cli.main( + [ + "--json", + "plan-set", + "--spec", + "full.json", + "--periods-per-shard", + "6", + ] + ) + == ReconstructionExitCode.SUCCESS + ) + assert json.loads(capsys.readouterr().out)["kind"] == ( + "reconstruction_plan_set_v1" + ) + assert ( + reconstruction_cli.main( + ["--json", "preflight-set", "--plan-set", "plan-set.json"] + ) + == ReconstructionExitCode.SUCCESS + ) + payload = json.loads(capsys.readouterr().out) + assert payload["verified_shard_count"] == 25 + assert payload["refusal_count"] == 3 + assert calls == [("plan-set", 6), ("preflight-set", "plan-set.json")] + + +def test_histdatacom_main_dispatches_reconstruction_command( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, tuple[str, ...]] = {} + + def fake_main(argv: list[str]) -> int: + captured["argv"] = tuple(argv) + return 0 + + monkeypatch.setattr(reconstruction_cli, "main", fake_main) + monkeypatch.setattr( + sys, + "argv", + ["histdatacom", "reconstruction", "preflight", "--request", "r.json"], + ) + + assert histdata_com.main() == 0 + assert captured["argv"] == ("preflight", "--request", "r.json") + + +def test_reconstruction_config_injects_command_and_explicit_flags_win( + tmp_path: Path, +) -> None: + config = tmp_path / "histdatacom.yaml" + config.write_text( + """ +histdatacom: + reconstruction: + command: preview + manifest: configured.json + limit: 7 + json: true +""".lstrip(), + encoding="utf-8", + ) + + configured = configured_reconstruction_argv(["--config", str(config)]) + explicit = configured_reconstruction_argv( + [ + "--config", + str(config), + "preview", + "--manifest", + "explicit.json", + "--limit", + "3", + ] + ) + + assert configured == [ + "--json", + "preview", + "--limit", + "7", + "--manifest", + "configured.json", + ] + assert explicit == [ + "--json", + "preview", + "--limit", + "7", + "--manifest", + "configured.json", + "--manifest", + "explicit.json", + "--limit", + "3", + ] + + +def test_cli_submit_only_writes_identity_checked_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + request = _request(tmp_path) + receipt = _receipt(tmp_path) + receipt_path = tmp_path / "receipt.json" + + class FakeClient: + def submit(self, supplied, *, wait): + assert supplied == request + assert not wait + return receipt + + monkeypatch.setattr(reconstruction_cli, "_client", lambda _: FakeClient()) + monkeypatch.setattr( + reconstruction_cli, "read_execution_request", lambda _: request + ) + + code = reconstruction_cli.main( + [ + "--json", + "run", + "--request", + "request.json", + "--submit-only", + "--receipt", + str(receipt_path), + ] + ) + + payload = json.loads(capsys.readouterr().out) + assert code == ReconstructionExitCode.SUCCESS + assert payload["receipt_path"] == str(receipt_path) + assert read_operation_receipt(receipt_path) == receipt + + +@pytest.mark.parametrize( + ("failure", "expected"), + ( + ( + ReconstructionUnsupportedError("M1 is unsupported"), + ReconstructionExitCode.INVALID_PLAN, + ), + ( + ReconstructionRefusedError("scientific refusal"), + ReconstructionExitCode.REFUSED, + ), + ( + RuntimeError("Temporal crashed"), + ReconstructionExitCode.RUNTIME_FAILURE, + ), + ), +) +def test_cli_maps_invalid_refused_and_runtime_failures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + failure: Exception, + expected: ReconstructionExitCode, +) -> None: + class FakeClient: + def submit(self, *_: Any, **__: Any) -> Any: + raise failure + + monkeypatch.setattr(reconstruction_cli, "_client", lambda _: FakeClient()) + monkeypatch.setattr( + reconstruction_cli, + "read_execution_request", + lambda _: _request(tmp_path), + ) + + code = reconstruction_cli.main( + [ + "--json", + "run", + "--request", + "request.json", + "--submit-only", + "--receipt", + str(tmp_path / "receipt.json"), + ] + ) + + payload = json.loads(capsys.readouterr().err) + assert code == expected + assert payload["exit_code"] == expected + + +def test_cli_maps_failed_local_report_to_validation_exit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + request = _request(tmp_path) + + class FakeClient: + def execute_local(self, supplied, *, window_id): + assert supplied == request + assert window_id == "window-1" + return _receipt(tmp_path, status="failed") + + monkeypatch.setattr(reconstruction_cli, "_client", lambda _: FakeClient()) + monkeypatch.setattr( + reconstruction_cli, "read_execution_request", lambda _: request + ) + + code = reconstruction_cli.main( + [ + "--json", + "run", + "--request", + "request.json", + "--local", + "--window-id", + "window-1", + "--receipt", + str(tmp_path / "failed.json"), + ] + ) + + assert json.loads(capsys.readouterr().out)["status"] == "failed" + assert code == ReconstructionExitCode.VALIDATION_FAILURE + + +def test_cli_preflight_refusal_is_distinct_from_unsupported( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + request = _request(tmp_path) + preflight = ReconstructionPreflightV1( + request_id=request.request_id, + plan_id=request.plan_id, + status="refused", + executable=False, + plan_status="ready_with_refusals", + dry_run={}, + evidence_refs={}, + refusal_reasons=( + { + "code": "market_context_unsupported", + "reason": "calendar evidence absent", + }, + ), + ) + + class FakeClient: + def preflight(self, supplied): + assert supplied == request + return preflight + + monkeypatch.setattr(reconstruction_cli, "_client", lambda _: FakeClient()) + monkeypatch.setattr( + reconstruction_cli, "read_execution_request", lambda _: request + ) + + code = reconstruction_cli.main( + ["--json", "preflight", "--request", "request.json"] + ) + + payload = json.loads(capsys.readouterr().out) + assert code == ReconstructionExitCode.REFUSED + assert payload["refusal_reasons"][0]["code"] == ( + "market_context_unsupported" + ) + + +def test_cli_status_cancel_and_resume_use_receipt_controls( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = _receipt(tmp_path) + source_path = tmp_path / "source.json" + source_path.write_text(json.dumps(source.to_dict()), encoding="utf-8") + calls: list[str] = [] + + class FakeClient: + def inspect(self, receipt, *, offline): + assert receipt == source + assert offline + calls.append("status") + return _receipt(tmp_path, status="running") + + def cancel(self, receipt, *, reason): + assert receipt == source + assert reason == "operator request" + calls.append("cancel") + return _receipt(tmp_path, status="cancellation_requested") + + def resume(self, receipt, *, wait, local): + assert receipt == source + assert not wait + assert not local + calls.append("resume") + return _receipt(tmp_path, status="submitted") + + monkeypatch.setattr(reconstruction_cli, "_client", lambda _: FakeClient()) + + assert ( + reconstruction_cli.main( + ["status", "--receipt", str(source_path), "--offline"] + ) + == ReconstructionExitCode.SUCCESS + ) + assert ( + reconstruction_cli.main( + [ + "cancel", + "--receipt", + str(source_path), + "--reason", + "operator request", + "--output", + str(tmp_path / "cancel.json"), + ] + ) + == ReconstructionExitCode.SUCCESS + ) + assert ( + reconstruction_cli.main( + [ + "resume", + "--receipt", + str(source_path), + "--submit-only", + "--output", + str(tmp_path / "resume.json"), + ] + ) + == ReconstructionExitCode.SUCCESS + ) + assert calls == ["status", "cancel", "resume"] + + +@pytest.mark.parametrize( + ("state", "expected"), + ( + ( + CertificationState.READY_FOR_PROMOTION, + ReconstructionExitCode.SUCCESS, + ), + (CertificationState.INCOMPLETE, ReconstructionExitCode.REFUSED), + (CertificationState.FAILED, ReconstructionExitCode.VALIDATION_FAILURE), + ), +) +def test_cli_certify_runs_public_campaign_and_maps_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + state: CertificationState, + expected: ReconstructionExitCode, +) -> None: + """The installed command publishes its receipt and stable exit category.""" + captured: dict[str, str] = {} + + class FakeDossier: + summary = {"passed_gate_count": 14, "missing_gate_count": 1} + + def __init__(self, selected_state: CertificationState) -> None: + self.state = selected_state + + class FakeResult: + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": "campaign-result.v1", + "state": state.value, + "dossier_id": "dossier:test", + } + + class FakeClient: + def certify(self, spec: str, *, output_directory: str): + captured["spec"] = spec + captured["output"] = output_directory + return FakeDossier(state), FakeResult() + + monkeypatch.setattr(reconstruction_cli, "_client", lambda _: FakeClient()) + + code = reconstruction_cli.main( + [ + "--json", + "certify", + "--spec", + "campaign.json", + "--output-directory", + str(tmp_path / "dossier"), + ] + ) + + payload = json.loads(capsys.readouterr().out) + assert code == expected + assert payload["state"] == state.value + assert payload["summary"]["passed_gate_count"] == 14 + assert captured == { + "spec": "campaign.json", + "output": str(tmp_path / "dossier"), + } diff --git a/tests/unit/test_reconstruction_plan.py b/tests/unit/test_reconstruction_plan.py new file mode 100644 index 00000000..4cdaa71d --- /dev/null +++ b/tests/unit/test_reconstruction_plan.py @@ -0,0 +1,909 @@ +"""Tests for the first-party reconstruction plan and artifact graph.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pyarrow as pa +import pyarrow.ipc as ipc +import pytest + +from histdatacom.manifest_store import ManifestStatusStore +from histdatacom.reconstruction import ( + ReconstructionClient, + ReconstructionExecutionRequestV1, + ReconstructionPlanError, + ReconstructionPlanSetV1, + ReconstructionPlanSpecV1, + ReconstructionRefusedError, + ReconstructionUnsupportedError, + read_execution_request, + read_operation_receipt, + read_reconstruction_plan_set, + write_execution_request, + write_operation_receipt, +) + +from histdatacom.orchestration.reconstruction import ( + RECONSTRUCTION_STAGE_ORDER, + ReconstructionStage, + artifact_ref_for_file, +) +from histdatacom.orchestration.queues import build_orchestration_worker_config +from histdatacom.synthetic import ( + ASCII_TICK_SOURCE_KIND, + FIRST_PARTY_RECONSTRUCTION_HANDLERS, + IMMUTABLE_ANCHOR_POLICY, + SCIENTIFIC_NONCLAIM, + TICK_ONLY_INPUT_POLICY, + InformationMode, + ModernReferenceMotifProfileV1, + ReconstructionDeliveryMode, + ReconstructionPlanCompatibilityError, + ReconstructionSourceInventoryV1, + ReconstructionStoragePolicyV1, + SyntheticInfillPlanV1, + build_synthetic_infill_plan, + load_reconstruction_stage_plan, + read_reconstruction_plan_execution_manifest, + read_reconstruction_source_inventory, + read_synthetic_infill_plan, + validate_synthetic_infill_plan_for_execution, + write_synthetic_infill_plan, +) +from histdatacom.synthetic import reconstruction_plan as plan_module + +_SYMBOLS = ("eurgbp", "eurusd", "gbpusd") +_PERIOD = "202001" +_START_MS = 1_578_268_800_000 + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _write_tick_partition(path: Path, offset: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + table = pa.table( + { + "datetime": [_START_MS + offset, _START_MS + 60_000 + offset], + "bid": [1.0 + offset / 1_000_000, 1.0001 + offset / 1_000_000], + "ask": [1.0002 + offset / 1_000_000, 1.0003 + offset / 1_000_000], + "vol": [0, 0], + } + ) + with pa.OSFile(str(path), "wb") as sink: + with ipc.new_file(sink, table.schema) as writer: + writer.write_table(table) + + +def _artifact(tmp_path: Path, role: str, kind: str) -> Any: + path = tmp_path / "inputs" / f"{role}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({"role": role}, sort_keys=True), encoding="utf-8" + ) + return artifact_ref_for_file(path, kind=kind) + + +def _resolved_inputs(tmp_path: Path, source_root: Path) -> Any: + lineage: list[dict[str, str]] = [] + for ordinal, symbol in enumerate(_SYMBOLS): + path = source_root / symbol / "2020" / "1" / ".data" + _write_tick_partition(path, ordinal) + lineage.append( + { + "period": _PERIOD, + "symbol": symbol.upper(), + "source_artifact_sha256": f"sha256:{_sha256(path)}", + "evidence_id": f"feed-evidence:{symbol}", + } + ) + definition = SimpleNamespace( + definition_id="feed-epochs-v2:test", + symbols=tuple(symbol.upper() for symbol in _SYMBOLS), + lineage={"sources": lineage}, + coverage_end_utc_ms=1_580_515_200_000, + assign=lambda **_: SimpleNamespace(assignment_kind="assigned"), + ) + artifacts = { + "feed_epochs": _artifact( + tmp_path, "feed-epochs", "feed_epoch_definition_v2" + ), + "observation_operator": _artifact( + tmp_path, "observation", "observation-operator" + ), + "market_context": _artifact( + tmp_path, "market-context", "market_context_corpus_v1" + ), + "cftc_positioning": _artifact( + tmp_path, "cftc-positioning", "cftc_positioning_corpus_v1" + ), + "benchmark_manifest": _artifact( + tmp_path, "benchmark", "reverse_degradation_manifest_v1" + ), + "motif_manifest": _artifact( + tmp_path, "motif-manifest", "modern_reference_motif_manifest_v1" + ), + "motif_index": _artifact( + tmp_path, "motif-index", "modern_reference_motif_index_v1" + ), + "motif_qualification": _artifact( + tmp_path, + "motif-qualification", + "modern_reference_motif_qualification_v1", + ), + "motif_leakage_audit": _artifact( + tmp_path, + "motif-leakage", + "modern_reference_motif_leakage_audit_v1", + ), + } + return plan_module._ResolvedPlanInputs( + feed_epoch_definition=definition, + observation_operator=SimpleNamespace( + operator_id="observation-operator:test", + required_left_halo_ns=0, + ), + market_context=SimpleNamespace(corpus_id="market-context:test"), + cftc_positioning=SimpleNamespace(corpus_id="cftc-positioning:test"), + benchmark_corpus=SimpleNamespace(corpus_id="benchmark:test"), + motif_profile=ModernReferenceMotifProfileV1(), + motif_index=SimpleNamespace(index_id="motif-index:test"), + artifacts=artifacts, + motif_manifest={"library_id": "modern-reference-motif:test"}, + motif_qualification={"qualified": True}, + motif_leakage_audit={"accepted": True}, + ) + + +def _builder_kwargs(tmp_path: Path) -> dict[str, Any]: + unused = tmp_path / "unused.json" + return { + "feed_epoch_definition_path": unused, + "observation_operator_path": unused, + "market_context_corpus_path": unused, + "cftc_positioning_corpus_path": unused, + "benchmark_manifest_path": unused, + "motif_manifest_path": unused, + "motif_index_path": unused, + "motif_qualification_path": unused, + "motif_leakage_audit_path": unused, + "artifact_root": tmp_path / "artifacts", + "output_root": tmp_path / "output", + "checkpoint_root": tmp_path / "checkpoints", + "scratch_root": tmp_path / "scratch", + "start_period": _PERIOD, + "end_period": _PERIOD, + } + + +@pytest.fixture # type: ignore[untyped-decorator] +def planned_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[Path, dict[str, Any]]: + source_root = tmp_path / "ASCII" / "T" + resolved = _resolved_inputs(tmp_path, source_root) + monkeypatch.setattr( + plan_module, "_resolve_plan_inputs", lambda **_: resolved + ) + monkeypatch.setattr( + plan_module, + "preflight_market_context_corpus", + lambda *_, **__: SimpleNamespace(reasons=()), + ) + monkeypatch.setattr( + plan_module, + "preflight_cftc_positioning_corpus", + lambda *_, **__: SimpleNamespace(ready=True, reasons=()), + ) + return source_root, _builder_kwargs(tmp_path) + + +def test_public_builder_is_deterministic_bounded_and_stage_consumable( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + + first = build_synthetic_infill_plan(source_root, **kwargs) + second = build_synthetic_infill_plan(source_root, **kwargs) + plan_ref = write_synthetic_infill_plan(first, kwargs["artifact_root"]) + restored = read_synthetic_infill_plan(plan_ref.path) + + assert first == second == restored + assert SyntheticInfillPlanV1.from_json(first.to_json()) == first + assert first.delivery_mode is ReconstructionDeliveryMode.MODERN_REFERENCE + assert first.status == "ready" + assert first.resources.planned_window_count == 2 + assert first.resources.executable_window_count == 2 + assert first.resources.ensemble_member_count == 4 + assert len(first.workflow_requests) == 4 + assert ( + max( + len(json.dumps(request.to_dict()).encode("utf-8")) + for request in first.workflow_requests + ) + < 1_048_576 + ) + assert '"rows"' not in first.to_json() + assert '"events"' not in first.to_json() + assert first.to_dict()["scientific_nonclaim"] == SCIENTIFIC_NONCLAIM + assert first.to_dict()["immutable_anchor_policy"] == IMMUTABLE_ANCHOR_POLICY + assert first.to_dict()["input_policy"] == TICK_ONLY_INPUT_POLICY + + validate_synthetic_infill_plan_for_execution(restored) + task = restored.workflow_requests[0].tasks[0] + assert tuple(command.stage for command in task.commands) == ( + RECONSTRUCTION_STAGE_ORDER + ) + for command in task.commands: + loaded = load_reconstruction_stage_plan(command) + assert loaded.command == command + assert ( + loaded.configuration.configuration_id == restored.configuration_id + ) + assert ( + command.handler_name + == FIRST_PARTY_RECONSTRUCTION_HANDLERS[command.stage] + ) + assert command.configuration_refs == ( + restored.artifact_graph["execution_manifest"], + ) + broker_command = next( + command + for command in task.commands + if command.stage is ReconstructionStage.BROKER_TRANSFER + ) + assert broker_command.input_manifest_refs == () + + +def test_typed_public_facade_constructs_requests_and_preflights( + planned_environment: tuple[Path, dict[str, Any]], + tmp_path: Path, +) -> None: + """Public callers need no private imports from construction to dry-run.""" + source_root, kwargs = planned_environment + spec = _public_spec(source_root, kwargs) + client = ReconstructionClient() + + plan_ref = client.construct_plan(spec) + request = client.create_request( + plan_ref.path, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + acknowledge_scientific_nonclaim=True, + ) + request_path = write_execution_request(request, tmp_path / "request.json") + restored = read_execution_request(request_path) + preflight = client.preflight(restored) + + assert restored == request + assert preflight.executable + assert preflight.status == "ready" + assert preflight.dry_run["information_mode"] == "ex_post_reconstruction" + assert preflight.dry_run["resources"]["workflow_request_count"] == 4 + assert "benchmark_manifest" in preflight.evidence_refs + assert "information_audit" in preflight.evidence_refs + + +def test_public_plan_set_shards_and_revalidates_bounded_full_range( + planned_environment: tuple[Path, dict[str, Any]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A range plan remains public and strong without one unbounded payload.""" + source_root, kwargs = planned_environment + spec = replace( + _public_spec(source_root, kwargs), + window_size_ns=24 * 60 * 60 * 1_000_000_000, + ) + client = ReconstructionClient() + ordinary_construct = client._construct_plan_model + + def resource_bounded_construct( + shard_spec: ReconstructionPlanSpecV1, + ) -> SyntheticInfillPlanV1: + assert shard_spec.requested_start_ns is not None + assert shard_spec.requested_end_ns is not None + if ( + shard_spec.requested_end_ns - shard_spec.requested_start_ns + > 8 * 24 * 60 * 60 * 1_000_000_000 + ): + raise ReconstructionPlanError( + "reconstruction persistence preflight failed: fixture bound" + ) + return ordinary_construct(shard_spec) + + monkeypatch.setattr( + client, "_construct_plan_model", resource_bounded_construct + ) + + ref = client.construct_plan_set(spec, periods_per_shard=1) + plan_set = read_reconstruction_plan_set(ref.path) + preflight = client.preflight_plan_set(ref.path) + + assert isinstance(plan_set, ReconstructionPlanSetV1) + assert plan_set.status == "ready" + assert plan_set.executable + assert len(plan_set.shards) == 4 + assert all(item.start_period == _PERIOD for item in plan_set.shards) + assert all(item.end_period == _PERIOD for item in plan_set.shards) + assert plan_set.resource_summary["plan_shard_count"] == 4 + assert plan_set.resource_summary["planned_window_count"] == 31 + first_plan = read_synthetic_infill_plan(plan_set.shards[0].plan_ref.path) + first_inventory = read_reconstruction_source_inventory( + first_plan.artifact_graph["source_inventory"].path + ) + assert plan_set.resource_summary["source_partition_count"] == 3 + assert plan_set.resource_summary["source_event_count"] == ( + first_inventory.total_row_count + ) + assert plan_set.resource_summary["source_size_bytes"] == ( + first_inventory.total_size_bytes + ) + assert preflight.executable + assert preflight.verified_shard_count == 4 + assert preflight.resource_summary == plan_set.resource_summary + assert ReconstructionPlanSetV1.from_dict(plan_set.to_dict()) == plan_set + + Path(plan_set.shards[0].plan_ref.path).write_bytes( + Path(plan_set.shards[0].plan_ref.path).read_bytes() + b"\n" + ) + with pytest.raises(ReconstructionPlanError, match="shard artifact differs"): + client.preflight_plan_set(ref.path) + + +def test_public_plan_set_preserves_a_contiguous_refusal_only_shard( + planned_environment: tuple[Path, dict[str, Any]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Full-range planning accounts for unsupported spans without fake work.""" + source_root, kwargs = planned_environment + monkeypatch.setattr( + plan_module, + "preflight_market_context_corpus", + lambda *_, **__: SimpleNamespace(reasons=("context unsupported",)), + ) + client = ReconstructionClient() + ref = client.construct_plan_set( + replace( + _public_spec(source_root, kwargs), + window_size_ns=24 * 60 * 60 * 1_000_000_000, + ), + periods_per_shard=1, + ) + + plan_set = read_reconstruction_plan_set(ref.path) + preflight = client.preflight_plan_set(ref.path) + + assert plan_set.status == "ready_with_refusals" + assert plan_set.executable + assert len(plan_set.shards) == 1 + assert plan_set.shards[0].preflight_status == "ready_with_refusals" + assert plan_set.resource_summary["executable_window_count"] == 0 + assert plan_set.resource_summary["refused_window_count"] == 31 + assert plan_set.resource_summary["estimated_output_bytes"] == 0 + assert preflight.executable + assert preflight.status == "ready_with_refusals" + assert preflight.refusal_count == 31 + + +def test_public_plan_spec_threads_bounded_window_size_into_resources( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + """Operators can split large monthly inputs before resource preflight.""" + source_root, kwargs = planned_environment + window_size_ns = 6 * 60 * 60 * 1_000_000_000 + spec = replace( + _public_spec(source_root, kwargs), + window_size_ns=window_size_ns, + ) + + restored_spec = ReconstructionPlanSpecV1.from_dict(spec.to_dict()) + plan_ref = ReconstructionClient().construct_plan(restored_spec) + plan = read_synthetic_infill_plan(plan_ref.path) + configuration = plan_module.read_reconstruction_plan_configuration( + plan.artifact_graph["configuration"].path + ) + + assert restored_spec.window_size_ns == window_size_ns + assert configuration.window_size_ns == window_size_ns + estimates = tuple( + task.resource_estimate + for request in plan.workflow_requests + for task in request.tasks + ) + nonempty = tuple( + estimate for estimate in estimates if estimate.input_event_count + ) + assert nonempty + assert all(estimate.estimated_batch_count >= 3 for estimate in nonempty) + assert all( + estimate.estimated_memory_bytes >= 512 * 1024 * 1024 + for estimate in estimates + ) + + +def test_public_request_requires_ack_and_exact_information_mode( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + """The operator cannot omit the nonclaim or relabel plan information.""" + source_root, kwargs = planned_environment + client = ReconstructionClient() + plan_ref = client.construct_plan(_public_spec(source_root, kwargs)) + + with pytest.raises(ReconstructionRefusedError, match="acknowledgement"): + client.create_request( + plan_ref.path, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + acknowledge_scientific_nonclaim=False, + ) + + mismatched = ReconstructionExecutionRequestV1( + plan_path=plan_ref.path, + plan_id=read_synthetic_infill_plan(plan_ref.path).plan_id, + information_mode=InformationMode.EX_ANTE_SIMULATION, + scientific_nonclaim_acknowledged=True, + ) + with pytest.raises(ReconstructionRefusedError, match="information mode"): + client.preflight(mismatched) + + +def test_typed_public_facade_accepts_a_point_in_time_ex_ante_plan( + planned_environment: tuple[Path, dict[str, Any]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An ex-ante request is executable when every fitted artifact predates it.""" + source_root, kwargs = planned_environment + resolved = plan_module._resolve_plan_inputs() + definition = SimpleNamespace( + **{ + **vars(resolved.feed_epoch_definition), + "coverage_end_utc_ms": 1_514_764_800_000, + } + ) + profile = ModernReferenceMotifProfileV1( + split_periods={ + "train": ("201501",), + "calibration": ("201601",), + "validation": ("201701",), + "final_holdout": ("201801",), + } + ) + point_in_time = replace( + resolved, + feed_epoch_definition=definition, + motif_profile=profile, + ) + monkeypatch.setattr( + plan_module, "_resolve_plan_inputs", lambda **_: point_in_time + ) + client = ReconstructionClient() + plan_ref = client.construct_plan( + replace( + _public_spec(source_root, kwargs), + information_mode=InformationMode.EX_ANTE_SIMULATION, + ) + ) + request = client.create_request( + plan_ref.path, + information_mode=InformationMode.EX_ANTE_SIMULATION, + acknowledge_scientific_nonclaim=True, + ) + + preflight = client.preflight(request) + + assert preflight.executable + assert preflight.dry_run["information_mode"] == "ex_ante_simulation" + + +def test_public_spec_rejects_m1_partial_triangle_and_broker_only( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + """Unsupported formats and delivery requests fail before plan building.""" + source_root, kwargs = planned_environment + spec = _public_spec(source_root, kwargs) + + with pytest.raises(ReconstructionUnsupportedError, match="timeframe"): + replace(spec, timeframe="M1") + with pytest.raises(ReconstructionUnsupportedError, match="symbols"): + replace(spec, symbols=("eurusd", "gbpusd")) + with pytest.raises( + ReconstructionUnsupportedError, match="broker_delivery_artifact" + ): + replace( + spec, + delivery_mode=ReconstructionDeliveryMode.BROKER_CONDITIONED, + ) + + +def test_public_submit_status_resume_and_receipt_round_trip( + planned_environment: tuple[Path, dict[str, Any]], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Public controls retain exact request stores and fresh resume attempts.""" + source_root, kwargs = planned_environment + plan_ref = ReconstructionClient().construct_plan( + _public_spec(source_root, kwargs) + ) + request = ReconstructionClient().create_request( + plan_ref.path, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + acknowledge_scientific_nonclaim=True, + ) + config = build_orchestration_worker_config( + workspace=tmp_path, + runtime_home=tmp_path / "runtime", + ) + + class FakeClient: + def __init__(self) -> None: + self.calls: list[tuple[Any, dict[str, Any], dict[str, Any]]] = [] + + async def start_workflow( + self, workflow: Any, payload: Any, **options: Any + ) -> Any: + self.calls.append((workflow, payload, options)) + return SimpleNamespace(id=options["id"], run_id="fake-run") + + temporal = FakeClient() + client = ReconstructionClient(config=config, temporal_client=temporal) + submitted = client.submit(request) + receipt_path = write_operation_receipt( + submitted, tmp_path / "submission.json" + ) + restored = read_operation_receipt(receipt_path) + status = client.inspect(restored, offline=True) + + async def fake_cancel_job(workflow_id: str, **options: Any) -> Any: + assert workflow_id + assert options["reason"] == "operator request" + assert isinstance(options["status_store"], ManifestStatusStore) + return SimpleNamespace( + to_dict=lambda: { + "workflow_id": workflow_id, + "status": "CANCELLED", + "lifecycle": "cancel_requested", + } + ) + + monkeypatch.setattr( + "histdatacom.reconstruction.cancel_job", fake_cancel_job + ) + cancelled = client.cancel(restored, reason="operator request") + resumed = client.resume(restored, wait=False) + + assert restored == submitted + assert len(submitted.handles) == 4 + assert status.status == "running" + assert cancelled.status == "cancellation_requested" + assert len(cancelled.job_snapshots) == 4 + assert resumed.operation == "resume" + assert resumed.execution_attempt_id == "resume-001" + resume_calls = [ + payload + for _, payload, _ in temporal.calls + if payload.get("execution_attempt_id") == "resume-001" + ] + assert len(resume_calls) == 4 + assert all( + call["request"]["request_id"] == workflow_request.request_id + for call, workflow_request in zip( + resume_calls, + read_synthetic_infill_plan(plan_ref.path).workflow_requests, + strict=True, + ) + ) + + +def _public_spec( + source_root: Path, kwargs: dict[str, Any] +) -> ReconstructionPlanSpecV1: + return ReconstructionPlanSpecV1( + source_root=str(source_root), + feed_epoch_definition_path=str(kwargs["feed_epoch_definition_path"]), + observation_operator_path=str(kwargs["observation_operator_path"]), + market_context_corpus_path=str(kwargs["market_context_corpus_path"]), + cftc_positioning_corpus_path=str( + kwargs["cftc_positioning_corpus_path"] + ), + benchmark_manifest_path=str(kwargs["benchmark_manifest_path"]), + motif_manifest_path=str(kwargs["motif_manifest_path"]), + motif_index_path=str(kwargs["motif_index_path"]), + motif_qualification_path=str(kwargs["motif_qualification_path"]), + motif_leakage_audit_path=str(kwargs["motif_leakage_audit_path"]), + artifact_root=str(kwargs["artifact_root"]), + output_root=str(kwargs["output_root"]), + checkpoint_root=str(kwargs["checkpoint_root"]), + scratch_root=str(kwargs["scratch_root"]), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + start_period=kwargs["start_period"], + end_period=kwargs["end_period"], + ) + + +def test_public_plan_spec_supports_exact_paired_window_bounds( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + month_start = datetime(2020, 1, 1, tzinfo=timezone.utc) + start_ns = int(month_start.timestamp() * 1_000_000_000) + 60_000_000_000 + end_ns = start_ns + 600_000_000_000 + spec = replace( + _public_spec(source_root, kwargs), + requested_start_ns=start_ns, + requested_end_ns=end_ns, + window_size_ns=600_000_000_000, + ) + + restored = ReconstructionPlanSpecV1.from_dict(spec.to_dict()) + plan_ref = ReconstructionClient().construct_plan(restored) + plan = read_synthetic_infill_plan(plan_ref.path) + + assert plan.requested_start_ns == start_ns + assert plan.requested_end_ns == end_ns + assert plan.resources.planned_window_count == 1 + assert ( + plan.resources.candidate_amplification + <= plan.run.storage_policy.max_candidate_amplification + ) + assert { + task.window.core_start_ns + for request in plan.workflow_requests + for task in request.tasks + } == {start_ns} + + payload = spec.to_dict() + payload["requested_end_ns"] = None + with pytest.raises( + ReconstructionUnsupportedError, + match="must be supplied together", + ): + ReconstructionPlanSpecV1.from_dict(payload) + + +def test_exact_period_resolution_does_not_round_month_boundary_nanoseconds() -> ( + None +): + """The last nanosecond in a month stays in that source partition.""" + february_start_ns = int( + datetime(2020, 2, 1, tzinfo=timezone.utc).timestamp() * 1_000_000_000 + ) + + assert plan_module._period_for_ns(february_start_ns - 1) == "202001" + assert plan_module._period_for_ns(february_start_ns) == "202002" + + +def test_source_inventory_declares_tick_only_ordinal_identity( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + plan = build_synthetic_infill_plan(source_root, **kwargs) + inventory = read_reconstruction_source_inventory( + plan.artifact_graph["source_inventory"].path + ) + + assert len(inventory.partitions) == 3 + assert all( + item.artifact.kind == ASCII_TICK_SOURCE_KIND + for item in inventory.partitions + ) + assert all( + item.to_dict()["row_identity_basis"] + == "zero-based-arrow-row-ordinal-v1" + for item in inventory.partitions + ) + assert inventory.to_dict()["input_contract"] == "ascii/T-tick-bid-ask-only" + + with pytest.raises(ValueError, match="complete synchronized triangle"): + ReconstructionSourceInventoryV1( + source_root=inventory.source_root, + symbols=inventory.symbols, + periods=inventory.periods, + partitions=inventory.partitions[:-1], + requested_start_ns=inventory.requested_start_ns, + requested_end_ns=inventory.requested_end_ns, + total_row_count=sum( + item.row_count for item in inventory.partitions[:-1] + ), + total_size_bytes=sum( + item.artifact.size_bytes or 0 + for item in inventory.partitions[:-1] + ), + ) + + +def test_builder_rejects_stale_source_hash( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + path = source_root / "eurusd" / "2020" / "1" / ".data" + path.write_bytes(path.read_bytes() + b"stale") + + with pytest.raises( + ReconstructionPlanCompatibilityError, match="source hash differs" + ): + build_synthetic_infill_plan(source_root, **kwargs) + + +def test_builder_rejects_partial_triangle_before_artifact_resolution( + tmp_path: Path, +) -> None: + kwargs = _builder_kwargs(tmp_path) + + with pytest.raises( + ReconstructionPlanCompatibilityError, match="complete.*triangle" + ): + build_synthetic_infill_plan( + tmp_path / "ASCII" / "T", + symbols=("EURUSD", "GBPUSD"), + **kwargs, + ) + + +def test_builder_rejects_plan_root_inside_immutable_source( + tmp_path: Path, +) -> None: + kwargs = _builder_kwargs(tmp_path) + source_root = tmp_path / "ASCII" / "T" + kwargs["output_root"] = source_root / "generated" + + with pytest.raises( + ReconstructionPlanCompatibilityError, + match="output root overlaps the immutable source tree", + ): + build_synthetic_infill_plan(source_root, **kwargs) + + +def test_builder_rejects_overlapping_durable_roots(tmp_path: Path) -> None: + kwargs = _builder_kwargs(tmp_path) + kwargs["checkpoint_root"] = Path(kwargs["output_root"]) / "checkpoints" + + with pytest.raises( + ReconstructionPlanCompatibilityError, + match="output and checkpoint roots overlap", + ): + build_synthetic_infill_plan(tmp_path / "ASCII" / "T", **kwargs) + + +def test_execution_manifest_rejects_artifact_inside_scratch( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + plan = build_synthetic_infill_plan(source_root, **kwargs) + execution = read_reconstruction_plan_execution_manifest( + plan.artifact_graph["execution_manifest"].path + ) + + with pytest.raises(ValueError, match="artifact .* inside the scratch root"): + replace( + execution, + scratch_root=Path(execution.artifacts["configuration"].path).parent, + manifest_id="", + ) + + +def test_builder_rejects_unsupported_period( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + kwargs = {**kwargs, "start_period": "201912", "end_period": "201912"} + + with pytest.raises( + ReconstructionPlanCompatibilityError, match="periods.*common" + ): + build_synthetic_infill_plan(source_root, **kwargs) + + +def test_builder_rejects_ex_ante_artifact_leakage( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + + with pytest.raises( + ReconstructionPlanCompatibilityError, + match="ex-ante plan refused.*observe the requested future", + ): + build_synthetic_infill_plan( + source_root, + information_mode=InformationMode.EX_ANTE_SIMULATION, + **kwargs, + ) + + +def test_builder_rejects_quota_overflow( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + policy = ReconstructionStoragePolicyV1( + max_memory_bytes=1, + max_scratch_bytes=1, + max_output_bytes=1, + ) + + with pytest.raises(ValueError, match="resource preflight failed"): + build_synthetic_infill_plan( + source_root, storage_policy=policy, **kwargs + ) + + +def test_builder_records_a_fully_refused_interval_without_executable_work( + planned_environment: tuple[Path, dict[str, Any]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Unsupported spans remain contiguous, explicit, and safe to skip.""" + source_root, kwargs = planned_environment + monkeypatch.setattr( + plan_module, + "preflight_market_context_corpus", + lambda *_, **__: SimpleNamespace(reasons=("context unsupported",)), + ) + + plan = build_synthetic_infill_plan(source_root, **kwargs) + plan_ref = write_synthetic_infill_plan(plan, kwargs["artifact_root"]) + restored = read_synthetic_infill_plan(plan_ref.path) + execution = read_reconstruction_plan_execution_manifest( + restored.artifact_graph["execution_manifest"].path + ) + + assert restored.status == "ready_with_refusals" + assert restored.workflow_requests == () + assert restored.resources.executable_window_count == 0 + assert restored.resources.refused_window_count == 2 + assert restored.resources.workflow_request_count == 0 + assert restored.resources.estimated_output_bytes == 0 + assert execution.executable_window_count == 0 + assert len(execution.refusal_ids) == 2 + validate_synthetic_infill_plan_for_execution(restored) + request = ReconstructionClient().create_request( + plan_ref.path, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + acknowledge_scientific_nonclaim=True, + ) + preflight = ReconstructionClient().preflight(request) + assert preflight.status == "refused" + assert not preflight.executable + + +def test_broker_only_delivery_requires_the_broker_artifact( + tmp_path: Path, +) -> None: + kwargs = _builder_kwargs(tmp_path) + + with pytest.raises( + ReconstructionPlanCompatibilityError, + match="requires a strong broker artifact", + ): + build_synthetic_infill_plan( + tmp_path / "ASCII" / "T", + delivery_mode=ReconstructionDeliveryMode.BROKER_CONDITIONED, + **kwargs, + ) + + +def test_stage_loader_rejects_graph_reference_substitution( + planned_environment: tuple[Path, dict[str, Any]], +) -> None: + source_root, kwargs = planned_environment + plan = build_synthetic_infill_plan(source_root, **kwargs) + command = plan.workflow_requests[0].tasks[0].commands[0] + foreign = kwargs["artifact_root"] / "foreign.json" + foreign.write_text("{}", encoding="utf-8") + substituted = replace( + command, + input_manifest_refs=( + artifact_ref_for_file( + foreign, + kind="unexpected", + ), + ), + command_id="", + ) + + with pytest.raises(ReconstructionPlanCompatibilityError): + load_reconstruction_stage_plan(substituted, verify_artifacts=False) diff --git a/tests/unit/test_release_workflow.py b/tests/unit/test_release_workflow.py index b97b3684..167dcb3b 100644 --- a/tests/unit/test_release_workflow.py +++ b/tests/unit/test_release_workflow.py @@ -94,6 +94,9 @@ def test_release_workflow_builds_platform_wheels_only_when_opted_in() -> None: assert isinstance(dispatch, dict) inputs = dispatch["inputs"] assert isinstance(inputs, dict) + push = triggers["push"] + assert isinstance(push, dict) + assert push["tags"] == ["*.*.*"] jobs = workflow["jobs"] assert isinstance(jobs, dict) expected_platforms = set(fetch_script.TEMPORAL_CLI_ASSETS) @@ -107,6 +110,10 @@ def test_release_workflow_builds_platform_wheels_only_when_opted_in() -> None: assert isinstance(size_confirm, dict) assert size_confirm["default"] is False assert "size policy" in str(size_confirm["description"]) + testpypi_run = inputs["testpypi_run_id"] + assert isinstance(testpypi_run, dict) + assert testpypi_run["default"] == "" + assert "exact files" in str(testpypi_run["description"]) env = workflow["env"] assert isinstance(env, dict) @@ -122,6 +129,13 @@ def test_release_workflow_builds_platform_wheels_only_when_opted_in() -> None: assert "build-only" in validation_command assert "bundled_platform_wheel_size_confirmed" in validation_command assert "private/offline" in validation_command + promotion_validation = _step_run( + validation, "Validate PyPI promotion input" + ) + assert "testpypi_dry_run_confirmed=true" in promotion_validation + assert "TESTPYPI_DRY_RUN_CONFIRMED" in promotion_validation + assert "TESTPYPI_RUN_ID" in promotion_validation + assert "successful dev Release run" in promotion_validation build_platform = jobs["build-platform-wheels"] assert isinstance(build_platform, dict) @@ -223,7 +237,7 @@ def test_release_workflow_builds_platform_wheels_only_when_opted_in() -> None: assert isinstance(assemble, dict) assert assemble["needs"] == "build-metadata" assert jobs["publish-testpypi"]["needs"] == "assemble-release-artifacts" - assert jobs["publish-pypi"]["needs"] == "assemble-release-artifacts" + assert jobs["publish-pypi"]["needs"] == "validate-release-inputs" assert jobs["publish-testpypi"]["if"] == ( "github.event_name == 'workflow_dispatch' && " "inputs.release_target == 'testpypi' && " @@ -245,6 +259,10 @@ def test_release_workflow_publishes_metadata_only_dist_artifact() -> None: build_metadata = jobs["build-metadata"] assert isinstance(build_metadata, dict) assert build_metadata["needs"] == "validate-release-inputs" + assert build_metadata["if"] == ( + "github.event_name != 'workflow_dispatch' || " + "inputs.release_target != 'pypi'" + ) assemble = jobs["assemble-release-artifacts"] assert isinstance(assemble, dict) @@ -275,7 +293,61 @@ def test_release_workflow_publishes_metadata_only_dist_artifact() -> None: } assert jobs["publish-testpypi"]["needs"] == "assemble-release-artifacts" - assert jobs["publish-pypi"]["needs"] == "assemble-release-artifacts" + assert jobs["publish-pypi"]["needs"] == "validate-release-inputs" + + +def test_pypi_promotion_reuses_exact_successful_testpypi_artifact() -> None: + """Production publishing must promote tested files instead of rebuilding.""" + workflow = _release_workflow() + jobs = workflow["jobs"] + assert isinstance(jobs, dict) + publish = jobs["publish-pypi"] + assert isinstance(publish, dict) + + assert publish["needs"] == "validate-release-inputs" + permissions = publish["permissions"] + assert isinstance(permissions, dict) + assert permissions["actions"] == "read" + assert permissions["contents"] == "read" + assert permissions["id-token"] == "write" + + checkout = _step(publish, "Check out production source and tags") + assert checkout["with"] == {"fetch-depth": 0} + + validate = _step_run(publish, "Validate successful TestPyPI source run") + assert "actions/runs/${TESTPYPI_RUN_ID}" in validate + assert 'head_branch == "dev"' in validate + assert '.path == ".github/workflows/release.yml"' in validate + assert '.name == "Publish to TestPyPI"' in validate + assert '.conclusion == "success"' in validate + assert "histdatacom-dist" in validate + assert "git merge-base --is-ancestor" in validate + assert ".github/workflows/release.yml" in validate + assert "RELEASE.md" in validate + assert "tests/unit/test_release_workflow.py" in validate + + download = _step_run(publish, "Download exact TestPyPI release artifacts") + assert 'gh run download "${TESTPYPI_RUN_ID}"' in download + assert "--name histdatacom-dist" in download + assert "--dir dist" in download + + testpypi_verify = _step_run( + publish, "Verify exact TestPyPI artifact hashes" + ) + assert "test.pypi.org/pypi/histdatacom" in testpypi_verify + assert "remote_hashes != local_hashes" in testpypi_verify + assert "histdatacom.pypi-artifact-promotion.v1" in testpypi_verify + + pypi_verify = _step_run(publish, "Verify exact PyPI artifact hashes") + assert "pypi.org/pypi/histdatacom" in pypi_verify + assert 'remote_hashes != report["files"]' in pypi_verify + assert 'report["pypi_verified"] = True' in pypi_verify + + step_names = [ + step.get("name") for step in publish["steps"] if isinstance(step, dict) + ] + assert "Download release artifacts" not in step_names + assert "Build sdist and fallback wheel" not in step_names def test_package_metadata_advertises_platform_wheel_support() -> None: @@ -292,6 +364,32 @@ def test_package_metadata_advertises_platform_wheel_support() -> None: } <= classifiers +def test_package_metadata_advertises_optional_models_provider() -> None: + """Rich fitted models should remain isolated in the models/all extras.""" + optional = _pyproject_config()["project"]["optional-dependencies"] + + assert optional["models"] == [ + "arch>=8.0.0,<9", + "statsmodels>=0.14.6,<0.15", + ] + assert "arch>=8.0.0,<9" in optional["all"] + assert "statsmodels>=0.14.6,<0.15" in optional["all"] + assert "arch==8.0.0" in optional["test"] + assert "statsmodels==0.14.6" in optional["test"] + assert "arch==8.0.0" in optional["dev"] + assert "statsmodels==0.14.6" in optional["dev"] + + +def test_package_metadata_advertises_optional_duckdb_query_provider() -> None: + """DuckDB stays optional at runtime and pinned in verification extras.""" + optional = _pyproject_config()["project"]["optional-dependencies"] + + assert optional["query"] == ["duckdb>=1.5.4,<2"] + assert "duckdb>=1.5.4,<2" in optional["all"] + assert "duckdb==1.5.4" in optional["test"] + assert "duckdb==1.5.4" in optional["dev"] + + def test_runtime_runbook_documents_windows_runtime_support_gap() -> None: """Release docs should state the current Windows runtime support boundary.""" runbook = _project_text("docs/temporal-orchestration-runtime-runbook.md") @@ -371,19 +469,25 @@ def test_local_pypi_install_smoke_uses_exact_version_verifier() -> None: assert "pypi_install)\n pypi_install\n ;;" in script -def test_release_docs_mark_local_publishing_as_current_path() -> None: - """Release docs should not imply Actions deployment is active today.""" +def test_release_docs_cover_trusted_promotion_and_local_fallback() -> None: + """Release docs should define exact promotion and the local fallback.""" release_docs = _project_text("RELEASE.md") readme = _project_text("README.md") assert ( - "Local publishing is the authoritative release path today." - in release_docs + "GitHub Actions Trusted Publishing is the authoritative" in release_docs + ) + assert "Local Publishing Fallback" in release_docs + assert "testpypi_run_id" in release_docs + assert ( + "publishes those same files to PyPI without rebuilding" in release_docs ) - assert "GitHub Actions" in release_docs - assert "publishing is future architecture" in release_docs + assert "does not rebuild production distributions" in release_docs assert "TestPyPI is only dispatchable from `dev`" in release_docs - assert "PyPI is only dispatchable from `main`" in release_docs + assert re.search( + r"PyPI is only dispatchable from\s+`main`", + release_docs, + ) assert ( "`bash pypi.sh testpypi` is guarded to run from `dev`" in release_docs ) diff --git a/tests/unit/test_resource_usage.py b/tests/unit/test_resource_usage.py new file mode 100644 index 00000000..7f7fbc7a --- /dev/null +++ b/tests/unit/test_resource_usage.py @@ -0,0 +1,167 @@ +"""Tests for portable process resource-usage probes.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace +from typing import Any + +import histdatacom.resource_usage as resource_usage + + +class _FakeResource: + RUSAGE_SELF = 0 + + @staticmethod + def getrusage(_target: int) -> SimpleNamespace: + return SimpleNamespace(ru_maxrss=123) + + +class _FakeFunction: + def __init__(self, callback: Any) -> None: + self.callback = callback + self.argtypes: list[Any] = [] + self.restype: Any = None + + def __call__(self, *args: Any) -> Any: + return self.callback(*args) + + +def test_resource_peak_rss_normalizes_linux_kib_and_preserves_macos_bytes() -> ( + None +): + linux = resource_usage._resource_peak_rss_measurement( + _FakeResource, + platform_name="linux", + ) + macos = resource_usage._resource_peak_rss_measurement( + _FakeResource, + platform_name="darwin", + ) + + assert linux == resource_usage.PeakRssMeasurement( + bytes=123 * 1024, + source="resource.ru_maxrss", + available=True, + ) + assert macos == resource_usage.PeakRssMeasurement( + bytes=123, + source="resource.ru_maxrss", + available=True, + ) + + +def test_windows_peak_rss_uses_peak_working_set_bytes(monkeypatch: Any) -> None: + get_current_process = _FakeFunction(lambda: 1234) + + def populate_counters( + _process: int, + counters_pointer: Any, + _size: int, + ) -> int: + counters_pointer._obj.PeakWorkingSetSize = 987_654 + return 1 + + get_process_memory_info = _FakeFunction(populate_counters) + + def fake_windll(name: str, *, use_last_error: bool) -> SimpleNamespace: + assert use_last_error is True + if name == "kernel32": + return SimpleNamespace(GetCurrentProcess=get_current_process) + assert name == "psapi" + return SimpleNamespace(GetProcessMemoryInfo=get_process_memory_info) + + monkeypatch.setattr( + resource_usage.ctypes, + "WinDLL", + fake_windll, + raising=False, + ) + + assert resource_usage._windows_peak_working_set_bytes() == 987_654 + + +def test_public_probe_records_windows_source_when_resource_is_unavailable( + monkeypatch: Any, +) -> None: + def missing_resource(_name: str) -> Any: + raise ModuleNotFoundError("resource") + + monkeypatch.setattr(resource_usage, "import_module", missing_resource) + monkeypatch.setattr( + resource_usage, + "sys", + SimpleNamespace(platform="win32"), + ) + monkeypatch.setattr( + resource_usage, + "_windows_peak_working_set_bytes", + lambda: 456_789, + ) + + assert resource_usage.peak_rss_measurement() == ( + resource_usage.PeakRssMeasurement( + bytes=456_789, + source="windows.PeakWorkingSetSize", + available=True, + ) + ) + assert resource_usage.peak_rss_bytes() == 456_789 + + +def test_public_probe_has_explicit_unavailable_sentinel( + monkeypatch: Any, +) -> None: + def missing_resource(_name: str) -> Any: + raise ModuleNotFoundError("resource") + + monkeypatch.setattr(resource_usage, "import_module", missing_resource) + monkeypatch.setattr( + resource_usage, + "sys", + SimpleNamespace(platform="unknown"), + ) + + assert resource_usage.peak_rss_measurement() == ( + resource_usage.PeakRssMeasurement( + bytes=0, + source="unavailable", + available=False, + ) + ) + assert resource_usage.peak_rss_bytes() == 0 + + +def test_public_release_surfaces_import_without_resource_module() -> None: + root = Path(__file__).resolve().parents[2] + program = """ +import builtins + +real_import = builtins.__import__ + +def guarded_import(name, *args, **kwargs): + if name == "resource": + raise ModuleNotFoundError("resource is unavailable") + return real_import(name, *args, **kwargs) + +builtins.__import__ = guarded_import + +import histdatacom.data_analytics.feed_epochs_v2 +import histdatacom.market_context.corpus +import histdatacom.market_context.positioning +import histdatacom.synthetic.benchmark_corpus +import histdatacom.synthetic.motif_library +import histdatacom.synthetic.observation_calibration +import histdatacom.synthetic.reconstruction_handlers +""" + completed = subprocess.run( + [sys.executable, "-c", program], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr diff --git a/tests/unit/test_runtime_contracts.py b/tests/unit/test_runtime_contracts.py index 2a8f29b8..3b5d040b 100644 --- a/tests/unit/test_runtime_contracts.py +++ b/tests/unit/test_runtime_contracts.py @@ -153,11 +153,13 @@ def test_run_request_round_trips_options_payload() -> None: options.repo_quality_columns = True options.no_overlap = True options.schedule_key = "eurusd-cache" + options.output_timezone = "America/New_York" request = RunRequest.from_options(options, request_id="run-test") restored = RunRequest.from_dict(json.loads(json.dumps(request.to_dict()))) assert restored == request + assert restored.metadata["output_timezone"] == "America/New_York" assert restored.request_id == "run-test" assert restored.pairs == ("eurusd", "gbpusd") assert restored.timeframes == ("T",) diff --git a/tests/unit/test_smoke_container.py b/tests/unit/test_smoke_container.py new file mode 100644 index 00000000..29a56ccb --- /dev/null +++ b/tests/unit/test_smoke_container.py @@ -0,0 +1,211 @@ +"""Tests for the histdatacom application-image smoke helper.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from typing import Any + + +def _module() -> Any: + script_path = ( + Path(__file__).resolve().parents[2] / "scripts/smoke_container.py" + ) + spec = importlib.util.spec_from_file_location( + "smoke_container", + script_path, + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _completed( + module: Any, + command: list[str], + *, + stdout: str = "", + stderr: str = "", + returncode: int = 0, +) -> Any: + return module.subprocess.CompletedProcess( + args=command, + returncode=returncode, + stdout=stdout, + stderr=stderr, + ) + + +def _inspect_payload() -> str: + env = [ + "HISTDATACOM_RUNTIME_HOME=/workspace/runtime", + "HISTDATACOM_RUNTIME_WORKSPACE=/workspace", + "HISTDATACOM_TEMPORAL_CACHE_DIR=/workspace/cache/temporal-cli", + ] + labels = { + name: "value" + for name in ( + "org.opencontainers.image.created", + "org.opencontainers.image.description", + "org.opencontainers.image.documentation", + "org.opencontainers.image.licenses", + "org.opencontainers.image.revision", + "org.opencontainers.image.source", + "org.opencontainers.image.title", + "org.opencontainers.image.version", + ) + } + return json.dumps( + [ + { + "Architecture": "arm64", + "Config": { + "Cmd": ["--help"], + "Entrypoint": ["/usr/bin/tini", "--", "histdatacom"], + "Env": env, + "Healthcheck": None, + "Labels": labels, + "User": "10001:10001", + "Volumes": None, + "WorkingDir": "/workspace", + }, + "Id": "sha256:image", + "Os": "linux", + } + ] + ) + + +def test_smoke_builds_inspects_executes_and_cleans_image() -> None: + """The default smoke should prove the image contract without residue.""" + module = _module() + calls: list[tuple[list[str], bool]] = [] + + def fake_run(command: list[str], *, check: bool = True) -> Any: + calls.append((list(command), check)) + if command[:2] == ["docker", "build"]: + return _completed(module, command) + if command[:3] == ["docker", "image", "inspect"]: + return _completed(module, command, stdout=_inspect_payload()) + if command[-1:] == ["--version"]: + return _completed(module, command, stdout="histdatacom 2.0.0\n") + if command[-1:] == ["--help"]: + return _completed( + module, + command, + stdout="usage: histdatacom [-h]\n", + ) + if "--entrypoint" in command and "python" in command: + return _completed( + module, + command, + stdout=json.dumps( + { + "gid": 10001, + "paths": ["/workspace/data"], + "uid": 10001, + } + ), + ) + if command[:4] == ["docker", "image", "rm", "--force"]: + return _completed(module, command) + raise AssertionError(command) + + report = module.run_container_smoke( + image="histdatacom:test", + context=Path("."), + created="2026-07-14T00:00:00Z", + run_command=fake_run, + ) + + assert report["status"] == "passed" + assert report["platform"] == {"architecture": "arm64", "os": "linux"} + build = calls[0][0] + assert "IMAGE_CREATED=2026-07-14T00:00:00Z" in build + assert "IMAGE_VERSION=local-smoke" in build + assert ( + ["docker", "image", "rm", "--force", "histdatacom:test"], + False, + ) in calls + + +def test_deep_smoke_proves_temporal_cache_across_containers() -> None: + """Connected mode should start/stop Temporal and reuse its named cache.""" + module = _module() + calls: list[tuple[list[str], bool]] = [] + + def fake_run(command: list[str], *, check: bool = True) -> Any: + calls.append((list(command), check)) + if command[:2] == ["docker", "build"]: + return _completed(module, command) + if command[:3] == ["docker", "image", "inspect"]: + return _completed(module, command, stdout=_inspect_payload()) + if command[-1:] == ["--version"]: + return _completed(module, command, stdout="histdatacom 2.0.0\n") + if command[-1:] == ["--help"]: + return _completed( + module, + command, + stdout="usage: histdatacom [-h]\n", + ) + if "--entrypoint" in command and "python" in command: + if "/usr/bin/tini" in command: + return _completed( + module, + command, + stdout=json.dumps( + { + "doctor": {}, + "start": {"state": "running"}, + "stop": {"state": "stopped"}, + } + ), + ) + return _completed( + module, + command, + stdout=json.dumps({"gid": 10001, "uid": 10001}), + ) + if command[:3] == ["docker", "volume", "create"]: + return _completed(module, command, stdout=f"{command[-1]}\n") + if command[-3:] == ["runtime", "doctor", "--json"]: + return _completed( + module, + command, + stdout=json.dumps( + { + "platform": {"key": "linux-arm64"}, + "runtime_provisioning": { + "cache_available": True, + "cache_entries": [{"valid": True}], + }, + } + ), + ) + if command[:4] in ( + ["docker", "volume", "rm", "--force"], + ["docker", "image", "rm", "--force"], + ): + return _completed(module, command) + raise AssertionError(command) + + report = module.run_container_smoke( + image="histdatacom:test", + deep_runtime=True, + created="2026-07-14T00:00:00Z", + run_command=fake_run, + ) + + assert report["runtime"]["start_state"] == "running" + assert report["runtime"]["stop_state"] == "stopped" + assert report["runtime"]["runtime_provisioning"]["cache_available"] + assert any(call[:3] == ["docker", "volume", "create"] for call, _ in calls) + assert any( + call[:4] == ["docker", "volume", "rm", "--force"] and check is False + for call, check in calls + ) diff --git a/tests/unit/test_smoke_influx_docker.py b/tests/unit/test_smoke_influx_docker.py index 80d01a8c..d1a358df 100644 --- a/tests/unit/test_smoke_influx_docker.py +++ b/tests/unit/test_smoke_influx_docker.py @@ -124,7 +124,7 @@ def get_value(self) -> int: return self.value class FakeTable: - records = [FakeRecord(2)] + records = [FakeRecord(module.EXPECTED_FIELD_COUNT)] class FakeQueryApi: def query(self, query: str, *, org: str) -> list[FakeTable]: @@ -155,7 +155,11 @@ def query_api(self) -> FakeQueryApi: assert written_batches == [[line] for line in module.SMOKE_LINES] assert report["written_lines"] == 2 - assert report["actual_field_count"] == 2 + assert report["actual_field_count"] == module.EXPECTED_FIELD_COUNT + assert "row_id=1" in module.SMOKE_LINES[0].split(" ", maxsplit=1)[0] + assert "quality_status_code=0i" in module.SMOKE_LINES[0] + assert "class_training_action_code=0i" in module.SMOKE_LINES[0] + assert "synth_bid" not in module.SMOKE_LINES[0] assert query_calls[0][1] == "org" assert 'from(bucket: "bucket")' in query_calls[0][0] assert 'r._measurement == "eurusd"' in query_calls[0][0] @@ -182,7 +186,9 @@ def fake_run(command: list[str], *, check: bool = True) -> Any: run_command=fake_run, wait_for_health=lambda url, timeout: {"status": "pass"}, writer_factory=lambda args: _NoopWriter(), - client_factory=lambda **kwargs: _CountClient(6), + client_factory=lambda **kwargs: _CountClient( + module.EXPECTED_FIELD_COUNT + ), ) assert report["status"] == "passed" diff --git a/tests/unit/test_smoke_orchestration_install.py b/tests/unit/test_smoke_orchestration_install.py index 375c5bc0..e4f2e59e 100644 --- a/tests/unit/test_smoke_orchestration_install.py +++ b/tests/unit/test_smoke_orchestration_install.py @@ -1317,10 +1317,10 @@ def _quality_job( } -def test_check_package_metadata_requires_core_temporal_dependency( +def test_check_package_metadata_requires_core_runtime_dependencies( monkeypatch, ) -> None: - """Base-install smoke should require temporalio as a core dependency.""" + """Base-install smoke should require runtime code and timezone data.""" import histdatacom module = _module() @@ -1375,3 +1375,4 @@ def select(self, *, group: str) -> "_EntryPoints": assert report["orchestration_contracts"] == ["RunRequest"] assert report["temporalio_version"] == "1.10.0" + assert report["tzdata_version"] == "1.10.0" diff --git a/tests/unit/test_synthetic_activity.py b/tests/unit/test_synthetic_activity.py new file mode 100644 index 00000000..dc75c00e --- /dev/null +++ b/tests/unit/test_synthetic_activity.py @@ -0,0 +1,359 @@ +"""Tests for honest final-event activity and liquidity-proxy semantics.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pytest + +from histdatacom.synthetic import ( + ACTIVITY_REVERSE_DEGRADATION_METRICS, + SYNTHETIC_EVENT_SCHEMA_VERSION, + ActivityAggregationSemantics, + ActivitySliceScope, + ActivityVolumeState, + InformationMode, + ReconstructionActivityBenchmarkEvidenceV1, + ReconstructionActivityManifestV1, + ReconstructionActivityPolicyV1, + activity_bar_projection_semantics, + read_reconstruction_activity_manifest, + reconstruction_activity_benchmark_evidence, + summarize_committed_reconstruction_activity, + summarize_reconstruction_activity, + summarize_reconstruction_activity_streams, + write_reconstruction_activity_manifest, +) +from histdatacom.synthetic.contracts import SYNTHETIC_EVENT_ARROW_COLUMNS +from histdatacom.synthetic.persistence import publish_reconstruction_group +from tests.unit.test_synthetic_benchmark import _run_scorecard +from tests.unit.test_synthetic_contracts import _stream +from tests.unit.test_synthetic_persistence import _publication_inputs + + +def test_activity_contract_round_trip_separates_origins_and_units() -> None: + """Observed, synthetic, and merged activity remain explicit.""" + stream = _stream(generated_count=2) + + manifest = summarize_reconstruction_activity_streams( + (stream,), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:activity-test", + calibration_report_id="ensemble-calibration:sha256:test", + ) + + assert ReconstructionActivityManifestV1.from_json(manifest.to_json()) == ( + manifest + ) + assert manifest.event_count == 4 + assert manifest.symbols == ("eurusd",) + assert manifest.to_dict()["event_schema_version"] == ( + SYNTHETIC_EVENT_SCHEMA_VERSION + ) + assert manifest.to_dict()["event_schema_augmented"] is False + assert manifest.to_dict()["centralized_traded_volume_claim"] is False + by_scope = {item.scope: item for item in manifest.slices} + assert by_scope[ActivitySliceScope.OBSERVED].event_count == 2 + assert by_scope[ActivitySliceScope.SYNTHETIC].event_count == 2 + assert by_scope[ActivitySliceScope.MERGED].event_count == 4 + + synthetic = by_scope[ActivitySliceScope.SYNTHETIC] + metrics = synthetic.metric_by_name + assert metrics["event_count"].value == 2 + assert metrics["quote_update_count"].value == 2 + assert metrics["exposure_duration_ns"].unit == "nanosecond" + assert metrics["tick_intensity_per_second"].unit == ("event_per_second") + assert metrics["mean_spread"].semantics.value == ("spread_liquidity_proxy") + assert metrics["mean_event_confidence"].value == 0.8 + assert synthetic.generator_ids == ("empirical-motif",) + assert synthetic.generator_versions == ("1.0.0",) + assert synthetic.reference_ids == ("reference:modern-2026",) + assert synthetic.motif_ids == ("motif:quiet-london-001",) + assert synthetic.feed_epoch_ids == ("feed-epoch:modern",) + assert synthetic.broker_profile_ids == ("broker-profile:demo-v1",) + assert synthetic.constraint_set_ids == ("constraint:sha256:historical-v1",) + assert synthetic.stream_ids == (stream.stream_id,) + assert synthetic.event_content_sha256 + + +def test_activity_manifest_persistence_is_content_addressed( + tmp_path: Path, +) -> None: + """Published activity evidence is atomic, compact, and hash verified.""" + manifest = summarize_reconstruction_activity_streams( + (_stream(generated_count=2),), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:activity-test", + ) + + first = write_reconstruction_activity_manifest(manifest, tmp_path) + second = write_reconstruction_activity_manifest(manifest, tmp_path) + + assert first == second + assert first.kind == "activity-manifest" + assert read_reconstruction_activity_manifest(first.path) == manifest + Path(first.path).write_bytes(Path(first.path).read_bytes() + b"changed") + with pytest.raises(ValueError, match="hash differs"): + read_reconstruction_activity_manifest(first.path) + + +def test_activity_manifest_compacts_high_cardinality_provenance() -> None: + """Large reference sets retain a bounded prefix plus count and digest.""" + source = _stream(generated_count=3) + events = tuple( + replace( + event, + reference_id=( + f"reference:unique-{index}" + if event.reference_id is not None + else None + ), + motif_id=( + f"motif:unique-{index}" if event.motif_id is not None else None + ), + event_id="", + ) + for index, event in enumerate(source.events) + ) + stream = replace(source, events=events, stream_id="") + + manifest = summarize_reconstruction_activity_streams( + (stream,), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:activity-test", + policy=ReconstructionActivityPolicyV1(max_provenance_values=1), + ) + synthetic = next( + item + for item in manifest.slices + if item.scope is ActivitySliceScope.SYNTHETIC + ) + + assert len(synthetic.reference_ids) == 1 + assert len(synthetic.motif_ids) == 1 + assert any( + value.startswith("reference_ids_truncated:occurrence_count=3:sha256=") + for value in synthetic.limitations + ) + assert any( + value.startswith("motif_ids_truncated:occurrence_count=3:sha256=") + for value in synthetic.limitations + ) + + +def test_activity_is_deterministic_and_keeps_event_schema_narrow() -> None: + """Derived evidence never becomes a fabricated final-event column.""" + stream = _stream(generated_count=3) + kwargs = { + "information_mode": InformationMode.EX_POST_RECONSTRUCTION, + "information_manifest_id": "information-manifest:sha256:stable", + } + + first = summarize_reconstruction_activity_streams((stream,), **kwargs) + second = summarize_reconstruction_activity_streams((stream,), **kwargs) + + assert first == second + assert first.manifest_id == second.manifest_id + assert len(SYNTHETIC_EVENT_ARROW_COLUMNS) == 26 + assert "volume" not in SYNTHETIC_EVENT_ARROW_COLUMNS + assert "activity" not in SYNTHETIC_EVENT_ARROW_COLUMNS + assert first.policy.volume_state is ActivityVolumeState.UNAVAILABLE + assert first.to_dict()["output_mode"] == "derived_metadata" + + +def test_activity_information_modes_fail_closed() -> None: + """Ex-ante evidence requires an as-of boundary; ex-post forbids one.""" + stream = _stream() + + with pytest.raises(ValueError, match="requires as_of_ns"): + summarize_reconstruction_activity_streams( + (stream,), + information_mode=InformationMode.EX_ANTE_SIMULATION, + information_manifest_id="information-manifest:sha256:mode", + ) + with pytest.raises(ValueError, match="rejects as_of_ns"): + summarize_reconstruction_activity_streams( + (stream,), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:mode", + as_of_ns=1_700_000_000_000_000_000, + ) + + ex_ante = summarize_reconstruction_activity_streams( + (stream,), + information_mode=InformationMode.EX_ANTE_SIMULATION, + information_manifest_id="information-manifest:sha256:mode", + as_of_ns=1_700_000_000_000_000_000, + ) + assert ex_ante.information_mode is InformationMode.EX_ANTE_SIMULATION + assert ex_ante.as_of_ns == 1_700_000_000_000_000_000 + + +def test_activity_rejects_unsupported_size_claims_and_unordered_events() -> ( + None +): + """Missing source sizes and unstable ordering cannot become evidence.""" + stream = _stream() + source_size_policy = ReconstructionActivityPolicyV1( + volume_state=ActivityVolumeState.OBSERVED_SOURCE_SIZE + ) + with pytest.raises(ValueError, match="no source-supported size fields"): + summarize_reconstruction_activity_streams( + (stream,), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:size", + policy=source_size_policy, + ) + + with pytest.raises(ValueError, match="strictly ordered"): + summarize_reconstruction_activity( + reversed(stream.events), + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:order", + ) + + +def test_activity_provenance_limits_publish_count_and_digest() -> None: + """Bounded metadata compacts excess provenance without losing evidence.""" + stream = _stream(generated_count=2) + events = tuple( + replace( + event, + source_version_id=f"source-version:{index}", + event_id="", + ) + for index, event in enumerate(stream.events, start=1) + ) + policy = ReconstructionActivityPolicyV1(max_provenance_values=1) + + manifest = summarize_reconstruction_activity( + events, + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:bounded", + policy=policy, + ) + + assert all(len(item.source_version_ids) == 1 for item in manifest.slices) + assert all( + any( + value.startswith("source_version_ids_truncated:occurrence_count=") + and ":sha256=" in value + for value in item.limitations + ) + for item in manifest.slices + ) + + +def test_bar_projection_semantics_are_explicit_and_non_volume() -> None: + """#18 receives sum/recompute/weighted-mean rules, not a volume guess.""" + projection = activity_bar_projection_semantics() + + assert projection["tick_count"]["operation"] == "sum" + assert projection["activity_duration_ns"]["operation"] == ( + "recompute_from_bar_event_bounds" + ) + assert projection["tick_intensity_per_second"]["operation"] == ( + "recompute_count_divided_by_activity_duration" + ) + assert projection["mean_spread"]["operation"] == ( + "event_support_weighted_mean" + ) + assert projection["volume"] == { + "source_metric": None, + "operation": "unavailable_unless_separately_sourced", + "unit": None, + } + + +def test_committed_parquet_activity_matches_in_memory_streaming( + tmp_path: Path, +) -> None: + """Projected one-row batches reproduce the in-memory final event slices.""" + rendered, anchors, storage_policy, retention = _publication_inputs(tmp_path) + published = publish_reconstruction_group( + tmp_path / "archive", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=storage_policy, + row_group_size=2, + ) + information_id = "information-manifest:sha256:committed" + + committed = summarize_committed_reconstruction_activity( + published.manifest_path, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id=information_id, + batch_size=1, + ) + in_memory = summarize_reconstruction_activity_streams( + rendered.streams, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id=information_id, + window_id=published.manifest.window_id, + synchronization_unit_id=(published.manifest.synchronization_unit_id), + product_manifest_id=published.manifest.manifest_id, + ) + + assert committed == in_memory + assert committed.product_manifest_id == published.manifest.manifest_id + assert committed.event_count == published.manifest.event_count + assert all(item.stream_ids for item in committed.slices) + + +def test_existing_reverse_degradation_scorecard_supplies_activity_evidence() -> ( + None +): + """Activity validation reuses the established benchmark and calibration.""" + _, scorecard = _run_scorecard() + + evidence = reconstruction_activity_benchmark_evidence(scorecard) + + assert ( + ReconstructionActivityBenchmarkEvidenceV1.from_dict(evidence.to_dict()) + == evidence + ) + assert set(evidence.metric_support_counts) == set( + ACTIVITY_REVERSE_DEGRADATION_METRICS + ) + assert all(value > 0 for value in evidence.metric_support_counts.values()) + assert evidence.calibration_supported_candidate_count > 0 + assert evidence.execution_failure_count == 0 + assert evidence.to_dict()["automatic_winner"] is False + assert evidence.to_dict()["winner_candidate_id"] is None + + +def test_activity_metric_definition_drift_is_rejected() -> None: + """Units and aggregation semantics are part of the versioned contract.""" + manifest = summarize_reconstruction_activity_streams( + (_stream(),), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:metric-drift", + ) + metric = manifest.slices[0].metrics[0] + + with pytest.raises(ValueError, match="definition differs"): + replace( + metric, + aggregation=ActivityAggregationSemantics.MAXIMUM, + ) + + +def test_activity_json_rejects_derived_claim_drift() -> None: + """Serialized evidence cannot be relabeled as volume or a winner.""" + manifest = summarize_reconstruction_activity_streams( + (_stream(),), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + information_manifest_id="information-manifest:sha256:claim-drift", + ) + payload = manifest.to_dict() + payload["centralized_traded_volume_claim"] = True + + with pytest.raises(ValueError, match="derived activity field"): + ReconstructionActivityManifestV1.from_dict(payload) diff --git a/tests/unit/test_synthetic_bars.py b/tests/unit/test_synthetic_bars.py new file mode 100644 index 00000000..c5a92188 --- /dev/null +++ b/tests/unit/test_synthetic_bars.py @@ -0,0 +1,621 @@ +"""Derived candlestick contracts and atomic publication regression tests.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pyarrow.parquet as pq +import pytest + +from histdatacom.histdata_ascii import columns_for_timeframe +from histdatacom.synthetic import ( + DERIVED_BAR_ARROW_COLUMNS, + STANDARD_DERIVED_BAR_INTERVALS, + ActivitySliceScope, + DerivedBarIntervalV1, + DerivedBarPersistenceError, + DerivedBarPolicyV1, + DerivedBarProductManifestV1, + DerivedBarV1, + commit_derived_bar_publication, + derive_reconstruction_bars, + discover_derived_bar_manifests, + iter_derived_bar_batches, + publish_derived_bars, + scan_derived_bars_polars, + stage_derived_bar_publication, + verify_derived_bar_publication, +) +from histdatacom.synthetic.contracts import ( + SYNTHETIC_EVENT_ARROW_COLUMNS, + SyntheticEventStreamV1, +) +from histdatacom.synthetic.persistence import ( + PublishedReconstructionV1, + publish_reconstruction_group, +) +from tests.unit.test_synthetic_contracts import ( + BASE_TIME_NS, + _generated, + _observed, +) +from tests.unit.test_synthetic_persistence import _publication_inputs + +MINUTE_NS = STANDARD_DERIVED_BAR_INTERVALS["1m"] +ALIGNED_NS = (BASE_TIME_NS // MINUTE_NS) * MINUTE_NS + + +def _boundary_stream() -> SyntheticEventStreamV1: + """Return observed and generated rows spanning an intentionally empty bin.""" + left = _observed( + 101, + event_time_ns=ALIGNED_NS + 10_000_000_000, + bid=1.1000, + ) + right = _observed( + 102, + event_time_ns=ALIGNED_NS + 180_000_000_000, + bid=1.1020, + ) + generated = replace( + _generated(left, right), + event_time_ns=ALIGNED_NS + 20_000_000_000, + bid=1.1010, + ask=1.1012, + event_id="", + ) + return SyntheticEventStreamV1.merge( + run_id=left.run_id, + ensemble_member_id=left.ensemble_member_id, + symbol=left.symbol, + observed_events=(right, left), + synthetic_events=(generated,), + ) + + +def _all_scope_policy(*, max_bars: int = 100) -> DerivedBarPolicyV1: + return DerivedBarPolicyV1( + intervals=("1m",), + scopes=("observed", "synthetic", "merged"), + max_bars=max_bars, + ) + + +def _derive_boundary_bars( + *, + start_ns: int | None = None, + end_ns: int | None = None, +) -> tuple[DerivedBarV1, ...]: + stream = _boundary_stream() + return derive_reconstruction_bars( + stream.events, + source_product_manifest_id="product-manifest:sha256:boundary", + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + policy=_all_scope_policy(), + start_ns=start_ns, + end_ns=end_ns, + ) + + +def test_interval_policy_and_bar_contracts_round_trip() -> None: + """Intervals, policy identity, and rows serialize without claim drift.""" + assert tuple(STANDARD_DERIVED_BAR_INTERVALS) == ( + "1m", + "5m", + "15m", + "30m", + "1h", + "4h", + "1d", + ) + interval = DerivedBarIntervalV1("5m") + assert DerivedBarIntervalV1.from_dict(interval.to_dict()) == interval + policy = DerivedBarPolicyV1( + intervals=("1d", "1m", "1h"), + scopes=("merged", "observed"), + ) + assert policy.intervals == ("1m", "1h", "1d") + assert DerivedBarPolicyV1.from_dict(policy.to_dict()) == policy + assert policy.to_dict()["raw_m1_input"] is False + assert policy.to_dict()["centralized_traded_volume_claim"] is False + + bar = _derive_boundary_bars()[0] + assert DerivedBarV1.from_json(bar.to_json()) == bar + assert tuple(bar.to_dict()) == DERIVED_BAR_ARROW_COLUMNS + assert bar.to_dict()["volume"] is None + assert bar.to_dict()["volume_state"] == "unavailable" + assert bar.policy_id == _all_scope_policy().policy_id + assert bar.rounding_digits == _all_scope_policy().rounding_digits + + stream = _boundary_stream() + low_precision = derive_reconstruction_bars( + stream.events, + source_product_manifest_id="product-manifest:sha256:rounded", + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + policy=DerivedBarPolicyV1( + intervals=("1m",), + rounding_digits=0, + ), + ) + assert low_precision + assert all(item.rounding_digits == 0 for item in low_precision) + + with pytest.raises(ValueError, match="unsupported derived bar interval"): + DerivedBarIntervalV1("2m") + with pytest.raises(ValueError, match="epoch alignment"): + DerivedBarIntervalV1("1m", alignment_epoch_ns=1) + + +def test_bar_contract_rejects_derived_claim_drift() -> None: + """Derived arithmetic and boolean fields cannot be independently forged.""" + bar = next( + item + for item in _derive_boundary_bars() + if item.scope is ActivitySliceScope.MERGED + and item.bar_start_ns == ALIGNED_NS + ) + with pytest.raises(ValueError, match="mid_open differs from bid/ask"): + replace(bar, mid_open=bar.mid_open + 0.00001, bar_id="") + assert bar.tick_intensity_per_second is not None + with pytest.raises(ValueError, match="tick intensity differs"): + replace( + bar, + tick_intensity_per_second=bar.tick_intensity_per_second * 2, + bar_id="", + ) + with pytest.raises(ValueError, match="stale rate differs"): + replace(bar, stale_quote_rate=0.5, bar_id="") + with pytest.raises(ValueError, match="transition support is incomplete"): + replace( + bar, + transition_count=0, + price_change_count=0, + stale_quote_count=0, + stale_quote_rate=None, + bar_id="", + ) + with pytest.raises(ValueError, match="endpoint identities"): + replace(bar, last_event_id=bar.first_event_id, bar_id="") + + payload = bar.to_dict() + payload["is_partial_start"] = 1 + payload["bar_id"] = "" + with pytest.raises(ValueError, match="partial flags must be boolean"): + DerivedBarV1.from_dict(payload) + + +def test_every_standard_interval_uses_half_open_utc_bins() -> None: + """An event on an interval edge starts the next UTC-aligned bar.""" + for ordinal, (interval_code, duration) in enumerate( + STANDARD_DERIVED_BAR_INTERVALS.items() + ): + start = (BASE_TIME_NS // duration) * duration + left = _observed(301 + ordinal * 2, event_time_ns=start + duration - 1) + right = _observed(302 + ordinal * 2, event_time_ns=start + duration) + + bars = derive_reconstruction_bars( + (left, right), + source_product_manifest_id="product-manifest:sha256:intervals", + run_id=left.run_id, + ensemble_member_id=left.ensemble_member_id, + policy=DerivedBarPolicyV1(intervals=(interval_code,)), + ) + + assert tuple(item.bar_start_ns for item in bars) == ( + start, + start + duration, + ) + assert all( + item.bar_end_ns - item.bar_start_ns == duration for item in bars + ) + + +def test_scopes_ohlc_empty_bins_and_boundary_carry_are_explicit() -> None: + """Bars preserve scopes, omit empty bins, and carry transitions forward.""" + bars = _derive_boundary_bars() + keyed = {(bar.scope, bar.bar_start_ns): bar for bar in bars} + + first_merged = keyed[(ActivitySliceScope.MERGED, ALIGNED_NS)] + assert first_merged.event_count == 2 + assert first_merged.observed_event_count == 1 + assert first_merged.synthetic_event_count == 1 + assert first_merged.bid_open == 1.1 + assert first_merged.bid_high == 1.101 + assert first_merged.bid_close == 1.101 + assert first_merged.ask_close == 1.1012 + assert first_merged.mid_close == 1.1011 + assert first_merged.spread_close == pytest.approx(0.0002) + assert first_merged.transition_count == 1 + assert first_merged.price_change_count == 1 + + last_merged = keyed[(ActivitySliceScope.MERGED, ALIGNED_NS + 3 * MINUTE_NS)] + assert last_merged.event_count == 1 + assert last_merged.transition_count == 1 + assert last_merged.price_change_count == 1 + assert last_merged.activity_duration_ns == 0 + assert last_merged.tick_intensity_per_second is None + assert last_merged.stale_quote_rate == 0.0 + + merged_starts = { + bar.bar_start_ns + for bar in bars + if bar.scope is ActivitySliceScope.MERGED + } + assert merged_starts == {ALIGNED_NS, ALIGNED_NS + 3 * MINUTE_NS} + assert (ActivitySliceScope.SYNTHETIC, ALIGNED_NS) in keyed + assert ( + ActivitySliceScope.SYNTHETIC, + ALIGNED_NS + 3 * MINUTE_NS, + ) not in keyed + + +def test_query_boundaries_flag_partial_bars_without_filling() -> None: + """Explicit range cuts flag edge bars and never synthesize empty rows.""" + bars = _derive_boundary_bars( + start_ns=ALIGNED_NS + 15_000_000_000, + end_ns=ALIGNED_NS + 185_000_000_000, + ) + merged = [bar for bar in bars if bar.scope is ActivitySliceScope.MERGED] + + assert len(merged) == 2 + assert merged[0].is_partial_start + assert not merged[0].is_partial_end + assert not merged[1].is_partial_start + assert merged[1].is_partial_end + assert merged[0].first_event_time_ns == ALIGNED_NS + 20_000_000_000 + assert merged[0].transition_count == 0 + + +def test_duplicate_timestamps_use_sequence_and_unordered_rows_fail() -> None: + """Within-time ordering is stable and reversed positions fail closed.""" + first = _observed( + 201, + event_time_ns=ALIGNED_NS + 1, + event_sequence=0, + bid=1.2, + ) + second = _observed( + 202, + event_time_ns=ALIGNED_NS + 1, + event_sequence=1, + bid=1.3, + ) + common = { + "source_product_manifest_id": "product-manifest:sha256:sequence", + "run_id": first.run_id, + "ensemble_member_id": first.ensemble_member_id, + "policy": DerivedBarPolicyV1(intervals=("1m",)), + } + + bar = derive_reconstruction_bars((first, second), **common)[0] + assert bar.bid_open == 1.2 + assert bar.bid_close == 1.3 + assert bar.transition_count == 1 + + with pytest.raises(ValueError, match="strictly ordered"): + derive_reconstruction_bars((second, first), **common) + + +def test_resource_and_provenance_limits_refuse() -> None: + """Bar count and lineage state are explicitly bounded.""" + stream = _boundary_stream() + with pytest.raises(ValueError, match="output exceeds policy"): + derive_reconstruction_bars( + stream.events, + source_product_manifest_id="product-manifest:sha256:limit", + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + policy=_all_scope_policy(max_bars=1), + ) + + second_source = replace( + stream.events[1], + source_version_id="source-artifact:sha256:second", + event_id="", + ) + events = (stream.events[0], second_source) + with pytest.raises(ValueError, match="source_version_ids exceeds"): + derive_reconstruction_bars( + events, + source_product_manifest_id="product-manifest:sha256:lineage", + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + policy=DerivedBarPolicyV1( + intervals=("1m",), + max_provenance_values=1, + ), + ) + + +def test_atomic_publication_round_trip_and_idempotent_commit( + tmp_path: Path, +) -> None: + """Scratch is invisible and one rename publishes exact narrow bars.""" + source = _publish_source(tmp_path) + root = tmp_path / "bars" + policy = DerivedBarPolicyV1( + intervals=("1m", "5m"), + scopes=("observed", "synthetic", "merged"), + ) + + staged = stage_derived_bar_publication( + root, + source.manifest_path, + policy=policy, + start_ns=source.manifest.min_event_time_ns, + end_ns=source.manifest.max_event_time_ns + 1, + batch_size=1, + row_group_size=2, + write_buffer_rows=1, + ) + assert not discover_derived_bar_manifests(root) + committed = commit_derived_bar_publication(staged) + manifest = verify_derived_bar_publication(committed.manifest_path) + + assert DerivedBarProductManifestV1.from_json(manifest.to_json()) == manifest + assert manifest.source_product_manifest_id == source.manifest.manifest_id + assert manifest.query_start_ns == source.manifest.min_event_time_ns + assert manifest.query_end_ns == source.manifest.max_event_time_ns + 1 + assert manifest.bar_count == sum( + partition.row_count for partition in manifest.partitions + ) + assert manifest.to_dict()["event_rows_inline"] is False + assert manifest.to_dict()["analytical_frame_columns_inline"] is False + assert discover_derived_bar_manifests(root) == (committed.manifest_path,) + for partition in manifest.partitions: + table = pq.ParquetFile( + committed.manifest_path.parent / partition.relative_path + ).read() + assert table.schema.names == list(DERIVED_BAR_ARROW_COLUMNS) + assert table.num_columns == 64 + assert table.num_columns < 521 + assert len(SYNTHETIC_EVENT_ARROW_COLUMNS) == 26 + assert "volume" not in SYNTHETIC_EVENT_ARROW_COLUMNS + + retry = commit_derived_bar_publication(staged) + assert retry.idempotent_retry + assert retry.manifest == manifest + + +def test_batch_and_buffer_boundaries_preserve_logical_bars( + tmp_path: Path, +) -> None: + """Input and output chunk sizes cannot change logical bar identities.""" + source = _publish_source(tmp_path) + policy = DerivedBarPolicyV1( + intervals=("1m", "5m"), + scopes=("merged",), + ) + first = publish_derived_bars( + tmp_path / "bars-a", + source.manifest_path, + policy=policy, + batch_size=1, + row_group_size=2, + write_buffer_rows=1, + ) + second = publish_derived_bars( + tmp_path / "bars-b", + source.manifest_path, + policy=policy, + batch_size=3, + row_group_size=3, + write_buffer_rows=3, + ) + + assert first.manifest.logical_content_sha256 == ( + second.manifest.logical_content_sha256 + ) + assert _bar_ids(first.manifest_path) == _bar_ids(second.manifest_path) + assert first.manifest.publication_id == second.manifest.publication_id + + +def test_projection_scans_and_raw_m1_remains_rejected(tmp_path: Path) -> None: + """Readers prune the 64-column product while raw M1 stays unsupported.""" + source = _publish_source(tmp_path) + bars = publish_derived_bars( + tmp_path / "bars", + source.manifest_path, + policy=DerivedBarPolicyV1(intervals=("1m",)), + batch_size=1, + ) + batches = tuple( + iter_derived_bar_batches( + bars.manifest_path, + columns=("symbol", "bar_start_ns", "mid_close"), + symbols=("eurusd",), + intervals=("1m",), + batch_size=1, + ) + ) + assert batches + assert all( + batch.schema.names == ["symbol", "bar_start_ns", "mid_close"] + for batch in batches + ) + lazy = scan_derived_bars_polars( + bars.manifest_path, + columns=("symbol", "bar_start_ns", "mid_close"), + symbols=("eurusd",), + ) + assert lazy.collect().columns == ["symbol", "bar_start_ns", "mid_close"] + assert "PROJECT 3/64 COLUMNS" in lazy.explain(optimized=True) + + eurusd_partition = next( + item + for item in bars.manifest.partitions + if item.symbol == "eurusd" and item.interval_code == "1m" + ) + overlap_start = eurusd_partition.min_bar_start_ns + 1 + overlap = tuple( + iter_derived_bar_batches( + bars.manifest_path, + columns=("bar_start_ns", "bar_end_ns"), + symbols=("eurusd",), + intervals=("1m",), + start_ns=overlap_start, + end_ns=overlap_start + 1, + ) + ) + assert overlap + assert overlap[0].to_pylist()[0]["bar_start_ns"] == ( + eurusd_partition.min_bar_start_ns + ) + + with pytest.raises(ValueError, match="symbols are outside"): + tuple(iter_derived_bar_batches(bars.manifest_path, symbols=("xauusd",))) + with pytest.raises(ValueError, match="scopes are outside"): + scan_derived_bars_polars( + bars.manifest_path, + scopes=(ActivitySliceScope.OBSERVED,), + ) + with pytest.raises(ValueError, match="intervals are outside"): + tuple(iter_derived_bar_batches(bars.manifest_path, intervals=("5m",))) + + with pytest.raises(ValueError, match="unsupported ASCII timeframe"): + columns_for_timeframe("M1") + + +def test_failed_stage_releases_writers_and_removes_scratch( + tmp_path: Path, +) -> None: + """A bounded aggregation failure leaves no open or discoverable product.""" + source = _publish_source(tmp_path) + root = tmp_path / "bars" + + with pytest.raises(ValueError, match="output exceeds policy"): + stage_derived_bar_publication( + root, + source.manifest_path, + policy=DerivedBarPolicyV1(intervals=("1m",), max_bars=1), + batch_size=1, + write_buffer_rows=1, + ) + + assert not discover_derived_bar_manifests(root) + assert not tuple(root.rglob("publication.tmp-*")) + + +def test_partition_and_manifest_tampering_fail_closed(tmp_path: Path) -> None: + """Truncated Parquet and serialized claim drift cannot pass verification.""" + source = _publish_source(tmp_path) + bars = publish_derived_bars( + tmp_path / "bars", + source.manifest_path, + policy=DerivedBarPolicyV1(intervals=("1m",)), + ) + manifest = bars.manifest + payload = manifest.to_dict() + payload["raw_m1_input"] = True + with pytest.raises(ValueError, match="raw_m1_input differs"): + DerivedBarProductManifestV1.from_dict(payload) + payload = manifest.to_dict() + payload["query_start_ns"] = True + with pytest.raises(ValueError, match="must be an integer"): + DerivedBarProductManifestV1.from_dict(payload) + with pytest.raises(ValueError, match="duplicate partitions"): + replace( + manifest, + partitions=manifest.partitions + (manifest.partitions[0],), + publication_id="", + manifest_id="", + ) + with pytest.raises(ValueError, match="cannot be empty"): + replace(manifest.partitions[0], size_bytes=0, partition_id="") + + partition = manifest.partitions[0] + path = bars.manifest_path.parent / partition.relative_path + path.write_bytes(path.read_bytes()[:32]) + with pytest.raises(DerivedBarPersistenceError, match="size differs"): + verify_derived_bar_publication(bars.manifest_path) + + +def test_unexpected_artifacts_are_not_accepted(tmp_path: Path) -> None: + """Committed directories are confined to manifest-declared artifacts.""" + source = _publish_source(tmp_path) + bars = publish_derived_bars( + tmp_path / "bars", + source.manifest_path, + policy=DerivedBarPolicyV1(intervals=("1m",)), + ) + empty = bars.manifest_path.parent / "unexpected-directory" + empty.mkdir() + with pytest.raises( + DerivedBarPersistenceError, match="artifact set differs" + ): + verify_derived_bar_publication(bars.manifest_path) + empty.rmdir() + + extra = bars.manifest_path.parent / "unexpected.csv" + extra.write_text("not part of the product", encoding="utf-8") + + with pytest.raises( + DerivedBarPersistenceError, match="artifact set differs" + ): + verify_derived_bar_publication(bars.manifest_path) + + +def test_symlink_artifacts_are_not_accepted(tmp_path: Path) -> None: + """Publication verification refuses even undeclared readable symlinks.""" + source = _publish_source(tmp_path) + bars = publish_derived_bars( + tmp_path / "bars", + source.manifest_path, + policy=DerivedBarPolicyV1(intervals=("1m",)), + ) + target = tmp_path / "outside.txt" + target.write_text("outside product", encoding="utf-8") + link = bars.manifest_path.parent / "linked.txt" + try: + link.symlink_to(target) + except OSError as err: # pragma: no cover - platform policy + pytest.skip(f"symlink creation is unavailable: {err}") + + with pytest.raises(DerivedBarPersistenceError, match="unsafe symlink"): + verify_derived_bar_publication(bars.manifest_path) + + +def test_derived_bar_contract_is_documented() -> None: + """README and detailed docs preserve the product-boundary decisions.""" + root = Path(__file__).parents[2] + readme = (root / "README.md").read_text(encoding="utf-8") + contract = (root / "docs/derived-bar-contracts.md").read_text( + encoding="utf-8" + ) + + assert "Derived Reconstruction Candlesticks" in readme + assert "publish_derived_bars()" in readme + assert "verified" in readme + assert "committed reconstruction manifest" in readme + assert "Empty bins emit no rows" in contract + assert "centralized_traded_volume_claim=false" in contract + assert "exact 64-column Arrow schema" in contract + assert "Raw `ascii/T`" in contract + assert "caches" in contract + + +def _publish_source(tmp_path: Path) -> PublishedReconstructionV1: + rendered, anchors, storage, retention = _publication_inputs(tmp_path) + return publish_reconstruction_group( + tmp_path / "events", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=storage, + row_group_size=2, + ) + + +def _bar_ids(manifest_path: Path) -> tuple[str, ...]: + return tuple( + str(row["bar_id"]) + for batch in iter_derived_bar_batches( + manifest_path, + columns=("bar_id",), + ) + for row in batch.to_pylist() + ) diff --git a/tests/unit/test_synthetic_benchmark.py b/tests/unit/test_synthetic_benchmark.py new file mode 100644 index 00000000..c1ba1621 --- /dev/null +++ b/tests/unit/test_synthetic_benchmark.py @@ -0,0 +1,890 @@ +"""Tests for the streaming reverse-degradation benchmark.""" + +from __future__ import annotations + +from dataclasses import replace +import hashlib + +import polars as pl +import pytest + +from histdatacom.synthetic import ( + BENCHMARK_EVENT_SCHEMA_VERSION, + BrokerBenchmarkComparisonV1, + BenchmarkCandidateKind, + BenchmarkCandidateV1, + BenchmarkCandidateWindowV1, + BenchmarkControlKind, + BenchmarkEventV1, + BenchmarkExecutionEvidenceV1, + BenchmarkProfileV1, + BenchmarkScenarioV1, + BenchmarkSplitKind, + BenchmarkSplitV1, + ReverseDegradationBenchmarkManifestV1, + ReverseDegradationBenchmarkV1, + ReverseDegradationScorecardV1, + benchmark_events_from_empirical_overlay, + build_benchmark_control_events, + build_benchmark_control_windows, + compare_broker_benchmark_results, + degrade_benchmark_window, + generate_benchmark_candidate_window, + validate_benchmark_information_boundary, +) +from histdatacom.synthetic.information import ( + InformationMode, + InformationSplitKind, + ReconstructionInformationManifestV1, + ReconstructionInformationSplitV1, +) +from histdatacom.synthetic.observation import ( + ObservationApplicationResultV1, + ObservationCarryStateV1, + ObservationInputEventV1, + ObservationOutputEventV1, +) +from histdatacom.synthetic.streaming import ReconstructionWindowV1 + +BASE_NS = 1_700_000_000_000_000_000 +RUN_ID = "reconstruction-run:sha256:benchmark-fixture" +INFORMATION_POLICY_ID = "information-policy:sha256:benchmark-fixture" +WINDOW_PLAN_ID = "window-plan:sha256:benchmark-fixture" + + +def _information_manifest() -> ReconstructionInformationManifestV1: + return ReconstructionInformationManifestV1( + run_id=RUN_ID, + policy_id=INFORMATION_POLICY_ID, + information_mode=InformationMode.EX_ANTE_SIMULATION, + window_plan_id=WINDOW_PLAN_ID, + inputs=(), + splits=( + ReconstructionInformationSplitV1( + InformationSplitKind.TRAIN, + BASE_NS, + BASE_NS + 1_000, + ), + ReconstructionInformationSplitV1( + InformationSplitKind.CALIBRATION, + BASE_NS + 1_000, + BASE_NS + 2_000, + ), + ReconstructionInformationSplitV1( + InformationSplitKind.VALIDATION, + BASE_NS + 2_000, + BASE_NS + 6_000, + ), + ), + ) + + +def _candidate( + kind: BenchmarkCandidateKind, + method: str, + *, + control: BenchmarkControlKind | None = None, + members: tuple[str, ...] = ("control",), + parameters: dict[str, object] | None = None, +) -> BenchmarkCandidateV1: + return BenchmarkCandidateV1( + kind=kind, + method_id=method, + implementation_version="fixture-v1", + parameters=parameters or {}, + ensemble_member_ids=members, + control_kind=control, + ) + + +def _manifest( + *, profile: BenchmarkProfileV1 | None = None +) -> ReverseDegradationBenchmarkManifestV1: + information = _information_manifest() + splits = ( + BenchmarkSplitV1( + BenchmarkSplitKind.CALIBRATION, + BASE_NS + 1_000, + BASE_NS + 2_000, + ), + BenchmarkSplitV1( + BenchmarkSplitKind.VALIDATION, + BASE_NS + 2_000, + BASE_NS + 4_000, + ), + BenchmarkSplitV1( + BenchmarkSplitKind.FINAL_HOLDOUT, + BASE_NS + 4_000, + BASE_NS + 6_000, + ), + ) + scenarios = tuple( + BenchmarkScenarioV1( + split_kind=split, + epoch_id=epoch, + severity_id=severity, + observation_operator_id=f"operator:{epoch}:{severity}", + degradation_parameters={ + "retention_probability": retention, + "severity": severity, + }, + ) + for split, epoch, severity, retention in ( + ( + BenchmarkSplitKind.VALIDATION, + "epoch-modern", + "mild", + 0.8, + ), + ( + BenchmarkSplitKind.FINAL_HOLDOUT, + "epoch-modern", + "severe", + 0.4, + ), + ( + BenchmarkSplitKind.VALIDATION, + "epoch-legacy", + "severe", + 0.4, + ), + ( + BenchmarkSplitKind.FINAL_HOLDOUT, + "epoch-legacy", + "mild", + 0.8, + ), + ) + ) + candidates = ( + _candidate( + BenchmarkCandidateKind.CONTROL, + "no-fill", + control=BenchmarkControlKind.NO_FILL, + ), + _candidate( + BenchmarkCandidateKind.CONTROL, + "linear", + control=BenchmarkControlKind.LINEAR_INTERPOLATION, + parameters={"interval_ns": 100}, + ), + _candidate( + BenchmarkCandidateKind.CONTROL, + "resample", + control=BenchmarkControlKind.RESAMPLE_LAST, + parameters={"interval_ns": 250}, + ), + _candidate( + BenchmarkCandidateKind.CONTROL, + "empirical-overlay", + control=BenchmarkControlKind.EMPIRICAL_OVERLAY, + parameters={"source_schema": "synthetic-tick-generation.v1"}, + ), + _candidate( + BenchmarkCandidateKind.CANDIDATE, + "fixture-generator", + members=("member-a", "member-b"), + parameters={"temperature": 0.0}, + ), + ) + return ReverseDegradationBenchmarkManifestV1( + run_id=RUN_ID, + information_manifest_id=information.manifest_id, + profile=profile or BenchmarkProfileV1(), + splits=splits, + scenarios=scenarios, + candidates=candidates, + ) + + +def _window(scenario: BenchmarkScenarioV1) -> ReconstructionWindowV1: + start = ( + BASE_NS + 2_000 + if scenario.split_kind is BenchmarkSplitKind.VALIDATION + else BASE_NS + 4_000 + ) + return ReconstructionWindowV1( + run_id=RUN_ID, + ensemble_member_id="member-a", + symbols=("EURUSD",), + core_start_ns=start, + core_end_ns=start + 2_000, + ) + + +def _event( + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + ordinal: int, + *, + member: str | None = None, + sparsity: str | None = None, + shift: float = 0.0, + support: bool = False, + anchor: bool = False, +) -> BenchmarkEventV1: + bid = 1.1000 + ordinal * 0.0001 + shift + ask = bid + 0.0001 + mid = (bid + ask) / 2 + return BenchmarkEventV1( + source_event_id=f"source-{ordinal}", + symbol="EURUSD", + event_time_ns=window.core_start_ns + 100 + ordinal * 100, + event_sequence=ordinal, + bid=bid, + ask=ask, + epoch_id=scenario.epoch_id, + session="london", + event_state="scheduled-event" if ordinal >= 3 else "normal", + sparsity=sparsity or scenario.severity_id, + ensemble_member_id=member, + anchor_id=f"anchor-{ordinal}" if anchor else None, + support_lower_mid=mid - 0.0002 if support else None, + support_upper_mid=mid + 0.0002 if support else None, + ) + + +def _events( + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + ordinals: tuple[int, ...], + **kwargs: object, +) -> tuple[BenchmarkEventV1, ...]: + return tuple( + _event(scenario, window, ordinal, **kwargs) for ordinal in ordinals + ) + + +def _candidate_windows( + manifest: ReverseDegradationBenchmarkManifestV1, + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + reference: tuple[BenchmarkEventV1, ...], + degraded: tuple[BenchmarkEventV1, ...], + *, + hard_violation: bool = False, + drop_candidate_anchors: bool = False, +) -> tuple[BenchmarkCandidateWindowV1, ...]: + overlay = tuple( + replace( + event, + ensemble_member_id="control", + benchmark_event_id="", + ) + for event in degraded + ) + windows = list( + build_benchmark_control_windows( + manifest, + scenario, + window, + degraded, + empirical_overlay_events=overlay, + ) + ) + candidate = next( + item + for item in manifest.candidates + if item.kind is BenchmarkCandidateKind.CANDIDATE + ) + for member, shift in (("member-a", 0.00001), ("member-b", -0.00001)): + generated = tuple( + replace( + item, + ensemble_member_id=member, + bid=item.bid + shift, + ask=item.ask + shift, + anchor_id=None if drop_candidate_anchors else item.anchor_id, + support_lower_mid=item.mid - 0.0002, + support_upper_mid=item.mid + 0.0002, + benchmark_event_id="", + ) + for item in reference + ) + windows.append( + BenchmarkCandidateWindowV1( + scenario_id=scenario.scenario_id, + candidate_id=candidate.candidate_id, + window_id=window.window_id, + ensemble_member_id=member, + events=generated, + execution=BenchmarkExecutionEvidenceV1( + attempted=True, + converged=True, + wall_time_ms=5, + peak_memory_bytes=1_024, + scratch_bytes=2_048, + durable_bytes=256, + ), + hard_constraint_violations=( + {"historical_anchor_violation": 1} if hard_violation else {} + ), + cross_series_hooks={"triangle_residual": 0.00001}, + strategy_hooks={"spread_cost_delta": 0.00002}, + ) + ) + return tuple(windows) + + +def _run_scorecard( + *, + hard_violation: bool = False, + protected_anchors: bool = False, + drop_candidate_anchors: bool = False, +) -> tuple[ + ReverseDegradationBenchmarkManifestV1, ReverseDegradationScorecardV1 +]: + manifest = _manifest() + engine = ReverseDegradationBenchmarkV1(manifest) + for scenario in manifest.scenarios: + window = _window(scenario) + reference = _events( + scenario, + window, + (0, 1, 2, 3, 4, 5), + anchor=protected_anchors, + ) + degraded = _events( + scenario, + window, + (0, 2, 5), + anchor=protected_anchors, + ) + engine.consume_window( + scenario_id=scenario.scenario_id, + window=window, + reference_events=reference, + degraded_events=degraded, + candidate_windows=_candidate_windows( + manifest, + scenario, + window, + reference, + degraded, + hard_violation=hard_violation, + drop_candidate_anchors=drop_candidate_anchors, + ), + ) + return manifest, engine.finalize() + + +def test_broker_conditioned_and_unconditioned_results_are_paired_without_winner() -> ( + None +): + """The same reverse-degradation scenarios compare both transfer modes.""" + manifest, scorecard = _run_scorecard() + conditioned = next( + item + for item in manifest.candidates + if item.kind is BenchmarkCandidateKind.CANDIDATE + ) + unconditioned = next( + item + for item in manifest.candidates + if item.control_kind is BenchmarkControlKind.NO_FILL + ) + + comparison = compare_broker_benchmark_results( + scorecard, + conditioned_candidate_id=conditioned.candidate_id, + unconditioned_candidate_id=unconditioned.candidate_id, + ) + + assert comparison.scenario_ids + assert comparison.to_dict()["automatic_winner"] is False + assert comparison.to_dict()["winner_candidate_id"] is None + assert all(comparison.aggregate_metric_deltas.values()) + assert ( + BrokerBenchmarkComparisonV1.from_json(comparison.to_json()) + == comparison + ) + + +def test_manifest_is_immutable_versioned_and_information_bound() -> None: + information = _information_manifest() + manifest = _manifest() + + validate_benchmark_information_boundary(manifest, information) + assert ( + ReverseDegradationBenchmarkManifestV1.from_json(manifest.to_json()) + == manifest + ) + assert tuple(item.kind for item in manifest.splits) == ( + BenchmarkSplitKind.CALIBRATION, + BenchmarkSplitKind.VALIDATION, + BenchmarkSplitKind.FINAL_HOLDOUT, + ) + assert len({item.epoch_id for item in manifest.scenarios}) == 2 + assert len({item.severity_id for item in manifest.scenarios}) == 2 + assert all( + candidate.generator_config_id != scenario.degradation_config_id + for candidate in manifest.candidates + for scenario in manifest.scenarios + ) + with pytest.raises(ValueError, match="calibration differs"): + validate_benchmark_information_boundary( + replace( + manifest, + splits=( + BenchmarkSplitV1( + BenchmarkSplitKind.CALIBRATION, + BASE_NS + 1_001, + BASE_NS + 2_000, + ), + *manifest.splits[1:], + ), + manifest_id="", + ), + information, + ) + + +def test_manifest_requires_full_controls_multiple_cells_and_no_winner() -> None: + manifest = _manifest() + controls = tuple( + item for item in manifest.candidates if item.control_kind is not None + ) + + with pytest.raises(ValueError, match="control set differs"): + replace(manifest, candidates=controls[:-1], manifest_id="") + duplicate_no_fill = _candidate( + BenchmarkCandidateKind.CONTROL, + "second-no-fill", + control=BenchmarkControlKind.NO_FILL, + ) + with pytest.raises(ValueError, match="exactly one"): + replace( + manifest, + candidates=manifest.candidates + (duplicate_no_fill,), + manifest_id="", + ) + with pytest.raises(ValueError, match="multiple feed epochs"): + replace( + manifest, + scenarios=tuple( + item + for item in manifest.scenarios + if item.epoch_id == "epoch-modern" + ), + manifest_id="", + ) + with pytest.raises(ValueError, match="automatic winner"): + replace(manifest, automatic_winner=True, manifest_id="") + + +def test_controls_are_transparent_bounded_and_overlay_is_same_cardinality() -> ( + None +): + manifest = _manifest() + scenario = manifest.scenarios[0] + window = _window(scenario) + degraded = _events(scenario, window, (0, 2, 5)) + by_control = { + item.control_kind: item + for item in manifest.candidates + if item.control_kind is not None + } + + no_fill = build_benchmark_control_events( + by_control[BenchmarkControlKind.NO_FILL], + degraded, + ensemble_member_id="control", + ) + interpolated = build_benchmark_control_events( + by_control[BenchmarkControlKind.LINEAR_INTERPOLATION], + degraded, + ensemble_member_id="control", + ) + resampled = build_benchmark_control_events( + by_control[BenchmarkControlKind.RESAMPLE_LAST], + degraded, + ensemble_member_id="control", + ) + + assert len(no_fill) == len(degraded) + assert len(interpolated) > len(degraded) + assert len(resampled) <= len(degraded) + with pytest.raises(ValueError, match="preserve degraded-row cardinality"): + build_benchmark_control_events( + by_control[BenchmarkControlKind.EMPIRICAL_OVERLAY], + degraded, + ensemble_member_id="control", + empirical_overlay_events=degraded[:-1], + ) + + +def test_empirical_overlay_adapter_consumes_augmented_columns() -> None: + frame = pl.DataFrame( + { + "timestamp_utc_ms": [1_700_000_000_000, 1_700_000_000_100], + "synth_bid": [1.1, 1.1001], + "synth_ask": [1.1002, 1.1003], + } + ) + + events = benchmark_events_from_empirical_overlay( + frame, + symbol="eurusd", + epoch_id="epoch-modern", + session="london", + event_state="normal", + ) + + assert len(events) == frame.height + assert events[0].event_time_ns == 1_700_000_000_000_000_000 + assert events[0].symbol == "EURUSD" + assert events[0].sparsity == "empirical_overlay" + + +class _FixtureGenerator: + """Deterministic generator implementing the public benchmark protocol.""" + + event_schema_version = BENCHMARK_EVENT_SCHEMA_VERSION + + def __init__(self, candidate_id: str) -> None: + self.candidate_id = candidate_id + + def generate( + self, + degraded_events: tuple[BenchmarkEventV1, ...], + *, + scenario: BenchmarkScenarioV1, + window: ReconstructionWindowV1, + ensemble_member_id: str, + ) -> tuple[BenchmarkEventV1, ...]: + del scenario, window + return tuple( + replace( + item, + ensemble_member_id=ensemble_member_id, + benchmark_event_id="", + ) + for item in degraded_events + ) + + +def test_generator_uses_shared_interface_and_independent_identity() -> None: + manifest = _manifest() + scenario = manifest.scenarios[0] + window = _window(scenario) + degraded = _events(scenario, window, (0, 2, 5)) + candidate = next( + item + for item in manifest.candidates + if item.kind is BenchmarkCandidateKind.CANDIDATE + ) + + generated = generate_benchmark_candidate_window( + _FixtureGenerator(candidate.candidate_id), + candidate, + degraded, + scenario=scenario, + window=window, + ensemble_member_id="member-a", + execution=BenchmarkExecutionEvidenceV1(attempted=True, converged=True), + ) + + assert len(generated.events) == len(degraded) + assert generated.metadata()["events_inline"] is False + assert "events" not in generated.metadata() + assert all( + item.schema_version == BENCHMARK_EVENT_SCHEMA_VERSION + for item in generated.events + ) + + +class _ObservationOperatorStub: + """Small valid observation result producer for adapter integration.""" + + operator_id = ( + "observation-operator:sha256:" + + hashlib.sha256(b"benchmark-operator").hexdigest() + ) + stratum_id = ( + "observation-stratum:sha256:" + + hashlib.sha256(b"benchmark-stratum").hexdigest() + ) + + def degrade( + self, + events: list[ObservationInputEventV1], + *, + window: ReconstructionWindowV1, + carry: ObservationCarryStateV1 | None, + protected_event_ids: tuple[str, ...], + source_start: bool, + ) -> ObservationApplicationResultV1: + del carry, source_start + outputs = tuple( + ObservationOutputEventV1( + source_event_id=event.source_event_id, + operator_id=self.operator_id, + stratum_id=self.stratum_id, + symbol=event.symbol, + source_time_ns=event.event_time_ns, + observed_time_ns=event.event_time_ns, + observed_sequence=0, + bid=event.bid, + ask=event.ask, + duplicate_ordinal=0, + transformations=(), + protected_anchor=event.source_event_id in protected_event_ids, + ) + for event in events + ) + last = outputs[-1] + return ObservationApplicationResultV1( + operator_id=self.operator_id, + window_id=window.window_id, + symbol=last.symbol, + application_mode="degrade", + input_count=len(events), + output_events=outputs, + reason_counts={"retained": len(events)}, + fallback_counts={"global": len(events)}, + diagnostic_samples=(), + samples_truncated=False, + carry_state=ObservationCarryStateV1( + operator_id=self.operator_id, + symbol=last.symbol, + last_source_time_ns=last.source_time_ns, + last_observed_time_ns=last.observed_time_ns, + last_bid=last.bid, + last_ask=last.ask, + ), + ) + + +def test_degradation_adapter_binds_operator_and_preserves_protected_anchor() -> ( + None +): + manifest = _manifest() + original = manifest.scenarios[0] + operator = _ObservationOperatorStub() + scenario = replace( + original, + observation_operator_id=operator.operator_id, + scenario_id="", + degradation_config_id="", + ) + window = _window(scenario) + reference = _events(scenario, window, (0,), anchor=True) + + degraded, result = degrade_benchmark_window( + operator, # type: ignore[arg-type] + reference, + scenario=scenario, + window=window, + protected_event_ids=(reference[0].source_event_id,), + source_start=True, + ) + + assert result.application_mode == "degrade" + assert degraded[0].anchor_id == reference[0].anchor_id + assert degraded[0].schema_version == BENCHMARK_EVENT_SCHEMA_VERSION + with pytest.raises(ValueError, match="operator identity differs"): + degrade_benchmark_window( + operator, # type: ignore[arg-type] + reference, + scenario=original, + window=window, + source_start=True, + ) + + +def test_scorecard_is_stratified_reproducible_and_never_selects_winner() -> ( + None +): + manifest, scorecard = _run_scorecard() + _, repeated = _run_scorecard() + + assert scorecard.scorecard_id == repeated.scorecard_id + assert ( + ReverseDegradationScorecardV1.from_json(scorecard.to_json()) + == scorecard + ) + assert scorecard.automatic_winner is False + assert scorecard.to_dict()["winner_candidate_id"] is None + assert len(scorecard.candidate_scores) == ( + len(manifest.scenarios) * len(manifest.candidates) + ) + candidate_scores = [ + item + for item in scorecard.candidate_scores + if manifest.candidate_by_id(item.candidate_id).kind + is BenchmarkCandidateKind.CANDIDATE + ] + assert all(item.promotion_eligible for item in candidate_scores) + assert all(len(item.slice_scores) >= 2 for item in candidate_scores) + assert all( + item.uncertainty_metrics["support_interval_count"] > 0 + for item in candidate_scores + ) + assert all( + item.uncertainty_metrics["ensemble_common_event_comparison_count"] > 0 + for item in candidate_scores + ) + assert all(item.cross_series_hooks for item in candidate_scores) + assert all(item.strategy_hooks for item in candidate_scores) + assert all( + item.execution_summary["attempted_count"] == 2 + and item.execution_summary["converged_count"] == 2 + and item.execution_summary["failure_count"] == 0 + and item.execution_summary["peak_memory_bytes"] == 1_024 + and item.execution_summary["scratch_bytes"] == 4_096 + and item.execution_summary["durable_bytes"] == 512 + for item in candidate_scores + ) + assert all( + "worst_slice_soft_loss" in item.aggregate_metrics + and "mean_soft_loss_delta" in item.relative_to_no_fill + for item in scorecard.candidate_scores + ) + metric_names = set(candidate_scores[0].slice_scores[0].metrics) + assert { + "event_count_relative_error", + "interarrival_hist_l1", + "burst_duration_relative_error", + "quiet_duration_relative_error", + "bid_mean_relative_error", + "ask_mean_relative_error", + "spread_hist_l1", + "bid_transition_relative_error", + "ask_transition_relative_error", + "mid_transition_relative_error", + "spread_transition_relative_error", + "mid_range_relative_error", + "endpoint_relative_error", + } <= metric_names + + +def test_hard_constraint_violation_blocks_soft_fit_promotion() -> None: + manifest, scorecard = _run_scorecard(hard_violation=True) + candidate_scores = [ + item + for item in scorecard.candidate_scores + if manifest.candidate_by_id(item.candidate_id).kind + is BenchmarkCandidateKind.CANDIDATE + ] + + assert all(not item.promotion_eligible for item in candidate_scores) + assert all( + item.hard_constraint_violations["historical_anchor_violation"] == 2 + for item in candidate_scores + ) + assert all( + item.aggregate_metrics["mean_soft_loss"] < 1 + for item in candidate_scores + ) + + +def test_missing_protected_anchor_is_an_automatic_hard_gate() -> None: + manifest, scorecard = _run_scorecard( + protected_anchors=True, + drop_candidate_anchors=True, + ) + candidate_scores = [ + item + for item in scorecard.candidate_scores + if manifest.candidate_by_id(item.candidate_id).kind + is BenchmarkCandidateKind.CANDIDATE + ] + + assert all(not item.promotion_eligible for item in candidate_scores) + assert all( + item.hard_constraint_violations["historical_anchor_missing"] > 0 + for item in candidate_scores + ) + with pytest.raises(ValueError, match="always block promotion"): + replace( + candidate_scores[0], + promotion_eligible=True, + candidate_score_id="", + ) + + +def test_finalize_rejects_partial_immutable_split_coverage() -> None: + manifest = _manifest() + engine = ReverseDegradationBenchmarkV1(manifest) + for index, scenario in enumerate(manifest.scenarios): + full_window = _window(scenario) + window = ( + replace( + full_window, + core_end_ns=full_window.core_start_ns + 1_000, + window_id="", + synchronization_unit_id="", + ) + if index == 0 + else full_window + ) + reference = _events(scenario, window, (0, 1, 2)) + degraded = _events(scenario, window, (0, 2)) + engine.consume_window( + scenario_id=scenario.scenario_id, + window=window, + reference_events=reference, + degraded_events=degraded, + candidate_windows=_candidate_windows( + manifest, scenario, window, reference, degraded + ), + ) + + with pytest.raises(ValueError, match="complete split"): + engine.finalize() + + +def test_engine_fails_closed_on_missing_members_bounds_and_reuse() -> None: + manifest = _manifest() + scenario = manifest.scenarios[0] + window = _window(scenario) + reference = _events(scenario, window, (0, 1, 2)) + degraded = _events(scenario, window, (0, 2)) + windows = _candidate_windows( + manifest, scenario, window, reference, degraded + ) + engine = ReverseDegradationBenchmarkV1(manifest) + + with pytest.raises(ValueError, match="cover configured ensemble members"): + engine.consume_window( + scenario_id=scenario.scenario_id, + window=window, + reference_events=reference, + degraded_events=degraded, + candidate_windows=windows[:-1], + ) + outside = replace( + reference[0], + event_time_ns=window.core_end_ns, + benchmark_event_id="", + ) + with pytest.raises(ValueError, match="outside window ownership"): + engine.consume_window( + scenario_id=scenario.scenario_id, + window=window, + reference_events=(outside,), + degraded_events=degraded, + candidate_windows=windows, + ) + with pytest.raises(ValueError, match="unprocessed scenario cells"): + engine.finalize() + + +def test_profile_bounds_refuse_unbounded_online_state() -> None: + profile = replace(BenchmarkProfileV1(), max_slices=1, profile_id="") + manifest = _manifest(profile=profile) + scenario = manifest.scenarios[0] + window = _window(scenario) + reference = _events(scenario, window, (0, 1, 2)) + degraded = _events(scenario, window, (0, 2)) + + with pytest.raises(ValueError, match="slice limit"): + ReverseDegradationBenchmarkV1(manifest).consume_window( + scenario_id=scenario.scenario_id, + window=window, + reference_events=reference, + degraded_events=degraded, + candidate_windows=_candidate_windows( + manifest, scenario, window, reference, degraded + ), + ) diff --git a/tests/unit/test_synthetic_broker_transfer.py b/tests/unit/test_synthetic_broker_transfer.py new file mode 100644 index 00000000..e689f7dd --- /dev/null +++ b/tests/unit/test_synthetic_broker_transfer.py @@ -0,0 +1,337 @@ +"""End-to-end broker rendering over reconciled synthetic event groups.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +from histdatacom.broker_capture import fit_broker_delivery_fingerprint +from histdatacom.synthetic import ( + BrokerRenderedGroupV1, + BrokerTransferConfigV1, + BrokerTransferStatus, + HistoricalCarvingConstraintSetV1, + SyntheticEventOrigin, + SyntheticEventStreamV1, + eurusd_triangle_reconciliation_config, + reconcile_cross_currency_window, + render_broker_delivery, +) +from tests.unit.test_broker_delivery_fingerprints import ( + BASE_WALL_NS, + _capture, +) +from tests.unit.test_synthetic_cross_currency import ( + START_NS, + _condition, + _reconciled_group, + _run, + _stream, + _window, +) + + +def test_render_is_deterministic_preserves_anchors_and_validates_final_group( + tmp_path: Path, +) -> None: + """Rendering binds profile lineage and emits only fully validated output.""" + run, window, group, constraints = _group_with_constraints() + manifest = _capture(tmp_path, seed=21, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + config = BrokerTransferConfigV1( + strength=0.25, + max_events_per_group=100, + ) + + rendered = render_broker_delivery( + run=run, + window=window, + group=group, + fingerprint=fingerprint, + constraints=constraints, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=config, + quality_period="202001", + ) + retry = render_broker_delivery( + run=run, + window=window, + group=group, + fingerprint=fingerprint, + constraints=constraints, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=config, + quality_period="202001", + ) + + assert rendered == retry + assert rendered.status is BrokerTransferStatus.APPLIED + assert rendered.post_broker_validation is not None + assert rendered.post_broker_validation.passed + assert rendered.manifest.local_validation_passed + assert rendered.manifest.cross_instrument_quality_status == "clean" + assert rendered.manifest.fingerprint_id == fingerprint.fingerprint_id + assert rendered.manifest.selections[0].profile_effective_start_utc_ns == ( + fingerprint.effective_start_utc_ns + ) + assert rendered.manifest.to_dict()[ + "profile_effective_periods_embedded_in_selections" + ] + assert rendered.manifest.synthetic_event_count == len( + rendered.event_lineage + ) + assert rendered.manifest.action_counts + before_anchors = _observed_payloads(group.streams) + after_anchors = _observed_payloads(rendered.streams) + assert after_anchors == before_anchors + assert { + event.broker_profile_id + for stream in rendered.streams + for event in stream.events + if event.origin is SyntheticEventOrigin.SYNTHETIC + } == {fingerprint.fingerprint_id} + assert BrokerRenderedGroupV1.from_json(rendered.to_json()) == rendered + + +def test_unsupported_profile_cell_refuses_without_exposing_partial_rows( + tmp_path: Path, +) -> None: + """A missing symbol cell never silently falls back to global support.""" + run, window, group, constraints = _group_with_constraints() + manifest = _capture(tmp_path, seed=22, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + + refused = render_broker_delivery( + run=run, + window=window, + group=group, + fingerprint=fingerprint, + constraints=constraints, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + requested_conditions={ + "EURGBP": {"symbol": "EURGBP"}, + "EURUSD": {"symbol": "EURUSD"}, + "GBPUSD": {"symbol": "GBPUSD"}, + }, + config=BrokerTransferConfigV1(max_events_per_group=100), + ) + + assert refused.status is BrokerTransferStatus.REFUSED + assert refused.streams == () + assert refused.event_lineage == () + assert any( + "requested_condition_absent" in reason + for reason in refused.manifest.reason_codes + ) + assert BrokerRenderedGroupV1.from_json(refused.to_json()) == refused + + +def test_render_resource_limit_refuses_before_materializing_output( + tmp_path: Path, +) -> None: + """Event amplification and in-flight rows stay bounded by config.""" + run, window, group, constraints = _group_with_constraints() + manifest = _capture(tmp_path, seed=23, wall_start_ns=BASE_WALL_NS) + fingerprint = fit_broker_delivery_fingerprint(tmp_path, (manifest,)) + + refused = render_broker_delivery( + run=run, + window=window, + group=group, + fingerprint=fingerprint, + constraints=constraints, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=BrokerTransferConfigV1(max_events_per_group=1), + ) + + assert refused.status is BrokerTransferStatus.REFUSED + assert refused.manifest.reason_codes == ("max_events_per_group_exceeded",) + assert refused.manifest.output_content_sha256 is None + + +def test_render_applies_measured_batching_to_dense_synthetic_rows( + tmp_path: Path, +) -> None: + """Batch presentation is exercised independently of stale behavior.""" + run, window, group, constraints = _group_with_constraints(dense=True) + manifest = _capture(tmp_path, seed=24, wall_start_ns=BASE_WALL_NS) + fingerprint = _profile_with_metrics( + fit_broker_delivery_fingerprint(tmp_path, (manifest,)), + source_batch_quote_count=3.0, + ) + + rendered = render_broker_delivery( + run=run, + window=window, + group=group, + fingerprint=fingerprint, + constraints=constraints, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=BrokerTransferConfigV1( + strength=1.0, + apply_stale_behavior=False, + apply_exact_duplicates=False, + max_events_per_group=100, + ), + quality_period="202001", + ) + + assert rendered.status is BrokerTransferStatus.APPLIED + assert rendered.manifest.action_counts["batched_timestamp"] > 0 + + +def test_render_applies_measured_stale_quotes_independently( + tmp_path: Path, +) -> None: + """A fully supported stale rate repeats prior synthetic presentation.""" + run, window, group, constraints = _group_with_constraints(dense=True) + manifest = _capture(tmp_path, seed=25, wall_start_ns=BASE_WALL_NS) + fingerprint = _profile_with_metrics( + fit_broker_delivery_fingerprint(tmp_path, (manifest,)), + stale_quote_rate=1.0, + exact_duplicate_rate=0.0, + ) + + rendered = render_broker_delivery( + run=run, + window=window, + group=group, + fingerprint=fingerprint, + constraints=constraints, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=BrokerTransferConfigV1( + strength=1.0, + apply_exact_duplicates=False, + apply_batching=False, + max_events_per_group=100, + ), + quality_period="202001", + ) + + assert rendered.status is BrokerTransferStatus.APPLIED + assert rendered.manifest.action_counts["stale_quote"] > 0 + assert "exact_duplicate_quote" not in rendered.manifest.action_counts + + +def test_render_applies_measured_exact_duplicates_independently( + tmp_path: Path, +) -> None: + """Exact-duplicate presentation remains distinct from stale behavior.""" + run, window, group, constraints = _group_with_constraints(dense=True) + manifest = _capture(tmp_path, seed=26, wall_start_ns=BASE_WALL_NS) + fingerprint = _profile_with_metrics( + fit_broker_delivery_fingerprint(tmp_path, (manifest,)), + stale_quote_rate=0.0, + exact_duplicate_rate=1.0, + ) + + rendered = render_broker_delivery( + run=run, + window=window, + group=group, + fingerprint=fingerprint, + constraints=constraints, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=BrokerTransferConfigV1( + strength=1.0, + apply_stale_behavior=False, + apply_batching=False, + max_events_per_group=100, + ), + quality_period="202001", + ) + + assert rendered.status is BrokerTransferStatus.APPLIED + assert rendered.manifest.action_counts["exact_duplicate_quote"] > 0 + assert "stale_quote" not in rendered.manifest.action_counts + + +def _group_with_constraints(*, dense: bool = False): + if dense: + config = eurusd_triangle_reconciliation_config() + run = _run(config) + window = _window(run) + inputs = { + symbol: _stream( + run, + symbol=symbol, + anchor_midpoint=midpoint, + synthetic_midpoint=midpoint, + extra_synthetic=( + (START_NS + 250_000_000, 2, midpoint), + (START_NS + 750_000_000, 3, midpoint), + ), + ) + for symbol, midpoint in { + "EURGBP": 0.8, + "EURUSD": 1.2, + "GBPUSD": 1.5, + }.items() + } + else: + run, window, inputs, initial = _reconciled_group() + config = initial.config + constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture" + ) + streams: dict[str, SyntheticEventStreamV1] = {} + for symbol, stream in inputs.items(): + events = tuple( + ( + replace( + event, + constraint_set_id=constraints.constraint_set_id, + event_id="", + ) + if event.origin is SyntheticEventOrigin.SYNTHETIC + else event + ) + for event in stream.events + ) + streams[symbol] = SyntheticEventStreamV1( + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + symbol=stream.symbol, + events=events, + ) + group = reconcile_cross_currency_window( + run=run, + window=window, + streams=streams, + config=config, + conditions=(_condition(),), + ) + return run, window, group, constraints + + +def _profile_with_metrics(fingerprint, **values: float): + cells = [] + for cell in fingerprint.cells: + if cell.condition.key != "global": + cells.append(cell) + continue + metrics = tuple( + ( + replace( + metric, + estimate=values[metric.name], + lower=values[metric.name], + upper=values[metric.name], + metric_id="", + ) + if metric.name in values + else metric + ) + for metric in cell.metrics + ) + cells.append(replace(cell, metrics=metrics, cell_id="")) + return replace(fingerprint, cells=tuple(cells), fingerprint_id="") + + +def _observed_payloads(streams): + return { + event.event_id: event.to_dict() + for stream in streams + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + } diff --git a/tests/unit/test_synthetic_certification.py b/tests/unit/test_synthetic_certification.py new file mode 100644 index 00000000..8b8b9aeb --- /dev/null +++ b/tests/unit/test_synthetic_certification.py @@ -0,0 +1,475 @@ +"""Regression tests for release-grade reconstruction certification.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +from typing import cast + +import pytest + +from histdatacom.synthetic import ( + CERTIFICATION_ARTIFACT_SCHEMA_VERSION, + EURUSD_TRIANGLE_COMMON_START_PERIOD, + PROMOTION_ONLY_CHECK_IDS, + CertificationArtifactV1, + CertificationCheckStatus, + CertificationComparator, + CertificationGate, + CertificationObservationV1, + CertificationRequirementV1, + CertificationState, + ReconstructionCertificationDossierV1, + ReconstructionCertificationPolicyV1, + eurusd_triangle_certification_policy, + evaluate_reconstruction_certification, + load_reconstruction_certification_dossier, + write_reconstruction_certification_dossier, +) +from histdatacom.synthetic.certification import EURUSD_TRIANGLE_SYMBOLS + +BROKER_ID = "broker-delivery-fingerprint:sha256:qualified-fixture" +METHODOLOGY = ( + "The fixture binds immutable source identities, final holdout scorecards, " + "post-render cross-currency validation, restart/replay evidence, resource " + "measurements, negative tests, and downstream strategy sensitivity to one " + "predeclared policy. Event rows remain outside the dossier." +) + + +def _policy() -> ReconstructionCertificationPolicyV1: + return eurusd_triangle_certification_policy( + broker_fingerprint_id=BROKER_ID, + common_end_period="202606", + peak_memory_budget_bytes=4_000_000_000, + scratch_budget_bytes=80_000_000_000, + runtime_budget_seconds=86_400.0, + storage_budget_bytes=80_000_000_000, + ) + + +def _artifact( + kind: str, + *, + policy: ReconstructionCertificationPolicyV1 | None = None, + verified: bool = True, +) -> CertificationArtifactV1: + selected_policy = policy or _policy() + subject_id = ( + BROKER_ID + if kind == "broker-delivery-fingerprint" + else f"{kind}:fixture" + ) + return CertificationArtifactV1.from_payload( + policy_id=selected_policy.policy_id, + kind=kind, + subject_id=subject_id, + subject_schema_version=f"histdatacom.{kind}.v1", + payload={ + "kind": kind, + "subject_id": subject_id, + "fixture": True, + }, + relative_path=f"certification/{kind}.json", + verified=verified, + metadata={"event_rows_inline": False}, + ) + + +def _artifacts( + policy: ReconstructionCertificationPolicyV1, +) -> tuple[CertificationArtifactV1, ...]: + kinds = sorted( + { + kind + for requirement in policy.requirements + for kind in requirement.required_artifact_kinds + } + ) + return tuple(_artifact(kind, policy=policy) for kind in kinds) + + +def _passing_value( + requirement: CertificationRequirementV1, +) -> bool | int | float | str | None: + if requirement.comparator is CertificationComparator.LESS_OR_EQUAL: + assert isinstance(requirement.expected, (int, float)) + return cast(int | float, requirement.expected) / 2 + if requirement.comparator is CertificationComparator.GREATER_OR_EQUAL: + return requirement.expected + return cast(bool | int | float | str | None, requirement.expected) + + +def _observations( + policy: ReconstructionCertificationPolicyV1, + artifacts: tuple[CertificationArtifactV1, ...], +) -> tuple[CertificationObservationV1, ...]: + by_kind = {item.kind: item for item in artifacts} + return tuple( + CertificationObservationV1( + check_id=requirement.check_id, + actual=_passing_value(requirement), + artifact_evidence_ids=tuple( + by_kind[kind].evidence_id + for kind in requirement.required_artifact_kinds + ), + note="measured by deterministic certification fixture", + ) + for requirement in policy.requirements + ) + + +def _dossier( + *, + omit_checks: frozenset[str] = frozenset(), + actual_overrides: dict[str, bool | int | float | str | None] | None = None, + artifacts: tuple[CertificationArtifactV1, ...] | None = None, + blocking_limitations: tuple[str, ...] = (), +) -> ReconstructionCertificationDossierV1: + policy = _policy() + selected_artifacts = artifacts or _artifacts(policy) + observations = [] + overrides = actual_overrides or {} + for item in _observations(policy, selected_artifacts): + if item.check_id in omit_checks: + continue + observations.append( + replace( + item, + actual=overrides.get(item.check_id, item.actual), + observation_id="", + ) + ) + return evaluate_reconstruction_certification( + policy, + artifacts=selected_artifacts, + observations=observations, + methodology=METHODOLOGY, + accepted_limitations=( + "The deterministic fixture demonstrates contract semantics, not historical truth.", + ), + blocking_limitations=blocking_limitations, + ) + + +def test_default_policy_predeclares_every_issue_gate_and_round_trips() -> None: + """The policy is complete, deterministic, and tied to the selected broker.""" + policy = _policy() + + assert set(item.gate for item in policy.requirements) == set( + CertificationGate + ) + assert policy.symbols == tuple(sorted(EURUSD_TRIANGLE_SYMBOLS)) + assert policy.common_start_period == EURUSD_TRIANGLE_COMMON_START_PERIOD + assert policy.broker_fingerprint_id == BROKER_ID + assert len({item.check_id for item in policy.requirements}) == len( + policy.requirements + ) + assert ( + ReconstructionCertificationPolicyV1.from_dict(policy.to_dict()) + == policy + ) + + +def test_complete_content_bound_evidence_certifies_without_product_claims() -> ( + None +): + """All measured gates certify while retaining only bounded metadata.""" + dossier = _dossier() + payload = dossier.to_dict() + + assert dossier.state is CertificationState.CERTIFIED + assert dossier.certified + assert payload["release_authorized"] is True + assert payload["event_rows_inline"] is False + assert payload["analytical_frame_columns_inline"] is False + assert payload["automatic_winner"] is False + assert payload["historical_truth_claim"] is False + assert payload["investment_recommendation"] is False + assert dossier.summary["passed_gate_count"] == 15 + assert ( + ReconstructionCertificationDossierV1.from_json(dossier.to_json()) + == dossier + ) + + +def test_only_promotion_coverage_missing_is_ready_for_promotion() -> None: + """Coverage stays deferred until dev-to-main without hiding other gates.""" + dossier = _dossier(omit_checks=PROMOTION_ONLY_CHECK_IDS) + + assert dossier.state is CertificationState.READY_FOR_PROMOTION + assert dossier.ready_for_promotion + assert not dossier.certified + missing = { + result.check_id + for gate in dossier.gate_results + for result in gate.check_results + if result.status is CertificationCheckStatus.MISSING + } + assert missing == PROMOTION_ONLY_CHECK_IDS + + +def test_missing_broker_artifact_remains_incomplete() -> None: + """A caller cannot replace a qualified broker fingerprint with a boolean.""" + policy = _policy() + artifacts = tuple( + item + for item in _artifacts(policy) + if item.kind != "broker-delivery-fingerprint" + ) + by_kind = {item.kind: item for item in artifacts} + observations = [] + for requirement in policy.requirements: + available = tuple( + by_kind[kind].evidence_id + for kind in requirement.required_artifact_kinds + if kind in by_kind + ) + if not available: + available = (artifacts[0].evidence_id,) + observations.append( + CertificationObservationV1( + check_id=requirement.check_id, + actual=_passing_value(requirement), + artifact_evidence_ids=available, + ) + ) + + dossier = evaluate_reconstruction_certification( + policy, + artifacts=artifacts, + observations=observations, + methodology=METHODOLOGY, + blocking_limitations=( + "No qualified live broker fingerprint is available.", + ), + ) + + assert dossier.state is CertificationState.INCOMPLETE + cross = next( + item + for item in dossier.gate_results + if item.gate is CertificationGate.CROSS_CURRENCY + ) + assert cross.status is CertificationCheckStatus.MISSING + + +def test_wrong_broker_identity_fails_certification() -> None: + """Verified content from another broker profile cannot satisfy the policy.""" + policy = _policy() + artifacts = tuple( + ( + replace( + item, + subject_id="broker-delivery-fingerprint:sha256:other", + evidence_id="", + ) + if item.kind == "broker-delivery-fingerprint" + else item + ) + for item in _artifacts(policy) + ) + dossier = _dossier(artifacts=artifacts) + + assert dossier.state is CertificationState.FAILED + cross = next( + item + for item in dossier.gate_results + if item.gate is CertificationGate.CROSS_CURRENCY + ) + assert cross.status is CertificationCheckStatus.FAILED + assert "fingerprint differs" in cross.check_results[0].reason + + +@pytest.mark.parametrize( # type: ignore[untyped-decorator] + "check_id,actual", + ( + ("raw_source_hash_mismatch_count", 1), + ("reverse_holdout_failure_count", 1), + ("actual_peak_memory_bytes", 4_000_000_001), + ("corruption_refused", False), + ("strategy_automatic_winner", True), + ("local_simple_registry_preflight_passed", False), + ), +) +def test_measured_failures_fail_closed( + check_id: str, actual: bool | int +) -> None: + """Scientific, operational, resource, and release failures are terminal.""" + dossier = _dossier(actual_overrides={check_id: actual}) + + assert dossier.state is CertificationState.FAILED + result = next( + result + for gate in dossier.gate_results + for result in gate.check_results + if result.check_id == check_id + ) + assert result.status is CertificationCheckStatus.FAILED + + +def test_unverified_artifact_cannot_support_a_pass() -> None: + """Content identity without verification is missing evidence, not a pass.""" + policy = _policy() + artifacts = tuple( + ( + _artifact(item.kind, policy=policy, verified=False) + if item.kind == "information-audit-report" + else item + ) + for item in _artifacts(policy) + ) + dossier = _dossier(artifacts=artifacts) + + assert dossier.state is CertificationState.INCOMPLETE + information = next( + item + for item in dossier.gate_results + if item.gate is CertificationGate.INFORMATION_SAFETY + ) + assert information.status is CertificationCheckStatus.MISSING + + +def test_blocking_limitation_prevents_certification() -> None: + """A contradictory limitation cannot be accepted away.""" + dossier = _dossier( + blocking_limitations=( + "Final historical reconstruction has not been executed.", + ) + ) + + assert dossier.state is CertificationState.INCOMPLETE + assert dossier.summary["blocking_limitation_count"] == 1 + assert not dossier.to_dict()["release_authorized"] + + +def test_measured_failure_outranks_a_blocking_limitation() -> None: + """Known threshold violations remain failures even when work is missing.""" + dossier = _dossier( + actual_overrides={"reverse_holdout_failure_count": 1}, + blocking_limitations=("A separate required artifact is unavailable.",), + ) + + assert dossier.state is CertificationState.FAILED + + +def test_artifacts_are_bound_to_the_predeclared_policy() -> None: + """Evidence produced under different thresholds cannot be repurposed.""" + policy = _policy() + other_policy = eurusd_triangle_certification_policy( + broker_fingerprint_id=BROKER_ID, + common_end_period="202606", + peak_memory_budget_bytes=4_000_000_001, + scratch_budget_bytes=80_000_000_000, + runtime_budget_seconds=86_400.0, + storage_budget_bytes=80_000_000_000, + ) + artifacts = _artifacts(other_policy) + + with pytest.raises(ValueError, match="differ from policy"): + evaluate_reconstruction_certification( + policy, + artifacts=artifacts, + observations=_observations(policy, artifacts), + methodology=METHODOLOGY, + ) + + +def test_dossier_publication_is_atomic_replayable_and_human_readable( + tmp_path: Path, +) -> None: + """Machine and human reports publish with strong content references.""" + dossier = _dossier() + json_path = tmp_path / "evidence" / "certification.json" + markdown_path = tmp_path / "evidence" / "certification.md" + + json_ref, markdown_ref = write_reconstruction_certification_dossier( + dossier, + json_path=json_path, + markdown_path=markdown_path, + ) + + assert load_reconstruction_certification_dossier(json_path) == dossier + assert json_ref.sha256 + assert markdown_ref.sha256 + assert json_ref.size_bytes == json_path.stat().st_size + markdown = markdown_path.read_text(encoding="utf-8") + assert "## Gate results" in markdown + assert "## Methodology" in markdown + assert "## Accepted limitations" in markdown + assert "## Blocking limitations" in markdown + assert "event rows" in markdown.lower() + + +def test_tampering_and_scope_drift_are_rejected() -> None: + """Deterministic identities and the three-symbol coverage scope fail closed.""" + dossier = _dossier() + payload = dossier.to_dict() + payload["dossier_id"] = ( + "reconstruction-certification-dossier:sha256:tampered" + ) + with pytest.raises(ValueError, match="dossier_id differs"): + ReconstructionCertificationDossierV1.from_dict(payload) + + policy = _policy() + with pytest.raises(ValueError, match="EURUSD triangle"): + replace(policy, symbols=("EURUSD",), policy_id="") + with pytest.raises(ValueError, match="common coverage"): + replace(policy, common_start_period="200005", policy_id="") + + +def test_unknown_or_duplicate_observations_are_rejected() -> None: + """The evaluator consumes exactly the checks declared by policy.""" + policy = _policy() + artifacts = _artifacts(policy) + observations = _observations(policy, artifacts) + unknown = CertificationObservationV1( + check_id="undeclared-check", + actual=True, + artifact_evidence_ids=(artifacts[0].evidence_id,), + ) + with pytest.raises(ValueError, match="outside policy"): + evaluate_reconstruction_certification( + policy, + artifacts=artifacts, + observations=(*observations, unknown), + methodology=METHODOLOGY, + ) + with pytest.raises(ValueError, match="duplicate check"): + evaluate_reconstruction_certification( + policy, + artifacts=artifacts, + observations=(*observations, observations[0]), + methodology=METHODOLOGY, + ) + + +def test_requirement_and_artifact_contract_bounds() -> None: + """Paths, hashes, scalar comparisons, and policy coverage remain bounded.""" + with pytest.raises(ValueError, match="relative and safe"): + CertificationArtifactV1( + policy_id=_policy().policy_id, + kind="report", + subject_id="report:one", + subject_schema_version="report.v1", + content_sha256="a" * 64, + relative_path="../secret.json", + size_bytes=1, + verified=True, + metadata={}, + ) + with pytest.raises(ValueError, match="SHA-256"): + replace(_artifact("report"), content_sha256="bad", evidence_id="") + with pytest.raises(ValueError, match="true comparator"): + CertificationRequirementV1( + gate=CertificationGate.REPOSITORY_GATES, + check_id="invalid-true", + comparator=CertificationComparator.TRUE, + expected=False, + required_artifact_kinds=("report",), + description="invalid comparator fixture", + ) + + artifact = _artifact("report") + assert artifact.schema_version == CERTIFICATION_ARTIFACT_SCHEMA_VERSION + assert CertificationArtifactV1.from_dict(artifact.to_dict()) == artifact diff --git a/tests/unit/test_synthetic_certification_campaign.py b/tests/unit/test_synthetic_certification_campaign.py new file mode 100644 index 00000000..f87246b3 --- /dev/null +++ b/tests/unit/test_synthetic_certification_campaign.py @@ -0,0 +1,228 @@ +"""Executable modern-reference certification campaign tests.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import cast + +import pytest + +from histdatacom.runtime_contracts import JSONScalar, JSONValue +from histdatacom.synthetic import CertificationComparator, CertificationState +from histdatacom.synthetic.certification import ( + CertificationRequirementV1, + ReconstructionCertificationPolicyV2, + modern_reference_triangle_certification_policy, +) +from histdatacom.synthetic.certification_campaign import ( + METHODOLOGY_REPORT_EVIDENCE_KEY, + CertificationCampaignArtifactV1, + CertificationCampaignObservationV1, + ModernReferenceCertificationCampaignSpecV1, + read_modern_reference_certification_campaign_spec, + run_modern_reference_certification_campaign, +) +from histdatacom.synthetic.contracts import canonical_contract_json + + +def _policy() -> ReconstructionCertificationPolicyV2: + return modern_reference_triangle_certification_policy( + common_end_period="202606", + peak_memory_budget_bytes=4_000_000_000, + scratch_budget_bytes=80_000_000_000, + runtime_budget_seconds=86_400.0, + storage_budget_bytes=80_000_000_000, + candidate_amplification_budget=10.0, + ) + + +def _passing_value(requirement: CertificationRequirementV1) -> JSONScalar: + if requirement.comparator is CertificationComparator.LESS_OR_EQUAL: + return cast(float, requirement.expected) / 2 + if requirement.comparator is CertificationComparator.GREATER_OR_EQUAL: + return requirement.expected + return requirement.expected + + +def _write_json(path: Path, payload: dict[str, JSONValue]) -> str: + encoded = canonical_contract_json(payload).encode("utf-8") + b"\n" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(encoded) + return hashlib.sha256(encoded).hexdigest() + + +def _complete_spec( + tmp_path: Path, +) -> ModernReferenceCertificationCampaignSpecV1: + policy = _policy() + requirements = { + item.check_id: item + for item in policy.requirements + if item.check_id + not in { + "coverage_promotion_run_count", + "methodology_and_limitations_published", + "machine_evidence_manifest_published", + } + } + checks_by_measurement_kind: dict[str, dict[str, JSONScalar]] = {} + for requirement in requirements.values(): + kind = next( + item + for item in requirement.required_artifact_kinds + if item != "methodology-report" + ) + checks_by_measurement_kind.setdefault(kind, {})[ + requirement.check_id + ] = _passing_value(requirement) + kinds = sorted( + { + kind + for requirement in requirements.values() + for kind in requirement.required_artifact_kinds + if kind != "methodology-report" + } + ) + artifacts = [] + for kind in kinds: + key = kind.replace("-", "_") + schema = f"histdatacom.{kind}.test.v1" + report_id = f"{kind}:fixture" + payload: dict[str, JSONValue] = { + "schema_version": schema, + "report_id": report_id, + "measurements": checks_by_measurement_kind.get(kind, {}), + "event_rows_inline": False, + } + path = tmp_path / "reports" / f"{key}.json" + digest = _write_json(path, payload) + artifacts.append( + CertificationCampaignArtifactV1( + evidence_key=key, + kind=kind, + path=str(path.relative_to(tmp_path)), + content_sha256=digest, + subject_id=report_id, + subject_id_pointer="/report_id", + subject_schema_version=schema, + relative_path=f"reports/{key}.json", + metadata={"event_rows_inline": False}, + ) + ) + keys_by_kind = {item.kind: item.evidence_key for item in artifacts} + observations = [] + for requirement in requirements.values(): + measurement_kind = next( + item + for item in requirement.required_artifact_kinds + if item != "methodology-report" + ) + evidence_keys = tuple( + ( + METHODOLOGY_REPORT_EVIDENCE_KEY + if kind == "methodology-report" + else keys_by_kind[kind] + ) + for kind in requirement.required_artifact_kinds + ) + observations.append( + CertificationCampaignObservationV1( + check_id=requirement.check_id, + measurement_evidence_key=keys_by_kind[measurement_kind], + measurement_pointer=f"/measurements/{requirement.check_id}", + artifact_evidence_keys=evidence_keys, + note="measured by a content-bound campaign fixture", + ) + ) + return ModernReferenceCertificationCampaignSpecV1( + common_end_period="202606", + peak_memory_budget_bytes=4_000_000_000, + scratch_budget_bytes=80_000_000_000, + runtime_budget_seconds=86_400.0, + storage_budget_bytes=80_000_000_000, + candidate_amplification_budget=10.0, + artifacts=tuple(artifacts), + observations=tuple(observations), + methodology=( + "All observations are extracted from hash-verified JSON reports; " + "the fixture proves campaign mechanics rather than product quality." + ), + accepted_limitations=("Fixture evidence is not release evidence.",), + blocking_limitations=(), + ) + + +def _write_spec( + tmp_path: Path, spec: ModernReferenceCertificationCampaignSpecV1 +) -> Path: + path = tmp_path / "campaign.json" + path.write_text(spec.to_json() + "\n", encoding="utf-8") + return path + + +def test_campaign_extracts_every_value_and_reaches_ready_for_promotion( + tmp_path: Path, +) -> None: + """A campaign can pass only with exact hash-bound scalar extractions.""" + spec = _complete_spec(tmp_path) + spec_path = _write_spec(tmp_path, spec) + + dossier, result = run_modern_reference_certification_campaign( + spec_path, output_directory=tmp_path / "output" + ) + + assert dossier.state is CertificationState.READY_FOR_PROMOTION + assert result.state is CertificationState.READY_FOR_PROMOTION + assert result.verified_input_count == len(spec.artifacts) + assert result.observation_count == len(spec.observations) + 2 + assert read_modern_reference_certification_campaign_spec(spec_path) == spec + assert Path(result.dossier_json.path).is_file() + assert Path(result.dossier_markdown.path).is_file() + assert (tmp_path / "output" / "campaign-result.json").is_file() + + +def test_campaign_rejects_changed_evidence_bytes(tmp_path: Path) -> None: + """A report changed after campaign freeze cannot be consumed.""" + spec = _complete_spec(tmp_path) + spec_path = _write_spec(tmp_path, spec) + changed = tmp_path / spec.artifacts[0].path + changed.write_text("{}\n", encoding="utf-8") + + with pytest.raises(ValueError, match="hash differs"): + run_modern_reference_certification_campaign( + spec_path, output_directory=tmp_path / "output" + ) + + +def test_campaign_rejects_inline_values_and_premature_coverage( + tmp_path: Path, +) -> None: + """Ordinary dev campaigns cannot smuggle values or promotion coverage.""" + spec = _complete_spec(tmp_path) + payload = spec.to_dict() + payload["observation_values_inline"] = True + with pytest.raises(ValueError, match="inline observation"): + ModernReferenceCertificationCampaignSpecV1.from_dict(payload) + + first = spec.observations[0] + coverage = CertificationCampaignObservationV1( + check_id="coverage_promotion_run_count", + measurement_evidence_key=first.measurement_evidence_key, + measurement_pointer=first.measurement_pointer, + artifact_evidence_keys=first.artifact_evidence_keys, + ) + with pytest.raises(ValueError, match="forbidden outside promotion"): + ModernReferenceCertificationCampaignSpecV1( + common_end_period=spec.common_end_period, + peak_memory_budget_bytes=spec.peak_memory_budget_bytes, + scratch_budget_bytes=spec.scratch_budget_bytes, + runtime_budget_seconds=spec.runtime_budget_seconds, + storage_budget_bytes=spec.storage_budget_bytes, + candidate_amplification_budget=spec.candidate_amplification_budget, + artifacts=spec.artifacts, + observations=(*spec.observations, coverage), + methodology=spec.methodology, + accepted_limitations=spec.accepted_limitations, + blocking_limitations=spec.blocking_limitations, + ) diff --git a/tests/unit/test_synthetic_certification_v2.py b/tests/unit/test_synthetic_certification_v2.py new file mode 100644 index 00000000..281562af --- /dev/null +++ b/tests/unit/test_synthetic_certification_v2.py @@ -0,0 +1,257 @@ +"""Modern-reference reconstruction certification regression tests.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +from typing import cast + +import pytest + +from histdatacom.synthetic import ( + MODERN_REFERENCE_DELIVERY_CLAIM, + MODERN_REFERENCE_DELIVERY_MODE, + PROMOTION_ONLY_CHECK_IDS, + CertificationArtifactV1, + CertificationCheckStatus, + CertificationComparator, + CertificationObservationV1, + CertificationRequirementV1, + CertificationState, + ReconstructionCertificationDossierV2, + ReconstructionCertificationPolicyV2, + evaluate_modern_reference_reconstruction_certification, + load_modern_reference_reconstruction_certification_dossier, + modern_reference_triangle_certification_policy, + write_modern_reference_reconstruction_certification_dossier, +) + +METHODOLOGY = ( + "The fixture binds independently produced readiness, scientific, product, " + "operational, repository, and publication reports to a frozen modern-reference " + "policy. It exercises contract semantics only and contains no event rows." +) + + +def _policy() -> ReconstructionCertificationPolicyV2: + return modern_reference_triangle_certification_policy( + common_end_period="202606", + peak_memory_budget_bytes=4_000_000_000, + scratch_budget_bytes=80_000_000_000, + runtime_budget_seconds=86_400.0, + storage_budget_bytes=80_000_000_000, + candidate_amplification_budget=10.0, + ) + + +def _passing_value( + requirement: CertificationRequirementV1, +) -> bool | int | float | str | None: + if requirement.comparator is CertificationComparator.LESS_OR_EQUAL: + assert isinstance(requirement.expected, (int, float)) + return cast(int | float, requirement.expected) / 2 + if requirement.comparator is CertificationComparator.GREATER_OR_EQUAL: + return requirement.expected + return cast(bool | int | float | str | None, requirement.expected) + + +def _artifacts( + policy: ReconstructionCertificationPolicyV2, +) -> tuple[CertificationArtifactV1, ...]: + kinds = sorted( + { + kind + for requirement in policy.requirements + for kind in requirement.required_artifact_kinds + } + ) + return tuple( + CertificationArtifactV1.from_payload( + policy_id=policy.policy_id, + kind=kind, + subject_id=f"{kind}:fixture", + subject_schema_version=f"histdatacom.{kind}.v1", + payload={ + "schema_version": f"histdatacom.{kind}.v1", + "subject_id": f"{kind}:fixture", + "fixture": True, + }, + relative_path=f"certification/{kind}.json", + metadata={"event_rows_inline": False}, + ) + for kind in kinds + ) + + +def _observations( + policy: ReconstructionCertificationPolicyV2, + artifacts: tuple[CertificationArtifactV1, ...], + *, + omit: frozenset[str] = frozenset(), +) -> tuple[CertificationObservationV1, ...]: + by_kind = {item.kind: item for item in artifacts} + return tuple( + CertificationObservationV1( + check_id=requirement.check_id, + actual=_passing_value(requirement), + artifact_evidence_ids=tuple( + by_kind[kind].evidence_id + for kind in requirement.required_artifact_kinds + ), + note="deterministic contract fixture", + ) + for requirement in policy.requirements + if requirement.check_id not in omit + ) + + +def _dossier( + *, omit: frozenset[str] = frozenset() +) -> ReconstructionCertificationDossierV2: + policy = _policy() + artifacts = _artifacts(policy) + return evaluate_modern_reference_reconstruction_certification( + policy, + artifacts=artifacts, + observations=_observations(policy, artifacts, omit=omit), + methodology=METHODOLOGY, + accepted_limitations=( + "This fixture proves certification mechanics, not product evidence.", + ), + ) + + +def test_policy_covers_all_live_issue_seams_without_broker_evidence() -> None: + """V2 binds the current #449 scope and leaves V1 semantics untouched.""" + policy = _policy() + checks = {item.check_id for item in policy.requirements} + kinds = { + kind + for requirement in policy.requirements + for kind in requirement.required_artifact_kinds + } + + assert policy.delivery_mode == MODERN_REFERENCE_DELIVERY_MODE + assert policy.delivery_claim == MODERN_REFERENCE_DELIVERY_CLAIM + assert all("broker" not in value for value in checks | kinds) + assert { + "source_inventory_reconciled", + "market_context_corpus_valid", + "cftc_positioning_corpus_valid", + "feed_epoch_artifact_valid", + "observation_operator_valid", + "benchmark_corpus_valid", + "motif_artifact_valid", + "representative_window_class_missing_count", + "substantial_multi_period_run_passed", + "cancellation_publishable_partial_count", + "invalid_information_mode_refused", + "quota_overflow_refused", + "public_cli_api_evidence_chain_passed", + "declared_test_dependencies_installed", + }.issubset(checks) + assert ( + ReconstructionCertificationPolicyV2.from_dict(policy.to_dict()) + == policy + ) + + +def test_complete_v2_fixture_certifies_and_round_trips() -> None: + """Complete bounded V2 evidence produces a broker-neutral dossier.""" + dossier = _dossier() + payload = dossier.to_dict() + + assert dossier.state is CertificationState.CERTIFIED + assert dossier.summary["passed_gate_count"] == 15 + assert payload["delivery_mode"] == MODERN_REFERENCE_DELIVERY_MODE + assert payload["delivery_claim"] == MODERN_REFERENCE_DELIVERY_CLAIM + assert payload["broker_specific_claim"] is False + assert ( + ReconstructionCertificationDossierV2.from_json(dossier.to_json()) + == dossier + ) + + +def test_only_promotion_coverage_missing_is_ready_for_promotion() -> None: + """The modern-reference policy retains the exactly-once coverage boundary.""" + dossier = _dossier(omit=PROMOTION_ONLY_CHECK_IDS) + + assert dossier.state is CertificationState.READY_FOR_PROMOTION + assert { + result.check_id + for gate in dossier.gate_results + for result in gate.check_results + if result.status is CertificationCheckStatus.MISSING + } == PROMOTION_ONLY_CHECK_IDS + + +def test_v2_rejects_broker_artifacts_and_nonexact_evidence_bindings() -> None: + """Broker evidence and unrelated extra artifacts cannot satisfy V2 checks.""" + policy = _policy() + artifacts = _artifacts(policy) + broker = CertificationArtifactV1.from_payload( + policy_id=policy.policy_id, + kind="broker-delivery-fingerprint", + subject_id="broker:fixture", + subject_schema_version="histdatacom.broker.v1", + payload={"schema_version": "histdatacom.broker.v1"}, + relative_path="certification/broker.json", + ) + with pytest.raises(ValueError, match="broker-specific evidence"): + evaluate_modern_reference_reconstruction_certification( + policy, + artifacts=(*artifacts, broker), + observations=_observations(policy, artifacts), + methodology=METHODOLOGY, + ) + + first = _observations(policy, artifacts)[0] + altered = replace( + first, + artifact_evidence_ids=( + *first.artifact_evidence_ids, + artifacts[-1].evidence_id, + ), + observation_id="", + ) + observations = tuple( + altered if item.check_id == first.check_id else item + for item in _observations(policy, artifacts) + ) + dossier = evaluate_modern_reference_reconstruction_certification( + policy, + artifacts=artifacts, + observations=observations, + methodology=METHODOLOGY, + ) + result = next( + item + for gate in dossier.gate_results + for item in gate.check_results + if item.check_id == first.check_id + ) + assert result.status is CertificationCheckStatus.MISSING + assert "evidence kinds differ" in result.reason + + +def test_v2_publication_is_atomic_replayable_and_explicitly_unconditioned( + tmp_path: Path, +) -> None: + """The published report names its delivery boundary without broker claims.""" + dossier = _dossier() + json_path = tmp_path / "dossier.json" + markdown_path = tmp_path / "dossier.md" + + refs = write_modern_reference_reconstruction_certification_dossier( + dossier, json_path=json_path, markdown_path=markdown_path + ) + + assert ( + load_modern_reference_reconstruction_certification_dossier(json_path) + == dossier + ) + assert all(ref.sha256 and ref.size_bytes for ref in refs) + markdown = markdown_path.read_text(encoding="utf-8") + assert "modern_reference" in markdown + assert "unconditioned_reference" in markdown + assert "Broker-specific claim: `false`" in markdown diff --git a/tests/unit/test_synthetic_contracts.py b/tests/unit/test_synthetic_contracts.py new file mode 100644 index 00000000..02113fc2 --- /dev/null +++ b/tests/unit/test_synthetic_contracts.py @@ -0,0 +1,532 @@ +"""Tests for versioned variable-cardinality synthetic event contracts.""" + +from __future__ import annotations + +from dataclasses import replace +import json +from pathlib import Path +import subprocess +import sys + +from hypothesis import given, settings +from hypothesis import strategies as st +import pyarrow as pa +import pytest + +from histdatacom.synthetic import ( + SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION, + SYNTHETIC_EVENT_SCHEMA_VERSION, + SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION, + SyntheticEnsembleManifestV1, + SyntheticEnsembleMemberV1, + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + derive_anchor_interval_id, + read_synthetic_event_stream_parquet, + synthetic_event_arrow_schema, + synthetic_event_stream_from_arrow, + synthetic_event_stream_from_parquet_bytes, + synthetic_event_stream_to_arrow, + synthetic_event_stream_to_parquet_bytes, + write_synthetic_event_stream_parquet, +) +from histdatacom.synthetic.contracts import SYNTHETIC_EVENT_ARROW_COLUMNS + +BASE_TIME_NS = 1_700_000_000_000_000_000 +RUN_ID = "run-contract-v1" +MEMBER_ID = "member-000" +SYMBOL = "eurusd" +OBSERVED_SOURCE = "source-artifact:sha256:observed-v1" +GENERATED_SOURCE = "source-manifest:sha256:inputs-v1" + + +def _observed( + row_id: int, + *, + event_time_ns: int | None = None, + event_sequence: int = 0, + member_id: str = MEMBER_ID, + bid: float | None = None, +) -> SyntheticEventV1: + selected_bid = bid if bid is not None else 1.08 + row_id / 100_000 + return SyntheticEventV1.observed( + symbol=SYMBOL, + event_time_ns=( + event_time_ns + if event_time_ns is not None + else BASE_TIME_NS + row_id * 1_000 + ), + event_sequence=event_sequence, + bid=selected_bid, + ask=selected_bid + 0.0001, + run_id=RUN_ID, + ensemble_member_id=member_id, + source_version_id=OBSERVED_SOURCE, + source_series_id="ascii:T:eurusd", + source_period="202401", + source_row_id=row_id, + ) + + +def _generated( + left: SyntheticEventV1, + right: SyntheticEventV1, + *, + ordinal: int = 1, + member_id: str = MEMBER_ID, +) -> SyntheticEventV1: + return SyntheticEventV1.generated( + symbol=SYMBOL, + event_time_ns=left.event_time_ns + ordinal, + event_sequence=0, + bid=1.08005 + ordinal / 1_000_000_000, + ask=1.08015 + ordinal / 1_000_000_000, + run_id=RUN_ID, + ensemble_member_id=member_id, + source_version_id=GENERATED_SOURCE, + left_anchor_event_id=left.event_id, + right_anchor_event_id=right.event_id, + generator_id="empirical-motif", + generator_version="1.0.0", + generator_config_id="config:sha256:motif-v1", + constraint_set_id="constraint:sha256:historical-v1", + confidence=0.8, + reference_id="reference:modern-2026", + motif_id="motif:quiet-london-001", + feed_epoch_id="feed-epoch:modern", + broker_profile_id="broker-profile:demo-v1", + ) + + +def _stream( + *, + member_id: str = MEMBER_ID, + generated_count: int = 1, +) -> SyntheticEventStreamV1: + left = _observed(1, member_id=member_id) + right = _observed( + 2, + event_time_ns=left.event_time_ns + 1_000, + member_id=member_id, + ) + generated = tuple( + _generated(left, right, ordinal=ordinal, member_id=member_id) + for ordinal in range(1, generated_count + 1) + ) + return SyntheticEventStreamV1.merge( + run_id=RUN_ID, + ensemble_member_id=member_id, + symbol=SYMBOL, + observed_events=(right, left), + synthetic_events=reversed(generated), + ) + + +def test_event_python_and_json_round_trip_verifies_identity() -> None: + left = _observed(1) + right = _observed(2) + generated = _generated(left, right) + + for event in (left, generated): + assert SyntheticEventV1.from_dict(event.to_dict()) == event + assert SyntheticEventV1.from_json(event.to_json()) == event + assert event.event_id.startswith("event:sha256:") + assert len(event.to_dict()) == len(SYNTHETIC_EVENT_ARROW_COLUMNS) + + assert left.origin is SyntheticEventOrigin.OBSERVED + assert generated.origin is SyntheticEventOrigin.SYNTHETIC + assert generated.anchor_interval_id == derive_anchor_interval_id( + left.event_id, + right.event_id, + ) + + +def test_event_reader_accepts_unknown_json_but_rejects_schema_or_id_drift() -> ( + None +): + payload = _observed(1).to_dict() + payload["future_envelope_key"] = "ignored" + assert SyntheticEventV1.from_dict(payload) == _observed(1) + + wrong_schema = dict(payload) + wrong_schema["schema_version"] = "histdatacom.synthetic-event.v2" + with pytest.raises(ValueError, match="unsupported schema version"): + SyntheticEventV1.from_dict(wrong_schema) + + wrong_id = dict(payload) + wrong_id["event_id"] = "event:sha256:" + "0" * 64 + with pytest.raises(ValueError, match="event_id does not match"): + SyntheticEventV1.from_dict(wrong_id) + + +@given( + field=st.sampled_from( + ( + "anchor_interval_id", + "left_anchor_event_id", + "right_anchor_event_id", + "generator_id", + "generator_version", + "generator_config_id", + "constraint_set_id", + ) + ), + invalid_value=st.sampled_from((None, "", " ")), +) +@settings(max_examples=30, deadline=None) +def test_synthetic_event_rejects_missing_reproducibility_lineage( + field: str, + invalid_value: str | None, +) -> None: + event = _generated(_observed(1), _observed(2)) + payload = event.to_dict() + payload[field] = invalid_value + payload["event_id"] = "" + + with pytest.raises(ValueError, match=f"requires {field}"): + SyntheticEventV1.from_dict(payload) + + +def test_origin_identity_cannot_be_misrepresented() -> None: + observed = _observed(1) + generated = _generated(observed, _observed(2)) + + with pytest.raises(ValueError, match="synthetic lineage"): + replace(observed, generator_id="not-observed", event_id="") + with pytest.raises(ValueError, match="observed row identity"): + replace( + generated, + source_series_id="fake-series", + source_period="202401", + source_row_id=99, + event_id="", + ) + + +def test_synthetic_confidence_is_optional_but_bounded_when_present() -> None: + event = _generated(_observed(1), _observed(2)) + without_confidence = replace(event, confidence=None, event_id="") + + assert without_confidence.confidence is None + assert SyntheticEventV1.from_dict(without_confidence.to_dict()) == ( + without_confidence + ) + with pytest.raises(ValueError, match="between zero and one"): + replace(event, confidence=1.1, event_id="") + + +@given( + row_id=st.integers(min_value=1, max_value=1_000_000), + event_time_ns=st.integers( + min_value=BASE_TIME_NS, + max_value=BASE_TIME_NS + 1_000_000_000, + ), + event_sequence=st.integers(min_value=0, max_value=100_000), + price_units=st.integers(min_value=1, max_value=10_000_000), + spread_units=st.integers(min_value=0, max_value=10_000), +) +@settings(max_examples=50, deadline=None) +def test_observed_identity_and_values_round_trip_as_a_property( + row_id: int, + event_time_ns: int, + event_sequence: int, + price_units: int, + spread_units: int, +) -> None: + bid = price_units / 1_000_000 + event = SyntheticEventV1.observed( + symbol="EURUSD", + event_time_ns=event_time_ns, + event_sequence=event_sequence, + bid=bid, + ask=bid + spread_units / 1_000_000, + run_id=RUN_ID, + ensemble_member_id=MEMBER_ID, + source_version_id=OBSERVED_SOURCE, + source_series_id="series-property", + source_period="202401", + source_row_id=row_id, + ) + + restored = SyntheticEventV1.from_json(event.to_json()) + assert restored == event + assert restored.event_id == event.event_id + assert restored.source_row_id == row_id + assert restored.bid == bid + + +@given(order=st.permutations((0, 1, 2, 3, 4))) +@settings(max_examples=30, deadline=None) +def test_duplicate_timestamp_ordering_is_deterministic_property( + order: list[int], +) -> None: + events = tuple( + _observed( + sequence + 1, + event_time_ns=BASE_TIME_NS, + event_sequence=sequence, + ) + for sequence in order + ) + stream = SyntheticEventStreamV1( + run_id=RUN_ID, + ensemble_member_id=MEMBER_ID, + symbol=SYMBOL, + events=events, + ) + + assert [event.event_sequence for event in stream.events] == list(range(5)) + + +def test_stream_rejects_ambiguous_duplicate_position() -> None: + first = _observed(1, event_time_ns=BASE_TIME_NS, event_sequence=0) + second = _observed(2, event_time_ns=BASE_TIME_NS, event_sequence=0) + + with pytest.raises(ValueError, match="duplicate event_time_ns"): + SyntheticEventStreamV1( + run_id=RUN_ID, + ensemble_member_id=MEMBER_ID, + symbol=SYMBOL, + events=(first, second), + ) + + +@pytest.mark.parametrize("generated_count", [0, 1, 4]) +def test_stream_permits_variable_cardinality_and_preserves_observations( + generated_count: int, +) -> None: + stream = _stream(generated_count=generated_count) + observed = tuple( + event + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + ) + + assert stream.observed_event_count == 2 + assert stream.synthetic_event_count == generated_count + assert observed == (_observed(1), _observed(2)) + assert SyntheticEventStreamV1.from_json(stream.to_json()) == stream + + +def test_generated_event_id_is_partition_and_retry_independent() -> None: + left = _observed(1) + right = _observed(2, event_time_ns=left.event_time_ns + 1_000) + first = _generated(left, right) + retry = _generated(left, right) + earlier = _observed(3, event_time_ns=left.event_time_ns - 1_000) + later = _observed(4, event_time_ns=right.event_time_ns + 1_000) + + narrow = SyntheticEventStreamV1( + run_id=RUN_ID, + ensemble_member_id=MEMBER_ID, + symbol=SYMBOL, + events=(left, first, right), + ) + wider = SyntheticEventStreamV1( + run_id=RUN_ID, + ensemble_member_id=MEMBER_ID, + symbol=SYMBOL, + events=(earlier, left, retry, right, later), + ) + + assert first.event_id == retry.event_id + assert narrow.stream_id != wider.stream_id + assert first.event_id in {event.event_id for event in wider.events} + + +@given( + ordinal=st.integers(min_value=1, max_value=999), + surrounding_count=st.integers(min_value=0, max_value=20), +) +@settings(max_examples=50, deadline=None) +def test_generated_event_id_stability_is_a_partition_context_property( + ordinal: int, + surrounding_count: int, +) -> None: + left = _observed(1) + right = _observed(2, event_time_ns=left.event_time_ns + 1_000) + generated = _generated(left, right, ordinal=ordinal) + retry = _generated(left, right, ordinal=ordinal) + surrounding = tuple( + _observed( + row_id + 10, + event_time_ns=right.event_time_ns + (row_id + 1) * 1_000, + ) + for row_id in range(surrounding_count) + ) + + narrow = SyntheticEventStreamV1( + run_id=RUN_ID, + ensemble_member_id=MEMBER_ID, + symbol=SYMBOL, + events=(left, generated, right), + ) + wider = SyntheticEventStreamV1( + run_id=RUN_ID, + ensemble_member_id=MEMBER_ID, + symbol=SYMBOL, + events=(left, retry, right, *surrounding), + ) + + assert generated.event_id == retry.event_id + assert narrow.stream_id != wider.stream_id or not surrounding + assert generated.event_id in {event.event_id for event in wider.events} + + +def test_arrow_and_parquet_round_trip_without_loss(tmp_path: Path) -> None: + stream = _stream(generated_count=3) + table = synthetic_event_stream_to_arrow(stream) + + assert synthetic_event_stream_from_arrow(table) == stream + assert table.schema.names == list(SYNTHETIC_EVENT_ARROW_COLUMNS) + assert table.schema.field("event_time_ns").type == pa.int64() + assert table.schema.field("bid").type == pa.float64() + assert not { + name + for name in table.schema.names + if name.startswith(("dq_", "cm_", "synth_")) + } + + first_bytes = synthetic_event_stream_to_parquet_bytes(stream) + second_bytes = synthetic_event_stream_to_parquet_bytes(stream) + assert first_bytes == second_bytes + assert synthetic_event_stream_from_parquet_bytes(first_bytes) == stream + + output = write_synthetic_event_stream_parquet( + stream, + tmp_path / "nested" / "events.parquet", + ) + assert read_synthetic_event_stream_parquet(output) == stream + + +def test_arrow_schema_drift_and_metadata_tampering_fail_closed() -> None: + stream = _stream() + table = synthetic_event_stream_to_arrow(stream) + drifted = table.drop(["confidence"]) + with pytest.raises(ValueError, match="schema does not match"): + synthetic_event_stream_from_arrow(drifted) + + metadata = dict(table.schema.metadata or {}) + header = json.loads( + metadata[b"histdatacom.synthetic_event_stream"].decode("utf-8") + ) + header["event_count"] += 1 + metadata[b"histdatacom.synthetic_event_stream"] = json.dumps( + header, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + tampered = table.replace_schema_metadata(metadata) + with pytest.raises(ValueError, match="event count"): + synthetic_event_stream_from_arrow(tampered) + + +def test_stream_json_derived_schema_and_counts_fail_closed() -> None: + payload = _stream().to_dict() + payload["event_count"] = 99 + with pytest.raises(ValueError, match="event count"): + SyntheticEventStreamV1.from_dict(payload) + + payload = _stream().to_dict() + payload["event_schema_version"] = "histdatacom.synthetic-event.v2" + with pytest.raises(ValueError, match="event_schema_version"): + SyntheticEventStreamV1.from_dict(payload) + + +def test_ensemble_manifest_is_compact_deterministic_and_reconciling() -> None: + primary = _stream(member_id="member-000", generated_count=1) + alternate = _stream(member_id="member-001", generated_count=3) + manifest = SyntheticEnsembleManifestV1.from_streams( + (alternate, primary), + primary_member_id="member-000", + configuration_ids=("config:sha256:motif-v1",), + ) + + assert manifest.schema_version == ( + SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION + ) + assert [member.member_id for member in manifest.members] == [ + "member-000", + "member-001", + ] + assert manifest.members[0].event_count == 3 + assert manifest.members[1].event_count == 5 + assert "events" not in manifest.to_dict() + assert SyntheticEnsembleManifestV1.from_json(manifest.to_json()) == ( + manifest + ) + + duplicate = SyntheticEnsembleMemberV1( + member_id="member-duplicate", + stream_id=manifest.members[0].stream_id, + event_count=manifest.members[0].event_count, + observed_event_count=manifest.members[0].observed_event_count, + synthetic_event_count=manifest.members[0].synthetic_event_count, + content_sha256=manifest.members[0].content_sha256, + ) + with pytest.raises(ValueError, match="stream IDs must be unique"): + SyntheticEnsembleManifestV1( + run_id=RUN_ID, + primary_member_id="member-000", + members=(manifest.members[0], duplicate), + source_version_ids=(OBSERVED_SOURCE,), + configuration_ids=("config:sha256:motif-v1",), + ) + + with pytest.raises(ValueError, match="do not cover generated events"): + SyntheticEnsembleManifestV1.from_streams( + (primary,), + primary_member_id="member-000", + configuration_ids=("config:other",), + ) + + payload = manifest.to_dict() + payload["stream_schema_version"] = "histdatacom.synthetic-event-stream.v2" + with pytest.raises(ValueError, match="stream_schema_version"): + SyntheticEnsembleManifestV1.from_dict(payload) + + +def test_contract_schema_versions_are_explicit_and_stable() -> None: + assert SYNTHETIC_EVENT_SCHEMA_VERSION.endswith(".v1") + assert SYNTHETIC_EVENT_STREAM_SCHEMA_VERSION.endswith(".v1") + assert SYNTHETIC_ENSEMBLE_MANIFEST_SCHEMA_VERSION.endswith(".v1") + assert synthetic_event_arrow_schema().names == list( + SYNTHETIC_EVENT_ARROW_COLUMNS + ) + + +def test_importing_synthetic_contracts_does_not_import_optional_arrow() -> None: + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; import histdatacom.synthetic; " + "assert 'pyarrow' not in sys.modules" + ), + ], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + + +def test_example_stream_artifact_is_valid_stable_and_narrow() -> None: + fixture = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "synthetic_event_stream_v1.json" + ) + text = fixture.read_text(encoding="utf-8") + stream = SyntheticEventStreamV1.from_dict(json.loads(text)) + + assert stream.observed_event_count == 2 + assert stream.synthetic_event_count == 1 + assert json.dumps(stream.to_dict(), indent=2, sort_keys=True) + "\n" == text + assert not any( + name.startswith(("dq_", "cm_", "synth_")) + for name in stream.events[0].to_dict() + ) diff --git a/tests/unit/test_synthetic_cross_currency.py b/tests/unit/test_synthetic_cross_currency.py new file mode 100644 index 00000000..dc10d42b --- /dev/null +++ b/tests/unit/test_synthetic_cross_currency.py @@ -0,0 +1,802 @@ +"""Tests for synchronized event-time cross-currency reconstruction.""" + +from __future__ import annotations + +from dataclasses import replace +import hashlib + +import pytest + +from histdatacom.data_quality import ( + CROSS_INSTRUMENT_METADATA_KEY, + QualityStatus, +) +from histdatacom.runtime_contracts import ArtifactRef +from histdatacom.synthetic import ( + CrossCurrencyConditionV1, + CrossCurrencyExcludedReason, + CrossCurrencyGroupStatus, + CrossCurrencyReconciledGroupV1, + CrossCurrencyReconciliationConfigV1, + CrossCurrencyRelationshipKind, + CrossCurrencyRelationshipV1, + CrossCurrencySymbolCoverageV1, + CrossCurrencyValidationStage, + CrossCurrencyValidationStatus, + CrossCurrencyWindowPlanStatus, + CrossCurrencyWindowPlanV1, + EventBatchV1, + PartitionManifestV1, + ReconstructionRunV1, + ReconstructionStoragePolicyV1, + ReconstructionWindowV1, + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + cross_currency_quality_report, + eurusd_triangle_reconciliation_config, + plan_cross_currency_windows, + reconcile_cross_currency_window, + validate_cross_currency_atomic_manifest, + validate_cross_currency_output, +) + +MEMBER = "member-000" +SOURCE = "source:sha256:historical-v1" +START_NS = 1_700_000_000_000_000_000 +END_NS = START_NS + 1_000_000_000 +SYNTHETIC_NS = START_NS + 500_000_000 + + +def _run( + config: CrossCurrencyReconciliationConfigV1, + *, + symbols: tuple[str, ...] = ("EURGBP", "EURUSD", "GBPUSD"), +) -> ReconstructionRunV1: + return ReconstructionRunV1( + symbols=symbols, + source_version_ids=(SOURCE,), + configuration_ids=(config.config_id,), + ensemble_member_ids=(MEMBER,), + base_seed=441, + storage_policy=ReconstructionStoragePolicyV1(), + ) + + +def _window(run: ReconstructionRunV1) -> ReconstructionWindowV1: + return ReconstructionWindowV1( + run_id=run.run_id, + ensemble_member_id=MEMBER, + symbols=run.symbols, + core_start_ns=START_NS, + core_end_ns=END_NS, + ) + + +def _observed( + run: ReconstructionRunV1, + *, + symbol: str, + timestamp_ns: int, + sequence: int, + row_id: int, + midpoint: float, +) -> SyntheticEventV1: + return SyntheticEventV1.observed( + symbol=symbol, + event_time_ns=timestamp_ns, + event_sequence=sequence, + bid=midpoint - 0.0001, + ask=midpoint + 0.0001, + run_id=run.run_id, + ensemble_member_id=MEMBER, + source_version_id=SOURCE, + source_series_id=f"ascii:T:{symbol}:histdata.com", + source_period="202001", + source_row_id=row_id, + ) + + +def _stream( + run: ReconstructionRunV1, + *, + symbol: str, + anchor_midpoint: float, + synthetic_midpoint: float, + extra_synthetic: tuple[tuple[int, int, float], ...] = (), + middle_origin: SyntheticEventOrigin = SyntheticEventOrigin.SYNTHETIC, +) -> SyntheticEventStreamV1: + left = _observed( + run, + symbol=symbol, + timestamp_ns=START_NS, + sequence=0, + row_id=1, + midpoint=anchor_midpoint, + ) + right = _observed( + run, + symbol=symbol, + timestamp_ns=END_NS - 1, + sequence=0, + row_id=3, + midpoint=anchor_midpoint, + ) + if middle_origin is SyntheticEventOrigin.OBSERVED: + middle = _observed( + run, + symbol=symbol, + timestamp_ns=SYNTHETIC_NS, + sequence=1, + row_id=2, + midpoint=synthetic_midpoint, + ) + else: + middle = SyntheticEventV1.generated( + symbol=symbol, + event_time_ns=SYNTHETIC_NS, + event_sequence=1, + bid=synthetic_midpoint - 0.0001, + ask=synthetic_midpoint + 0.0001, + run_id=run.run_id, + ensemble_member_id=MEMBER, + source_version_id=SOURCE, + left_anchor_event_id=left.event_id, + right_anchor_event_id=right.event_id, + generator_id="empirical-motif", + generator_version="1.0.0", + generator_config_id="motif-config:fixture", + constraint_set_id="carving-constraints:fixture", + confidence=0.8, + motif_id="motif:fixture", + reference_id="reference:fixture", + feed_epoch_id="epoch:modern", + ) + extras = tuple( + SyntheticEventV1.generated( + symbol=symbol, + event_time_ns=timestamp_ns, + event_sequence=sequence, + bid=midpoint - 0.0001, + ask=midpoint + 0.0001, + run_id=run.run_id, + ensemble_member_id=MEMBER, + source_version_id=SOURCE, + left_anchor_event_id=left.event_id, + right_anchor_event_id=right.event_id, + generator_id="empirical-motif", + generator_version="1.0.0", + generator_config_id="motif-config:fixture", + constraint_set_id="carving-constraints:fixture", + confidence=0.7, + motif_id=f"motif:extra:{sequence}", + reference_id="reference:fixture", + feed_epoch_id="epoch:modern", + ) + for timestamp_ns, sequence, midpoint in extra_synthetic + ) + return SyntheticEventStreamV1( + run_id=run.run_id, + ensemble_member_id=MEMBER, + symbol=symbol, + events=(left, middle, *extras, right), + ) + + +def _triangle_streams( + run: ReconstructionRunV1, + *, + direct_midpoint: float = 0.82, + middle_origin: SyntheticEventOrigin = SyntheticEventOrigin.SYNTHETIC, + eurusd_extras: tuple[tuple[int, int, float], ...] = (), +) -> dict[str, SyntheticEventStreamV1]: + return { + "EURGBP": _stream( + run, + symbol="EURGBP", + anchor_midpoint=0.8, + synthetic_midpoint=direct_midpoint, + middle_origin=middle_origin, + ), + "EURUSD": _stream( + run, + symbol="EURUSD", + anchor_midpoint=1.2, + synthetic_midpoint=1.2, + extra_synthetic=eurusd_extras, + middle_origin=middle_origin, + ), + "GBPUSD": _stream( + run, + symbol="GBPUSD", + anchor_midpoint=1.5, + synthetic_midpoint=1.5, + middle_origin=middle_origin, + ), + } + + +def _condition() -> CrossCurrencyConditionV1: + return CrossCurrencyConditionV1( + start_ns=START_NS, + end_ns=END_NS, + session_key="london", + event_key="ordinary", + feed_epoch_key="modern-electronic", + ) + + +def _reconciled_group() -> tuple[ + ReconstructionRunV1, + ReconstructionWindowV1, + dict[str, SyntheticEventStreamV1], + CrossCurrencyReconciledGroupV1, +]: + config = eurusd_triangle_reconciliation_config() + run = _run(config) + window = _window(run) + streams = _triangle_streams(run) + group = reconcile_cross_currency_window( + run=run, + window=window, + streams=streams, + config=config, + conditions=(_condition(),), + ) + return run, window, streams, group + + +def test_common_window_plan_records_unequal_and_missing_coverage() -> None: + """Only common coverage is planned and every excluded span is explicit.""" + config = eurusd_triangle_reconciliation_config() + run = _run(config) + plan = plan_cross_currency_windows( + run, + ensemble_member_id=MEMBER, + requested_start_ns=START_NS, + requested_end_ns=END_NS, + window_size_ns=200_000_000, + coverages=( + CrossCurrencySymbolCoverageV1( + "EURGBP", + START_NS + 200_000_000, + END_NS - 100_000_000, + ("200203", "200204"), + ), + CrossCurrencySymbolCoverageV1( + "EURUSD", START_NS, END_NS, ("200005",) + ), + CrossCurrencySymbolCoverageV1( + "GBPUSD", START_NS, END_NS, ("200005",) + ), + ), + ) + + assert plan.status is CrossCurrencyWindowPlanStatus.PLANNED + assert plan.common_start_ns == START_NS + 200_000_000 + assert plan.common_end_ns == END_NS - 100_000_000 + assert plan.windows[0].symbols == run.symbols + reasons = {(item.symbol, item.reason) for item in plan.excluded_spans} + assert ( + "eurgbp", + CrossCurrencyExcludedReason.SYMBOL_NOT_YET_AVAILABLE, + ) in reasons + assert ( + "eurgbp", + CrossCurrencyExcludedReason.SYMBOL_NO_LONGER_AVAILABLE, + ) in reasons + assert CrossCurrencyWindowPlanV1.from_json(plan.to_json()) == plan + + missing = plan_cross_currency_windows( + run, + ensemble_member_id=MEMBER, + requested_start_ns=START_NS, + requested_end_ns=END_NS, + window_size_ns=200_000_000, + coverages=( + CrossCurrencySymbolCoverageV1("EURUSD", START_NS, END_NS), + CrossCurrencySymbolCoverageV1("GBPUSD", START_NS, END_NS), + ), + ) + assert missing.status is CrossCurrencyWindowPlanStatus.REFUSED + assert missing.missing_symbols == ("eurgbp",) + assert missing.windows == () + assert any( + item.reason is CrossCurrencyExcludedReason.MISSING_SYMBOL + for item in missing.excluded_spans + ) + + no_overlap = plan_cross_currency_windows( + run, + ensemble_member_id=MEMBER, + requested_start_ns=START_NS, + requested_end_ns=END_NS, + window_size_ns=200_000_000, + coverages=( + CrossCurrencySymbolCoverageV1( + "EURGBP", START_NS, START_NS + 400_000_000 + ), + CrossCurrencySymbolCoverageV1( + "EURUSD", START_NS + 500_000_000, END_NS + ), + CrossCurrencySymbolCoverageV1("GBPUSD", START_NS, END_NS), + ), + ) + assert no_overlap.status is CrossCurrencyWindowPlanStatus.REFUSED + assert no_overlap.windows == () + assert any( + item.symbol == "*" + and item.reason is CrossCurrencyExcludedReason.NO_COMMON_SUPPORT + for item in no_overlap.excluded_spans + ) + + +def test_triangle_reconciliation_is_deterministic_and_preserves_anchors() -> ( + None +): + """The dependent synthetic leg is projected and observations stay exact.""" + run, window, inputs, group = _reconciled_group() + + assert group.status is CrossCurrencyGroupStatus.RECONCILED + assert group.generation_ready is True + assert group.requires_post_broker_validation is True + assert group.generation_validation.passed is True + direct = group.stream_for("EURGBP") + projected = next( + event for event in direct.events if event.event_time_ns == SYNTHETIC_NS + ) + assert projected.bid == pytest.approx(1.1999 / 1.5001) + assert projected.ask == pytest.approx(1.2001 / 1.4999) + assert (projected.bid + projected.ask) / 2.0 == pytest.approx(0.8) + assert len(group.projection_lineage) == 1 + lineage = group.projection_lineage[0] + assert lineage.symbol == "eurgbp" + assert lineage.input_event_id == lineage.output_event_id + assert lineage.input_content_sha256 != lineage.output_content_sha256 + assert lineage.post_residual <= lineage.allowed_residual + assert { + (item.dimension, item.key) + for item in group.generation_validation.residual_slices + } >= { + ("session", "london"), + ("event", "ordinary"), + ("feed_epoch", "modern-electronic"), + } + for symbol, stream in inputs.items(): + before = { + event.event_id: event.to_dict() + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + } + after = { + event.event_id: event.to_dict() + for event in group.stream_for(symbol).events + if event.origin is SyntheticEventOrigin.OBSERVED + } + assert after == before + + retry = reconcile_cross_currency_window( + run=run, + window=window, + streams=inputs, + config=group.config, + conditions=(_condition(),), + ) + assert retry == group + assert CrossCurrencyReconciledGroupV1.from_json(group.to_json()) == group + + +def test_triangle_projection_falls_back_after_negative_spread_target() -> None: + """A later synthetic priority leg is tried when the first is infeasible.""" + config = eurusd_triangle_reconciliation_config() + run = _run(config) + window = _window(run) + + def requote( + stream: SyntheticEventStreamV1, + *, + bid: float, + ask: float, + ) -> SyntheticEventStreamV1: + return SyntheticEventStreamV1( + run_id=stream.run_id, + ensemble_member_id=stream.ensemble_member_id, + symbol=stream.symbol, + events=tuple( + ( + replace(event, bid=bid, ask=ask, event_id="") + if event.event_time_ns == SYNTHETIC_NS + else event + ) + for event in stream.events + ), + source_version_ids=stream.source_version_ids, + ) + + direct = requote( + _stream( + run, + symbol="EURGBP", + anchor_midpoint=0.8, + synthetic_midpoint=0.86711, + middle_origin=SyntheticEventOrigin.OBSERVED, + ), + bid=0.86699, + ask=0.86723, + ) + numerator = requote( + _stream( + run, + symbol="EURUSD", + anchor_midpoint=1.2, + synthetic_midpoint=1.09413, + ), + bid=1.09401, + ask=1.09425, + ) + denominator = requote( + _stream( + run, + symbol="GBPUSD", + anchor_midpoint=1.5, + synthetic_midpoint=1.26171, + ), + bid=1.26152, + ask=1.26190, + ) + + group = reconcile_cross_currency_window( + run=run, + window=window, + streams={ + "EURGBP": direct, + "EURUSD": numerator, + "GBPUSD": denominator, + }, + config=config, + ) + + assert group.status is CrossCurrencyGroupStatus.RECONCILED + assert len(group.projection_lineage) == 1 + assert group.projection_lineage[0].symbol == "gbpusd" + preserved = next( + event + for event in group.stream_for("EURGBP").events + if event.event_time_ns == SYNTHETIC_NS + ) + projected = next( + event + for event in group.stream_for("GBPUSD").events + if event.event_time_ns == SYNTHETIC_NS + ) + assert preserved.bid == 0.86699 + assert preserved.ask == 0.86723 + assert projected.ask >= projected.bid + assert group.generation_validation.anchor_preserved is True + + +def test_observed_conflict_and_missing_leg_refuse_without_projection() -> None: + """Infeasible anchors and absent legs remain visible refused results.""" + config = eurusd_triangle_reconciliation_config() + run = _run(config) + window = _window(run) + observed_conflict = _triangle_streams( + run, + direct_midpoint=0.9, + middle_origin=SyntheticEventOrigin.OBSERVED, + ) + refused = reconcile_cross_currency_window( + run=run, + window=window, + streams=observed_conflict, + config=config, + ) + assert refused.status is CrossCurrencyGroupStatus.REFUSED + assert refused.generation_validation.anchor_preserved is True + assert refused.projection_lineage == () + assert any( + reason.startswith("infeasible_relationship_point:") + for reason in refused.generation_validation.failure_reasons + ) + + missing = reconcile_cross_currency_window( + run=run, + window=window, + streams={ + symbol: stream + for symbol, stream in observed_conflict.items() + if symbol != "EURGBP" + }, + config=config, + ) + assert missing.status is CrossCurrencyGroupStatus.REFUSED + assert missing.missing_symbols == ("eurgbp",) + assert "missing_symbol:eurgbp" in ( + missing.generation_validation.failure_reasons + ) + + +def test_many_observed_conflicts_return_bounded_auditable_refusal() -> None: + """Large immutable-anchor conflicts refuse instead of overflowing evidence.""" + config = eurusd_triangle_reconciliation_config() + run = _run(config) + window = _window(run) + midpoints = {"eurgbp": 0.9, "eurusd": 1.2, "gbpusd": 1.5} + streams = { + symbol: SyntheticEventStreamV1( + run_id=run.run_id, + ensemble_member_id=MEMBER, + symbol=symbol, + events=tuple( + _observed( + run, + symbol=symbol, + timestamp_ns=START_NS + index * 1_000_000, + sequence=0, + row_id=index + 1, + midpoint=midpoint, + ) + for index in range(130) + ), + ) + for symbol, midpoint in midpoints.items() + } + + refused = reconcile_cross_currency_window( + run=run, + window=window, + streams=streams, + config=config, + ) + + assert refused.status is CrossCurrencyGroupStatus.REFUSED + reasons = refused.generation_validation.failure_reasons + assert len(reasons) == 128 + assert any( + reason.startswith("failure_reasons_truncated:total=130:sha256=") + for reason in reasons + ) + assert refused.generation_validation.anchor_preserved is True + assert ( + CrossCurrencyReconciledGroupV1.from_json(refused.to_json()) == refused + ) + + +def test_duplicate_and_asynchronous_times_are_bounded_without_forward_fill() -> ( + None +): + """Duplicate ordinals pair deterministically while sparse support is surfaced.""" + config = eurusd_triangle_reconciliation_config() + run = _run(config) + window = _window(run) + streams = _triangle_streams( + run, + eurusd_extras=( + (SYNTHETIC_NS, 2, 1.2), + (SYNTHETIC_NS + 100_000_000, 3, 1.21), + (SYNTHETIC_NS + 200_000_000, 4, 1.22), + (SYNTHETIC_NS + 300_000_000, 5, 1.23), + ), + ) + group = reconcile_cross_currency_window( + run=run, + window=window, + streams=streams, + config=config, + ) + + assert group.status is CrossCurrencyGroupStatus.RECONCILED + assert group.generation_validation.duplicate_timestamp_event_count == 1 + assert group.generation_validation.union_timestamp_count == 6 + assert group.generation_validation.common_timestamp_count == 3 + assert group.generation_validation.asynchronous_timestamp_count == 3 + assert group.generation_validation.stale_join_risk_count == 2 + support = group.generation_validation.relationship_support[0] + assert support.support_count == 3 + assert len(group.stream_for("EURUSD").events) == 7 + + +def test_inverse_relationship_projects_a_synthetic_leg() -> None: + """The extensible relationship contract also enforces inverse pairs.""" + relationship = CrossCurrencyRelationshipV1.inverse( + left="EURUSD", + right="USDEUR", + projection_priority=("EURUSD", "USDEUR"), + ) + assert relationship.kind is CrossCurrencyRelationshipKind.INVERSE + config = CrossCurrencyReconciliationConfigV1(relationships=(relationship,)) + run = _run(config, symbols=("EURUSD", "USDEUR")) + window = _window(run) + streams = { + "EURUSD": _stream( + run, + symbol="EURUSD", + anchor_midpoint=1.2, + synthetic_midpoint=1.3, + ), + "USDEUR": _stream( + run, + symbol="USDEUR", + anchor_midpoint=1.0 / 1.2, + synthetic_midpoint=0.8, + ), + } + group = reconcile_cross_currency_window( + run=run, + window=window, + streams=streams, + config=config, + ) + assert group.status is CrossCurrencyGroupStatus.RECONCILED + left = next( + event + for event in group.stream_for("EURUSD").events + if event.event_time_ns == SYNTHETIC_NS + ) + right = next( + event + for event in group.stream_for("USDEUR").events + if event.event_time_ns == SYNTHETIC_NS + ) + assert left.bid == pytest.approx(1.0 / right.ask) + assert left.ask == pytest.approx(1.0 / right.bid) + assert ((left.bid + left.ask) / 2.0) * ( + (right.bid + right.ask) / 2.0 + ) == pytest.approx(1.0) + + +def test_existing_cross_instrument_rule_consumes_reconciled_group() -> None: + """The #331 diagnostic validates group rows without a cache roundtrip.""" + _, _, _, group = _reconciled_group() + report = cross_currency_quality_report(group, period="202001") + payload = report.metadata[CROSS_INSTRUMENT_METADATA_KEY] + + assert report.status is QualityStatus.CLEAN + assert payload["triangular_candidate_count"] == 1 + assert payload["triangular_compared_timestamp_count"] == 3 + assert payload["triangular_warning_count"] == 0 + assert payload["triangular_error_count"] == 0 + + +def test_post_broker_validation_and_atomic_manifest_gate() -> None: + """A complete group cannot commit without content-bound final validation.""" + run, window, inputs, group = _reconciled_group() + post_broker = validate_cross_currency_output( + run=run, + window=window, + streams={item.symbol: item for item in group.streams}, + config=group.config, + stage=CrossCurrencyValidationStage.POST_BROKER, + observed_anchors=( + event + for stream in inputs.values() + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + ), + conditions=(_condition(),), + ) + assert post_broker.status is CrossCurrencyValidationStatus.PASSED + manifest = _manifest(window, group) + group.validate_atomic_manifest( + manifest, + post_broker_validation=post_broker, + ) + + direct = group.stream_for("EURGBP") + bad_events = tuple( + ( + replace(event, bid=0.8999, ask=0.9001, event_id="") + if event.event_time_ns == SYNTHETIC_NS + else event + ) + for event in direct.events + ) + bad_streams = { + item.symbol: ( + SyntheticEventStreamV1( + run_id=item.run_id, + ensemble_member_id=item.ensemble_member_id, + symbol=item.symbol, + events=bad_events, + source_version_ids=item.source_version_ids, + ) + if item.symbol == "eurgbp" + else item + ) + for item in group.streams + } + failed = validate_cross_currency_output( + run=run, + window=window, + streams=bad_streams, + config=group.config, + stage=CrossCurrencyValidationStage.POST_BROKER, + observed_anchors=( + event + for stream in inputs.values() + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + ), + ) + assert failed.status is CrossCurrencyValidationStatus.FAILED + with pytest.raises(ValueError, match="passing post-broker"): + validate_cross_currency_atomic_manifest( + window_scope=( + run.run_id, + window.window_id, + window.synchronization_unit_id, + MEMBER, + run.symbols, + ), + streams=tuple(bad_streams.values()), + manifest=manifest, + post_broker_validation=failed, + ) + with pytest.raises(ValueError, match="every synchronized symbol"): + validate_cross_currency_atomic_manifest( + window_scope=( + run.run_id, + window.window_id, + window.synchronization_unit_id, + MEMBER, + run.symbols, + ), + streams=group.streams[:-1], + manifest=manifest, + post_broker_validation=post_broker, + ) + + +def _ref(kind: str, path: str, content: str) -> ArtifactRef: + encoded = content.encode("utf-8") + return ArtifactRef( + kind=kind, + path=path, + size_bytes=len(encoded), + sha256=hashlib.sha256(encoded).hexdigest(), + ) + + +def _manifest( + window: ReconstructionWindowV1, + group: CrossCurrencyReconciledGroupV1, +) -> PartitionManifestV1: + batches = tuple( + EventBatchV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + symbol=stream.symbol, + batch_ordinal=0, + event_count=len(stream.events), + ownership_start_ns=window.core_start_ns, + ownership_end_ns=window.core_end_ns, + first_event_time_ns=window.core_start_ns, + last_event_time_ns=window.core_end_ns - 1, + content_sha256=hashlib.sha256( + stream.to_json().encode("utf-8") + ).hexdigest(), + artifact=_ref( + "synthetic-event-batch", + f"scratch/{stream.symbol}.parquet", + stream.to_json(), + ), + ) + for stream in group.streams + ) + return PartitionManifestV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + symbols=window.symbols, + symbol_event_counts={ + stream.symbol: len(stream.events) for stream in group.streams + }, + event_batches=batches, + rejection_summary_ref=_ref( + "rejection-summary", "scratch/rejections.json", "{}" + ), + carry_state_ref=_ref("carry-state", "scratch/carry.json", "{}"), + ) diff --git a/tests/unit/test_synthetic_ensembles.py b/tests/unit/test_synthetic_ensembles.py new file mode 100644 index 00000000..fb5c82c9 --- /dev/null +++ b/tests/unit/test_synthetic_ensembles.py @@ -0,0 +1,502 @@ +"""Tests for calibrated reconstruction-ensemble contracts.""" + +from __future__ import annotations + +from dataclasses import replace +import hashlib + +import pytest + +from histdatacom.synthetic import ( + ENSEMBLE_CALIBRATION_METRIC_NAMES, + ENSEMBLE_CONFIDENCE_QUANTITY, + BenchmarkCandidateWindowV1, + BenchmarkEventV1, + BenchmarkExecutionEvidenceV1, + BenchmarkScenarioV1, + BenchmarkSplitKind, + EnsembleCalibrationConfigV1, + EnsembleCalibrationReportV1, + EnsembleCalibrationSampleV1, + EnsembleCalibrationStratumV1, + EnsembleDiversityStatus, + EnsembleMemberCalibrationV1, + EnsembleMemberStatus, + EnsembleReportStatus, + EnsembleStorageEstimateV1, + ReconstructionEnsemblePlanV1, + ReconstructionResourceLimitError, + benchmark_ensemble_calibration_sample, + benchmark_logical_content_sha256, + build_ensemble_regeneration_request, + calibrate_reconstruction_ensemble, + estimate_reconstruction_ensemble_resources, + plan_reconstruction_ensemble, + verify_ensemble_regeneration, +) + +HORIZON_NS = 60_000_000_000 +BASE_METRICS = { + "event_count": 100.0, + "observed_duration_ns": 60_000_000_000.0, + "mean_interarrival_ns": 600_000_000.0, + "mean_spread": 0.0002, + "mid_path_range": 0.002, + "endpoint_mid": 1.1, + "downstream_sensitivity": 0.5, +} + + +def _sha(label: str) -> str: + return hashlib.sha256(label.encode()).hexdigest() + + +def _config(**changes: object) -> EnsembleCalibrationConfigV1: + values: dict[str, object] = { + "member_count": 3, + "retained_member_count": 2, + "horizons_ns": (HORIZON_NS,), + "minimum_fit_samples": 2, + "nominal_coverage": 0.8, + "minimum_achieved_coverage": 0.5, + } + values.update(changes) + return EnsembleCalibrationConfigV1(**values) # type: ignore[arg-type] + + +def _plan( + *, + config: EnsembleCalibrationConfigV1 | None = None, +) -> ReconstructionEnsemblePlanV1: + return plan_reconstruction_ensemble( + symbols=("eurusd",), + source_artifact_hashes={"source-eurusd": _sha("source-eurusd")}, + configuration_artifact_hashes={"generator-v1": _sha("generator-v1")}, + base_seed=20260713, + config=config or _config(), + ) + + +def _metrics(scale: float) -> dict[str, float]: + return {name: value * scale for name, value in BASE_METRICS.items()} + + +def _stratum() -> EnsembleCalibrationStratumV1: + return EnsembleCalibrationStratumV1( + epoch_id="modern-dense", + session="london", + event_state="ordinary", + symbol="eurusd", + horizon_ns=HORIZON_NS, + sparsity="sparse-20pct", + ) + + +def _sample( + plan: ReconstructionEnsemblePlanV1, + *, + split: BenchmarkSplitKind, + ordinal: int, + collapsed: bool = False, + false_diversity: bool = False, +) -> EnsembleCalibrationSampleV1: + members = [] + for index, member in enumerate(plan.members): + scale = 1.0 if false_diversity else (0.9 + index * 0.1) + digest_label = ( + f"shared-{split.value}-{ordinal}" + if collapsed + else f"{member.member_id}-{split.value}-{ordinal}" + ) + members.append( + EnsembleMemberCalibrationV1( + member_id=member.member_id, + status=EnsembleMemberStatus.COMPLETED, + metrics=_metrics(scale), + logical_content_sha256=_sha(digest_label), + ) + ) + return EnsembleCalibrationSampleV1( + benchmark_manifest_id="benchmark-manifest-v1", + scenario_id=f"scenario-{split.value}-{ordinal}", + candidate_id="empirical-motif-v1", + window_id=f"window-{split.value}-{ordinal}", + split_kind=split, + stratum=_stratum(), + reference_metrics=_metrics(1.0), + reference_content_sha256=_sha(f"reference-{split.value}-{ordinal}"), + members=tuple(members), + ) + + +def _calibrated_report( + plan: ReconstructionEnsemblePlanV1, +) -> tuple[EnsembleCalibrationReportV1, EnsembleStorageEstimateV1]: + counts = {member.member_id: 100 for member in plan.members} + estimate = estimate_reconstruction_ensemble_resources( + plan, + input_event_count=100, + member_event_counts=counts, + ) + samples = ( + _sample(plan, split=BenchmarkSplitKind.VALIDATION, ordinal=1), + _sample(plan, split=BenchmarkSplitKind.VALIDATION, ordinal=2), + _sample(plan, split=BenchmarkSplitKind.FINAL_HOLDOUT, ordinal=1), + ) + return ( + calibrate_reconstruction_ensemble( + plan, + samples=samples, + storage_estimate=estimate, + ), + estimate, + ) + + +def _event( + source_id: str, + *, + time_ns: int, + sequence: int, + mid: float, + member_id: str | None = None, +) -> BenchmarkEventV1: + return BenchmarkEventV1( + source_event_id=source_id, + symbol="eurusd", + event_time_ns=time_ns, + event_sequence=sequence, + bid=mid - 0.0001, + ask=mid + 0.0001, + epoch_id="modern-dense", + session="london", + event_state="ordinary", + sparsity="sparse-20pct", + ensemble_member_id=member_id, + ) + + +def test_plan_is_deterministic_hash_bound_and_round_trips() -> None: + first = _plan() + second = _plan() + + assert first == second + assert first.plan_id == second.plan_id + assert len({item.member_id for item in first.members}) == 3 + assert len({item.seed for item in first.members}) == 3 + assert ReconstructionEnsemblePlanV1.from_json(first.to_json()) == first + + with pytest.raises(ValueError, match="config hash differs"): + plan_reconstruction_ensemble( + symbols=("eurusd",), + source_artifact_hashes={"source-eurusd": _sha("source-eurusd")}, + configuration_artifact_hashes={ + first.config.config_id: _sha("tampered-config") + }, + base_seed=20260713, + config=first.config, + ) + + +def test_resource_preflight_accounts_for_all_members_and_retained_quota() -> ( + None +): + plan = _plan() + counts = { + plan.members[0].member_id: 10, + plan.members[1].member_id: 20, + plan.members[2].member_id: 30, + } + estimate = estimate_reconstruction_ensemble_resources( + plan, + input_event_count=10, + member_event_counts=counts, + ) + + assert estimate.conservative_retained_event_count == 50 + assert estimate.resource_estimate.candidate_event_count == 60 + assert estimate.resource_estimate.estimated_output_bytes == ( + 50 * plan.config.estimated_bytes_per_event + ) + assert EnsembleStorageEstimateV1.from_json(estimate.to_json()) == estimate + with pytest.raises(ValueError, match="resource estimate arithmetic"): + replace( + estimate, + resource_estimate=replace( + estimate.resource_estimate, + estimated_output_bytes=1, + estimate_id="", + ), + estimate_id="", + ) + + with pytest.raises(ReconstructionResourceLimitError, match="amplification"): + estimate_reconstruction_ensemble_resources( + plan, + input_event_count=1, + member_event_counts=counts, + ) + + +def test_reverse_degradation_adapter_is_compact_and_records_failures() -> None: + plan = _plan() + scenario = BenchmarkScenarioV1( + split_kind=BenchmarkSplitKind.VALIDATION, + epoch_id="modern-dense", + severity_id="sparse-20pct", + observation_operator_id="operator-v1", + degradation_parameters={"retention_rate": 0.2}, + ) + reference = tuple( + _event( + f"reference-{index}", time_ns=index * 1_000, sequence=index, mid=1.1 + ) + for index in range(1, 4) + ) + windows = [] + for index, member in enumerate(plan.members): + execution = BenchmarkExecutionEvidenceV1( + attempted=True, + converged=index != 2, + failure_reason="worker_failed" if index == 2 else None, + ) + events = tuple( + _event( + f"candidate-{index}-{event_index}", + time_ns=event_index * 1_000, + sequence=event_index, + mid=1.1 + index * 0.0001, + member_id=member.member_id, + ) + for event_index in range(1, 4) + ) + windows.append( + BenchmarkCandidateWindowV1( + scenario_id=scenario.scenario_id, + candidate_id="empirical-motif-v1", + window_id="window-1", + ensemble_member_id=member.member_id, + events=events, + execution=execution, + hard_constraint_violations=( + {"negative_spread": 1} if index == 1 else {} + ), + strategy_hooks={"downstream_sensitivity": 0.5}, + ) + ) + + sample = benchmark_ensemble_calibration_sample( + plan, + benchmark_manifest_id="benchmark-manifest-v1", + scenario=scenario, + candidate_id="empirical-motif-v1", + window_id="window-1", + horizon_ns=HORIZON_NS, + reference_events=reference, + member_windows=windows, + reference_downstream_sensitivity=0.5, + ) + + assert sorted(item.status.value for item in sample.members) == [ + "completed", + "failed", + "refused", + ] + assert sample.stratum.to_dict().keys() >= { + "epoch_id", + "session", + "event_state", + "symbol", + "horizon_ns", + "sparsity", + } + assert sample.to_dict()["event_rows_inline"] is False + assert EnsembleCalibrationSampleV1.from_json(sample.to_json()) == sample + assert benchmark_logical_content_sha256(reference) == ( + benchmark_logical_content_sha256(reversed(reference)) + ) + reidentified = tuple( + replace( + event, + source_event_id=f"retry-or-seed-{index}", + ensemble_member_id="different-member", + benchmark_event_id="", + ) + for index, event in enumerate(reference) + ) + assert benchmark_logical_content_sha256(reference) == ( + benchmark_logical_content_sha256(reidentified) + ) + + +def test_calibration_reports_defined_coverage_and_no_automatic_winner() -> None: + plan = _plan() + report, _ = _calibrated_report(plan) + + assert report.status is EnsembleReportStatus.CALIBRATED + assert report.candidate_id == "empirical-motif-v1" + assert report.primary_member_id in report.retained_member_ids + assert len(report.retained_member_ids) == 2 + assert len(report.regenerable_member_ids) == 1 + assert report.automatic_winner is False + assert report.default_generator_id is None + assert len(report.metric_calibrations) == len( + ENSEMBLE_CALIBRATION_METRIC_NAMES + ) + assert all( + item.calibrated_coverage_rate == 1.0 + for item in report.metric_calibrations + ) + payload = report.to_dict() + assert payload["confidence_quantity"] == ENSEMBLE_CONFIDENCE_QUANTITY + assert payload["confidence_scope"] == ( + "stratum_metric_horizon_summary_not_per_event" + ) + assert payload["primary_interpretation"] == ( + "representative_member_not_historical_truth" + ) + assert payload["winner_member_id"] is None + assert payload["event_rows_inline"] is False + assert EnsembleCalibrationReportV1.from_json(report.to_json()) == report + with pytest.raises(ValueError, match="status differs from evidence"): + replace( + report, + status=EnsembleReportStatus.MISCALIBRATED, + report_id="", + ) + + +def test_every_configured_horizon_requires_calibration_evidence() -> None: + plan = _plan(config=_config(horizons_ns=(HORIZON_NS, HORIZON_NS * 2))) + estimate = estimate_reconstruction_ensemble_resources( + plan, + input_event_count=100, + member_event_counts={item.member_id: 100 for item in plan.members}, + ) + + with pytest.raises(ValueError, match="configured horizons"): + calibrate_reconstruction_ensemble( + plan, + samples=( + _sample( + plan, + split=BenchmarkSplitKind.VALIDATION, + ordinal=1, + ), + _sample( + plan, + split=BenchmarkSplitKind.VALIDATION, + ordinal=2, + ), + _sample( + plan, + split=BenchmarkSplitKind.FINAL_HOLDOUT, + ordinal=1, + ), + ), + storage_estimate=estimate, + ) + + +@pytest.mark.parametrize( # type: ignore[untyped-decorator] + ("collapsed", "false_diversity", "expected"), + ( + (True, False, EnsembleDiversityStatus.COLLAPSED), + (False, True, EnsembleDiversityStatus.FALSE_DIVERSITY), + ), +) +def test_non_substantive_member_diversity_blocks_calibrated_status( + collapsed: bool, + false_diversity: bool, + expected: EnsembleDiversityStatus, +) -> None: + plan = _plan() + estimate = estimate_reconstruction_ensemble_resources( + plan, + input_event_count=100, + member_event_counts={item.member_id: 100 for item in plan.members}, + ) + samples = tuple( + _sample( + plan, + split=split, + ordinal=ordinal, + collapsed=collapsed, + false_diversity=false_diversity, + ) + for split, ordinal in ( + (BenchmarkSplitKind.VALIDATION, 1), + (BenchmarkSplitKind.VALIDATION, 2), + (BenchmarkSplitKind.FINAL_HOLDOUT, 1), + ) + ) + + report = calibrate_reconstruction_ensemble( + plan, + samples=samples, + storage_estimate=estimate, + ) + + assert report.status is EnsembleReportStatus.MISCALIBRATED + assert {item.status for item in report.diversity_summaries} == {expected} + + +def test_regeneration_requires_calibration_omission_and_exact_hashes() -> None: + plan = _plan() + report, _ = _calibrated_report(plan) + omitted = report.regenerable_member_ids + request = build_ensemble_regeneration_request( + plan, + report, + member_ids=omitted, + ) + assert ( + tuple( + item.member_id + for item in verify_ensemble_regeneration( + plan, + report, + request, + available_source_artifact_hashes=plan.source_hashes, + available_configuration_artifact_hashes=( + plan.configuration_hashes + ), + ) + ) + == omitted + ) + with pytest.raises(ValueError, match="retained/unknown"): + verify_ensemble_regeneration( + plan, + report, + build_ensemble_regeneration_request( + plan, + report, + member_ids=(report.retained_member_ids[0],), + ), + available_source_artifact_hashes=plan.source_hashes, + available_configuration_artifact_hashes=(plan.configuration_hashes), + ) + tampered = replace( + request, + source_artifact_hashes={"source-eurusd": _sha("different-source")}, + request_id="", + ) + with pytest.raises(ValueError, match="source hashes differ"): + verify_ensemble_regeneration( + plan, + report, + tampered, + available_source_artifact_hashes=plan.source_hashes, + available_configuration_artifact_hashes=(plan.configuration_hashes), + ) + with pytest.raises(ValueError, match="available regeneration source"): + verify_ensemble_regeneration( + plan, + report, + request, + available_source_artifact_hashes={ + "source-eurusd": _sha("different-source") + }, + available_configuration_artifact_hashes=(plan.configuration_hashes), + ) diff --git a/tests/unit/test_synthetic_generation.py b/tests/unit/test_synthetic_generation.py new file mode 100644 index 00000000..359a5f16 --- /dev/null +++ b/tests/unit/test_synthetic_generation.py @@ -0,0 +1,1915 @@ +"""Tests for variable-cardinality empirical-motif candidate generation.""" + +from __future__ import annotations + +from dataclasses import replace +import hashlib +import json + +import pytest + +from histdatacom.data_quality.synthetic_constraints import ( + SYNTHETIC_VALIDATION_SCHEMA_VERSION, +) +from histdatacom.market_context import ( + MarketContextCalendarStateV1, + MarketContextMissingReason, + MarketContextQueryStatus, + MarketContextQueryV1, + MarketContextView, +) +from histdatacom.runtime_contracts import ArtifactRef +from histdatacom.orchestration.reconstruction import artifact_ref_for_file +from histdatacom.synthetic import ( + CANDIDATE_ONLY_CONSTRAINT_SET_ID, + EMPIRICAL_MOTIF_GENERATOR_ID, + MOTIF_TRANSFORMATION_CONFIDENCE_QUANTITY, + BenchmarkCandidateKind, + BenchmarkCandidateV1, + BenchmarkControlKind, + BenchmarkEventV1, + BenchmarkExecutionEvidenceV1, + BenchmarkProfileV1, + BenchmarkScenarioV1, + BenchmarkSplitKind, + BenchmarkSplitV1, + CarvingBatchStatus, + CarvingFingerprintEvidenceV1, + CarvingReason, + EmpiricalMotifBenchmarkGeneratorV1, + EmpiricalMotifGeneratorConfigV1, + HistoricalCarvedCandidateBatchV1, + HistoricalCarvingConditionPolicyV1, + HistoricalCarvingConstraintSetV1, + HistoricalCarvingQuarantineV1, + InformationMode, + MotifGenerationDecision, + MotifGenerationStatus, + ReferenceMotifConditionV1, + ReferenceMotifIndexConfigV1, + ReferenceMotifQueryV1, + ReferenceMotifSourceEventV1, + ReferenceMotifSourceWindowV1, + ReferenceMotifSplitKind, + ReferenceMotifSplitV1, + ReferenceMotifTransformPolicyV1, + ReconstructionRunV1, + ReconstructionStoragePolicyV1, + ReconstructionWindowV1, + ReverseDegradationBenchmarkManifestV1, + ReverseDegradationBenchmarkV1, + SyntheticEventOrigin, + SyntheticEventStreamV1, + SyntheticEventV1, + build_benchmark_control_windows, + build_reference_motif_index, + carve_empirical_motif_candidates, + generate_benchmark_candidate_window, + generate_empirical_motif_candidates, + query_reference_motifs, + write_synthetic_event_stream_parquet, +) +from histdatacom.synthetic.reconstruction_handlers import ( + _candidate_evidence, + _proposal_synchronization_event_time, + _restore_candidate_batches, + _source_row_order_key, +) + +BASE_NS = 1_700_000_000_000_000_000 +SECOND = 1_000_000_000 +SOURCE_VERSION_ID = "source-version:fixture-v1" +MEMBER_ID = "member-a" + + +def _digest(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _condition( + *, + feed_epoch_id: str = "feed-epoch:modern-v1", + intensity: float = 8.0, + interarrival_ns: float | None = None, + timestamp_precision_ns: float = 1_000_000.0, + session_state: str = "active", + special_tags: tuple[str, ...] = ("ordinary",), + event_tags: tuple[str, ...] = (), +) -> ReferenceMotifConditionV1: + cadence = interarrival_ns or SECOND / intensity if intensity else SECOND + return ReferenceMotifConditionV1( + symbol="EURUSD", + feed_epoch_id=feed_epoch_id, + session_state=session_state, + currencies=("EUR", "USD"), + active_sessions=("london",), + special_tags=special_tags, + event_tags=event_tags, + return_regime="small-positive", + range_regime="normal", + volatility_regime="normal", + spread_regime="tight", + activity_regime="active" if intensity else "inactive", + interarrival_regime="dense" if intensity >= 8 else "sparse", + timestamp_precision="millisecond", + price_precision="five-decimal", + source_quality_state="eligible", + metrics={ + "return_value": 0.0004, + "range_value": 0.0015, + "volatility": 0.001, + "spread": 0.0002, + "tick_intensity": intensity, + "interarrival_ns": cadence, + "timestamp_precision_ns": timestamp_precision_ns, + "price_precision_digits": 5.0, + "source_quality_score": 1.0, + }, + ) + + +def _splits() -> tuple[ReferenceMotifSplitV1, ...]: + return ( + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.TRAIN, + BASE_NS - 100 * SECOND, + BASE_NS - 50 * SECOND, + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.CALIBRATION, + BASE_NS + 10 * SECOND, + BASE_NS + 20 * SECOND, + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.VALIDATION, + BASE_NS + 30 * SECOND, + BASE_NS + 40 * SECOND, + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.FINAL_HOLDOUT, + BASE_NS + 50 * SECOND, + BASE_NS + 60 * SECOND, + ), + ) + + +def _source_window( + condition: ReferenceMotifConditionV1, + *, + event_offsets_ns: tuple[int, ...] = ( + 0, + 125_000_000, + 250_000_000, + ), + quotes: tuple[tuple[float, float], ...] = ( + (1.1000, 1.1002), + (1.1001, 1.1003), + (1.10005, 1.10025), + ), + allow_spread_scaling: bool = False, + name: str = "ordinary", +) -> ReferenceMotifSourceWindowV1: + start = BASE_NS - 90 * SECOND + events = tuple( + ReferenceMotifSourceEventV1( + event_time_ns=start + offset, + event_sequence=index, + bid=bid, + ask=ask, + source_row_id=index, + ) + for index, (offset, (bid, ask)) in enumerate( + zip(event_offsets_ns, quotes), start=1 + ) + ) + artifact = ArtifactRef( + kind="augmented-tick-partition", + path=f"artifacts/{name}.data", + size_bytes=4_096, + sha256=_digest(name), + metadata={"projection": "motif-source-v1"}, + ) + return ReferenceMotifSourceWindowV1( + source_series_id=f"ascii:T:EURUSD:histdata.com:{name}", + period="202001", + source_artifact=artifact, + split_kind=ReferenceMotifSplitKind.TRAIN, + condition=condition, + events=events, + first_known_at_ns=events[-1].event_time_ns, + available_at_ns=events[-1].event_time_ns, + transform_policy=ReferenceMotifTransformPolicyV1( + min_time_scale=0.5, + max_time_scale=2.0, + min_price_scale=0.5, + max_price_scale=2.0, + max_time_warp_ratio=1.25, + allow_spread_scaling=allow_spread_scaling, + ), + ) + + +def _index( + condition: ReferenceMotifConditionV1, + **window_kwargs: object, +): + return build_reference_motif_index( + (_source_window(condition, **window_kwargs),), + splits=_splits(), + config=ReferenceMotifIndexConfigV1(min_cell_support=1), + ) + + +def _result( + index, + condition: ReferenceMotifConditionV1, + *, + minimum_support: int = 1, + used_at_ns: int = BASE_NS + SECOND, +): + return query_reference_motifs( + index, + ReferenceMotifQueryV1( + condition=condition, + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + used_at_ns=used_at_ns, + min_cell_support=minimum_support, + ), + ) + + +def _run( + config: EmpiricalMotifGeneratorConfigV1, + *, + members: tuple[str, ...] = (MEMBER_ID,), + storage_policy: ReconstructionStoragePolicyV1 | None = None, + additional_configuration_ids: tuple[str, ...] = (), +) -> ReconstructionRunV1: + return ReconstructionRunV1( + symbols=("EURUSD",), + source_version_ids=(SOURCE_VERSION_ID,), + configuration_ids=(config.config_id, *additional_configuration_ids), + ensemble_member_ids=members, + base_seed=7, + storage_policy=storage_policy or ReconstructionStoragePolicyV1(), + ) + + +def _anchor( + run: ReconstructionRunV1, + event_time_ns: int, + *, + sequence: int, + row_id: int, + member: str = MEMBER_ID, + bid: float = 1.1000, + ask: float = 1.1002, +) -> SyntheticEventV1: + return SyntheticEventV1.observed( + symbol="EURUSD", + event_time_ns=event_time_ns, + event_sequence=sequence, + bid=bid, + ask=ask, + run_id=run.run_id, + ensemble_member_id=member, + source_version_id=SOURCE_VERSION_ID, + source_series_id="ascii:T:EURUSD:histdata.com", + source_period="202001", + source_row_id=row_id, + ) + + +def _window( + run: ReconstructionRunV1, + start_ns: int, + end_ns: int, + *, + member: str = MEMBER_ID, + left_halo_ns: int = 0, + right_lookahead_ns: int = 0, +) -> ReconstructionWindowV1: + return ReconstructionWindowV1( + run_id=run.run_id, + ensemble_member_id=member, + symbols=("EURUSD",), + core_start_ns=start_ns, + core_end_ns=end_ns, + left_halo_ns=left_halo_ns, + right_lookahead_ns=right_lookahead_ns, + ) + + +def _market_context( + window: ReconstructionWindowV1, + *, + session_state: str = "active", + special_tags: tuple[str, ...] = ("ordinary",), + event_tags: tuple[str, ...] = (), + profile_complete: bool = True, + missing_reason: MarketContextMissingReason = ( + MarketContextMissingReason.NO_MATCHING_EVENT + ), +) -> MarketContextQueryV1: + calendar = MarketContextCalendarStateV1( + timestamp_utc_ns=window.core_start_ns, + session_state=session_state, + clock_sessions=("london",), + active_sessions=("london",) if session_state == "active" else (), + overlaps=(), + special_tags=special_tags, + holiday_tags=(), + event_tags=event_tags, + calendar_tags=(*special_tags, *event_tags), + profile_source="calendar-profile:fixture", + profile_version="1.0.0", + profile_complete=profile_complete, + limitations=("deterministic unit-test fixture",), + ) + return MarketContextQueryV1( + timeline_id="market-context-timeline:fixture-v1", + view=MarketContextView.EX_POST, + start_ns=window.core_start_ns, + end_ns=window.core_end_ns, + as_of_ns=None, + events=(), + status=MarketContextQueryStatus.MISSING, + missing_reason=missing_reason, + calendar_state=calendar, + requested_symbols=("EURUSD",), + window_id=window.window_id, + limitations=("deterministic unit-test fixture",), + ) + + +def _fingerprint_evidence( + *batches, + status: str = "match", +) -> CarvingFingerprintEvidenceV1: + return CarvingFingerprintEvidenceV1( + validation_payload={ + "schema_version": SYNTHETIC_VALIDATION_SCHEMA_VERSION, + "status": status, + "advisory": True, + "compared_target_count": 1, + "matching_target_count": 1 if status == "match" else 0, + "mismatched_target_count": 0 if status == "match" else 1, + }, + candidate_batch_ids=tuple(item.batch_id for item in batches), + reference_report_id="quality-report:reference-fixture", + candidate_report_id="quality-report:candidate-fixture", + ) + + +def _carving_inputs( + constraints: HistoricalCarvingConstraintSetV1, + *, + condition: ReferenceMotifConditionV1 | None = None, + source_name: str = "ordinary", +): + selected_condition = condition or _condition() + index = _index(selected_condition, name=source_name) + config = EmpiricalMotifGeneratorConfigV1() + run = _run( + config, + additional_configuration_ids=(constraints.constraint_set_id,), + ) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + window = _window(run, BASE_NS, BASE_NS + SECOND + 1) + result = _result(index, selected_condition) + batch = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=result, + config=config, + ) + return config, run, left, right, window, result, batch + + +@pytest.mark.parametrize( + ("intensity", "expected_count"), + ((2.0, 1), (4.0, 3), (8.0, 7), (16.0, 15)), +) +def test_cardinality_tracks_conditioned_tick_intensity( + intensity: float, + expected_count: int, +) -> None: + source_condition = _condition() + target_condition = _condition(intensity=intensity) + index = _index(source_condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor( + run, + BASE_NS + SECOND, + sequence=99, + row_id=2, + bid=1.1004, + ask=1.1006, + ) + + batch = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, target_condition), + config=config, + ) + + assert batch.status is MotifGenerationStatus.GENERATED + assert batch.target_event_count == expected_count + assert len(batch.events) == expected_count + assert all( + event.constraint_set_id == CANDIDATE_ONLY_CONSTRAINT_SET_ID + for event in batch.events + ) + assert all(event.confidence is None for event in batch.events) + + +def test_required_cross_series_time_survives_empirical_motif_warp() -> None: + condition = _condition(intensity=8.0) + index = _index(condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + required = BASE_NS + 333_000_000 + + batch = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + required_event_time_ns=required, + ) + replay = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + required_event_time_ns=required, + ) + + assert required in {event.event_time_ns for event in batch.events} + assert replay.batch_id == batch.batch_id + assert replay.events == batch.events + + with pytest.raises(ValueError, match="inside the anchor interval"): + generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + required_event_time_ns=right.event_time_ns, + ) + assert all(0.0 <= item.confidence <= 1.0 for item in batch.transformations) + assert MOTIF_TRANSFORMATION_CONFIDENCE_QUANTITY == ( + "uncalibrated-motif-match-similarity-v1" + ) + + +def test_proposal_query_evidence_is_compact_and_restores_incrementally( + tmp_path, +) -> None: + condition = _condition() + index = _index(condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + result = _result(index, condition) + batch = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=result, + config=config, + ) + evidence = _candidate_evidence(batch) + candidate_path = tmp_path / "candidates.parquet" + write_synthetic_event_stream_parquet( + SyntheticEventStreamV1( + run_id=run.run_id, + ensemble_member_id=MEMBER_ID, + symbol=batch.symbol, + events=batch.events, + source_version_ids=run.source_version_ids, + ), + candidate_path, + ) + candidate_ref = artifact_ref_for_file( + candidate_path, kind="reconstruction_candidate_stream_v1" + ) + ledger_path = tmp_path / "candidate-batches.ndjson" + ledger_path.write_text( + json.dumps(evidence, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + ledger_ref = artifact_ref_for_file( + ledger_path, + kind="reconstruction_candidate_batch_ledger_v1", + metadata={ + "batch_count": 1, + "format": "canonical-json-lines-v1", + }, + ) + manifest = { + "batch_ledger_ref": ledger_ref.to_dict(), + "batches_inline": False, + "batch_count": 1, + "candidate_stream_refs": {batch.symbol: candidate_ref.to_dict()}, + "generator_config": config.to_dict(), + "query_conditions": {batch.symbol: condition.to_dict()}, + "motif_index_id": index.index_id, + } + + restored = tuple(_restore_candidate_batches(manifest, index=index)) + + assert restored == (batch,) + assert "query_result" not in evidence + assert evidence["query_evidence"]["retrieval_rows_inline"] is False + assert len(json.dumps(evidence["query_evidence"])) < len(result.to_json()) + + +def test_source_duplicate_order_uses_partition_row_not_quote_value() -> None: + """Equal timestamps retain immutable Arrow order across quote changes.""" + later_row_with_lower_bid = ( + BASE_NS, + 1.10, + 1.11, + "201101", + 8, + "ascii-tick:eurgbp:201101", + ) + earlier_row_with_higher_bid = ( + BASE_NS, + 1.20, + 1.21, + "201101", + 7, + "ascii-tick:eurgbp:201101", + ) + + ordered = sorted( + (later_row_with_lower_bid, earlier_row_with_higher_bid), + key=_source_row_order_key, + ) + + assert ordered == [ + earlier_row_with_higher_bid, + later_row_with_lower_bid, + ] + + +def test_triangle_synchronization_time_uses_sparsest_aligned_observed_anchor() -> ( + None +): + config = EmpiricalMotifGeneratorConfigV1() + run = ReconstructionRunV1( + symbols=("EURUSD", "GBPUSD", "EURGBP"), + source_version_ids=(SOURCE_VERSION_ID,), + configuration_ids=(config.config_id,), + ensemble_member_ids=(MEMBER_ID,), + base_seed=7, + ) + precision = 250_000_000 + feed_phase = 243_000_000 + streams: dict[str, SyntheticEventStreamV1] = {} + conditions: dict[str, ReferenceMotifConditionV1] = {} + for symbol, offsets_ns in { + "EURUSD": (143_000_000, 643_000_000, 1_143_000_000), + "GBPUSD": (193_000_000, 693_000_000, 1_193_000_000), + "EURGBP": (feed_phase, 743_000_000, 1_243_000_000), + }.items(): + events = tuple( + SyntheticEventV1.observed( + symbol=symbol, + event_time_ns=BASE_NS + offset_ns, + event_sequence=0, + bid=1.1, + ask=1.1002, + run_id=run.run_id, + ensemble_member_id=MEMBER_ID, + source_version_id=SOURCE_VERSION_ID, + source_series_id=f"ascii:T:{symbol}:histdata.com", + source_period="202001", + source_row_id=row_id, + ) + for row_id, offset_ns in enumerate(offsets_ns, start=1) + ) + streams[symbol] = SyntheticEventStreamV1.merge( + run_id=run.run_id, + ensemble_member_id=MEMBER_ID, + symbol=symbol, + observed_events=events, + synthetic_events=(), + source_version_ids=run.source_version_ids, + ) + conditions[symbol] = replace( + _condition( + intensity=1.0 if symbol == "EURGBP" else 8.0, + timestamp_precision_ns=float(precision), + ), + symbol=symbol, + ) + + selected = _proposal_synchronization_event_time( + streams, + conditions=conditions, + start_ns=BASE_NS, + end_ns=BASE_NS + 1_500_000_000, + ) + + assert selected == BASE_NS + 743_000_000 + assert selected % precision == feed_phase + assert ( + sum( + selected in {event.event_time_ns for event in stream.events} + for stream in streams.values() + ) + == 1 + ) + assert selected in { + event.event_time_ns for event in streams["EURGBP"].events + } + + +def test_cardinality_falls_back_to_conditioned_interarrival_cadence() -> None: + source_condition = _condition() + target_condition = _condition(intensity=1.0, interarrival_ns=200_000_000.0) + target_condition = replace( + target_condition, + metrics={ + key: value + for key, value in target_condition.metrics.items() + if key != "tick_intensity" + }, + ) + index = _index(source_condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + + batch = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, target_condition), + config=config, + ) + + assert batch.status is MotifGenerationStatus.GENERATED + assert batch.target_event_count == 4 + assert [item.event_time_ns - BASE_NS for item in batch.events] == [ + 200_000_000, + 400_000_000, + 600_000_000, + 800_000_000, + ] + + +def test_empirical_clock_and_quote_transition_marks_are_materialized() -> None: + condition = _condition(intensity=8.0) + index = _index( + condition, + event_offsets_ns=(0, 50_000_000, 250_000_000), + quotes=( + (1.10000, 1.10020), + (1.10010, 1.10020), + (1.10010, 1.10030), + ), + ) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor( + run, + BASE_NS + SECOND, + sequence=99, + row_id=2, + bid=1.1004, + ask=1.1006, + ) + + first = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + ) + replay = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + ) + + assert first.events == replay.events + gaps = [ + right_time - left_time + for left_time, right_time in zip( + ( + left.event_time_ns, + *(item.event_time_ns for item in first.events), + ), + ( + *(item.event_time_ns for item in first.events), + right.event_time_ns, + ), + ) + ] + assert len(set(gaps)) > 1 + stream = (left, *first.events, right) + transitions = { + ( + current.bid != previous.bid, + current.ask != previous.ask, + ) + for previous, current in zip(stream, stream[1:]) + } + assert (True, False) in transitions + assert (False, True) in transitions + + +def test_lineage_anchor_seams_and_merged_observations_are_preserved() -> None: + condition = _condition() + index = _index(condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor( + run, + BASE_NS + SECOND, + sequence=99, + row_id=2, + bid=1.1004, + ask=1.1006, + ) + batch = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + ) + stream = batch.merged_stream((left, right)) + + observed = tuple( + event + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + ) + assert observed == (left, right) + assert [event.to_dict() for event in observed] == [ + left.to_dict(), + right.to_dict(), + ] + assert stream.observed_event_count == 2 + assert stream.synthetic_event_count == 7 + assert all( + left.event_time_ns < item.event_time_ns < right.event_time_ns + for item in batch.events + ) + assert all(item.bid > 0.0 and item.ask >= item.bid for item in batch.events) + assert len(batch.event_lineage) == len(batch.events) + assert all( + batch.lineage_for(item.event_id).global_event_ordinal > 0 + for item in batch.events + ) + assert all(item.time_warp_ratio <= 1.25 for item in batch.transformations) + event_by_ordinal = {item.event_sequence: item for item in batch.events} + left_mid = (left.bid + left.ask) / 2.0 + right_mid = (right.bid + right.ask) / 2.0 + left_spread = left.ask - left.bid + right_spread = right.ask - right.bid + for transform in batch.transformations: + seam = event_by_ordinal[transform.output_end_ordinal] + progress = (seam.event_time_ns - left.event_time_ns) / ( + right.event_time_ns - left.event_time_ns + ) + expected_mid = left_mid + progress * (right_mid - left_mid) + expected_spread = left_spread + progress * (right_spread - left_spread) + assert seam.bid == round(expected_mid - expected_spread / 2.0, 5) + assert seam.ask == round(expected_mid + expected_spread / 2.0, 5) + assert EmpiricalMotifGeneratorConfigV1.from_json(config.to_json()) == config + assert all( + type(item).from_json(item.to_json()) == item + for item in batch.transformations + ) + assert all( + type(item).from_json(item.to_json()) == item + for item in batch.event_lineage + ) + metadata = batch.metadata() + assert metadata["candidate_only"] is True + assert metadata["hard_carving_status"] == "not_evaluated" + assert metadata["broker_conditioning_status"] == "not_applied" + assert metadata["final_storage_status"] == "not_persisted" + assert metadata["generator_config"] == config.to_dict() + assert metadata["transformations_inline"] is False + assert "transformations" not in metadata + assert len(metadata["event_content_sha256"]) == 64 + + changed_event = replace( + batch.events[0], + bid=batch.events[0].bid + 0.00001, + ask=batch.events[0].ask + 0.00001, + ) + changed_batch = replace( + batch, + events=(changed_event, *batch.events[1:]), + batch_id="", + ) + assert changed_batch.batch_id != batch.batch_id + with pytest.raises(ValueError, match="event-to-transform lineage"): + replace( + batch, + event_lineage=( + replace( + batch.event_lineage[0], + requested_event_time_ns=( + batch.event_lineage[0].requested_event_time_ns + 1 + ), + ), + *batch.event_lineage[1:], + ), + batch_id="", + ) + + +def test_retry_and_partitioning_do_not_change_candidate_identity_or_values() -> ( + None +): + condition = _condition() + index = _index(condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config, members=(MEMBER_ID, "member-b")) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor( + run, + BASE_NS + SECOND, + sequence=99, + row_id=2, + bid=1.1004, + ask=1.1006, + ) + result = _result(index, condition) + whole_window = _window(run, BASE_NS, BASE_NS + SECOND + 1) + whole = generate_empirical_motif_candidates( + run=run, + window=whole_window, + left_anchor=left, + right_anchor=right, + query_result=result, + config=config, + ) + retry = generate_empirical_motif_candidates( + run=run, + window=whole_window, + left_anchor=left, + right_anchor=right, + query_result=result, + config=config, + ) + edge = BASE_NS + SECOND // 2 + first = generate_empirical_motif_candidates( + run=run, + window=_window( + run, + BASE_NS, + edge, + right_lookahead_ns=BASE_NS + SECOND + 1 - edge, + ), + left_anchor=left, + right_anchor=right, + query_result=result, + config=config, + ) + second = generate_empirical_motif_candidates( + run=run, + window=_window( + run, + edge, + BASE_NS + SECOND + 1, + left_halo_ns=edge - BASE_NS, + ), + left_anchor=left, + right_anchor=right, + query_result=result, + config=config, + ) + + assert whole.events == retry.events + partitioned = tuple( + sorted( + (*first.events, *second.events), + key=lambda item: (item.event_time_ns, item.event_sequence), + ) + ) + assert partitioned == whole.events + assert len({item.event_id for item in partitioned}) == len(partitioned) + + +def test_ensemble_members_have_isolated_deterministic_candidate_identity() -> ( + None +): + condition = _condition() + index = _index(condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config, members=(MEMBER_ID, "member-b")) + + def generate(member: str): + left = _anchor(run, BASE_NS, sequence=0, row_id=1, member=member) + right = _anchor( + run, + BASE_NS + SECOND, + sequence=99, + row_id=2, + member=member, + ) + return generate_empirical_motif_candidates( + run=run, + window=_window( + run, + BASE_NS, + BASE_NS + SECOND + 1, + member=member, + ), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + ) + + first = generate(MEMBER_ID) + first_retry = generate(MEMBER_ID) + second = generate("member-b") + + assert first.events == first_retry.events + assert [item.event_id for item in first.events] != [ + item.event_id for item in second.events + ] + assert all(item.ensemble_member_id == MEMBER_ID for item in first.events) + assert all(item.ensemble_member_id == "member-b" for item in second.events) + + +def test_resource_estimation_tuning_does_not_change_semantic_event_identity() -> ( + None +): + condition = _condition() + index = _index(condition) + first_config = EmpiricalMotifGeneratorConfigV1( + estimated_bytes_per_event=512 + ) + second_config = EmpiricalMotifGeneratorConfigV1( + estimated_bytes_per_event=1_024 + ) + assert first_config.config_id == second_config.config_id + run = _run(first_config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + result = _result(index, condition) + window = _window(run, BASE_NS, BASE_NS + SECOND + 1) + + first = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=result, + config=first_config, + ) + second = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=result, + config=second_config, + ) + + assert first.events == second.events + assert ( + second.resource_estimate.estimated_memory_bytes + == 2 * first.resource_estimate.estimated_memory_bytes + ) + + +def test_same_timestamp_and_reversed_anchors_refuse_explicitly() -> None: + condition = _condition() + index = _index(condition) + result = _result(index, condition, used_at_ns=BASE_NS) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + window = _window(run, BASE_NS, BASE_NS + SECOND + 1) + same_left = _anchor(run, BASE_NS, sequence=0, row_id=1) + same_right = _anchor(run, BASE_NS, sequence=1, row_id=2) + zero = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=same_left, + right_anchor=same_right, + query_result=result, + config=config, + ) + reversed_batch = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=_anchor(run, BASE_NS + SECOND, sequence=2, row_id=3), + right_anchor=same_left, + query_result=result, + config=config, + ) + + assert zero.status is MotifGenerationStatus.REFUSED + assert zero.decision is MotifGenerationDecision.ZERO_WIDTH_ANCHOR + assert reversed_batch.decision is MotifGenerationDecision.REVERSED_ANCHOR + assert not zero.events and not reversed_batch.events + with pytest.raises(ValueError, match="right anchor boundary"): + generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=same_left, + right_anchor=same_right, + query_result=_result(index, condition), + config=config, + ) + + +def test_duplicate_precision_timestamps_remain_stably_sequenced() -> None: + condition = _condition( + intensity=2_000.0, + timestamp_precision_ns=1_000_000.0, + ) + index = _index( + condition, + event_offsets_ns=(0, 500_000, 1_000_000), + ) + config = EmpiricalMotifGeneratorConfigV1() + run = _run( + config, + storage_policy=ReconstructionStoragePolicyV1( + max_candidate_amplification=100.0 + ), + ) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor( + run, + BASE_NS + 5_000_000, + sequence=99, + row_id=2, + ) + batch = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + 5_000_001), + left_anchor=left, + right_anchor=right, + query_result=_result( + index, + condition, + used_at_ns=BASE_NS + 5_000_000, + ), + config=config, + ) + + positions = [ + (item.event_time_ns, item.event_sequence) for item in batch.events + ] + assert batch.status is MotifGenerationStatus.GENERATED + assert len({item.event_time_ns for item in batch.events}) < len( + batch.events + ) + assert positions == sorted(positions) + assert len(set(positions)) == len(positions) + + +def test_sparse_closed_and_large_gap_decisions_are_observable() -> None: + source_condition = _condition() + index = _index(source_condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + window = _window(run, BASE_NS, BASE_NS + SECOND + 1) + + sparse = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=_result(index, source_condition, minimum_support=2), + config=config, + ) + closed_condition = _condition( + session_state="weekend_closed", + intensity=0.0, + ) + closed = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=_result(index, closed_condition, minimum_support=2), + config=config, + ) + large_right = _anchor( + run, + BASE_NS + 100 * SECOND, + sequence=100, + row_id=3, + ) + large = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + 100 * SECOND + 1), + left_anchor=left, + right_anchor=large_right, + query_result=_result( + index, + source_condition, + used_at_ns=BASE_NS + 100 * SECOND, + ), + config=config, + ) + + assert sparse.decision is MotifGenerationDecision.NO_SUPPORTED_CELL + assert any( + item["outcome"] == "sparse" + for item in sparse.metadata()["backoff_attempts"] + ) + assert closed.status is MotifGenerationStatus.EMPTY + assert closed.decision is MotifGenerationDecision.CLOSED_SESSION + assert large.status is MotifGenerationStatus.REFUSED + assert large.decision is MotifGenerationDecision.RESOURCE_LIMIT + assert large.resource_estimate.candidate_amplification > 25.0 + + +@pytest.mark.parametrize( + ("special_tags", "event_tags"), + (("daily_rollover", ()), ("ordinary", ("us_cpi",))), +) +def test_rollover_and_event_window_conditioning_remains_in_lineage( + special_tags: str, + event_tags: tuple[str, ...], +) -> None: + condition = _condition( + special_tags=(special_tags,), + event_tags=event_tags, + ) + index = _index(condition, name=f"{special_tags}-{'-'.join(event_tags)}") + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + batch = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + ) + + assert batch.status is MotifGenerationStatus.GENERATED + assert batch.metadata()["condition"]["special_tags"] == [special_tags] + assert batch.metadata()["condition"]["event_tags"] == list(event_tags) + assert all( + item.feed_epoch_id == condition.feed_epoch_id for item in batch.events + ) + + +def test_unsafe_spread_shape_refuses_the_entire_anchor_interval() -> None: + condition = _condition() + index = _index( + condition, + quotes=( + (1.1000, 1.1200), + (1.1095, 1.1105), + (1.1000, 1.1200), + ), + allow_spread_scaling=True, + name="unsafe-spread-shape", + ) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + batch = generate_empirical_motif_candidates( + run=run, + window=_window(run, BASE_NS, BASE_NS + SECOND + 1), + left_anchor=left, + right_anchor=right, + query_result=_result(index, condition), + config=config, + ) + + assert batch.status is MotifGenerationStatus.REFUSED + assert batch.decision is MotifGenerationDecision.INVALID_TRANSFORMED_QUOTE + assert not batch.events + + +def test_benchmark_adapter_uses_the_shared_reverse_degradation_interface() -> ( + None +): + condition = _condition() + index = _index(condition) + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + candidate = BenchmarkCandidateV1( + kind=BenchmarkCandidateKind.CANDIDATE, + method_id=EMPIRICAL_MOTIF_GENERATOR_ID, + implementation_version="1.0.0", + parameters={"motif_generator_config_id": config.config_id}, + ensemble_member_ids=(MEMBER_ID,), + ) + scenario = BenchmarkScenarioV1( + split_kind=BenchmarkSplitKind.VALIDATION, + epoch_id=condition.feed_epoch_id, + severity_id="sparse", + observation_operator_id="observation-operator:fixture", + degradation_parameters={"retention_probability": 0.25}, + ) + window = _window(run, BASE_NS, BASE_NS + SECOND + 1) + degraded = ( + BenchmarkEventV1( + source_event_id="left", + symbol="EURUSD", + event_time_ns=BASE_NS, + event_sequence=0, + bid=1.1000, + ask=1.1002, + epoch_id=condition.feed_epoch_id, + session="london", + event_state="ordinary", + sparsity="sparse", + anchor_id="left", + ), + BenchmarkEventV1( + source_event_id="right", + symbol="EURUSD", + event_time_ns=BASE_NS + SECOND, + event_sequence=99, + bid=1.1004, + ask=1.1006, + epoch_id=condition.feed_epoch_id, + session="london", + event_state="ordinary", + sparsity="sparse", + anchor_id="right", + ), + ) + adapter = EmpiricalMotifBenchmarkGeneratorV1( + candidate=candidate, + run=run, + motif_index=index, + condition=condition, + config=config, + ) + + candidate_window = generate_benchmark_candidate_window( + adapter, + candidate, + degraded, + scenario=scenario, + window=window, + ensemble_member_id=MEMBER_ID, + execution=BenchmarkExecutionEvidenceV1( + attempted=True, + converged=True, + peak_memory_bytes=7 * config.estimated_bytes_per_event, + ), + ) + + assert candidate_window.candidate_id == candidate.candidate_id + assert len(candidate_window.events) == 9 + assert {item.source_event_id for item in candidate_window.events} >= { + "left", + "right", + } + assert ( + sum( + item.sparsity == "empirical-motif-candidate" + for item in candidate_window.events + ) + == 7 + ) + + +def test_reverse_degradation_scorecard_compares_motif_generator_to_controls() -> ( + None +): + config = EmpiricalMotifGeneratorConfigV1() + run = _run(config) + candidate = BenchmarkCandidateV1( + kind=BenchmarkCandidateKind.CANDIDATE, + method_id=EMPIRICAL_MOTIF_GENERATOR_ID, + implementation_version="1.0.0", + parameters={"motif_generator_config_id": config.config_id}, + ensemble_member_ids=(MEMBER_ID,), + ) + + def control( + kind: BenchmarkControlKind, + method_id: str, + parameters: dict[str, object] | None = None, + ) -> BenchmarkCandidateV1: + return BenchmarkCandidateV1( + kind=BenchmarkCandidateKind.CONTROL, + method_id=method_id, + implementation_version="fixture-v1", + parameters=parameters or {}, + ensemble_member_ids=("control",), + control_kind=kind, + ) + + calibration_start = BASE_NS - 20 * SECOND + validation_start = BASE_NS + validation_end = BASE_NS + SECOND + 1 + holdout_start = BASE_NS + 20 * SECOND + holdout_end = holdout_start + SECOND + 1 + splits = ( + BenchmarkSplitV1( + BenchmarkSplitKind.CALIBRATION, + calibration_start, + BASE_NS - 10 * SECOND, + ), + BenchmarkSplitV1( + BenchmarkSplitKind.VALIDATION, + validation_start, + validation_end, + ), + BenchmarkSplitV1( + BenchmarkSplitKind.FINAL_HOLDOUT, + holdout_start, + holdout_end, + ), + ) + scenarios = tuple( + BenchmarkScenarioV1( + split_kind=split_kind, + epoch_id=epoch_id, + severity_id=severity_id, + observation_operator_id=f"operator:{epoch_id}:{severity_id}", + degradation_parameters={"retention_probability": retention}, + ) + for split_kind, epoch_id, severity_id, retention in ( + ( + BenchmarkSplitKind.VALIDATION, + "feed-epoch:modern-v1", + "mild", + 0.5, + ), + ( + BenchmarkSplitKind.VALIDATION, + "feed-epoch:legacy-v1", + "severe", + 0.25, + ), + ( + BenchmarkSplitKind.FINAL_HOLDOUT, + "feed-epoch:modern-v1", + "severe", + 0.25, + ), + ( + BenchmarkSplitKind.FINAL_HOLDOUT, + "feed-epoch:legacy-v1", + "mild", + 0.5, + ), + ) + ) + manifest = ReverseDegradationBenchmarkManifestV1( + run_id=run.run_id, + information_manifest_id="information-manifest:fixture", + profile=BenchmarkProfileV1(), + splits=splits, + scenarios=scenarios, + candidates=( + control(BenchmarkControlKind.NO_FILL, "no-fill"), + control( + BenchmarkControlKind.LINEAR_INTERPOLATION, + "linear", + {"interval_ns": 125_000_000}, + ), + control( + BenchmarkControlKind.RESAMPLE_LAST, + "resample-last", + {"interval_ns": 125_000_000}, + ), + control( + BenchmarkControlKind.EMPIRICAL_OVERLAY, + "empirical-overlay", + {"source_schema": "synthetic-tick-generation.v1"}, + ), + candidate, + ), + ) + engine = ReverseDegradationBenchmarkV1(manifest) + for scenario in manifest.scenarios: + start_ns = ( + validation_start + if scenario.split_kind is BenchmarkSplitKind.VALIDATION + else holdout_start + ) + end_ns = ( + validation_end + if scenario.split_kind is BenchmarkSplitKind.VALIDATION + else holdout_end + ) + condition = _condition(feed_epoch_id=scenario.epoch_id) + index = _index(condition, name=f"benchmark-{scenario.scenario_id}") + window = _window(run, start_ns, end_ns) + reference = tuple( + BenchmarkEventV1( + source_event_id=f"{scenario.scenario_id}:source:{ordinal}", + symbol="EURUSD", + event_time_ns=start_ns + ordinal * 125_000_000, + event_sequence=ordinal, + bid=1.1000 + ordinal * 0.00005, + ask=1.1002 + ordinal * 0.00005, + epoch_id=scenario.epoch_id, + session=condition.session_state, + event_state=condition.activity_regime, + sparsity=scenario.severity_id, + anchor_id=(f"anchor:{ordinal}" if ordinal in {0, 8} else None), + ) + for ordinal in range(9) + ) + degraded = (reference[0], reference[-1]) + adapter = EmpiricalMotifBenchmarkGeneratorV1( + candidate=candidate, + run=run, + motif_index=index, + condition=condition, + config=config, + ) + motif_window = generate_benchmark_candidate_window( + adapter, + candidate, + degraded, + scenario=scenario, + window=window, + ensemble_member_id=MEMBER_ID, + execution=BenchmarkExecutionEvidenceV1( + attempted=True, + converged=True, + ), + ) + controls = build_benchmark_control_windows( + manifest, + scenario, + window, + degraded, + empirical_overlay_events=degraded, + ) + engine.consume_window( + scenario_id=scenario.scenario_id, + window=window, + reference_events=reference, + degraded_events=degraded, + candidate_windows=(*controls, motif_window), + ) + + scorecard = engine.finalize() + motif_scores = tuple( + item + for item in scorecard.candidate_scores + if item.candidate_id == candidate.candidate_id + ) + assert scorecard.automatic_winner is False + assert len(motif_scores) == len(scenarios) + assert all( + "mean_soft_loss_delta" in item.relative_to_no_fill + for item in motif_scores + ) + assert all( + item.execution_summary["converged_count"] == 1 for item in motif_scores + ) + + +def test_historical_carving_projects_with_bound_lineage_and_roundtrips() -> ( + None +): + condition = _condition( + special_tags=("daily_rollover", "crisis"), + event_tags=("news_window",), + ) + policies = ( + HistoricalCarvingConditionPolicyV1( + name="news intensity and spread", + match_tags=("news_window",), + spread_multiplier=1.5, + priority=30, + ), + HistoricalCarvingConditionPolicyV1( + name="rollover liquidity", + match_tags=("daily_rollover",), + spread_multiplier=2.0, + priority=20, + ), + HistoricalCarvingConditionPolicyV1( + name="crisis state", + match_tags=("crisis",), + priority=10, + ), + ) + constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture-v1", + condition_policies=policies, + ) + _, run, left, right, window, _, batch = _carving_inputs( + constraints, + condition=condition, + ) + context = _market_context( + window, + special_tags=("daily_rollover", "crisis"), + event_tags=("news_window",), + ) + evidence = _fingerprint_evidence(batch) + + carved = carve_empirical_motif_candidates( + run=run, + window=window, + candidate_batch=batch, + observed_events=(left, right), + market_context=context, + constraints=constraints, + fingerprint_evidence=evidence, + ) + + assert carved.status is CarvingBatchStatus.ACCEPTED + assert carved.projected_event_count == len(batch.events) + assert carved.rejection_summary.rejected_count == 0 + assert all( + item.constraint_set_id == constraints.constraint_set_id + for item in carved.accepted_events + ) + assert all( + item.candidate_event_id != item.output_event_id + and item.original_bid is not None + and item.original_ask is not None + and item.spread_multiplier == pytest.approx(3.0) + and set(item.policy_ids) == {policy.policy_id for policy in policies} + for item in carved.accepted_lineage + ) + merged = carved.merged_stream((left, right)) + assert merged.observed_event_count == 2 + assert {left.event_id, right.event_id}.issubset( + {item.event_id for item in merged.events} + ) + assert carved.metadata()["rejected_rows_retained"] is False + assert "accepted_events" not in carved.metadata() + assert ( + HistoricalCarvingConstraintSetV1.from_json(constraints.to_json()) + == constraints + ) + assert ( + HistoricalCarvedCandidateBatchV1.from_json(carved.to_json()) == carved + ) + + +def test_hard_closure_and_quarantine_precede_conditioned_projection() -> None: + projection = HistoricalCarvingConditionPolicyV1( + name="ordinary projection", + match_tags=("ordinary",), + spread_multiplier=2.0, + ) + closed_constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture-v1", + condition_policies=(projection,), + ) + _, run, left, right, window, _, batch = _carving_inputs(closed_constraints) + closed = carve_empirical_motif_candidates( + run=run, + window=window, + candidate_batch=batch, + observed_events=(left, right), + market_context=_market_context( + window, + session_state="weekend_closed", + ), + constraints=closed_constraints, + fingerprint_evidence=_fingerprint_evidence(batch), + ) + assert closed.status is CarvingBatchStatus.REFUSED + assert closed.refusal_reason is CarvingReason.CLOSED_SESSION + assert not closed.accepted_events + assert closed.projected_event_count == 0 + assert closed.rejection_summary.reason_counts == { + CarvingReason.CLOSED_SESSION.value: len(batch.events) + } + + quarantine_constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture-v1", + quarantines=( + HistoricalCarvingQuarantineV1( + symbol="EURUSD", + start_ns=BASE_NS, + end_ns=BASE_NS + SECOND, + reason="source quality quarantine", + source_id="quality-report:fixture", + ), + ), + max_rejection_examples=2, + ) + _, run, left, right, window, _, batch = _carving_inputs( + quarantine_constraints + ) + quarantined = carve_empirical_motif_candidates( + run=run, + window=window, + candidate_batch=batch, + observed_events=(left, right), + market_context=_market_context(window), + constraints=quarantine_constraints, + fingerprint_evidence=_fingerprint_evidence(batch), + ) + assert quarantined.status is CarvingBatchStatus.REFUSED + assert quarantined.refusal_reason is CarvingReason.QUARANTINED_INTERVAL + assert len(quarantined.rejection_examples) == 2 + assert quarantined.rejection_summary.reason_counts == { + CarvingReason.QUARANTINED_INTERVAL.value: len(batch.events) + } + + +def test_conditioned_thinning_is_retry_stable_and_evidence_is_bounded() -> None: + constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture-v1", + condition_policies=( + HistoricalCarvingConditionPolicyV1( + name="zero intensity envelope", + match_tags=("ordinary",), + acceptance_rate=0.0, + ), + ), + max_rejection_examples=1, + ) + _, run, left, right, window, _, batch = _carving_inputs(constraints) + arguments = { + "run": run, + "window": window, + "candidate_batch": batch, + "observed_events": (left, right), + "market_context": _market_context(window), + "constraints": constraints, + "fingerprint_evidence": _fingerprint_evidence(batch), + } + first = carve_empirical_motif_candidates(**arguments) + retry = carve_empirical_motif_candidates(**arguments) + + assert first == retry + assert first.status is CarvingBatchStatus.REFUSED + assert first.refusal_reason is CarvingReason.INTENSITY_THINNED + assert len(first.rejection_examples) == 1 + assert ( + first.rejection_summary.accepted_count + + first.rejection_summary.rejected_count + == first.rejection_summary.candidate_count + == len(batch.events) + ) + assert first.rejection_summary.reason_counts == { + CarvingReason.INTENSITY_THINNED.value: len(batch.events) + } + + failed_validation = carve_empirical_motif_candidates( + **{ + **arguments, + "fingerprint_evidence": _fingerprint_evidence( + batch, status="mismatch" + ), + } + ) + assert failed_validation.refusal_reason is ( + CarvingReason.FINGERPRINT_VALIDATION_FAILED + ) + assert not failed_validation.accepted_events + + +def test_motif_incompatibility_uses_same_position_substitution() -> None: + condition = _condition() + primary_index = _index(condition, name="primary") + alternate_index = _index(condition, name="alternate") + eligible_motif_id = alternate_index.fragments[0].fragment_id + constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture-v1", + condition_policies=( + HistoricalCarvingConditionPolicyV1( + name="eligible crisis motif", + match_tags=("ordinary",), + eligible_motif_ids=(eligible_motif_id,), + ), + ), + ) + config = EmpiricalMotifGeneratorConfigV1() + run = _run( + config, + additional_configuration_ids=(constraints.constraint_set_id,), + ) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + window = _window(run, BASE_NS, BASE_NS + SECOND + 1) + primary = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=_result(primary_index, condition), + config=config, + ) + alternate = generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=_result(alternate_index, condition), + config=config, + ) + + without_substitution = carve_empirical_motif_candidates( + run=run, + window=window, + candidate_batch=primary, + observed_events=(left, right), + market_context=_market_context(window), + constraints=constraints, + fingerprint_evidence=_fingerprint_evidence(primary), + ) + substituted = carve_empirical_motif_candidates( + run=run, + window=window, + candidate_batch=primary, + substitution_batches=(alternate,), + observed_events=(left, right), + market_context=_market_context(window), + constraints=constraints, + fingerprint_evidence=_fingerprint_evidence(primary, alternate), + ) + + assert ( + without_substitution.refusal_reason is CarvingReason.MOTIF_INCOMPATIBLE + ) + assert substituted.status is CarvingBatchStatus.ACCEPTED + assert substituted.substituted_event_count == len(primary.events) + assert all( + item.candidate_batch_id == alternate.batch_id + and item.action.value == "substituted" + for item in substituted.accepted_lineage + ) + + +@pytest.mark.parametrize( + ("constraint_kwargs", "expected_reason"), + ( + ( + {"max_input_candidate_events": 1}, + CarvingReason.RESOURCE_LIMIT, + ), + ( + {"max_anchor_gap_ns": SECOND // 2}, + CarvingReason.ANCHOR_GAP_LIMIT, + ), + ), +) +def test_carving_refuses_resource_and_pathological_gap_limits( + constraint_kwargs: dict[str, int], + expected_reason: CarvingReason, +) -> None: + constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture-v1", + **constraint_kwargs, + ) + _, run, left, right, window, _, batch = _carving_inputs(constraints) + carved = carve_empirical_motif_candidates( + run=run, + window=window, + candidate_batch=batch, + observed_events=(left, right), + market_context=_market_context(window), + constraints=constraints, + fingerprint_evidence=_fingerprint_evidence(batch), + ) + assert carved.status is CarvingBatchStatus.REFUSED + assert carved.refusal_reason is expected_reason + assert not carved.accepted_events + + +def test_carving_preserves_an_upstream_empty_window_as_explicit_empty() -> None: + constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture-v1" + ) + _, run, left, right, window, _, batch = _carving_inputs( + constraints, + condition=_condition(intensity=0.0), + ) + assert batch.status is MotifGenerationStatus.EMPTY + + carved = carve_empirical_motif_candidates( + run=run, + window=window, + candidate_batch=batch, + observed_events=(left, right), + market_context=_market_context(window), + constraints=constraints, + fingerprint_evidence=None, + ) + + assert carved.status is CarvingBatchStatus.EMPTY + assert carved.refusal_reason is CarvingReason.UPSTREAM_EMPTY + assert carved.rejection_summary.candidate_count == 0 + assert carved.rejection_summary.reason_counts == {} + assert not carved.accepted_events and not carved.rejection_examples + + +def test_carving_output_is_window_partition_invariant() -> None: + condition = _condition() + constraints = HistoricalCarvingConstraintSetV1( + fingerprint_constraint_id="fingerprint-constraints:fixture-v1" + ) + config = EmpiricalMotifGeneratorConfigV1() + run = _run( + config, + additional_configuration_ids=(constraints.constraint_set_id,), + ) + left = _anchor(run, BASE_NS, sequence=0, row_id=1) + right = _anchor(run, BASE_NS + SECOND, sequence=99, row_id=2) + result = _result(_index(condition), condition) + midpoint = BASE_NS + SECOND // 2 + whole_window = _window(run, BASE_NS, BASE_NS + SECOND + 1) + left_window = _window( + run, + BASE_NS, + midpoint, + right_lookahead_ns=BASE_NS + SECOND + 1 - midpoint, + ) + right_window = _window( + run, + midpoint, + BASE_NS + SECOND + 1, + left_halo_ns=midpoint - BASE_NS, + ) + + def generated(window: ReconstructionWindowV1): + return generate_empirical_motif_candidates( + run=run, + window=window, + left_anchor=left, + right_anchor=right, + query_result=result, + config=config, + ) + + whole_batch = generated(whole_window) + left_batch = generated(left_window) + right_batch = generated(right_window) + + def carved(window: ReconstructionWindowV1, batch): + return carve_empirical_motif_candidates( + run=run, + window=window, + candidate_batch=batch, + observed_events=(left, right), + market_context=_market_context(window), + constraints=constraints, + fingerprint_evidence=_fingerprint_evidence(batch), + ) + + whole = carved(whole_window, whole_batch) + left_part = carved(left_window, left_batch) + right_part = carved(right_window, right_batch) + partitioned = (*left_part.accepted_events, *right_part.accepted_events) + + assert whole == carved(whole_window, whole_batch) + assert [item.event_id for item in whole.accepted_events] == [ + item.event_id + for item in sorted( + partitioned, + key=lambda item: (item.event_time_ns, item.event_sequence), + ) + ] + assert [item.to_dict() for item in whole.accepted_events] == [ + item.to_dict() + for item in sorted( + partitioned, + key=lambda item: (item.event_time_ns, item.event_sequence), + ) + ] diff --git a/tests/unit/test_synthetic_information.py b/tests/unit/test_synthetic_information.py new file mode 100644 index 00000000..71fd9c95 --- /dev/null +++ b/tests/unit/test_synthetic_information.py @@ -0,0 +1,802 @@ +"""Tests for reconstruction information modes and leakage auditing.""" + +from __future__ import annotations + +from dataclasses import replace +import json + +import pytest + +from histdatacom.synthetic import ( + RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION, + InformationAuditReportV1, + InformationAuditRule, + InformationInputKind, + InformationLeakageError, + InformationMode, + InformationScope, + InformationSplitKind, + InformationStage, + ReconstructionInformationInputV1, + ReconstructionInformationManifestV1, + ReconstructionInformationPolicyV1, + ReconstructionInformationSplitV1, + ReconstructionRunV1, + ReconstructionWindowV1, + audit_reconstruction_information, + plan_reconstruction_windows, + reconstruction_information_window_plan_id, + require_reconstruction_information_audit, +) + +BASE_NS = 1_700_000_000_000_000_000 +SYMBOLS = ("eurusd", "gbpusd", "eurgbp") + + +def _policy( + mode: InformationMode = InformationMode.EX_ANTE_SIMULATION, + *, + lookahead_ns: int = 0, + max_findings: int = 128, +) -> ReconstructionInformationPolicyV1: + return ReconstructionInformationPolicyV1( + information_mode=mode, + max_allowed_lookahead_ns=lookahead_ns, + max_retained_findings=max_findings, + ) + + +def _run( + policy: ReconstructionInformationPolicyV1, + *, + bind_policy: bool = True, +) -> ReconstructionRunV1: + configuration_ids = ["config:sha256:reconstruction-v1"] + if bind_policy: + configuration_ids.append(policy.policy_id) + return ReconstructionRunV1( + symbols=SYMBOLS, + source_version_ids=("source:sha256:historical-v1",), + configuration_ids=tuple(configuration_ids), + ensemble_member_ids=("member-000",), + base_seed=20260713, + ) + + +def _splits() -> tuple[ReconstructionInformationSplitV1, ...]: + return ( + ReconstructionInformationSplitV1( + InformationSplitKind.TRAIN, + BASE_NS, + BASE_NS + 100, + ), + ReconstructionInformationSplitV1( + InformationSplitKind.CALIBRATION, + BASE_NS + 100, + BASE_NS + 200, + ), + ReconstructionInformationSplitV1( + InformationSplitKind.VALIDATION, + BASE_NS + 200, + BASE_NS + 300, + ), + ) + + +def _windows( + run: ReconstructionRunV1, + *, + right_lookahead_ns: int = 0, +) -> tuple[ReconstructionWindowV1, ...]: + return plan_reconstruction_windows( + run, + ensemble_member_id="member-000", + start_ns=BASE_NS, + end_ns=BASE_NS + 300, + window_size_ns=100, + right_lookahead_ns=right_lookahead_ns, + ) + + +def _input( + run: ReconstructionRunV1, + *, + artifact_id: str, + mode: InformationMode = InformationMode.EX_ANTE_SIMULATION, + input_kind: InformationInputKind = InformationInputKind.EXTERNAL, + stage: InformationStage = InformationStage.SOURCE, + scope: InformationScope = InformationScope.POINT_IN_TIME, + event_offset: int = 10, + available_offset: int = 10, + used_offset: int = 90, + observation_start_offset: int | None = None, + observation_end_offset: int | None = None, + vintage_id: str = "vintage:initial", + reason: str = "declared synthetic test input", + revision_sequence: int = 0, + supersedes_input_id: str | None = None, + allowed_lookahead_ns: int = 0, + parent_input_ids: tuple[str, ...] = (), + split_kind: InformationSplitKind | None = None, +) -> ReconstructionInformationInputV1: + return ReconstructionInformationInputV1( + run_id=run.run_id, + artifact_id=artifact_id, + information_mode=mode, + input_kind=input_kind, + stage=stage, + scope=scope, + event_time_ns=BASE_NS + event_offset, + available_at_ns=BASE_NS + available_offset, + used_at_ns=BASE_NS + used_offset, + observation_start_ns=BASE_NS + + ( + event_offset + if observation_start_offset is None + else observation_start_offset + ), + observation_end_ns=BASE_NS + + ( + event_offset + if observation_end_offset is None + else observation_end_offset + ), + vintage_id=vintage_id, + reason=reason, + revision_sequence=revision_sequence, + supersedes_input_id=supersedes_input_id, + allowed_lookahead_ns=allowed_lookahead_ns, + parent_input_ids=parent_input_ids, + split_kind=split_kind, + ) + + +def _valid_bundle( + *, + max_findings: int = 128, +) -> tuple[ + ReconstructionInformationPolicyV1, + ReconstructionRunV1, + ReconstructionInformationManifestV1, + tuple[ReconstructionWindowV1, ...], +]: + policy = _policy(max_findings=max_findings) + run = _run(policy) + windows = _windows(run) + source = _input( + run, + artifact_id="artifact:source-ticks", + observation_start_offset=0, + observation_end_offset=90, + split_kind=InformationSplitKind.TRAIN, + ) + model = _input( + run, + artifact_id="artifact:model-fit", + input_kind=InformationInputKind.DERIVED, + stage=InformationStage.MODEL_FIT, + event_offset=90, + available_offset=90, + used_offset=99, + observation_start_offset=0, + observation_end_offset=90, + parent_input_ids=(source.input_id,), + split_kind=InformationSplitKind.TRAIN, + ) + calibration = _input( + run, + artifact_id="artifact:calibration", + input_kind=InformationInputKind.DERIVED, + stage=InformationStage.CALIBRATION, + event_offset=150, + available_offset=150, + used_offset=175, + observation_start_offset=100, + observation_end_offset=150, + parent_input_ids=(model.input_id,), + split_kind=InformationSplitKind.CALIBRATION, + ) + validation = _input( + run, + artifact_id="artifact:validation", + input_kind=InformationInputKind.DERIVED, + stage=InformationStage.VALIDATION, + event_offset=250, + available_offset=250, + used_offset=275, + observation_start_offset=200, + observation_end_offset=250, + parent_input_ids=(calibration.input_id,), + split_kind=InformationSplitKind.VALIDATION, + ) + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id(windows), + inputs=(validation, source, calibration, model), + splits=_splits(), + ) + return policy, run, manifest, windows + + +def _rules(report: InformationAuditReportV1) -> set[InformationAuditRule]: + return {finding.rule_id for finding in report.findings} + + +def _with_inputs( + manifest: ReconstructionInformationManifestV1, + *inputs: ReconstructionInformationInputV1, +) -> ReconstructionInformationManifestV1: + return replace( + manifest, + inputs=manifest.inputs + tuple(inputs), + manifest_id="", + ) + + +def test_ex_ante_manifest_is_run_bound_serializable_and_strategy_valid() -> ( + None +): + policy, run, manifest, windows = _valid_bundle() + + report = require_reconstruction_information_audit( + manifest, + policy, + run=run, + windows=windows, + ) + + assert report.accepted is True + assert report.total_violation_count == 0 + assert report.valid_for_strategy_usefulness_claim is True + assert "point_in_time_simulation" in report.valid_for + assert ( + ReconstructionInformationPolicyV1.from_json(policy.to_json()) == policy + ) + assert ReconstructionInformationManifestV1.from_json( + manifest.to_json() + ) == (manifest) + assert InformationAuditReportV1.from_json(report.to_json()) == report + assert tuple(item.input_id for item in manifest.inputs) == tuple( + sorted(item.input_id for item in manifest.inputs) + ) + + +def test_diagnostic_retention_does_not_change_policy_run_or_seed_identity() -> ( + None +): + narrow = _policy(max_findings=1) + verbose = _policy(max_findings=256) + narrow_run = _run(narrow) + verbose_run = _run(verbose) + + assert narrow.policy_id == verbose.policy_id + assert narrow_run.run_id == verbose_run.run_id + assert narrow_run.seed_for( + "member-000", "anchor-42" + ) == verbose_run.seed_for("member-000", "anchor-42") + assert narrow.to_dict()["max_retained_findings"] == 1 + assert verbose.to_dict()["max_retained_findings"] == 256 + + +def test_ex_post_future_anchor_is_labeled_bounded_and_not_strategy_valid() -> ( + None +): + policy = _policy( + InformationMode.EX_POST_RECONSTRUCTION, + lookahead_ns=100, + ) + run = _run(policy) + windows = _windows(run, right_lookahead_ns=50) + anchor = _input( + run, + artifact_id="artifact:future-anchor", + mode=policy.information_mode, + stage=InformationStage.CARVING, + scope=InformationScope.FUTURE_ANCHOR, + event_offset=50, + available_offset=50, + used_offset=0, + allowed_lookahead_ns=50, + ) + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id(windows), + inputs=(anchor,), + splits=_splits(), + ) + + report = require_reconstruction_information_audit( + manifest, + policy, + run=run, + windows=windows, + ) + + assert report.accepted is True + assert report.valid_for_strategy_usefulness_claim is False + assert "historically_informed_reconstruction" in report.valid_for + assert "prospective_simulation" in report.invalid_for + + +def test_revision_unavailable_at_use_is_rejected_in_ex_ante_mode() -> None: + policy = _policy() + run = _run(policy) + windows = _windows(run) + initial = _input( + run, + artifact_id="macro:gdp:2025q4:initial", + event_offset=20, + available_offset=30, + used_offset=60, + vintage_id="2026-01-30-initial", + ) + revision = _input( + run, + artifact_id="macro:gdp:2025q4:revision-1", + scope=InformationScope.REVISION, + event_offset=20, + available_offset=80, + used_offset=60, + vintage_id="2026-03-30-revision-1", + revision_sequence=1, + supersedes_input_id=initial.input_id, + ) + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id(windows), + inputs=(initial, revision), + splits=_splits(), + ) + + report = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=windows, + ) + + assert report.accepted is False + assert InformationAuditRule.EX_ANTE_REVISION_NOT_AVAILABLE in _rules(report) + finding = next( + item + for item in report.findings + if item.rule_id is InformationAuditRule.EX_ANTE_REVISION_NOT_AVAILABLE + ) + assert finding.evidence["vintage_id"] == "2026-03-30-revision-1" + + +@pytest.mark.parametrize( + ("scope", "expected_rule"), + ( + ( + InformationScope.FUTURE_ANCHOR, + InformationAuditRule.EX_ANTE_FUTURE_ANCHOR, + ), + ( + InformationScope.FULL_PERIOD_SUMMARY, + InformationAuditRule.EX_ANTE_FULL_PERIOD_SUMMARY, + ), + ( + InformationScope.GLOBAL_NORMALIZATION, + InformationAuditRule.EX_ANTE_GLOBAL_NORMALIZATION, + ), + ), +) +def test_ex_post_only_scopes_cannot_be_reused_as_ex_ante_features( + scope: InformationScope, + expected_rule: InformationAuditRule, +) -> None: + policy, run, manifest, windows = _valid_bundle() + source = manifest.inputs[0] + feature = _input( + run, + artifact_id=f"artifact:{scope.value}", + input_kind=InformationInputKind.DERIVED, + stage=InformationStage.FEATURE, + scope=scope, + event_offset=50, + available_offset=80, + used_offset=90, + observation_start_offset=0, + observation_end_offset=90, + parent_input_ids=(source.input_id,), + split_kind=InformationSplitKind.TRAIN, + ) + + report = audit_reconstruction_information( + _with_inputs(manifest, feature), + policy, + run=run, + windows=windows, + ) + + assert expected_rule in _rules(report) + + +def test_future_event_and_motif_selection_leakage_have_distinct_rules() -> None: + policy, run, manifest, windows = _valid_bundle() + source = manifest.inputs[0] + future_event = _input( + run, + artifact_id="calendar:event:future", + stage=InformationStage.CALENDAR_CONTEXT, + event_offset=95, + available_offset=80, + used_offset=90, + ) + motif = _input( + run, + artifact_id="motif:selected-from-future", + input_kind=InformationInputKind.DERIVED, + stage=InformationStage.MOTIF_SELECTION, + scope=InformationScope.EMPIRICAL_MOTIF, + event_offset=50, + available_offset=80, + used_offset=90, + observation_start_offset=0, + observation_end_offset=95, + parent_input_ids=(source.input_id,), + split_kind=InformationSplitKind.TRAIN, + ) + + report = audit_reconstruction_information( + _with_inputs(manifest, future_event, motif), + policy, + run=run, + windows=windows, + ) + + assert InformationAuditRule.EX_ANTE_FUTURE_EVENT in _rules(report) + assert InformationAuditRule.EX_ANTE_MOTIF_SELECTION_LEAKAGE in _rules( + report + ) + + +def test_same_run_cannot_mix_information_modes() -> None: + policy, run, manifest, windows = _valid_bundle() + mixed = _input( + run, + artifact_id="artifact:mixed-mode", + mode=InformationMode.EX_POST_RECONSTRUCTION, + event_offset=20, + available_offset=20, + used_offset=30, + ) + + report = audit_reconstruction_information( + _with_inputs(manifest, mixed), + policy, + run=run, + windows=windows, + ) + + assert InformationAuditRule.INPUT_MODE_MISMATCH in _rules(report) + + +def test_exact_window_plan_and_right_lookahead_are_audited() -> None: + policy, run, manifest, windows = _valid_bundle() + leaking_windows = _windows(run, right_lookahead_ns=1) + + mismatch = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=leaking_windows, + ) + empty = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=(), + ) + + assert reconstruction_information_window_plan_id( + tuple(reversed(windows)) + ) == (manifest.window_plan_id) + assert InformationAuditRule.WINDOW_PLAN_MISMATCH in _rules(mismatch) + assert InformationAuditRule.WINDOW_LOOKAHEAD_EXCEEDS_POLICY in _rules( + mismatch + ) + assert InformationAuditRule.EX_ANTE_WINDOW_LOOKAHEAD in _rules(mismatch) + assert InformationAuditRule.WINDOW_PLAN_EMPTY in _rules(empty) + + shifted = replace( + windows[1], + core_start_ns=windows[1].core_start_ns + 1, + window_id="", + synchronization_unit_id="", + ) + gapped_windows = (windows[0], shifted, windows[2]) + gapped_manifest = replace( + manifest, + window_plan_id=reconstruction_information_window_plan_id( + gapped_windows + ), + manifest_id="", + ) + gapped = audit_reconstruction_information( + gapped_manifest, + policy, + run=run, + windows=gapped_windows, + ) + assert InformationAuditRule.WINDOW_PLAN_INVALID in _rules(gapped) + + +def test_window_plan_must_cover_every_run_ensemble_member() -> None: + policy = _policy() + run = ReconstructionRunV1( + symbols=SYMBOLS, + source_version_ids=("source:sha256:historical-v1",), + configuration_ids=(policy.policy_id,), + ensemble_member_ids=("member-000", "member-001"), + base_seed=20260713, + ) + windows = _windows(run) + source = _input(run, artifact_id="artifact:source") + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id(windows), + inputs=(source,), + splits=_splits(), + ) + + report = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=windows, + ) + + assert InformationAuditRule.WINDOW_MEMBER_MISSING in _rules(report) + + +def test_policy_must_be_bound_to_the_reconstruction_run() -> None: + policy, _, manifest, _ = _valid_bundle() + unbound_run = _run(policy, bind_policy=False) + unbound_windows = _windows(unbound_run) + rebound_inputs = tuple( + replace(item, run_id=unbound_run.run_id, input_id="") + for item in manifest.inputs + ) + rebound_manifest = ReconstructionInformationManifestV1( + run_id=unbound_run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id( + unbound_windows + ), + inputs=rebound_inputs, + splits=manifest.splits, + ) + + report = audit_reconstruction_information( + rebound_manifest, + policy, + run=unbound_run, + windows=unbound_windows, + ) + + assert InformationAuditRule.POLICY_NOT_BOUND_TO_RUN in _rules(report) + + +def test_splits_are_required_ordered_and_stage_specific() -> None: + policy, run, manifest, windows = _valid_bundle() + model = next( + item + for item in manifest.inputs + if item.stage is InformationStage.MODEL_FIT + ) + no_split_model = replace(model, split_kind=None, input_id="") + inputs = tuple( + no_split_model if item.input_id == model.input_id else item + for item in manifest.inputs + ) + unordered = replace( + manifest, + inputs=inputs, + splits=tuple(reversed(manifest.splits)), + manifest_id="", + ) + + report = audit_reconstruction_information( + unordered, + policy, + run=run, + windows=windows, + ) + + assert InformationAuditRule.SPLIT_DECLARATION_ORDER in _rules(report) + assert InformationAuditRule.INPUT_SPLIT_MISSING in _rules(report) + + +def test_missing_parent_and_revision_vintage_fail_graph_audit() -> None: + policy, run, manifest, windows = _valid_bundle() + derived = _input( + run, + artifact_id="artifact:orphan-derived", + input_kind=InformationInputKind.DERIVED, + stage=InformationStage.FEATURE, + parent_input_ids=("information-input:sha256:missing",), + ) + revision = _input( + run, + artifact_id="macro:revision:orphan", + scope=InformationScope.REVISION, + vintage_id="revision-2", + revision_sequence=2, + supersedes_input_id="information-input:sha256:missing-vintage", + ) + + report = audit_reconstruction_information( + _with_inputs(manifest, derived, revision), + policy, + run=run, + windows=windows, + ) + + assert InformationAuditRule.MISSING_PARENT_INPUT in _rules(report) + assert InformationAuditRule.REVISION_PARENT_MISSING in _rules(report) + + +def test_findings_are_deterministic_bounded_and_fail_closed() -> None: + policy, run, manifest, windows = _valid_bundle(max_findings=2) + violations = tuple( + _input( + run, + artifact_id=f"artifact:future:{ordinal}", + stage=InformationStage.NEWS_CONTEXT, + event_offset=95 + ordinal, + available_offset=95 + ordinal, + used_offset=90, + ) + for ordinal in range(5) + ) + invalid = _with_inputs(manifest, *violations) + + first = audit_reconstruction_information( + invalid, + policy, + run=run, + windows=windows, + ) + second = audit_reconstruction_information( + invalid, + policy, + run=run, + windows=windows, + ) + + assert first == second + assert first.audit_id == second.audit_id + assert first.total_violation_count > len(first.findings) == 2 + assert first.evidence_truncated is True + assert first.valid_for == () + assert "strategy_usefulness_claims" in first.invalid_for + assert first.findings[0].from_json(first.findings[0].to_json()) == ( + first.findings[0] + ) + with pytest.raises(InformationLeakageError) as caught: + require_reconstruction_information_audit( + invalid, + policy, + run=run, + windows=windows, + ) + assert caught.value.report == first + + +def test_ex_post_future_use_must_be_labeled_and_within_lookahead() -> None: + policy = _policy( + InformationMode.EX_POST_RECONSTRUCTION, + lookahead_ns=20, + ) + run = _run(policy) + windows = _windows(run, right_lookahead_ns=20) + unlabeled = _input( + run, + artifact_id="artifact:unlabeled-future", + mode=policy.information_mode, + stage=InformationStage.CARVING, + event_offset=50, + available_offset=50, + used_offset=0, + allowed_lookahead_ns=10, + ) + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id(windows), + inputs=(unlabeled,), + splits=_splits(), + ) + + report = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=windows, + ) + + assert InformationAuditRule.EX_POST_UNLABELED_FUTURE_INFORMATION in _rules( + report + ) + assert InformationAuditRule.EX_POST_LOOKAHEAD_EXCEEDED in _rules(report) + + +def test_contract_readers_reject_identity_and_schema_drift() -> None: + policy, _, manifest, _ = _valid_bundle() + payload = manifest.to_dict() + payload["future_key"] = "ignored" + assert ReconstructionInformationManifestV1.from_dict(payload) == manifest + + wrong_id = dict(payload) + wrong_id["manifest_id"] = "information-manifest:sha256:" + "0" * 64 + with pytest.raises(ValueError, match="manifest_id does not match"): + ReconstructionInformationManifestV1.from_dict(wrong_id) + + wrong_schema = policy.to_dict() + wrong_schema["schema_version"] = ( + "histdatacom.reconstruction-information-policy.v2" + ) + with pytest.raises(ValueError, match="unsupported schema version"): + ReconstructionInformationPolicyV1.from_dict(wrong_schema) + + +def test_contract_validation_requires_complete_temporal_and_revision_metadata() -> ( + None +): + policy = _policy() + run = _run(policy) + with pytest.raises(ValueError, match="event_time_ns must lie inside"): + _input( + run, + artifact_id="artifact:bad-time", + event_offset=10, + observation_start_offset=20, + observation_end_offset=30, + ) + with pytest.raises(ValueError, match="requires supersedes_input_id"): + _input( + run, + artifact_id="artifact:bad-revision", + revision_sequence=1, + ) + with pytest.raises(ValueError, match="requires zero look-ahead"): + _policy(InformationMode.EX_ANTE_SIMULATION, lookahead_ns=1) + + +def test_schema_versions_and_json_are_stable_and_explicit() -> None: + policy, run, manifest, windows = _valid_bundle() + report = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=windows, + ) + versions = ( + RECONSTRUCTION_INFORMATION_POLICY_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_INPUT_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_SPLIT_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_AUDIT_FINDING_SCHEMA_VERSION, + RECONSTRUCTION_INFORMATION_AUDIT_REPORT_SCHEMA_VERSION, + ) + + assert all(version.endswith(".v1") for version in versions) + assert json.loads(manifest.to_json())["manifest_id"] == manifest.manifest_id + assert json.loads(report.to_json())["accepted"] is True diff --git a/tests/unit/test_synthetic_motifs.py b/tests/unit/test_synthetic_motifs.py new file mode 100644 index 00000000..6d3e3412 --- /dev/null +++ b/tests/unit/test_synthetic_motifs.py @@ -0,0 +1,733 @@ +"""Tests for deterministic, split-safe empirical reference motifs.""" + +from __future__ import annotations + +from dataclasses import replace +import hashlib +import json +from pathlib import Path + +import polars as pl +import pytest + +from histdatacom.runtime_contracts import ArtifactRef +from histdatacom.synthetic import ( + REFERENCE_MOTIF_ARTIFACT_KIND, + REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION, + REFERENCE_MOTIF_INDEX_SCHEMA_VERSION, + InformationMode, + InformationScope, + InformationSplitKind, + InformationStage, + ReferenceMotifConditionV1, + ReferenceMotifFragmentV1, + ReferenceMotifIndexConfigV1, + ReferenceMotifIndexV1, + ReferenceMotifLeakageError, + ReferenceMotifQueryResultV1, + ReferenceMotifQueryStatus, + ReferenceMotifQueryV1, + ReferenceMotifResourceLimitError, + ReferenceMotifSourceEventV1, + ReferenceMotifSourceWindowV1, + ReferenceMotifSplitKind, + ReferenceMotifSplitV1, + ReferenceMotifTransformPolicyV1, + ReferenceMotifTransition, + ReconstructionInformationManifestV1, + ReconstructionInformationPolicyV1, + ReconstructionInformationSplitV1, + ReconstructionRunV1, + ReconstructionWindowV1, + audit_reconstruction_information, + build_reference_motif_index, + extract_reference_motif_fragment, + query_reference_motifs, + read_reference_motif_index, + reference_motif_condition_from_quotes, + reconstruction_information_window_plan_id, + reference_motif_information_inputs, + reference_motif_source_window_from_training_frame, + write_reference_motif_index, +) + +BASE_NS = 1_700_000_000_000_000_000 +STEP = 1_000_000_000 + + +def _digest(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _artifact(name: str = "eurusd-modern") -> ArtifactRef: + return ArtifactRef( + kind="augmented-tick-partition", + path=f"artifacts/{name}.data", + size_bytes=4096, + sha256=_digest(name), + metadata={"training_columns": 521, "projection": "motif-source-v1"}, + ) + + +def _splits() -> tuple[ReferenceMotifSplitV1, ...]: + return ( + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.TRAIN, + BASE_NS, + BASE_NS + 1500 * STEP, + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.CALIBRATION, + BASE_NS + 2000 * STEP, + BASE_NS + 3000 * STEP, + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.VALIDATION, + BASE_NS + 4000 * STEP, + BASE_NS + 5000 * STEP, + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.FINAL_HOLDOUT, + BASE_NS + 6000 * STEP, + BASE_NS + 7000 * STEP, + ), + ) + + +def _condition( + *, + symbol: str = "EURUSD", + session: str = "active", + active_sessions: tuple[str, ...] = ("london",), + event_tags: tuple[str, ...] = (), + spread: float = 0.0002, + volatility: float = 0.001, +) -> ReferenceMotifConditionV1: + return ReferenceMotifConditionV1( + symbol=symbol, + feed_epoch_id="feed-epoch:modern-v1", + session_state=session, + active_sessions=active_sessions, + overlap_tags=("london_new_york",) if len(active_sessions) > 1 else (), + special_tags=("ordinary",), + holiday_tags=(), + event_tags=event_tags, + return_regime="small-positive", + range_regime="normal", + volatility_regime="normal", + spread_regime="tight", + activity_regime="active", + interarrival_regime="dense", + timestamp_precision="millisecond", + price_precision="five-decimal", + source_quality_state="eligible", + metrics={ + "return_value": 0.0004, + "range_value": 0.0015, + "volatility": volatility, + "spread": spread, + "tick_intensity": 8.0, + "interarrival_ns": 125_000_000.0, + "timestamp_precision_ns": 1_000_000.0, + "price_precision_digits": 5.0, + "source_quality_score": 1.0, + }, + ) + + +def _split_start(kind: ReferenceMotifSplitKind) -> int: + return { + ReferenceMotifSplitKind.TRAIN: BASE_NS, + ReferenceMotifSplitKind.CALIBRATION: BASE_NS + 2000 * STEP, + ReferenceMotifSplitKind.VALIDATION: BASE_NS + 4000 * STEP, + ReferenceMotifSplitKind.FINAL_HOLDOUT: BASE_NS + 6000 * STEP, + }[kind] + + +def _window( + offset: int, + *, + split: ReferenceMotifSplitKind = ReferenceMotifSplitKind.TRAIN, + condition: ReferenceMotifConditionV1 | None = None, + shape: tuple[float, ...] = (0.0, 0.0001, -0.00005), + time_shape: tuple[int, ...] = (0, 10, 30), + source_name: str = "eurusd-modern", + eligible: bool = True, + available_at_ns: int | None = None, +) -> ReferenceMotifSourceWindowV1: + start = _split_start(split) + offset * STEP + events = tuple( + ReferenceMotifSourceEventV1( + event_time_ns=start + time_offset * 1_000_000, + event_sequence=index, + bid=round(1.1 + movement, 8), + ask=round(1.1002 + movement, 8), + source_row_id=offset * 10 + index, + ) + for index, (time_offset, movement) in enumerate( + zip(time_shape, shape), start=1 + ) + ) + availability = ( + available_at_ns + if available_at_ns is not None + else events[-1].event_time_ns + ) + return ReferenceMotifSourceWindowV1( + source_series_id=f"ascii:T:EURUSD:histdata.com:{source_name}", + period="202001", + source_artifact=_artifact(source_name), + split_kind=split, + condition=condition or _condition(), + events=events, + first_known_at_ns=availability, + available_at_ns=availability, + data_quality_eligible=eligible, + data_quality_reasons=() if eligible else ("DQ_NEGATIVE_SPREAD",), + transform_policy=ReferenceMotifTransformPolicyV1( + min_time_scale=0.75, + max_time_scale=1.5, + min_price_scale=0.8, + max_price_scale=1.25, + max_time_warp_ratio=1.1, + ), + ) + + +def _index( + windows: tuple[ReferenceMotifSourceWindowV1, ...] | None = None, + *, + config: ReferenceMotifIndexConfigV1 | None = None, +) -> ReferenceMotifIndexV1: + values = windows or ( + _window(10, condition=_condition(spread=0.00018)), + _window( + 20, + condition=_condition(spread=0.00022), + shape=(0.0, 0.0002, 0.0001), + ), + _window( + 30, + condition=_condition( + active_sessions=("new_york",), + event_tags=("us_cpi",), + spread=0.0004, + ), + shape=(0.0, -0.0002, 0.0003), + ), + ) + return build_reference_motif_index( + values, + splits=_splits(), + config=config or ReferenceMotifIndexConfigV1(min_cell_support=1), + ) + + +def test_augmented_training_frame_projects_to_compact_fragment() -> None: + rows = { + "series_id": ["ascii:T:EURUSD:histdata.com"] * 3, + "period": ["202001"] * 3, + "row_id": [101, 102, 103], + "event_seq": [1, 2, 3], + "symbol": ["EURUSD"] * 3, + "timestamp_utc_ms": [ + 1_700_000_000_000, + 1_700_000_000_010, + 1_700_000_000_030, + ], + "bid": [1.1, 1.1001, 1.1001], + "ask": [1.1002, 1.1003, 1.1004], + "training_usable": [True, True, True], + "training_exclusion_reason_code": [0, 0, 0], + } + rows.update( + {f"augmented_{index}": [float(index)] * 3 for index in range(521)} + ) + frame = pl.DataFrame(rows) + + window = reference_motif_source_window_from_training_frame( + frame, + source_artifact=_artifact(), + split_kind=ReferenceMotifSplitKind.TRAIN, + condition=_condition(), + first_known_at_ns=1_700_000_000_030_000_000, + available_at_ns=1_700_000_000_030_000_000, + ) + fragment = extract_reference_motif_fragment(window) + payload = fragment.to_dict() + + assert fragment.schema_version == REFERENCE_MOTIF_FRAGMENT_SCHEMA_VERSION + assert fragment.event_offsets_ns == (0, 10_000_000, 30_000_000) + assert fragment.transitions == ( + ReferenceMotifTransition.START, + ReferenceMotifTransition.BOTH, + ReferenceMotifTransition.ASK, + ) + assert fragment.source_row_ids == (101, 102, 103) + assert fragment.end_bid == pytest.approx(1.1001) + assert fragment.end_ask == pytest.approx(1.1004) + assert not any("augmented_" in key for key in payload) + assert "events" not in payload + assert len(json.dumps(payload)) < 16_384 + assert ReferenceMotifSourceWindowV1.from_dict(window.to_dict()) == window + assert ReferenceMotifFragmentV1.from_dict(payload) == fragment + + inconsistent = frame.with_columns(pl.lit(False).alias("training_usable")) + with pytest.raises(ValueError, match="usability and exclusion reason"): + reference_motif_source_window_from_training_frame( + inconsistent, + source_artifact=_artifact(), + split_kind=ReferenceMotifSplitKind.TRAIN, + condition=_condition(), + first_known_at_ns=1_700_000_000_030_000_000, + available_at_ns=1_700_000_000_030_000_000, + ) + + +def test_quote_feature_schema_is_fixed_and_encodes_weekday() -> None: + condition = reference_motif_condition_from_quotes( + symbol="EURUSD", + feed_epoch_id="technology_epoch_04", + session_state="london", + event_times_ns=( + 1_704_067_200_000_000_000, + 1_704_067_200_100_000_000, + 1_704_067_200_400_000_000, + ), + bids=(1.10000, 1.10002, 1.10008), + asks=(1.10010, 1.10012, 1.10018), + event_tags=("market_context:scheduled_macro",), + ) + + assert condition.special_tags == ("weekday:monday",) + assert condition.event_tags == ("market_context:scheduled_macro",) + assert condition.return_regime == "up_large" + assert condition.volatility_regime in {"medium", "high"} + assert condition.activity_regime == "high" + assert condition.timestamp_precision == "millisecond" + assert condition.metrics["timestamp_precision_ns"] == 100_000_000.0 + + with pytest.raises(ValueError, match="aligned quote rows"): + reference_motif_condition_from_quotes( + symbol="EURUSD", + feed_epoch_id="technology_epoch_04", + session_state="london", + event_times_ns=(1, 2), + bids=(1.0,), + asks=(1.1, 1.2), + ) + + +def test_compact_fragment_revalidates_quote_and_availability_invariants() -> ( + None +): + window = _window(10) + fragment = extract_reference_motif_fragment(window) + + with pytest.raises(ValueError, match="negative spread"): + replace( + fragment, + ask_deltas=(0.0, -0.0002, 0.0002), + transitions=( + ReferenceMotifTransition.START, + ReferenceMotifTransition.BOTH, + ReferenceMotifTransition.BOTH, + ), + fragment_id="", + ) + with pytest.raises(ValueError, match="transition marks disagree"): + replace( + fragment, + transitions=( + ReferenceMotifTransition.START, + ReferenceMotifTransition.BID, + ReferenceMotifTransition.ASK, + ), + fragment_id="", + ) + with pytest.raises(ValueError, match="before its last observation"): + replace( + window, + first_known_at_ns=window.end_ns - 1, + available_at_ns=window.end_ns, + source_window_id="", + ) + + +def test_build_is_order_independent_train_only_and_fully_accounted() -> None: + windows = ( + _window(10), + _window(20, shape=(0.0, 0.0002, 0.0001)), + _window(30, shape=(0.0, -0.0002, 0.0003)), + _window( + 10, + split=ReferenceMotifSplitKind.CALIBRATION, + shape=(0.0, 0.0003, 0.0005, 0.0004), + time_shape=(0, 3, 17, 51), + source_name="calibration", + ), + _window( + 10, + split=ReferenceMotifSplitKind.VALIDATION, + shape=(0.0, -0.0004, 0.0002, 0.0007, 0.0001), + time_shape=(0, 2, 9, 28, 90), + source_name="validation", + ), + _window( + 10, + split=ReferenceMotifSplitKind.FINAL_HOLDOUT, + shape=(0.0, 0.0002, -0.0006, 0.0009, 0.0004, -0.0001), + time_shape=(0, 5, 12, 25, 61, 130), + source_name="holdout", + ), + _window(40, eligible=False, source_name="ineligible"), + ) + config = ReferenceMotifIndexConfigV1( + max_fragments=2, + min_cell_support=1, + ) + + first = build_reference_motif_index( + windows, splits=_splits(), config=config + ) + second = build_reference_motif_index( + reversed(windows), splits=_splits(), config=config + ) + + assert first == second + assert first.schema_version == REFERENCE_MOTIF_INDEX_SCHEMA_VERSION + assert first.source_window_count == 7 + assert len(first.fragments) == 2 + assert {item.split_kind for item in first.fragments} == { + ReferenceMotifSplitKind.TRAIN + } + assert first.excluded_split_counts == { + "calibration": 1, + "final_holdout": 1, + "validation": 1, + } + assert first.ineligible_window_count == 1 + assert first.selection_omitted_count == 1 + assert first.leakage_comparison_count >= 0 + assert first.index_id == second.index_id + + +def test_cross_split_normalized_shape_near_duplicate_fails_closed() -> None: + train = _window(10, shape=(0.0, 0.0001, -0.00005)) + holdout = _window( + 10, + split=ReferenceMotifSplitKind.FINAL_HOLDOUT, + shape=(0.0, 0.0002, -0.0001), + time_shape=(0, 20, 60), + source_name="holdout-near-duplicate", + ) + + with pytest.raises(ReferenceMotifLeakageError) as raised: + build_reference_motif_index( + (train, holdout), + splits=_splits(), + config=ReferenceMotifIndexConfigV1(min_cell_support=1), + ) + + assert ( + raised.value.findings[0]["rule"] == "cross_split_near_duplicate_shape" + ) + assert raised.value.findings[0]["split_kinds"] == ["final_holdout", "train"] + + +def test_cross_split_source_guard_catches_adjacent_window_leakage() -> None: + splits = ( + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.TRAIN, BASE_NS, BASE_NS + 1000 + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.CALIBRATION, BASE_NS + 1000, BASE_NS + 2000 + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.VALIDATION, BASE_NS + 2000, BASE_NS + 3000 + ), + ReferenceMotifSplitV1( + ReferenceMotifSplitKind.FINAL_HOLDOUT, + BASE_NS + 3000, + BASE_NS + 4000, + ), + ) + train = replace( + _window(1), + events=( + ReferenceMotifSourceEventV1(BASE_NS + 900, 1, 1.1, 1.1002, 1), + ReferenceMotifSourceEventV1(BASE_NS + 990, 2, 1.1001, 1.1003, 2), + ), + source_window_id="", + ) + calibration = replace( + _window(1, split=ReferenceMotifSplitKind.CALIBRATION), + source_artifact=train.source_artifact, + events=( + ReferenceMotifSourceEventV1(BASE_NS + 1001, 1, 1.2, 1.2003, 3), + ReferenceMotifSourceEventV1(BASE_NS + 1100, 2, 1.2002, 1.2005, 4), + ), + source_window_id="", + ) + + with pytest.raises(ReferenceMotifLeakageError) as raised: + build_reference_motif_index( + (train, calibration), + splits=splits, + config=ReferenceMotifIndexConfigV1( + min_events_per_fragment=2, + min_cell_support=1, + source_overlap_guard_ns=20, + ), + ) + + assert any( + item["rule"] == "cross_split_source_window_overlap" + for item in raised.value.findings + ) + + +def test_artifact_round_trip_verifies_content_identity(tmp_path: Path) -> None: + index = _index() + path = tmp_path / "reference-motifs.json" + + reference = write_reference_motif_index(index, path) + restored = read_reference_motif_index(path, artifact_ref=reference) + + assert reference.kind == REFERENCE_MOTIF_ARTIFACT_KIND + assert reference.metadata["index_id"] == index.index_id + assert restored == index + path.write_text(f"{index.to_json()} \n", encoding="utf-8") + with pytest.raises(ValueError, match="artifact (size|sha256) differs"): + read_reference_motif_index(path, artifact_ref=reference) + + +def test_query_uses_distance_deterministic_ties_and_explicit_backoff() -> None: + index = _index() + exact_query = ReferenceMotifQueryV1( + condition=_condition(spread=0.00019), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + used_at_ns=BASE_NS + 8000 * STEP, + max_results=2, + min_cell_support=2, + ) + + first = query_reference_motifs(index, exact_query) + second = query_reference_motifs( + index, ReferenceMotifQueryV1.from_dict(exact_query.to_dict()) + ) + + assert first == second + assert first.status is ReferenceMotifQueryStatus.MATCHED + assert [item.rank for item in first.matches] == [1, 2] + assert [item.distance for item in first.matches] == sorted( + item.distance for item in first.matches + ) + assert first.matches[0].fragment.source_artifact.sha256 + assert first.matches[0].fragment.source_row_ids + assert first.matches[0].cell_support == 2 + assert first.backoff_attempts[-1].outcome == "selected" + assert ReferenceMotifQueryResultV1.from_json(first.to_json()) == first + + fallback = query_reference_motifs( + index, + ReferenceMotifQueryV1( + condition=_condition( + active_sessions=("london",), + event_tags=("ecb_press_conference",), + ), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + used_at_ns=BASE_NS + 8000 * STEP, + min_cell_support=2, + ), + ) + assert fallback.status is ReferenceMotifQueryStatus.MATCHED + assert fallback.backoff_attempts[0].level == "exact" + assert fallback.backoff_attempts[0].outcome == "sparse" + assert fallback.matches[0].backoff_level == "symbol_epoch_session" + + +def test_point_in_time_query_filters_future_motifs_and_binds_information() -> ( + None +): + available = BASE_NS + 500 * STEP + index = _index( + ( + _window(10, available_at_ns=available), + _window( + 20, + available_at_ns=BASE_NS + 900 * STEP, + shape=(0.0, 0.0002, 0.0001), + ), + ), + config=ReferenceMotifIndexConfigV1(min_cell_support=1), + ) + query = ReferenceMotifQueryV1( + condition=_condition(), + information_mode=InformationMode.EX_ANTE_SIMULATION, + used_at_ns=BASE_NS + 700 * STEP, + as_of_ns=BASE_NS + 700 * STEP, + min_cell_support=1, + ) + + result = query_reference_motifs(index, query) + inputs = reference_motif_information_inputs( + result, run_id="run:point-in-time" + ) + + assert result.status is ReferenceMotifQueryStatus.MATCHED + assert result.hidden_by_availability_count == 1 + assert len(result.matches) == 1 + assert len(inputs) == 1 + assert inputs[0].information_mode is InformationMode.EX_ANTE_SIMULATION + assert inputs[0].stage is InformationStage.MOTIF_SELECTION + assert inputs[0].scope is InformationScope.EMPIRICAL_MOTIF + assert inputs[0].split_kind is InformationSplitKind.TRAIN + assert inputs[0].allowed_lookahead_ns == 0 + assert inputs[0].observation_end_ns <= inputs[0].used_at_ns + + policy = ReconstructionInformationPolicyV1( + InformationMode.EX_ANTE_SIMULATION + ) + run = ReconstructionRunV1( + symbols=("EURUSD",), + source_version_ids=(index.index_id,), + configuration_ids=(policy.policy_id, index.config.config_id), + ensemble_member_ids=("member-1",), + base_seed=438, + ) + target_window = ReconstructionWindowV1( + run_id=run.run_id, + ensemble_member_id="member-1", + symbols=run.symbols, + core_start_ns=query.used_at_ns, + core_end_ns=query.used_at_ns + STEP, + ) + manifest = ReconstructionInformationManifestV1( + run_id=run.run_id, + policy_id=policy.policy_id, + information_mode=policy.information_mode, + window_plan_id=reconstruction_information_window_plan_id( + (target_window,) + ), + inputs=reference_motif_information_inputs(result, run_id=run.run_id), + splits=( + ReconstructionInformationSplitV1( + InformationSplitKind.TRAIN, + BASE_NS, + BASE_NS + 1500 * STEP, + ), + ReconstructionInformationSplitV1( + InformationSplitKind.CALIBRATION, + BASE_NS + 2000 * STEP, + BASE_NS + 3000 * STEP, + ), + ReconstructionInformationSplitV1( + InformationSplitKind.VALIDATION, + BASE_NS + 4000 * STEP, + BASE_NS + 7000 * STEP, + ), + ), + ) + audit = audit_reconstruction_information( + manifest, + policy, + run=run, + windows=(target_window,), + ) + assert audit.accepted is True + + with pytest.raises(ValueError, match="requires as_of_ns"): + ReferenceMotifQueryV1( + condition=_condition(), + information_mode=InformationMode.EX_ANTE_SIMULATION, + used_at_ns=BASE_NS, + ) + + +def test_point_in_time_query_refuses_when_every_match_is_future() -> None: + index = _index( + (_window(10, available_at_ns=BASE_NS + 900 * STEP),), + config=ReferenceMotifIndexConfigV1(min_cell_support=1), + ) + result = query_reference_motifs( + index, + ReferenceMotifQueryV1( + condition=_condition(), + information_mode=InformationMode.EX_ANTE_SIMULATION, + used_at_ns=BASE_NS + 100 * STEP, + as_of_ns=BASE_NS + 100 * STEP, + min_cell_support=1, + ), + ) + + assert result.status is ReferenceMotifQueryStatus.NOT_AVAILABLE_AS_OF + assert not result.matches + assert result.hidden_by_availability_count == 1 + + +def test_resource_limits_and_split_order_fail_before_indexing() -> None: + config = ReferenceMotifIndexConfigV1( + max_source_windows=2, + min_cell_support=1, + ) + with pytest.raises(ReferenceMotifResourceLimitError): + build_reference_motif_index( + (_window(10), _window(20), _window(30)), + splits=_splits(), + config=config, + ) + with pytest.raises(ValueError, match="must be train"): + build_reference_motif_index( + (_window(10),), + splits=tuple(reversed(_splits())), + config=ReferenceMotifIndexConfigV1(min_cell_support=1), + ) + + +def test_period_scale_build_and_retrieval_remain_config_bounded() -> None: + count = 1000 + windows = tuple( + _window( + index + 1, + shape=( + 0.0, + ((index % 19) + 1) * 0.000001, + ((index % 23) - 11) * 0.000001, + ), + time_shape=(0, 10 + index % 7, 40 + index % 13), + source_name=f"period-{index % 17}", + condition=_condition( + spread=0.0001 + (index % 9) * 0.00001, + volatility=0.0005 + (index % 11) * 0.0001, + ), + ) + for index in range(count) + ) + config = ReferenceMotifIndexConfigV1( + max_source_windows=count, + max_fragments=64, + min_cell_support=1, + max_matches=8, + ) + + index = build_reference_motif_index( + windows, splits=_splits(), config=config + ) + result = query_reference_motifs( + index, + ReferenceMotifQueryV1( + condition=_condition(), + information_mode=InformationMode.EX_POST_RECONSTRUCTION, + used_at_ns=BASE_NS + 8000 * STEP, + max_results=8, + min_cell_support=1, + ), + ) + + assert index.source_window_count == count + assert len(index.fragments) == 64 + assert index.selection_omitted_count == count - 64 + assert len(result.matches) <= 8 + assert result.scanned_fragment_count <= 64 * len(config.backoff_levels) + assert len(index.to_json().encode("utf-8")) < config.max_artifact_bytes diff --git a/tests/unit/test_synthetic_observation.py b/tests/unit/test_synthetic_observation.py new file mode 100644 index 00000000..3db9309d --- /dev/null +++ b/tests/unit/test_synthetic_observation.py @@ -0,0 +1,726 @@ +"""Tests for versioned historical feed-observation operators.""" + +from __future__ import annotations + +from dataclasses import replace +import hashlib +import math +from pathlib import Path + +import pytest + +from histdatacom.data_analytics import ( + DEFAULT_FEED_EPOCH_FEATURES, + FeedEpochDefinitionV1, + FeedEpochEvidenceV1, + fit_feed_epochs, +) +from histdatacom.synthetic import ( + InformationMode, + OBSERVATION_PARAMETER_NAMES, + ObservationApplicationResultV1, + ObservationCarryStateV1, + ObservationContextV1, + ObservationFitEvidenceV1, + ObservationInputEventV1, + ObservationOperatorFitConfigV1, + ObservationOperatorV1, + ObservationOutputEventV1, + ReconstructionWindowV1, + fit_observation_operator, + read_observation_operator_artifact, + write_observation_operator, +) + +SYMBOL = "EURUSD" +BASE_PARAMETERS = { + "retention_probability": 1.0, + "unchanged_retention_probability": 1.0, + "timestamp_quantum_ns": 10.0, + "price_precision_digits": 4.0, + "quote_transition_threshold": 0.0, + "batch_window_ns": 0.0, + "duplicate_probability": 0.0, + "rate_cap_per_second": 0.0, + "burst_window_ns": 100.0, + "quiet_gap_probability": 0.0, + "outage_window_ns": 100.0, + "reconnect_duplicate_probability": 0.0, +} + + +def _sha256(value: str) -> str: + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _epoch_evidence(index: int) -> FeedEpochEvidenceV1: + modern = index >= 6 + period = f"2020{index + 1:02d}" + tick_rate = 1_000.0 if modern else 100.0 + interarrival = 100.0 if modern else 1_000.0 + precision = 6.0 if modern else 4.0 + spread = 0.0001 if modern else 0.0008 + features = { + "log_tick_rate_per_hour": math.log1p(tick_rate), + "log_median_interarrival_ms": math.log1p(interarrival), + "log_p95_interarrival_ms": math.log1p(interarrival * 2.5), + "minimum_observed_interval_ms": 10.0 if modern else 1_000.0, + "price_precision_digits": precision, + "spread_median": spread, + "conditioned_spread_median": spread * 1.05, + "absolute_spread_change_median": spread / 4.0, + "stale_repeat_rate": 0.02 if modern else 0.45, + "burst_rate": 0.8 if modern else 0.1, + "duplicate_timestamp_rate": 0.0 if modern else 0.1, + "suspicious_gap_rate": 0.0 if modern else 0.05, + "source_quality_penalty": 0.0 if modern else 0.2, + } + assert set(features) == set(DEFAULT_FEED_EPOCH_FEATURES) + start = index * 10_000 + return FeedEpochEvidenceV1( + symbol=SYMBOL, + period=period, + start_timestamp_utc_ms=start, + end_timestamp_utc_ms=start + 9_000, + fingerprint_id=_sha256(f"fingerprint:{period}"), + source_artifact_sha256=_sha256(f"source:{period}"), + source_hash_basis="persisted_fingerprint_artifact_sha256", + source_kind="cache", + feature_values=features, + feature_provenance={name: (f"fixture.{name}",) for name in features}, + conditioning={ + "calendar_status": "ok", + "session_counts": {"london": 50, "new_york": 50}, + }, + quality={"sequence_status": "ok", "limitations": []}, + profile={ + "row_count": int(tick_rate), + "tick_rate_per_hour": tick_rate, + "median_interarrival_ms": interarrival, + "p95_interarrival_ms": interarrival * 2.5, + "max_interarrival_ms": interarrival * 5.0, + }, + ) + + +@pytest.fixture(scope="module") +def epoch_definition() -> FeedEpochDefinitionV1: + definition = fit_feed_epochs(tuple(_epoch_evidence(i) for i in range(12))) + assert definition.valid_for_observation_models + assert len(definition.epochs) == 2 + return definition + + +def _fit_evidence( + definition: FeedEpochDefinitionV1, + *, + parameters: dict[str, float] | None = None, + support: int = 100, + source_number: int = 1, + state: str | None = "epoch", + session: str | None = "london", + event_tag: str | None = "none", +) -> ObservationFitEvidenceV1: + selected = dict(BASE_PARAMETERS) + selected.update(parameters or {}) + assert set(selected) == set(OBSERVATION_PARAMETER_NAMES) + return ObservationFitEvidenceV1( + context=ObservationContextV1( + symbol=SYMBOL, + epoch_id="epoch-001", + state=state, + session=session, + event_tag=event_tag, + ), + period=f"2020{source_number:02d}", + start_timestamp_ns=source_number * 1_000_000_000, + end_timestamp_ns=source_number * 1_000_000_000 + 100_000_000, + source_evidence_id=f"fixture-evidence-{source_number}", + source_artifact_sha256=_sha256(f"fit-source:{source_number}"), + source_hash_basis="controlled_fixture_sha256", + evidence_kind="controlled_fixture", + parameter_values=selected, + parameter_lower_bounds=selected, + parameter_upper_bounds=selected, + parameter_support_counts={name: support for name in selected}, + parameter_basis={name: "controlled_fixture" for name in selected}, + parameter_provenance={ + name: (f"fixture.parameters.{name}",) for name in selected + }, + ) + + +def _operator( + definition: FeedEpochDefinitionV1, + *, + parameters: dict[str, float] | None = None, + config: ObservationOperatorFitConfigV1 | None = None, +) -> ObservationOperatorV1: + return fit_observation_operator( + (_fit_evidence(definition, parameters=parameters),), + epoch_definition=definition, + config=config + or ObservationOperatorFitConfigV1( + min_stratum_support=1, + min_parameter_support=1, + min_supported_parameters=len(OBSERVATION_PARAMETER_NAMES), + ), + ) + + +def _context( + *, + state: str = "epoch", + session: str = "london", + event_tag: str = "none", +) -> ObservationContextV1: + return ObservationContextV1( + symbol=SYMBOL, + epoch_id="epoch-001", + state=state, + session=session, + event_tag=event_tag, + ) + + +def _event( + ordinal: int, + *, + timestamp: int, + bid: float = 1.23456, + ask: float = 1.23467, + protected: bool = False, + context: ObservationContextV1 | None = None, +) -> ObservationInputEventV1: + return ObservationInputEventV1( + source_event_id=f"market-event-{ordinal}", + symbol=SYMBOL, + event_time_ns=timestamp, + event_sequence=ordinal, + bid=bid, + ask=ask, + context=context or _context(), + protected_anchor=protected, + ) + + +def _window(start: int = 0, end: int = 1_000) -> ReconstructionWindowV1: + return ReconstructionWindowV1( + run_id="run-observation-v1", + ensemble_member_id="member-000", + symbols=(SYMBOL,), + core_start_ns=start, + core_end_ns=end, + ) + + +def test_canonical_epoch_projection_is_replayable_and_does_not_overclaim( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + """Canonical evidence should fit proxies while exposing unknown thinning.""" + source = tuple(_epoch_evidence(index) for index in range(12)) + projected = tuple( + ObservationFitEvidenceV1.from_feed_epoch_evidence( + item, epoch_definition + ) + for item in source + ) + + first = fit_observation_operator( + projected, + epoch_definition=epoch_definition, + ) + second = fit_observation_operator( + tuple(reversed(projected)), + epoch_definition=epoch_definition, + ) + + assert first.operator_id == second.operator_id + assert first.to_dict() == second.to_dict() + assert ObservationOperatorV1.from_json(first.to_json()) == first + assert len(first.source_hashes) == len(source) + assert "retention_probability" in ( + first.diagnostics.unsupported_parameter_names + ) + global_parameter = next( + stratum for stratum in first.strata if stratum.level == "global" + ).parameter_map["retention_probability"] + assert global_parameter.support_status == "unsupported" + assert global_parameter.estimation_bases == ( + "identity_without_dense_denominator", + ) + assert first.lineage["evidence_count"] == len(source) + + +def test_evidence_and_operator_readers_reject_identity_or_lineage_forgery( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + evidence = _fit_evidence(epoch_definition) + evidence_payload = evidence.to_dict() + evidence_payload["evidence_id"] = "observation-evidence:sha256:" + "0" * 64 + with pytest.raises(ValueError, match="evidence_id"): + ObservationFitEvidenceV1.from_dict(evidence_payload) + + operator = _operator(epoch_definition) + payload = operator.to_dict() + lineage = dict(payload["lineage"]) + lineage["evidence_count"] = 99 + payload["lineage"] = lineage + payload["operator_id"] = "" + with pytest.raises(ValueError, match="lineage evidence count"): + ObservationOperatorV1.from_dict(payload) + + payload = operator.to_dict() + lineage = dict(payload["lineage"]) + lineage["unversioned_extra"] = "not-allowed" + payload["lineage"] = lineage + payload["operator_id"] = "" + with pytest.raises(ValueError, match="lineage fields"): + ObservationOperatorV1.from_dict(payload) + + +def test_fit_requires_stability_passing_epoch_definition( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + unstable_stability = replace( + epoch_definition.stability, + status="fail", + unstable_boundary_ids=(epoch_definition.boundaries[0].boundary_id,), + ) + unstable_definition = replace( + epoch_definition, + stability=unstable_stability, + definition_id="", + ) + + with pytest.raises(ValueError, match="not passed stability"): + fit_observation_operator( + (_fit_evidence(epoch_definition),), + epoch_definition=unstable_definition, + ) + + +def test_evidence_periods_and_large_supported_source_sets_are_bounded( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + source = _fit_evidence(epoch_definition) + with pytest.raises(ValueError, match="period must use"): + replace(source, period="202013", evidence_id="") + + evidence = tuple( + replace( + source, + source_evidence_id=f"large-fixture-{index}", + source_artifact_sha256=_sha256(f"large-source-{index}"), + evidence_id="", + ) + for index in range(65) + ) + operator = fit_observation_operator( + evidence, + epoch_definition=epoch_definition, + config=ObservationOperatorFitConfigV1( + min_stratum_support=1, + min_parameter_support=1, + min_supported_parameters=len(OBSERVATION_PARAMETER_NAMES), + ), + ) + + assert operator.diagnostics.evidence_count == 65 + assert len(operator.lineage["evidence_ids"]) == 65 + + +def test_sparse_specific_strata_back_off_to_supported_parent( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + sparse = _fit_evidence( + epoch_definition, + support=1, + source_number=1, + session="asia", + event_tag="release", + ) + supported = _fit_evidence( + epoch_definition, + support=100, + source_number=2, + session="london", + event_tag="none", + ) + operator = fit_observation_operator( + (sparse, supported), + epoch_definition=epoch_definition, + config=ObservationOperatorFitConfigV1( + min_stratum_support=10, + min_parameter_support=10, + min_supported_parameters=len(OBSERVATION_PARAMETER_NAMES), + ), + ) + + stratum, attempted = operator.resolve_stratum(sparse.context) + + assert len(attempted) > 1 + assert stratum.level == "symbol_epoch_state" + assert stratum.status == "ready" + exact = next( + item + for item in operator.strata + if item.key + == sparse.context.key_for_level("symbol_epoch_state_session_event") + ) + assert exact.status == "unsupported" + assert stratum.key in exact.fallback_keys + + +def test_apply_preserves_anchor_and_quantizes_delivery_observation( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + operator = _operator( + epoch_definition, + parameters={"batch_window_ns": 20.0}, + ) + anchor = _event(1, timestamp=15, protected=True) + candidate = _event(2, timestamp=37) + + result = operator.apply( + (candidate, anchor), + window=_window(), + source_start=True, + ) + + assert result.input_count == 2 + assert result.output_count == 2 + protected = next( + event for event in result.output_events if event.protected_anchor + ) + transformed = next( + event for event in result.output_events if not event.protected_anchor + ) + assert (protected.observed_time_ns, protected.bid, protected.ask) == ( + anchor.event_time_ns, + anchor.bid, + anchor.ask, + ) + assert protected.transformations == () + assert transformed.observed_time_ns == 20 + assert (transformed.bid, transformed.ask) == (1.2346, 1.2347) + assert set(transformed.transformations) == { + "batched", + "timestamp_quantized", + "price_quantized", + } + assert ( + ObservationOutputEventV1.from_dict(transformed.to_dict()) == transformed + ) + assert ObservationCarryStateV1.from_dict(result.carry_state.to_dict()) == ( + result.carry_state + ) + assert ObservationApplicationResultV1.from_dict(result.to_dict()) == result + + +def test_ex_ante_apply_refuses_anchors_but_benchmark_degrade_controls_them( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + operator = _operator( + epoch_definition, + parameters={"retention_probability": 0.0}, + ) + anchor = _event(1, timestamp=20, protected=True) + + with pytest.raises(ValueError, match="ex-ante.*anchors"): + operator.apply( + (anchor,), + window=_window(), + information_mode=InformationMode.EX_ANTE_SIMULATION, + source_start=True, + ) + + unprotected = operator.degrade( + (anchor,), + window=_window(), + source_start=True, + ) + protected = operator.degrade( + (anchor,), + window=_window(), + protected_event_ids=(anchor.source_event_id,), + source_start=True, + ) + assert unprotected.output_count == 0 + assert unprotected.reason_counts == {"thinning": 1} + assert protected.output_count == 1 + assert protected.output_events[0].protected_anchor + + +@pytest.mark.parametrize( + ("parameters", "events", "reason", "output_count", "transformation"), + ( + ( + {"retention_probability": 0.0}, + (_event(1, timestamp=20),), + "thinning", + 0, + None, + ), + ( + {"duplicate_probability": 1.0}, + (_event(1, timestamp=20),), + "retained", + 2, + "duplicated", + ), + ( + { + "quiet_gap_probability": 1.0, + "outage_window_ns": 100.0, + }, + (_event(1, timestamp=20),), + "outage", + 0, + None, + ), + ( + {"unchanged_retention_probability": 0.0}, + ( + _event(1, timestamp=20), + _event(2, timestamp=30), + ), + "unchanged_quote_filter", + 1, + None, + ), + ), +) +def test_controlled_fixtures_distinguish_observation_mechanisms( + epoch_definition: FeedEpochDefinitionV1, + parameters: dict[str, float], + events: tuple[ObservationInputEventV1, ...], + reason: str, + output_count: int, + transformation: str | None, +) -> None: + """Thinning, duplication, outages, and unchanged filters stay distinct.""" + operator = _operator(epoch_definition, parameters=parameters) + + result = operator.degrade( + events, + window=_window(), + source_start=True, + ) + + assert result.output_count == output_count + assert result.reason_counts[reason] >= 1 + if transformation is not None: + assert transformation in result.output_events[-1].transformations + + +def test_rate_cap_uses_the_declared_burst_window( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + operator = _operator( + epoch_definition, + parameters={"rate_cap_per_second": 10.0}, + ) + + result = operator.degrade( + (_event(1, timestamp=20), _event(2, timestamp=30)), + window=_window(), + source_start=True, + ) + + assert result.reason_counts == {"rate_cap": 1, "retained": 1} + assert result.output_count == 1 + + +def test_reconnect_behavior_survives_a_streaming_boundary( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + operator = _operator( + epoch_definition, + parameters={ + "quiet_gap_probability": 0.5, + "reconnect_duplicate_probability": 1.0, + }, + ) + events = tuple( + _event( + bucket, + timestamp=bucket * 100 + 20, + bid=1.2 + bucket / 100_000, + ask=1.2001 + bucket / 100_000, + ) + for bucket in range(100) + ) + + whole = operator.degrade( + events, + window=_window(0, 10_000), + source_start=True, + ) + reconnect = next( + event + for event in whole.output_events + if "reconnect_duplicate" in event.transformations + ) + split = reconnect.source_time_ns // 100 * 100 + first_events = tuple( + event for event in events if event.event_time_ns < split + ) + second_events = tuple( + event for event in events if event.event_time_ns >= split + ) + + first = operator.degrade( + first_events, + window=_window(0, split), + source_start=True, + ) + assert first.carry_state.reconnect_pending + second = operator.degrade( + second_events, + window=_window(split, 10_000), + carry=first.carry_state, + ) + + assert tuple(event.to_dict() for event in whole.output_events) == tuple( + event.to_dict() + for event in (*first.output_events, *second.output_events) + ) + assert whole.carry_state == second.carry_state + + +def test_window_carry_matches_single_window_and_is_required_when_declared( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + config = ObservationOperatorFitConfigV1( + min_stratum_support=1, + min_parameter_support=1, + min_supported_parameters=len(OBSERVATION_PARAMETER_NAMES), + required_left_halo_ns=20, + ) + operator = _operator(epoch_definition, config=config) + first_event = _event(1, timestamp=20) + second_event = _event(2, timestamp=120, bid=1.2347, ask=1.2348) + + whole = operator.apply( + (first_event, second_event), + window=_window(0, 200), + source_start=True, + ) + first = operator.apply( + (first_event,), + window=_window(0, 100), + source_start=True, + ) + with pytest.raises(ValueError, match="requires carry"): + operator.apply((second_event,), window=_window(100, 200)) + second = operator.apply( + (second_event,), + window=_window(100, 200), + carry=first.carry_state, + ) + + assert tuple(event.to_dict() for event in whole.output_events) == tuple( + event.to_dict() + for event in (*first.output_events, *second.output_events) + ) + assert whole.carry_state == second.carry_state + + with pytest.raises(ValueError, match="watermark is stale or overlapping"): + operator.apply( + (first_event,), + window=_window(0, 100), + carry=first.carry_state, + ) + + +def test_operator_artifact_replays_by_hash_and_rejects_tampering( + epoch_definition: FeedEpochDefinitionV1, + tmp_path: Path, +) -> None: + operator = _operator(epoch_definition) + path = tmp_path / "operator.json" + + artifact = write_observation_operator(operator, path) + + assert read_observation_operator_artifact(artifact) == operator + path.write_text(operator.to_json() + " \n", encoding="utf-8") + with pytest.raises(ValueError, match="size differs|hash differs"): + read_observation_operator_artifact(artifact) + + +def test_fit_and_application_resource_limits_fail_before_unbounded_work( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + evidence = _fit_evidence(epoch_definition) + with pytest.raises(ValueError, match="evidence.*limit"): + fit_observation_operator( + (evidence, replace(evidence, evidence_id="")), + epoch_definition=epoch_definition, + config=ObservationOperatorFitConfigV1(max_evidence=1), + ) + + operator = _operator( + epoch_definition, + config=ObservationOperatorFitConfigV1( + min_stratum_support=1, + min_parameter_support=1, + min_supported_parameters=len(OBSERVATION_PARAMETER_NAMES), + max_input_events=1, + ), + ) + with pytest.raises(ValueError, match="input event limit"): + operator.apply( + (_event(1, timestamp=20), _event(2, timestamp=30)), + window=_window(), + source_start=True, + ) + + +def test_input_order_does_not_change_delivery_result( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + operator = _operator( + epoch_definition, + parameters={"duplicate_probability": 1.0}, + ) + events = ( + _event(1, timestamp=20), + _event(2, timestamp=30, bid=1.2347, ask=1.2348), + ) + + forward = operator.degrade(events, window=_window(), source_start=True) + reverse = operator.degrade( + tuple(reversed(events)), window=_window(), source_start=True + ) + + assert forward.to_dict() == reverse.to_dict() + + +def test_application_result_rejects_incomplete_reason_accounting( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + result = _operator(epoch_definition).apply( + (_event(1, timestamp=20),), + window=_window(), + source_start=True, + ) + payload = result.to_dict() + payload["reason_counts"] = {"retained": 2} + payload["result_id"] = "" + + with pytest.raises(ValueError, match="reason counts do not cover"): + ObservationApplicationResultV1.from_dict(payload) + + +def test_window_boundaries_must_align_with_quantization_contract( + epoch_definition: FeedEpochDefinitionV1, +) -> None: + operator = _operator(epoch_definition) + + with pytest.raises(ValueError, match="not aligned"): + operator.apply( + (_event(1, timestamp=21),), + window=_window(1, 101), + source_start=True, + ) diff --git a/tests/unit/test_synthetic_observation_calibration.py b/tests/unit/test_synthetic_observation_calibration.py new file mode 100644 index 00000000..18231c3a --- /dev/null +++ b/tests/unit/test_synthetic_observation_calibration.py @@ -0,0 +1,428 @@ +"""Tests for real-evidence observation calibration and holdout gates.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timezone +import hashlib +from pathlib import Path + +import polars as pl +import pytest + +from histdatacom.data_analytics import ( + FeedEpochDefinitionV2, + FeedEpochEvidenceV2, + FeedEpochFitConfigV2, + FeedEpochIntervalV2, + FeedEpochStabilityV2, +) +from histdatacom.synthetic import ( + BenchmarkSplitKind, + ObservationCalibrationCampaignV2, + ObservationCalibrationProfileV2, + ObservationContextV1, + ObservationFitEvidenceV1, + ObservationInputEventV1, + ObservationOperatorFitConfigV1, + ReconstructionWindowV1, + calibrate_historical_observation_operators, + estimate_paired_observation_evidence, + fit_observation_operator, + read_observation_calibration_campaign, + write_observation_calibration_campaign, +) + +SYMBOLS = ("EURGBP", "EURUSD", "GBPUSD") + + +def _month_start(period: str) -> int: + return int( + datetime( + int(period[:4]), + int(period[4:]), + 1, + tzinfo=timezone.utc, + ).timestamp() + * 1000 + ) + + +def _sha256(value: str) -> str: + return "sha256:" + hashlib.sha256(value.encode()).hexdigest() + + +def _definition() -> FeedEpochDefinitionV2: + start = _month_start("201901") + end = _month_start("202301") - 1 + config = FeedEpochFitConfigV2() + return FeedEpochDefinitionV2( + config=config, + symbols=SYMBOLS, + coverage_start_utc_ms=start, + coverage_end_utc_ms=end, + evidence_count=144, + period_count=48, + feature_names=config.feature_names, + boundaries=(), + epochs=( + FeedEpochIntervalV2( + label="technology_epoch_01", + period_start="201901", + period_end="202212", + start_timestamp_utc_ms=start, + end_timestamp_utc_ms=end, + evidence_count=144, + feature_medians={name: 0.0 for name in config.feature_names}, + ), + ), + symbol_deviations=(), + stability=FeedEpochStabilityV2( + status="pass", + reasons=(), + run_count=1, + run_counts={"fixture": 1}, + boundary_support={}, + boundary_support_by_family={}, + rejected_candidates={}, + feature_coverage={name: 1.0 for name in config.feature_names}, + common_period_count=48, + symbol_count=3, + ), + lineage={"fixture": True}, + ) + + +def _tick_cache(path: Path) -> str: + start = _month_start("202001") + rows = 24 * 60 * 4 + bids = [1.1 + (index % 11) * 0.00001 for index in range(rows)] + frame = pl.DataFrame( + { + "datetime": [start + index * 15_000 for index in range(rows)], + "bid": bids, + "ask": [value + 0.0001 for value in bids], + "vol": [0] * rows, + } + ) + path.parent.mkdir(parents=True, exist_ok=True) + frame.write_ipc(path) + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def _epoch_evidence( + *, + symbol: str, + period: str, + path: Path, + source_hash: str, + row_count: int = 5760, +) -> FeedEpochEvidenceV2: + start = _month_start(period) + features = { + "log_active_window_tick_rate_per_hour": 7.0, + "timestamp_exact_second_rate": 1.0, + "timestamp_last_digit_entropy": 0.0, + "price_precision_digits": 5.0, + "duplicate_timestamp_rate": 0.1, + "log_interarrival_median_ms": 5.0, + "bid_only_rate": 0.0, + "ask_only_rate": 0.0, + "joint_move_rate": 1.0, + "unchanged_rate": 0.0, + "session_activity_share_asia": 0.25, + "session_activity_share_london": 0.25, + "session_activity_share_new_york": 0.25, + "session_activity_share_off_session": 0.25, + } + return FeedEpochEvidenceV2( + symbol=symbol, + period=period, + source_path=str(path), + source_artifact_sha256=source_hash, + source_size_bytes=path.stat().st_size, + start_timestamp_utc_ms=start, + end_timestamp_utc_ms=start + 2_000_000, + row_count=row_count, + denominators_ms={ + "calendar_duration_ms": 2_000_000, + "market_open_duration_ms": 1_500_000, + "active_window_duration_ms": 1_000_000, + }, + counts={ + "transition_count": row_count - 1, + "market_open_row_count": row_count - 100, + "active_window_interval_count": row_count - 200, + }, + feature_values=features, + feature_provenance={name: (f"fixture.{name}",) for name in features}, + activity_bin_counts={str(start // 3_600_000): 100}, + calendar_policy={"active_time_policy_version": "fixture.v1"}, + ) + + +def _campaign_evidence(tmp_path: Path) -> tuple[FeedEpochEvidenceV2, ...]: + cache_by_symbol: dict[str, tuple[Path, str]] = {} + for symbol in SYMBOLS: + path = tmp_path / symbol / ".data" + cache_by_symbol[symbol] = (path, _tick_cache(path)) + result = [] + for year in range(2019, 2023): + for month in range(1, 13): + period = f"{year}{month:02d}" + for symbol in SYMBOLS: + path, source_hash = cache_by_symbol[symbol] + result.append( + _epoch_evidence( + symbol=symbol, + period=period, + path=path, + source_hash=source_hash, + ) + ) + return tuple(result) + + +def test_real_campaign_is_blocked_bounded_replayable_and_fail_closed( + tmp_path: Path, +) -> None: + """A ready claim requires three time blocks and passing final holdouts.""" + definition = _definition() + profile = ObservationCalibrationProfileV2( + split_periods={ + "calibration": "202001", + "validation": "202101", + "final_holdout": "202201", + }, + sessions=("asia", "london", "new_york"), + max_events_per_window=128, + minimum_events_per_window=64, + ) + + evidence = _campaign_evidence(tmp_path) + campaign = calibrate_historical_observation_operators( + evidence, + epoch_definition=definition, + profile=profile, + ) + repeated = calibrate_historical_observation_operators( + evidence, + epoch_definition=definition, + profile=profile, + ) + + assert campaign.valid_for_application + assert repeated.campaign_id == campaign.campaign_id + assert repeated.operator.operator_id == campaign.operator.operator_id + assert {item.split_kind for item in campaign.windows} == set( + ( + BenchmarkSplitKind.CALIBRATION, + BenchmarkSplitKind.VALIDATION, + BenchmarkSplitKind.FINAL_HOLDOUT, + ) + ) + assert all(item.input_count <= 128 for item in campaign.windows) + assert all( + item.passed + for item in campaign.windows + if item.split_kind is BenchmarkSplitKind.FINAL_HOLDOUT + ) + assert all( + target.parameter_status["retention_probability"] == "supported" + for target in campaign.targets + ) + assert all( + target.parameter_status["outage_window_ns"] == "unsupported" + for target in campaign.targets + ) + assert all( + target.parameter_support_counts["retention_probability"] > 0 + and target.parameter_support_counts["outage_window_ns"] == 0 + for target in campaign.targets + ) + assert all( + target.mechanism_diagnostics["calendar_closure"] + != target.mechanism_diagnostics["archive_gap"] + for target in campaign.targets + ) + context = ObservationContextV1( + symbol="EURUSD", + epoch_id="technology_epoch_01", + state="update_joint", + session="london", + ) + campaign.require_application_ready(context) + with pytest.raises(ValueError, match="unsupported"): + campaign.require_application_ready( + context, + required_parameters=("outage_window_ns",), + ) + + artifacts = write_observation_calibration_campaign( + campaign, tmp_path / "artifacts" + ) + restored = read_observation_calibration_campaign(artifacts["campaign"].path) + assert restored.campaign_id == campaign.campaign_id + assert restored.operator.operator_id == campaign.operator.operator_id + + target = campaign.targets[0] + unsafe = replace( + target, + parameter_reasons={ + **target.parameter_reasons, + "retention_probability": "identity_without_dense_denominator", + }, + target_id="", + ) + with pytest.raises(ValueError, match="readiness evidence differs"): + ObservationCalibrationCampaignV2( + feed_epoch_definition_id=campaign.feed_epoch_definition_id, + calibration_corpus_sha256=campaign.calibration_corpus_sha256, + profile=campaign.profile, + operator=campaign.operator, + targets=(unsafe, *campaign.targets[1:]), + fit_evidence=campaign.fit_evidence, + windows=campaign.windows, + readiness_status="ready", + readiness_reasons=(), + runtime_seconds=campaign.runtime_seconds, + peak_memory_bytes=campaign.peak_memory_bytes, + ) + + +def test_controlled_pair_recovers_known_state_dependent_parameters() -> None: + """Known state-specific thinning, timestamp grid, and duplicates recover.""" + definition = _definition() + start_ns = _month_start("202001") * 1_000_000 + contexts = { + "update_bid_only": ObservationContextV1( + symbol="EURUSD", + epoch_id="technology_epoch_01", + state="update_bid_only", + session="london", + ), + "update_joint": ObservationContextV1( + symbol="EURUSD", + epoch_id="technology_epoch_01", + state="update_joint", + session="london", + ), + } + source_hash = _sha256("controlled-pair") + fit_rows = [] + for ordinal, (state, probability) in enumerate( + (("update_bid_only", 0.25), ("update_joint", 0.75)), start=1 + ): + values = { + "retention_probability": probability, + "timestamp_quantum_ns": 1_000_000_000.0, + "duplicate_probability": 0.2, + } + fit_rows.append( + ObservationFitEvidenceV1( + context=contexts[state], + period="202001", + start_timestamp_ns=start_ns, + end_timestamp_ns=start_ns + 1_000_000_000_000, + source_evidence_id=f"controlled-{ordinal}", + source_artifact_sha256=source_hash, + source_hash_basis="controlled_fixture_sha256", + evidence_kind="controlled_fixture", + parameter_values=values, + parameter_lower_bounds=values, + parameter_upper_bounds=values, + parameter_support_counts={name: 2500 for name in values}, + parameter_basis={name: "controlled_fixture" for name in values}, + parameter_provenance={ + name: (f"fixture.{name}",) for name in values + }, + ) + ) + operator = fit_observation_operator( + fit_rows, + epoch_definition=definition, + config=ObservationOperatorFitConfigV1( + min_stratum_support=1, + min_parameter_support=1, + min_supported_parameters=3, + max_input_events=5000, + ), + ) + events = tuple( + ObservationInputEventV1( + source_event_id=f"controlled-event-{index}", + symbol="EURUSD", + event_time_ns=start_ns + index * 137_000_000, + event_sequence=index, + bid=1.1 + (index % 7) * 0.00001, + ask=1.1001 + (index % 7) * 0.00001, + context=contexts[ + "update_bid_only" if index % 2 == 0 else "update_joint" + ], + ) + for index in range(5000) + ) + applied = operator.degrade( + events, + window=ReconstructionWindowV1( + run_id="controlled-recovery", + ensemble_member_id="member-000", + symbols=("EURUSD",), + core_start_ns=start_ns, + core_end_ns=(events[-1].event_time_ns // 1_000_000_000 + 1) + * 1_000_000_000, + ), + source_start=True, + ) + + recovered = estimate_paired_observation_evidence( + events, + applied, + period="202001", + source_artifact_sha256=source_hash, + ) + by_state = {item.context.state: item for item in recovered} + assert by_state["update_bid_only"].parameter_values[ + "retention_probability" + ] == pytest.approx(0.25, abs=0.04) + assert by_state["update_joint"].parameter_values[ + "retention_probability" + ] == pytest.approx(0.75, abs=0.04) + assert all( + item.parameter_values["timestamp_quantum_ns"] == 1_000_000_000 + for item in recovered + ) + assert all( + item.parameter_values["duplicate_probability"] + == pytest.approx(0.2, abs=0.04) + for item in recovered + ) + + +def test_real_v2_projection_caps_support_and_keeps_integral_durations( + tmp_path: Path, +) -> None: + """Monthly real row counts must fit the bounded v1 evidence bridge.""" + path = tmp_path / ".data" + source_hash = _tick_cache(path) + evidence = _epoch_evidence( + symbol="EURUSD", + period="202001", + path=path, + source_hash=source_hash, + row_count=500_000, + ) + + projected = ObservationFitEvidenceV1.from_feed_epoch_evidence( + evidence, _definition() + ) + + assert max(projected.parameter_support_counts.values()) == 250_000 + assert projected.parameter_support_counts["retention_probability"] == 0 + for name in ( + "timestamp_quantum_ns", + "batch_window_ns", + "burst_window_ns", + "outage_window_ns", + ): + assert projected.parameter_values[name].is_integer() diff --git a/tests/unit/test_synthetic_persistence.py b/tests/unit/test_synthetic_persistence.py new file mode 100644 index 00000000..3e5759cb --- /dev/null +++ b/tests/unit/test_synthetic_persistence.py @@ -0,0 +1,673 @@ +"""Atomic final-product persistence for reconstructed event streams.""" + +from __future__ import annotations + +from dataclasses import replace +import json +from pathlib import Path + +import duckdb +import polars as pl +import pyarrow.parquet as pq +import pytest + +from histdatacom.broker_capture import fit_broker_delivery_fingerprint +from histdatacom.synthetic import ( + RECONSTRUCTION_PRODUCT_DIRECTORY, + BrokerTransferConfigV1, + ReconstructionCheckpointV1, + ReconstructionCommitPhase, + ReconstructionProductManifestV2, + ReconstructionPersistenceError, + ReconstructionProductManifestV1, + ReconstructionStoragePolicyV1, + ReconstructionStoragePreflightError, + SyntheticEventOrigin, + cleanup_reconstruction_scratch, + commit_delivery_reconstruction_publication, + commit_reconstruction_publication, + discover_reconstruction_manifests, + estimate_reconstruction_retention, + iter_reconstruction_event_batches, + publish_reconstruction_group, + read_reconstruction_streams, + reconstruction_parquet_paths, + project_modern_reference_delivery, + render_broker_delivery, + scan_reconstruction_events_polars, + stage_reconstruction_publication, + stage_delivery_reconstruction_publication, + validate_cross_currency_output, + verify_reconstruction_publication, +) +from histdatacom.synthetic.cross_currency import CrossCurrencyValidationStage +from histdatacom.synthetic.contracts import SYNTHETIC_EVENT_ARROW_COLUMNS +from tests.unit.test_broker_delivery_fingerprints import ( + BASE_WALL_NS, + _capture, +) +from tests.unit.test_synthetic_broker_transfer import _group_with_constraints + + +def test_publish_exact_narrow_schema_reconciles_anchors_and_replays( + tmp_path: Path, +) -> None: + """The committed product contains exact #431 rows and compact evidence.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + + published = publish_reconstruction_group( + tmp_path / "archive", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + row_group_size=2, + ) + + assert not published.idempotent_retry + manifest = verify_reconstruction_publication(published.manifest_path) + assert manifest == published.manifest + assert ReconstructionProductManifestV1.from_json(manifest.to_json()) == ( + manifest + ) + assert manifest.event_count == sum( + len(stream.events) for stream in rendered.streams + ) + assert manifest.observed_event_count == len(anchors) + assert manifest.synthetic_event_count == ( + manifest.event_count - len(anchors) + ) + assert manifest.quality.broker_transfer_manifest_id == ( + rendered.manifest.manifest_id + ) + assert manifest.source.observed_content_sha256 + assert manifest.constraints.constraint_set_ids + assert manifest.replay.logical_content_sha256 + assert manifest.ensemble.materialized_member_id == ( + rendered.manifest.ensemble_member_id + ) + assert manifest.ensemble.retention_plan_id == retention.plan_id + assert manifest.ensemble.to_dict()["automatic_winner"] is False + assert manifest.retention.plan_id == retention.plan_id + payload = manifest.to_dict() + assert payload["event_rows_inline"] is False + assert payload["analytical_frame_columns_inline"] is False + assert "events" not in json.dumps(payload) + + for partition in manifest.partitions: + table = pq.ParquetFile( + published.manifest_path.parent / partition.relative_path + ).read() + assert table.schema.names == list(SYNTHETIC_EVENT_ARROW_COLUMNS) + assert table.num_columns == 26 + assert table.num_columns < 521 + assert table.num_rows == partition.row_count + assert partition.row_group_count >= 1 + assert any(item.row_group_count > 1 for item in manifest.partitions) + assert read_reconstruction_streams(published.manifest_path) == ( + rendered.streams + ) + + +def test_staging_is_invisible_and_commit_is_atomic_and_idempotent( + tmp_path: Path, +) -> None: + """Only directory promotion makes a synchronized unit discoverable.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + root = tmp_path / "archive" + + staged = stage_reconstruction_publication( + root, + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + + assert staged.staging_directory.exists() + assert staged.manifest_ref.path == str(staged.manifest_path) + assert discover_reconstruction_manifests(root) == () + + committed = commit_reconstruction_publication(staged) + assert not committed.idempotent_retry + assert not staged.staging_directory.exists() + assert discover_reconstruction_manifests(root) == (committed.manifest_path,) + + retry = commit_reconstruction_publication(staged) + assert retry.idempotent_retry + assert retry.manifest == committed.manifest + assert retry.manifest_ref.sha256 == committed.manifest_ref.sha256 + + publish_retry = publish_reconstruction_group( + root, + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + assert publish_retry.idempotent_retry + assert publish_retry.manifest == committed.manifest + + +def test_generic_delivery_commit_recovers_after_atomic_rename( + tmp_path: Path, +) -> None: + """A retry after rename does not depend on the vanished staging path.""" + run, window, group, _constraints = _group_with_constraints() + anchors = tuple( + event + for stream in group.streams + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + ) + validation = validate_cross_currency_output( + run=run, + window=window, + streams={stream.symbol: stream for stream in group.streams}, + config=group.config, + stage=CrossCurrencyValidationStage.POST_BROKER, + observed_anchors=anchors, + ) + delivered = project_modern_reference_delivery( + group, delivery_profile_id="modern-reference:fixture" + ) + retention = estimate_reconstruction_retention( + run_id=run.run_id, + primary_member_id=window.ensemble_member_id, + retained_member_event_counts={ + window.ensemble_member_id: sum( + len(stream.events) for stream in delivered.streams + ) + }, + estimated_partition_count=len(delivered.streams), + storage_policy=run.storage_policy, + ) + root = tmp_path / "archive" + staged = stage_delivery_reconstruction_publication( + root, + delivered, + final_validation=validation, + benchmark_artifact_ids=("benchmark:fixture",), + benchmark_evidence={"gate": "passed"}, + immutable_source_anchors=anchors, + symbol_group_id=window.synchronization_unit_id, + retention_plan=retention, + storage_policy=run.storage_policy, + staging_root=tmp_path / "window-scratch" / "publication", + row_group_size=2, + ) + + assert discover_reconstruction_manifests(root) == () + committed = commit_delivery_reconstruction_publication(staged) + assert not committed.idempotent_retry + assert not staged.staging_directory.exists() + assert isinstance(committed.manifest, ReconstructionProductManifestV2) + assert discover_reconstruction_manifests( + root, + delivery_profile_id="modern-reference:fixture", + ) == (committed.manifest_path,) + + recovered = commit_delivery_reconstruction_publication(staged) + assert recovered.idempotent_retry + assert recovered.manifest == committed.manifest + assert read_reconstruction_streams(committed.manifest_path) == ( + delivered.streams + ) + + +def test_idempotent_publish_retry_revalidates_committed_partitions( + tmp_path: Path, +) -> None: + """A matching manifest cannot hide corrupt committed Parquet on retry.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + root = tmp_path / "archive" + published = publish_reconstruction_group( + root, + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + partition = published.manifest.partitions[0] + partition_path = published.manifest_path.parent / partition.relative_path + partition_path.write_bytes(partition_path.read_bytes()[:32]) + + with pytest.raises( + ReconstructionPersistenceError, + match="byte size differs", + ): + publish_reconstruction_group( + root, + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + + +def test_manifest_refs_complete_the_checkpoint_two_phase_commit( + tmp_path: Path, +) -> None: + """Staged and promoted byte identities plug into the #432 state machine.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + staged = stage_reconstruction_publication( + tmp_path / "archive", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + checkpoint = ReconstructionCheckpointV1( + run_id=rendered.manifest.run_id, + window_id=rendered.manifest.window_id, + synchronization_unit_id=rendered.manifest.synchronization_unit_id, + revision=1, + phase=ReconstructionCommitPhase.RUNNING, + ) + staged_checkpoint = checkpoint.transition( + ReconstructionCommitPhase.STAGED, + expected_checkpoint_id=checkpoint.checkpoint_id, + staged_manifest_ref=staged.manifest_ref, + ) + validated_checkpoint = staged_checkpoint.transition( + ReconstructionCommitPhase.VALIDATED, + expected_checkpoint_id=staged_checkpoint.checkpoint_id, + ) + + published = commit_reconstruction_publication(staged) + committed_checkpoint = validated_checkpoint.transition( + ReconstructionCommitPhase.COMMITTED, + expected_checkpoint_id=validated_checkpoint.checkpoint_id, + committed_manifest_ref=published.manifest_ref, + ) + + assert committed_checkpoint.advertised_manifest_ref == ( + published.manifest_ref + ) + assert committed_checkpoint.staged_manifest_ref is None + assert staged_checkpoint.advertised_manifest_ref is None + + +def test_truncated_staging_fails_closed_before_publication( + tmp_path: Path, +) -> None: + """A damaged Parquet footer can never cross the commit boundary.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + root = tmp_path / "archive" + staged = stage_reconstruction_publication( + root, + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + partition = staged.manifest.partitions[0] + partition_path = staged.staging_directory / partition.relative_path + partition_path.write_bytes(partition_path.read_bytes()[:32]) + + with pytest.raises( + ReconstructionPersistenceError, + match="byte size differs", + ): + commit_reconstruction_publication(staged) + + assert discover_reconstruction_manifests(root) == () + assert cleanup_reconstruction_scratch(root) == (staged.staging_directory,) + assert not staged.staging_directory.exists() + + +def test_anchor_value_drift_refuses_even_when_observed_id_is_unchanged( + tmp_path: Path, +) -> None: + """Publication compares immutable values as well as stable source IDs.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + first = anchors[0] + changed = replace(first, bid=first.bid + 0.0001) + + with pytest.raises( + ReconstructionPersistenceError, + match="values or IDs differ", + ): + publish_reconstruction_group( + tmp_path / "archive", + rendered, + immutable_source_anchors=(changed, *anchors[1:]), + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + + assert discover_reconstruction_manifests(tmp_path / "archive") == () + + +def test_retention_preflight_estimates_primary_and_all_retained_members() -> ( + None +): + """Primary, retained-member, manifest, and policy bytes reconcile early.""" + policy = ReconstructionStoragePolicyV1( + max_output_bytes=10_000_000, + max_retained_ensemble_members=3, + ) + plan = estimate_reconstruction_retention( + run_id="run:fixture", + primary_member_id="member-000", + retained_member_event_counts={ + "member-000": 1_000, + "member-001": 1_500, + "member-002": 2_000, + }, + estimated_partition_count=9, + storage_policy=policy, + estimated_bytes_per_event=200, + estimated_compression_ratio=0.5, + ) + + assert plan.estimated_primary_bytes == 100_000 + assert plan.estimated_retained_bytes == 450_000 + assert plan.estimated_manifest_bytes == 9 * 4_096 + assert plan.estimated_total_output_bytes == ( + plan.estimated_retained_bytes + plan.estimated_manifest_bytes + ) + assert plan.storage_policy_id == policy.policy_id + + limited = ReconstructionStoragePolicyV1( + max_output_bytes=100, + max_retained_ensemble_members=1, + ) + with pytest.raises(ReconstructionStoragePreflightError) as raised: + estimate_reconstruction_retention( + run_id="run:fixture", + primary_member_id="member-000", + retained_member_event_counts={ + "member-000": 1_000, + "member-001": 1_500, + }, + estimated_partition_count=2, + storage_policy=limited, + ) + assert len(raised.value.violations) == 2 + + +def test_cleanup_removes_only_transaction_scratch( + tmp_path: Path, +) -> None: + """Cleanup cannot touch committed products or immutable source evidence.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + root = tmp_path / "archive" + committed = publish_reconstruction_group( + root, + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + staged = stage_reconstruction_publication( + root, + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + source = root / "immutable-source.data" + source.write_text("evidence", encoding="utf-8") + unrelated = staged.staging_directory.parent / "operator-note.txt" + unrelated.write_text("keep", encoding="utf-8") + + removed = cleanup_reconstruction_scratch(root) + + assert removed == (staged.staging_directory,) + assert committed.manifest_path.exists() + assert source.read_text(encoding="utf-8") == "evidence" + assert unrelated.read_text(encoding="utf-8") == "keep" + assert discover_reconstruction_manifests(root) == (committed.manifest_path,) + + +def test_arrow_and_polars_scans_prune_files_columns_and_rows( + tmp_path: Path, +) -> None: + """Manifest pruning and lazy readers avoid unrelated product columns.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + published = publish_reconstruction_group( + tmp_path / "archive", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + eurusd = next( + stream for stream in rendered.streams if stream.symbol == "eurusd" + ) + start = eurusd.events[0].event_time_ns + end = eurusd.events[-1].event_time_ns + 1 + paths = reconstruction_parquet_paths( + published.manifest_path, + symbols=("eurusd",), + start_ns=start, + end_ns=end, + ) + + assert paths + assert all("symbol=eurusd" in str(path) for path in paths) + assert all("symbol=gbpusd" not in str(path) for path in paths) + batches = tuple( + iter_reconstruction_event_batches( + published.manifest_path, + columns=("symbol", "event_time_ns", "bid"), + symbols=("eurusd",), + start_ns=start, + end_ns=end, + batch_size=1, + ) + ) + assert batches + assert all( + batch.schema.names == ["symbol", "event_time_ns", "bid"] + for batch in batches + ) + lazy = scan_reconstruction_events_polars( + published.manifest_path, + columns=("symbol", "event_time_ns", "bid"), + symbols=("eurusd",), + start_ns=start, + end_ns=end, + ) + plan = lazy.explain(optimized=True) + frame = lazy.collect() + assert frame.columns == ["symbol", "event_time_ns", "bid"] + assert frame["symbol"].unique().to_list() == ["eurusd"] + assert frame.height == len(eurusd.events) + assert "PROJECT 3/26 COLUMNS" in plan + assert "SELECTION" in plan + + +def test_duckdb_smoke_proves_parquet_projection_and_filter_pushdown( + tmp_path: Path, +) -> None: + """DuckDB reads the committed paths with projected columns and filters.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + published = publish_reconstruction_group( + tmp_path / "archive", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + eurusd = next( + stream for stream in rendered.streams if stream.symbol == "eurusd" + ) + start = eurusd.events[1].event_time_ns + end = eurusd.events[-1].event_time_ns + 1 + paths = reconstruction_parquet_paths( + published.manifest_path, + symbols=("eurusd",), + start_ns=start, + end_ns=end, + ) + literals = ",".join( + "'" + str(path).replace("'", "''") + "'" for path in paths + ) + query = ( + "SELECT symbol, event_time_ns, bid " + f"FROM read_parquet([{literals}]) " + f"WHERE event_time_ns >= {start} AND event_time_ns < {end}" + ) + + connection = duckdb.connect() + try: + plan_rows = connection.execute("EXPLAIN " + query).fetchall() + rows = connection.execute(query).fetchall() + finally: + connection.close() + plan = "\n".join(str(value) for row in plan_rows for value in row) + + expected_rows = sum(event.event_time_ns >= start for event in eurusd.events) + assert len(rows) == expected_rows + assert {row[0] for row in rows} == {"eurusd"} + assert "Projections:" in plan + assert "symbol" in plan and "event_time_ns" in plan and "bid" in plan + assert "Filters:" in plan + + +def test_same_writer_runtime_produces_stable_partition_byte_hashes( + tmp_path: Path, +) -> None: + """Physical hashes are stable across roots under the pinned writer.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + first = publish_reconstruction_group( + tmp_path / "archive-a", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + row_group_size=2, + ) + second = publish_reconstruction_group( + tmp_path / "archive-b", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + row_group_size=2, + ) + + assert first.manifest.publication_id == second.manifest.publication_id + assert first.manifest.manifest_id == second.manifest.manifest_id + assert first.manifest.replay.partition_byte_sha256 == ( + second.manifest.replay.partition_byte_sha256 + ) + assert [item.byte_sha256 for item in first.manifest.partitions] == [ + item.byte_sha256 for item in second.manifest.partitions + ] + + +def test_manifest_tampering_fails_closed( + tmp_path: Path, +) -> None: + """Derived counts and deterministic manifest identity reject mutation.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + published = publish_reconstruction_group( + tmp_path / "archive", + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + payload = json.loads(published.manifest_path.read_text(encoding="utf-8")) + payload["event_count"] += 1 + published.manifest_path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="event_count"): + verify_reconstruction_publication(published.manifest_path) + + +def test_discovery_filters_member_run_broker_and_group_axes( + tmp_path: Path, +) -> None: + """Committed lookup uses manifest axes without scanning unrelated rows.""" + rendered, anchors, policy, retention = _publication_inputs(tmp_path) + root = tmp_path / "archive" + published = publish_reconstruction_group( + root, + rendered, + immutable_source_anchors=anchors, + symbol_group_id="eurusd-triangle", + retention_plan=retention, + storage_policy=policy, + ) + + assert discover_reconstruction_manifests( + root, + run_id=rendered.manifest.run_id, + broker_profile_id=rendered.manifest.fingerprint_id, + ensemble_member_id=rendered.manifest.ensemble_member_id, + symbol_group_id="eurusd-triangle", + ) == (published.manifest_path,) + assert ( + discover_reconstruction_manifests( + root, ensemble_member_id="other-member" + ) + == () + ) + assert RECONSTRUCTION_PRODUCT_DIRECTORY in str(published.manifest_path) + + +def _publication_inputs(tmp_path: Path): + run, window, group, constraints = _group_with_constraints() + capture = _capture( + tmp_path / "broker-capture", + seed=446, + wall_start_ns=BASE_WALL_NS, + ) + fingerprint = fit_broker_delivery_fingerprint( + tmp_path / "broker-capture", (capture,) + ) + rendered = render_broker_delivery( + run=run, + window=window, + group=group, + fingerprint=fingerprint, + constraints=constraints, + selected_at_utc_ns=fingerprint.effective_start_utc_ns, + config=BrokerTransferConfigV1( + strength=0.25, + max_events_per_group=100, + ), + quality_period="202001", + ) + anchors = tuple( + event + for stream in group.streams + for event in stream.events + if event.origin is SyntheticEventOrigin.OBSERVED + ) + policy = run.storage_policy + event_count = sum(len(stream.events) for stream in rendered.streams) + retention = estimate_reconstruction_retention( + run_id=run.run_id, + primary_member_id=rendered.manifest.ensemble_member_id, + retained_member_event_counts={ + rendered.manifest.ensemble_member_id: event_count + }, + estimated_partition_count=len(rendered.streams), + storage_policy=policy, + ) + return rendered, anchors, policy, retention + + +def test_polars_dependency_is_the_supported_runtime() -> None: + """Keep the smoke-test import visible to dependency and packaging audits.""" + assert tuple(int(part) for part in pl.__version__.split(".")[:2]) >= (1, 41) diff --git a/tests/unit/test_synthetic_strategy_sensitivity.py b/tests/unit/test_synthetic_strategy_sensitivity.py new file mode 100644 index 00000000..4bb03e70 --- /dev/null +++ b/tests/unit/test_synthetic_strategy_sensitivity.py @@ -0,0 +1,801 @@ +"""Regression tests for bounded reconstructed-history strategy sensitivity.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Sequence + +import pytest + +from histdatacom.synthetic import ( + ReferenceMomentumStrategyV1, + StrategyEvaluationCaseV1, + StrategyEvaluationFailure, + StrategyEvaluationPlanV1, + StrategyEvaluationPolicyV1, + StrategyExecutionSpecificationV1, + StrategyQuoteV1, + StrategySensitivityReportV1, + StrategySignalStateV1, + StrategySignalV1, + StrategySourceKind, + StrategyWindowStatus, + evaluate_strategy_sensitivity, + strategy_sensitivity_benchmark_hooks, +) +from histdatacom.synthetic.benchmark import BenchmarkEventV1 +from histdatacom.synthetic.information import ( + InformationAuditReportV1, + InformationMode, +) +from tests.unit.test_synthetic_bars import _derive_boundary_bars +from tests.unit.test_synthetic_contracts import _generated, _observed + +RUN_ID = "reconstruction-run:sha256:strategy-fixture" +MANIFEST_ID = "information-manifest:sha256:strategy-fixture" +SECOND = 1_000_000_000 + + +def _audit( + mode: InformationMode = InformationMode.EX_ANTE_SIMULATION, + *, + strategy_valid: bool | None = None, +) -> InformationAuditReportV1: + if strategy_valid is None: + strategy_valid = mode is InformationMode.EX_ANTE_SIMULATION + return InformationAuditReportV1( + run_id=RUN_ID, + policy_id="information-policy:sha256:strategy-fixture", + manifest_id=MANIFEST_ID, + window_plan_id="window-plan:sha256:strategy-fixture", + information_mode=mode, + accepted=True, + total_violation_count=0, + findings=(), + evidence_truncated=False, + valid_for=("strategy_usefulness_claims",) if strategy_valid else (), + invalid_for=() if strategy_valid else ("strategy_usefulness_claims",), + summary="accepted fixture information boundary", + ) + + +def _case( + source_kind: StrategySourceKind, + *, + audit: InformationAuditReportV1, + member: str, + alignment: str = "window-a", + start_ns: int = 0, + end_ns: int = 8 * SECOND, + broker_profile_id: str | None = None, + invalid_for_backtest_reason: str | None = None, + source_scope: str | None = None, + bar_interval_code: str | None = None, +) -> StrategyEvaluationCaseV1: + return StrategyEvaluationCaseV1( + run_id=RUN_ID, + alignment_window_id=alignment, + source_kind=source_kind, + source_artifact_id=f"artifact:{source_kind.value}:{member}:{alignment}", + symbol="EURUSD", + start_ns=start_ns, + end_ns=end_ns, + information_mode=audit.information_mode, + information_manifest_id=MANIFEST_ID, + information_audit_id=audit.audit_id, + ensemble_member_id=member, + broker_profile_id=broker_profile_id, + source_scope=source_scope, + bar_interval_code=bar_interval_code, + invalid_for_backtest_reason=invalid_for_backtest_reason, + ) + + +def _engine() -> ReferenceMomentumStrategyV1: + return ReferenceMomentumStrategyV1( + lookback_ns=SECOND, + decision_interval_ns=20 * SECOND, + threshold_bps=0.0, + ) + + +def _policy( + *, + horizons_ns: tuple[int, ...] = (SECOND,), + max_quotes_per_window: int = 100, + max_payload_bytes: int = 4_194_304, +) -> StrategyEvaluationPolicyV1: + return StrategyEvaluationPolicyV1( + horizons_ns=horizons_ns, + max_cases=32, + max_quotes_per_window=max_quotes_per_window, + max_signals_per_window=16, + max_pending_signals=16, + max_slices=128, + max_payload_bytes=max_payload_bytes, + ) + + +def _plan( + cases: tuple[StrategyEvaluationCaseV1, ...], + *, + engine: ReferenceMomentumStrategyV1 | None = None, + execution: StrategyExecutionSpecificationV1 | None = None, + policy: StrategyEvaluationPolicyV1 | None = None, + invalid_for_backtest_reason: str | None = None, +) -> StrategyEvaluationPlanV1: + selected = engine or _engine() + return StrategyEvaluationPlanV1( + run_id=RUN_ID, + strategy=selected.specification, + execution=execution or StrategyExecutionSpecificationV1(), + policy=policy or _policy(), + cases=cases, + invalid_for_backtest_reason=invalid_for_backtest_reason, + ) + + +def _quotes( + member: str, + values: Sequence[float], + *, + times: Sequence[int] | None = None, + epoch: str = "modern", + session: str = "london", + event_state: str = "quiet", + sparsity: str = "dense", + broker_profile_id: str | None = None, + spread: float = 0.0, +) -> tuple[StrategyQuoteV1, ...]: + selected_times = times or tuple( + index * SECOND for index in range(len(values)) + ) + return tuple( + StrategyQuoteV1( + source_event_id=f"{member}:{index}", + symbol="eurusd", + event_time_ns=selected_times[index], + event_sequence=0, + bid=value, + ask=value + spread, + epoch_id=epoch, + session=session, + event_state=event_state, + sparsity=sparsity, + ensemble_member_id=member, + broker_profile_id=broker_profile_id, + ) + for index, value in enumerate(values) + ) + + +def _three_cases( + audit: InformationAuditReportV1, +) -> tuple[StrategyEvaluationCaseV1, ...]: + return ( + _case(StrategySourceKind.OBSERVED, audit=audit, member="reference"), + _case( + StrategySourceKind.DEGRADED_HOLDOUT, + audit=audit, + member="degraded", + ), + _case( + StrategySourceKind.RECONSTRUCTED, + audit=audit, + member="member-a", + ), + ) + + +def _streams( + cases: Sequence[StrategyEvaluationCaseV1], + *, + dense: Sequence[float] = (1.0, 1.001, 1.002, 1.003), + degraded: Sequence[float] = (1.0, 1.001, 1.0005, 1.003), + reconstructed: Sequence[float] = (1.0, 1.001, 1.0018, 1.003), +) -> dict[str, tuple[StrategyQuoteV1, ...]]: + values = { + StrategySourceKind.OBSERVED: dense, + StrategySourceKind.DEGRADED_HOLDOUT: degraded, + StrategySourceKind.RECONSTRUCTED: reconstructed, + StrategySourceKind.UNCONDITIONED_RECONSTRUCTION: reconstructed, + StrategySourceKind.BROKER_CONDITIONED: reconstructed, + StrategySourceKind.DERIVED_BARS: reconstructed, + } + return { + item.case_id: _quotes( + item.ensemble_member_id, + values[item.source_kind], + sparsity=item.source_kind.value, + broker_profile_id=item.broker_profile_id, + ) + for item in cases + } + + +def test_versioned_inputs_and_report_round_trip_without_profit_claims() -> None: + """Every assumption and result is deterministic, bounded metadata.""" + audit = _audit() + engine = _engine() + cases = _three_cases(audit) + execution = StrategyExecutionSpecificationV1( + entry_latency_ns=SECOND, + max_execution_wait_ns=SECOND, + slippage_bps_per_side=0.25, + fixed_cost_bps_per_side=0.1, + ) + policy = _policy(horizons_ns=(SECOND, 2 * SECOND)) + plan = _plan(cases, engine=engine, execution=execution, policy=policy) + + assert StrategyEvaluationPlanV1.from_dict(plan.to_dict()) == plan + assert ( + StrategyExecutionSpecificationV1.from_dict(execution.to_dict()) + == execution + ) + assert StrategyEvaluationPolicyV1.from_dict(policy.to_dict()) == policy + assert all( + StrategyEvaluationCaseV1.from_dict(item.to_dict()) == item + for item in cases + ) + + report = evaluate_strategy_sensitivity( + plan, + _streams(cases), + {audit.audit_id: audit}, + engine, + ) + payload = report.to_dict() + + assert StrategySensitivityReportV1.from_json(report.to_json()) == report + assert payload["output_mode"] == "bounded-derived-metadata" + assert payload["event_schema_augmented"] is False + assert payload["profit_claim"] is False + assert payload["investment_recommendation"] is False + assert payload["automatic_winner"] is False + assert all( + item.to_dict()["quotes_retained"] is False + for item in report.window_results + ) + assert all( + item.to_dict()["outcomes_retained"] is False + for item in report.window_results + ) + + +def test_ex_post_and_mixed_information_require_invalid_backtest_labels() -> ( + None +): + """Ex-post evidence cannot silently enter a prospective comparison.""" + ex_ante = _audit() + ex_post = _audit(InformationMode.EX_POST_RECONSTRUCTION) + with pytest.raises(ValueError, match="ex-post.*invalid-for-backtest"): + _case( + StrategySourceKind.RECONSTRUCTED, + audit=ex_post, + member="member-a", + ) + observed = _case( + StrategySourceKind.OBSERVED, + audit=ex_ante, + member="reference", + ) + reconstructed = _case( + StrategySourceKind.RECONSTRUCTED, + audit=ex_post, + member="member-a", + invalid_for_backtest_reason="ex-post historical counterfactual", + ) + with pytest.raises(ValueError, match="mixed information modes"): + _plan((observed, reconstructed)) + + engine = _engine() + plan = _plan( + (observed, reconstructed), + engine=engine, + invalid_for_backtest_reason="mixed ex-ante and ex-post comparison", + ) + report = evaluate_strategy_sensitivity( + plan, + _streams((observed, reconstructed)), + {ex_ante.audit_id: ex_ante, ex_post.audit_id: ex_post}, + engine, + ) + + assert report.valid_for_backtest is False + assert report.to_dict()["backtest_label"] == "invalid-for-backtest" + assert all(not item.valid_for_backtest for item in report.window_results) + + +def test_information_audit_must_open_strategy_usefulness_gate() -> None: + """An accepted ex-ante audit without strategy validity still fails closed.""" + audit = _audit(strategy_valid=False) + engine = _engine() + cases = _three_cases(audit) + + with pytest.raises(ValueError, match="strategy-usefulness"): + evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + _streams(cases), + {audit.audit_id: audit}, + engine, + ) + + +def test_alignment_requires_identical_symbol_and_half_open_window() -> None: + """Source surfaces cannot be compared across different time support.""" + audit = _audit() + first = _case(StrategySourceKind.OBSERVED, audit=audit, member="reference") + misaligned = _case( + StrategySourceKind.RECONSTRUCTED, + audit=audit, + member="member-a", + end_ns=7 * SECOND, + ) + with pytest.raises(ValueError, match="differ in symbol or time bounds"): + _plan((first, misaligned)) + + duplicate = replace( + first, + source_artifact_id="artifact:observed:duplicate", + case_id="", + ) + with pytest.raises(ValueError, match="role is duplicated"): + _plan((first, duplicate)) + + +def test_reference_accounting_applies_latency_spread_slippage_and_costs() -> ( + None +): + """The transparent fixture has reviewable execution arithmetic.""" + audit = _audit() + engine = _engine() + cases = _three_cases(audit) + execution = StrategyExecutionSpecificationV1( + entry_latency_ns=SECOND, + max_execution_wait_ns=SECOND, + slippage_bps_per_side=0.5, + fixed_cost_bps_per_side=0.25, + ) + plan = _plan( + cases, + engine=engine, + execution=execution, + policy=_policy(horizons_ns=(SECOND, 2 * SECOND)), + ) + streams = { + item.case_id: _quotes( + item.ensemble_member_id, + (1.0, 1.001, 1.002, 1.003, 1.004), + sparsity=item.source_kind.value, + spread=0.0002, + ) + for item in cases + } + + report = evaluate_strategy_sensitivity( + plan, streams, {audit.audit_id: audit}, engine + ) + observed = next( + item + for item in report.window_results + if item.source_kind is StrategySourceKind.OBSERVED + ) + + assert observed.status is StrategyWindowStatus.COMPLETED + assert observed.signal_count == 1 + assert {item.horizon_ns for item in observed.slices} == { + SECOND, + 2 * SECOND, + } + assert all(item.mean_entry_delay_ns == SECOND for item in observed.slices) + assert all( + item.mean_net_execution_response_bps < item.mean_gross_response_bps + for item in observed.slices + ) + assert all(item.mean_cost_drag_bps > 0 for item in observed.slices) + assert observed.mean_spread_bps > 0 + + +def test_results_stratify_and_retain_member_uncertainty() -> None: + """Epoch/session/event/broker/member strata and dispersion remain explicit.""" + audit = _audit() + engine = _engine() + cases = ( + _case(StrategySourceKind.OBSERVED, audit=audit, member="reference"), + _case( + StrategySourceKind.RECONSTRUCTED, + audit=audit, + member="member-a", + ), + _case( + StrategySourceKind.RECONSTRUCTED, + audit=audit, + member="member-b", + ), + _case( + StrategySourceKind.BROKER_CONDITIONED, + audit=audit, + member="member-c", + broker_profile_id="broker:demo-v1", + ), + ) + streams = { + cases[0].case_id: _quotes("reference", (1.0, 1.001, 1.002, 1.003)), + cases[1].case_id: _quotes( + "member-a", + (1.0, 1.001, 1.0018, 1.003), + epoch="epoch-modern", + session="new_york", + event_state="news", + sparsity="reconstructed", + ), + cases[2].case_id: _quotes( + "member-b", + (1.0, 1.001, 1.0014, 1.003), + epoch="epoch-modern", + session="new_york", + event_state="news", + sparsity="reconstructed", + ), + cases[3].case_id: _quotes( + "member-c", + (1.0, 1.001, 1.0016, 1.003), + epoch="epoch-modern", + session="new_york", + event_state="news", + sparsity="reconstructed", + broker_profile_id="broker:demo-v1", + ), + } + + report = evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + streams, + {audit.audit_id: audit}, + engine, + ) + reconstructed = next( + item + for item in report.uncertainty_summaries + if item.source_kind is StrategySourceKind.RECONSTRUCTED + ) + broker = next( + item + for item in report.uncertainty_summaries + if item.source_kind is StrategySourceKind.BROKER_CONDITIONED + ) + + assert reconstructed.epoch_id == "epoch-modern" + assert reconstructed.session == "new_york" + assert reconstructed.event_state == "news" + assert reconstructed.ensemble_member_ids == ("member-a", "member-b") + assert reconstructed.standard_deviation_bps > 0 + assert broker.broker_profile_id == "broker:demo-v1" + + +def test_reverse_degradation_reports_approach_to_dense_reference() -> None: + """Reconstructed execution response is compared with dense and degraded.""" + audit = _audit() + engine = _engine() + cases = _three_cases(audit) + + report = evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + _streams(cases), + {audit.audit_id: audit}, + engine, + ) + + assert len(report.restoration_results) == 1 + restoration = report.restoration_results[0] + assert restoration.candidate_source_kind is StrategySourceKind.RECONSTRUCTED + assert restoration.candidate_absolute_error_bps < ( + restoration.degraded_absolute_error_bps + ) + assert restoration.restoration_gain_bps > 0 + assert restoration.approaches_dense_reference is True + assert report.restoration_unavailable_count == 0 + + reconstructed = next( + item + for item in report.window_results + if item.source_kind is StrategySourceKind.RECONSTRUCTED + ) + hooks = strategy_sensitivity_benchmark_hooks(reconstructed) + assert ( + hooks["downstream_sensitivity"] + == reconstructed.slices[0].mean_net_execution_response_bps + ) + assert hooks["strategy_missing_support_rate"] == 0.0 + + +def test_missing_horizon_support_and_missing_stream_are_reported() -> None: + """Incomplete exits and absent case streams remain explicit rates.""" + audit = _audit() + engine = _engine() + cases = _three_cases(audit) + plan = _plan( + cases, + engine=engine, + policy=_policy(horizons_ns=(10 * SECOND,)), + ) + streams = _streams(cases) + del streams[cases[2].case_id] + + report = evaluate_strategy_sensitivity( + plan, streams, {audit.audit_id: audit}, engine + ) + + assert report.summary["missing_support_window_rate"] == pytest.approx(1.0) + assert report.summary["missing_support_outcome_rate"] == pytest.approx(1.0) + assert all( + item.status is StrategyWindowStatus.MISSING_SUPPORT + for item in report.window_results + ) + assert report.summary["status_counts"]["missing_support"] == 3 + + +def test_no_trade_failure_and_resource_refusal_rates_are_bounded() -> None: + """Every non-result status has a bounded reason and rate.""" + audit = _audit() + engine = _engine() + cases = _three_cases(audit) + constant_streams = { + item.case_id: _quotes( + item.ensemble_member_id, + (1.0, 1.0, 1.0, 1.0), + sparsity=item.source_kind.value, + ) + for item in cases + } + no_trade = evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + constant_streams, + {audit.audit_id: audit}, + engine, + ) + assert no_trade.summary["no_trade_window_rate"] == 1.0 + with pytest.raises(ValueError, match="completed window"): + strategy_sensitivity_benchmark_hooks(no_trade.window_results[0]) + + refusal = evaluate_strategy_sensitivity( + _plan( + cases, + engine=engine, + policy=_policy(max_quotes_per_window=2), + ), + _streams(cases), + {audit.audit_id: audit}, + engine, + ) + assert refusal.summary["refused_window_rate"] == 1.0 + assert all(not item.slices for item in refusal.window_results) + + failing = _FailingEngine(engine) + failed = evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + _streams(cases), + {audit.audit_id: audit}, + failing, + ) + assert failed.summary["failure_window_rate"] == 1.0 + assert all( + item.reason == "fixture strategy failure" + for item in failed.window_results + ) + + +class _FailingState: + def observe(self, quote: StrategyQuoteV1) -> Sequence[StrategySignalV1]: + del quote + raise StrategyEvaluationFailure("fixture strategy failure") + + +class _FailingEngine: + def __init__(self, reference: ReferenceMomentumStrategyV1) -> None: + self.specification = reference.specification + + def start_window( + self, evaluation_case: StrategyEvaluationCaseV1 + ) -> StrategySignalStateV1: + del evaluation_case + return _FailingState() + + +def test_quote_stream_order_and_support_are_fail_closed() -> None: + """Malformed streams are contract errors, not plausible result artifacts.""" + audit = _audit() + engine = _engine() + cases = _three_cases(audit) + streams = _streams(cases) + streams[cases[0].case_id] = tuple(reversed(streams[cases[0].case_id])) + with pytest.raises(ValueError, match="not strictly ordered"): + evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + streams, + {audit.audit_id: audit}, + engine, + ) + + streams = _streams(cases) + outside = replace( + streams[cases[0].case_id][0], + event_time_ns=cases[0].end_ns, + quote_id="", + ) + streams[cases[0].case_id] = (outside,) + with pytest.raises(ValueError, match="outside aligned"): + evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + streams, + {audit.audit_id: audit}, + engine, + ) + + +def test_engine_specification_and_payload_bounds_fail_closed() -> None: + """Logic drift and oversized report metadata cannot pass silently.""" + audit = _audit() + engine = _engine() + other = ReferenceMomentumStrategyV1(threshold_bps=1.0) + cases = _three_cases(audit) + with pytest.raises(ValueError, match="engine specification differs"): + evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + _streams(cases), + {audit.audit_id: audit}, + other, + ) + + with pytest.raises(ValueError, match="payload exceeds"): + evaluate_strategy_sensitivity( + _plan( + cases, + engine=engine, + policy=_policy(max_payload_bytes=256), + ), + _streams(cases), + {audit.audit_id: audit}, + engine, + ) + + +def test_market_surface_adapters_cover_benchmark_reconstruction_and_bars() -> ( + None +): + """All source families enter through one minimal normalized quote contract.""" + benchmark = BenchmarkEventV1( + source_event_id="benchmark-source:1", + symbol="EURUSD", + event_time_ns=SECOND, + event_sequence=2, + bid=1.1, + ask=1.1002, + epoch_id="modern", + session="london", + event_state="quiet", + sparsity="degraded", + ensemble_member_id="member-a", + ) + benchmark_quote = StrategyQuoteV1.from_benchmark_event( + benchmark, broker_profile_id="broker:test" + ) + assert ( + StrategyQuoteV1.from_dict(benchmark_quote.to_dict()) == benchmark_quote + ) + assert benchmark_quote.session == "london" + assert benchmark_quote.broker_profile_id == "broker:test" + + left = _observed(1) + right = _observed(2) + generated = _generated(left, right) + reconstructed = StrategyQuoteV1.from_synthetic_event( + generated, + epoch_id="legacy", + session="new_york", + event_state="news", + sparsity="reconstructed", + ) + assert reconstructed.source_event_id == generated.event_id + assert reconstructed.ensemble_member_id == generated.ensemble_member_id + + bar = _derive_boundary_bars()[0] + bar_quote = StrategyQuoteV1.from_derived_bar( + bar, session="london", event_state="quiet" + ) + assert bar_quote.source_event_id == bar.bar_id + assert bar_quote.bid == bar.bid_close + assert bar_quote.ask == bar.ask_close + assert bar_quote.sparsity == "derived_bar" + assert bar_quote.source_scope == bar.scope.value + assert bar_quote.bar_interval_code == bar.interval_code + + +def test_derived_bar_cases_require_manifest_scope_and_interval() -> None: + """Bar comparisons cannot hide product scope or aggregation semantics.""" + audit = _audit() + with pytest.raises(ValueError, match="requires scope and interval"): + _case( + StrategySourceKind.DERIVED_BARS, + audit=audit, + member="member-a", + ) + case = _case( + StrategySourceKind.DERIVED_BARS, + audit=audit, + member="member-a", + source_scope="merged", + bar_interval_code="1m", + ) + assert case.source_scope == "merged" + assert case.bar_interval_code == "1m" + observed = _case( + StrategySourceKind.OBSERVED, + audit=audit, + member="reference", + ) + engine = _engine() + with pytest.raises(ValueError, match="bar quote scope differs"): + evaluate_strategy_sensitivity( + _plan((observed, case), engine=engine), + _streams((observed, case)), + {audit.audit_id: audit}, + engine, + ) + + +def test_multiple_alignment_windows_contribute_rolling_stability() -> None: + """Repeated aligned windows remain distinct inputs to uncertainty summaries.""" + audit = _audit() + engine = _engine() + first = ( + _case( + StrategySourceKind.OBSERVED, + audit=audit, + member="reference-a", + alignment="window-a", + ), + _case( + StrategySourceKind.RECONSTRUCTED, + audit=audit, + member="member-a", + alignment="window-a", + ), + ) + second = ( + _case( + StrategySourceKind.OBSERVED, + audit=audit, + member="reference-b", + alignment="window-b", + ), + _case( + StrategySourceKind.RECONSTRUCTED, + audit=audit, + member="member-b", + alignment="window-b", + ), + ) + cases = first + second + streams = _streams(cases) + streams[second[1].case_id] = _quotes( + "member-b", + (1.0, 1.001, 1.0012, 1.003), + sparsity="reconstructed", + ) + + report = evaluate_strategy_sensitivity( + _plan(cases, engine=engine), + streams, + {audit.audit_id: audit}, + engine, + ) + reconstructed = next( + item + for item in report.uncertainty_summaries + if item.source_kind is StrategySourceKind.RECONSTRUCTED + ) + + assert reconstructed.window_count == 2 + assert reconstructed.ensemble_member_ids == ("member-a", "member-b") + assert reconstructed.standard_deviation_bps > 0 diff --git a/tests/unit/test_synthetic_streaming.py b/tests/unit/test_synthetic_streaming.py new file mode 100644 index 00000000..fa783fa5 --- /dev/null +++ b/tests/unit/test_synthetic_streaming.py @@ -0,0 +1,799 @@ +"""Tests for bounded reconstruction stream and checkpoint contracts.""" + +from __future__ import annotations + +import asyncio +from dataclasses import replace +import hashlib +import json +from pathlib import Path + +from hypothesis import given, settings +from hypothesis import strategies as st +import pytest +from temporalio.converter import DataConverter + +from histdatacom.runtime_contracts import ArtifactRef +from histdatacom.synthetic import ( + CARRY_STATE_SCHEMA_VERSION, + EVENT_BATCH_SCHEMA_VERSION, + PARTITION_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_CHECKPOINT_SCHEMA_VERSION, + RECONSTRUCTION_HEARTBEAT_SCHEMA_VERSION, + RECONSTRUCTION_RESOURCE_ESTIMATE_SCHEMA_VERSION, + RECONSTRUCTION_RUN_SCHEMA_VERSION, + RECONSTRUCTION_STORAGE_POLICY_SCHEMA_VERSION, + RECONSTRUCTION_WINDOW_SCHEMA_VERSION, + REJECTION_SUMMARY_SCHEMA_VERSION, + CarryStateV1, + EventBatchV1, + PartitionManifestV1, + ReconstructionCheckpointV1, + ReconstructionCommitPhase, + ReconstructionHeartbeatV1, + ReconstructionResourceEstimateV1, + ReconstructionResourceLimitError, + ReconstructionRunV1, + ReconstructionStoragePolicyV1, + ReconstructionWindowV1, + RejectionSummaryV1, + artifact_ref_for_json_contract, + plan_reconstruction_windows, + validate_reconstruction_window_plan, +) + +START_NS = 1_700_000_000_000_000_000 +END_NS = START_NS + 12_000 +SYMBOLS = ("eurusd", "gbpusd", "eurgbp") + + +def _ref( + kind: str, + path: str, + content: str, + *, + metadata: dict | None = None, +) -> ArtifactRef: + encoded = content.encode("utf-8") + return ArtifactRef( + kind=kind, + path=path, + size_bytes=len(encoded), + sha256=hashlib.sha256(encoded).hexdigest(), + metadata=dict(metadata or {}), + ) + + +def _run( + *, + policy: ReconstructionStoragePolicyV1 | None = None, +) -> ReconstructionRunV1: + return ReconstructionRunV1( + symbols=SYMBOLS, + source_version_ids=("source:sha256:historical-v1",), + configuration_ids=("config:sha256:reconstruction-v1",), + ensemble_member_ids=("member-000", "member-001"), + base_seed=20260713, + storage_policy=policy or ReconstructionStoragePolicyV1(), + ) + + +def _window() -> ReconstructionWindowV1: + return plan_reconstruction_windows( + _run(), + ensemble_member_id="member-000", + start_ns=START_NS, + end_ns=END_NS, + window_size_ns=4_000, + left_halo_ns=1_000, + right_lookahead_ns=2_000, + )[0] + + +def _batch( + window: ReconstructionWindowV1, + symbol: str, + ordinal: int, + *, + event_count: int = 3, +) -> EventBatchV1: + normalized_symbol = symbol.lower() + content = f"{window.window_id}:{normalized_symbol}:{ordinal}:{event_count}" + content_sha256 = hashlib.sha256(content.encode("utf-8")).hexdigest() + return EventBatchV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + symbol=normalized_symbol, + batch_ordinal=ordinal, + event_count=event_count, + ownership_start_ns=window.core_start_ns, + ownership_end_ns=window.core_end_ns, + first_event_time_ns=window.core_start_ns + ordinal, + last_event_time_ns=window.core_start_ns + ordinal + 100, + content_sha256=content_sha256, + artifact=_ref( + "synthetic-event-batch", + f"scratch/{window.window_id}/{normalized_symbol}/{ordinal}.parquet", + content, + metadata={"event_count": event_count}, + ), + ) + + +def _carry(window: ReconstructionWindowV1) -> CarryStateV1: + return CarryStateV1( + run_id=window.run_id, + ensemble_member_id=window.ensemble_member_id, + symbol_watermarks_ns={ + symbol: window.core_end_ns for symbol in window.symbols + }, + last_event_ids={ + symbol: f"event:sha256:{symbol}" for symbol in window.symbols + }, + state_artifacts=( + _ref( + "reconstruction-carry-detail", + f"scratch/{window.window_id}/carry.parquet", + "bounded carry detail", + ), + ), + ) + + +def _rejections(window: ReconstructionWindowV1) -> RejectionSummaryV1: + return RejectionSummaryV1( + run_id=window.run_id, + window_id=window.window_id, + candidate_count=12, + accepted_count=9, + rejected_count=3, + reason_counts={"negative_spread": 1, "weekend_closure": 2}, + ) + + +def _manifest( + window: ReconstructionWindowV1, +) -> tuple[PartitionManifestV1, tuple[EventBatchV1, ...]]: + batches = tuple(_batch(window, symbol, 0) for symbol in window.symbols) + carry = _carry(window) + rejections = _rejections(window) + manifest = PartitionManifestV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + ensemble_member_id=window.ensemble_member_id, + symbols=window.symbols, + symbol_event_counts={symbol: 3 for symbol in window.symbols}, + event_batches=batches, + rejection_summary_ref=artifact_ref_for_json_contract( + rejections, + kind="rejection-summary", + path=f"scratch/{window.window_id}/rejections.json", + ), + carry_state_ref=artifact_ref_for_json_contract( + carry, + kind="carry-state", + path=f"scratch/{window.window_id}/carry.json", + ), + ) + return manifest, batches + + +def test_schema_versions_and_round_trips_are_explicit() -> None: + run = _run() + policy = run.storage_policy + estimate = ReconstructionResourceEstimateV1( + input_event_count=100, + candidate_event_count=200, + retained_ensemble_members=2, + inflight_batches=2, + peak_events_per_batch=50, + estimated_memory_bytes=1_000, + estimated_scratch_bytes=2_000, + estimated_output_bytes=3_000, + estimated_batch_count=4, + ) + window = _window() + batch = _batch(window, "eurusd", 0) + carry = _carry(window) + rejections = _rejections(window) + manifest, _ = _manifest(window) + checkpoint = ReconstructionCheckpointV1.planned(window) + heartbeat = ReconstructionHeartbeatV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + phase=ReconstructionCommitPhase.RUNNING, + sequence=1, + completed_units=1, + total_units=3, + ) + + assert ReconstructionStoragePolicyV1.from_json(policy.to_json()) == policy + assert ReconstructionResourceEstimateV1.from_json(estimate.to_json()) == ( + estimate + ) + assert ReconstructionRunV1.from_json(run.to_json()) == run + assert ReconstructionWindowV1.from_json(window.to_json()) == window + assert EventBatchV1.from_json(batch.to_json()) == batch + assert CarryStateV1.from_json(carry.to_json()) == carry + assert RejectionSummaryV1.from_json(rejections.to_json()) == rejections + assert PartitionManifestV1.from_json(manifest.to_json()) == manifest + assert ReconstructionCheckpointV1.from_json(checkpoint.to_json()) == ( + checkpoint + ) + assert ReconstructionHeartbeatV1.from_json(heartbeat.to_json()) == ( + heartbeat + ) + assert all( + version.endswith(".v1") + for version in ( + RECONSTRUCTION_STORAGE_POLICY_SCHEMA_VERSION, + RECONSTRUCTION_RESOURCE_ESTIMATE_SCHEMA_VERSION, + RECONSTRUCTION_RUN_SCHEMA_VERSION, + RECONSTRUCTION_WINDOW_SCHEMA_VERSION, + EVENT_BATCH_SCHEMA_VERSION, + CARRY_STATE_SCHEMA_VERSION, + REJECTION_SUMMARY_SCHEMA_VERSION, + PARTITION_MANIFEST_SCHEMA_VERSION, + RECONSTRUCTION_CHECKPOINT_SCHEMA_VERSION, + RECONSTRUCTION_HEARTBEAT_SCHEMA_VERSION, + ) + ) + + +def test_execution_policy_does_not_change_semantic_run_or_seed() -> None: + narrow = _run(policy=ReconstructionStoragePolicyV1(max_inflight_batches=1)) + parallel = _run( + policy=ReconstructionStoragePolicyV1(max_inflight_batches=32) + ) + + assert narrow.run_id == parallel.run_id + assert narrow.seed_for("member-000", "anchor-interval-42") == ( + parallel.seed_for("member-000", "anchor-interval-42") + ) + assert narrow.storage_policy.policy_id != parallel.storage_policy.policy_id + + policy_payload = narrow.storage_policy.to_dict() + policy_payload["atomic_promotion_required"] = "true" + policy_payload["policy_id"] = "" + with pytest.raises(ValueError, match="must be a boolean"): + ReconstructionStoragePolicyV1.from_dict(policy_payload) + + +@given( + first_size=st.integers(min_value=1, max_value=4_000), + second_size=st.integers(min_value=1, max_value=4_000), + offset=st.integers(min_value=0, max_value=11_999), +) +@settings(max_examples=60, deadline=None) +def test_legal_window_partitioning_has_single_ownership_and_same_seed( + first_size: int, + second_size: int, + offset: int, +) -> None: + run = _run() + first = plan_reconstruction_windows( + run, + ensemble_member_id="member-000", + start_ns=START_NS, + end_ns=END_NS, + window_size_ns=first_size, + left_halo_ns=500, + right_lookahead_ns=750, + ) + second = plan_reconstruction_windows( + run, + ensemble_member_id="member-000", + start_ns=START_NS, + end_ns=END_NS, + window_size_ns=second_size, + left_halo_ns=500, + right_lookahead_ns=750, + ) + event_time = START_NS + offset + + assert sum(window.owns_event_time(event_time) for window in first) == 1 + assert sum(window.owns_event_time(event_time) for window in second) == 1 + assert run.seed_for("member-000", f"anchor:{event_time}") == ( + run.seed_for("member-000", f"anchor:{event_time}") + ) + + +def test_window_halo_is_readable_but_never_generation_owned() -> None: + window = _window() + + assert window.reads_event_time(window.input_start_ns) + assert not window.owns_event_time(window.input_start_ns) + assert window.owns_event_time(window.core_start_ns) + assert not window.owns_event_time(window.core_end_ns) + assert window.reads_event_time(window.core_end_ns) + assert not window.reads_event_time(window.input_end_ns) + + +def test_window_plan_rejects_gaps_overlaps_and_scope_drift() -> None: + first = _window() + gap = ReconstructionWindowV1( + run_id=first.run_id, + ensemble_member_id=first.ensemble_member_id, + symbols=first.symbols, + core_start_ns=first.core_end_ns + 1, + core_end_ns=first.core_end_ns + 100, + ) + overlap = replace( + gap, + core_start_ns=first.core_end_ns - 1, + window_id="", + synchronization_unit_id="", + ) + drift = ReconstructionWindowV1( + run_id=first.run_id, + ensemble_member_id="member-001", + symbols=first.symbols, + core_start_ns=first.core_end_ns, + core_end_ns=first.core_end_ns + 100, + ) + + with pytest.raises(ValueError, match="contiguous"): + validate_reconstruction_window_plan((first, gap)) + with pytest.raises(ValueError, match="contiguous"): + validate_reconstruction_window_plan((first, overlap)) + with pytest.raises(ValueError, match="scope drifted"): + validate_reconstruction_window_plan((first, drift)) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + ( + ("candidate_event_count", 251, "amplification"), + ("peak_events_per_batch", 100_001, "peak_events_per_batch"), + ("retained_ensemble_members", 5, "retained_ensemble_members"), + ("inflight_batches", 9, "inflight_batches"), + ("estimated_memory_bytes", 2 * 1024**3 + 1, "memory"), + ("estimated_scratch_bytes", 100 * 1024**3 + 1, "scratch"), + ("estimated_output_bytes", 100 * 1024**3 + 1, "output"), + ), +) +def test_resource_preflight_fails_early_with_full_estimate( + field: str, + value: int, + message: str, +) -> None: + policy = ReconstructionStoragePolicyV1() + estimate = ReconstructionResourceEstimateV1( + input_event_count=10, + candidate_event_count=100, + retained_ensemble_members=2, + inflight_batches=2, + peak_events_per_batch=100, + estimated_memory_bytes=1_000, + estimated_scratch_bytes=2_000, + estimated_output_bytes=3_000, + estimated_batch_count=4, + ) + refused = replace(estimate, **{field: value}, estimate_id="") + + with pytest.raises(ReconstructionResourceLimitError, match=message) as err: + policy.preflight(refused) + + assert err.value.estimate == refused + assert err.value.violations + assert policy.preflight(estimate) == estimate + + +def test_event_batches_are_strong_refs_and_retry_deterministic() -> None: + window = _window() + original = _batch(window, "EURUSD", 0) + retry = _batch(window, "eurusd", 0) + relocated_retry = replace( + retry, + artifact=replace( + retry.artifact, + path="scratch/worker-99/retry.parquet", + metadata={"worker": "99"}, + ), + batch_id="", + ) + payload = original.to_dict() + + assert original == retry + assert original.batch_id == retry.batch_id + assert relocated_retry != original + assert relocated_retry.batch_id == original.batch_id + checkpoint = ReconstructionCheckpointV1.planned(window) + assert checkpoint.pending_batches((original, relocated_retry)) == ( + original, + ) + assert "events" not in payload + assert payload["artifact"]["sha256"] + assert payload["artifact"]["size_bytes"] > 0 + with pytest.raises(ValueError, match="SHA-256"): + replace( + original, + artifact=replace(original.artifact, sha256="weak"), + batch_id="", + ) + with pytest.raises(ValueError, match="inline data"): + replace( + original, + artifact=replace( + original.artifact, + metadata={"events": [{"bid": 1.1}]}, + ), + batch_id="", + ) + with pytest.raises(ValueError, match="finite numbers"): + replace( + original, + artifact=replace( + original.artifact, + metadata={"quality_score": float("nan")}, + ), + batch_id="", + ) + with pytest.raises(ValueError, match="outside half-open"): + replace( + original, + last_event_time_ns=original.ownership_end_ns, + batch_id="", + ) + + +def test_carry_and_rejection_contracts_are_bounded_aggregates() -> None: + window = _window() + carry = _carry(window) + rejections = _rejections(window) + + assert "events" not in carry.to_dict() + assert len(carry.to_json().encode("utf-8")) < 262_144 + assert sum(rejections.reason_counts.values()) == rejections.rejected_count + with pytest.raises(ValueError, match="reconcile"): + replace(rejections, reason_counts={"one": 1}, summary_id="") + with pytest.raises(ValueError, match="unknown symbol"): + replace( + carry, + last_event_ids={"usdjpy": "event:sha256:unknown"}, + carry_id="", + ) + + +def test_partition_manifest_commits_all_symbols_as_one_unit() -> None: + window = _window() + manifest, batches = _manifest(window) + + assert manifest.symbols == tuple(sorted(SYMBOLS)) + assert manifest.event_count == 9 + assert {batch.symbol for batch in manifest.event_batches} == set(SYMBOLS) + assert PartitionManifestV1.from_json(manifest.to_json()) == manifest + with pytest.raises(ValueError, match="reconcile"): + replace( + manifest, + symbol_event_counts={**manifest.symbol_event_counts, "eurusd": 2}, + manifest_id="", + ) + foreign = replace( + batches[0], + synchronization_unit_id="sync:foreign", + batch_id="", + ) + with pytest.raises(ValueError, match="scope"): + replace( + manifest, + event_batches=(foreign, *batches[1:]), + manifest_id="", + ) + + +def test_checkpoint_crash_retry_deduplicates_completed_batches() -> None: + window = _window() + manifest, batches = _manifest(window) + planned = ReconstructionCheckpointV1.planned(window) + running = planned.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=planned.checkpoint_id, + ) + after_first = running.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=running.checkpoint_id, + completed_batches=(batches[0],), + input_watermark_ns=window.core_end_ns, + ) + recovered = ReconstructionCheckpointV1.from_json(after_first.to_json()) + + assert recovered.pending_batches((batches[0], *batches)) == batches[1:] + finished = recovered.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=recovered.checkpoint_id, + completed_batches=(batches[0], *batches[1:]), + output_watermark_ns=window.core_end_ns, + ) + assert set(finished.completed_batch_ids) == { + batch.batch_id for batch in batches + } + assert not finished.pending_batches(batches) + assert manifest.event_count == sum(batch.event_count for batch in batches) + + +def test_two_phase_commit_never_advertises_partial_output() -> None: + window = _window() + manifest, batches = _manifest(window) + carry = _carry(window) + rejections = _rejections(window) + carry_ref = artifact_ref_for_json_contract( + carry, + kind="carry-state", + path=f"scratch/{window.window_id}/carry.json", + ) + rejection_ref = artifact_ref_for_json_contract( + rejections, + kind="rejection-summary", + path=f"scratch/{window.window_id}/rejections.json", + ) + staged_ref = artifact_ref_for_json_contract( + manifest, + kind="partition-manifest-temp", + path=f"scratch/{window.window_id}/manifest.partial.json", + ) + committed_ref = artifact_ref_for_json_contract( + manifest, + kind="partition-manifest", + path=f"products/{window.window_id}/manifest.json", + ) + checkpoint = ReconstructionCheckpointV1.planned(window) + checkpoint = checkpoint.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=checkpoint.checkpoint_id, + completed_batches=batches, + carry_state_ref=carry_ref, + rejection_summary_ref=rejection_ref, + ) + staged = checkpoint.transition( + ReconstructionCommitPhase.STAGED, + expected_checkpoint_id=checkpoint.checkpoint_id, + staged_manifest_ref=staged_ref, + ) + validated = staged.transition( + ReconstructionCommitPhase.VALIDATED, + expected_checkpoint_id=staged.checkpoint_id, + ) + + assert staged.advertised_manifest_ref is None + assert validated.advertised_manifest_ref is None + committed = validated.transition( + ReconstructionCommitPhase.COMMITTED, + expected_checkpoint_id=validated.checkpoint_id, + committed_manifest_ref=committed_ref, + ) + assert committed.advertised_manifest_ref == committed_ref + assert committed.staged_manifest_ref is None + assert ( + committed.transition( + ReconstructionCommitPhase.COMMITTED, + expected_checkpoint_id=committed.checkpoint_id, + committed_manifest_ref=committed_ref, + ) + is committed + ) + with pytest.raises(ValueError, match="cannot change manifest"): + committed.transition( + ReconstructionCommitPhase.COMMITTED, + expected_checkpoint_id=committed.checkpoint_id, + committed_manifest_ref=replace( + committed_ref, + path="products/conflicting/manifest.json", + ), + ) + + +def test_commit_rejects_promoted_bytes_that_differ_from_validated_stage() -> ( + None +): + window = _window() + manifest, _ = _manifest(window) + staged_ref = artifact_ref_for_json_contract( + manifest, + kind="partition-manifest-temp", + path=f"scratch/{window.window_id}/manifest.partial.json", + ) + checkpoint = ReconstructionCheckpointV1.planned(window) + checkpoint = checkpoint.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=checkpoint.checkpoint_id, + ) + checkpoint = checkpoint.transition( + ReconstructionCommitPhase.STAGED, + expected_checkpoint_id=checkpoint.checkpoint_id, + staged_manifest_ref=staged_ref, + ) + checkpoint = checkpoint.transition( + ReconstructionCommitPhase.VALIDATED, + expected_checkpoint_id=checkpoint.checkpoint_id, + ) + different = _ref( + "partition-manifest", + f"products/{window.window_id}/manifest.json", + "different bytes", + ) + + with pytest.raises(ValueError, match="do not match"): + checkpoint.transition( + ReconstructionCommitPhase.COMMITTED, + expected_checkpoint_id=checkpoint.checkpoint_id, + committed_manifest_ref=different, + ) + + +def test_checkpoint_rejects_stale_transition_and_watermark_regression() -> None: + window = _window() + checkpoint = ReconstructionCheckpointV1.planned(window) + running = checkpoint.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=checkpoint.checkpoint_id, + input_watermark_ns=window.core_end_ns, + ) + + with pytest.raises(ValueError, match="stale checkpoint"): + running.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=checkpoint.checkpoint_id, + ) + with pytest.raises(ValueError, match="cannot move backwards"): + running.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=running.checkpoint_id, + input_watermark_ns=window.core_start_ns, + ) + + +def test_cancellation_stops_advertising_and_resume_clears_partial_ref() -> None: + window = _window() + manifest, _ = _manifest(window) + staged_ref = artifact_ref_for_json_contract( + manifest, + kind="partition-manifest-temp", + path=f"scratch/{window.window_id}/manifest.partial.json", + ) + checkpoint = ReconstructionCheckpointV1.planned(window) + checkpoint = checkpoint.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=checkpoint.checkpoint_id, + ) + staged = checkpoint.transition( + ReconstructionCommitPhase.STAGED, + expected_checkpoint_id=checkpoint.checkpoint_id, + staged_manifest_ref=staged_ref, + ) + cancelled = staged.transition( + ReconstructionCommitPhase.CANCELLED, + expected_checkpoint_id=staged.checkpoint_id, + interruption_reason="operator requested cancellation", + ) + + assert cancelled.advertised_manifest_ref is None + assert cancelled.staged_manifest_ref == staged_ref + resumed = cancelled.transition( + ReconstructionCommitPhase.RUNNING, + expected_checkpoint_id=cancelled.checkpoint_id, + ) + assert resumed.staged_manifest_ref is None + assert resumed.interruption_reason == "" + + +def test_checkpoint_policy_enforces_bounded_workflow_payload() -> None: + window = _window() + checkpoint = ReconstructionCheckpointV1.planned(window) + tiny_policy = ReconstructionStoragePolicyV1(max_checkpoint_bytes=100) + + with pytest.raises(ValueError, match="checkpoint payload"): + checkpoint.assert_within(tiny_policy) + assert checkpoint.assert_within(_run().storage_policy) == checkpoint + + +def test_heartbeat_is_bounded_and_explicit_about_cancel_resume() -> None: + window = _window() + checkpoint = ReconstructionCheckpointV1.planned(window) + heartbeat = ReconstructionHeartbeatV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + phase=ReconstructionCommitPhase.RUNNING, + sequence=3, + completed_units=2, + total_units=4, + observed_event_count=100, + candidate_event_count=200, + accepted_event_count=150, + scratch_bytes=10_000, + output_bytes=5_000, + checkpoint_id=checkpoint.checkpoint_id, + cancellation_requested=True, + message="draining current bounded batch", + ) + payload = heartbeat.to_dict() + + assert payload["percent_complete"] == 50.0 + assert payload["stops_future_work_on_cancel"] is True + assert payload["resume_mode"] == "last_valid_checkpoint" + assert "events" not in json.dumps(payload) + assert len(heartbeat.to_json().encode("utf-8")) < 65_536 + + +def test_checkpoint_and_heartbeat_survive_temporal_data_conversion() -> None: + window = _window() + checkpoint = ReconstructionCheckpointV1.planned(window) + heartbeat = ReconstructionHeartbeatV1( + run_id=window.run_id, + window_id=window.window_id, + synchronization_unit_id=window.synchronization_unit_id, + phase=ReconstructionCommitPhase.PLANNED, + sequence=0, + completed_units=0, + total_units=3, + checkpoint_id=checkpoint.checkpoint_id, + ) + payload = { + "checkpoint": checkpoint.to_dict(), + "heartbeat": heartbeat.to_dict(), + } + + async def round_trip() -> dict: + encoded = await DataConverter.default.encode([payload]) + [decoded] = await DataConverter.default.decode( + encoded, + type_hints=[dict], + ) + return decoded + + assert asyncio.run(round_trip()) == payload + + +def test_derived_fields_and_ids_fail_closed() -> None: + window = _window() + manifest, _ = _manifest(window) + checkpoint = ReconstructionCheckpointV1.planned(window) + + window_payload = window.to_dict() + window_payload["window_id"] = "reconstruction-window:sha256:" + "0" * 64 + with pytest.raises(ValueError, match="window_id"): + ReconstructionWindowV1.from_dict(window_payload) + + manifest_payload = manifest.to_dict() + manifest_payload["event_count"] = 999 + with pytest.raises(ValueError, match="event_count"): + PartitionManifestV1.from_dict(manifest_payload) + + checkpoint_payload = checkpoint.to_dict() + checkpoint_payload["advertised_manifest_ref"] = _ref( + "partition-manifest", + "products/uncommitted.json", + "not committed", + ).to_dict() + with pytest.raises(ValueError, match="advertised manifest"): + ReconstructionCheckpointV1.from_dict(checkpoint_payload) + + duplicate_payload = checkpoint.to_dict() + duplicate_payload["phase"] = "running" + duplicate_payload["revision"] = 1 + duplicate_payload["completed_batch_ids"] = ["batch:one", "batch:one"] + duplicate_payload["checkpoint_id"] = "" + with pytest.raises(ValueError, match="must be unique"): + ReconstructionCheckpointV1.from_dict(duplicate_payload) + + +def test_checkpoint_fixture_is_stable_bounded_and_reconstructable() -> None: + fixture = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "reconstruction_checkpoint_v1.json" + ) + text = fixture.read_text(encoding="utf-8") + checkpoint = ReconstructionCheckpointV1.from_dict(json.loads(text)) + + assert checkpoint == ReconstructionCheckpointV1.planned(_window()) + assert json.dumps( + checkpoint.to_dict(), indent=2, sort_keys=True + ) + "\n" == (text) + assert len(text.encode("utf-8")) < 2_048 + assert not {"events", "rows", "records"}.intersection(checkpoint.to_dict()) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6588ae2d..96fa7fcd 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -15,6 +15,7 @@ get_now_utc_timestamp, load_influx_yaml, normalize_api_return_type, + normalize_output_timezone, set_working_data_dir, ) @@ -64,6 +65,32 @@ def test_api_return_type_contract_is_explicit() -> None: } +@pytest.mark.parametrize( + ("timezone_name", "expected"), + ( + ("UTC", "UTC"), + ("America/New_York", "America/New_York"), + (" Europe/London ", "Europe/London"), + (None, ""), + ("", ""), + ), +) +def test_normalize_output_timezone( + timezone_name: str | None, + expected: str, +) -> None: + """IANA output zones should be normalized without choosing a local default.""" + assert normalize_output_timezone(timezone_name) == expected + + +def test_normalize_output_timezone_rejects_unknown_zone() -> None: + """Unknown zones should fail with actionable IANA guidance.""" + with pytest.raises(ValueError, match="unsupported output timezone") as err: + normalize_output_timezone("Mars/Olympus_Mons") + + assert "America/New_York" in str(err.value) + + def test_get_now_utc_timestamp_uses_aware_utc_without_warnings() -> None: """Current UTC timestamp should not rely on naive datetime helpers.""" before = datetime.now(timezone.utc).timestamp()