From 5242b99893344550124b81711d6dd9d41c5948bc Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Sun, 19 Jul 2026 14:29:31 +0100 Subject: [PATCH 1/2] Declare UK capital gains target facts natively The published populace-uk release carries 149 targets and none constrain capital gains, so the calibrated weights leave the gains distribution unanchored. Reading capital_gains weighted by household_weight straight out of populace_uk_2023.h5 gives 1.47m CGT taxpayers holding GBP 96.0bn at a GBP 65,374 mean, against HMRC's 378k, GBP 65.9bn and ~GBP 174,000 - 3.9x the taxpayer count with a mean gain 62% below the administrative figure. That is distributional error rather than a level shift, so it survives any revenue-side correction: on an uncalibrated baseline the share of people affected by a CGT rate reform comes out roughly three times too high. Declares hmrc/capital_gains_total and hmrc/cgt_taxpayers as household-grain facts with HMRC citations, following us_runtime.fiscal_targets, plus a coverage requirement so dropping either fails the gate instead of passing quietly. Both measures are prepared columns (the registry refuses callables); the taxpayer count must track the annual exempt amount in force for the period, which moved GBP 12,300 -> 6,000 -> 3,000 across 2022-23 to 2024-25. Co-Authored-By: Claude Fable 5 --- .../src/populace/build/uk_runtime/__init__.py | 10 ++ .../build/uk_runtime/fiscal_targets.py | 133 ++++++++++++++++++ .../tests/test_uk_cgt_targets.py | 66 +++++++++ 3 files changed, 209 insertions(+) create mode 100644 packages/populace-build/src/populace/build/uk_runtime/fiscal_targets.py create mode 100644 packages/populace-build/tests/test_uk_cgt_targets.py diff --git a/packages/populace-build/src/populace/build/uk_runtime/__init__.py b/packages/populace-build/src/populace/build/uk_runtime/__init__.py index cfabe562..b9bd17b3 100644 --- a/packages/populace-build/src/populace/build/uk_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/uk_runtime/__init__.py @@ -32,6 +32,12 @@ validate_uk_firm_population, write_uk_firm_population, ) +from populace.build.uk_runtime.fiscal_targets import ( + UK_CGT_REQUIRED_COLUMNS, + UK_CGT_TARGET_COVERAGE_REQUIREMENTS, + UK_CGT_TARGET_SPECS, + UK_FISCAL_TARGET_REGISTRY, +) from populace.build.uk_runtime.frs_hmrc_leaves import ( FRS_HMRC_INCPBEN_COLUMN, FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, @@ -299,6 +305,10 @@ ) __all__ = [ + "UK_CGT_REQUIRED_COLUMNS", + "UK_CGT_TARGET_COVERAGE_REQUIREMENTS", + "UK_CGT_TARGET_SPECS", + "UK_FISCAL_TARGET_REGISTRY", "AGE_BANDS", "AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL", "AREA_TYPES", diff --git a/packages/populace-build/src/populace/build/uk_runtime/fiscal_targets.py b/packages/populace-build/src/populace/build/uk_runtime/fiscal_targets.py new file mode 100644 index 00000000..46a12c4e --- /dev/null +++ b/packages/populace-build/src/populace/build/uk_runtime/fiscal_targets.py @@ -0,0 +1,133 @@ +"""UK fiscal target facts, declared natively in Populace. + +The UK national calibration surface is currently ingested from +policyengine-uk-data via :func:`populace.calibrate.specs_from_pe_surface`, whose +docstring calls that "the transitional path from harness-extracted surfaces to +declared facts". This module starts the declared-fact side for the UK, following +``us_runtime.fiscal_targets``. + +The first facts declared here are capital gains, because their absence is +measurable. The published ``populace-uk`` release (``populace-uk-2023-dd68c73``) +carries 149 targets and none of them constrain capital gains, so the calibrated +weights leave the gains distribution unanchored: reading ``capital_gains`` +weighted by ``household_weight`` straight out of ``populace_uk_2023.h5`` gives +1.47m CGT taxpayers holding £96.0bn of gains at a £65,374 mean, against HMRC's +378k, £65.9bn and ~£174,000. The taxpayer count is 3.9x the administrative +figure and the mean gain is 62% below it — the imputation spreads gains across +far too many households in amounts that are individually far too small. + +That is distributional error, not a level shift, so it survives any +revenue-side correction. It matters for any CGT analysis: on an uncalibrated +baseline the share of people affected by a rate reform comes out roughly three +times too high. + +Two targets are declared: + +``hmrc/capital_gains_total`` + Total chargeable gains of CGT taxpayers. + +``hmrc/cgt_taxpayers`` + Number of CGT taxpayers. + +Both are household-entity, matching the household grain of the UK weights, and +both therefore need prepared columns on the household frame — the registry +refuses callables so that it can serialize, and count-like facts are documented +to use prepared indicator/count columns. See ``UK_CGT_REQUIRED_COLUMNS``. + +Open question for reviewers: ``us_runtime.fiscal_targets`` has since moved to +value-free references whose values arrive from an external Ledger artifact at +build time. The values here are inline with citations, which is what the +registry's own ``TargetSpec`` contract describes ("a value, optionally a +standard error, and always a citation"). If the UK should follow the US onto +Ledger-sourced values instead, these specs are the shape to migrate. +""" + +from __future__ import annotations + +from populace.build.gates import TargetCoverageRequirement +from populace.calibrate import TargetRegistry, TargetSpec + +__all__ = [ + "UK_CGT_REQUIRED_COLUMNS", + "UK_CGT_TARGET_COVERAGE_REQUIREMENTS", + "UK_CGT_TARGET_SPECS", + "UK_FISCAL_TARGET_REGISTRY", +] + +#: HMRC Capital Gains Tax statistics, tax year 2023-24 (published August 2025). +_HMRC_CGT_SOURCE = ( + "HMRC Capital Gains Tax statistics, tax year 2023-24, table 1: " + "https://www.gov.uk/government/statistics/capital-gains-tax-statistics" +) + +#: The reference period of the declared facts. HMRC's 2023-24 outturn is the +#: latest published tax year; build-side aging carries it to forecast years. +_HMRC_CGT_PERIOD = 2023 + +#: Columns the household frame must expose for these facts to compile. +#: +#: ``capital_gains`` is the household sum of person-level chargeable gains. +#: ``cgt_taxpayer_count`` is the household count of people whose gains exceed +#: the annual exempt amount **in force for the period** — the AEA moved +#: £12,300 -> £6,000 -> £3,000 across 2022-23 to 2024-25, so a fixed threshold +#: would silently mean different things in different years. +UK_CGT_REQUIRED_COLUMNS: tuple[str, ...] = ( + "capital_gains", + "cgt_taxpayer_count", +) + +UK_CGT_TARGET_SPECS: tuple[TargetSpec, ...] = ( + TargetSpec( + name="hmrc/capital_gains_total", + entity="household", + measure="capital_gains", + value=65_900_000_000.0, + period=_HMRC_CGT_PERIOD, + source=_HMRC_CGT_SOURCE, + family="hmrc", + notes=( + "Total chargeable gains of CGT taxpayers in 2023-24. Declared " + "unsigned: the fact is a positive aggregate. Note that a household " + "measure summing gains net of losses can be negative for " + "individual records even though the total is positive, so a build " + "aggregating losses into this column should confirm the sign " + "handling it wants." + ), + ), + TargetSpec( + name="hmrc/cgt_taxpayers", + entity="household", + measure="cgt_taxpayer_count", + value=378_000.0, + period=_HMRC_CGT_PERIOD, + source=_HMRC_CGT_SOURCE, + family="hmrc", + notes=( + "Number of CGT taxpayers in 2023-24. The measure column counts " + "people above the annual exempt amount in force for the period; " + "the AEA is policy-dependent and must track the period rather " + "than being hard-coded." + ), + ), +) + +UK_CGT_TARGET_COVERAGE_REQUIREMENTS: tuple[TargetCoverageRequirement, ...] = ( + TargetCoverageRequirement( + requirement_id="uk_capital_gains", + label="HMRC capital gains totals and taxpayer counts", + accepted_names=( + "hmrc/capital_gains_total", + "hmrc/cgt_taxpayers", + ), + min_matches=2, + notes=( + "Without both facts the gains distribution is unanchored: the " + "published populace-uk release has 1.47m CGT taxpayers against " + "HMRC's 378k. A revenue-side target constrains what CGT raises, " + "not how gains are spread across households, so it does not " + "substitute for these." + ), + ), +) + +UK_FISCAL_TARGET_REGISTRY = TargetRegistry(UK_CGT_TARGET_SPECS, country="uk") diff --git a/packages/populace-build/tests/test_uk_cgt_targets.py b/packages/populace-build/tests/test_uk_cgt_targets.py new file mode 100644 index 00000000..80ab6f39 --- /dev/null +++ b/packages/populace-build/tests/test_uk_cgt_targets.py @@ -0,0 +1,66 @@ +"""Tests for the declared UK capital gains target facts. + +The published populace-uk surface carries no capital gains targets, which +leaves the gains distribution unanchored (1.47m CGT taxpayers against HMRC's +378k). These tests pin the declared facts, their provenance, and the coverage +requirement that makes their absence a build failure rather than a silence. +""" + +import pytest + +from populace.build.uk_runtime.fiscal_targets import ( + UK_CGT_REQUIRED_COLUMNS, + UK_CGT_TARGET_COVERAGE_REQUIREMENTS, + UK_CGT_TARGET_SPECS, + UK_FISCAL_TARGET_REGISTRY, +) + + +def test_declares_gains_total_and_taxpayer_count(): + """Both facts are needed: a total alone does not pin the distribution.""" + assert {spec.name for spec in UK_CGT_TARGET_SPECS} == { + "hmrc/capital_gains_total", + "hmrc/cgt_taxpayers", + } + + +def test_every_fact_carries_provenance(): + """A fact without a citation is not a fact.""" + for spec in UK_CGT_TARGET_SPECS: + assert "gov.uk" in spec.source + assert spec.family == "hmrc" + + +def test_facts_match_hmrc_2023_24_outturn(): + """Values are HMRC's published outturn, not uprated approximations.""" + by_name = {spec.name: spec for spec in UK_CGT_TARGET_SPECS} + assert by_name["hmrc/capital_gains_total"].value == pytest.approx(65.9e9) + assert by_name["hmrc/cgt_taxpayers"].value == pytest.approx(378_000) + assert all(spec.period == 2023 for spec in UK_CGT_TARGET_SPECS) + + +def test_measures_are_declared_columns(): + """The registry refuses callables, so measures must be prepared columns.""" + measures = {spec.measure for spec in UK_CGT_TARGET_SPECS} + assert measures == set(UK_CGT_REQUIRED_COLUMNS) + assert all(isinstance(spec.measure, str) for spec in UK_CGT_TARGET_SPECS) + + +def test_facts_are_household_grain(): + """UK weights are household-level, so the facts must constrain households.""" + assert all(spec.entity == "household" for spec in UK_CGT_TARGET_SPECS) + + +def test_registry_is_uk_and_content_addressed(): + assert UK_FISCAL_TARGET_REGISTRY.country == "uk" + assert UK_FISCAL_TARGET_REGISTRY.version + + +def test_coverage_requires_both_facts(): + """A build that drops either fact must fail the gate, not pass quietly.""" + (requirement,) = UK_CGT_TARGET_COVERAGE_REQUIREMENTS + assert requirement.min_matches == 2 + assert set(requirement.accepted_names) == { + "hmrc/capital_gains_total", + "hmrc/cgt_taxpayers", + } From 4647f818e575eada131901aedf6bcff79aba6278 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Sun, 19 Jul 2026 14:52:45 +0100 Subject: [PATCH 2/2] Materialize the UK capital gains constraint frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The declared CGT facts were person-measures wearing a household entity, and nothing built the columns they compile against. Correct both specs to entity="person" — matching the uk_runtime.hmrc_calibration convention where constraint rows are person-grain and the calibrated mass stays on household Weights — and rename their measures to the prepared columns uk_cgt_measure_gains_amount and uk_cgt_measure_taxpayer_count. Add uk_runtime.cgt_calibration.materialize_uk_cgt_calibration_frame, which prepares those columns from the persisted person-level capital_gains input (no simulation needed), assembles the person+household Frame with household Weights and the dataset mass log, and returns the declared facts as a TargetRegistry. The CGT taxpayer indicator is capital_gains > annual exempt amount, and the AEA is policy-dependent (12,300 -> 6,000 -> 3,000 across 2022-23 to 2024-25). The default is derived from dataset.time_period through an explicit mapping and an unmapped period raises: silently defaulting would change what "CGT taxpayer" means without changing the declared target value. Missing capital_gains and zero positive-mass support both fail closed rather than calibrating the positive HMRC facts to zero rows. Co-Authored-By: Claude Fable 5 --- .../src/populace/build/uk_runtime/__init__.py | 16 ++ .../build/uk_runtime/cgt_calibration.py | 198 ++++++++++++++++++ .../build/uk_runtime/fiscal_targets.py | 43 ++-- .../tests/test_uk_cgt_calibration.py | 137 ++++++++++++ .../tests/test_uk_cgt_targets.py | 17 +- 5 files changed, 389 insertions(+), 22 deletions(-) create mode 100644 packages/populace-build/src/populace/build/uk_runtime/cgt_calibration.py create mode 100644 packages/populace-build/tests/test_uk_cgt_calibration.py diff --git a/packages/populace-build/src/populace/build/uk_runtime/__init__.py b/packages/populace-build/src/populace/build/uk_runtime/__init__.py index b9bd17b3..21d47e2c 100644 --- a/packages/populace-build/src/populace/build/uk_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/uk_runtime/__init__.py @@ -1,5 +1,14 @@ """UK build helpers for Populace-owned local-geography artifacts.""" +from populace.build.uk_runtime.cgt_calibration import ( + UK_CGT_ANNUAL_EXEMPT_AMOUNTS, + UK_CGT_GAINS_AMOUNT_COLUMN, + UK_CGT_SOURCE_COLUMN, + UK_CGT_TAXPAYER_COUNT_COLUMN, + UKCGTTargetMaterialization, + materialize_uk_cgt_calibration_frame, + uk_cgt_annual_exempt_amount, +) from populace.build.uk_runtime.firm_generation import ( EMPLOYMENT_BANDS, HMRC_BAND_COLUMNS, @@ -305,6 +314,13 @@ ) __all__ = [ + "UK_CGT_ANNUAL_EXEMPT_AMOUNTS", + "UK_CGT_GAINS_AMOUNT_COLUMN", + "UK_CGT_SOURCE_COLUMN", + "UK_CGT_TAXPAYER_COUNT_COLUMN", + "UKCGTTargetMaterialization", + "materialize_uk_cgt_calibration_frame", + "uk_cgt_annual_exempt_amount", "UK_CGT_REQUIRED_COLUMNS", "UK_CGT_TARGET_COVERAGE_REQUIREMENTS", "UK_CGT_TARGET_SPECS", diff --git a/packages/populace-build/src/populace/build/uk_runtime/cgt_calibration.py b/packages/populace-build/src/populace/build/uk_runtime/cgt_calibration.py new file mode 100644 index 00000000..a566e724 --- /dev/null +++ b/packages/populace-build/src/populace/build/uk_runtime/cgt_calibration.py @@ -0,0 +1,198 @@ +"""Materialize the declared UK capital gains constraint frame. + +``uk_runtime.fiscal_targets`` declares two HMRC capital gains facts. This +module prepares the person-level measure columns those facts compile against +and assembles the constraint :class:`~populace.frame.Frame`, following the +shape of :mod:`populace.build.uk_runtime.hmrc_calibration`. + +Unlike the HMRC SPI surface, no simulation is needed: ``capital_gains`` is a +persisted person-level input on the UK national dataset, so the measures read +straight off ``dataset.person``. + +The CGT taxpayer indicator is ``capital_gains > annual_exempt_amount``. The AEA +is policy-dependent — it moved £12,300 -> £6,000 -> £3,000 across 2022-23 to +2024-25 — so the default is derived from ``dataset.time_period`` through the +explicit :data:`UK_CGT_ANNUAL_EXEMPT_AMOUNTS` mapping and an unmapped period +raises. Silently defaulting would change what "CGT taxpayer" means without +changing the declared target value. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from populace.build.uk_runtime.fiscal_targets import ( + UK_CGT_REQUIRED_COLUMNS, + UK_CGT_TARGET_SPECS, +) +from populace.build.uk_runtime.national_build import UKNationalDataset +from populace.calibrate import TargetRegistry +from populace.frame import EntitySchema, Frame, Weights + +__all__ = [ + "UK_CGT_ANNUAL_EXEMPT_AMOUNTS", + "UK_CGT_GAINS_AMOUNT_COLUMN", + "UK_CGT_SOURCE_COLUMN", + "UK_CGT_TAXPAYER_COUNT_COLUMN", + "UKCGTTargetMaterialization", + "materialize_uk_cgt_calibration_frame", + "uk_cgt_annual_exempt_amount", +] + +#: The persisted person-level input the measures are derived from. +UK_CGT_SOURCE_COLUMN = "capital_gains" + +UK_CGT_GAINS_AMOUNT_COLUMN = "uk_cgt_measure_gains_amount" +UK_CGT_TAXPAYER_COUNT_COLUMN = "uk_cgt_measure_taxpayer_count" + +#: CGT annual exempt amount by build period (tax year starting in that year). +#: +#: HMRC Capital Gains Tax rates and allowances: +#: https://www.gov.uk/government/publications/rates-and-allowances-capital-gains-tax +UK_CGT_ANNUAL_EXEMPT_AMOUNTS: dict[str, float] = { + "2022": 12_300.0, + "2023": 6_000.0, + "2024": 3_000.0, + "2025": 3_000.0, + "2026": 3_000.0, +} + + +@dataclass(frozen=True) +class UKCGTTargetMaterialization: + """Person-level CGT constraint frame and the declared HMRC facts.""" + + frame: Frame + registry: TargetRegistry + annual_exempt_amount: float + taxpayer_rows: int + minimum_positive_support_rows: int + + +def uk_cgt_annual_exempt_amount(time_period: int | str) -> float: + """Return the CGT annual exempt amount in force for ``time_period``. + + Raises: + ValueError: If the period has no reviewed AEA. A wrong threshold + silently changes what the declared taxpayer-count fact means, so + an unmapped period is refused rather than defaulted. + """ + + period = str(time_period) + try: + return UK_CGT_ANNUAL_EXEMPT_AMOUNTS[period] + except KeyError: + raise ValueError( + f"No reviewed CGT annual exempt amount for period {period!r}; " + "the AEA is policy-dependent (£12,300 -> £6,000 -> £3,000 across " + "2022-23 to 2024-25) and must be declared explicitly. Known " + f"periods: {sorted(UK_CGT_ANNUAL_EXEMPT_AMOUNTS)}." + ) from None + + +def materialize_uk_cgt_calibration_frame( + dataset: UKNationalDataset, + *, + annual_exempt_amount: float | None = None, +) -> UKCGTTargetMaterialization: + """Prepare the CGT measure columns and assemble the constraint frame. + + Args: + dataset: UK national dataset carrying the persisted person-level + ``capital_gains`` input and strictly positive household weights. + annual_exempt_amount: Override for the CGT threshold. Defaults to the + reviewed AEA for ``dataset.time_period``. + + Returns: + The frame, the two declared facts as a :class:`TargetRegistry`, and + support diagnostics. + + Raises: + ValueError: If ``capital_gains`` is absent or non-finite, if any + household prior weight is not strictly positive, or if either + target has no strictly positive-mass support. + """ + + if annual_exempt_amount is None: + annual_exempt_amount = uk_cgt_annual_exempt_amount(dataset.time_period) + if not np.isfinite(annual_exempt_amount) or annual_exempt_amount < 0.0: + raise ValueError( + "CGT annual exempt amount must be finite and non-negative; got " + f"{annual_exempt_amount!r}." + ) + + person = dataset.person + household = dataset.household + if UK_CGT_SOURCE_COLUMN not in person: + raise ValueError( + "UK CGT target materialization requires the person-level " + f"{UK_CGT_SOURCE_COLUMN!r} input column; it is absent from the " + "dataset person table, and the declared HMRC capital gains facts " + "cannot be compiled without it." + ) + capital_gains = pd.to_numeric( + person[UK_CGT_SOURCE_COLUMN], errors="coerce" + ).to_numpy(dtype=float, na_value=np.nan) + if not np.isfinite(capital_gains).all(): + raise ValueError( + f"UK CGT person {UK_CGT_SOURCE_COLUMN!r} values must be finite." + ) + + mapped_mass = person["person_household_id"].map( + household.set_index("household_id")["household_weight"] + ) + if mapped_mass.isna().any() or not mapped_mass.gt(0.0).all(): + raise ValueError( + "UK CGT calibration requires strictly positive prior household " + "mass for every person row." + ) + positive_mass = mapped_mass.to_numpy(dtype=float) > 0.0 + + support = capital_gains > float(annual_exempt_amount) + measure_values = { + UK_CGT_TAXPAYER_COUNT_COLUMN: support.astype(float), + UK_CGT_GAINS_AMOUNT_COLUMN: np.where(support, capital_gains, 0.0), + } + if set(measure_values) != set(UK_CGT_REQUIRED_COLUMNS): + raise RuntimeError( # pragma: no cover - guarded by the declared facts + "UK CGT prepared columns diverged from the declared measures: " + f"{sorted(measure_values)} != {sorted(UK_CGT_REQUIRED_COLUMNS)}." + ) + + n_positive = int((support & positive_mass).sum()) + if n_positive == 0: + raise ValueError( + "UK CGT targets have no strictly positive-mass support; refusing " + "to calibrate positive HMRC capital gains facts (£65.9bn, 378k " + "taxpayers) to zero constraint rows. Check the annual exempt " + f"amount ({annual_exempt_amount:,.0f}) against the gains input." + ) + + calibration_person = person[["person_id", "person_household_id"]].reset_index( + drop=True + ) + calibration_person = calibration_person.assign(**measure_values) + frame = Frame( + { + "person": calibration_person, + "household": household[["household_id"]].copy(), + }, + EntitySchema(group_entities=("household",)), + { + "household": Weights( + household["household_weight"].to_numpy(dtype=float), + dataset.household_weight_kind, + ) + }, + mass_log=dataset.mass_log, + ) + return UKCGTTargetMaterialization( + frame=frame, + registry=TargetRegistry(UK_CGT_TARGET_SPECS, country="uk"), + annual_exempt_amount=float(annual_exempt_amount), + taxpayer_rows=int(support.sum()), + minimum_positive_support_rows=n_positive, + ) diff --git a/packages/populace-build/src/populace/build/uk_runtime/fiscal_targets.py b/packages/populace-build/src/populace/build/uk_runtime/fiscal_targets.py index 46a12c4e..3d97e6ed 100644 --- a/packages/populace-build/src/populace/build/uk_runtime/fiscal_targets.py +++ b/packages/populace-build/src/populace/build/uk_runtime/fiscal_targets.py @@ -29,10 +29,14 @@ ``hmrc/cgt_taxpayers`` Number of CGT taxpayers. -Both are household-entity, matching the household grain of the UK weights, and -both therefore need prepared columns on the household frame — the registry -refuses callables so that it can serialize, and count-like facts are documented -to use prepared indicator/count columns. See ``UK_CGT_REQUIRED_COLUMNS``. +Both are person-entity, following the UK convention established by +``uk_runtime.hmrc_calibration``: the measures are person-level, so the +constraint rows live on the person table while the calibrated weights stay on +the household table via the frame's household ``Weights``. Both therefore need +prepared columns on the person frame — the registry refuses callables so that +it can serialize, and count-like facts are documented to use prepared +indicator/count columns. See ``UK_CGT_REQUIRED_COLUMNS``, and +``uk_runtime.cgt_calibration`` for the materialization that prepares them. Open question for reviewers: ``us_runtime.fiscal_targets`` has since moved to value-free references whose values arrive from an external Ledger artifact at @@ -64,47 +68,48 @@ #: latest published tax year; build-side aging carries it to forecast years. _HMRC_CGT_PERIOD = 2023 -#: Columns the household frame must expose for these facts to compile. +#: Prepared person columns the frame must expose for these facts to compile. #: -#: ``capital_gains`` is the household sum of person-level chargeable gains. -#: ``cgt_taxpayer_count`` is the household count of people whose gains exceed -#: the annual exempt amount **in force for the period** — the AEA moved +#: ``uk_cgt_measure_gains_amount`` is each person's chargeable gains, zeroed on +#: people who are not CGT taxpayers. +#: ``uk_cgt_measure_taxpayer_count`` is the 0/1 indicator for people whose +#: gains exceed the annual exempt amount **in force for the period** — the AEA moved #: £12,300 -> £6,000 -> £3,000 across 2022-23 to 2024-25, so a fixed threshold #: would silently mean different things in different years. UK_CGT_REQUIRED_COLUMNS: tuple[str, ...] = ( - "capital_gains", - "cgt_taxpayer_count", + "uk_cgt_measure_gains_amount", + "uk_cgt_measure_taxpayer_count", ) UK_CGT_TARGET_SPECS: tuple[TargetSpec, ...] = ( TargetSpec( name="hmrc/capital_gains_total", - entity="household", - measure="capital_gains", + entity="person", + measure="uk_cgt_measure_gains_amount", value=65_900_000_000.0, period=_HMRC_CGT_PERIOD, source=_HMRC_CGT_SOURCE, family="hmrc", notes=( "Total chargeable gains of CGT taxpayers in 2023-24. Declared " - "unsigned: the fact is a positive aggregate. Note that a household " - "measure summing gains net of losses can be negative for " - "individual records even though the total is positive, so a build " + "unsigned: the fact is a positive aggregate. Note that a measure " + "carrying gains net of losses can be negative on individual " + "person records even though the total is positive, so a build " "aggregating losses into this column should confirm the sign " "handling it wants." ), ), TargetSpec( name="hmrc/cgt_taxpayers", - entity="household", - measure="cgt_taxpayer_count", + entity="person", + measure="uk_cgt_measure_taxpayer_count", value=378_000.0, period=_HMRC_CGT_PERIOD, source=_HMRC_CGT_SOURCE, family="hmrc", notes=( - "Number of CGT taxpayers in 2023-24. The measure column counts " - "people above the annual exempt amount in force for the period; " + "Number of CGT taxpayers in 2023-24. The measure column is a " + "person-level indicator for people above the annual exempt amount in force for the period; " "the AEA is policy-dependent and must track the period rather " "than being hard-coded." ), diff --git a/packages/populace-build/tests/test_uk_cgt_calibration.py b/packages/populace-build/tests/test_uk_cgt_calibration.py new file mode 100644 index 00000000..e280e84a --- /dev/null +++ b/packages/populace-build/tests/test_uk_cgt_calibration.py @@ -0,0 +1,137 @@ +"""Tests for the UK capital gains constraint-frame materialization. + +The declared HMRC facts only bind if the prepared person columns mean what the +facts say they mean. The threshold is policy-dependent, so these tests pin the +per-period annual exempt amount, the refusal to guess for an unmapped period, +and the two error paths that would otherwise calibrate a positive fact to a +zero row. +""" + +import numpy as np +import pandas as pd +import pytest + +from populace.build.uk_runtime.cgt_calibration import ( + UK_CGT_GAINS_AMOUNT_COLUMN, + UK_CGT_TAXPAYER_COUNT_COLUMN, + materialize_uk_cgt_calibration_frame, + uk_cgt_annual_exempt_amount, +) +from populace.build.uk_runtime.national_build import UKNationalDataset +from populace.frame import WeightKind + + +def _dataset(gains, *, time_period="2023", drop_gains=False, weights=None): + ids = np.arange(len(gains), dtype=int) + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "capital_gains": np.asarray(gains, dtype=float), + } + ) + if drop_gains: + person = person.drop(columns=["capital_gains"]) + return UKNationalDataset( + person=person, + benunit=pd.DataFrame({"benunit_id": ids}), + household=pd.DataFrame( + { + "household_id": ids, + "household_weight": ( + np.ones(len(ids)) if weights is None else np.asarray(weights, float) + ), + } + ), + time_period=time_period, + household_weight_kind=WeightKind.IMPORTANCE, + ) + + +@pytest.mark.parametrize( + ("period", "expected"), + [("2022", 12_300.0), ("2023", 6_000.0), ("2024", 3_000.0)], +) +def test_annual_exempt_amount_tracks_the_period(period, expected): + """The AEA moved £12,300 -> £6,000 -> £3,000; the threshold must follow.""" + assert uk_cgt_annual_exempt_amount(period) == expected + + +def test_unmapped_period_raises_rather_than_defaulting(): + """A wrong threshold silently changes what the taxpayer-count fact means.""" + with pytest.raises(ValueError, match="No reviewed CGT annual exempt amount"): + uk_cgt_annual_exempt_amount("1999") + + +def test_default_threshold_comes_from_the_dataset_period(): + """2022-23 and 2023-24 must not classify the same person identically.""" + gains = [0.0, 8_000.0, 100_000.0] + assert ( + materialize_uk_cgt_calibration_frame( + _dataset(gains, time_period="2023") + ).taxpayer_rows + == 2 + ) + assert ( + materialize_uk_cgt_calibration_frame( + _dataset(gains, time_period="2022") + ).taxpayer_rows + == 1 + ) + + +def test_measure_columns_are_indicator_and_gated_amount(): + """Counts are 0/1; amounts are zeroed off-support, not merely copied.""" + result = materialize_uk_cgt_calibration_frame( + _dataset([0.0, 5_000.0, 20_000.0, 100_000.0]) + ) + person = result.frame.table("person") + assert person[UK_CGT_TAXPAYER_COUNT_COLUMN].tolist() == [0.0, 0.0, 1.0, 1.0] + assert person[UK_CGT_GAINS_AMOUNT_COLUMN].tolist() == [ + 0.0, + 0.0, + 20_000.0, + 100_000.0, + ] + assert result.annual_exempt_amount == 6_000.0 + assert result.taxpayer_rows == 2 + assert result.minimum_positive_support_rows == 2 + + +def test_explicit_threshold_overrides_the_period_default(): + result = materialize_uk_cgt_calibration_frame( + _dataset([0.0, 5_000.0, 20_000.0]), annual_exempt_amount=10_000.0 + ) + assert result.annual_exempt_amount == 10_000.0 + assert result.taxpayer_rows == 1 + + +def test_frame_carries_person_rows_and_household_weights(): + """Constraint rows are person-grain; the calibrated mass stays household.""" + result = materialize_uk_cgt_calibration_frame( + _dataset([0.0, 20_000.0], weights=[3.0, 5.0]) + ) + assert set(result.frame.entities) == {"person", "household"} + weights = result.frame.resolve_weights("household") + assert weights.kind is WeightKind.IMPORTANCE + assert np.asarray(weights.values).tolist() == [3.0, 5.0] + + +def test_registry_contains_exactly_the_two_declared_facts(): + result = materialize_uk_cgt_calibration_frame(_dataset([0.0, 20_000.0])) + assert result.registry.country == "uk" + assert {spec.name for spec in result.registry} == { + "hmrc/capital_gains_total", + "hmrc/cgt_taxpayers", + } + + +def test_missing_capital_gains_column_fails_loudly(): + with pytest.raises(ValueError, match="capital_gains"): + materialize_uk_cgt_calibration_frame(_dataset([0.0, 1.0], drop_gains=True)) + + +def test_zero_support_refuses_to_calibrate_a_positive_fact(): + """Every person below the AEA means the £65.9bn fact has no support.""" + with pytest.raises(ValueError, match="no strictly positive-mass support"): + materialize_uk_cgt_calibration_frame(_dataset([0.0, 100.0, 5_999.0])) diff --git a/packages/populace-build/tests/test_uk_cgt_targets.py b/packages/populace-build/tests/test_uk_cgt_targets.py index 80ab6f39..a3005eca 100644 --- a/packages/populace-build/tests/test_uk_cgt_targets.py +++ b/packages/populace-build/tests/test_uk_cgt_targets.py @@ -46,9 +46,20 @@ def test_measures_are_declared_columns(): assert all(isinstance(spec.measure, str) for spec in UK_CGT_TARGET_SPECS) -def test_facts_are_household_grain(): - """UK weights are household-level, so the facts must constrain households.""" - assert all(spec.entity == "household" for spec in UK_CGT_TARGET_SPECS) +def test_facts_are_person_grain(): + """UK measures are person-level, matching the hmrc_calibration convention. + + The weights stay household-level; the frame carries them as household + ``Weights`` while the constraint rows live on the person table. + """ + assert all(spec.entity == "person" for spec in UK_CGT_TARGET_SPECS) + + +def test_measure_columns_are_prepared_names(): + """Measures name prepared columns, not raw model inputs.""" + by_name = {spec.name: spec for spec in UK_CGT_TARGET_SPECS} + assert by_name["hmrc/capital_gains_total"].measure == "uk_cgt_measure_gains_amount" + assert by_name["hmrc/cgt_taxpayers"].measure == "uk_cgt_measure_taxpayer_count" def test_registry_is_uk_and_content_addressed():