diff --git a/.github/workflows/linkml.yml b/.github/workflows/linkml.yml new file mode 100644 index 0000000..f52e61a --- /dev/null +++ b/.github/workflows/linkml.yml @@ -0,0 +1,63 @@ +name: LinkML schema + +# Validate the LinkML target schema, its example instances, and the committed +# generated artifacts (pydantic + JSON Schema). + +on: + push: + paths: + - "src/schemas/target_schema.yaml" + - "examples/**" + - "project/**" + - "tests/test_target_schema_examples.py" + - ".github/workflows/linkml.yml" + - "Makefile" + pull_request: + paths: + - "src/schemas/target_schema.yaml" + - "examples/**" + - "project/**" + - "tests/test_target_schema_examples.py" + - ".github/workflows/linkml.yml" + - "Makefile" + workflow_dispatch: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install LinkML + run: | + python -m pip install --upgrade pip + pip install "linkml>=1.7" pytest pyyaml + + - name: Lint schema + run: linkml-lint --config .linkmllint.yaml src/schemas/target_schema.yaml + + - name: Validate examples against schema + run: | + mkdir -p examples/output + linkml-run-examples \ + --schema src/schemas/target_schema.yaml \ + --input-directory examples/valid \ + --counter-example-input-directory examples/invalid \ + --output-directory examples/output + + - name: Run pytest example checks + run: pytest tests/test_target_schema_examples.py -q -o addopts="" + + - name: Check generated artifacts are up to date + run: | + gen-pydantic src/schemas/target_schema.yaml > project/pydantic/target_schema.py + gen-json-schema --inline src/schemas/target_schema.yaml > project/jsonschema/target_schema.schema.json + if ! git diff --exit-code -- project/; then + echo "::error::Generated artifacts under project/ are out of date." \ + "Run 'make gen' and commit the result." + exit 1 + fi diff --git a/.gitignore b/.gitignore index 01b75fa..d190757 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ results/tables/* # OS .DS_Store Thumbs.db + +# LinkML example-runner output (regenerated by `make test-examples`) +examples/output/ diff --git a/.linkmllint.yaml b/.linkmllint.yaml new file mode 100644 index 0000000..a3bbd59 --- /dev/null +++ b/.linkmllint.yaml @@ -0,0 +1,9 @@ +# Configuration for `linkml-lint` (see `make lint`). +extends: recommended +rules: + # The harmonized column names (datetime_UTC, gravimetric_water_content_gH2O_gs, + # water_potential_kPa, ...) are a fixed external contract: they must match the + # canonical column list in target_schema.py exactly, so they cannot be renamed + # to snake_case. Disable the naming rule rather than fight that contract. + standard_naming: + level: disabled diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c726eca --- /dev/null +++ b/Makefile @@ -0,0 +1,46 @@ +# Makefile for the LinkML target schema. +# +# Source of truth: src/schemas/target_schema.yaml +# Generated artifacts (committed): +# project/pydantic/target_schema.py (pydantic v2 models) +# project/jsonschema/target_schema.schema.json +# +# Requires the `linkml` package: pip install -e ".[linkml]" + +SCHEMA := src/schemas/target_schema.yaml +PYDANTIC_OUT := project/pydantic/target_schema.py +JSONSCHEMA_OUT := project/jsonschema/target_schema.schema.json +EXAMPLES_OUT := examples/output + +.PHONY: all gen gen-pydantic gen-json-schema lint test-examples test-pytest test clean + +all: gen test + +# ---- generation --------------------------------------------------------- +gen: gen-pydantic gen-json-schema ## regenerate all artifacts from the schema + +gen-pydantic: $(SCHEMA) ## generate pydantic v2 models + gen-pydantic $(SCHEMA) > $(PYDANTIC_OUT) + +gen-json-schema: $(SCHEMA) ## generate JSON Schema + gen-json-schema --inline $(SCHEMA) > $(JSONSCHEMA_OUT) + +# ---- validation --------------------------------------------------------- +lint: $(SCHEMA) ## lint the schema itself + linkml-lint $(SCHEMA) + +test-examples: $(SCHEMA) ## examples/valid must pass, examples/invalid must fail + mkdir -p $(EXAMPLES_OUT) + linkml-run-examples \ + --schema $(SCHEMA) \ + --input-directory examples/valid \ + --counter-example-input-directory examples/invalid \ + --output-directory $(EXAMPLES_OUT) + +test-pytest: ## run the pytest wrapper around example validation + pytest tests/test_target_schema_examples.py -o addopts="" + +test: lint test-examples test-pytest ## run all schema checks + +clean: + rm -rf $(EXAMPLES_OUT) diff --git a/README.md b/README.md index 07f6f5c..08f17ad 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ wfsfa-harmonization-eval/ │ ├── schemas/ # Pydantic data models │ │ ├── skill1_bundle.py # Curator output schema │ │ ├── skill2_mapping.py # Harmonizer output schema -│ │ └── target_schema.py # Target harmonized schema +│ │ ├── target_schema.py # Target harmonized schema +│ │ └── target_schema.yaml # LinkML version of the target schema │ │ │ ├── harness/ # Skill runners │ │ ├── run_skill1.py # Curator runner @@ -151,6 +152,14 @@ wfsfa-harmonization-eval/ │ ├── phase_b_prospective.py # Phase B: novel data │ └── metric_validation.py # Validate automated vs expert scores │ +├── examples/ # LinkML schema example instances +│ ├── valid/ # Instances that MUST validate +│ └── invalid/ # Counter-examples that MUST fail +│ +├── project/ # Generated LinkML artifacts (do not edit) +│ ├── pydantic/ # gen-pydantic output +│ └── jsonschema/ # gen-json-schema output +│ ├── results/ # Outputs │ ├── raw_runs/ # Per-run outputs │ ├── scored/ # Scored metrics (tidy CSV) @@ -166,6 +175,33 @@ wfsfa-harmonization-eval/ └── test_metrics.py # Known-answer test cases ``` +## LinkML Target Schema + +The canonical 9-column harmonized schema is also defined declaratively in +[`src/schemas/target_schema.yaml`](src/schemas/target_schema.yaml), a +[LinkML](https://linkml.io) schema equivalent to `target_schema.py`. It is the +single source of truth for two committed, generated artifacts: + +- `project/pydantic/target_schema.py` — Pydantic v2 models (`gen-pydantic`) +- `project/jsonschema/target_schema.schema.json` — JSON Schema (`gen-json-schema`) + +Example instances live under `examples/`: everything in `examples/valid/` must +validate against the schema, and every counter-example in `examples/invalid/` +must fail (one per constraint — missing required field, out-of-range value, +wrong type, and the "at least one moisture variable" rule). + +```bash +pip install -e ".[linkml]" # install the LinkML toolchain + +make gen # regenerate the pydantic + JSON Schema artifacts +make test # lint the schema + validate all examples + run pytest checks +``` + +CI (`.github/workflows/linkml.yml`) runs the same checks and fails if the +committed artifacts under `project/` drift from the schema. The pytest wrapper +(`tests/test_target_schema_examples.py`) runs as part of the normal test suite +when `linkml` is installed and is skipped otherwise. + ## Installation ```bash diff --git a/examples/invalid/HarmonizedRecord-bad-boolean.yaml b/examples/invalid/HarmonizedRecord-bad-boolean.yaml new file mode 100644 index 0000000..b66b4d1 --- /dev/null +++ b/examples/invalid/HarmonizedRecord-bad-boolean.yaml @@ -0,0 +1,5 @@ +# INVALID: is_timeseries must be a boolean, not a string. +datetime_UTC: '2021-06-15T12:00:00+00:00' +site_id: SITE_A +is_timeseries: maybe +volumetric_water_content_m3_m3: 0.32 diff --git a/examples/invalid/HarmonizedRecord-depth-out-of-range.yaml b/examples/invalid/HarmonizedRecord-depth-out-of-range.yaml new file mode 100644 index 0000000..a75e07e --- /dev/null +++ b/examples/invalid/HarmonizedRecord-depth-out-of-range.yaml @@ -0,0 +1,6 @@ +# INVALID: depth_m exceeds the maximum of 100 m. +datetime_UTC: '2021-06-15T12:00:00+00:00' +site_id: SITE_A +depth_m: 250.0 +is_timeseries: false +volumetric_water_content_m3_m3: 0.32 diff --git a/examples/invalid/HarmonizedRecord-missing-required-site.yaml b/examples/invalid/HarmonizedRecord-missing-required-site.yaml new file mode 100644 index 0000000..c2e6cf0 --- /dev/null +++ b/examples/invalid/HarmonizedRecord-missing-required-site.yaml @@ -0,0 +1,5 @@ +# INVALID: site_id is required but absent. +datetime_UTC: '2021-06-15T12:00:00+00:00' +depth_m: 0.10 +is_timeseries: true +volumetric_water_content_m3_m3: 0.32 diff --git a/examples/invalid/HarmonizedRecord-no-moisture.yaml b/examples/invalid/HarmonizedRecord-no-moisture.yaml new file mode 100644 index 0000000..2ed44a9 --- /dev/null +++ b/examples/invalid/HarmonizedRecord-no-moisture.yaml @@ -0,0 +1,5 @@ +# INVALID: no moisture variable present (violates the at-least-one rule). +datetime_UTC: '2021-06-15T12:00:00+00:00' +site_id: SITE_A +depth_m: 0.10 +is_timeseries: false diff --git a/examples/invalid/HarmonizedRecord-positive-water-potential.yaml b/examples/invalid/HarmonizedRecord-positive-water-potential.yaml new file mode 100644 index 0000000..f7c0e19 --- /dev/null +++ b/examples/invalid/HarmonizedRecord-positive-water-potential.yaml @@ -0,0 +1,5 @@ +# INVALID: water potential must be non-positive (<= 0 kPa). +datetime_UTC: '2021-06-15T12:00:00+00:00' +site_id: SITE_C +is_timeseries: false +water_potential_kPa: 50.0 diff --git a/examples/invalid/HarmonizedRecord-vwc-above-one.yaml b/examples/invalid/HarmonizedRecord-vwc-above-one.yaml new file mode 100644 index 0000000..32432a7 --- /dev/null +++ b/examples/invalid/HarmonizedRecord-vwc-above-one.yaml @@ -0,0 +1,6 @@ +# INVALID: volumetric water content is a fraction and must be <= 1.0. +datetime_UTC: '2021-06-15T12:00:00+00:00' +site_id: SITE_A +depth_m: 0.10 +is_timeseries: false +volumetric_water_content_m3_m3: 1.8 diff --git a/examples/valid/HarmonizedDataset-mixed.yaml b/examples/valid/HarmonizedDataset-mixed.yaml new file mode 100644 index 0000000..c6224cd --- /dev/null +++ b/examples/valid/HarmonizedDataset-mixed.yaml @@ -0,0 +1,20 @@ +# A small dataset combining records that report different moisture variables. +records: + - datetime_UTC: '2021-06-15T12:00:00+00:00' + site_id: SITE_A + depth_m: 0.10 + replicate: 1.0 + is_timeseries: true + interval_min: 60.0 + volumetric_water_content_m3_m3: 0.32 + - datetime_UTC: '2021-06-15T13:00:00+00:00' + site_id: SITE_A + depth_m: 0.10 + replicate: 1.0 + is_timeseries: true + interval_min: 60.0 + volumetric_water_content_m3_m3: 0.31 + - datetime_UTC: '2019-03-01T00:00:00+00:00' + site_id: SITE_B + is_timeseries: false + gravimetric_water_content_gH2O_gs: 0.18 diff --git a/examples/valid/HarmonizedRecord-gravimetric-minimal.yaml b/examples/valid/HarmonizedRecord-gravimetric-minimal.yaml new file mode 100644 index 0000000..23539f6 --- /dev/null +++ b/examples/valid/HarmonizedRecord-gravimetric-minimal.yaml @@ -0,0 +1,5 @@ +# Minimal record: only the required fields plus one moisture variable. +datetime_UTC: '2019-03-01T00:00:00+00:00' +site_id: SITE_B +is_timeseries: false +gravimetric_water_content_gH2O_gs: 0.18 diff --git a/examples/valid/HarmonizedRecord-volumetric-timeseries.yaml b/examples/valid/HarmonizedRecord-volumetric-timeseries.yaml new file mode 100644 index 0000000..35878ca --- /dev/null +++ b/examples/valid/HarmonizedRecord-volumetric-timeseries.yaml @@ -0,0 +1,8 @@ +# A full time-series record reporting volumetric water content. +datetime_UTC: '2021-06-15T12:00:00+00:00' +site_id: SITE_A +depth_m: 0.10 +replicate: 1.0 +is_timeseries: true +interval_min: 60.0 +volumetric_water_content_m3_m3: 0.32 diff --git a/examples/valid/HarmonizedRecord-water-potential.yaml b/examples/valid/HarmonizedRecord-water-potential.yaml new file mode 100644 index 0000000..f4e132c --- /dev/null +++ b/examples/valid/HarmonizedRecord-water-potential.yaml @@ -0,0 +1,7 @@ +# A grab sample reporting (negative) soil water potential. +datetime_UTC: '2020-09-21T18:30:00+00:00' +site_id: SITE_C +depth_m: 0.30 +replicate: 2.0 +is_timeseries: false +water_potential_kPa: -1500.0 diff --git a/project/jsonschema/target_schema.schema.json b/project/jsonschema/target_schema.schema.json new file mode 100644 index 0000000..2dc15b2 --- /dev/null +++ b/project/jsonschema/target_schema.schema.json @@ -0,0 +1,156 @@ +{ + "$defs": { + "HarmonizedDataset": { + "additionalProperties": false, + "description": "A harmonized dataset: an ordered collection of ``HarmonizedRecord`` rows conforming to the target schema.", + "properties": { + "records": { + "description": "The harmonized rows that make up the dataset.", + "items": { + "$ref": "#/$defs/HarmonizedRecord" + }, + "type": [ + "array", + "null" + ] + } + }, + "title": "HarmonizedDataset", + "type": "object" + }, + "HarmonizedRecord": { + "additionalProperties": false, + "description": "A single row of a harmonized soil moisture dataset. The ordered list of slots corresponds exactly to ``HARMONIZED_COLUMNS`` (order matters). ``datetime_UTC`` and ``site_id`` are required, and at least one of the three moisture variables must be present.", + "if": { + "properties": { + "datetime_UTC": {} + }, + "required": [ + "datetime_UTC" + ] + }, + "properties": { + "datetime_UTC": { + "description": "Observation timestamp in UTC. Maps to a pandas ``datetime64[ns, UTC]`` column. Required.", + "format": "date-time", + "type": "string" + }, + "depth_m": { + "description": "Sampling depth below the surface, in meters.", + "maximum": 100.0, + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "gravimetric_water_content_gH2O_gs": { + "description": "Gravimetric water content (grams of water per gram of dry soil). The generous upper bound accommodates organic-rich soils.", + "maximum": 10.0, + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "interval_min": { + "description": "Sampling interval in minutes for time-series records. Stored as a float to allow NaN (missing) values. Nullable.", + "maximum": 1000000.0, + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "is_timeseries": { + "description": "Whether the record belongs to a time series (true) or is a one-off / point measurement (false).", + "type": [ + "boolean", + "null" + ] + }, + "replicate": { + "description": "Replicate number for the observation. Stored as a float to allow NaN (missing) values. Nullable.", + "type": [ + "number", + "null" + ] + }, + "site_id": { + "description": "Identifier for the sampling site. Required.", + "type": "string" + }, + "volumetric_water_content_m3_m3": { + "description": "Volumetric water content as a fraction (cubic meters of water per cubic meter of soil).", + "maximum": 1.0, + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "water_potential_kPa": { + "description": "Soil water potential in kilopascals. Values are non-positive (zero at saturation, increasingly negative as the soil dries).", + "maximum": 0.0, + "minimum": -1000000.0, + "type": [ + "number", + "null" + ] + } + }, + "required": [ + "datetime_UTC", + "site_id" + ], + "then": { + "anyOf": [ + { + "properties": { + "volumetric_water_content_m3_m3": {} + }, + "required": [ + "volumetric_water_content_m3_m3" + ] + }, + { + "properties": { + "gravimetric_water_content_gH2O_gs": {} + }, + "required": [ + "gravimetric_water_content_gH2O_gs" + ] + }, + { + "properties": { + "water_potential_kPa": {} + }, + "required": [ + "water_potential_kPa" + ] + } + ] + }, + "title": "HarmonizedRecord", + "type": "object" + }, + "QCFlagVocabulary": { + "description": "Controlled vocabulary for QC flag values. Equivalent to the ``QCFlagVocabulary`` enum in ``target_schema.py``.", + "enum": [ + "d1", + "g1", + "g2" + ], + "title": "QCFlagVocabulary", + "type": "string" + } + }, + "$id": "https://w3id.org/bioepic/data-harmonization-eval/target-schema", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "additionalProperties": true, + "metamodel_version": "1.11.0", + "title": "target_schema", + "type": "object", + "version": null +} + diff --git a/project/pydantic/target_schema.py b/project/pydantic/target_schema.py new file mode 100644 index 0000000..2de6c05 --- /dev/null +++ b/project/pydantic/target_schema.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import re +import sys +from datetime import ( + date, + datetime, + time +) +from decimal import Decimal +from enum import Enum +from typing import ( + Any, + ClassVar, + Literal, + Optional, + Union +) + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + RootModel, + SerializationInfo, + SerializerFunctionWrapHandler, + field_validator, + model_serializer +) + + +metamodel_version = "1.11.0" +version = "None" + + +class ConfiguredBaseModel(BaseModel): + model_config = ConfigDict( + serialize_by_alias = True, + validate_by_name = True, + validate_assignment = True, + validate_default = True, + extra = "forbid", + arbitrary_types_allowed = True, + use_enum_values = True, + strict = False, + ) + + + + + +class LinkMLMeta(RootModel): + root: dict[str, Any] = {} + model_config = ConfigDict(frozen=True) + + def __getattr__(self, key:str): + return getattr(self.root, key) + + def __getitem__(self, key:str): + return self.root[key] + + def __setitem__(self, key:str, value): + self.root[key] = value + + def __contains__(self, key:str) -> bool: + return key in self.root + + +linkml_meta = LinkMLMeta({'default_prefix': 'harmon', + 'default_range': 'string', + 'description': 'LinkML representation of the canonical harmonized dataset ' + 'schema. Every harmonized dataset must conform to this exact ' + '9-column schema. This is the LinkML equivalent of ' + '``src/schemas/target_schema.py``: the ``HarmonizedRecord`` ' + 'class describes a single row, with one slot per harmonized ' + 'column. Slot ranges, units, and value constraints mirror the ' + '``HARMONIZED_COLUMNS``, ``EXPECTED_DTYPES`` and ' + '``VALID_RANGES`` constants in that module.', + 'id': 'https://w3id.org/bioepic/data-harmonization-eval/target-schema', + 'imports': ['linkml:types'], + 'license': 'MIT', + 'name': 'target_schema', + 'prefixes': {'harmon': {'prefix_prefix': 'harmon', + 'prefix_reference': 'https://w3id.org/bioepic/data-harmonization-eval/target-schema/'}, + 'linkml': {'prefix_prefix': 'linkml', + 'prefix_reference': 'https://w3id.org/linkml/'}}, + 'source_file': 'src/schemas/target_schema.yaml', + 'title': 'Harmonized Soil Moisture Target Schema'} ) + +class QCFlagVocabulary(str, Enum): + """ + Controlled vocabulary for QC flag values. Equivalent to the ``QCFlagVocabulary`` enum in ``target_schema.py``. + """ + d1 = "d1" + """ + Depth approximated from a reported range. + """ + g1 = "g1" + """ + Coordinates from Varadharajan et al. (not present in the source dataset). + """ + g2 = "g2" + """ + Coordinates not available from any source. + """ + + + +class HarmonizedRecord(ConfiguredBaseModel): + """ + A single row of a harmonized soil moisture dataset. The ordered list of slots corresponds exactly to ``HARMONIZED_COLUMNS`` (order matters). ``datetime_UTC`` and ``site_id`` are required, and at least one of the three moisture variables must be present. + """ + linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({'from_schema': 'https://w3id.org/bioepic/data-harmonization-eval/target-schema', + 'rules': [{'description': 'At least one moisture variable (volumetric water ' + 'content, gravimetric water content, or water ' + 'potential) must be present in a record.', + 'postconditions': {'any_of': [{'slot_conditions': {'volumetric_water_content_m3_m3': {'name': 'volumetric_water_content_m3_m3', + 'value_presence': 'PRESENT'}}}, + {'slot_conditions': {'gravimetric_water_content_gH2O_gs': {'name': 'gravimetric_water_content_gH2O_gs', + 'value_presence': 'PRESENT'}}}, + {'slot_conditions': {'water_potential_kPa': {'name': 'water_potential_kPa', + 'value_presence': 'PRESENT'}}}]}, + 'preconditions': {'slot_conditions': {'datetime_UTC': {'name': 'datetime_UTC', + 'value_presence': 'PRESENT'}}}}]}) + + datetime_UTC: datetime = Field(default=..., description="""Observation timestamp in UTC. Maps to a pandas ``datetime64[ns, UTC]`` column. Required.""", json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 1} }) + site_id: str = Field(default=..., description="""Identifier for the sampling site. Required.""", json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 2} }) + depth_m: Optional[float] = Field(default=None, description="""Sampling depth below the surface, in meters.""", ge=0.0, le=100.0, json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 3, 'unit': {'ucum_code': 'm'}} }) + replicate: Optional[float] = Field(default=None, description="""Replicate number for the observation. Stored as a float to allow NaN (missing) values. Nullable.""", json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 4} }) + is_timeseries: Optional[bool] = Field(default=None, description="""Whether the record belongs to a time series (true) or is a one-off / point measurement (false).""", json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 5} }) + interval_min: Optional[float] = Field(default=None, description="""Sampling interval in minutes for time-series records. Stored as a float to allow NaN (missing) values. Nullable.""", ge=0.0, le=1000000.0, json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 6, 'unit': {'ucum_code': 'min'}} }) + volumetric_water_content_m3_m3: Optional[float] = Field(default=None, description="""Volumetric water content as a fraction (cubic meters of water per cubic meter of soil).""", ge=0.0, le=1.0, json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 7, 'unit': {'ucum_code': 'm3/m3'}} }) + gravimetric_water_content_gH2O_gs: Optional[float] = Field(default=None, description="""Gravimetric water content (grams of water per gram of dry soil). The generous upper bound accommodates organic-rich soils.""", ge=0.0, le=10.0, json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 8, 'unit': {'ucum_code': 'g/g'}} }) + water_potential_kPa: Optional[float] = Field(default=None, description="""Soil water potential in kilopascals. Values are non-positive (zero at saturation, increasingly negative as the soil dries).""", ge=-1000000.0, le=0.0, json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedRecord'], 'rank': 9, 'unit': {'ucum_code': 'kPa'}} }) + + +class HarmonizedDataset(ConfiguredBaseModel): + """ + A harmonized dataset: an ordered collection of ``HarmonizedRecord`` rows conforming to the target schema. + """ + linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({'from_schema': 'https://w3id.org/bioepic/data-harmonization-eval/target-schema'}) + + records: Optional[list[HarmonizedRecord]] = Field(default=None, description="""The harmonized rows that make up the dataset.""", json_schema_extra = { "linkml_meta": {'domain_of': ['HarmonizedDataset']} }) + + +# Model rebuild +# see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model +HarmonizedRecord.model_rebuild() +HarmonizedDataset.model_rebuild() diff --git a/pyproject.toml b/pyproject.toml index fe30f80..9157c1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,12 @@ notebooks = [ "ipywidgets>=8.0.0", ] +# LinkML schema tooling: validation, linting, and code generation for +# src/schemas/target_schema.yaml (see the Makefile and .github/workflows/linkml.yml). +linkml = [ + "linkml>=1.7", +] + [tool.setuptools] packages = ["src"] diff --git a/src/schemas/target_schema.yaml b/src/schemas/target_schema.yaml new file mode 100644 index 0000000..0b3a815 --- /dev/null +++ b/src/schemas/target_schema.yaml @@ -0,0 +1,188 @@ +id: https://w3id.org/bioepic/data-harmonization-eval/target-schema +name: target_schema +title: Harmonized Soil Moisture Target Schema +description: >- + LinkML representation of the canonical harmonized dataset schema. + Every harmonized dataset must conform to this exact 9-column schema. + This is the LinkML equivalent of ``src/schemas/target_schema.py``: the + ``HarmonizedRecord`` class describes a single row, with one slot per + harmonized column. Slot ranges, units, and value constraints mirror the + ``HARMONIZED_COLUMNS``, ``EXPECTED_DTYPES`` and ``VALID_RANGES`` constants + in that module. +license: MIT +default_prefix: harmon +default_range: string + +prefixes: + harmon: https://w3id.org/bioepic/data-harmonization-eval/target-schema/ + linkml: https://w3id.org/linkml/ + +imports: + - linkml:types + +#================================= +# Classes +#================================= +classes: + + HarmonizedRecord: + description: >- + A single row of a harmonized soil moisture dataset. The ordered list of + slots corresponds exactly to ``HARMONIZED_COLUMNS`` (order matters). + ``datetime_UTC`` and ``site_id`` are required, and at least one of the + three moisture variables must be present. + slots: + - datetime_UTC + - site_id + - depth_m + - replicate + - is_timeseries + - interval_min + - volumetric_water_content_m3_m3 + - gravimetric_water_content_gH2O_gs + - water_potential_kPa + rules: + - description: >- + At least one moisture variable (volumetric water content, gravimetric + water content, or water potential) must be present in a record. + # The precondition is always satisfied (datetime_UTC is required), so the + # postcondition is enforced unconditionally. It is stated explicitly so + # the generated JSON Schema emits a well-formed if/then conditional. + preconditions: + slot_conditions: + datetime_UTC: + value_presence: PRESENT + postconditions: + any_of: + - slot_conditions: + volumetric_water_content_m3_m3: + value_presence: PRESENT + - slot_conditions: + gravimetric_water_content_gH2O_gs: + value_presence: PRESENT + - slot_conditions: + water_potential_kPa: + value_presence: PRESENT + + HarmonizedDataset: + description: >- + A harmonized dataset: an ordered collection of ``HarmonizedRecord`` rows + conforming to the target schema. + attributes: + records: + description: The harmonized rows that make up the dataset. + range: HarmonizedRecord + multivalued: true + inlined_as_list: true + +#================================= +# Slots +#================================= +slots: + + datetime_UTC: + description: >- + Observation timestamp in UTC. Maps to a pandas + ``datetime64[ns, UTC]`` column. Required. + range: datetime + required: true + rank: 1 + + site_id: + description: Identifier for the sampling site. Required. + range: string + required: true + rank: 2 + + depth_m: + description: Sampling depth below the surface, in meters. + range: float + required: false + unit: + ucum_code: m + minimum_value: 0.0 + maximum_value: 100.0 + rank: 3 + + replicate: + description: >- + Replicate number for the observation. Stored as a float to allow NaN + (missing) values. Nullable. + range: float + required: false + rank: 4 + + is_timeseries: + description: >- + Whether the record belongs to a time series (true) or is a one-off / + point measurement (false). + range: boolean + required: false + rank: 5 + + interval_min: + description: >- + Sampling interval in minutes for time-series records. Stored as a float + to allow NaN (missing) values. Nullable. + range: float + required: false + unit: + ucum_code: min + minimum_value: 0.0 + maximum_value: 1000000.0 + rank: 6 + + volumetric_water_content_m3_m3: + description: >- + Volumetric water content as a fraction (cubic meters of water per cubic + meter of soil). + range: float + required: false + unit: + ucum_code: m3/m3 + minimum_value: 0.0 + maximum_value: 1.0 + rank: 7 + + gravimetric_water_content_gH2O_gs: + description: >- + Gravimetric water content (grams of water per gram of dry soil). The + generous upper bound accommodates organic-rich soils. + range: float + required: false + unit: + ucum_code: g/g + minimum_value: 0.0 + maximum_value: 10.0 + rank: 8 + + water_potential_kPa: + description: >- + Soil water potential in kilopascals. Values are non-positive (zero at + saturation, increasingly negative as the soil dries). + range: float + required: false + unit: + ucum_code: kPa + minimum_value: -1000000.0 + maximum_value: 0.0 + rank: 9 + +#================================= +# Enumerations +#================================= +enums: + + QCFlagVocabulary: + description: >- + Controlled vocabulary for QC flag values. Equivalent to the + ``QCFlagVocabulary`` enum in ``target_schema.py``. + permissible_values: + d1: + description: Depth approximated from a reported range. + g1: + description: >- + Coordinates from Varadharajan et al. (not present in the source + dataset). + g2: + description: Coordinates not available from any source. diff --git a/tests/test_target_schema_examples.py b/tests/test_target_schema_examples.py new file mode 100644 index 0000000..37d8e3d --- /dev/null +++ b/tests/test_target_schema_examples.py @@ -0,0 +1,59 @@ +"""Validate the LinkML target schema and its example instances. + +Each file under ``examples/valid/`` MUST validate against +``src/schemas/target_schema.yaml``; each file under ``examples/invalid/`` MUST +fail validation. The target class for a file is taken from the part of its +filename before the first ``-`` (e.g. ``HarmonizedRecord-foo.yaml`` -> +``HarmonizedRecord``), matching the convention used by ``linkml-run-examples``. + +These tests require the ``linkml`` package (see the ``linkml`` optional +dependency group in ``pyproject.toml``); they are skipped if it is not +installed so the rest of the suite can run without it. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +pytest.importorskip("linkml.validator", reason="linkml is not installed") + +from linkml.validator import validate # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCHEMA = REPO_ROOT / "src" / "schemas" / "target_schema.yaml" +VALID_DIR = REPO_ROOT / "examples" / "valid" +INVALID_DIR = REPO_ROOT / "examples" / "invalid" + + +def _target_class(path: Path) -> str: + """Infer the schema class a given example file should be validated against.""" + return path.stem.split("-", 1)[0] + + +def _instances(directory: Path) -> list[Path]: + return sorted(directory.glob("*.yaml")) + + +def _validate(path: Path): + instance = yaml.safe_load(path.read_text()) + return validate(instance, str(SCHEMA), _target_class(path)) + + +@pytest.mark.parametrize("path", _instances(VALID_DIR), ids=lambda p: p.name) +def test_valid_examples_pass(path: Path) -> None: + report = _validate(path) + messages = [r.message for r in report.results] + assert not report.results, f"{path.name} should validate but got: {messages}" + + +@pytest.mark.parametrize("path", _instances(INVALID_DIR), ids=lambda p: p.name) +def test_invalid_examples_fail(path: Path) -> None: + report = _validate(path) + assert report.results, f"{path.name} should fail validation but passed" + + +def test_example_directories_are_populated() -> None: + assert _instances(VALID_DIR), "no valid examples found" + assert _instances(INVALID_DIR), "no invalid examples found"