Add LinkML schema for harmonized soil moisture data

.github/workflows/linkml.yml

@@ -0,0 +1,63 @@
+name: LinkML schema
+
+# Validate the LinkML target schema, its example instances, and the committed
+# generated artifacts (pydantic + JSON Schema).
+
+on:
+  push:
+    paths:
+      - "src/schemas/target_schema.yaml"
+      - "examples/**"
+      - "project/**"
+      - "tests/test_target_schema_examples.py"
+      - ".github/workflows/linkml.yml"
+      - "Makefile"
+  pull_request:
+    paths:
+      - "src/schemas/target_schema.yaml"
+      - "examples/**"
+      - "project/**"
+      - "tests/test_target_schema_examples.py"
+      - ".github/workflows/linkml.yml"
+      - "Makefile"
+  workflow_dispatch:
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install LinkML
+        run: |
+          python -m pip install --upgrade pip
+          pip install "linkml>=1.7" pytest pyyaml
+
+      - name: Lint schema
+        run: linkml-lint --config .linkmllint.yaml src/schemas/target_schema.yaml
+
+      - name: Validate examples against schema
+        run: |
+          mkdir -p examples/output
+          linkml-run-examples \
+            --schema src/schemas/target_schema.yaml \
+            --input-directory examples/valid \
+            --counter-example-input-directory examples/invalid \
+            --output-directory examples/output
+
+      - name: Run pytest example checks
+        run: pytest tests/test_target_schema_examples.py -q -o addopts=""
+
+      - name: Check generated artifacts are up to date
+        run: |
+          gen-pydantic src/schemas/target_schema.yaml > project/pydantic/target_schema.py
+          gen-json-schema --inline src/schemas/target_schema.yaml > project/jsonschema/target_schema.schema.json
+          if ! git diff --exit-code -- project/; then
+            echo "::error::Generated artifacts under project/ are out of date." \
+                 "Run 'make gen' and commit the result."
+            exit 1
+          fi

.gitignore

@@ -81,3 +81,6 @@ results/tables/*
 # OS
 .DS_Store
 Thumbs.db
+
+# LinkML example-runner output (regenerated by `make test-examples`)
+examples/output/

.linkmllint.yaml

@@ -0,0 +1,9 @@
+# Configuration for `linkml-lint` (see `make lint`).
+extends: recommended
+rules:
+  # The harmonized column names (datetime_UTC, gravimetric_water_content_gH2O_gs,
+  # water_potential_kPa, ...) are a fixed external contract: they must match the
+  # canonical column list in target_schema.py exactly, so they cannot be renamed
+  # to snake_case. Disable the naming rule rather than fight that contract.
+  standard_naming:
+    level: disabled

Makefile

@@ -0,0 +1,46 @@
+# Makefile for the LinkML target schema.
+#
+# Source of truth:  src/schemas/target_schema.yaml
+# Generated artifacts (committed):
+#   project/pydantic/target_schema.py          (pydantic v2 models)
+#   project/jsonschema/target_schema.schema.json
+#
+# Requires the `linkml` package:  pip install -e ".[linkml]"
+
+SCHEMA       := src/schemas/target_schema.yaml
+PYDANTIC_OUT := project/pydantic/target_schema.py
+JSONSCHEMA_OUT := project/jsonschema/target_schema.schema.json
+EXAMPLES_OUT := examples/output
+
+.PHONY: all gen gen-pydantic gen-json-schema lint test-examples test-pytest test clean
+
+all: gen test
+
+# ---- generation ---------------------------------------------------------
+gen: gen-pydantic gen-json-schema  ## regenerate all artifacts from the schema
+
+gen-pydantic: $(SCHEMA)  ## generate pydantic v2 models
+	gen-pydantic $(SCHEMA) > $(PYDANTIC_OUT)
+
+gen-json-schema: $(SCHEMA)  ## generate JSON Schema
+	gen-json-schema --inline $(SCHEMA) > $(JSONSCHEMA_OUT)
+
+# ---- validation ---------------------------------------------------------
+lint: $(SCHEMA)  ## lint the schema itself
+	linkml-lint $(SCHEMA)
+
+test-examples: $(SCHEMA)  ## examples/valid must pass, examples/invalid must fail
+	mkdir -p $(EXAMPLES_OUT)
+	linkml-run-examples \
+		--schema $(SCHEMA) \
+		--input-directory examples/valid \
+		--counter-example-input-directory examples/invalid \
+		--output-directory $(EXAMPLES_OUT)
+
+test-pytest:  ## run the pytest wrapper around example validation
+	pytest tests/test_target_schema_examples.py -o addopts=""
+
+test: lint test-examples test-pytest  ## run all schema checks
+
+clean:
+	rm -rf $(EXAMPLES_OUT)

README.md

@@ -119,7 +119,8 @@ wfsfa-harmonization-eval/
 │   ├── schemas/                 # Pydantic data models
 │   │   ├── skill1_bundle.py     # Curator output schema
 │   │   ├── skill2_mapping.py    # Harmonizer output schema
-│   │   └── target_schema.py     # Target harmonized schema
+│   │   ├── target_schema.py     # Target harmonized schema
+│   │   └── target_schema.yaml   # LinkML version of the target schema
 │   │
 │   ├── harness/                 # Skill runners
 │   │   ├── run_skill1.py        # Curator runner
@@ -151,6 +152,14 @@ wfsfa-harmonization-eval/
 │   ├── phase_b_prospective.py   # Phase B: novel data
 │   └── metric_validation.py     # Validate automated vs expert scores
 │
+├── examples/                    # LinkML schema example instances
+│   ├── valid/                   # Instances that MUST validate
+│   └── invalid/                 # Counter-examples that MUST fail
+│
+├── project/                     # Generated LinkML artifacts (do not edit)
+│   ├── pydantic/                # gen-pydantic output
+│   └── jsonschema/              # gen-json-schema output
+│
 ├── results/                     # Outputs
 │   ├── raw_runs/                # Per-run outputs
 │   ├── scored/                  # Scored metrics (tidy CSV)
@@ -166,6 +175,33 @@ wfsfa-harmonization-eval/
     └── test_metrics.py          # Known-answer test cases
 ```
 
+## LinkML Target Schema
+
+The canonical 9-column harmonized schema is also defined declaratively in
+[`src/schemas/target_schema.yaml`](src/schemas/target_schema.yaml), a
+[LinkML](https://linkml.io) schema equivalent to `target_schema.py`. It is the
+single source of truth for two committed, generated artifacts:
+
+- `project/pydantic/target_schema.py` — Pydantic v2 models (`gen-pydantic`)
+- `project/jsonschema/target_schema.schema.json` — JSON Schema (`gen-json-schema`)
+
+Example instances live under `examples/`: everything in `examples/valid/` must
+validate against the schema, and every counter-example in `examples/invalid/`
+must fail (one per constraint — missing required field, out-of-range value,
+wrong type, and the "at least one moisture variable" rule).
+
+```bash
+pip install -e ".[linkml]"   # install the LinkML toolchain
+
+make gen      # regenerate the pydantic + JSON Schema artifacts
+make test     # lint the schema + validate all examples + run pytest checks
+```
+
+CI (`.github/workflows/linkml.yml`) runs the same checks and fails if the
+committed artifacts under `project/` drift from the schema. The pytest wrapper
+(`tests/test_target_schema_examples.py`) runs as part of the normal test suite
+when `linkml` is installed and is skipped otherwise.
+
 ## Installation
 
 ```bash

examples/invalid/HarmonizedRecord-bad-boolean.yaml

@@ -0,0 +1,5 @@
+# INVALID: is_timeseries must be a boolean, not a string.
+datetime_UTC: '2021-06-15T12:00:00+00:00'
+site_id: SITE_A
+is_timeseries: maybe
+volumetric_water_content_m3_m3: 0.32

examples/invalid/HarmonizedRecord-depth-out-of-range.yaml

@@ -0,0 +1,6 @@
+# INVALID: depth_m exceeds the maximum of 100 m.
+datetime_UTC: '2021-06-15T12:00:00+00:00'
+site_id: SITE_A
+depth_m: 250.0
+is_timeseries: false
+volumetric_water_content_m3_m3: 0.32

examples/invalid/HarmonizedRecord-missing-required-site.yaml

@@ -0,0 +1,5 @@
+# INVALID: site_id is required but absent.
+datetime_UTC: '2021-06-15T12:00:00+00:00'
+depth_m: 0.10
+is_timeseries: true
+volumetric_water_content_m3_m3: 0.32

examples/invalid/HarmonizedRecord-no-moisture.yaml

@@ -0,0 +1,5 @@
+# INVALID: no moisture variable present (violates the at-least-one rule).
+datetime_UTC: '2021-06-15T12:00:00+00:00'
+site_id: SITE_A
+depth_m: 0.10
+is_timeseries: false

examples/invalid/HarmonizedRecord-positive-water-potential.yaml

@@ -0,0 +1,5 @@
+# INVALID: water potential must be non-positive (<= 0 kPa).
+datetime_UTC: '2021-06-15T12:00:00+00:00'
+site_id: SITE_C
+is_timeseries: false
+water_potential_kPa: 50.0

examples/invalid/HarmonizedRecord-vwc-above-one.yaml

@@ -0,0 +1,6 @@
+# INVALID: volumetric water content is a fraction and must be <= 1.0.
+datetime_UTC: '2021-06-15T12:00:00+00:00'
+site_id: SITE_A
+depth_m: 0.10
+is_timeseries: false
+volumetric_water_content_m3_m3: 1.8

examples/valid/HarmonizedDataset-mixed.yaml

@@ -0,0 +1,20 @@
+# A small dataset combining records that report different moisture variables.
+records:
+  - datetime_UTC: '2021-06-15T12:00:00+00:00'
+    site_id: SITE_A
+    depth_m: 0.10
+    replicate: 1.0
+    is_timeseries: true
+    interval_min: 60.0
+    volumetric_water_content_m3_m3: 0.32
+  - datetime_UTC: '2021-06-15T13:00:00+00:00'
+    site_id: SITE_A
+    depth_m: 0.10
+    replicate: 1.0
+    is_timeseries: true
+    interval_min: 60.0
+    volumetric_water_content_m3_m3: 0.31
+  - datetime_UTC: '2019-03-01T00:00:00+00:00'
+    site_id: SITE_B
+    is_timeseries: false
+    gravimetric_water_content_gH2O_gs: 0.18

examples/valid/HarmonizedRecord-gravimetric-minimal.yaml

@@ -0,0 +1,5 @@
+# Minimal record: only the required fields plus one moisture variable.
+datetime_UTC: '2019-03-01T00:00:00+00:00'
+site_id: SITE_B
+is_timeseries: false
+gravimetric_water_content_gH2O_gs: 0.18

examples/valid/HarmonizedRecord-volumetric-timeseries.yaml

@@ -0,0 +1,8 @@
+# A full time-series record reporting volumetric water content.
+datetime_UTC: '2021-06-15T12:00:00+00:00'
+site_id: SITE_A
+depth_m: 0.10
+replicate: 1.0
+is_timeseries: true
+interval_min: 60.0
+volumetric_water_content_m3_m3: 0.32

examples/valid/HarmonizedRecord-water-potential.yaml

@@ -0,0 +1,7 @@
+# A grab sample reporting (negative) soil water potential.
+datetime_UTC: '2020-09-21T18:30:00+00:00'
+site_id: SITE_C
+depth_m: 0.30
+replicate: 2.0
+is_timeseries: false
+water_potential_kPa: -1500.0