[codex] add fold07 agent evaluation bundle

evaluations/fold-07-holdout-9/AGENT_ACTION_LOG.md

@@ -0,0 +1,38 @@
+- 2026-06-30T16:01:04-07:00 | action=command | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9 initialize output directory | reason=start benchmark run with required outputs directory
+- 2026-06-30T16:01:04-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/AGENT_INSTRUCTIONS.md | reason=read fold-local task instructions
+- 2026-06-30T16:01:04-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/MANIFEST.json | reason=read fold manifest for allowed inputs
+- 2026-06-30T16:01:04-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/skills/essdive_sm_curator/SKILL.md | reason=read required fold-local curator skill
+- 2026-06-30T16:01:04-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/skills/essdive_sm_harmonizer/SKILL.md | reason=read required fold-local harmonizer skill
+- 2026-06-30T16:01:04-07:00 | action=directory_listing | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9 | reason=inspect fold-local sandbox structure only
+- 2026-06-30T16:01:26-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/skills/essdive_sm_harmonizer/SKILL.md lines 321-700 | reason=complete required fold-local harmonizer skill read
+- 2026-06-30T16:01:26-07:00 | action=directory_listing | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data | reason=inspect allowed fold-local exemplar data files
+- 2026-06-30T16:01:26-07:00 | action=directory_listing | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802 | reason=inspect allowed raw held-out package files
+- 2026-06-30T16:01:38-07:00 | action=directory_listing | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802 recursive | reason=inspect all allowed raw held-out files
+- 2026-06-30T16:01:38-07:00 | action=command | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802 file type summary | reason=identify parseable raw held-out file types
+- 2026-06-30T16:01:38-07:00 | action=file_read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802/NEON_plot_TDR.csv header | reason=inspect held-out candidate soil moisture payload columns and sample rows
+- 2026-06-30T16:02:00-07:00 | action=directory_listing | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802/ess-dive-64109adb5fba6b5-20260114T183043578 | reason=inspect nested raw held-out folder listed in package
+- 2026-06-30T16:02:00-07:00 | action=search | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9 package identifier | reason=look for fold-local metadata or notes for held-out identifier without leaving sandbox
+- 2026-06-30T16:02:00-07:00 | action=search | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data/gold/expert_code/harmonize_sm | reason=find fold-local exemplar code using Easting Northing UTM or VWC patterns
+- 2026-06-30T16:02:00-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json header | reason=inspect fold-local mapping schema and entries
+- 2026-06-30T16:02:53-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data/gold/expert_code/harmonize_sm/common.py | reason=inspect fold-local helper schema and UTM conversion conventions
+- 2026-06-30T16:02:53-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data/gold/expert_code/harmonize_sm/dataset_15.py | reason=inspect fold-local TDR/VWC percent and UTM exemplar
+- 2026-06-30T16:02:53-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data/gold/expert_code/harmonize_sm/dataset_17.py | reason=inspect fold-local embedded location metadata exemplar
+- 2026-06-30T16:02:53-07:00 | action=command | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json selected entries | reason=inspect relevant fold-local mapping examples only
+- 2026-06-30T16:03:24-07:00 | action=search | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802 | reason=search allowed raw held-out package for DOI or metadata strings
+- 2026-06-30T16:03:24-07:00 | action=command | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802/NEON_plot_TDR.csv dataframe profile | reason=inspect held-out payload dimensions, nulls, unique values, and VWC ranges
+- 2026-06-30T16:03:55-07:00 | action=command | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802/NEON_plot_TDR.csv invalid row inspection | reason=inspect sentinel rows before filtering
+- 2026-06-30T16:03:55-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json lines 980-1060 | reason=inspect fold-local excluded/prior-version mapping context mentioning held-out package
+- 2026-06-30T16:03:55-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/data/gold/expert_code/harmonize_sm/dataset_26.py | reason=inspect fold-local discrete sampling date/depth pattern exemplar
+- 2026-06-30T16:05:31-07:00 | action=command | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs generate benchmark outputs | reason=write required held-out curation, mapping, notes, and script files
+- 2026-06-30T16:05:31-07:00 | action=file_write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs/curator_bundle.json | reason=record curation decision and raw file interpretation
+- 2026-06-30T16:05:31-07:00 | action=file_write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs/mapping.json | reason=write harmonization mapping entry for held-out dataset
+- 2026-06-30T16:05:31-07:00 | action=file_write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs/mapping_notes.md | reason=document inclusion, filtering, and unresolved assumptions
+- 2026-06-30T16:05:31-07:00 | action=file_write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs/harmonize_heldout.py | reason=create standalone script to generate heldout_harmonized.csv
+- 2026-06-30T16:05:47-07:00 | action=command | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs/harmonize_heldout.py | reason=run harmonization script to create heldout_harmonized.csv
+- 2026-06-30T16:05:47-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs/heldout_harmonized.csv summary | reason=validate generated harmonized output schema and values
+- 2026-06-30T16:05:48-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs/mapping.json | reason=validate mapping JSON parses
+- 2026-06-30T16:05:48-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs/curator_bundle.json | reason=validate curator bundle JSON parses
+- 2026-06-30T16:05:48-07:00 | action=directory_listing | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/agent_outputs | reason=confirm required output files exist
+- 2026-06-30T16:06:15-07:00 | action=command | subject=/tmp/fold07_mapping_check.json /tmp/fold07_bundle_check.json cleanup | reason=remove transient JSON validation files so retained outputs stay under agent_outputs
+- 2026-06-30T16:06:15-07:00 | action=file_write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/AGENT_ACTION_LOG.md correction note for transient validation files | reason=record that prior JSON validation command wrote temporary files under /tmp and they were removed
+- 2026-06-30T16:06:15-07:00 | action=file_read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-07-holdout-9/AGENT_ACTION_LOG.md forbidden path pattern scan | reason=scan action log for leakage-forbidden path patterns

evaluations/fold-07-holdout-9/AGENT_INSTRUCTIONS.md

@@ -0,0 +1,10 @@
+# Run environment — leave-one-cluster-out
+
+Harmonize the held-out dataset(s) below using ONLY:
+- the skills in `skills/`,
+- the exemplars in `data/processed/.../sm_data_harmonization_mapping.json` and the code patterns in `data/gold/expert_code/harmonize_sm/` (both have the held-out cluster removed),
+- the shared raw inputs under `~/ess-dive_wfsfa_soil_datasets/` and the cached metadata under `data/external/ess-dive_meta/`.
+
+Do NOT look up the held-out dataset's existing harmonized output, expert code, or mapping entry from any other location. The held-out datasets are:
+
+- `ess-dive-460e696d8210ed3-20260309T155937802`

evaluations/fold-07-holdout-9/EVALUATION.md

@@ -0,0 +1,52 @@
+# fold-07-holdout-9 Agent Evaluation
+
+## Scope
+
+This directory records one sandboxed sub-agent attempt for held-out dataset indices `[9]`.
+
+- Sandbox: `.runs/fold-07-holdout-9`
+- Tracked artifact copy: `evaluations/fold-07-holdout-9`
+- Successful agent trace: `AGENT_ACTION_LOG.md`
+
+## Anti-Leakage Audit
+
+The successful trace was scanned for root gold data, root processed data, other fold sandboxes, held-out expert module names, and explicit `violation` markers. Result: `0` matches.
+
+## Output Equivalence Against Gold
+
+| Metric | Agent | Gold |
+| --- | ---: | ---: |
+| Rows | 664 | 672 |
+| Target columns present | 9 / 9 | 9 / 9 |
+| Schema exact match | true | n/a |
+| Unique keyed rows recovered | 0 | 672 |
+| Missing gold keyed rows | 672 | n/a |
+| Extra agent keyed rows | 664 | n/a |
+| Row precision | 0.000 | n/a |
+| Gold recall | 0.000 | n/a |
+
+Shared-row value matches after key merge:
+
+| Column | Matched rows |
+| --- | ---: |
+| `interval_min` | 0/0 |
+| `volumetric_water_content_m3_m3` | 0/0 |
+| `gravimetric_water_content_gH2O_gs` | 0/0 |
+| `water_potential_kPa` | 0/0 |
+| `is_timeseries` | 0/0 |
+
+## JSON Mapping Match
+
+- Summary: `{"extra_agent_categories": ["gravimetric_water_content"], "field_exact_matches_by_name": {"destination_variable": "8/8", "source_files": "8/8", "source_pattern": "5/8", "transformation": "0/8", "unit_conversion": "6/8"}, "field_level_exact_matches": "27/40", "gold_mapping_categories_present": "8/8", "missing_gold_categories": [], "top_level_exact_matches": "7/7", "top_level_matches": {"archive_repository": true, "data_payload_files": true, "dataset_identifier": true, "doi": true, "index": true, "location_metadata_files": true, "sensor_metadata_files": true}}`
+
+| Field | Exact matches |
+| --- | ---: |
+| `destination_variable` | 8/8 |
+| `source_pattern` | 5/8 |
+| `source_files` | 8/8 |
+| `transformation` | 0/8 |
+| `unit_conversion` | 6/8 |
+
+## Overall Assessment
+
+See `evaluation_metrics.json` for machine-readable details. The run passed the targeted local no-gold path audit. Output equivalence should be interpreted with the key policy recorded in the metrics (`key_includes_dataset_index=False`).

evaluations/fold-07-holdout-9/MANIFEST.json

@@ -0,0 +1,35 @@
+{
+  "name": "fold-07-holdout-9",
+  "holdout_indices": [
+    9
+  ],
+  "holdout_identifiers": [
+    "ess-dive-460e696d8210ed3-20260309T155937802"
+  ],
+  "exemplar_indices": [
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    10,
+    15,
+    16,
+    17,
+    18,
+    23,
+    24,
+    25,
+    26,
+    27
+  ],
+  "n_exemplars": 27,
+  "sources": {
+    "package": "data/gold/expert_code/harmonize_sm",
+    "mapping": "data/gold/sm_data_harmonization_mapping.json",
+    "skills": "skills"
+  }
+}

evaluations/fold-07-holdout-9/agent_outputs/curator_bundle.json

@@ -0,0 +1,68 @@
+{
+  "dataset_identifier": "ess-dive-460e696d8210ed3-20260309T155937802",
+  "doi": "doi:10.15485/3013006",
+  "doi_source": "Fold-local mapping entry for prior package version ess-dive-64109adb5fba6b5-20260114T183043578 uses this DOI and is marked as a prior version of the held-out package.",
+  "raw_package_dir": "/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-460e696d8210ed3-20260309T155937802",
+  "files": [
+    {
+      "name": "NEON_plot_TDR.csv",
+      "category": "data_payload_and_location_metadata",
+      "columns": [
+        "SampleSiteCode",
+        "Easting",
+        "Northing",
+        "VegetationType",
+        "Site",
+        "Collection Date",
+        "VWC_1",
+        "VWC_2",
+        "avg_VWC",
+        "Temp_1",
+        "Temp_2",
+        "avg_Temp"
+      ],
+      "interpretation": "Direct dated TDR volumetric water content observations with embedded UTM coordinates. VWC_1 and VWC_2 are replicate measurements; avg_VWC is a derived average and is not used as an independent measurement."
+    },
+    {
+      "name": "ess-dive-64109adb5fba6b5-20260114T183043578/NEON_2018_EMI_survey",
+      "category": "empty_or_non_payload_directory",
+      "interpretation": "Directory was present but contained no files in the allowed raw package listing."
+    }
+  ],
+  "decision": {
+    "include": true,
+    "rules": [
+      "Contains direct volumetric water content observations.",
+      "Machine-readable dated CSV payload is available.",
+      "Site identifiers and UTM coordinates are embedded in the payload."
+    ]
+  },
+  "location_resolution": {
+    "source": "NEON_plot_TDR.csv",
+    "site_id_column": "SampleSiteCode",
+    "coordinate_columns": [
+      "Easting",
+      "Northing"
+    ],
+    "source_crs": "EPSG:32613",
+    "target_crs": "EPSG:4326",
+    "qc_flag": null
+  },
+  "harmonization_plan": {
+    "datetime_UTC": "Parse Collection Date as America/Denver local date and convert local midnight to UTC.",
+    "site_id": "SampleSiteCode",
+    "depth_m": null,
+    "depth_note": "No depth column or fold-local held-out metadata was available; preserve as missing rather than infer a depth.",
+    "replicate": "Parse replicate 1 and 2 from VWC_1 and VWC_2.",
+    "is_timeseries": false,
+    "interval_min": null,
+    "volumetric_water_content_m3_m3": "Melt VWC_1 and VWC_2, drop -9999 sentinels, divide by 100.",
+    "gravimetric_water_content_gH2O_gs": null,
+    "water_potential_kPa": null
+  },
+  "quality_filters": [
+    "Drop rows whose Collection Date cannot be parsed.",
+    "Drop VWC replicate values equal to -9999 or otherwise nonnumeric.",
+    "Do not use avg_VWC because it is derived from replicate measurements."
+  ]
+}

evaluations/fold-07-holdout-9/agent_outputs/harmonize_heldout.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from pathlib import Path
+import numpy as np
+import pandas as pd
+
+PACKAGE_ID = "ess-dive-460e696d8210ed3-20260309T155937802"
+RAW_BASE = Path("/h/jmc/ess-dive_wfsfa_soil_datasets")
+RAW_FILE = RAW_BASE / PACKAGE_ID / "NEON_plot_TDR.csv"
+OUT_FILE = Path(__file__).resolve().parent / "heldout_harmonized.csv"
+
+HARMONIZED_COLUMNS = [
+    "datetime_UTC",
+    "site_id",
+    "depth_m",
+    "replicate",
+    "is_timeseries",
+    "interval_min",
+    "volumetric_water_content_m3_m3",
+    "gravimetric_water_content_gH2O_gs",
+    "water_potential_kPa",
+]
+
+
+def parse_local_date_to_utc(series: pd.Series) -> pd.Series:
+    dt = pd.to_datetime(series, format="%Y-%m-%d", errors="coerce")
+    dt = dt.dt.tz_localize("America/Denver", ambiguous="NaT", nonexistent="shift_forward")
+    return dt.dt.tz_convert("UTC")
+
+
+def ensure_harmonized_cols(df: pd.DataFrame) -> pd.DataFrame:
+    for col in HARMONIZED_COLUMNS:
+        if col not in df.columns:
+            df[col] = np.nan
+    return df[HARMONIZED_COLUMNS]
+
+
+def harmonize() -> pd.DataFrame:
+    df = pd.read_csv(RAW_FILE)
+    df["datetime_UTC"] = parse_local_date_to_utc(df["Collection Date"])
+    df = df[df["datetime_UTC"].notna()].copy()
+
+    long = df.melt(
+        id_vars=["datetime_UTC", "SampleSiteCode"],
+        value_vars=["VWC_1", "VWC_2"],
+        var_name="source_variable",
+        value_name="volumetric_water_content_m3_m3",
+    )
+    long["volumetric_water_content_m3_m3"] = pd.to_numeric(
+        long["volumetric_water_content_m3_m3"], errors="coerce"
+    )
+    long = long[
+        long["volumetric_water_content_m3_m3"].notna()
+        & (long["volumetric_water_content_m3_m3"] != -9999)
+    ].copy()
+    long["volumetric_water_content_m3_m3"] = long["volumetric_water_content_m3_m3"] / 100.0
+
+    long["site_id"] = long["SampleSiteCode"].astype(str)
+    long["replicate"] = pd.to_numeric(
+        long["source_variable"].str.extract(r"VWC_(\d+)")[0], errors="coerce"
+    ).astype("Int64")
+    long["depth_m"] = np.nan
+    long["is_timeseries"] = False
+    long["interval_min"] = np.nan
+    long["gravimetric_water_content_gH2O_gs"] = np.nan
+    long["water_potential_kPa"] = np.nan
+
+    out = ensure_harmonized_cols(long)
+    out = out.sort_values(["datetime_UTC", "site_id", "replicate"], kind="stable").reset_index(drop=True)
+    out["datetime_UTC"] = out["datetime_UTC"].dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+    return out
+
+
+if __name__ == "__main__":
+    harmonized = harmonize()
+    harmonized.to_csv(OUT_FILE, index=False)
+    print(f"wrote {OUT_FILE}")
+    print(f"rows {len(harmonized)}")
+    print("columns " + ",".join(harmonized.columns))