[codex] add fold13 agent evaluation bundle

evaluations/fold-13-holdout-25/AGENT_ACTION_LOG.md

@@ -0,0 +1,52 @@
+- 2026-06-30T16:08:13-07:00 | action=command+write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/AGENT_ACTION_LOG.md and agent_outputs | reason=initialize mandatory trace log and output directory
+- 2026-06-30T16:08:29-07:00 | action=command | subject=read fold-local instructions and skills | reason=load required benchmark guidance before task actions
+- 2026-06-30T16:08:29-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/AGENT_INSTRUCTIONS.md | reason=required benchmark instructions
+- 2026-06-30T16:08:29-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/MANIFEST.json | reason=identify held-out dataset and allowed fold-local inputs
+- 2026-06-30T16:08:29-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/skills/essdive_sm_curator/SKILL.md | reason=required fold-local curator workflow
+- 2026-06-30T16:08:29-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/skills/essdive_sm_harmonizer/SKILL.md | reason=required fold-local harmonizer workflow
+- 2026-06-30T16:08:40-07:00 | action=command | subject=continue reading fold-local skill files | reason=ensure required skill instructions are read completely
+- 2026-06-30T16:08:40-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/skills/essdive_sm_curator/SKILL.md lines 221-520 | reason=complete fold-local curator workflow
+- 2026-06-30T16:08:40-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/skills/essdive_sm_harmonizer/SKILL.md lines 301-700 | reason=complete fold-local harmonizer workflow
+- 2026-06-30T16:08:48-07:00 | action=command | subject=read remaining curator skill tail | reason=complete required fold-local skill read
+- 2026-06-30T16:08:48-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/skills/essdive_sm_curator/SKILL.md lines 521-900 | reason=complete fold-local curator workflow
+- 2026-06-30T16:09:00-07:00 | action=command | subject=list allowed fold-local inputs and held-out raw directory | reason=locate exemplars, metadata cache, and package files without leaving allowed paths
+- 2026-06-30T16:09:00-07:00 | action=list | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/data | reason=locate fold-local exemplar and metadata paths
+- 2026-06-30T16:09:00-07:00 | action=list | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767 | reason=inspect held-out raw package files
+- 2026-06-30T16:09:38-07:00 | action=command | subject=read fold-local mapping and exemplar code | reason=learn schema and local code patterns from allowed examples
+- 2026-06-30T16:09:38-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json | reason=reference allowed mapping examples and schema
+- 2026-06-30T16:09:38-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/data/gold/expert_code/harmonize_sm/common.py | reason=reference allowed harmonizer helper functions
+- 2026-06-30T16:09:38-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/data/gold/expert_code/harmonize_sm/dataset_26.py | reason=reference similar fold-local time-series code pattern
+- 2026-06-30T16:09:38-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/data/gold/expert_code/harmonize_sm/dataset_27.py | reason=reference similar fold-local time-series code pattern
+- 2026-06-30T16:09:53-07:00 | action=command | subject=inspect held-out raw CSVs and fold-local metadata cache | reason=classify files and infer columns, rows, time coverage, and DOI
+- 2026-06-30T16:09:53-07:00 | action=list | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/data/external | reason=find cached ESS-DIVE metadata inside fold sandbox if present
+- 2026-06-30T16:09:53-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_aspen.csv | reason=inspect held-out raw data columns and sample values
+- 2026-06-30T16:09:53-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_conifer.csv | reason=inspect held-out raw data columns and sample values
+- 2026-06-30T16:10:13-07:00 | action=command | subject=inspect raw CSV first lines and line counts | reason=detect header presence and exact row counts
+- 2026-06-30T16:10:13-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_aspen.csv first lines and line count | reason=confirm headerless/aspen row structure
+- 2026-06-30T16:10:13-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_conifer.csv first lines and line count | reason=confirm conifer header and row structure
+- 2026-06-30T16:10:37-07:00 | action=command | subject=profile held-out raw VWC and timestamps | reason=validate units, missing values, intervals, and output row count expectation
+- 2026-06-30T16:10:37-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_aspen.csv full CSV profile | reason=validate aspen VWC and timestamp data before harmonization
+- 2026-06-30T16:10:37-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_conifer.csv full CSV profile | reason=validate conifer VWC and timestamp data before harmonization
+- 2026-06-30T16:10:52-07:00 | action=command | subject=ESS-DIVE API lookup for ess-dive-e67ab1151ebc525-20230929T190307767 | reason=retrieve DOI and package metadata absent from fold-local cache
+- 2026-06-30T16:11:02-07:00 | action=command | subject=ESS-DIVE API lookup for ess-dive-e67ab1151ebc525-20230929T190307767 with network approval | reason=retrieve DOI and package metadata after sandbox DNS failure
+- 2026-06-30T16:11:29-07:00 | action=command | subject=inspect invalid timestamp rows in conifer CSV | reason=decide whether to drop unparsable records during harmonization
+- 2026-06-30T16:11:29-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_conifer.csv invalid timestamp rows | reason=identify malformed year/month/day/hour records
+- 2026-06-30T16:11:52-07:00 | action=command | subject=simulate held-out harmonization row counts | reason=verify filtering rules and expected output dimensions before writing files
+- 2026-06-30T16:11:52-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_aspen.csv full CSV simulation | reason=estimate harmonized rows after valid timestamp and VWC filtering
+- 2026-06-30T16:11:52-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_conifer.csv full CSV simulation | reason=estimate harmonized rows after valid timestamp and VWC filtering
+- 2026-06-30T16:13:29-07:00 | action=command | subject=write generated benchmark outputs except harmonized CSV | reason=create curator bundle, mapping, notes, and executable harmonization script
+- 2026-06-30T16:13:29-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/curator_bundle.json | reason=record curation decision and inspected source metadata
+- 2026-06-30T16:13:29-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/mapping.json | reason=record held-out dataset harmonization mapping entry
+- 2026-06-30T16:13:29-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/mapping_notes.md | reason=document harmonization assumptions and quality decisions
+- 2026-06-30T16:13:29-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/harmonize_heldout.py | reason=generate reproducible held-out harmonization script
+- 2026-06-30T16:13:40-07:00 | action=command | subject=run harmonize_heldout.py | reason=generate heldout_harmonized.csv from raw held-out files
+- 2026-06-30T16:13:40-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/harmonize_heldout.py | reason=execute generated harmonization script
+- 2026-06-30T16:13:40-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_aspen.csv | reason=script input for held-out harmonized CSV
+- 2026-06-30T16:13:40-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-e67ab1151ebc525-20230929T190307767/Carbone_conifer.csv | reason=script input for held-out harmonized CSV
+- 2026-06-30T16:13:40-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/heldout_harmonized.csv | reason=write final held-out harmonized output
+- 2026-06-30T16:14:09-07:00 | action=command | subject=validate outputs and scan action log | reason=confirm files, schema, row count, JSON validity, and anti-leakage compliance
+- 2026-06-30T16:14:09-07:00 | action=list | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs | reason=confirm required output files only
+- 2026-06-30T16:14:09-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/heldout_harmonized.csv | reason=validate final row count and schema
+- 2026-06-30T16:14:09-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/mapping.json | reason=validate JSON mapping output
+- 2026-06-30T16:14:09-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/agent_outputs/curator_bundle.json | reason=validate JSON curator output
+- 2026-06-30T16:14:09-07:00 | action=read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-13-holdout-25/AGENT_ACTION_LOG.md | reason=scan mandatory trace for forbidden path patterns

evaluations/fold-13-holdout-25/AGENT_INSTRUCTIONS.md

@@ -0,0 +1,10 @@
+# Run environment — leave-one-cluster-out
+
+Harmonize the held-out dataset(s) below using ONLY:
+- the skills in `skills/`,
+- the exemplars in `data/processed/.../sm_data_harmonization_mapping.json` and the code patterns in `data/gold/expert_code/harmonize_sm/` (both have the held-out cluster removed),
+- the shared raw inputs under `~/ess-dive_wfsfa_soil_datasets/` and the cached metadata under `data/external/ess-dive_meta/`.
+
+Do NOT look up the held-out dataset's existing harmonized output, expert code, or mapping entry from any other location. The held-out datasets are:
+
+- `ess-dive-e67ab1151ebc525-20230929T190307767`

evaluations/fold-13-holdout-25/EVALUATION.md

@@ -0,0 +1,52 @@
+# fold-13-holdout-25 Agent Evaluation
+
+## Scope
+
+This directory records one sandboxed sub-agent attempt for held-out dataset indices `[25]`.
+
+- Sandbox: `.runs/fold-13-holdout-25`
+- Tracked artifact copy: `evaluations/fold-13-holdout-25`
+- Successful agent trace: `AGENT_ACTION_LOG.md`
+
+## Anti-Leakage Audit
+
+The successful trace was scanned for root gold data, root processed data, other fold sandboxes, held-out expert module names, and explicit `violation` markers. Result: `0` matches.
+
+## Output Equivalence Against Gold
+
+| Metric | Agent | Gold |
+| --- | ---: | ---: |
+| Rows | 471,671 | 471,893 |
+| Target columns present | 9 / 9 | 9 / 9 |
+| Schema exact match | true | n/a |
+| Unique keyed rows recovered | 0 | 467,529 |
+| Missing gold keyed rows | 467,529 | n/a |
+| Extra agent keyed rows | 467,624 | n/a |
+| Row precision | 0.000 | n/a |
+| Gold recall | 0.000 | n/a |
+
+Shared-row value matches after key merge:
+
+| Column | Matched rows |
+| --- | ---: |
+| `interval_min` | 0/0 |
+| `volumetric_water_content_m3_m3` | 0/0 |
+| `gravimetric_water_content_gH2O_gs` | 0/0 |
+| `water_potential_kPa` | 0/0 |
+| `is_timeseries` | 0/0 |
+
+## JSON Mapping Match
+
+- Summary: `{"extra_agent_categories": ["gravimetric_water_content"], "field_exact_matches_by_name": {"destination_variable": "8/8", "source_files": "8/8", "source_pattern": "3/8", "transformation": "0/8", "unit_conversion": "6/8"}, "field_level_exact_matches": "25/40", "gold_mapping_categories_present": "8/8", "missing_gold_categories": [], "top_level_exact_matches": "7/7", "top_level_matches": {"archive_repository": true, "data_payload_files": true, "dataset_identifier": true, "doi": true, "index": true, "location_metadata_files": true, "sensor_metadata_files": true}}`
+
+| Field | Exact matches |
+| --- | ---: |
+| `destination_variable` | 8/8 |
+| `source_pattern` | 3/8 |
+| `source_files` | 8/8 |
+| `transformation` | 0/8 |
+| `unit_conversion` | 6/8 |
+
+## Overall Assessment
+
+See `evaluation_metrics.json` for machine-readable details. The run passed the targeted local no-gold path audit. Output equivalence should be interpreted with the key policy recorded in the metrics (`key_includes_dataset_index=False`).

evaluations/fold-13-holdout-25/MANIFEST.json

@@ -0,0 +1,35 @@
+{
+  "name": "fold-13-holdout-25",
+  "holdout_indices": [
+    25
+  ],
+  "holdout_identifiers": [
+    "ess-dive-e67ab1151ebc525-20230929T190307767"
+  ],
+  "exemplar_indices": [
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    15,
+    16,
+    17,
+    18,
+    23,
+    24,
+    26,
+    27
+  ],
+  "n_exemplars": 27,
+  "sources": {
+    "package": "data/gold/expert_code/harmonize_sm",
+    "mapping": "data/gold/sm_data_harmonization_mapping.json",
+    "skills": "skills"
+  }
+}

evaluations/fold-13-holdout-25/agent_outputs/curator_bundle.json

@@ -0,0 +1,114 @@
+{
+  "package_id": "ess-dive-e67ab1151ebc525-20230929T190307767",
+  "doi": "doi:10.6084/M9.FIGSHARE.7834406.V2",
+  "curator_decision": "INCLUDE",
+  "exclusion_reason": null,
+  "title": "Raw soil carbon dioxide, moisture, and temperature data from a mixed conifer and aspen forest stands in the East River Watershed of Colorado June 2011-December 2021.",
+  "data_payload_files": [
+    {
+      "filename": "Carbone_aspen.csv",
+      "columns": [
+        "Year",
+        "Month",
+        "Day",
+        "Hour",
+        "CO2-50cm",
+        "CO2-15cm",
+        "CO2-5cm",
+        "CO2-0cm",
+        "CO2-above",
+        "T-50cm",
+        "T-15cm",
+        "T-5cm",
+        "T-0cm",
+        "T-above",
+        "VWC-50cm",
+        "VWC-15cm",
+        "VWC-5cm",
+        "DecagonT-50cm",
+        "DecagonT-15cm",
+        "DecagonT-5cm",
+        "EC-50cm",
+        "EC-15cm",
+        "EC-5cm",
+        "BatteryVoltage"
+      ],
+      "row_count_estimate": 79025,
+      "file_notes": "Headerless CSV; columns inferred from the conifer header and package metadata headers file description. Contains hourly records for the deciduous/aspen forest stand.",
+      "column_preview": "2011,8,20,0,8351.7,2117,1231.6,1085,663.61,10.67,11.82,11.72,11.48,10.37,0.106,0.093,0.072,10.9,12.1,12.5,0,0.01,0,12.8"
+    },
+    {
+      "filename": "Carbone_conifer.csv",
+      "columns": [
+        "Year",
+        "Month",
+        "Day",
+        "Hour",
+        "CO2-50cm",
+        "CO2-15cm",
+        "CO2-5cm",
+        "CO2-0cm",
+        "CO2-above",
+        "T-50cm",
+        "T-15cm",
+        "T-5cm",
+        "T-0cm",
+        "T-above",
+        "VWC-50cm",
+        "VWC-15cm",
+        "VWC-5cm",
+        "DecagonT-50cm",
+        "DecagonT-15cm",
+        "DecagonT-5cm",
+        "EC-50cm",
+        "EC-15cm",
+        "EC-5cm",
+        "BatteryVoltage"
+      ],
+      "row_count_estimate": 79024,
+      "file_notes": "Headered CSV; contains hourly records for the mixed conifer forest stand.",
+      "column_preview": "Year,Month,Day,Hour,CO2-50cm,CO2-15cm,CO2-5cm,CO2-0cm,CO2-above,T-50cm,T-15cm,T-5cm,T-0cm,T-above,VWC-50cm,VWC-15cm,VWC-5cm,DecagonT-50cm,DecagonT-15cm,DecagonT-5cm,EC-50cm,EC-15cm,EC-5cm,BatteryVoltage"
+    }
+  ],
+  "location_metadata_files": null,
+  "sensor_metadata_files": null,
+  "readme_content": null,
+  "location_resolution": {
+    "source": "package_metadata",
+    "qc_flag_recommendation": null,
+    "site_coordinates": [
+      {
+        "site_id": "Carbone_aspen",
+        "latitude": 38.9592,
+        "longitude": -106.9898
+      },
+      {
+        "site_id": "Carbone_conifer",
+        "latitude": 38.9592,
+        "longitude": -106.9898
+      }
+    ],
+    "notes": "ESS-DIVE spatialCoverage reports the two soil pits are within 15 meters of each other at 38.9592 N, -106.9898 W, elevation 2880 m."
+  },
+  "time_series_inference": {
+    "is_timeseries": true,
+    "interval_min": 60.0,
+    "reasoning": "Package metadata states soil temperature, moisture, and CO2 were recorded each hour; sorted timestamp differences are predominantly 60 minutes."
+  },
+  "experimental_context": {
+    "manipulation_detected": false,
+    "manipulation_type": null,
+    "has_control_data": null,
+    "recommendation": "include_all",
+    "notes": "Dataset compares adjacent natural mixed-conifer and deciduous/aspen stands; no warming, irrigation, drought, fertilization, or other manipulation was reported."
+  },
+  "similar_dataset_reference": {
+    "index": 27,
+    "reason": "Wide hourly sensor time series with VWC depth encoded in column names and site_id encoded by file/source."
+  },
+  "open_questions": [
+    "Source files do not include timezone. Harmonization assumes fixed Mountain Standard Time (UTC-7) for logger timestamps.",
+    "Carbone_aspen.csv is headerless in the raw package copy; columns are inferred from Carbone_conifer.csv and the API metadata description of a shared headers file.",
+    "Non-leap-year February 29 rows in Carbone_conifer.csv are dropped because they cannot be converted to valid timestamps."
+  ]
+}