[codex] add fold12 agent evaluation bundle

evaluations/fold-12-holdout-24/AGENT_ACTION_LOG.md

@@ -0,0 +1,64 @@
+2026-06-30T16:05:54-07:00 | action=init | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-12-holdout-24/AGENT_ACTION_LOG.md | reason=initialize mandatory trace log before benchmark actions
+2026-06-30T16:05:54-07:00 | action=command | subject=mkdir agent_outputs | reason=initialize output directory
+2026-06-30T16:06:12-07:00 | action=command | subject=read required fold instructions and skills | reason=load benchmark workflow and fold-local skill guidance
+2026-06-30T16:06:12-07:00 | action=read | subject=.runs/fold-12-holdout-24/AGENT_INSTRUCTIONS.md | reason=required benchmark instructions
+2026-06-30T16:06:12-07:00 | action=read | subject=.runs/fold-12-holdout-24/MANIFEST.json | reason=required fold manifest
+2026-06-30T16:06:12-07:00 | action=read | subject=.runs/fold-12-holdout-24/skills/essdive_sm_curator/SKILL.md | reason=required fold-local curator skill
+2026-06-30T16:06:12-07:00 | action=read | subject=.runs/fold-12-holdout-24/skills/essdive_sm_harmonizer/SKILL.md | reason=required fold-local harmonizer skill
+2026-06-30T16:06:28-07:00 | action=command | subject=continue reading fold-local skills | reason=complete required skill reads after initial chunk
+2026-06-30T16:06:28-07:00 | action=read | subject=.runs/fold-12-holdout-24/skills/essdive_sm_curator/SKILL.md line count | reason=determine whether full skill was read
+2026-06-30T16:06:28-07:00 | action=read | subject=.runs/fold-12-holdout-24/skills/essdive_sm_curator/SKILL.md lines 261-520 | reason=complete curator skill read
+2026-06-30T16:06:28-07:00 | action=read | subject=.runs/fold-12-holdout-24/skills/essdive_sm_harmonizer/SKILL.md line count | reason=determine remaining harmonizer skill length
+2026-06-30T16:06:28-07:00 | action=read | subject=.runs/fold-12-holdout-24/skills/essdive_sm_harmonizer/SKILL.md lines 261-620 | reason=complete harmonizer skill read
+2026-06-30T16:06:37-07:00 | action=command | subject=read final curator skill lines | reason=complete required fold-local curator skill read
+2026-06-30T16:06:37-07:00 | action=read | subject=.runs/fold-12-holdout-24/skills/essdive_sm_curator/SKILL.md lines 521-620 | reason=finish full curator skill read
+2026-06-30T16:06:45-07:00 | action=command | subject=list fold sandbox root | reason=locate allowed fold-local data and outputs
+2026-06-30T16:06:45-07:00 | action=list | subject=.runs/fold-12-holdout-24 root | reason=inspect allowed sandbox structure
+2026-06-30T16:06:52-07:00 | action=command | subject=list fold-local exemplar files | reason=identify allowed mapping and code examples
+2026-06-30T16:06:52-07:00 | action=list | subject=.runs/fold-12-holdout-24/data | reason=inspect allowed fold-local exemplar data
+2026-06-30T16:07:01-07:00 | action=command | subject=locate raw heldout package | reason=find allowed raw files for dataset 24
+2026-06-30T16:07:01-07:00 | action=search | subject=/h/jmc/ess-dive_wfsfa_soil_datasets heldout package directory | reason=locate raw held-out input files
+2026-06-30T16:07:09-07:00 | action=command | subject=list raw heldout package files | reason=classify payload and metadata files
+2026-06-30T16:07:09-07:00 | action=list | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-daa156d2129c471-20250716T160748658 | reason=inspect allowed raw package contents
+2026-06-30T16:07:31-07:00 | action=command | subject=inspect heldout raw csv structure | reason=identify columns, row counts, sample values, and payload roles
+2026-06-30T16:07:31-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-daa156d2129c471-20250716T160748658/Iso_MP_Sap_DataDaily_ESSDiveUpload_R1.csv | reason=inspect candidate measurement payload
+2026-06-30T16:07:31-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-daa156d2129c471-20250716T160748658/Locations.csv | reason=inspect candidate location metadata
+2026-06-30T16:07:45-07:00 | action=command | subject=search fold-local examples for water potential and delimiter patterns | reason=find structurally similar exemplar code and mapping
+2026-06-30T16:07:45-07:00 | action=search | subject=.runs/fold-12-holdout-24/data/gold/expert_code/harmonize_sm and mapping | reason=locate allowed exemplar patterns for water potential/semicolon/wide data
+2026-06-30T16:08:00-07:00 | action=command | subject=read fold-local common and similar dataset code | reason=mirror allowed schema helpers and water-potential wide-format pattern
+2026-06-30T16:08:00-07:00 | action=read | subject=.runs/fold-12-holdout-24/data/gold/expert_code/harmonize_sm/common.py | reason=inspect allowed harmonized schema helper behavior
+2026-06-30T16:08:00-07:00 | action=read | subject=.runs/fold-12-holdout-24/data/gold/expert_code/harmonize_sm/dataset_03.py | reason=inspect allowed MP wide-format exemplar
+2026-06-30T16:08:00-07:00 | action=read | subject=.runs/fold-12-holdout-24/data/gold/expert_code/harmonize_sm/dataset_01.py | reason=inspect allowed depth-replicate wide-format exemplar
+2026-06-30T16:08:16-07:00 | action=command | subject=profile semicolon-parsed heldout data | reason=determine MP columns, units, non-null counts, dates, and location rows
+2026-06-30T16:08:16-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-daa156d2129c471-20250716T160748658/Iso_MP_Sap_DataDaily_ESSDiveUpload_R1.csv | reason=profile semicolon-delimited measurement payload
+2026-06-30T16:08:16-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-daa156d2129c471-20250716T160748658/Locations.csv | reason=profile semicolon-delimited location metadata
+2026-06-30T16:08:41-07:00 | action=command | subject=read selected fold-local mapping entries | reason=align mapping.json with allowed exemplar schema and wording
+2026-06-30T16:08:41-07:00 | action=read | subject=.runs/fold-12-holdout-24/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json selected entries | reason=inspect allowed comparable mapping entries only
+2026-06-30T16:08:51-07:00 | action=command | subject=fetch ESS-DIVE heldout package metadata | reason=resolve DOI without reading prohibited local gold/processed outputs
+2026-06-30T16:08:51-07:00 | action=search | subject=https://api.ess-dive.lbl.gov/packages/ess-dive-daa156d2129c471-20250716T160748658 | reason=retrieve allowed package metadata DOI from ESS-DIVE API
+2026-06-30T16:09:03-07:00 | action=command | subject=fetch ESS-DIVE heldout package metadata with network access | reason=resolve DOI after sandbox DNS failure
+2026-06-30T16:09:03-07:00 | action=search | subject=https://api.ess-dive.lbl.gov/packages/ess-dive-daa156d2129c471-20250716T160748658 | reason=retrieve package metadata DOI from ESS-DIVE API after sandbox DNS failure
+2026-06-30T16:10:54-07:00 | action=command | subject=write heldout curator bundle mapping notes and harmonization script | reason=generate required benchmark outputs in agent_outputs
+2026-06-30T16:10:54-07:00 | action=write | subject=.runs/fold-12-holdout-24/agent_outputs/curator_bundle.json | reason=record curation decisions and inspected raw file structure
+2026-06-30T16:10:54-07:00 | action=write | subject=.runs/fold-12-holdout-24/agent_outputs/mapping.json | reason=record dataset 24 harmonization mapping entry
+2026-06-30T16:10:54-07:00 | action=write | subject=.runs/fold-12-holdout-24/agent_outputs/mapping_notes.md | reason=document harmonization reasoning and open questions
+2026-06-30T16:10:54-07:00 | action=write | subject=.runs/fold-12-holdout-24/agent_outputs/harmonize_heldout.py | reason=create executable harmonization script for heldout dataset
+2026-06-30T16:11:03-07:00 | action=command | subject=run harmonize_heldout.py | reason=generate required heldout_harmonized.csv output
+2026-06-30T16:11:16-07:00 | action=command | subject=fix harmonize_heldout.py action log timestamp helper | reason=pandas does not accept tz=local in this environment
+2026-06-30T16:11:16-07:00 | action=read | subject=.runs/fold-12-holdout-24/agent_outputs/harmonize_heldout.py | reason=load script text for timestamp helper fix
+2026-06-30T16:11:16-07:00 | action=write | subject=.runs/fold-12-holdout-24/agent_outputs/harmonize_heldout.py | reason=replace unsupported pandas local timezone timestamp
+2026-06-30T16:11:23-07:00 | action=command | subject=rerun harmonize_heldout.py | reason=generate heldout CSV after timestamp helper fix
+2026-06-30T16:11:24.416925-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-daa156d2129c471-20250716T160748658/Iso_MP_Sap_DataDaily_ESSDiveUpload_R1.csv | reason=load heldout semicolon-delimited measurement payload
+2026-06-30T16:11:24.445118-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-daa156d2129c471-20250716T160748658/Locations.csv | reason=load heldout location metadata for selected site ids
+2026-06-30T16:11:24.476853-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-12-holdout-24/agent_outputs/heldout_harmonized.csv | reason=write heldout harmonized CSV with target schema
+2026-06-30T16:11:40-07:00 | action=command | subject=validate required agent output files | reason=confirm files, row count, schema, and script syntax
+2026-06-30T16:11:40-07:00 | action=list | subject=.runs/fold-12-holdout-24/agent_outputs | reason=confirm required output files exist only in output directory
+2026-06-30T16:11:40-07:00 | action=read | subject=.runs/fold-12-holdout-24/agent_outputs/heldout_harmonized.csv | reason=validate row count and schema
+2026-06-30T16:11:40-07:00 | action=read | subject=.runs/fold-12-holdout-24/agent_outputs/harmonize_heldout.py | reason=compile-check generated harmonization script
+2026-06-30T16:12:05-07:00 | action=command | subject=update interval documentation in outputs | reason=align notes and curator bundle with validated DST-aware interval values
+2026-06-30T16:12:05-07:00 | action=read | subject=.runs/fold-12-holdout-24/agent_outputs/curator_bundle.json | reason=load curator bundle for interval documentation correction
+2026-06-30T16:12:05-07:00 | action=read | subject=.runs/fold-12-holdout-24/agent_outputs/mapping_notes.md | reason=load notes for interval documentation correction
+2026-06-30T16:12:05-07:00 | action=write | subject=.runs/fold-12-holdout-24/agent_outputs/curator_bundle.json | reason=correct interval documentation after validation
+2026-06-30T16:12:06-07:00 | action=write | subject=.runs/fold-12-holdout-24/agent_outputs/mapping_notes.md | reason=correct interval documentation after validation
+2026-06-30T16:12:26-07:00 | action=command | subject=scan action log for leakage patterns | reason=mandatory final anti-leakage validation
+2026-06-30T16:12:26-07:00 | action=read | subject=.runs/fold-12-holdout-24/AGENT_ACTION_LOG.md | reason=scan own action log for forbidden access patterns

evaluations/fold-12-holdout-24/AGENT_INSTRUCTIONS.md

@@ -0,0 +1,10 @@
+# Run environment — leave-one-cluster-out
+
+Harmonize the held-out dataset(s) below using ONLY:
+- the skills in `skills/`,
+- the exemplars in `data/processed/.../sm_data_harmonization_mapping.json` and the code patterns in `data/gold/expert_code/harmonize_sm/` (both have the held-out cluster removed),
+- the shared raw inputs under `~/ess-dive_wfsfa_soil_datasets/` and the cached metadata under `data/external/ess-dive_meta/`.
+
+Do NOT look up the held-out dataset's existing harmonized output, expert code, or mapping entry from any other location. The held-out datasets are:
+
+- `ess-dive-daa156d2129c471-20250716T160748658`

evaluations/fold-12-holdout-24/EVALUATION.md

@@ -0,0 +1,52 @@
+# fold-12-holdout-24 Agent Evaluation
+
+## Scope
+
+This directory records one sandboxed sub-agent attempt for held-out dataset indices `[24]`.
+
+- Sandbox: `.runs/fold-12-holdout-24`
+- Tracked artifact copy: `evaluations/fold-12-holdout-24`
+- Successful agent trace: `AGENT_ACTION_LOG.md`
+
+## Anti-Leakage Audit
+
+The successful trace was scanned for root gold data, root processed data, other fold sandboxes, held-out expert module names, and explicit `violation` markers. Result: `0` matches.
+
+## Output Equivalence Against Gold
+
+| Metric | Agent | Gold |
+| --- | ---: | ---: |
+| Rows | 5,736 | 3,204 |
+| Target columns present | 9 / 9 | 9 / 9 |
+| Schema exact match | true | n/a |
+| Unique keyed rows recovered | 3,204 | 3,204 |
+| Missing gold keyed rows | 0 | n/a |
+| Extra agent keyed rows | 2,532 | n/a |
+| Row precision | 0.559 | n/a |
+| Gold recall | 1.000 | n/a |
+
+Shared-row value matches after key merge:
+
+| Column | Matched rows |
+| --- | ---: |
+| `interval_min` | 3204/3204 |
+| `volumetric_water_content_m3_m3` | 3204/3204 |
+| `gravimetric_water_content_gH2O_gs` | 3204/3204 |
+| `water_potential_kPa` | 3204/3204 |
+| `is_timeseries` | 3204/3204 |
+
+## JSON Mapping Match
+
+- Summary: `{"extra_agent_categories": ["gravimetric_water_content"], "field_exact_matches_by_name": {"destination_variable": "8/8", "source_files": "8/8", "source_pattern": "5/8", "transformation": "0/8", "unit_conversion": "6/8"}, "field_level_exact_matches": "27/40", "gold_mapping_categories_present": "8/8", "missing_gold_categories": [], "top_level_exact_matches": "7/7", "top_level_matches": {"archive_repository": true, "data_payload_files": true, "dataset_identifier": true, "doi": true, "index": true, "location_metadata_files": true, "sensor_metadata_files": true}}`
+
+| Field | Exact matches |
+| --- | ---: |
+| `destination_variable` | 8/8 |
+| `source_pattern` | 5/8 |
+| `source_files` | 8/8 |
+| `transformation` | 0/8 |
+| `unit_conversion` | 6/8 |
+
+## Overall Assessment
+
+See `evaluation_metrics.json` for machine-readable details. The run passed the targeted local no-gold path audit. Output equivalence should be interpreted with the key policy recorded in the metrics (`key_includes_dataset_index=False`).

evaluations/fold-12-holdout-24/MANIFEST.json

@@ -0,0 +1,35 @@
+{
+  "name": "fold-12-holdout-24",
+  "holdout_indices": [
+    24
+  ],
+  "holdout_identifiers": [
+    "ess-dive-daa156d2129c471-20250716T160748658"
+  ],
+  "exemplar_indices": [
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    15,
+    16,
+    17,
+    18,
+    23,
+    25,
+    26,
+    27
+  ],
+  "n_exemplars": 27,
+  "sources": {
+    "package": "data/gold/expert_code/harmonize_sm",
+    "mapping": "data/gold/sm_data_harmonization_mapping.json",
+    "skills": "skills"
+  }
+}

evaluations/fold-12-holdout-24/agent_outputs/curator_bundle.json

@@ -0,0 +1,125 @@
+{
+  "package_id": "ess-dive-daa156d2129c471-20250716T160748658",
+  "doi": "doi:10.15485/2462766",
+  "curator_decision": "INCLUDE",
+  "exclusion_reason": null,
+  "data_payload_files": [
+    {
+      "filename": "Iso_MP_Sap_DataDaily_ESSDiveUpload_R1.csv",
+      "columns": [
+        "Timestamp",
+        "P1_MP_15",
+        "P1_MP_30",
+        "P1_MP_60",
+        "P2_MP_15",
+        "P2_MP_30",
+        "P2_MP_60",
+        "d2H_*",
+        "d18O_*",
+        "meteorology fields",
+        "sap flow fields",
+        "root water uptake depth fields"
+      ],
+      "row_count_estimate": 956,
+      "file_size_mb": 0.621,
+      "delimiter": "semicolon",
+      "column_preview": "Timestamp;P1_MP_15;P1_MP_30;P1_MP_60;P2_MP_15;P2_MP_30;P2_MP_60;...",
+      "measurement_columns_selected": [
+        "P1_MP_15",
+        "P1_MP_30",
+        "P1_MP_60",
+        "P2_MP_15",
+        "P2_MP_30",
+        "P2_MP_60"
+      ],
+      "measurement_summary": {
+        "P1_MP_15": {
+          "non_null": 534,
+          "min": -2022.0,
+          "max": -0.2
+        },
+        "P1_MP_30": {
+          "non_null": 534,
+          "min": -1376.0,
+          "max": -0.1
+        },
+        "P1_MP_60": {
+          "non_null": 534,
+          "min": -1013.5,
+          "max": 0.0
+        },
+        "P2_MP_15": {
+          "non_null": 534,
+          "min": -3638.0,
+          "max": -0.1
+        },
+        "P2_MP_30": {
+          "non_null": 534,
+          "min": -1618.5,
+          "max": 0.0
+        },
+        "P2_MP_60": {
+          "non_null": 534,
+          "min": -1371.0,
+          "max": 0.0
+        }
+      }
+    }
+  ],
+  "location_metadata_files": [
+    {
+      "filename": "Locations.csv",
+      "columns": [
+        "ID",
+        "Latitude",
+        "Longitude",
+        "Altitude",
+        "esrignss_f",
+        "DataCollec",
+        "GlobalID",
+        "Creator",
+        "Editor",
+        "Research_P",
+        "Notes"
+      ],
+      "row_count": 8,
+      "content_preview": "ID;Latitude;Longitude;Altitude;...\nP1;38.96071917;-106.9910977;2895.414399;...\nP2;38.96067806;-106.9910133;2897.958633;..."
+    }
+  ],
+  "sensor_metadata_files": null,
+  "readme_content": null,
+  "location_resolution": {
+    "source": "location_metadata_file",
+    "qc_flag_recommendation": null,
+    "site_coordinates": [
+      {
+        "site_id": "P1",
+        "latitude": 38.96071917,
+        "longitude": -106.9910977
+      },
+      {
+        "site_id": "P2",
+        "latitude": 38.96067806,
+        "longitude": -106.9910133
+      }
+    ]
+  },
+  "time_series_inference": {
+    "is_timeseries": true,
+    "interval_min": 1440.0,
+    "reasoning": "Daily date sequence from 2022-01-01 through 2024-08-13 with repeated observations by soil profile and depth; median interval is 1440 minutes, with 1380/1500 minute UTC differences at daylight-saving transitions after local-midnight conversion."
+  },
+  "experimental_context": {
+    "manipulation_detected": false,
+    "manipulation_type": null,
+    "has_control_data": null,
+    "recommendation": "include_all"
+  },
+  "similar_dataset_reference": {
+    "index": 3,
+    "reason": "Wide-format soil matric potential columns with depth encoded in column names and separate latitude/longitude location metadata."
+  },
+  "open_questions": [
+    "No source data dictionary file was present in the local raw package; MP units are inferred as kPa from column naming, value ranges, and fold-local MP exemplars."
+  ]
+}