[codex] add fold10 agent evaluation bundle

evaluations/fold-10-holdout-18/AGENT_ACTION_LOG.md

@@ -0,0 +1,48 @@
+2026-06-30T16:04:12-07:00 | action=command | subject=.runs/fold-10-holdout-18 initialize log/output dir | reason=start benchmark trace and output workspace
+2026-06-30T16:04:12-07:00 | action=write | subject=.runs/fold-10-holdout-18/agent_outputs | reason=create required output directory
+2026-06-30T16:04:30-07:00 | action=command | subject=read fold-local instructions and skills | reason=load allowed benchmark instructions
+2026-06-30T16:04:30-07:00 | action=read | subject=.runs/fold-10-holdout-18/AGENT_INSTRUCTIONS.md | reason=mandatory benchmark instructions
+2026-06-30T16:04:30-07:00 | action=read | subject=.runs/fold-10-holdout-18/MANIFEST.json | reason=identify held-out raw files and allowed exemplars
+2026-06-30T16:04:30-07:00 | action=read | subject=.runs/fold-10-holdout-18/skills/essdive_sm_curator/SKILL.md | reason=fold-local curator workflow
+2026-06-30T16:04:30-07:00 | action=read | subject=.runs/fold-10-holdout-18/skills/essdive_sm_harmonizer/SKILL.md | reason=fold-local harmonizer workflow
+2026-06-30T16:04:39-07:00 | action=command | subject=continue reading fold-local harmonizer skill | reason=complete required skill instructions
+2026-06-30T16:04:39-07:00 | action=read | subject=.runs/fold-10-holdout-18/skills/essdive_sm_harmonizer/SKILL.md lines 321-700 | reason=complete fold-local harmonizer instructions
+2026-06-30T16:04:55-07:00 | action=command | subject=list fold sandbox structure | reason=find allowed local data and output locations
+2026-06-30T16:04:55-07:00 | action=list | subject=.runs/fold-10-holdout-18 | reason=inspect allowed sandbox structure only
+2026-06-30T16:04:55-07:00 | action=list | subject=.runs/fold-10-holdout-18/data | reason=locate fold-local exemplar and metadata directories
+2026-06-30T16:04:55-07:00 | action=list | subject=.runs/fold-10-holdout-18/data/gold | reason=locate fold-local exemplar mapping and expert code
+2026-06-30T16:04:55-07:00 | action=list | subject=.runs/fold-10-holdout-18/data/external | reason=locate fold-local cached ESS-DIVE metadata
+2026-06-30T16:05:08-07:00 | action=command | subject=list fold-local data files | reason=identify allowed exemplar files inside sandbox
+2026-06-30T16:05:08-07:00 | action=search | subject=.runs/fold-10-holdout-18/data | reason=list fold-local allowed data files without leaving sandbox
+2026-06-30T16:05:08-07:00 | action=command | subject=list permitted raw held-out package directory | reason=inspect available raw files for held-out dataset
+2026-06-30T16:05:08-07:00 | action=list | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-b924878d23c9dd7-20250214T163427929 | reason=identify raw held-out files
+2026-06-30T16:05:25-07:00 | action=command | subject=inspect fold-local exemplar mapping and helper code | reason=learn output schema and code conventions from allowed examples
+2026-06-30T16:05:25-07:00 | action=read | subject=.runs/fold-10-holdout-18/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json | reason=allowed exemplar mapping schema with held-out removed
+2026-06-30T16:05:25-07:00 | action=read | subject=.runs/fold-10-holdout-18/data/gold/expert_code/harmonize_sm/common.py | reason=allowed harmonizer helper conventions
+2026-06-30T16:05:48-07:00 | action=command | subject=inspect mapping list and raw held-out CSV | reason=derive allowed exemplar schema and held-out column semantics
+2026-06-30T16:05:48-07:00 | action=read | subject=.runs/fold-10-holdout-18/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json | reason=summarize allowed exemplar entries
+2026-06-30T16:05:48-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-b924878d23c9dd7-20250214T163427929/2017_East_River_Pumphouse_Soil_Water_Content_and_pH__1_.csv | reason=inspect held-out raw payload columns and values
+2026-06-30T16:06:03-07:00 | action=command | subject=inspect held-out CSV with latin1 encoding | reason=UTF-8 read failed and raw file uses non-UTF8 bytes
+2026-06-30T16:06:03-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-b924878d23c9dd7-20250214T163427929/2017_East_River_Pumphouse_Soil_Water_Content_and_pH__1_.csv | reason=inspect non-UTF8 held-out payload with latin1
+2026-06-30T16:06:20-07:00 | action=command | subject=analyze held-out depth and coordinate edge cases | reason=resolve harmonization ambiguities before writing outputs
+2026-06-30T16:06:20-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-b924878d23c9dd7-20250214T163427929/2017_East_River_Pumphouse_Soil_Water_Content_and_pH__1_.csv | reason=inspect depth labels and missing-coordinate rows
+2026-06-30T16:06:34-07:00 | action=command | subject=read fold-local discrete exemplar modules | reason=match code pattern for discrete gravimetric water content datasets
+2026-06-30T16:06:34-07:00 | action=read | subject=.runs/fold-10-holdout-18/data/gold/expert_code/harmonize_sm/dataset_26.py | reason=allowed discrete soil physical/GWC exemplar
+2026-06-30T16:06:34-07:00 | action=read | subject=.runs/fold-10-holdout-18/data/gold/expert_code/harmonize_sm/dataset_25.py | reason=allowed non-timeseries exemplar with embedded coordinates/site fields
+2026-06-30T16:07:01-07:00 | action=command | subject=read exemplar JSON mapping entries | reason=match mapping.json structure for held-out entry
+2026-06-30T16:07:01-07:00 | action=read | subject=.runs/fold-10-holdout-18/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json entries 23 25 26 | reason=copy allowed mapping conventions for similar datasets
+2026-06-30T16:08:58-07:00 | action=command | subject=write held-out benchmark outputs | reason=generate curator bundle mapping notes harmonizer script and harmonized CSV
+2026-06-30T16:08:58-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-b924878d23c9dd7-20250214T163427929/2017_East_River_Pumphouse_Soil_Water_Content_and_pH__1_.csv | reason=read held-out raw payload for harmonization output
+2026-06-30T16:08:58-07:00 | action=write | subject=.runs/fold-10-holdout-18/agent_outputs/harmonize_heldout.py | reason=save reproducible held-out harmonizer script
+2026-06-30T16:08:58-07:00 | action=write | subject=.runs/fold-10-holdout-18/agent_outputs/heldout_harmonized.csv | reason=save required harmonized held-out CSV
+2026-06-30T16:08:58-07:00 | action=write | subject=.runs/fold-10-holdout-18/agent_outputs/mapping.json | reason=save held-out mapping entry
+2026-06-30T16:08:58-07:00 | action=write | subject=.runs/fold-10-holdout-18/agent_outputs/curator_bundle.json | reason=save curator inspection bundle
+2026-06-30T16:08:58-07:00 | action=write | subject=.runs/fold-10-holdout-18/agent_outputs/mapping_notes.md | reason=save harmonization decision notes
+2026-06-30T16:09:37-07:00 | action=command | subject=run agent_outputs/harmonize_heldout.py | reason=execute generated harmonizer script as required
+2026-06-30T16:09:37-07:00 | action=read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-b924878d23c9dd7-20250214T163427929/2017_East_River_Pumphouse_Soil_Water_Content_and_pH__1_.csv | reason=script reads held-out raw payload
+2026-06-30T16:09:37-07:00 | action=write | subject=.runs/fold-10-holdout-18/agent_outputs/heldout_harmonized.csv | reason=script rewrites harmonized CSV
+2026-06-30T16:10:01-07:00 | action=command | subject=verify generated outputs and scan action log | reason=final QA for files schema row count and leakage trace
+2026-06-30T16:10:01-07:00 | action=list | subject=.runs/fold-10-holdout-18/agent_outputs | reason=confirm required output files exist
+2026-06-30T16:10:01-07:00 | action=read | subject=.runs/fold-10-holdout-18/agent_outputs/heldout_harmonized.csv | reason=verify row count schema and value ranges
+2026-06-30T16:10:01-07:00 | action=read | subject=.runs/fold-10-holdout-18/agent_outputs/mapping.json | reason=verify mapping JSON parses
+2026-06-30T16:10:01-07:00 | action=read | subject=.runs/fold-10-holdout-18/AGENT_ACTION_LOG.md | reason=scan action trace for forbidden paths and patterns

evaluations/fold-10-holdout-18/AGENT_INSTRUCTIONS.md

@@ -0,0 +1,10 @@
+# Run environment — leave-one-cluster-out
+
+Harmonize the held-out dataset(s) below using ONLY:
+- the skills in `skills/`,
+- the exemplars in `data/processed/.../sm_data_harmonization_mapping.json` and the code patterns in `data/gold/expert_code/harmonize_sm/` (both have the held-out cluster removed),
+- the shared raw inputs under `~/ess-dive_wfsfa_soil_datasets/` and the cached metadata under `data/external/ess-dive_meta/`.
+
+Do NOT look up the held-out dataset's existing harmonized output, expert code, or mapping entry from any other location. The held-out datasets are:
+
+- `ess-dive-b924878d23c9dd7-20250214T163427929`

evaluations/fold-10-holdout-18/EVALUATION.md

@@ -0,0 +1,52 @@
+# fold-10-holdout-18 Agent Evaluation
+
+## Scope
+
+This directory records one sandboxed sub-agent attempt for held-out dataset indices `[18]`.
+
+- Sandbox: `.runs/fold-10-holdout-18`
+- Tracked artifact copy: `evaluations/fold-10-holdout-18`
+- Successful agent trace: `AGENT_ACTION_LOG.md`
+
+## Anti-Leakage Audit
+
+The successful trace was scanned for root gold data, root processed data, other fold sandboxes, held-out expert module names, and explicit `violation` markers. Result: `0` matches.
+
+## Output Equivalence Against Gold
+
+| Metric | Agent | Gold |
+| --- | ---: | ---: |
+| Rows | 416 | 420 |
+| Target columns present | 9 / 9 | 9 / 9 |
+| Schema exact match | true | n/a |
+| Unique keyed rows recovered | 0 | 393 |
+| Missing gold keyed rows | 393 | n/a |
+| Extra agent keyed rows | 398 | n/a |
+| Row precision | 0.000 | n/a |
+| Gold recall | 0.000 | n/a |
+
+Shared-row value matches after key merge:
+
+| Column | Matched rows |
+| --- | ---: |
+| `interval_min` | 0/0 |
+| `volumetric_water_content_m3_m3` | 0/0 |
+| `gravimetric_water_content_gH2O_gs` | 0/0 |
+| `water_potential_kPa` | 0/0 |
+| `is_timeseries` | 0/0 |
+
+## JSON Mapping Match
+
+- Summary: `{"extra_agent_categories": [], "field_exact_matches_by_name": {"destination_variable": "8/9", "source_files": "9/9", "source_pattern": "7/9", "transformation": "2/9", "unit_conversion": "8/9"}, "field_level_exact_matches": "34/45", "gold_mapping_categories_present": "9/9", "missing_gold_categories": [], "top_level_exact_matches": "5/7", "top_level_matches": {"archive_repository": true, "data_payload_files": true, "dataset_identifier": true, "doi": false, "index": true, "location_metadata_files": false, "sensor_metadata_files": true}}`
+
+| Field | Exact matches |
+| --- | ---: |
+| `destination_variable` | 8/9 |
+| `source_pattern` | 7/9 |
+| `source_files` | 9/9 |
+| `transformation` | 2/9 |
+| `unit_conversion` | 8/9 |
+
+## Overall Assessment
+
+See `evaluation_metrics.json` for machine-readable details. The run passed the targeted local no-gold path audit. Output equivalence should be interpreted with the key policy recorded in the metrics (`key_includes_dataset_index=False`).

evaluations/fold-10-holdout-18/MANIFEST.json

@@ -0,0 +1,35 @@
+{
+  "name": "fold-10-holdout-18",
+  "holdout_indices": [
+    18
+  ],
+  "holdout_identifiers": [
+    "ess-dive-b924878d23c9dd7-20250214T163427929"
+  ],
+  "exemplar_indices": [
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    15,
+    16,
+    17,
+    23,
+    24,
+    25,
+    26,
+    27
+  ],
+  "n_exemplars": 27,
+  "sources": {
+    "package": "data/gold/expert_code/harmonize_sm",
+    "mapping": "data/gold/sm_data_harmonization_mapping.json",
+    "skills": "skills"
+  }
+}

evaluations/fold-10-holdout-18/agent_outputs/curator_bundle.json

@@ -0,0 +1,87 @@
+{
+  "dataset_identifier": "ess-dive-b924878d23c9dd7-20250214T163427929",
+  "doi": "UNKNOWN_FROM_ALLOWED_INPUTS",
+  "index": 18,
+  "decision": "INCLUDE",
+  "decision_reason": "Contains a machine-readable table of direct gravimetric soil water content observations with dates, depths, site hierarchy, replicates, and mostly embedded coordinates.",
+  "files": [
+    {
+      "filename": "2017_East_River_Pumphouse_Soil_Water_Content_and_pH__1_.csv",
+      "category": "data_payload",
+      "encoding": "latin1",
+      "rows": 420,
+      "columns": [
+        "Lab_ID_LBNL",
+        "IGSN",
+        "Latitude",
+        "Longitude",
+        "Location",
+        "Field_Site",
+        "Elevation",
+        "Block",
+        "Plot",
+        "Replicate",
+        "Topographic_Position",
+        "Depth_Increment",
+        "Date_Collected",
+        "Soil Water Content (g H2O per gram  soil)",
+        "Soil pH "
+      ],
+      "soil_moisture_columns": [
+        "Soil Water Content (g H2O per gram  soil)"
+      ],
+      "date_columns": [
+        "Date_Collected"
+      ],
+      "depth_columns": [
+        "Depth_Increment"
+      ],
+      "site_columns": [
+        "Location",
+        "Field_Site",
+        "Block",
+        "Plot",
+        "Replicate",
+        "Topographic_Position"
+      ],
+      "coordinate_columns": [
+        "Latitude",
+        "Longitude"
+      ]
+    }
+  ],
+  "raw_shape": {
+    "rows": 420,
+    "columns": 15
+  },
+  "harmonized_shape": {
+    "rows": 416,
+    "columns": 9
+  },
+  "excluded_rows": {
+    "missing_gravimetric_water_content": 4
+  },
+  "location_resolution": {
+    "source": "payload Latitude/Longitude where present",
+    "rows_missing_payload_coordinates": 57,
+    "qc_recommendation_for_missing_coordinates": "g2 if producing companion location rows from payload only",
+    "unique_site_coordinate_records_observed": 133
+  },
+  "depth_resolution": {
+    "0-5 cm": 0.025,
+    "5-15 cm": 0.1,
+    "15-May": 0.1,
+    "15 cm +": 0.15,
+    "qc_recommendation": "d1 for midpoint/lower-bound depth approximations if qc_flag is tracked outside the required nine-column CSV"
+  },
+  "time_series": {
+    "is_timeseries": false,
+    "interval_min": null,
+    "reason": "Discrete field sampling dates, not a continuous sensor/logger time series."
+  },
+  "open_questions": [
+    "DOI was not available in the fold-local instructions, fold-local metadata cache, or held-out raw CSV; set to UNKNOWN_FROM_ALLOWED_INPUTS.",
+    "Depth token '15-May' is interpreted as an Excel-mangled '5-15 cm' label based on date/depth patterning.",
+    "Rows with '15 cm +' use 0.15 m as the reported lower-bound representative depth."
+  ]
+}

evaluations/fold-10-holdout-18/agent_outputs/harmonize_heldout.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+from pathlib import Path
+import numpy as np
+import pandas as pd
+
+RAW_DIR = Path('/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-b924878d23c9dd7-20250214T163427929')
+RAW_FILE = '2017_East_River_Pumphouse_Soil_Water_Content_and_pH__1_.csv'
+OUT_DIR = Path('/scratch/jmc/data-harmonization-eval/.runs/fold-10-holdout-18/agent_outputs')
+COLS = [
+    'datetime_UTC',
+    'site_id',
+    'depth_m',
+    'replicate',
+    'is_timeseries',
+    'interval_min',
+    'volumetric_water_content_m3_m3',
+    'gravimetric_water_content_gH2O_gs',
+    'water_potential_kPa',
+]
+
+def site_id_from_row(row: pd.Series) -> str:
+    parts = [
+        row.get('Location'),
+        row.get('Field_Site'),
+        row.get('Block'),
+        row.get('Plot'),
+        f"rep{row.get('Replicate')}",
+    ]
+    topo = row.get('Topographic_Position')
+    if pd.notna(topo) and str(topo).strip():
+        parts.append(topo)
+    cleaned = []
+    for part in parts:
+        s = str(part).strip().replace(' ', '_').replace('/', '_')
+        cleaned.append(s)
+    return '|'.join(cleaned)
+
+def depth_to_m(value: object) -> float:
+    return {
+        '0-5 cm': 0.025,
+        '5-15 cm': 0.10,
+        '15-May': 0.10,
+        '15 cm +': 0.15,
+    }.get(str(value).strip(), np.nan)
+
+def harmonize() -> pd.DataFrame:
+    df = pd.read_csv(RAW_DIR / RAW_FILE, encoding='latin1')
+    out = pd.DataFrame()
+    dt = pd.to_datetime(df['Date_Collected'], format='%m/%d/%Y', errors='coerce')
+    dt = dt.dt.tz_localize('America/Denver', ambiguous='NaT', nonexistent='shift_forward').dt.tz_convert('UTC')
+    out['datetime_UTC'] = dt.dt.strftime('%Y-%m-%dT%H:%M:%SZ')
+    out['site_id'] = df.apply(site_id_from_row, axis=1)
+    out['depth_m'] = df['Depth_Increment'].map(depth_to_m).astype(float)
+    out['replicate'] = pd.to_numeric(df['Replicate'], errors='coerce')
+    out['is_timeseries'] = False
+    out['interval_min'] = np.nan
+    out['volumetric_water_content_m3_m3'] = np.nan
+    out['gravimetric_water_content_gH2O_gs'] = pd.to_numeric(
+        df['Soil Water Content (g H2O per gram  soil)'], errors='coerce'
+    )
+    out['water_potential_kPa'] = np.nan
+    out = out[out['gravimetric_water_content_gH2O_gs'].notna()].copy()
+    return out[COLS]
+
+if __name__ == '__main__':
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    harmonize().to_csv(OUT_DIR / 'heldout_harmonized.csv', index=False)