[codex] add fold05 agent evaluation bundle

evaluations/fold-05-holdout-7/AGENT_ACTION_LOG.md

@@ -0,0 +1,35 @@
+# Agent Action Log
+
+- 2026-06-30T15:38:27-07:00 | action=command+write | subject=`initialize AGENT_ACTION_LOG.md and agent_outputs` | reason=initialize mandatory benchmark trace and output directory
+- 2026-06-30T15:38:27-07:00 | action=write | subject=`/scratch/jmc/data-harmonization-eval/.runs/fold-05-holdout-7/agent_outputs` | reason=create required output directory
+- 2026-06-30T15:38:36-07:00 | action=command+read | subject=`sed -n 1,220p fold AGENT_INSTRUCTIONS/MANIFEST/local SKILL.md files` | reason=load fold-local task instructions and skills
+- 2026-06-30T15:38:44-07:00 | action=command+read | subject=`wc -l local SKILL.md files and sed remaining local skill sections` | reason=ensure both fold-local skill files are read completely
+- 2026-06-30T15:38:52-07:00 | action=command+read | subject=`sed -n 521,620p essdive_sm_curator/SKILL.md` | reason=read final missing portion of fold-local curator skill
+- 2026-06-30T15:39:00-07:00 | action=command+directory_listing | subject=`find fold sandbox maxdepth 4` | reason=locate fold-local exemplars, metadata cache, and output paths without leaving sandbox
+- 2026-06-30T15:39:11-07:00 | action=command+directory_listing | subject=`find fold expert_code and allowed raw package directories` | reason=identify allowed exemplar code files and held-out raw files
+- 2026-06-30T15:39:20-07:00 | action=command+directory_listing | subject=`find held-out raw package maxdepth 4` | reason=classify available raw files for held-out dataset
+- 2026-06-30T15:39:34-07:00 | action=command+directory_listing | subject=`find held-out raw nested package directory` | reason=verify whether nested package directory contains additional files
+- 2026-06-30T15:39:55-07:00 | action=command+read | subject=`python inspect held-out CSV headers row counts previews` | reason=identify measurement payload columns, metadata columns, and file usability
+- 2026-06-30T15:40:12-07:00 | action=command+read | subject=`python summarize fold-local mapping JSON and read common.py` | reason=use allowed exemplar schema and helper conventions
+- 2026-06-30T15:40:20-07:00 | action=command+search | subject=`rg depth_m|volumetric_water_content over fold expert_code` | reason=find fold-local exemplar code patterns for direct depth and VWC mappings
+- 2026-06-30T15:40:31-07:00 | action=command+read | subject=`read fold-local dataset_23.py dataset_26.py and mapping entries 23/26` | reason=inspect similar allowed exemplar transformations for direct VWC and depth columns
+- 2026-06-30T15:40:59-07:00 | action=command+read | subject=`python profile held-out CSV contents for timestamps depths values and coordinate hints` | reason=derive curation facts and harmonization transformations from allowed raw files
+- 2026-06-30T15:41:11-07:00 | action=command+search | subject=`rg DOI/package identifier in fold sandbox and held-out raw package only` | reason=look for package DOI without reading forbidden external gold/processed or other folds
+- 2026-06-30T15:41:48-07:00 | action=command+read | subject=`python inspect non-null Soil.Moisture rows in BM_EGM_Surface_Flux.csv` | reason=decide whether ancillary flux file contains harmonizable direct soil moisture observations
+- 2026-06-30T15:42:18-07:00 | action=command+write | subject=`curl ESS-DIVE package API for held-out package metadata to agent_outputs/package_metadata_probe.json` | reason=retrieve DOI through curator-skill metadata workflow without reading forbidden local gold/processed outputs
+- 2026-06-30T15:42:41-07:00 | action=command+write | subject=`curl ESS-DIVE package API with escalated network for held-out package metadata` | reason=retrieve DOI through curator-skill metadata workflow after sandbox DNS failure
+- 2026-06-30T15:42:59-07:00 | action=command+read | subject=`python parse agent_outputs/package_metadata_probe.json` | reason=extract DOI, title, description, and distribution metadata from own downloaded package metadata
+- 2026-06-30T15:42:59-07:00 | action=command+write | subject=`rm -f agent_outputs/package_metadata_probe.json` | reason=remove temporary metadata probe so final outputs match requested artifact set
+- 2026-06-30T15:44:28-07:00 | action=command+write | subject=`python generate curator_bundle.json mapping.json mapping_notes.md harmonize_heldout.py` | reason=create required held-out benchmark artifacts under agent_outputs
+- 2026-06-30T15:44:28.579869-07:00 | action=write | subject=`/scratch/jmc/data-harmonization-eval/.runs/fold-05-holdout-7/agent_outputs/curator_bundle.json` | reason=write required artifact curator_bundle.json
+- 2026-06-30T15:44:28.580215-07:00 | action=write | subject=`/scratch/jmc/data-harmonization-eval/.runs/fold-05-holdout-7/agent_outputs/mapping.json` | reason=write required artifact mapping.json
+- 2026-06-30T15:44:28.580396-07:00 | action=write | subject=`/scratch/jmc/data-harmonization-eval/.runs/fold-05-holdout-7/agent_outputs/mapping_notes.md` | reason=write required artifact mapping_notes.md
+- 2026-06-30T15:44:28.580556-07:00 | action=write | subject=`/scratch/jmc/data-harmonization-eval/.runs/fold-05-holdout-7/agent_outputs/harmonize_heldout.py` | reason=write required artifact harmonize_heldout.py
+- 2026-06-30T15:44:28.580771-07:00 | action=command+write | subject=`chmod 755 /scratch/jmc/data-harmonization-eval/.runs/fold-05-holdout-7/agent_outputs/harmonize_heldout.py` | reason=make required harmonization script executable
+- 2026-06-30T15:44:35-07:00 | action=command | subject=`agent_outputs/harmonize_heldout.py` | reason=run required harmonization script to generate heldout_harmonized.csv
+- 2026-06-30T15:44:36.244216-07:00 | action=read | subject=`/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-38e901ec3d7bd24-20230504T211548257225/BM_Merged_T_VWC_0616_1018.csv` | reason=read held-out primary VWC payload for harmonization
+- 2026-06-30T15:44:36.867023-07:00 | action=write | subject=`/scratch/jmc/data-harmonization-eval/.runs/fold-05-holdout-7/agent_outputs/heldout_harmonized.csv` | reason=write required heldout_harmonized.csv output
+- 2026-06-30T15:44:49-07:00 | action=command+read+directory_listing | subject=`ls agent_outputs and validate generated CSV/JSON outputs` | reason=verify required artifacts, row count, schema, and mapping fields
+- 2026-06-30T15:45:11-07:00 | action=command+read+write | subject=`scan AGENT_ACTION_LOG.md for forbidden path patterns` | reason=required final anti-leakage verification
+- 2026-06-30T15:45:21-07:00 | action=command+write | subject=`rm -f agent_outputs/forbidden_path_scan.txt` | reason=remove temporary scan report and leave only requested deliverables in agent_outputs
+- 2026-06-30T15:45:21-07:00 | action=command+directory_listing | subject=`ls agent_outputs final deliverables` | reason=confirm final output directory contains requested files only

evaluations/fold-05-holdout-7/AGENT_INSTRUCTIONS.md

@@ -0,0 +1,10 @@
+# Run environment — leave-one-cluster-out
+
+Harmonize the held-out dataset(s) below using ONLY:
+- the skills in `skills/`,
+- the exemplars in `data/processed/.../sm_data_harmonization_mapping.json` and the code patterns in `data/gold/expert_code/harmonize_sm/` (both have the held-out cluster removed),
+- the shared raw inputs under `~/ess-dive_wfsfa_soil_datasets/` and the cached metadata under `data/external/ess-dive_meta/`.
+
+Do NOT look up the held-out dataset's existing harmonized output, expert code, or mapping entry from any other location. The held-out datasets are:
+
+- `ess-dive-38e901ec3d7bd24-20230504T211548257225`

evaluations/fold-05-holdout-7/EVALUATION.md

@@ -0,0 +1,167 @@
+# Fold 05 Holdout 7 Agent Evaluation
+
+## Scope
+
+This directory records one sandboxed sub-agent attempt on fold 05, where dataset
+index 7 was held out:
+
+- Dataset: `ess-dive-38e901ec3d7bd24-20230504T211548257225`
+- Sandbox: `.runs/fold-05-holdout-7`
+- Tracked artifact copy: `evaluations/fold-05-holdout-7`
+- Successful agent trace: `AGENT_ACTION_LOG.md`
+
+The worker was run without inherited parent context. It was instructed to use
+only the fold-local Skill 1 curator and Skill 2 harmonizer materials, raw ESS-DIVE
+inputs, and fold-local exemplars, and to log each read/list/search/command/write
+before performing it.
+
+## Anti-Leakage Audit
+
+The successful trace was scanned for direct evidence of forbidden access to:
+
+- root gold data: `/scratch/jmc/data-harmonization-eval/data/gold`
+- root processed data: `/scratch/jmc/data-harmonization-eval/data/processed`
+- other fold sandboxes
+- the held-out expert module name: `dataset_07.py`
+- explicit `violation` markers
+
+Command:
+
+```bash
+rg -n '(/scratch/jmc/data-harmonization-eval/data/gold|/scratch/jmc/data-harmonization-eval/data/processed|/scratch/jmc/data-harmonization-eval/.runs/fold-(01|02|03|04|06|07|08|09|10|11|12|13)|dataset_07\.py|violation)' .runs/fold-05-holdout-7/AGENT_ACTION_LOG.md
+```
+
+Result: no matches. The trace therefore passes this targeted local path audit.
+
+The trace records one external metadata lookup through the ESS-DIVE package API
+to retrieve the DOI/title. That is not local gold leakage, but it is external
+metadata access and should be accounted for separately if the benchmark protocol
+requires all metadata to come only from staged raw package files.
+
+## Output Equivalence Against Gold
+
+The gold comparison used the current expert module for dataset 7. Running that
+module required an evaluation-only compatibility shim for `add_loc_qc`, because
+NumPy 2 raises a dtype promotion error when mixing string QC flags and `np.nan`.
+The shim affected only the location-QC side output after the harmonized table was
+constructed; it did not alter the soil-moisture transformation being compared.
+
+Summary:
+
+| Metric | Agent | Gold |
+| --- | ---: | ---: |
+| Rows | 56,861 | 56,861 |
+| Columns | 9 | 9 |
+| Schema exact match | yes | yes |
+| Unique keyed rows recovered | 56,861 | 56,861 |
+| Missing gold keyed rows | 0 | n/a |
+| Extra keyed rows | 0 | n/a |
+
+Site and depth counts matched exactly:
+
+| Field | Value |
+| --- | --- |
+| `site_id` | `BM` for all 56,861 rows |
+| Depths | `0.1`, `0.2`, `0.3`, `0.5`, `0.75`, `1.0`, `1.3` m |
+| Rows per depth | 8,123 |
+
+For keyed rows, target-value matches were:
+
+| Column | Matched rows |
+| --- | ---: |
+| `volumetric_water_content_m3_m3` | 56,861 / 56,861 |
+| `water_potential_kPa` | 56,861 / 56,861 |
+| `gravimetric_water_content_gH2O_gs` | 56,861 / 56,861 |
+| `is_timeseries` | 56,861 / 56,861 |
+| `interval_min` | 56,855 / 56,861 |
+
+The only table mismatch was six `interval_min` cells. The agent computed
+intervals within each `site_id`/`depth_m`/`replicate` stream. The gold expert
+computes a raw sequential diff across the input file after parsing timestamps,
+which creates six large negative interval values at depth transitions. The
+agent's behavior is scientifically cleaner, but it is not literal gold
+equivalence for those six cells.
+
+After sorting and dropping `interval_min`, the agent and gold target tables were
+exactly equal.
+
+## JSON Mapping Match
+
+The generated `mapping.json` was compared to gold mapping entry 7 from
+`data/gold/sm_data_harmonization_mapping.json`.
+
+Top-level exact matches:
+
+| Field | Match |
+| --- | --- |
+| `index` | yes |
+| `dataset_identifier` | yes |
+| `doi` | yes |
+| `archive_repository` | yes |
+| `data_payload_files` | yes |
+| `location_metadata_files` | yes |
+| `sensor_metadata_files` | yes |
+
+Mapping category coverage:
+
+- Gold categories present: 8 / 8
+- Extra agent category: `gravimetric_water_content`
+- Missing gold categories: none
+
+Field-level exact matches across shared categories:
+
+| Field | Exact matches |
+| --- | ---: |
+| `destination_variable` | 8 / 8 |
+| `source_pattern` | 3 / 8 |
+| `source_files` | 7 / 8 |
+| `transformation` | 0 / 8 |
+| `unit_conversion` | 7 / 8 |
+| Total | 25 / 40 |
+
+Important JSON differences:
+
+- Agent added `gravimetric_water_content`, which is absent from the gold mapping
+  JSON even though the output schema contains
+  `gravimetric_water_content_gH2O_gs`.
+- Agent used the actual raw column names, such as `Depth (cm)` and
+  `Volumetric Water Content`; gold uses normalized/source-pattern strings such
+  as `Depth..cm.` and `Volumetric.Water.Content`.
+- Transformation prose did not match gold text literally.
+- For `site_id`, the agent cited both the BM file prefix and the location file;
+  gold cites only `BM_EGM_Well_CO2.csv`.
+
+## Python Match
+
+The generated `harmonize_heldout.py` captured nearly all of the gold expert
+transformation:
+
+- Reads `BM_Merged_T_VWC_0616_1018.csv`.
+- Parses `date.time` with `%m/%d/%y %H:%M` in `America/Denver` and converts to
+  UTC.
+- Sets constant `site_id = "BM"`.
+- Converts `Depth (cm)` to `depth_m` by dividing by 100.
+- Sets `replicate` to `1`.
+- Sets `is_timeseries` to `True`.
+- Maps `Volumetric Water Content` to `volumetric_water_content_m3_m3`.
+- Sets water potential and gravimetric water content to null.
+
+Python differences relative to gold:
+
+- Computes `interval_min` within depth streams instead of raw file order,
+  causing six literal gold mismatches.
+- Sorts by `site_id`, `depth_m`, `replicate`, and timestamp before interval
+  calculation; gold preserves raw input order for interval diffs.
+- Does not implement the gold module interface using `Context` and
+  `DatasetResult`.
+- Does not emit the gold location side dataframe, though it documents the
+  `BM_EGM_Well_CO2.csv` coordinates in the curator bundle and notes.
+
+## Overall Assessment
+
+The index 7 run was substantially stronger than the index 4 run. It recovered
+the complete row set with no extras or omissions and matched all non-interval
+target values exactly. The remaining output difference is narrowly localized to
+six interval cells caused by a defensible but non-gold grouping choice. The JSON
+mapping is structurally strong and perfect on top-level metadata, but only a
+moderate literal match to gold prose and source-pattern fields.

evaluations/fold-05-holdout-7/MANIFEST.json

@@ -0,0 +1,35 @@
+{
+  "name": "fold-05-holdout-7",
+  "holdout_indices": [
+    7
+  ],
+  "holdout_identifiers": [
+    "ess-dive-38e901ec3d7bd24-20230504T211548257225"
+  ],
+  "exemplar_indices": [
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    8,
+    9,
+    10,
+    15,
+    16,
+    17,
+    18,
+    23,
+    24,
+    25,
+    26,
+    27
+  ],
+  "n_exemplars": 27,
+  "sources": {
+    "package": "data/gold/expert_code/harmonize_sm",
+    "mapping": "data/gold/sm_data_harmonization_mapping.json",
+    "skills": "skills"
+  }
+}

evaluations/fold-05-holdout-7/agent_outputs/curator_bundle.json

@@ -0,0 +1,86 @@
+{
+  "package_id": "ess-dive-38e901ec3d7bd24-20230504T211548257225",
+  "doi": "doi:10.15485/1660455",
+  "title": "Soil Environmental Conditions in the East River Watershed, CO",
+  "curator_decision": "INCLUDE",
+  "exclusion_reason": null,
+  "data_payload_files": [
+    {
+      "filename": "BM_Merged_T_VWC_0616_1018.csv",
+      "classification": "primary_soil_moisture_timeseries",
+      "columns": [
+        "date.time",
+        "record",
+        "Volumetric Water Content",
+        "Electrical Conductivity (dS/m)",
+        "T (degrees celcius)",
+        "Depth (cm)"
+      ],
+      "row_count_estimate": 56861,
+      "file_size_mb": 2.419095,
+      "column_preview": "date.time,record,Volumetric Water Content,Electrical Conductivity (dS/m),T (degrees celcius),Depth (cm)\n6/20/16 14:00,0,0.1194,0.0201,19.2250,10\n6/20/16 15:00,1,0.1200,0.0208,21.0667,10",
+      "notes": "Depth-resolved Campbell CS655 datalogger output; VWC values are already fractional m3/m3 based on 0.0557-0.5043 observed range."
+    }
+  ],
+  "non_harmonized_files": [
+    {
+      "filename": "BM_EGM_Surface_Flux.csv",
+      "classification": "surface_flux_context_with_incidental_soil_moisture",
+      "reason_not_harmonized": "Primary purpose is CO2 surface flux; only 36 non-null Soil.Moisture values and no depth metadata. Used as context for Bradley Meadow naming only."
+    }
+  ],
+  "location_metadata_files": [
+    {
+      "filename": "BM_EGM_Well_CO2.csv",
+      "columns": [
+        "Location",
+        "Latitude (deg N)",
+        "Longitude (deg E)",
+        "Date",
+        "Time",
+        "Depth",
+        "CO2.Max",
+        "Notes"
+      ],
+      "content_preview": "Location,Latitude (deg N),Longitude (deg E),Date,Time,Depth,CO2.Max,Notes",
+      "site_coordinates": [
+        {
+          "site_id": "BM",
+          "latitude": 38.98715,
+          "longitude": -107.003863
+        }
+      ]
+    }
+  ],
+  "sensor_metadata_files": null,
+  "location_resolution": {
+    "source": "location_metadata_file",
+    "qc_flag_recommendation": null,
+    "site_coordinates": [
+      {
+        "site_id": "BM",
+        "latitude": 38.98715,
+        "longitude": -107.003863
+      }
+    ]
+  },
+  "time_series_inference": {
+    "is_timeseries": true,
+    "interval_min": 60.0,
+    "reasoning": "Each of seven depths has 8123 observations from 2016-06-20 14:00 to 2018-10-19 09:00 with median 60-minute spacing."
+  },
+  "experimental_context": {
+    "manipulation_detected": false,
+    "manipulation_type": null,
+    "has_control_data": null,
+    "recommendation": "include_all_primary_payload"
+  },
+  "similar_dataset_reference": {
+    "index": 23,
+    "reason": "Fold-local exemplar with time series VWC in m3/m3 and explicit depth metadata converted from cm to m."
+  },
+  "open_questions": [
+    "No README/methods file was present in the allowed raw package directory; timezone inferred as America/Denver from East River, CO conventions in fold-local examples.",
+    "Surface flux file contains incidental Soil.Moisture values but lacks depth metadata and is not treated as the primary soil moisture payload."
+  ]
+}