diff --git a/evaluations/fold-11-holdout-23/AGENT_ACTION_LOG.md b/evaluations/fold-11-holdout-23/AGENT_ACTION_LOG.md new file mode 100644 index 0000000..cef4388 --- /dev/null +++ b/evaluations/fold-11-holdout-23/AGENT_ACTION_LOG.md @@ -0,0 +1,72 @@ +2026-06-30T16:04:39-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/AGENT_ACTION_LOG.md | reason=initialize mandatory trace log +2026-06-30T16:04:39-07:00 | action=directory create | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs | reason=initialize output directory +2026-06-30T16:04:39-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/AGENT_INSTRUCTIONS.md | reason=read benchmark instructions +2026-06-30T16:04:39-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/MANIFEST.json | reason=read sandbox manifest +2026-06-30T16:04:39-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/skills/essdive_sm_curator/SKILL.md | reason=read fold-local curator skill +2026-06-30T16:04:39-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/skills/essdive_sm_harmonizer/SKILL.md | reason=read fold-local harmonizer skill +2026-06-30T16:04:47-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/skills/essdive_sm_harmonizer/SKILL.md lines 261-520 | reason=complete fold-local harmonizer skill read +2026-06-30T16:04:56-07:00 | action=directory listing | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23 | reason=inspect sandbox top-level files +2026-06-30T16:04:56-07:00 | action=directory listing | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data | reason=inspect fold-local data files only +2026-06-30T16:04:56-07:00 | action=directory listing | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379 | reason=inspect allowed held-out raw package files +2026-06-30T16:05:20-07:00 | action=command | subject=python inspect fold-local exemplars and held-out raw files | reason=summarize allowed mapping/code/raw inputs +2026-06-30T16:05:21.293887-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/processed/ess-dive_wfsfa_soil_datasets/sm_data_harmonization_mapping.json | reason=inspect fold-local mapping schema and exemplar entries +2026-06-30T16:05:21.294834-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/gold/expert_code/harmonize_sm/common.py | reason=inspect allowed common harmonizer helpers +2026-06-30T16:05:21.294976-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/gold/expert_code/harmonize_sm/dataset_24.py | reason=inspect allowed exemplar code dataset_24 +2026-06-30T16:05:21.295094-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/gold/expert_code/harmonize_sm/dataset_25.py | reason=inspect allowed exemplar code dataset_25 +2026-06-30T16:05:21.295189-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/gold/expert_code/harmonize_sm/dataset_26.py | reason=inspect allowed exemplar code dataset_26 +2026-06-30T16:05:21.295276-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/gold/expert_code/harmonize_sm/dataset_27.py | reason=inspect allowed exemplar code dataset_27 +2026-06-30T16:05:21.295389-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/gold/expert_code/harmonize_sm/dataset_18.py | reason=inspect allowed exemplar code dataset_18 +2026-06-30T16:05:21.295489-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/gold/expert_code/harmonize_sm/dataset_16.py | reason=inspect allowed exemplar code dataset_16 +2026-06-30T16:05:21.295576-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/data/gold/expert_code/harmonize_sm/dataset_15.py | reason=inspect allowed exemplar code dataset_15 +2026-06-30T16:05:21.295668-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/WM_SWC.csv | reason=inspect held-out raw CSV header and sample rows +2026-06-30T16:05:21.472686-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/WM_SWC.csv | reason=inspect held-out raw CSV shape and dtypes +2026-06-30T16:05:23.952391-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/plot_metadata.csv | reason=inspect held-out raw CSV header and sample rows +2026-06-30T16:05:23.957491-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/plot_metadata.csv | reason=inspect held-out raw CSV shape and dtypes +2026-06-30T16:05:23.986915-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/sensor_metadata.csv | reason=inspect held-out raw CSV header and sample rows +2026-06-30T16:05:23.993878-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/sensor_metadata.csv | reason=inspect held-out raw CSV shape and dtypes +2026-06-30T16:05:52-07:00 | action=command | subject=curl ESS-DIVE package metadata for ess-dive-a99be52b7a6114c-20230504T210134503379 | reason=retrieve DOI and package-level metadata when fold cache is absent +2026-06-30T16:05:52-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/_essdive_meta_probe.json | reason=temporary metadata probe inside agent_outputs +2026-06-30T16:07:59-07:00 | action=command | subject=curl ESS-DIVE package metadata for ess-dive-a99be52b7a6114c-20230504T210134503379 with network | reason=retrieve DOI and package-level metadata when fold cache is absent +2026-06-30T16:07:59-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/_essdive_meta_probe.json | reason=temporary metadata probe inside agent_outputs +2026-06-30T16:08:01-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/_essdive_meta_probe.json | reason=inspect retrieved ESS-DIVE metadata probe +2026-06-30T16:08:15-07:00 | action=command | subject=python parse ESS-DIVE metadata probe | reason=extract DOI, spatial coverage, temporal coverage, distribution filenames +2026-06-30T16:08:15.212319-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/_essdive_meta_probe.json | reason=parse retrieved metadata fields +2026-06-30T16:08:40-07:00 | action=command | subject=python profile held-out sensor joins and cadence | reason=verify merge keys, depth mapping, replicate strategy, interval +2026-06-30T16:08:41.315174-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/sensor_metadata.csv | reason=profile sensor metadata keys +2026-06-30T16:08:41.319556-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/WM_SWC.csv | reason=profile payload keys and timestamp cadence +2026-06-30T16:09:38-07:00 | action=command | subject=python inspect post-merge output candidates | reason=verify missing metadata, exact duplicates, replicate grouping +2026-06-30T16:09:38.502999-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/WM_SWC.csv | reason=inspect payload for harmonization candidate rows +2026-06-30T16:09:39.852104-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/sensor_metadata.csv | reason=inspect sensor metadata for harmonization candidate rows +2026-06-30T16:10:01-07:00 | action=command | subject=python inspect missing-depth payload rows | reason=document handling for rows without point/depth metadata +2026-06-30T16:10:02.383473-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/WM_SWC.csv | reason=read missing-depth payload rows +2026-06-30T16:10:03.729952-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/sensor_metadata.csv | reason=read missing-depth sensor metadata rows +2026-06-30T16:11:18-07:00 | action=command | subject=write harmonize_heldout.py | reason=create held-out harmonization script +2026-06-30T16:11:18-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/harmonize_heldout.py | reason=create standalone harmonizer output +2026-06-30T16:12:21-07:00 | action=command | subject=python write curator_bundle mapping and notes | reason=create required JSON and markdown outputs +2026-06-30T16:12:22.118709-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/_essdive_meta_probe.json | reason=read public metadata probe for DOI/title and package context +2026-06-30T16:12:22.118940-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/WM_SWC.csv | reason=compute payload summary for curator bundle +2026-06-30T16:12:23.780294-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/plot_metadata.csv | reason=compute plot metadata summary for curator bundle +2026-06-30T16:12:23.783355-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/sensor_metadata.csv | reason=compute sensor metadata summary for curator bundle +2026-06-30T16:12:24.813846-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/curator_bundle.json | reason=write curator bundle JSON +2026-06-30T16:12:24.814199-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/mapping.json | reason=write mapping JSON entry +2026-06-30T16:12:24.814444-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/mapping_notes.md | reason=write mapping notes markdown +2026-06-30T16:12:32-07:00 | action=command | subject=python /scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/harmonize_heldout.py | reason=run held-out harmonization script +2026-06-30T16:12:32.615715-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/WM_SWC.csv | reason=read held-out SWC payload for harmonization +2026-06-30T16:12:33.945648-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/sensor_metadata.csv | reason=read held-out sensor metadata for depth/site joins +2026-06-30T16:12:53-07:00 | action=command | subject=perl edit harmonize_heldout.py sensor serial casting | reason=fix SWC-only Sensor.SN normalization +2026-06-30T16:12:53-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/harmonize_heldout.py | reason=update harmonizer to cast Sensor.SN after SWC filtering +2026-06-30T16:12:53-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/harmonize_heldout.py | reason=verify modified harmonizer section +2026-06-30T16:13:01-07:00 | action=command | subject=python /scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/harmonize_heldout.py rerun | reason=generate held-out harmonized CSV after cast fix +2026-06-30T16:13:01.671650-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/WM_SWC.csv | reason=read held-out SWC payload for harmonization +2026-06-30T16:13:03.334949-07:00 | action=file read | subject=/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379/sensor_metadata.csv | reason=read held-out sensor metadata for depth/site joins +2026-06-30T16:13:21.848342-07:00 | action=write | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/heldout_harmonized.csv | reason=write held-out harmonized CSV +2026-06-30T16:13:45-07:00 | action=command | subject=rm -f temporary metadata probe | reason=remove non-deliverable temporary file from agent_outputs +2026-06-30T16:13:45-07:00 | action=delete | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/_essdive_meta_probe.json | reason=remove non-deliverable temporary file +2026-06-30T16:14:09-07:00 | action=command | subject=python validate outputs and scan action log | reason=final schema/count validation and anti-leakage scan +2026-06-30T16:14:09.672234-07:00 | action=directory listing | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs | reason=verify only required deliverables are present +2026-06-30T16:14:09.672441-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/curator_bundle.json | reason=validate curator_bundle.json parses as JSON +2026-06-30T16:14:09.672699-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/mapping.json | reason=validate mapping.json parses as JSON +2026-06-30T16:14:09.672901-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/mapping_notes.md | reason=validate mapping notes exists and preview +2026-06-30T16:14:09.673045-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/harmonize_heldout.py | reason=validate harmonizer script exists and contains schema +2026-06-30T16:14:09.673229-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/agent_outputs/heldout_harmonized.csv | reason=validate harmonized CSV schema and row count +2026-06-30T16:14:11.809979-07:00 | action=file read | subject=/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23/AGENT_ACTION_LOG.md | reason=scan mandatory action log for forbidden path patterns diff --git a/evaluations/fold-11-holdout-23/AGENT_INSTRUCTIONS.md b/evaluations/fold-11-holdout-23/AGENT_INSTRUCTIONS.md new file mode 100644 index 0000000..1996946 --- /dev/null +++ b/evaluations/fold-11-holdout-23/AGENT_INSTRUCTIONS.md @@ -0,0 +1,10 @@ +# Run environment — leave-one-cluster-out + +Harmonize the held-out dataset(s) below using ONLY: +- the skills in `skills/`, +- the exemplars in `data/processed/.../sm_data_harmonization_mapping.json` and the code patterns in `data/gold/expert_code/harmonize_sm/` (both have the held-out cluster removed), +- the shared raw inputs under `~/ess-dive_wfsfa_soil_datasets/` and the cached metadata under `data/external/ess-dive_meta/`. + +Do NOT look up the held-out dataset's existing harmonized output, expert code, or mapping entry from any other location. The held-out datasets are: + +- `ess-dive-a99be52b7a6114c-20230504T210134503379` diff --git a/evaluations/fold-11-holdout-23/EVALUATION.md b/evaluations/fold-11-holdout-23/EVALUATION.md new file mode 100644 index 0000000..7d2f278 --- /dev/null +++ b/evaluations/fold-11-holdout-23/EVALUATION.md @@ -0,0 +1,58 @@ +# fold-11-holdout-23 Agent Evaluation + +## Scope + +This directory records one sandboxed sub-agent attempt for held-out dataset indices `[23]`. + +- Sandbox: `.runs/fold-11-holdout-23` +- Tracked artifact copy: `evaluations/fold-11-holdout-23` +- Successful agent trace: `AGENT_ACTION_LOG.md` + +Note: the raw `heldout_harmonized.csv` is 116,186,889 bytes, which exceeds +GitHub's 100 MB per-file limit. It is tracked as +`agent_outputs/heldout_harmonized.csv.gz`; see +`agent_outputs/heldout_harmonized_csv_manifest.json` for raw and compressed +artifact hashes. + +## Anti-Leakage Audit + +The successful trace was scanned for root gold data, root processed data, other fold sandboxes, held-out expert module names, and explicit `violation` markers. Result: `0` matches. + +## Output Equivalence Against Gold + +| Metric | Agent | Gold | +| --- | ---: | ---: | +| Rows | 1,722,716 | 912,417 | +| Target columns present | 9 / 9 | 9 / 9 | +| Schema exact match | true | n/a | +| Unique keyed rows recovered | 0 | 912,226 | +| Missing gold keyed rows | 912,226 | n/a | +| Extra agent keyed rows | 1,722,706 | n/a | +| Row precision | 0.000 | n/a | +| Gold recall | 0.000 | n/a | + +Shared-row value matches after key merge: + +| Column | Matched rows | +| --- | ---: | +| `interval_min` | 0/0 | +| `volumetric_water_content_m3_m3` | 0/0 | +| `gravimetric_water_content_gH2O_gs` | 0/0 | +| `water_potential_kPa` | 0/0 | +| `is_timeseries` | 0/0 | + +## JSON Mapping Match + +- Summary: `{"extra_agent_categories": ["gravimetric_water_content"], "field_exact_matches_by_name": {"destination_variable": "8/8", "source_files": "6/8", "source_pattern": "5/8", "transformation": "0/8", "unit_conversion": "6/8"}, "field_level_exact_matches": "25/40", "gold_mapping_categories_present": "8/8", "missing_gold_categories": [], "top_level_exact_matches": "5/7", "top_level_matches": {"archive_repository": true, "data_payload_files": true, "dataset_identifier": true, "doi": true, "index": true, "location_metadata_files": false, "sensor_metadata_files": false}}` + +| Field | Exact matches | +| --- | ---: | +| `destination_variable` | 8/8 | +| `source_pattern` | 5/8 | +| `source_files` | 6/8 | +| `transformation` | 0/8 | +| `unit_conversion` | 6/8 | + +## Overall Assessment + +See `evaluation_metrics.json` for machine-readable details. The run passed the targeted local no-gold path audit. Output equivalence should be interpreted with the key policy recorded in the metrics (`key_includes_dataset_index=False`). diff --git a/evaluations/fold-11-holdout-23/MANIFEST.json b/evaluations/fold-11-holdout-23/MANIFEST.json new file mode 100644 index 0000000..19d600c --- /dev/null +++ b/evaluations/fold-11-holdout-23/MANIFEST.json @@ -0,0 +1,35 @@ +{ + "name": "fold-11-holdout-23", + "holdout_indices": [ + 23 + ], + "holdout_identifiers": [ + "ess-dive-a99be52b7a6114c-20230504T210134503379" + ], + "exemplar_indices": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 15, + 16, + 17, + 18, + 24, + 25, + 26, + 27 + ], + "n_exemplars": 27, + "sources": { + "package": "data/gold/expert_code/harmonize_sm", + "mapping": "data/gold/sm_data_harmonization_mapping.json", + "skills": "skills" + } +} \ No newline at end of file diff --git a/evaluations/fold-11-holdout-23/agent_outputs/curator_bundle.json b/evaluations/fold-11-holdout-23/agent_outputs/curator_bundle.json new file mode 100644 index 0000000..6ef0557 --- /dev/null +++ b/evaluations/fold-11-holdout-23/agent_outputs/curator_bundle.json @@ -0,0 +1,121 @@ +{ + "dataset_identifier": "ess-dive-a99be52b7a6114c-20230504T210134503379", + "doi": "doi:10.15485/1842908", + "title": "Surface soil temperature and water content from warming experiment located at the Rocky Mountain Biological Laboratory, Colorado, 2015 to 2019", + "decision": "INCLUDE", + "decision_reason": "WM_SWC.csv contains direct hourly soil water content observations from in-ground sensors; rows documented as out-of-ground and lacking depth/site metadata are excluded.", + "experimental_context": { + "manipulation_detected": true, + "manipulation_type": "warming experiment", + "evidence": "Package title/description identify a warming experiment; plot_metadata.csv marks plots as control or heated." + }, + "files": { + "data_payload_files": [ + "WM_SWC.csv" + ], + "location_metadata_files": [ + "plot_metadata.csv", + "sensor_metadata.csv" + ], + "sensor_metadata_files": [ + "sensor_metadata.csv" + ], + "documentation_files": [] + }, + "raw_file_summaries": { + "WM_SWC.csv": { + "rows": 1722752, + "columns": [ + "Date Time", + "Plot.Location", + "Point.Location", + "Veg", + "Sensor.SN", + "Sensor.Type", + "Measurement" + ], + "sensor_type_counts": { + "SWC": 1722752 + }, + "timestamp_min": "2015-06-25 00:00:00", + "timestamp_max": "2019-07-17 15:00:00", + "measurement_min": 0.0003, + "measurement_max": 0.5654 + }, + "plot_metadata.csv": { + "rows": 10, + "columns": [ + "Plot.Location", + "Treatment" + ], + "treatments": [ + "control", + "heated" + ] + }, + "sensor_metadata.csv": { + "rows": 105, + "columns": [ + "Logger", + "Sensor Type", + "Sensor.SN", + "Point.Location", + "Plot.Location", + "Zone", + "Depth.cm", + "Veg", + "Notes", + "start", + "stop" + ], + "swc_sensor_rows": 71, + "swc_depth_cm_values": [ + 6.0, + 14.0 + ] + } + }, + "harmonization_plan": { + "included_rows": 1722716, + "excluded_rows": 36, + "excluded_row_reason": "Sensor 9731453 has note \"couldn't find a good spot; left plugged into mantis but out of the ground\" and lacks Point.Location, Veg, and Depth.cm metadata.", + "datetime": "Parse Date Time as fixed Mountain Standard Time (Etc/GMT+7) and convert to UTC ISO-8601 string.", + "site_id": "Construct from resolved Plot.Location, Point.Location, and Veg.", + "depth_m": "Look up Depth.cm by Sensor.SN in sensor_metadata.csv and divide by 100.", + "replicate": "Dense integer rank of Sensor.SN within each site_id and depth_m.", + "volumetric_water_content_m3_m3": "Use Measurement directly; source values are already fractional m3/m3 soil water content.", + "is_timeseries": true, + "interval_min": "Calculate elapsed minutes between consecutive timestamps within each site_id/depth_m/replicate series." + }, + "location_resolution": { + "specific_coordinates_available": false, + "package_spatial_coverage": [ + { + "@type": "Place", + "description": "The East River (ER) is a snow\u2010dominated, headwater basin of the Upper Colorado River Basin (UCRB) located in the western United States. The ER is the designated testbed of Lawrence Berkeley National Laboratory's Watershed Function Scientific Focus Area (WFSFA). Through WFSFA, observational networks have been established to measure stream discharge and precipitation chemistry. The ER is considered representative of many snow\u2010dominated headwaters of the Rocky Mountains. The study domain encompasses nearly 85 square km, a 1.4\u2010km vertical drop in elevation (4,120 to 2,760 m) and pristine alpine, subalpine, montane, and riparian ecosystems. The ER contains high\u2010energy mountain streams to low\u2010energy meandering floodplains and is eroding primarily into the Cretaceous, carbon\u2010rich, marine shale of the Mancos Formation. Additional metadata on specific locations within the watershed are provided in the following related data package: Varadharajan C. et al. (2020) doi:10.15485/1660962", + "geo": [ + { + "@type": "GeoCoordinates", + "name": "Northwest", + "latitude": 39.034, + "longitude": -107.05 + }, + { + "@type": "GeoCoordinates", + "name": "Southeast", + "latitude": 38.88, + "longitude": -106.88 + } + ] + } + ], + "qc_flag_recommendation": "g2", + "reason": "Raw files provide plot/point identifiers but no plot-level latitude/longitude; package metadata provides only a broad East River bounding box." + }, + "temporal_coverage": { + "startDate": "2015-06-25", + "endDate": "2019-07-17", + "@type": "DateTime" + }, + "source_metadata_view_url": "https://data.ess-dive.lbl.gov/view/doi:10.15485/1842908" +} diff --git a/evaluations/fold-11-holdout-23/agent_outputs/harmonize_heldout.py b/evaluations/fold-11-holdout-23/agent_outputs/harmonize_heldout.py new file mode 100644 index 0000000..fe138d8 --- /dev/null +++ b/evaluations/fold-11-holdout-23/agent_outputs/harmonize_heldout.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import pandas as pd + +RUN_DIR = Path('/scratch/jmc/data-harmonization-eval/.runs/fold-11-holdout-23') +RAW_DIR = Path('/h/jmc/ess-dive_wfsfa_soil_datasets/ess-dive-a99be52b7a6114c-20230504T210134503379') +OUT_DIR = RUN_DIR / 'agent_outputs' +LOG_PATH = RUN_DIR / 'AGENT_ACTION_LOG.md' +OUTPUT_PATH = OUT_DIR / 'heldout_harmonized.csv' + +SCHEMA = [ + 'datetime_UTC', + 'site_id', + 'depth_m', + 'replicate', + 'is_timeseries', + 'interval_min', + 'volumetric_water_content_m3_m3', + 'gravimetric_water_content_gH2O_gs', + 'water_potential_kPa', +] + + +def log_action(action: str, subject: Path | str, reason: str) -> None: + ts = datetime.now(timezone.utc).astimezone().isoformat() + with LOG_PATH.open('a') as log: + log.write(f'{ts} | action={action} | subject={subject} | reason={reason}\n') + + +def read_csv_logged(path: Path, reason: str, **kwargs) -> pd.DataFrame: + log_action('file read', path, reason) + return pd.read_csv(path, **kwargs) + + +def local_mst_to_utc_string(series: pd.Series) -> pd.Series: + dt = pd.to_datetime(series, errors='coerce') + utc = dt.dt.tz_localize('Etc/GMT+7').dt.tz_convert('UTC') + return utc.dt.strftime('%Y-%m-%dT%H:%M:%SZ') + + +def main() -> None: + OUT_DIR.mkdir(parents=True, exist_ok=True) + + swc = read_csv_logged(RAW_DIR / 'WM_SWC.csv', 'read held-out SWC payload for harmonization') + sensors = read_csv_logged(RAW_DIR / 'sensor_metadata.csv', 'read held-out sensor metadata for depth/site joins') + + swc = swc.copy() + sensors = sensors.copy() + swc_sensors = sensors.loc[sensors['Sensor Type'].eq('SWC')].copy() + swc['Sensor.SN'] = pd.to_numeric(swc['Sensor.SN'], errors='coerce').astype('Int64') + swc_sensors['Sensor.SN'] = pd.to_numeric(swc_sensors['Sensor.SN'], errors='coerce').round().astype('Int64') + meta_cols = [ + 'Sensor.SN', + 'Point.Location', + 'Plot.Location', + 'Zone', + 'Depth.cm', + 'Veg', + 'Notes', + 'start', + 'stop', + ] + merged = swc.merge(swc_sensors[meta_cols], on='Sensor.SN', how='left', suffixes=('', '.meta')) + + for col in ['Plot.Location', 'Point.Location', 'Veg']: + meta_col = f'{col}.meta' + if meta_col in merged.columns: + merged[f'{col}.resolved'] = merged[meta_col].combine_first(merged[col]) + else: + merged[f'{col}.resolved'] = merged[col] + + merged['depth_m'] = pd.to_numeric(merged['Depth.cm'], errors='coerce') / 100.0 + valid = ( + merged['depth_m'].notna() + & merged['Plot.Location.resolved'].notna() + & merged['Point.Location.resolved'].notna() + & merged['Veg.resolved'].notna() + ) + x = merged.loc[valid].copy() + + x['site_id'] = ( + x['Plot.Location.resolved'].astype(str) + + '_' + + x['Point.Location.resolved'].astype(str) + + '_' + + x['Veg.resolved'].astype(str) + ) + + present_sensors = x[['Sensor.SN', 'site_id', 'depth_m']].drop_duplicates().copy() + present_sensors = present_sensors.sort_values(['site_id', 'depth_m', 'Sensor.SN']) + present_sensors['replicate'] = present_sensors.groupby(['site_id', 'depth_m'], dropna=False).cumcount() + 1 + x = x.merge(present_sensors[['Sensor.SN', 'replicate']], on='Sensor.SN', how='left') + + dt_local = pd.to_datetime(x['Date Time'], errors='coerce') + dt_utc = dt_local.dt.tz_localize('Etc/GMT+7').dt.tz_convert('UTC') + x['datetime_UTC'] = dt_utc.dt.strftime('%Y-%m-%dT%H:%M:%SZ') + x['_dt_utc'] = dt_utc + + x = x.sort_values(['site_id', 'depth_m', 'replicate', '_dt_utc', 'Sensor.SN']).copy() + x['interval_min'] = ( + x.groupby(['site_id', 'depth_m', 'replicate'], dropna=False)['_dt_utc'] + .diff() + .dt.total_seconds() + .div(60.0) + ) + x['is_timeseries'] = True + x['volumetric_water_content_m3_m3'] = pd.to_numeric(x['Measurement'], errors='coerce') + x['gravimetric_water_content_gH2O_gs'] = np.nan + x['water_potential_kPa'] = np.nan + x['replicate'] = pd.to_numeric(x['replicate'], errors='coerce').astype('Int64') + + out = x[SCHEMA].copy() + log_action('write', OUTPUT_PATH, 'write held-out harmonized CSV') + out.to_csv(OUTPUT_PATH, index=False) + + +if __name__ == '__main__': + main() diff --git a/evaluations/fold-11-holdout-23/agent_outputs/heldout_harmonized.csv.gz b/evaluations/fold-11-holdout-23/agent_outputs/heldout_harmonized.csv.gz new file mode 100644 index 0000000..83ce5f4 Binary files /dev/null and b/evaluations/fold-11-holdout-23/agent_outputs/heldout_harmonized.csv.gz differ diff --git a/evaluations/fold-11-holdout-23/agent_outputs/heldout_harmonized_csv_manifest.json b/evaluations/fold-11-holdout-23/agent_outputs/heldout_harmonized_csv_manifest.json new file mode 100644 index 0000000..8443550 --- /dev/null +++ b/evaluations/fold-11-holdout-23/agent_outputs/heldout_harmonized_csv_manifest.json @@ -0,0 +1,10 @@ +{ + "artifact": "heldout_harmonized.csv", + "tracked_as": "heldout_harmonized.csv.gz", + "reason": "Raw CSV is 116186889 bytes, above GitHub's 100 MB per-file limit.", + "raw_size_bytes": 116186889, + "raw_sha256": "ca637ee9e64fa07e8170fd04f029db48fd27a1669e8cae406f4e3905033e0764", + "gzip_size_bytes": 7224816, + "gzip_sha256": "c9ada32726ed6c48d1a3691b7f8d9f2c80da6da0611f40719d21e9554ecb959d", + "local_raw_path": ".runs/fold-11-holdout-23/agent_outputs/heldout_harmonized.csv" +} diff --git a/evaluations/fold-11-holdout-23/agent_outputs/mapping.json b/evaluations/fold-11-holdout-23/agent_outputs/mapping.json new file mode 100644 index 0000000..9bfe7ca --- /dev/null +++ b/evaluations/fold-11-holdout-23/agent_outputs/mapping.json @@ -0,0 +1,111 @@ +{ + "index": 23, + "dataset_identifier": "ess-dive-a99be52b7a6114c-20230504T210134503379", + "doi": "doi:10.15485/1842908", + "archive_repository": "ESS-DIVE", + "data_payload_files": [ + "WM_SWC.csv" + ], + "location_metadata_files": [ + "plot_metadata.csv", + "sensor_metadata.csv" + ], + "sensor_metadata_files": [ + "sensor_metadata.csv" + ], + "harmonization_mappings": { + "datetime": { + "pattern_1": { + "source_pattern": "Date Time", + "source_files": [ + "WM_SWC.csv" + ], + "destination_variable": "datetime_UTC", + "transformation": "Parse logger timestamp as fixed Mountain Standard Time (Etc/GMT+7) and convert to UTC ISO-8601 string.", + "unit_conversion": null + } + }, + "site_id": { + "pattern_1": { + "source_pattern": "Plot.Location, Point.Location, Veg", + "source_files": [ + "WM_SWC.csv", + "sensor_metadata.csv" + ], + "destination_variable": "site_id", + "transformation": "Resolve plot, point, and vegetation class from sensor metadata by Sensor.SN with payload values as fallback; concatenate as Plot_Point_Veg.", + "unit_conversion": null + } + }, + "depth": { + "pattern_1": { + "source_pattern": "Depth.cm", + "source_files": [ + "sensor_metadata.csv" + ], + "destination_variable": "depth_m", + "transformation": "Look up sensor installation depth by Sensor.SN. Exclude rows for sensor 9731453 because metadata states it was out of the ground and no depth is reported.", + "unit_conversion": "Divide by 100 to convert cm to m." + } + }, + "replicate": { + "pattern_1": { + "source_pattern": "Sensor.SN", + "source_files": [ + "WM_SWC.csv", + "sensor_metadata.csv" + ], + "destination_variable": "replicate", + "transformation": "Sort Sensor.SN values within each site_id and depth_m and assign dense integer replicate numbers starting at 1.", + "unit_conversion": null + } + }, + "volumetric_water_content": { + "pattern_1": { + "source_pattern": "Measurement where Sensor.Type == SWC", + "source_files": [ + "WM_SWC.csv" + ], + "destination_variable": "volumetric_water_content_m3_m3", + "transformation": "Coerce Measurement to numeric after retaining valid in-ground SWC sensor rows.", + "unit_conversion": "None; source values are fractional soil water content in m3/m3." + } + }, + "gravimetric_water_content": { + "pattern_1": { + "source_pattern": null, + "source_files": null, + "destination_variable": "gravimetric_water_content_gH2O_gs", + "transformation": "Not reported in source; populate with NA.", + "unit_conversion": null + } + }, + "soil_water_potential": { + "pattern_1": { + "source_pattern": null, + "source_files": null, + "destination_variable": "water_potential_kPa", + "transformation": "Not reported in source; populate with NA.", + "unit_conversion": null + } + }, + "latitude": { + "pattern_1": { + "source_pattern": null, + "source_files": null, + "destination_variable": "latitude", + "transformation": "Specific plot coordinates are not present in raw files or package metadata; package-level metadata only provides an East River bounding box. Recommend location qc_flag g2 if location records are produced.", + "unit_conversion": null + } + }, + "longitude": { + "pattern_1": { + "source_pattern": null, + "source_files": null, + "destination_variable": "longitude", + "transformation": "Specific plot coordinates are not present in raw files or package metadata; package-level metadata only provides an East River bounding box. Recommend location qc_flag g2 if location records are produced.", + "unit_conversion": null + } + } + } +} diff --git a/evaluations/fold-11-holdout-23/agent_outputs/mapping_notes.md b/evaluations/fold-11-holdout-23/agent_outputs/mapping_notes.md new file mode 100644 index 0000000..9162198 --- /dev/null +++ b/evaluations/fold-11-holdout-23/agent_outputs/mapping_notes.md @@ -0,0 +1,13 @@ +# Mapping Notes: dataset 23 + +Dataset: `ess-dive-a99be52b7a6114c-20230504T210134503379` +DOI: `doi:10.15485/1842908` +Decision: INCLUDE + +`WM_SWC.csv` contains direct hourly soil water content measurements (`Sensor.Type == SWC`) from the warming experiment near the Rocky Mountain Biological Laboratory. `plot_metadata.csv` identifies plots as control or heated, so the dataset is included but documented as experimentally manipulated. + +The harmonizer joins `WM_SWC.csv` to `sensor_metadata.csv` by `Sensor.SN`. It constructs `site_id` as `Plot.Location_Point.Location_Veg`, converts `Depth.cm` to `depth_m`, assigns replicate numbers from sorted sensor serials within each site/depth, and uses `Measurement` directly as fractional volumetric water content (`m3/m3`). Timestamps are interpreted as fixed Mountain Standard Time (`Etc/GMT+7`) logger timestamps and written as UTC ISO strings. + +Rows retained: 1722716. Rows excluded: 36. The excluded rows are from sensor `9731453`; the sensor metadata note says it was left plugged in but out of the ground, and it has no point, vegetation, or depth metadata. + +Specific plot coordinates were not available in the raw files. Public package metadata provides only a broad East River bounding box, so plot-level location records would need `qc_flag = g2` unless an allowed external location registry is used. diff --git a/evaluations/fold-11-holdout-23/evaluation_metrics.json b/evaluations/fold-11-holdout-23/evaluation_metrics.json new file mode 100644 index 0000000..6a7a109 --- /dev/null +++ b/evaluations/fold-11-holdout-23/evaluation_metrics.json @@ -0,0 +1,60 @@ +{ + "fold": "fold-11-holdout-23", + "dataset_indices": [ + 23 + ], + "trace_forbidden_path_scan_matches": 0, + "agent_output": { + "rows": 1722716, + "target_columns_present": 9, + "schema_equal": true, + "extra_columns": [] + }, + "gold_output": { + "rows": 912417, + "columns": 9 + }, + "output_equivalence": { + "key_includes_dataset_index": false, + "unique_agent_keys": 1722706, + "unique_gold_keys": 912226, + "key_intersection": 0, + "agent_extra_keys": 1722706, + "agent_missing_gold_keys": 912226, + "common_rows": 0, + "shared_row_value_matches": { + "interval_min": "0/0", + "volumetric_water_content_m3_m3": "0/0", + "gravimetric_water_content_gH2O_gs": "0/0", + "water_potential_kPa": "0/0", + "is_timeseries": "0/0" + }, + "row_precision": 0.0, + "gold_recall": 0.0 + }, + "json_mapping_equivalence": { + "top_level_exact_matches": "5/7", + "top_level_matches": { + "index": true, + "dataset_identifier": true, + "doi": true, + "archive_repository": true, + "data_payload_files": true, + "location_metadata_files": false, + "sensor_metadata_files": false + }, + "gold_mapping_categories_present": "8/8", + "extra_agent_categories": [ + "gravimetric_water_content" + ], + "missing_gold_categories": [], + "field_level_exact_matches": "25/40", + "field_exact_matches_by_name": { + "destination_variable": "8/8", + "source_pattern": "5/8", + "source_files": "6/8", + "transformation": "0/8", + "unit_conversion": "6/8" + } + } +}