From 6201fcc4f0032555d81a38156032b07b8cc73e20 Mon Sep 17 00:00:00 2001
From: FBumann <117816358+FBumann@users.noreply.github.com>
Date: Thu, 23 Jul 2026 08:07:34 +0200
Subject: [PATCH] fix: load pre-7.0 clustered NetCDF files with legacy
 original_data refs

Files written before flixopt 7.0 stored clustering.original_data and
clustering._metrics as ':::original_data|...' / ':::metrics|...' references
in the clustering attrs JSON. The target arrays are no longer serialized
(they only fed the removed plot.compare()), and the current Clustering
constructor rejects the _original_data_refs / _metrics_refs keys outright.

As a result, FlowSystem.from_netcdf() on any such file raised
"Referenced DataArray 'original_data|...' not found in dataset".

Drop the two legacy keys in _restore_clustering before resolving so old
files remain loadable. 7.2.x files themselves are unaffected (they write
no such references).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 flixopt/io.py                               |  7 +++
 tests/test_clustering/test_clustering_io.py | 51 +++++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/flixopt/io.py b/flixopt/io.py
index e812f925d..3d5488404 100644
--- a/flixopt/io.py
+++ b/flixopt/io.py
@@ -1839,6 +1839,13 @@ def _restore_clustering(
             return
 
         clustering_structure = json.loads(reference_structure['clustering'])
+        # Backward-compat: files written before flixopt 7.0 stored clustering.original_data
+        # and clustering._metrics as ':::original_data|...' / ':::metrics|...' references whose
+        # target arrays are no longer serialized (they only fed the removed plot.compare()).
+        # These keys are also not accepted by the current Clustering.__init__. Drop them so
+        # such files remain loadable.
+        clustering_structure.pop('_original_data_refs', None)
+        clustering_structure.pop('_metrics_refs', None)
         clustering = fs_cls._resolve_reference_structure(clustering_structure, {})
         flow_system.clustering = clustering
 
diff --git a/tests/test_clustering/test_clustering_io.py b/tests/test_clustering/test_clustering_io.py
index e1399cf0a..e96468eb1 100644
--- a/tests/test_clustering/test_clustering_io.py
+++ b/tests/test_clustering/test_clustering_io.py
@@ -710,3 +710,54 @@ def test_expand_after_load_and_optimize(self, system_with_periods_and_scenarios,
         # Solution should be expanded
         assert fs_expanded.solution is not None
         assert 'source(out)|flow_rate' in fs_expanded.solution
+
+
+class TestLegacyClusteringBackwardCompat:
+    """Loading files written before flixopt 7.0, which stored clustering.original_data
+    and clustering._metrics as ':::original_data|...' / ':::metrics|...' references whose
+    target arrays are no longer serialized. See hotfix 7.2.2."""
+
+    def _inject_legacy_refs(self, ds: xr.Dataset) -> xr.Dataset:
+        """Mutate a clustered dataset's clustering attrs to look like a pre-7.0 file."""
+        import json
+
+        clustering = json.loads(ds.attrs['clustering'])
+        clustering['_original_data_refs'] = [
+            ':::original_data|demand(in)|fixed_relative_profile',
+        ]
+        clustering['_metrics_refs'] = None
+        ds.attrs['clustering'] = json.dumps(clustering, ensure_ascii=False)
+        return ds
+
+    def test_legacy_refs_reproduce_failure_without_shim(self, simple_system_8_days):
+        """Sanity check: the legacy references do point at arrays absent from the dataset."""
+        fs_clustered = simple_system_8_days.transform.cluster(n_clusters=2, cluster_duration='1D')
+        ds = self._inject_legacy_refs(fs_clustered.to_dataset(include_solution=False))
+        assert 'original_data|demand(in)|fixed_relative_profile' not in ds.variables
+
+    def test_load_legacy_clustered_dataset(self, simple_system_8_days):
+        """A pre-7.0 clustered dataset (with dangling original_data/metrics refs) loads cleanly."""
+        from flixopt.clustering import Clustering
+
+        fs_clustered = simple_system_8_days.transform.cluster(n_clusters=2, cluster_duration='1D')
+        ds = self._inject_legacy_refs(fs_clustered.to_dataset(include_solution=False))
+
+        fs_restored = fx.FlowSystem.from_dataset(ds)
+
+        assert isinstance(fs_restored.clustering, Clustering)
+        assert fs_restored.clustering.n_clusters == 2
+
+    def test_load_legacy_clustered_netcdf(self, simple_system_8_days, tmp_path):
+        """Same as above but through a real NetCDF file roundtrip."""
+        from flixopt.clustering import Clustering
+        from flixopt.io import save_dataset_to_netcdf
+
+        fs_clustered = simple_system_8_days.transform.cluster(n_clusters=2, cluster_duration='1D')
+        ds = self._inject_legacy_refs(fs_clustered.to_dataset(include_solution=False))
+
+        nc_path = tmp_path / 'legacy_clustered.nc'
+        save_dataset_to_netcdf(ds, nc_path)
+        fs_restored = fx.FlowSystem.from_netcdf(nc_path)
+
+        assert isinstance(fs_restored.clustering, Clustering)
+        assert fs_restored.clustering.n_clusters == 2