From aeebdb3208012804bd88b4d92a2caf057b590ee7 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 23 Jul 2026 13:57:08 +0200 Subject: [PATCH 1/2] fix: dimension-aware effect-total consistency check The post-solve validation in _create_effects_dataset compared ds[effect].sum(...) against solution[label] via np.allclose on the raw .values. When the two arrays carry the same dims in a different order -- e.g. a clustered system expanded back yields (scenario, period) while the computed total is (period, scenario) -- numpy cannot broadcast (3,2) against (2,3) and raises ValueError, turning a soft warning into a hard crash. Align the two label-aware before comparing: warn on a genuine dimension-set mismatch, otherwise transpose solution[label] to the computed dim order. Applies to both copies of the check (StatisticsAccessor and Results). Co-Authored-By: Claude Opus 4.8 --- flixopt/results.py | 11 ++++- flixopt/statistics_accessor.py | 11 ++++- tests/test_effects_dataset_validation.py | 53 ++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 tests/test_effects_dataset_validation.py diff --git a/flixopt/results.py b/flixopt/results.py index 8ec860244..e0518e499 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -966,7 +966,16 @@ def _create_effects_dataset(self, mode: Literal['temporal', 'periodic', 'total'] label = f'{effect}{suffix[mode]}' computed = ds[effect].sum('component') found = self.solution[label] - if not np.allclose(computed.values, found.fillna(0).values): + if set(computed.dims) != set(found.dims): + logger.critical( + f'Results for {effect}({mode}) in effects_dataset doesnt match {label}: ' + f'dimension mismatch {computed.dims=} vs {found.dims=}' + ) + elif not np.allclose( + computed.fillna(0).values, + found.transpose(*computed.dims).fillna(0).values, + equal_nan=True, + ): logger.critical( f'Results for {effect}({mode}) in effects_dataset doesnt match {label}\n{computed=}\n, {found=}' ) diff --git a/flixopt/statistics_accessor.py b/flixopt/statistics_accessor.py index 3a1438ba8..3e705261e 100644 --- a/flixopt/statistics_accessor.py +++ b/flixopt/statistics_accessor.py @@ -935,7 +935,16 @@ def get_contributor_type(contributor: str) -> str: if label in solution: computed = ds[effect].sum('contributor') found = solution[label] - if not np.allclose(computed.fillna(0).values, found.fillna(0).values, equal_nan=True): + if set(computed.dims) != set(found.dims): + logger.critical( + f'Results for {effect}({mode}) in effects_dataset doesnt match {label}: ' + f'dimension mismatch {computed.dims=} vs {found.dims=}' + ) + elif not np.allclose( + computed.fillna(0).values, + found.transpose(*computed.dims).fillna(0).values, + equal_nan=True, + ): logger.critical( f'Results for {effect}({mode}) in effects_dataset doesnt match {label}\n{computed=}\n, {found=}' ) diff --git a/tests/test_effects_dataset_validation.py b/tests/test_effects_dataset_validation.py new file mode 100644 index 000000000..d8073009b --- /dev/null +++ b/tests/test_effects_dataset_validation.py @@ -0,0 +1,53 @@ +"""Regression tests for the effect-total consistency check in ``_create_effects_dataset``. + +The check compares ``ds[effect].sum(...)`` against ``solution[label]``. When those two +arrays carry the same dimensions in a different order (e.g. a clustered system expanded +back produces ``(scenario, period)`` where the computed total is ``(period, scenario)``), +comparing the raw ``.values`` with ``np.allclose`` used to raise a ``ValueError`` on the +shape mismatch -- turning a soft warning into a hard crash. The comparison must instead be +dimension-aware. +""" + +import numpy as np +import pandas as pd +import pytest + +import flixopt as fx + + +@pytest.fixture +def solved_multiperiod_scenario_system(): + """A solved system with 3 periods x 2 scenarios (different sizes on purpose).""" + timesteps = pd.date_range('2024-01-01', periods=24, freq='h') + periods = pd.Index([2024, 2025, 2026], name='period') + scenarios = pd.Index(['high', 'low'], name='scenario') + + fs = fx.FlowSystem(timesteps, periods=periods, scenarios=scenarios) + fs.add_elements( + fx.Bus('heat'), + fx.Effect('costs', unit='EUR', description='costs', is_objective=True, is_standard=True), + fx.Sink('demand', inputs=[fx.Flow('in', bus='heat', fixed_relative_profile=np.ones(24), size=10)]), + fx.Source('source', outputs=[fx.Flow('out', bus='heat', size=50, effects_per_flow_hour={'costs': 0.05})]), + ) + fs.optimize(fx.solvers.HighsSolver(mip_gap=0, time_limit_seconds=60, log_to_console=False)) + return fs + + +@pytest.mark.parametrize('mode', ['temporal', 'periodic', 'total']) +def test_effects_dataset_tolerates_transposed_solution_dims(solved_multiperiod_scenario_system, mode): + """Transposing an effect total in the solution must not crash the validation.""" + fs = solved_multiperiod_scenario_system + suffix = {'temporal': '(temporal)|per_timestep', 'periodic': '(periodic)', 'total': ''}[mode] + label = f'costs{suffix}' + assert label in fs.solution + + found = fs.solution[label] + transposable = [d for d in found.dims if d != 'time'] + assert len(transposable) >= 2 # need >=2 non-time dims to reorder into a shape mismatch + + # Force the opposite dim order the bug is about. + reordered = [d for d in reversed(found.dims)] + fs._solution[label] = found.transpose(*reordered) + + ds = fs.stats._create_effects_dataset(mode) + assert 'costs' in ds From 11ae5dcb14c1a5958db7dfe6b50e496d40aa6e0a Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:31:46 +0200 Subject: [PATCH 2/2] test: clustered FlowSystem save/load/expand round-trip matrix Curated from property-based fuzzing that ran save -> load -> expand over random combinations of periods (0-3), scenarios (0-3), storage (all initial-charge modes), converters and optimize on/off, same-version and cross-version (7.0.0/7.1.0 -> 7.2.3). No round-trip failures were found; these 24 cases pin the representative dimension layouts so the path stays intact, and assert the effect-total validation does not crash on any layout. Co-Authored-By: Claude Opus 4.8 --- .../test_clustered_roundtrip.py | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 tests/test_clustering/test_clustered_roundtrip.py diff --git a/tests/test_clustering/test_clustered_roundtrip.py b/tests/test_clustering/test_clustered_roundtrip.py new file mode 100644 index 000000000..9e8afbcac --- /dev/null +++ b/tests/test_clustering/test_clustered_roundtrip.py @@ -0,0 +1,118 @@ +"""Round-trip regression matrix for clustered FlowSystems. + +Derived from property-based fuzzing (save -> load -> expand) that exercised random +combinations of periods, scenarios, storage (all initial-charge modes), converters and +optimize on/off. These curated cases pin the representative combinations so the +save/load/expand path stays intact across dimension layouts. +""" + +import numpy as np +import pandas as pd +import pytest + +import flixopt as fx + +SOLVER = fx.solvers.HighsSolver(mip_gap=0, time_limit_seconds=60, log_to_console=False) + + +def _build_system(n_days, periods, scenarios, storage, storage_init, converter): + hours = n_days * 24 + timesteps = pd.date_range('2024-01-01', periods=hours, freq='h') + period_idx = pd.Index(periods, name='period') if periods else None + scenario_idx = pd.Index(scenarios, name='scenario') if scenarios else None + + daily = 0.5 + 0.5 * np.sin(np.linspace(0, 2 * np.pi, 24)) + profile = np.clip(np.tile(daily, n_days) * 0.9, 0.01, None) + + extra = {'weight_of_last_period': 1.0} if period_idx is not None else {} + fs = fx.FlowSystem(timesteps, periods=period_idx, scenarios=scenario_idx, **extra) + elements = [ + fx.Bus('heat'), + fx.Effect('costs', 'EUR', 'costs', is_objective=True, is_standard=True), + fx.Sink('demand', inputs=[fx.Flow('in', bus='heat', fixed_relative_profile=profile, size=10)]), + fx.Source('grid', outputs=[fx.Flow('g', bus='heat', size=1000, effects_per_flow_hour={'costs': 0.1})]), + ] + if converter: + elements += [ + fx.Bus('gas'), + fx.Source('gas_src', outputs=[fx.Flow('gs', bus='gas', size=1000, effects_per_flow_hour={'costs': 0.05})]), + fx.LinearConverter( + 'boiler', + inputs=[fx.Flow('bi', bus='gas', size=1000)], + outputs=[fx.Flow('bo', bus='heat', size=1000)], + conversion_factors=[{'bi': 1.0, 'bo': 0.9}], + ), + ] + if storage: + init = {'equals_final': 'equals_final', 'none': None, 'value': 2.0}[storage_init] + elements.append( + fx.Storage( + 'battery', + charging=fx.Flow('sc', bus='heat', size=10), + discharging=fx.Flow('sd', bus='heat', size=10), + capacity_in_flow_hours=20, + initial_charge_state=init, + ) + ) + fs.add_elements(*elements) + return fs + + +# (label, n_days, periods, scenarios, storage, storage_init, converter, n_clusters) +CONFIGS = [ + ('plain', 3, None, None, False, 'none', False, 2), + ('periods', 4, [2020, 2025], None, False, 'none', False, 2), + ('scenarios', 4, None, ['a', 'b'], False, 'none', False, 2), + ('periods+scenarios', 3, [2020, 2025, 2030], ['a', 'b'], False, 'none', False, 2), + ('storage-cyclic', 4, None, None, True, 'equals_final', False, 2), + ('storage-free', 4, None, None, True, 'none', False, 2), + ('storage-value', 4, None, None, True, 'value', False, 3), + ('converter', 3, None, None, False, 'none', True, 2), + ('storage+conv+scen', 4, None, ['a', 'b'], True, 'equals_final', True, 2), + ('storage+periods', 4, [2020, 2025], None, True, 'value', False, 2), + ('k1', 3, None, None, True, 'equals_final', False, 1), + ('k-max', 3, None, None, False, 'none', False, 3), +] + + +@pytest.mark.parametrize('cfg', CONFIGS, ids=[c[0] for c in CONFIGS]) +def test_clustered_roundtrip_and_expand(cfg, tmp_path): + label, n_days, periods, scenarios, storage, storage_init, converter, n_clusters = cfg + fs = _build_system(n_days, periods, scenarios, storage, storage_init, converter) + clustered = fs.transform.cluster(n_clusters=n_clusters, cluster_duration='1D') + clustered.optimize(SOLVER) + + costs_before = float(clustered.solution['costs'].sum().item()) + + path = tmp_path / f'{label}.nc4' + clustered.to_netcdf(path) + reloaded = fx.FlowSystem.from_netcdf(path) + + assert reloaded.clustering is not None + assert reloaded.clustering.n_clusters == n_clusters + assert len(reloaded.timesteps) == len(clustered.timesteps) + + costs_after = float(reloaded.solution['costs'].sum().item()) + assert np.isclose(costs_before, costs_after, rtol=1e-6, atol=1e-4) + + # effect-total validation must not crash on any dimension layout + reloaded.stats._create_effects_dataset('total') + + expanded = reloaded.transform.expand() + assert len(expanded.timesteps) == n_days * 24 + + +@pytest.mark.parametrize('cfg', CONFIGS, ids=[c[0] for c in CONFIGS]) +def test_clustered_structure_only_roundtrip(cfg, tmp_path): + """Round-trip of an unsolved (no-solution) clustered system must also reload.""" + label, n_days, periods, scenarios, storage, storage_init, converter, n_clusters = cfg + fs = _build_system(n_days, periods, scenarios, storage, storage_init, converter) + clustered = fs.transform.cluster(n_clusters=n_clusters, cluster_duration='1D') + + path = tmp_path / f'{label}_structure.nc4' + clustered.to_netcdf(path) + reloaded = fx.FlowSystem.from_netcdf(path) + + assert reloaded.clustering is not None + assert reloaded.clustering.n_clusters == n_clusters + assert len(reloaded.timesteps) == len(clustered.timesteps)