From bf73cf02b521f1ea0ea5ca1e9b4f26236f17b2b7 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 6 Jul 2026 15:20:22 +0200 Subject: [PATCH 1/2] Add hfb_pwn dataset and doorsnedes GeoJSON to mockup data Register hfb_pwn (horizontal flow barriers / sheet pile walls, v1.0.0) in repository.yaml and add its mockup (hfb_pwn.geojson + generator). Add doorsnedes.geojson (converted from doorsnedes.gpkg, EPSG:28992) to the doorsnedes_nh mockup so model scripts can read the cross-section lines as GeoJSON. Add test_geojson_content.py asserting required property columns and a CRS for the drains_pwn and hfb_pwn GeoJSON artifacts. Fix the yamale extent rule (length= is ignored; use min/max=4) and normalize path separators in the mockup folder-structure test so it also passes on Windows. --- .../doorsnedes_nh/v1.0.0/doorsnedes.geojson | 12 ++++ .../mockup/hfb_pwn/v1.0.0/hfb_pwn.geojson | 11 ++++ .../data/mockup/hfb_pwn/v1.0.0/hfb_pwn.py | 64 +++++++++++++++++++ src/nhflodata/data/repository.yaml | 21 ++++++ tests/schema_repository.yaml | 3 +- tests/test_geojson_content.py | 49 ++++++++++++++ tests/test_mockup_folder_structure.py | 4 +- 7 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 src/nhflodata/data/mockup/doorsnedes_nh/v1.0.0/doorsnedes.geojson create mode 100644 src/nhflodata/data/mockup/hfb_pwn/v1.0.0/hfb_pwn.geojson create mode 100644 src/nhflodata/data/mockup/hfb_pwn/v1.0.0/hfb_pwn.py create mode 100644 tests/test_geojson_content.py diff --git a/src/nhflodata/data/mockup/doorsnedes_nh/v1.0.0/doorsnedes.geojson b/src/nhflodata/data/mockup/doorsnedes_nh/v1.0.0/doorsnedes.geojson new file mode 100644 index 0000000..c8a9e1b --- /dev/null +++ b/src/nhflodata/data/mockup/doorsnedes_nh/v1.0.0/doorsnedes.geojson @@ -0,0 +1,12 @@ +{ +"type": "FeatureCollection", +"name": "doorsnedes", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::28992" } }, +"features": [ +{ "type": "Feature", "properties": { "index": "A", "name": "A" }, "geometry": { "type": "LineString", "coordinates": [ [ 96500.0, 502701.0 ], [ 151000.0, 502701.0 ] ] } }, +{ "type": "Feature", "properties": { "index": "B", "name": "B" }, "geometry": { "type": "LineString", "coordinates": [ [ 96500.0, 529001.0 ], [ 151000.0, 529001.0 ] ] } }, +{ "type": "Feature", "properties": { "index": "C", "name": "C" }, "geometry": { "type": "LineString", "coordinates": [ [ 121001.0, 555000.0 ], [ 121001.0, 487000.0 ] ] } }, +{ "type": "Feature", "properties": { "index": "D", "name": "D" }, "geometry": { "type": "LineString", "coordinates": [ [ 130273.0, 552068.0 ], [ 130273.0, 527871.0 ] ] } }, +{ "type": "Feature", "properties": { "index": "E", "name": "E" }, "geometry": { "type": "LineString", "coordinates": [ [ 116548.0, 539289.0 ], [ 141011.0, 539289.0 ] ] } } +] +} diff --git a/src/nhflodata/data/mockup/hfb_pwn/v1.0.0/hfb_pwn.geojson b/src/nhflodata/data/mockup/hfb_pwn/v1.0.0/hfb_pwn.geojson new file mode 100644 index 0000000..bee9ad2 --- /dev/null +++ b/src/nhflodata/data/mockup/hfb_pwn/v1.0.0/hfb_pwn.geojson @@ -0,0 +1,11 @@ +{ +"type": "FeatureCollection", +"name": "hfb_pwn", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::28992" } }, +"xy_coordinate_resolution": 0.01, +"bbox": [ 107034.61, 521536.91, 107511.14, 521642.41 ], +"features": [ +{ "type": "Feature", "properties": { "name": "damwand_pompgebouw", "hydchr": 0.01, "depth": 5.0, "elevation": null, "comment": "" }, "bbox": [ 107280.34, 521589.56, 107511.14, 521619.63 ], "geometry": { "type": "LineString", "coordinates": [ [ 107280.34, 521589.56 ], [ 107511.14, 521619.63 ] ] } }, +{ "type": "Feature", "properties": { "name": "damwand_dwk", "hydchr": 0.01, "depth": 5.0, "elevation": null, "comment": "tot in veenlaag (0mNAP) OF SDL dieper (-5mNAP)" }, "bbox": [ 107034.61, 521536.91, 107280.34, 521642.41 ], "geometry": { "type": "LineString", "coordinates": [ [ 107034.61, 521642.41 ], [ 107058.75, 521592.47 ], [ 107070.99, 521536.91 ], [ 107280.34, 521589.56 ] ] } } +] +} diff --git a/src/nhflodata/data/mockup/hfb_pwn/v1.0.0/hfb_pwn.py b/src/nhflodata/data/mockup/hfb_pwn/v1.0.0/hfb_pwn.py new file mode 100644 index 0000000..cd4f95c --- /dev/null +++ b/src/nhflodata/data/mockup/hfb_pwn/v1.0.0/hfb_pwn.py @@ -0,0 +1,64 @@ +from pathlib import Path + +import geopandas as gpd +import numpy as np +from nhflotools.geoconverter.utils import round_bounds, validate_crs, validate_geometry + +from nhflodata import get_abs_data_path + +# Properties are those consumed by nlmod.gwf.hfb.get_hfb_spd(ds, linestrings, hydchr, +# depth=None, elevation=None), which places a horizontal flow barrier on the cell faces +# crossed by the line and spans the model layers vertically: +# - hydchr: hydraulic characteristic of the barrier [1/day]; 1/100 == resistance of 100 days. +# - depth: depth of the barrier below groundlevel [m, positive]; the top is at groundlevel. +# - elevation: alternative to depth: absolute elevation of the barrier BOTTOM [m NAP]. +# Use either depth or elevation (not both); here depth is populated and elevation left null. +columns_to_keep = [ + "name", + "hydchr", + "depth", + "elevation", + "comment", + "geometry", +] + +# %% Create horizontal flow barrier (sheet pile wall / "damwand") input +# The damwand features live in the same source dataset as the drains. +gdf_dir = get_abs_data_path("drains_en_damwand_bergen") +gdf = gpd.read_file(gdf_dir / "drains_en_damwand_bergen.geojson") +gdf = gdf.loc[gdf.naam.str.startswith("damwand")] +gdf = gdf.rename(columns={"naam": "name", "opmerking": "comment"}) +gdf["comment"] = gdf["comment"].fillna("") + +# diepte_van/diepte_tot are DEPTHS below groundlevel (negative = downward), so the barrier is +# defined by its construction depth, not by an absolute elevation. `depth` is the deepest extent +# below groundlevel as a POSITIVE number of metres -- exactly what nlmod.gwf.hfb.get_hfb_spd +# expects: the barrier top sits at groundlevel and extends `depth` metres straight down. +# `elevation` (absolute m NAP) is therefore left null; get_hfb_spd requires exactly one of the two. +# +# Deliberate modelling choice: representing the wall by construction depth (not an absolute tie-in) +# means that where the ground surface is high (dunes; AHN up to ~7.5 m NAP along this alignment) +# the wall bottom stays above the veen/SDL aquitard (0/-5 m NAP) and provides little cutoff there. +# This is intended -- the damwand is its physical 5 m of sheet pile, not forced down to the aquitard. +gdf["depth"] = -gdf[["diepte_van", "diepte_tot"]].min(axis=1) +gdf["elevation"] = np.nan + +# Hydraulic characteristic [1/day]. Bergen model (04v2pwnbergenmodel) used 1/100, +# i.e. a resistance of 100 days for a unit gradient. +gdf["hydchr"] = 1.0 / 100.0 + +gdf = gdf[columns_to_keep] + +# Checks +validate_crs(gdf) +for idx, geom in enumerate(gdf.geometry): + is_valid, error = validate_geometry(geom) + if not is_valid: + msg = f"Geometry at index {idx} is invalid: {error}" + raise ValueError(msg) + +out_geojson = Path(__file__).parent / "hfb_pwn.geojson" +gdf[columns_to_keep].to_file(out_geojson, driver="GeoJSON", write_bbox=True, coordinate_precision=2) + +bounds = dict(zip(["minx", "miny", "maxx", "maxy"], gdf.total_bounds, strict=False)) +print("extent:", round_bounds(bounds, rounding_interval=1000.0).values()) # noqa: T201 diff --git a/src/nhflodata/data/repository.yaml b/src/nhflodata/data/repository.yaml index de52be4..1d39abe 100644 --- a/src/nhflodata/data/repository.yaml +++ b/src/nhflodata/data/repository.yaml @@ -45,6 +45,27 @@ data: changelog: previous_version: 0.0.0 log: Initial version of drains of PWN. Uses mostly data from drains_en_damwand_bergen v1.0.0. + hfb_pwn: + - version_nhflo: 1.0.0 + owner: PWN + publication_date: "2026-07-01" + version_owner: 1.0.0 + description_short: Horizontal flow barriers (sheet pile walls) of PWN + description_long: Horizontal flow barriers (sheet pile walls / damwanden) of PWN, near pompstation Bergen + contact: bas.des.tombe@pwn.nl + timezone: Europe/Amsterdam + extent: + - 107000.0 + - 521000.0 + - 108000.0 + - 522000.0 + paths: + local: hfb_pwn/v1.0.0 + nhflo_server: /data/hfb_pwn/v1.0.0 + mockup: mockup/hfb_pwn/v1.0.0 + changelog: + previous_version: 0.0.0 + log: Initial version of horizontal flow barriers of PWN. Uses mostly data from drains_en_damwand_bergen v1.0.0. rivers_pwn: - version_nhflo: 1.0.0 owner: PWN diff --git a/tests/schema_repository.yaml b/tests/schema_repository.yaml index 636dfb8..66362a5 100644 --- a/tests/schema_repository.yaml +++ b/tests/schema_repository.yaml @@ -44,7 +44,8 @@ dataset_version: timezone: str(required=True) # Spatial extent [minx, miny, maxx, maxy] - extent: list(num(), length=4, required=True) + # Note: yamale ignores `length=` on list(); use min/max to enforce exactly 4 elements. + extent: list(num(), min=4, max=4, required=True) # Path information paths: include('paths', required=True) diff --git a/tests/test_geojson_content.py b/tests/test_geojson_content.py new file mode 100644 index 0000000..31cd50d --- /dev/null +++ b/tests/test_geojson_content.py @@ -0,0 +1,49 @@ +"""Content contract for the committed GeoJSON artifacts of the drain/HFB datasets. + +The registry test (test_repository_yaml) and the folder test (test_mockup_folder_structure) +verify metadata and folder structure but never open the GeoJSON files. This lightweight, +dependency-free (stdlib json) check verifies that the committed artifacts consumers rely on are +structurally sound: the expected file exists, has non-empty features, declares a CRS, and every +feature carries the property columns its model consumer reads. It deliberately does NOT assert +specific data values, which are allowed to change for a data repository. +""" + +import json + +import pytest + +from nhflodata.get_paths import get_abs_data_path + +# Required property columns per dataset, matching what the model consumers read. +REQUIRED_COLUMNS = { + "drains_pwn": { + "name", + "elevation", + "conductance_per_meter", + "conductance_per_squared_meter", + "bgt-identificatie", + "mover_lake_name", + "comment", + }, + "hfb_pwn": {"name", "hydchr", "elevation", "depth", "comment"}, +} + + +@pytest.mark.parametrize("name", sorted(REQUIRED_COLUMNS)) +def test_geojson_content_contract(name): + """Committed .geojson has non-empty features, a CRS, and all required columns.""" + path = get_abs_data_path(name=name, version="latest", location="mockup") + geojson_file = path / f"{name}.geojson" + assert geojson_file.exists(), f"{geojson_file} is missing" + + with open(geojson_file, encoding="utf-8") as f: + gj = json.load(f) + + features = gj.get("features", []) + assert features, f"{name}.geojson has no features" + assert gj.get("crs"), f"{name}.geojson has no CRS" + + required = REQUIRED_COLUMNS[name] + for i, feat in enumerate(features): + missing = required - set(feat.get("properties", {})) + assert not missing, f"{name}.geojson feature {i} is missing columns: {sorted(missing)}" diff --git a/tests/test_mockup_folder_structure.py b/tests/test_mockup_folder_structure.py index 21b4212..4348a5e 100644 --- a/tests/test_mockup_folder_structure.py +++ b/tests/test_mockup_folder_structure.py @@ -38,7 +38,9 @@ def test_mockup_folder_structure(): location="mockup", ) ) - if not path.endswith(version["paths"]["mockup"]): + # Normalize separators so the comparison also holds on Windows, where + # str(Path(...)) uses backslashes while the mockup path literal uses "/". + if not os.path.normpath(path).endswith(os.path.normpath(version["paths"]["mockup"])): pytest.fail(f"Path in repository.yaml does not match the expected path: {version['paths']['mockup']}") assert os.path.exists(path), f"Path {path} does not exist" From 92e0e896e3ac531b0839d92b4ae3054f3831b8de Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 6 Jul 2026 16:21:52 +0200 Subject: [PATCH 2/2] Sort imports in get_paths.py to satisfy ruff I001 Pre-existing lint failure surfaced by this PR's lint job; groups the stdlib zoneinfo import with the other stdlib imports. No behaviour change. --- src/nhflodata/get_paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nhflodata/get_paths.py b/src/nhflodata/get_paths.py index 9389255..2d65038 100644 --- a/src/nhflodata/get_paths.py +++ b/src/nhflodata/get_paths.py @@ -7,10 +7,10 @@ import re from datetime import datetime from pathlib import Path -from zoneinfo import ZoneInfo import yaml from yaml.loader import SafeLoader +from zoneinfo import ZoneInfo logger = logging.getLogger(__name__)