Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
75bb535
feat: add streaming write API for iterator-based array writing
theokb7 Mar 9, 2026
3665d74
feat: add xarray Dataset write support
theokb7 Mar 9, 2026
7fdaad5
feat: add dask array integration
theokb7 Mar 9, 2026
c3aafdc
refactor: Remove `resource` import
theokb7 Mar 11, 2026
ca8b232
feat: Add memory usage test
theokb7 Mar 11, 2026
b1f2664
feat: add and test fsspec xarray support
theokb7 Mar 16, 2026
55497a9
Merge branch 'main' into feat/streaming-write-and-dataset-api
terraputix Mar 19, 2026
a423c24
fix linter
terraputix Mar 19, 2026
0dc5035
revert unnecessary changes on this branch
terraputix Mar 24, 2026
c4a7879
more
terraputix Mar 24, 2026
18157b3
remove dask related changes
terraputix Mar 24, 2026
e88fca0
failable import not needed
terraputix Mar 24, 2026
ee8354f
Merge branch 'main' into xarray-write-support
terraputix Mar 25, 2026
96abe58
Revert "remove dask related changes"
terraputix Mar 25, 2026
cffd0fb
minor fixes
terraputix Mar 25, 2026
89b2781
Merge branch 'main' into xarray-write-support
terraputix Apr 2, 2026
a342a13
Merge branch 'main' into xarray-write-support
terraputix Apr 10, 2026
06782fd
Merge branch 'main' into xarray-write-support
terraputix Apr 17, 2026
e088d4e
Merge branch 'main' into xarray-write-support
terraputix Apr 21, 2026
ef9d495
fix: separator
terraputix Apr 21, 2026
9686457
Merge branch 'main' into xarray-write-support
terraputix Jul 21, 2026
0b6572d
move _validate_chunk_alignment to separate private module so xarray d…
terraputix Jul 21, 2026
97f29ff
cache xarray metadata discovery
terraputix Jul 21, 2026
a84ab99
pipe through scalar attributes
terraputix Jul 21, 2026
eac5353
Merge branch 'main' into xarray-write-support
terraputix Jul 21, 2026
565edee
remove _COORDINATE_VARIABLES
terraputix Jul 21, 2026
3daea40
align coordinate metadata with open-meteo conventions
terraputix Jul 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions python/omfiles/_chunk_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import math


def _validate_chunk_alignment(
data_chunks: tuple,
om_chunks: list[int],
array_shape: tuple,
) -> None:
"""
Validate chunked array blocks for compatibility with OM chunk streaming.

Every non-last data chunk along each dimension must be an exact multiple
of the corresponding OM chunk size (the last chunk may be smaller).
Additionally, for the leftmost dimension where a data block contains more
than one OM chunk, every trailing dimension must be fully covered by each
data block. This ensures the local chunk traversal inside a block matches
the global file order.
"""
ndim = len(om_chunks)

for d in range(ndim):
dim_chunks = data_chunks[d]
for i, c in enumerate(dim_chunks[:-1]):
if c % om_chunks[d] != 0:
raise ValueError(
f"Dask chunk size {c} along dimension {d} (block {i}) "
f"is not a multiple of the OM chunk size {om_chunks[d]}."
)

first_multi = None
for d in range(ndim):
local_n = max(math.ceil(c / om_chunks[d]) for c in data_chunks[d])
if local_n > 1:
first_multi = d
break

if first_multi is not None:
for d in range(first_multi + 1, ndim):
dim_chunks = data_chunks[d]
if not (len(dim_chunks) == 1 and dim_chunks[0] == array_shape[d]):
raise ValueError(
f"Dask blocks have multiple OM chunks in dimension {first_multi}, "
f"but dimension {d} is not fully covered by each dask block "
f"(dask chunks {dim_chunks} vs array size {array_shape[d]}). "
f"Rechunk so trailing dimensions are fully covered."
)
47 changes: 1 addition & 46 deletions python/omfiles/dask.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
"""Dask array integration for writing to OM files."""

import math
from typing import Iterator, Optional, Sequence

import numpy as np

from omfiles._chunk_utils import _validate_chunk_alignment
from omfiles._rust import OmFileWriter, OmWriterVariable

try:
Expand All @@ -13,51 +13,6 @@
raise ImportError("omfiles[dask] is required for dask functionality")


def _validate_chunk_alignment(
data_chunks: tuple,
om_chunks: list[int],
array_shape: tuple,
) -> None:
"""
Validate dask chunks are compatible with OM chunks for block-level streaming.

Every non-last dask chunk along each dimension must be an exact multiple
of the corresponding OM chunk size (the last chunk may be smaller).
Additionally, for the leftmost dimension where a dask block contains more
than one OM chunk, every trailing dimension must be fully covered by each
dask block. This ensures the local chunk traversal inside a block matches
the global file order.
"""
ndim = len(om_chunks)

for d in range(ndim):
dim_chunks = data_chunks[d]
for i, c in enumerate(dim_chunks[:-1]):
if c % om_chunks[d] != 0:
raise ValueError(
f"Dask chunk size {c} along dimension {d} (block {i}) "
f"is not a multiple of the OM chunk size {om_chunks[d]}."
)

first_multi = None
for d in range(ndim):
local_n = max(math.ceil(c / om_chunks[d]) for c in data_chunks[d])
if local_n > 1:
first_multi = d
break

if first_multi is not None:
for d in range(first_multi + 1, ndim):
dim_chunks = data_chunks[d]
if not (len(dim_chunks) == 1 and dim_chunks[0] == array_shape[d]):
raise ValueError(
f"Dask blocks have multiple OM chunks in dimension {first_multi}, "
f"but dimension {d} is not fully covered by each dask block "
f"(dask chunks {dim_chunks} vs array size {array_shape[d]}). "
f"Rechunk so trailing dimensions are fully covered."
)


def _dask_block_iterator(dask_array: da.Array) -> Iterator[np.ndarray]:
"""
Yield computed numpy arrays from a dask array in C-order block traversal.
Expand Down
Loading