From c366e9613f89961f10639f3ae0526276d55796b2 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 8 Jul 2026 10:32:30 -0700 Subject: [PATCH 01/21] Fixing beegfs path and ensuring recursive transfers --- orchestration/flows/bl832/nersc.py | 17 ++++++++++------- orchestration/globus/transfer.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index f5850a61..e98d7704 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1909,18 +1909,21 @@ def nersc_recon_flow( logger.info("All transfers complete.") # Register the reconstructed TIFFs in tiled - register_file_to_tiled( - path=Path(config.beegfs_scratch.root_path+tiff_file_path), - prefix="beamlines/bl832/scratch", - overwrite=False, - tags=["scratch", "bl832"], - ) + # register_file_to_tiled( + # path=Path(config.beegfs_scratch.root_path+tiff_file_path), + # prefix="beamlines/bl832/scratch", + # overwrite=False, + # tags=["scratch", "bl832"], + # ) + + #TODO: get project ID from the raw.h5 metadata and add it to the tags for the zarr registration + # Probably will only work if the data is still on Spot832 # Register the reconstructed ZARRs in tiled register_file_to_tiled( path=Path(config.beegfs_scratch.root_path+zarr_file_path), prefix="beamlines/bl832/scratch", - overwrite=False, + overwrite=True, tags=["8.3.2", folder_name], ) diff --git a/orchestration/globus/transfer.py b/orchestration/globus/transfer.py index 7a9c010a..f8be2c62 100644 --- a/orchestration/globus/transfer.py +++ b/orchestration/globus/transfer.py @@ -127,7 +127,7 @@ def start_transfer( relative_path = item.relative_to(source_path.parent) tdata.add_item(str(item), os.path.join(dest_path, str(relative_path))) else: - tdata.add_item(str(source_path), dest_path) + tdata.add_item(str(source_path), dest_path, recursive=True) logger.info( f"starting transfer {source_endpoint.uri}:{source_path} to {dest_endpoint.uri}:{dest_path}" ) From 9e15aff60841f8a2692a1ca5f5ca4a7986e56ffd Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 8 Jul 2026 13:06:19 -0700 Subject: [PATCH 02/21] Missing config parameter in transfer task submit call --- orchestration/flows/bl832/nersc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index e98d7704..2afed15a 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1896,7 +1896,8 @@ def nersc_recon_flow( nersc_to_beegfs_zarr_future = globus_transfer_task.submit( file_path=zarr_file_path, source=config.nersc832_alsdev_pscratch_scratch, - destination=config.beegfs_scratch + destination=config.beegfs_scratch, + config=config ) # Resolve before pruning (which needs to know what landed where) From 62fdfe3a1af89018c7e9f602a6eda62652b863a0 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 8 Jul 2026 13:18:25 -0700 Subject: [PATCH 03/21] Installing tiled[all] for Zarr support --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8c007895..539fdba4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@4828273f5f49ba4eba5442728729e0545b3f5b79", "sfapi_client", "starlette==1.0.0", - "tiled[client]", + "tiled[all]==0.2.12", "watchfiles" ] From d8cba2cd4f65125c7bf956f10eeaac3a3a4c90c4 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 8 Jul 2026 13:39:22 -0700 Subject: [PATCH 04/21] config --- config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config.yml b/config.yml index fb0b9989..c849453d 100644 --- a/config.yml +++ b/config.yml @@ -140,15 +140,15 @@ globus: name: nersc832 bl832-beegfs-raw: - root_path: /global/beegfs/beamlines/bl832/raw/ + root_path: /raw/ uri: beegfs.als.lbl.gov - uuid: d33b5d6e-1603-414e-93cb-bcb732b7914a + uuid: ad1d70a4-649e-4e1a-8cd4-3074b825bd82 name: bl832-beegfs-raw bl832-beegfs-scratch: - root_path: /global/beegfs/beamlines/bl832/scratch/ + root_path: /scratch/ uri: beegfs.als.lbl.gov - uuid: d33b5d6e-1603-414e-93cb-bcb732b7914a + uuid: ad1d70a4-649e-4e1a-8cd4-3074b825bd82 name: bl832-beegfs-scratch globus_apps: From e27d327f76ff347d7ed1653be96e11f39338e9f4 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 8 Jul 2026 14:17:36 -0700 Subject: [PATCH 05/21] Adjusting ingest prefix --- orchestration/flows/bl832/nersc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 2afed15a..17ea9696 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1923,7 +1923,7 @@ def nersc_recon_flow( # Register the reconstructed ZARRs in tiled register_file_to_tiled( path=Path(config.beegfs_scratch.root_path+zarr_file_path), - prefix="beamlines/bl832/scratch", + prefix=f"beamlines/bl832/scratch/{folder_name}", overwrite=True, tags=["8.3.2", folder_name], ) From 56fb8175519c870fd386de70ef376d6165e76f5a Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 8 Jul 2026 14:21:24 -0700 Subject: [PATCH 06/21] Trying to ingest up 1 level --- orchestration/flows/bl832/nersc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 17ea9696..abdedb0b 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1922,7 +1922,8 @@ def nersc_recon_flow( # Register the reconstructed ZARRs in tiled register_file_to_tiled( - path=Path(config.beegfs_scratch.root_path+zarr_file_path), + # path=Path(config.beegfs_scratch.root_path+zarr_file_path), + path=Path(config.beegfs_scratch.root_path+folder_name), prefix=f"beamlines/bl832/scratch/{folder_name}", overwrite=True, tags=["8.3.2", folder_name], From 2ac13b81a29bb7ee580c39c442a2211dcef20af4 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 8 Jul 2026 14:41:46 -0700 Subject: [PATCH 07/21] Reverting the ingest path, but keeping the prefix --- orchestration/flows/bl832/nersc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index abdedb0b..17ea9696 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1922,8 +1922,7 @@ def nersc_recon_flow( # Register the reconstructed ZARRs in tiled register_file_to_tiled( - # path=Path(config.beegfs_scratch.root_path+zarr_file_path), - path=Path(config.beegfs_scratch.root_path+folder_name), + path=Path(config.beegfs_scratch.root_path+zarr_file_path), prefix=f"beamlines/bl832/scratch/{folder_name}", overwrite=True, tags=["8.3.2", folder_name], From 042d84383a3b1f1b5c3bcdab00d020b805c1310c Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 8 Jul 2026 15:05:51 -0700 Subject: [PATCH 08/21] Fixing ingest path --- orchestration/flows/bl832/nersc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 17ea9696..208d7cea 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1922,7 +1922,8 @@ def nersc_recon_flow( # Register the reconstructed ZARRs in tiled register_file_to_tiled( - path=Path(config.beegfs_scratch.root_path+zarr_file_path), + # path=Path(config.beegfs_scratch.root_path+zarr_file_path), + path=Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + zarr_file_path), prefix=f"beamlines/bl832/scratch/{folder_name}", overwrite=True, tags=["8.3.2", folder_name], From 2361e17f821cb173c7833b9ac1965d9ab8607b4b Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 9 Jul 2026 09:47:00 -0700 Subject: [PATCH 09/21] Changing scratch -> processed --- config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.yml b/config.yml index c849453d..a59f09a4 100644 --- a/config.yml +++ b/config.yml @@ -146,7 +146,7 @@ globus: name: bl832-beegfs-raw bl832-beegfs-scratch: - root_path: /scratch/ + root_path: /processed/ uri: beegfs.als.lbl.gov uuid: ad1d70a4-649e-4e1a-8cd4-3074b825bd82 name: bl832-beegfs-scratch From 126b6677ecb06c356e65aef8758748b56782f4be Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 9 Jul 2026 09:47:49 -0700 Subject: [PATCH 10/21] Changing scratch -> processed --- orchestration/flows/bl832/nersc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 208d7cea..ea048551 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1924,7 +1924,7 @@ def nersc_recon_flow( register_file_to_tiled( # path=Path(config.beegfs_scratch.root_path+zarr_file_path), path=Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + zarr_file_path), - prefix=f"beamlines/bl832/scratch/{folder_name}", + prefix=f"beamlines/bl832/processed/{folder_name}", overwrite=True, tags=["8.3.2", folder_name], ) From 5ec84e3ea1ea4a83f391ed0c06446099cbc51a60 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 9 Jul 2026 13:08:19 -0700 Subject: [PATCH 11/21] Downgrading tiled==0.2.8 to match the server.. hopefully Zarr metadata is ingested correctly with this --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 539fdba4..1a71c9f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@4828273f5f49ba4eba5442728729e0545b3f5b79", "sfapi_client", "starlette==1.0.0", - "tiled[all]==0.2.12", + "tiled[all]==0.2.8", "watchfiles" ] From b2543a7597e5bc1e2d600b334cdd1b5ec855eb28 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 9 Jul 2026 13:48:59 -0700 Subject: [PATCH 12/21] Downgrading zarr==2.18.3 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 1a71c9f8..400c050e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "starlette==1.0.0", "tiled[all]==0.2.8", "watchfiles" + "zarr==2.18.3" ] [dependency-groups] From 46e1b1f80729de62972e068f07f1fa90991d4fb2 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 9 Jul 2026 13:53:14 -0700 Subject: [PATCH 13/21] =?UTF-8?q?Missing=20comma=20=F0=9F=99=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 400c050e..5ca0a3ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "sfapi_client", "starlette==1.0.0", "tiled[all]==0.2.8", - "watchfiles" + "watchfiles", "zarr==2.18.3" ] From 9bddc0a33f16e73560bc3aa14ff5aafc91b0dc3d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 9 Jul 2026 14:45:52 -0700 Subject: [PATCH 14/21] Trying to ingest up 1 level to match what worked with the cli ingestion --- orchestration/flows/bl832/nersc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index ea048551..dd9de7bf 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1923,7 +1923,7 @@ def nersc_recon_flow( # Register the reconstructed ZARRs in tiled register_file_to_tiled( # path=Path(config.beegfs_scratch.root_path+zarr_file_path), - path=Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + zarr_file_path), + path=Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + f"/{folder_name}"), prefix=f"beamlines/bl832/processed/{folder_name}", overwrite=True, tags=["8.3.2", folder_name], From ae61cee62d56e1ad4dbaecdc1a868d0804dc6b35 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 9 Jul 2026 14:46:14 -0700 Subject: [PATCH 15/21] pinning numcodecs --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5ca0a3ab..00644e09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "mkdocs-material", "mkdocs-mermaid2-plugin", "mlflow==2.22.0", + "numcodecs==0.13.1", "numpy>=1.26.4", "pillow", "prefect==3.4.2", From f50e738711371f94cfe2ac1e63fabee6abcc3bc0 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 9 Jul 2026 14:46:37 -0700 Subject: [PATCH 16/21] pinning tiled==0.2.11 to match what worked with the cli ingestion --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 00644e09..77e050f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@4828273f5f49ba4eba5442728729e0545b3f5b79", "sfapi_client", "starlette==1.0.0", - "tiled[all]==0.2.8", + "tiled[all]==0.2.11", "watchfiles", "zarr==2.18.3" ] From aa9f2ebdac1cc45874839b468aebd69bbb26f588 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 10 Jul 2026 11:22:06 -0700 Subject: [PATCH 17/21] pinning tiled==0.2.8, unpinning zarr --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 77e050f6..a9ef7724 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ dependencies = [ "mkdocs-material", "mkdocs-mermaid2-plugin", "mlflow==2.22.0", - "numcodecs==0.13.1", "numpy>=1.26.4", "pillow", "prefect==3.4.2", @@ -29,9 +28,8 @@ dependencies = [ "scicat_beamline @ git+https://github.com/als-computing/scicat_beamline.git@4828273f5f49ba4eba5442728729e0545b3f5b79", "sfapi_client", "starlette==1.0.0", - "tiled[all]==0.2.11", + "tiled[all]==0.2.8", "watchfiles", - "zarr==2.18.3" ] [dependency-groups] From 8aa2995d31f662f4d03ff778e891647813a49d52 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 10 Jul 2026 11:26:22 -0700 Subject: [PATCH 18/21] fixing file path --- orchestration/flows/bl832/nersc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index dd9de7bf..ea048551 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1923,7 +1923,7 @@ def nersc_recon_flow( # Register the reconstructed ZARRs in tiled register_file_to_tiled( # path=Path(config.beegfs_scratch.root_path+zarr_file_path), - path=Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + f"/{folder_name}"), + path=Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + zarr_file_path), prefix=f"beamlines/bl832/processed/{folder_name}", overwrite=True, tags=["8.3.2", folder_name], From f50009b33e7edbfd399b26eb20ecbfef1148d319 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 10 Jul 2026 14:16:32 -0700 Subject: [PATCH 19/21] setting overwrite=False --- orchestration/flows/bl832/nersc.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index ea048551..4942cdcc 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1921,14 +1921,23 @@ def nersc_recon_flow( # Probably will only work if the data is still on Spot832 # Register the reconstructed ZARRs in tiled + p = Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + zarr_file_path) + logger.info(f"Registering Zarr file in Tiled: {p}") register_file_to_tiled( # path=Path(config.beegfs_scratch.root_path+zarr_file_path), - path=Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + zarr_file_path), + path=p, prefix=f"beamlines/bl832/processed/{folder_name}", - overwrite=True, + overwrite=False, tags=["8.3.2", folder_name], ) +# register_file_to_tiled( +# path=Path("/global/beegfs/beamlines/bl832/processed/dabramov/rec20260224_140520_petiole5.zarr"), +# prefix=f"beamlines/bl832/processed/dabramov", +# overwrite=False, +# tags=["8.3.2", "dabramov"], +# ) + logger.info("Scheduling pruning tasks.") schedule_pruning( config=config, From 6e1e8e5864ef3c8167724e590b81518b8136ec3c Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 10 Jul 2026 15:10:06 -0700 Subject: [PATCH 20/21] Adding docstrings --- orchestration/tiled.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/orchestration/tiled.py b/orchestration/tiled.py index 90687c7b..1a25b2f4 100644 --- a/orchestration/tiled.py +++ b/orchestration/tiled.py @@ -23,6 +23,19 @@ def register_file_to_tiled( overwrite: bool = False, tags: list[str] | None = None, ) -> None: + """ + Register a file or Zarr/HDF5 store to the Tiled catalog. + + Args: + path: The path to the file or Zarr/HDF5 store to register. + prefix: The sub-path within the Tiled catalog where the entry should be registered. + overwrite: Whether to overwrite an existing entry with the same key. + tags: A list of tags to apply to the registered entry. + Raises: + RuntimeError: If registration fails for any reason. + Returns: + None + """ logger = get_run_logger() path = Path(path) tiled_uri = os.environ["TILED_URI"] @@ -76,6 +89,17 @@ def register_file_to_tiled( @task(name="apply-tags", task_run_name="apply-tags-{tags}") def _apply_tags(entry_node, tags: list[str]) -> None: + """ + Apply tags to a Tiled entry node. If the entry already has tags, merge them with the new tags. + + Args: + entry_node: The Tiled entry node to which tags should be applied. + tags: A list of tags to apply to the entry node. + Raises: + Exception: If the tagging operation fails for any reason. + Returns: + None + """ logger = get_run_logger() existing_blob = entry_node.access_blob existing_tags = (existing_blob or {}).get("tags", []) From b74689a15f5fc07989dfa8e2f543035db494bffb Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 10 Jul 2026 15:10:45 -0700 Subject: [PATCH 21/21] Cleanup --- orchestration/flows/bl832/nersc.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 4942cdcc..bae6f63d 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1924,20 +1924,12 @@ def nersc_recon_flow( p = Path("/global/beegfs/beamlines/bl832/" + config.beegfs_scratch.root_path + zarr_file_path) logger.info(f"Registering Zarr file in Tiled: {p}") register_file_to_tiled( - # path=Path(config.beegfs_scratch.root_path+zarr_file_path), path=p, prefix=f"beamlines/bl832/processed/{folder_name}", overwrite=False, tags=["8.3.2", folder_name], ) -# register_file_to_tiled( -# path=Path("/global/beegfs/beamlines/bl832/processed/dabramov/rec20260224_140520_petiole5.zarr"), -# prefix=f"beamlines/bl832/processed/dabramov", -# overwrite=False, -# tags=["8.3.2", "dabramov"], -# ) - logger.info("Scheduling pruning tasks.") schedule_pruning( config=config,