ALCF Reconstruction and Segmentation for bl832

config.yml

@@ -73,17 +73,35 @@ globus:
       uuid: 75b478b2-37af-46df-bfbd-71ed692c6506
       name: data832_scratch
 
-    alcf832_raw:
+    alcf832_synaps_raw:
+      root_path: /data/bl832/raw
+      uri: alcf.anl.gov
+      uuid: 728a8e30-32ef-4000-814c-f9ccbc00bf13
+      name: alcf832_synaps_raw
+
+    alcf832_synaps_recon:
+      root_path: /data/bl832/scratch/reconstruction/
+      uri: alcf.anl.gov
+      uuid: 728a8e30-32ef-4000-814c-f9ccbc00bf13
+      name: alcf832_synaps_recon
+
+    alcf832_synaps_segment:
+      root_path: /data/bl832/scratch/segmentation/
+      uri: alcf.anl.gov
+      uuid: 728a8e30-32ef-4000-814c-f9ccbc00bf13
+      name: alcf832_synaps_segment
+
+    alcf832_iri_raw:
       root_path: /data/raw
       uri: alcf.anl.gov
       uuid: 55c3adf6-31f1-4647-9a38-52591642f7e7
-      name: alcf_raw
+      name: alcf_iri_raw
 
-    alcf832_scratch:
+    alcf832_iri_scratch:
       root_path: /data/scratch
       uri: alcf.anl.gov
       uuid: 55c3adf6-31f1-4647-9a38-52591642f7e7
-      name: alcf_scratch
+      name: alcf_iri_scratch
 
     alcf_eagle832:
       root_path: /IRIBeta/als/example

orchestration/_tests/test_globus_flow.py

@@ -149,7 +149,9 @@ def __init__(self) -> None:
             "alcf832_raw": MockEndpoint("mock_alcf832_raw_path", MockSecret.for_endpoint("alcf832_raw")),
             "alcf832_scratch": MockEndpoint("mock_alcf832_scratch_path", MockSecret.for_endpoint("alcf832_scratch")),
             "beegfs_raw": MockEndpoint("mock_beegfs_raw_path", MockSecret.for_endpoint("beegfs_raw")),
-            "beegfs_scratch": MockEndpoint("mock_beegfs_scratch_path", MockSecret.for_endpoint("beegfs_scratch"))
+            "beegfs_scratch": MockEndpoint("mock_beegfs_scratch_path", MockSecret.for_endpoint("beegfs_scratch")),
+            "alcf832_iri_raw": MockEndpoint("mock_alcf832_raw_path", MockSecret.for_endpoint("alcf832_iri_raw")),
+            "alcf832_iri_scratch": MockEndpoint("mock_alcf832_scratch_path", MockSecret.for_endpoint("alcf832_iri_scratch")),
         }
 
         # Mock apps
@@ -164,8 +166,8 @@ def __init__(self) -> None:
         self.spot832 = self.endpoints["spot832"]
         self.data832 = self.endpoints["data832"]
         self.nersc832 = self.endpoints["nersc832"]
-        self.alcf832_raw = self.endpoints["alcf832_raw"]
-        self.alcf832_scratch = self.endpoints["alcf832_scratch"]
+        self.alcf832_iri_raw = self.endpoints["alcf832_iri_raw"]
+        self.alcf832_iri_scratch = self.endpoints["alcf832_iri_scratch"]
         self.data832_raw = self.endpoints["data832_raw"]
         self.data832_scratch = self.endpoints["data832_scratch"]
         self.nersc832_alsdev_scratch = self.endpoints["nersc832_alsdev_scratch"]
@@ -205,7 +207,12 @@ def test_alcf_recon_flow(mocker: MockFixture):
             "alcf832_raw": mocker.MagicMock(),
             "alcf832_scratch": mocker.MagicMock(),
             "bl832-beegfs-raw": mocker.MagicMock(),
-            "bl832-beegfs-scratch": mocker.MagicMock()
+            "bl832-beegfs-scratch": mocker.MagicMock(),
+            "alcf832_iri_raw": mocker.MagicMock(),
+            "alcf832_iri_scratch": mocker.MagicMock(),
+            "alcf832_synaps_raw": mocker.MagicMock(),
+            "alcf832_synaps_recon": mocker.MagicMock(),
+            "alcf832_synaps_segment": mocker.MagicMock(),
         }
     )
     mocker.patch(
@@ -256,10 +263,12 @@ def test_alcf_recon_flow(mocker: MockFixture):
         return_value=mock_transfer_controller
     )
 
-    # 7) Patch schedule_pruning => skip real scheduling
-    mock_schedule_pruning = mocker.patch(
-        "orchestration.flows.bl832.alcf.schedule_pruning",
-        return_value=True
+    # 7) Patch get_prune_controller(...) => skip real scheduling
+    mock_prune_controller = mocker.MagicMock()
+    mock_prune_controller.prune.return_value = True
+    mocker.patch(
+        "orchestration.flows.bl832.alcf.get_prune_controller",
+        return_value=mock_prune_controller
     )
 
     mocker.patch(
@@ -285,13 +294,13 @@ def test_alcf_recon_flow(mocker: MockFixture):
     assert mock_transfer_controller.copy.call_count == 4, "Should do 4 transfers in success path"
     mock_hpc_reconstruct.assert_called_once()
     mock_hpc_multires.assert_called_once()
-    mock_schedule_pruning.assert_called_once()
+    assert mock_prune_controller.prune.call_count == 5, "Should schedule 5 prune operations in success path"
 
     # Reset for next scenario
     mock_transfer_controller.copy.reset_mock()
     mock_hpc_reconstruct.reset_mock()
     mock_hpc_multires.reset_mock()
-    mock_schedule_pruning.reset_mock()
+    mock_prune_controller.prune.reset_mock()
 
     #
     # ---------- CASE 2: HPC reconstruction fails ----------
@@ -308,13 +317,13 @@ def test_alcf_recon_flow(mocker: MockFixture):
     assert mock_transfer_controller.copy.call_count == 1, (
         "Should only do the first data832->alcf copy before HPC fails"
     )
-    mock_schedule_pruning.assert_not_called()
+    mock_prune_controller.prune.assert_not_called()
 
     # Reset
     mock_transfer_controller.copy.reset_mock()
     mock_hpc_reconstruct.reset_mock()
     mock_hpc_multires.reset_mock()
-    mock_schedule_pruning.reset_mock()
+    mock_prune_controller.prune.reset_mock()
 
     # ---------- CASE 3: Tiff->Zarr fails ----------
     mock_transfer_controller.copy.return_value = True
@@ -329,13 +338,13 @@ def test_alcf_recon_flow(mocker: MockFixture):
     # HPC is done, so there's 2 successful transfer (data832->alcf).
     # We have not transferred tiff or zarr => total 2 copies
     assert mock_transfer_controller.copy.call_count == 2
-    mock_schedule_pruning.assert_not_called()
+    mock_prune_controller.prune.assert_not_called()
 
     # Reset
     mock_transfer_controller.copy.reset_mock()
     mock_hpc_reconstruct.reset_mock()
     mock_hpc_multires.reset_mock()
-    mock_schedule_pruning.reset_mock()
+    mock_prune_controller.prune.reset_mock()
 
     # ---------- CASE 4: data832->ALCF fails immediately ----------
     mock_transfer_controller.copy.return_value = False
@@ -349,4 +358,4 @@ def test_alcf_recon_flow(mocker: MockFixture):
     mock_hpc_multires.assert_not_called()
     # The only call is the failing copy
     mock_transfer_controller.copy.assert_called_once()
-    mock_schedule_pruning.assert_not_called()
+    mock_prune_controller.prune.assert_not_called()