@@ -138,7 +138,7 @@ def guess_type(path):
138138
139139
140140def get_disk_usage ():
141- if settings .quota :
141+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
142142 return quota_coordinator ().usage ()["used" ]
143143 exclude = {"db.json" , "db.sqlite" }
144144 return sum (
@@ -184,13 +184,18 @@ def account_chunk_written(nbytes: int) -> None:
184184
185185def quota_coordinator ():
186186 """Independent of authentication; one connection is opened per DB operation."""
187- if not settings .quota :
187+ if not settings .quota and remote_proxy . policy . cache_backend != "sparse" :
188188 return None
189189 work_bytes = settings .parse_size (settings .conf .get (".quota_work_bytes" , "1G" ))
190- key = (str (settings .statedir ), settings .quota , work_bytes )
190+ key = (str (settings .statedir ), settings .quota , work_bytes , remote_proxy . policy . cache_backend )
191191 coordinator = _quota_instances .get (key )
192192 if coordinator is None :
193- coordinator = storage_quota .StorageQuota (settings .statedir , settings .quota , work_bytes = work_bytes )
193+ coordinator = storage_quota .StorageQuota (
194+ settings .statedir ,
195+ settings .quota ,
196+ work_bytes = work_bytes ,
197+ cache_backend = remote_proxy .policy .cache_backend ,
198+ )
194199 _quota_instances [key ] = coordinator
195200 return coordinator
196201
@@ -244,7 +249,12 @@ def copy_dataset(source, destination):
244249 if not entry .name .endswith (".b2lock" ):
245250 copy_dataset (entry , destination / entry .name )
246251 else :
247- write_dataset (destination , source .read_bytes ())
252+ data = source .read_bytes ()
253+ if remote_proxy .policy .cache_backend == "sparse" :
254+ reference = remote_proxy .inspect (source )
255+ if reference is not None :
256+ data = remote_proxy .cold_cframe (* reference )
257+ write_dataset (destination , data )
248258
249259
250260def move_dataset (source , destination ):
@@ -266,6 +276,10 @@ def move_dataset(source, destination):
266276 data , generation = quota .snapshot (source )
267277 if generation is None :
268278 raise storage_quota .StorageBusy ("move source was removed" )
279+ if remote_proxy .policy .cache_backend == "sparse" and source .suffix in {".b2nd" , ".b2frame" }:
280+ carrier = blosc2 .ndarray_from_cframe (data )
281+ if carrier .schunk .vlmeta .get ("b2o" , {}).get ("kind" ) == "remote_proxy" :
282+ data = remote_proxy .cold_cframe (carrier , carrier .schunk .vlmeta ["b2o" ])
269283 write_dataset (destination , data )
270284 quota .publish (source , None , expected = generation , prune = False )
271285
@@ -299,7 +313,7 @@ async def read_remote_proxy(proxy, item, abspath):
299313 """Read one remote selection while serializing and accounting cache mutation."""
300314 lock = dataset_lock (abspath )
301315 async with lock :
302- if settings .quota :
316+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
303317 return await concurrency .run_in_threadpool (
304318 lambda : blosc2 .asarray (quota_proxy_operation (proxy , item )).to_cframe ()
305319 )
@@ -471,7 +485,7 @@ def _setup_plugin_globals():
471485
472486@contextlib .asynccontextmanager
473487async def lifespan (app : FastAPI ):
474- if settings .quota :
488+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
475489 await concurrency .run_in_threadpool (quota_coordinator )
476490 # Initialize the (users) database
477491 if user_login_enabled ():
@@ -486,7 +500,24 @@ async def lifespan(app: FastAPI):
486500 for p in providers .active :
487501 await p .startup ()
488502
489- yield
503+ async def cache_maintenance ():
504+ while True :
505+ await asyncio .sleep (30 )
506+ try :
507+ await concurrency .run_in_threadpool (quota_coordinator ().remote .maintain )
508+ except (OSError , sqlite3 .Error , ValueError ):
509+ remote_proxy .log .exception ("Sparse cache maintenance deferred" )
510+
511+ maintenance = (
512+ asyncio .create_task (cache_maintenance ()) if remote_proxy .policy .cache_backend == "sparse" else None
513+ )
514+ try :
515+ yield
516+ finally :
517+ if maintenance is not None :
518+ maintenance .cancel ()
519+ with contextlib .suppress (asyncio .CancelledError ):
520+ await maintenance
490521
491522 for p in providers .active :
492523 await p .shutdown ()
@@ -1411,6 +1442,23 @@ async def post_fetch_data(
14111442 )
14121443
14131444
1445+ class RemoteCacheFileResponse (responses .FileResponse ):
1446+ """Release artifact ownership even if streaming disconnects or fails."""
1447+
1448+ def __init__ (self , path , * , cleanup , ** kwargs ):
1449+ super ().__init__ (path , ** kwargs )
1450+ self .cleanup = cleanup
1451+
1452+ async def __call__ (self , scope , receive , send ):
1453+ try :
1454+ return await super ().__call__ (scope , receive , send )
1455+ finally :
1456+ import anyio
1457+
1458+ with anyio .CancelScope (shield = True ):
1459+ await concurrency .run_in_threadpool (self .cleanup )
1460+
1461+
14141462@app .get ("/api/download/{path:path}" )
14151463async def download_data (
14161464 path : pathlib .Path ,
@@ -1445,6 +1493,19 @@ async def download_data(
14451493 headers .update (srv_utils .NO_RANGES )
14461494 return responses .StreamingResponse (body , media_type = media_type , headers = headers )
14471495
1496+ if remote_proxy .policy .cache_backend == "sparse" and include_cache :
1497+ abspath = get_abspath (path , user )
1498+ reference = remote_proxy .inspect (abspath ) if abspath .suffix in {".b2nd" , ".b2frame" } else None
1499+ if reference is not None and reference [1 ]["cache_policy" ] == "disk" :
1500+ proxy = await concurrency .run_in_threadpool (lambda : remote_proxy .resolve (* reference ))
1501+ if proxy .src .stamp is not None :
1502+ artifact , etag , cleanup = await concurrency .run_in_threadpool (
1503+ lambda : quota_coordinator ().remote .export (proxy )
1504+ )
1505+ return RemoteCacheFileResponse (
1506+ artifact , cleanup = cleanup , filename = path .name , headers = {"ETag" : f'"{ etag } "' }
1507+ )
1508+
14481509 decompress = accept_encoding != "blosc2"
14491510 # Read before creating the response: a bad path must 404 up front, not
14501511 # abort the stream after the 200 headers already went out.
@@ -1547,7 +1608,7 @@ async def get_chunk(
15471608 # In case we do, this would have to be changed.
15481609 chunk = container .get_chunk (nchunk )
15491610 elif isinstance (container , remote_proxy .ServerRemoteProxy ):
1550- if settings .quota :
1611+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
15511612 chunk = await concurrency .run_in_threadpool (
15521613 lambda : quota_proxy_operation (container , nchunk = nchunk )
15531614 )
@@ -1751,12 +1812,14 @@ def publish_dataset(abspath: pathlib.Path, path: pathlib.Path) -> str:
17511812 srv_utils .raise_bad_request ("publishing needs fsspec, which is not installed here" )
17521813 destination = publish_destination (path )
17531814 fs , target = fsspec .url_to_fs (destination )
1754- if settings .quota and isinstance (fs , fsspec .implementations .local .LocalFileSystem ):
1815+ if (settings .quota or remote_proxy .policy .cache_backend == "sparse" ) and isinstance (
1816+ fs , fsspec .implementations .local .LocalFileSystem
1817+ ):
17551818 target_path = pathlib .Path (target ).resolve ()
17561819 state = pathlib .Path (settings .statedir ).resolve ()
17571820 if target_path == state or state in target_path .parents :
17581821 srv_utils .raise_bad_request ("local publish_root must be outside the server state directory" )
1759- if settings .quota :
1822+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
17601823 quota = quota_coordinator ()
17611824 frame , generation = quota .snapshot (abspath )
17621825 if generation is None :
@@ -1781,7 +1844,7 @@ def publish_dataset(abspath: pathlib.Path, path: pathlib.Path) -> str:
17811844 fs .makedirs (parent , exist_ok = True )
17821845 try :
17831846 with (
1784- io .BytesIO (frame ) if settings . quota else open (abspath , "rb" ) as source ,
1847+ io .BytesIO (frame ) if quota_coordinator () is not None else open (abspath , "rb" ) as source ,
17851848 fs .open (staging , "wb" ) as target_file ,
17861849 ):
17871850 shutil .copyfileobj (source , target_file )
@@ -1792,7 +1855,7 @@ def publish_dataset(abspath: pathlib.Path, path: pathlib.Path) -> str:
17921855 with contextlib .suppress (Exception ):
17931856 fs .rm (staging )
17941857 raise
1795- if settings .quota :
1858+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
17961859 array = blosc2 .ndarray_from_cframe (frame , copy = True )
17971860 array .schunk .vlmeta [PUBLISHED_URL ] = destination
17981861 array .schunk .vlmeta [FILL_STATE ] = PUBLISHED
@@ -1826,7 +1889,7 @@ def store_chunk(abspath: pathlib.Path, nchunk: int, chunk: bytes) -> dict:
18261889 both find the slot free would otherwise both write it, and the second would
18271890 move every chunk that came after the first.
18281891 """
1829- if settings .quota :
1892+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
18301893 quota = quota_coordinator ()
18311894 frame , generation = quota .snapshot (abspath )
18321895 try :
@@ -2150,7 +2213,7 @@ def make_expr(
21502213
21512214 abspath .mkdir (exist_ok = True , parents = True )
21522215
2153- if settings .quota :
2216+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
21542217 # Serialize before admission: metadata and compression determine the charge.
21552218 result = arr .compute () if compute else arr
21562219 try :
@@ -2294,7 +2357,7 @@ async def move(
22942357
22952358 # Make sure the destination directory exists
22962359 dest_abspath .parent .mkdir (exist_ok = True , parents = True )
2297- if settings .quota :
2360+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
22982361 # Reserve the copy before removing the source. No fictitious free-space
22992362 # credit; directory operations retain their existing non-atomic semantics.
23002363 move_dataset (abspath , dest_abspath )
@@ -2353,7 +2416,7 @@ async def copy(
23532416 # raise fastapi.HTTPException(status_code=409, detail="The new path already exists")
23542417
23552418 dest_abspath .parent .mkdir (exist_ok = True , parents = True )
2356- if settings .quota :
2419+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
23572420 copy_dataset (abspath , dest_abspath )
23582421 elif abspath .is_dir ():
23592422 shutil .copytree (abspath , dest_abspath )
@@ -2543,7 +2606,7 @@ async def append_file(
25432606
25442607 # Append the data
25452608 # The original dataset (open in append mode so it can be resized/written)
2546- if settings .quota :
2609+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
25472610 frame , generation = quota_coordinator ().snapshot (abspath )
25482611 orig = blosc2 .ndarray_from_cframe (frame , copy = True )
25492612 if "b2o" in orig .schunk .meta or "proxy-source" in orig .schunk .meta :
@@ -2565,7 +2628,7 @@ async def append_file(
25652628 orig .resize (result_shape )
25662629 # Append the new data to orig along the first axis
25672630 orig [orig .shape [0 ] - new_len :] = new_data
2568- if settings .quota :
2631+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
25692632 quota_coordinator ().publish (abspath , orig .to_cframe (), expected = generation )
25702633
25712634 # Return the new shape
@@ -2606,7 +2669,11 @@ async def unfold_file(
26062669 # Unfold the container
26072670 if abspath .suffix in {".h5" , ".hdf5" }:
26082671 # Create proxies for each dataset in HDF5 file
2609- all_dsets = list (hdf5 .create_hdf5_proxies (abspath , writer = write_dataset if settings .quota else None ))
2672+ all_dsets = list (
2673+ hdf5 .create_hdf5_proxies (
2674+ abspath , writer = write_dataset if quota_coordinator () is not None else None
2675+ )
2676+ )
26102677 if len (all_dsets ) == 0 :
26112678 detail = "No arrays found in HDF5 file"
26122679 raise fastapi .HTTPException (detail = detail , status_code = 400 )
@@ -4001,7 +4068,7 @@ async def htmx_upload(
40014068 suffix = filename .suffix
40024069 suffixes = filename .suffixes [- 2 :]
40034070 if suffix in [".tar" , ".tgz" , ".zip" ] or suffixes == [".tar" , ".gz" ]:
4004- if settings .quota :
4071+ if settings .quota or remote_proxy . policy . cache_backend == "sparse" :
40054072 # Admit encoded members independently. Never extract an archive into
40064073 # managed storage before measuring its final serialized files.
40074074 first = None
@@ -4171,6 +4238,23 @@ async def get_file_content(path, user, decompress=True, include_cache=True):
41714238 lock = dataset_lock (abspath )
41724239 async with lock :
41734240 carrier , payload = remote_proxy .inspect (abspath )
4241+ if (
4242+ remote_proxy .policy .cache_backend == "sparse"
4243+ and include_cache
4244+ and payload ["cache_policy" ] == "disk"
4245+ ):
4246+
4247+ def snapshot_sparse ():
4248+ proxy = remote_proxy .resolve (carrier , payload )
4249+ if proxy .src .stamp is None :
4250+ return remote_proxy .cold_cframe (carrier , payload )
4251+ artifact , _ , cleanup = quota_coordinator ().remote .export (proxy )
4252+ try :
4253+ return artifact .read_bytes ()
4254+ finally :
4255+ cleanup ()
4256+
4257+ return await concurrency .run_in_threadpool (snapshot_sparse )
41744258 return await concurrency .run_in_threadpool (
41754259 lambda : remote_proxy .export_cframe (carrier , payload , include_cache = include_cache )
41764260 )
0 commit comments