Skip to content

Commit bad629d

Browse files
committed
Use sparse remote proxy caches by default
1 parent a1de98c commit bad629d

14 files changed

Lines changed: 2407 additions & 104 deletions

‎caterva2-server.sample.toml‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,17 @@ register = true # allow users to register
3333
# peer_cache_quota = "1G"
3434

3535
# Persisted RemoteProxy objects are discoverable but cannot make outbound
36-
# requests by default. The first opt-in backend is credential-free HTTPS.
36+
# requests by default. The runtime cache uses private sparse generations and
37+
# credential-free HTTPS is the supported source transport.
3738
# Every destination must be listed exactly; redirects, URL queries, private or
3839
# otherwise non-public destination addresses, and embedded expression
3940
# references are refused. MEMORY carriers are accepted under the same source
4041
# policy but execute without retained caching (same as NONE). DISK proxies cache
41-
# chunks inside their carrier up to persisted max_cache_bytes (or unbounded if None).
42-
# With a customer quota, shared SQLite admission permits DISK growth when capacity
43-
# is available; denied fills still return data without retention. See server docs
44-
# for accounting exclusions, operational headroom, and staged replacement costs.
42+
# chunks in private runtime generations up to persisted max_cache_bytes (256 MiB
43+
# by default, or unbounded if None). With a customer quota, shared SQLite admission
44+
# permits DISK growth when capacity is available; denied fills still return data
45+
# without retention. See server docs for accounting exclusions, operational
46+
# headroom, and sparse lifecycle behavior.
4547
# [server.remote_proxy]
4648
# enabled = true
4749
# allowed_hosts = ["datasets.example.org", "objects.example.org:8443"]

‎caterva2/services/remote_proxy.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from __future__ import annotations
1818

1919
import ipaddress
20+
import logging
2021
import math
2122
import socket
2223
import sqlite3
@@ -33,6 +34,8 @@
3334

3435
from caterva2.services import storage_quota
3536

37+
log = logging.getLogger(__name__)
38+
3639

3740
class RemoteProxyDenied(ValueError):
3841
"""The server policy refuses a remote reference."""
@@ -47,6 +50,7 @@ class Policy:
4750
max_rank: int = 16
4851
max_chunks: int = 10_000_000
4952
max_concurrency: int = 8
53+
cache_backend: str = "sparse"
5054

5155

5256
policy = Policy()
@@ -377,6 +381,7 @@ def __init__(self, source, geometry, carrier, payload):
377381
self.shape, self.dtype, self.chunks, self.blocks = geometry
378382
self.cparams = source.cparams
379383
self.path = carrier.schunk.urlpath
384+
self.carrier_generation = storage_quota.signature(self.path)
380385
self.requested_cache_policy = payload["cache_policy"]
381386
self.requested_payload = dict(payload)
382387
self.requested_max_cache_bytes = payload["max_cache_bytes"]
@@ -400,6 +405,8 @@ def current_cache_bytes(self) -> int:
400405

401406
def quota_read(self, quota, item=(), *, nchunk=None):
402407
"""Assemble on an immutable candidate and admit its exact physical size."""
408+
if quota.cache_backend == "sparse":
409+
return quota.remote.read(self, item, nchunk=nchunk)
403410
if self.cache_policy != "disk" or getattr(self.src, "stamp", None) is None:
404411
return (
405412
self.get_chunk(nchunk, cache_limit=0)
@@ -526,7 +533,17 @@ def cold_cframe(carrier, payload) -> bytes:
526533
chunks=carrier.chunks,
527534
blocks=carrier.blocks,
528535
cparams=carrier.cparams,
536+
meta={
537+
key: carrier.schunk.meta[key]
538+
for key in carrier.schunk.meta
539+
if key not in {"b2nd", "b2o", "proxy"}
540+
},
529541
)
542+
from blosc2.proxy import _RESERVED_VLMETA
543+
544+
for key in carrier.schunk.vlmeta:
545+
if key not in _RESERVED_VLMETA and key != "b2o":
546+
cold.schunk.vlmeta[key] = carrier.schunk.vlmeta[key]
530547
write_b2object_payload(cold, payload)
531548
return cold.to_cframe()
532549

‎caterva2/services/server.py‎

Lines changed: 105 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ def guess_type(path):
138138

139139

140140
def get_disk_usage():
141-
if settings.quota:
141+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
142142
return quota_coordinator().usage()["used"]
143143
exclude = {"db.json", "db.sqlite"}
144144
return sum(
@@ -184,13 +184,18 @@ def account_chunk_written(nbytes: int) -> None:
184184

185185
def quota_coordinator():
186186
"""Independent of authentication; one connection is opened per DB operation."""
187-
if not settings.quota:
187+
if not settings.quota and remote_proxy.policy.cache_backend != "sparse":
188188
return None
189189
work_bytes = settings.parse_size(settings.conf.get(".quota_work_bytes", "1G"))
190-
key = (str(settings.statedir), settings.quota, work_bytes)
190+
key = (str(settings.statedir), settings.quota, work_bytes, remote_proxy.policy.cache_backend)
191191
coordinator = _quota_instances.get(key)
192192
if coordinator is None:
193-
coordinator = storage_quota.StorageQuota(settings.statedir, settings.quota, work_bytes=work_bytes)
193+
coordinator = storage_quota.StorageQuota(
194+
settings.statedir,
195+
settings.quota,
196+
work_bytes=work_bytes,
197+
cache_backend=remote_proxy.policy.cache_backend,
198+
)
194199
_quota_instances[key] = coordinator
195200
return coordinator
196201

@@ -244,7 +249,12 @@ def copy_dataset(source, destination):
244249
if not entry.name.endswith(".b2lock"):
245250
copy_dataset(entry, destination / entry.name)
246251
else:
247-
write_dataset(destination, source.read_bytes())
252+
data = source.read_bytes()
253+
if remote_proxy.policy.cache_backend == "sparse":
254+
reference = remote_proxy.inspect(source)
255+
if reference is not None:
256+
data = remote_proxy.cold_cframe(*reference)
257+
write_dataset(destination, data)
248258

249259

250260
def move_dataset(source, destination):
@@ -266,6 +276,10 @@ def move_dataset(source, destination):
266276
data, generation = quota.snapshot(source)
267277
if generation is None:
268278
raise storage_quota.StorageBusy("move source was removed")
279+
if remote_proxy.policy.cache_backend == "sparse" and source.suffix in {".b2nd", ".b2frame"}:
280+
carrier = blosc2.ndarray_from_cframe(data)
281+
if carrier.schunk.vlmeta.get("b2o", {}).get("kind") == "remote_proxy":
282+
data = remote_proxy.cold_cframe(carrier, carrier.schunk.vlmeta["b2o"])
269283
write_dataset(destination, data)
270284
quota.publish(source, None, expected=generation, prune=False)
271285

@@ -299,7 +313,7 @@ async def read_remote_proxy(proxy, item, abspath):
299313
"""Read one remote selection while serializing and accounting cache mutation."""
300314
lock = dataset_lock(abspath)
301315
async with lock:
302-
if settings.quota:
316+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
303317
return await concurrency.run_in_threadpool(
304318
lambda: blosc2.asarray(quota_proxy_operation(proxy, item)).to_cframe()
305319
)
@@ -471,7 +485,7 @@ def _setup_plugin_globals():
471485

472486
@contextlib.asynccontextmanager
473487
async def lifespan(app: FastAPI):
474-
if settings.quota:
488+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
475489
await concurrency.run_in_threadpool(quota_coordinator)
476490
# Initialize the (users) database
477491
if user_login_enabled():
@@ -486,7 +500,24 @@ async def lifespan(app: FastAPI):
486500
for p in providers.active:
487501
await p.startup()
488502

489-
yield
503+
async def cache_maintenance():
504+
while True:
505+
await asyncio.sleep(30)
506+
try:
507+
await concurrency.run_in_threadpool(quota_coordinator().remote.maintain)
508+
except (OSError, sqlite3.Error, ValueError):
509+
remote_proxy.log.exception("Sparse cache maintenance deferred")
510+
511+
maintenance = (
512+
asyncio.create_task(cache_maintenance()) if remote_proxy.policy.cache_backend == "sparse" else None
513+
)
514+
try:
515+
yield
516+
finally:
517+
if maintenance is not None:
518+
maintenance.cancel()
519+
with contextlib.suppress(asyncio.CancelledError):
520+
await maintenance
490521

491522
for p in providers.active:
492523
await p.shutdown()
@@ -1411,6 +1442,23 @@ async def post_fetch_data(
14111442
)
14121443

14131444

1445+
class RemoteCacheFileResponse(responses.FileResponse):
1446+
"""Release artifact ownership even if streaming disconnects or fails."""
1447+
1448+
def __init__(self, path, *, cleanup, **kwargs):
1449+
super().__init__(path, **kwargs)
1450+
self.cleanup = cleanup
1451+
1452+
async def __call__(self, scope, receive, send):
1453+
try:
1454+
return await super().__call__(scope, receive, send)
1455+
finally:
1456+
import anyio
1457+
1458+
with anyio.CancelScope(shield=True):
1459+
await concurrency.run_in_threadpool(self.cleanup)
1460+
1461+
14141462
@app.get("/api/download/{path:path}")
14151463
async def download_data(
14161464
path: pathlib.Path,
@@ -1445,6 +1493,19 @@ async def download_data(
14451493
headers.update(srv_utils.NO_RANGES)
14461494
return responses.StreamingResponse(body, media_type=media_type, headers=headers)
14471495

1496+
if remote_proxy.policy.cache_backend == "sparse" and include_cache:
1497+
abspath = get_abspath(path, user)
1498+
reference = remote_proxy.inspect(abspath) if abspath.suffix in {".b2nd", ".b2frame"} else None
1499+
if reference is not None and reference[1]["cache_policy"] == "disk":
1500+
proxy = await concurrency.run_in_threadpool(lambda: remote_proxy.resolve(*reference))
1501+
if proxy.src.stamp is not None:
1502+
artifact, etag, cleanup = await concurrency.run_in_threadpool(
1503+
lambda: quota_coordinator().remote.export(proxy)
1504+
)
1505+
return RemoteCacheFileResponse(
1506+
artifact, cleanup=cleanup, filename=path.name, headers={"ETag": f'"{etag}"'}
1507+
)
1508+
14481509
decompress = accept_encoding != "blosc2"
14491510
# Read before creating the response: a bad path must 404 up front, not
14501511
# abort the stream after the 200 headers already went out.
@@ -1547,7 +1608,7 @@ async def get_chunk(
15471608
# In case we do, this would have to be changed.
15481609
chunk = container.get_chunk(nchunk)
15491610
elif isinstance(container, remote_proxy.ServerRemoteProxy):
1550-
if settings.quota:
1611+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
15511612
chunk = await concurrency.run_in_threadpool(
15521613
lambda: quota_proxy_operation(container, nchunk=nchunk)
15531614
)
@@ -1751,12 +1812,14 @@ def publish_dataset(abspath: pathlib.Path, path: pathlib.Path) -> str:
17511812
srv_utils.raise_bad_request("publishing needs fsspec, which is not installed here")
17521813
destination = publish_destination(path)
17531814
fs, target = fsspec.url_to_fs(destination)
1754-
if settings.quota and isinstance(fs, fsspec.implementations.local.LocalFileSystem):
1815+
if (settings.quota or remote_proxy.policy.cache_backend == "sparse") and isinstance(
1816+
fs, fsspec.implementations.local.LocalFileSystem
1817+
):
17551818
target_path = pathlib.Path(target).resolve()
17561819
state = pathlib.Path(settings.statedir).resolve()
17571820
if target_path == state or state in target_path.parents:
17581821
srv_utils.raise_bad_request("local publish_root must be outside the server state directory")
1759-
if settings.quota:
1822+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
17601823
quota = quota_coordinator()
17611824
frame, generation = quota.snapshot(abspath)
17621825
if generation is None:
@@ -1781,7 +1844,7 @@ def publish_dataset(abspath: pathlib.Path, path: pathlib.Path) -> str:
17811844
fs.makedirs(parent, exist_ok=True)
17821845
try:
17831846
with (
1784-
io.BytesIO(frame) if settings.quota else open(abspath, "rb") as source,
1847+
io.BytesIO(frame) if quota_coordinator() is not None else open(abspath, "rb") as source,
17851848
fs.open(staging, "wb") as target_file,
17861849
):
17871850
shutil.copyfileobj(source, target_file)
@@ -1792,7 +1855,7 @@ def publish_dataset(abspath: pathlib.Path, path: pathlib.Path) -> str:
17921855
with contextlib.suppress(Exception):
17931856
fs.rm(staging)
17941857
raise
1795-
if settings.quota:
1858+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
17961859
array = blosc2.ndarray_from_cframe(frame, copy=True)
17971860
array.schunk.vlmeta[PUBLISHED_URL] = destination
17981861
array.schunk.vlmeta[FILL_STATE] = PUBLISHED
@@ -1826,7 +1889,7 @@ def store_chunk(abspath: pathlib.Path, nchunk: int, chunk: bytes) -> dict:
18261889
both find the slot free would otherwise both write it, and the second would
18271890
move every chunk that came after the first.
18281891
"""
1829-
if settings.quota:
1892+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
18301893
quota = quota_coordinator()
18311894
frame, generation = quota.snapshot(abspath)
18321895
try:
@@ -2150,7 +2213,7 @@ def make_expr(
21502213

21512214
abspath.mkdir(exist_ok=True, parents=True)
21522215

2153-
if settings.quota:
2216+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
21542217
# Serialize before admission: metadata and compression determine the charge.
21552218
result = arr.compute() if compute else arr
21562219
try:
@@ -2294,7 +2357,7 @@ async def move(
22942357

22952358
# Make sure the destination directory exists
22962359
dest_abspath.parent.mkdir(exist_ok=True, parents=True)
2297-
if settings.quota:
2360+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
22982361
# Reserve the copy before removing the source. No fictitious free-space
22992362
# credit; directory operations retain their existing non-atomic semantics.
23002363
move_dataset(abspath, dest_abspath)
@@ -2353,7 +2416,7 @@ async def copy(
23532416
# raise fastapi.HTTPException(status_code=409, detail="The new path already exists")
23542417

23552418
dest_abspath.parent.mkdir(exist_ok=True, parents=True)
2356-
if settings.quota:
2419+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
23572420
copy_dataset(abspath, dest_abspath)
23582421
elif abspath.is_dir():
23592422
shutil.copytree(abspath, dest_abspath)
@@ -2543,7 +2606,7 @@ async def append_file(
25432606

25442607
# Append the data
25452608
# The original dataset (open in append mode so it can be resized/written)
2546-
if settings.quota:
2609+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
25472610
frame, generation = quota_coordinator().snapshot(abspath)
25482611
orig = blosc2.ndarray_from_cframe(frame, copy=True)
25492612
if "b2o" in orig.schunk.meta or "proxy-source" in orig.schunk.meta:
@@ -2565,7 +2628,7 @@ async def append_file(
25652628
orig.resize(result_shape)
25662629
# Append the new data to orig along the first axis
25672630
orig[orig.shape[0] - new_len :] = new_data
2568-
if settings.quota:
2631+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
25692632
quota_coordinator().publish(abspath, orig.to_cframe(), expected=generation)
25702633

25712634
# Return the new shape
@@ -2606,7 +2669,11 @@ async def unfold_file(
26062669
# Unfold the container
26072670
if abspath.suffix in {".h5", ".hdf5"}:
26082671
# Create proxies for each dataset in HDF5 file
2609-
all_dsets = list(hdf5.create_hdf5_proxies(abspath, writer=write_dataset if settings.quota else None))
2672+
all_dsets = list(
2673+
hdf5.create_hdf5_proxies(
2674+
abspath, writer=write_dataset if quota_coordinator() is not None else None
2675+
)
2676+
)
26102677
if len(all_dsets) == 0:
26112678
detail = "No arrays found in HDF5 file"
26122679
raise fastapi.HTTPException(detail=detail, status_code=400)
@@ -4001,7 +4068,7 @@ async def htmx_upload(
40014068
suffix = filename.suffix
40024069
suffixes = filename.suffixes[-2:]
40034070
if suffix in [".tar", ".tgz", ".zip"] or suffixes == [".tar", ".gz"]:
4004-
if settings.quota:
4071+
if settings.quota or remote_proxy.policy.cache_backend == "sparse":
40054072
# Admit encoded members independently. Never extract an archive into
40064073
# managed storage before measuring its final serialized files.
40074074
first = None
@@ -4171,6 +4238,23 @@ async def get_file_content(path, user, decompress=True, include_cache=True):
41714238
lock = dataset_lock(abspath)
41724239
async with lock:
41734240
carrier, payload = remote_proxy.inspect(abspath)
4241+
if (
4242+
remote_proxy.policy.cache_backend == "sparse"
4243+
and include_cache
4244+
and payload["cache_policy"] == "disk"
4245+
):
4246+
4247+
def snapshot_sparse():
4248+
proxy = remote_proxy.resolve(carrier, payload)
4249+
if proxy.src.stamp is None:
4250+
return remote_proxy.cold_cframe(carrier, payload)
4251+
artifact, _, cleanup = quota_coordinator().remote.export(proxy)
4252+
try:
4253+
return artifact.read_bytes()
4254+
finally:
4255+
cleanup()
4256+
4257+
return await concurrency.run_in_threadpool(snapshot_sparse)
41744258
return await concurrency.run_in_threadpool(
41754259
lambda: remote_proxy.export_cframe(carrier, payload, include_cache=include_cache)
41764260
)

0 commit comments

Comments
 (0)