Skip to content

Commit bf37df2

Browse files
committed
Prevent remote cache quota growth and fix test isolation
1 parent a1287b9 commit bf37df2

6 files changed

Lines changed: 93 additions & 33 deletions

File tree

‎caterva2-server.sample.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ register = true # allow users to register
3636
# otherwise non-public destination addresses, and embedded expression
3737
# references are refused. MEMORY carriers are accepted under the same source
3838
# policy but execute without retained caching (same as NONE). DISK proxies cache
39-
# chunks inside their carrier up to persisted max_cache_bytes (or unbounded if None), subject to server quota.
39+
# chunks inside their carrier up to persisted max_cache_bytes (or unbounded if None).
40+
# With a customer quota configured, warm DISK chunks are reused but misses are not retained.
4041
# [server.remote_proxy]
4142
# enabled = true
4243
# allowed_hosts = ["datasets.example.org", "objects.example.org:8443"]

‎caterva2/services/remote_proxy.py‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -390,15 +390,18 @@ def current_cache_bytes(self) -> int:
390390
with carrier_thread_lock(self.path):
391391
carrier = raw_carrier(self.path, locking=True)
392392
with carrier.schunk.holding_lock():
393-
sizes = carrier.schunk.vlmeta.get("proxy-cache-sizes")
394-
if isinstance(sizes, dict):
395-
return sum(size for size in sizes.values() if isinstance(size, int) and size >= 0)
393+
# The physical payload is authoritative. Uploaded size tables
394+
# can be stale after older unbounded writers (or user supplied).
396395
return carrier.schunk.cbytes
397396

398397
def _backend(self, cache_limit=None, *, carrier=None):
399-
if self.cache_policy != "disk" or cache_limit == 0:
398+
if self.cache_policy != "disk":
400399
return blosc2.Proxy(self.src, _refresh_source=False)
401-
carrier = raw_carrier(self.path, mode="a", locking=True) if carrier is None else carrier
400+
mode = "r" if cache_limit == 0 else "a"
401+
carrier = raw_carrier(self.path, mode=mode, locking=True) if carrier is None else carrier
402+
if cache_limit == 0:
403+
# Consume warm data without writing metadata or retaining misses.
404+
return blosc2.Proxy(self.src, _cache=carrier, _refresh_source=False)
402405
if cache_limit is None:
403406
limit = self.max_cache_bytes
404407
elif self.max_cache_bytes is None:
@@ -413,10 +416,10 @@ def _backend(self, cache_limit=None, *, carrier=None):
413416
)
414417

415418
def read(self, item, *, cache_limit=None):
416-
if self.cache_policy != "disk" or cache_limit == 0:
419+
if self.cache_policy != "disk":
417420
return self._backend(cache_limit)[item]
418421
with carrier_thread_lock(self.path):
419-
carrier = raw_carrier(self.path, mode="a", locking=True)
422+
carrier = raw_carrier(self.path, mode="r" if cache_limit == 0 else "a", locking=True)
420423
with carrier.schunk.holding_lock():
421424
backend = self._backend(cache_limit, carrier=carrier)
422425
return backend[item]
@@ -425,7 +428,7 @@ def __getitem__(self, item):
425428
return self.read(item)
426429

427430
def get_chunk(self, nchunk, *, cache_limit=None):
428-
if self.cache_policy != "disk" or cache_limit == 0:
431+
if self.cache_policy != "disk":
429432
return self.src.get_chunk(nchunk)
430433
item = tuple(
431434
slice(coord * chunk, min((coord + 1) * chunk, size))
@@ -442,10 +445,15 @@ def get_chunk(self, nchunk, *, cache_limit=None):
442445
)
443446
)
444447
with carrier_thread_lock(self.path):
445-
carrier = raw_carrier(self.path, mode="a", locking=True)
448+
carrier = raw_carrier(self.path, mode="r" if cache_limit == 0 else "a", locking=True)
446449
with carrier.schunk.holding_lock():
447450
backend = self._backend(cache_limit, carrier=carrier)
448-
backend.fetch(item)
451+
try:
452+
backend.fetch(item)
453+
except ValueError as exc:
454+
if cache_limit != 0 or "reading mode" not in str(exc):
455+
raise
456+
return self.src.get_chunk(nchunk)
449457
chunk = backend.schunk.get_chunk(nchunk)
450458
backend._enforce_cache_limit(item)
451459
return chunk

‎caterva2/services/server.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -176,17 +176,18 @@ def account_chunk_written(nbytes: int) -> None:
176176

177177

178178
def remote_proxy_cache_limit(proxy: remote_proxy.ServerRemoteProxy) -> int | None:
179-
"""Return the per-carrier cache bound after applying remaining customer quota."""
179+
"""Return the cache allowance; quota-enabled servers consume caches read-only.
180+
181+
Payload limits cannot reserve physical metadata growth, and per-dataset locks
182+
do not coordinate other carriers, uploads, or workers. Until all writers share
183+
a physical-storage reservation mechanism, automatic fills must not grow disk
184+
usage under a customer quota. Zero still permits reuse of warm carrier data.
185+
"""
180186
if proxy.cache_policy != "disk":
181187
return 0
182188
if not settings.quota:
183189
return proxy.max_cache_bytes
184-
retained = proxy.current_cache_bytes()
185-
available = max(0, settings.quota - get_disk_usage_written(0))
186-
quota_bound = retained + available
187-
if proxy.max_cache_bytes is None:
188-
return quota_bound
189-
return min(proxy.max_cache_bytes, quota_bound)
190+
return 0
190191

191192

192193
async def read_remote_proxy(proxy, item, abspath):

‎caterva2/tests/test_api.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -246,8 +246,8 @@ def __init__(self, values):
246246
def get(self, key, default=None):
247247
return self.values.get(key, default)
248248

249-
server.settings.statedir = tmp_path / "statedir"
250-
server.settings.public = server.settings.statedir / "public"
249+
monkeypatch.setattr(server.settings, "statedir", tmp_path / "statedir")
250+
monkeypatch.setattr(server.settings, "public", server.settings.statedir / "public")
251251
server.settings.public.mkdir(parents=True, exist_ok=True)
252252

253253
data = np.arange(40, dtype=np.int32)
@@ -271,6 +271,7 @@ def get(self, key, default=None):
271271
payload["source"]["urlpath"] = url
272272
carrier.schunk.vlmeta["b2o"] = payload
273273

274+
monkeypatch.setattr(remote_proxy, "policy", remote_proxy.policy)
274275
remote_proxy.configure(
275276
_Conf(
276277
{
@@ -292,7 +293,7 @@ def traced_cat_file(path, *args, **kwargs):
292293
upstream_reads.append(path)
293294
return orig_cat_file(path, *args, **kwargs)
294295

295-
mem_fs.cat_file = traced_cat_file
296+
monkeypatch.setattr(mem_fs, "cat_file", traced_cat_file)
296297

297298
monkeypatch.setattr(remote_proxy, "_public_addresses", lambda host, port: ("93.184.216.34",))
298299
monkeypatch.setattr(remote_proxy, "_https_filesystem", lambda host, addr: mem_fs)

‎caterva2/tests/test_remote_proxy.py‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,51 @@ def test_zero_effective_quota_reads_without_retaining(tmp_path):
553553
assert carrier_path.read_bytes() == before
554554

555555

556-
def test_customer_quota_reduces_the_proxy_cache_limit(monkeypatch):
556+
@pytest.mark.parametrize("limit", [None, 1_000_000])
557+
def test_read_only_cache_reuses_warm_chunks_and_does_not_retain_misses(tmp_path, limit):
558+
proxy, data, path = _server_proxy(tmp_path, "readonly-cache", max_cache_bytes=limit)
559+
proxy.get_chunk(0)
560+
before = path.read_bytes()
561+
mtime = path.stat().st_mtime_ns
562+
proxy.src.traffic.reset()
563+
np.testing.assert_array_equal(proxy.read(slice(0, 10), cache_limit=0), data[:10])
564+
proxy.get_chunk(0, cache_limit=0)
565+
assert proxy.src.traffic.requests == 0
566+
np.testing.assert_array_equal(proxy.read(slice(10, 20), cache_limit=0), data[10:20])
567+
assert proxy.src.traffic.requests > 0
568+
proxy.src.traffic.reset()
569+
proxy.get_chunk(1, cache_limit=0)
570+
assert proxy.src.traffic.requests > 0
571+
assert path.read_bytes() == before
572+
assert path.stat().st_mtime_ns == mtime
573+
574+
575+
def test_concurrent_quota_reads_do_not_grow_distinct_carriers(tmp_path, monkeypatch):
576+
monkeypatch.setenv("CATERVA2_SECRET", "remote-proxy-test-secret")
577+
from caterva2.services import server
578+
579+
monkeypatch.setattr(server.settings, "quota", 1)
580+
entries = [_server_proxy(tmp_path, name, max_cache_bytes=None) for name in ("left", "right")]
581+
before = [path.read_bytes() for _, _, path in entries]
582+
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
583+
futures = [
584+
pool.submit(proxy.read, (), cache_limit=server.remote_proxy_cache_limit(proxy))
585+
for proxy, _, _ in entries
586+
]
587+
for future, (_, data, _) in zip(futures, entries, strict=True):
588+
np.testing.assert_array_equal(future.result(timeout=10), data)
589+
assert [path.read_bytes() for _, _, path in entries] == before
590+
591+
592+
def test_cache_accounting_ignores_stale_size_table(tmp_path):
593+
proxy, _, path = _server_proxy(tmp_path, "stale-table", max_cache_bytes=None)
594+
proxy.read(())
595+
carrier = remote_proxy.raw_carrier(path, mode="a")
596+
carrier.schunk.vlmeta["proxy-cache-sizes"] = {"0": 1}
597+
assert proxy.current_cache_bytes() == carrier.schunk.cbytes
598+
599+
600+
def test_customer_quota_disables_cache_growth(monkeypatch):
557601
monkeypatch.setenv("CATERVA2_SECRET", "remote-proxy-test-secret")
558602
from caterva2.services import server
559603

@@ -567,10 +611,10 @@ def current_cache_bytes():
567611

568612
monkeypatch.setattr(server.settings, "quota", 1_000)
569613
monkeypatch.setattr(server, "get_disk_usage_written", lambda pending: 900)
570-
assert server.remote_proxy_cache_limit(Proxy()) == 300
614+
assert server.remote_proxy_cache_limit(Proxy()) == 0
571615

572616
monkeypatch.setattr(server, "get_disk_usage_written", lambda pending: 1_000)
573-
assert server.remote_proxy_cache_limit(Proxy()) == 200
617+
assert server.remote_proxy_cache_limit(Proxy()) == 0
574618

575619
class UnlimitedProxy:
576620
cache_policy = "disk"
@@ -580,13 +624,13 @@ class UnlimitedProxy:
580624
def current_cache_bytes():
581625
return 200
582626

583-
# Under quota, unlimited disk cache is bounded by available quota (200 + 100 = 300)
627+
# Any configured quota disables automatic growth, regardless of remaining space.
584628
monkeypatch.setattr(server, "get_disk_usage_written", lambda pending: 900)
585-
assert server.remote_proxy_cache_limit(UnlimitedProxy()) == 300
629+
assert server.remote_proxy_cache_limit(UnlimitedProxy()) == 0
586630

587-
# When quota is exhausted, bounded by retained bytes (200 + 0 = 200)
631+
# Exhausted quota also leaves the carrier read-only.
588632
monkeypatch.setattr(server, "get_disk_usage_written", lambda pending: 1_000)
589-
assert server.remote_proxy_cache_limit(UnlimitedProxy()) == 200
633+
assert server.remote_proxy_cache_limit(UnlimitedProxy()) == 0
590634

591635
# Without quota, unlimited disk cache returns None
592636
monkeypatch.setattr(server.settings, "quota", 0)

‎doc/utilities/cat2-server.md‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,9 @@ policy but execute without retained caching (using the same no-retention executi
4040
path as `NONE`), avoiding unmanaged memory use on the server while preserving
4141
the requested client limit for download. With a `DISK` cache, fetched compressed
4242
chunks are retained inside the carrier up to the proxy's `max_cache_bytes` (or
43-
unbounded when `max_cache_bytes` is `None`, still subject to customer quota if
44-
configured). Caterva2 can inspect and report its stored shape, dtype, chunk, block, and proxy
43+
unbounded when `max_cache_bytes` is `None`) when no customer quota is configured.
44+
With a quota, existing warm disk chunks are reused but misses are not retained.
45+
Caterva2 can inspect and report its stored shape, dtype, chunk, block, and proxy
4546
metadata without contacting the source. Outbound resolution is disabled by
4647
default.
4748

@@ -69,9 +70,13 @@ created by a client that performed its own validation.
6970
The limits validate the remote array's structure and bound connection time and
7071
concurrent range fetches. They do not impose a network-work budget on each API
7172
request. The proxy's own cache limit bounds its retained compressed payload,
72-
while automatic carrier growth is also charged to the virtual server's existing
73-
shared `quota`. If no quota remains, reads still succeed but misses are not
74-
retained.
73+
while a configured customer `quota` makes all DISK proxy caches read-only.
74+
Valid warm chunks are reused, but misses are served without retention, even when
75+
quota space remains. This conservative restriction prevents cache fills from
76+
overspending physical storage through metadata growth or concurrent workers.
77+
Automatic fills under quota require a future storage reservation mechanism shared
78+
with uploads and other writers. Without a quota, normal bounded or unbounded
79+
DISK caching applies.
7580

7681
Public S3 objects are supported through credential-free HTTPS object URLs.
7782
Native `s3://` resolution, private-source credentials, and remote references

0 commit comments

Comments
 (0)