Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion hypex/dataset/abstract.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
StatisticRole,
default_roles,
)
from .storage_manager import StorageManager


class DatasetBase:
Expand Down Expand Up @@ -371,11 +370,16 @@ def persist(
triggered the action;
- :meth:`unpersist` releases all accumulated references.
"""
# Load after DatasetBase is defined to avoid a module-level import cycle.
from .storage_manager import StorageManager

StorageManager.instance().persist(self, storage_level, action)
return self

def unpersist(self, blocking: bool = False):
"""Release ALL references accumulated by the dataset lineage."""
from .storage_manager import StorageManager

StorageManager.instance().release_all(self.persist_key, blocking=blocking)
return self

Expand Down Expand Up @@ -429,6 +433,8 @@ def checkpoint(self, eager: bool = True) -> Self:
"""
self._backend_data.checkpoint(eager=eager)
if eager:
from .storage_manager import StorageManager

StorageManager.instance().release_all(self.persist_key)
return self

Expand Down
35 changes: 9 additions & 26 deletions hypex/dataset/storage_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,10 @@
import time
import uuid
import warnings
from typing import Any, Literal, Protocol, cast
from typing import TYPE_CHECKING, Literal


class _CacheFrame(Protocol):
def same_cache(self, other: Any) -> bool:
"""Return whether both frames refer to the same physical cache."""

def release_cache(self, blocking: bool = False) -> None:
"""Release this frame's cache."""


class _PersistableDataset(_CacheFrame, Protocol):
@property
def persist_key(self) -> str | None:
"""Return the lineage key of the persisted dataset."""

def _acquire_cache(self, storage_level: str) -> _PersistableDataset | None:
"""Acquire a cache and return a stable dataset snapshot."""

def materialize(self, action: Literal["count", "head", "none"]) -> None:
"""Materialize all, part, or none of the current data."""
if TYPE_CHECKING:
from .abstract import DatasetBase


class PersistEntry:
Expand All @@ -39,7 +22,7 @@ class PersistEntry:

__slots__ = ("created_at", "frame", "storage_level")

def __init__(self, frame: _CacheFrame, storage_level: str | None = None):
def __init__(self, frame: DatasetBase, storage_level: str | None = None):
self.frame = frame
self.storage_level = storage_level
self.created_at = time.monotonic()
Expand Down Expand Up @@ -100,7 +83,7 @@ def _same_plan(first, second):

def persist(
self,
data: _PersistableDataset,
data: DatasetBase,
storage_level: str,
action: Literal["count", "head", "none"],
) -> None:
Expand All @@ -112,7 +95,7 @@ def persist(
cached = data._acquire_cache(storage_level)
if cached is None:
return
key = cast(str, cached.persist_key)
key = cached.persist_key
self.register(key, cached, storage_level=storage_level)
cached.materialize(action)
if action == "count":
Expand All @@ -136,7 +119,7 @@ def _release_unowned(self, entries, blocking):
)

def register(
self, key: str, data: _CacheFrame, storage_level: str | None = None
self, key: str, data: DatasetBase, storage_level: str | None = None
) -> PersistEntry:
"""Register a persisted ``frame`` in the lineage bucket ``key``.

Expand All @@ -163,7 +146,7 @@ def register(
def release(
self,
key: str,
keep: _CacheFrame | None = None,
keep: DatasetBase | None = None,
blocking: bool = True,
) -> int:
"""Release all accumulated references of the lineage except ``keep``.
Expand Down Expand Up @@ -218,7 +201,7 @@ def entries(self, key: str) -> list[PersistEntry]:
with self._lock:
return list(self._storage.get(key, []))

def frames(self, key: str) -> list[_CacheFrame]:
def frames(self, key: str) -> list[DatasetBase]:
"""Snapshot of the persisted frames of the lineage."""
return [entry.frame for entry in self.entries(key)]

Expand Down
Loading