Files
DumasNodes/v3/ref_encode_cache.py
T
chris.dumas bf8f76505f Squashed 'vendor/dumas_h3_continuum/' content from commit d07ed31
git-subtree-dir: vendor/dumas_h3_continuum
git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
2026-10-20 13:43:38 +00:00

341 lines
10 KiB
Python

"""Process-local, VAE-scoped cache for V3.8 reference encode tensors.
Only detached, contiguous CPU tensor copies are retained. The cache never
owns source images, video frames, audio waveforms, conditioning objects, or
sampling outputs.
"""
from __future__ import annotations
from collections import Counter, OrderedDict
from dataclasses import dataclass
import threading
from typing import Any, Callable
import weakref
import torch
REF_ENCODE_CACHE_VERSION = 1
DEFAULT_MAX_ENTRIES = 16
DEFAULT_MAX_BYTES = 512 * 1024 * 1024
@dataclass(frozen=True, slots=True)
class RefEncodeCacheKey:
kind: str
preprocess_version: int
identity: str
cache_version: int = REF_ENCODE_CACHE_VERSION
@dataclass(frozen=True, slots=True)
class RefEncodeCacheEvent:
kind: str
action: str
reason: str = ""
payload_bytes: int = 0
@dataclass(slots=True)
class _CacheEntry:
tensor: torch.Tensor
payload_bytes: int
last_used: int
CacheEventSink = Callable[[RefEncodeCacheEvent], None]
def make_ref_encode_cache_key(
kind: str,
preprocess_version: int,
identity: str,
) -> RefEncodeCacheKey:
return RefEncodeCacheKey(
kind=str(kind),
preprocess_version=int(preprocess_version),
identity=str(identity),
)
class RefEncodeCache:
"""Small process-local LRU split into weak VAE object namespaces."""
def __init__(
self,
*,
max_entries: int = DEFAULT_MAX_ENTRIES,
max_bytes: int = DEFAULT_MAX_BYTES,
) -> None:
if int(max_entries) < 1:
raise ValueError("max_entries must be positive")
if int(max_bytes) < 1:
raise ValueError("max_bytes must be positive")
self.max_entries = int(max_entries)
self.max_bytes = int(max_bytes)
self._lock = threading.RLock()
self._namespaces: weakref.WeakKeyDictionary[
Any, OrderedDict[RefEncodeCacheKey, _CacheEntry]
] = weakref.WeakKeyDictionary()
self._clock = 0
self._stats: Counter[str] = Counter()
self._kind_stats: dict[str, Counter[str]] = {}
@staticmethod
def _vae_cacheable(vae: Any) -> bool:
try:
weakref.ref(vae)
hash(vae)
except TypeError:
return False
return True
def supports_vae(self, vae: Any) -> bool:
return self._vae_cacheable(vae)
def _next_clock_locked(self) -> int:
self._clock += 1
return self._clock
def _record_locked(
self,
*,
key: RefEncodeCacheKey,
action: str,
reason: str = "",
payload_bytes: int = 0,
) -> RefEncodeCacheEvent:
self._stats[str(action)] += 1
self._kind_stats.setdefault(key.kind, Counter())[str(action)] += 1
return RefEncodeCacheEvent(
kind=key.kind,
action=str(action),
reason=str(reason),
payload_bytes=int(payload_bytes),
)
@staticmethod
def _emit(event: RefEncodeCacheEvent, sink: CacheEventSink | None) -> None:
if sink is not None:
sink(event)
def _resident_locked(self) -> tuple[int, int]:
entries = 0
payload_bytes = 0
for namespace in list(self._namespaces.values()):
entries += len(namespace)
payload_bytes += sum(entry.payload_bytes for entry in namespace.values())
return entries, payload_bytes
def _evict_locked(self) -> None:
while True:
entries, payload_bytes = self._resident_locked()
if entries <= self.max_entries and payload_bytes <= self.max_bytes:
return
oldest_namespace = None
oldest_key = None
oldest_clock = None
for namespace in list(self._namespaces.values()):
for key, entry in namespace.items():
if oldest_clock is None or entry.last_used < oldest_clock:
oldest_namespace = namespace
oldest_key = key
oldest_clock = entry.last_used
if oldest_namespace is None or oldest_key is None:
return
oldest_namespace.pop(oldest_key, None)
self._stats["evictions"] += 1
def lookup(
self,
vae: Any,
key: RefEncodeCacheKey,
*,
event_sink: CacheEventSink | None = None,
) -> torch.Tensor | None:
if not self._vae_cacheable(vae):
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="vae_not_weakrefable_or_hashable",
)
self._emit(event, event_sink)
return None
master = None
with self._lock:
namespace = self._namespaces.get(vae)
entry = namespace.get(key) if namespace is not None else None
if entry is None:
event = self._record_locked(key=key, action="miss")
else:
entry.last_used = self._next_clock_locked()
namespace.move_to_end(key)
master = entry.tensor
event = self._record_locked(
key=key,
action="hit",
payload_bytes=entry.payload_bytes,
)
self._emit(event, event_sink)
return None if master is None else master.clone()
def store(
self,
vae: Any,
key: RefEncodeCacheKey,
tensor: Any,
*,
event_sink: CacheEventSink | None = None,
) -> bool:
if not self._vae_cacheable(vae):
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="vae_not_weakrefable_or_hashable",
)
self._emit(event, event_sink)
return False
if not torch.is_tensor(tensor):
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="payload_not_tensor",
)
self._emit(event, event_sink)
return False
if tensor.device.type != "cpu":
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="payload_not_cpu",
)
self._emit(event, event_sink)
return False
payload_bytes = int(tensor.numel()) * int(tensor.element_size())
if payload_bytes > self.max_bytes:
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="entry_exceeds_byte_limit",
payload_bytes=payload_bytes,
)
self._emit(event, event_sink)
return False
try:
master = tensor.detach().contiguous().clone()
except Exception:
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="payload_copy_failed",
payload_bytes=payload_bytes,
)
self._emit(event, event_sink)
return False
with self._lock:
namespace = self._namespaces.get(vae)
if namespace is None:
namespace = OrderedDict()
self._namespaces[vae] = namespace
namespace[key] = _CacheEntry(
tensor=master,
payload_bytes=payload_bytes,
last_used=self._next_clock_locked(),
)
namespace.move_to_end(key)
self._evict_locked()
event = self._record_locked(
key=key,
action="store",
payload_bytes=payload_bytes,
)
self._emit(event, event_sink)
return True
def clear(self) -> None:
with self._lock:
self._namespaces = weakref.WeakKeyDictionary()
self._clock = 0
self._stats.clear()
self._kind_stats.clear()
def inspect(self) -> dict[str, Any]:
with self._lock:
entries, payload_bytes = self._resident_locked()
return {
"cache_version": REF_ENCODE_CACHE_VERSION,
"resident_entries": entries,
"resident_bytes": payload_bytes,
"max_entries": self.max_entries,
"max_bytes": self.max_bytes,
"stats": dict(self._stats),
"kind_stats": {
kind: dict(values)
for kind, values in sorted(self._kind_stats.items())
},
"vae_namespaces": len(self._namespaces),
}
_REF_ENCODE_CACHE = RefEncodeCache()
def get_ref_encode_cache() -> RefEncodeCache:
return _REF_ENCODE_CACHE
def clear_ref_encode_cache() -> None:
_REF_ENCODE_CACHE.clear()
def inspect_ref_encode_cache() -> dict[str, Any]:
return _REF_ENCODE_CACHE.inspect()
def format_ref_encode_cache_diagnostics(
events: list[RefEncodeCacheEvent] | tuple[RefEncodeCacheEvent, ...],
) -> str:
if not events:
return ""
labels = {
"reference_image": "image",
"video_guide": "video",
"reference_audio": "audio",
}
by_kind: dict[str, Counter[str]] = {}
totals: Counter[str] = Counter()
for event in events:
by_kind.setdefault(event.kind, Counter())[event.action] += 1
totals[event.action] += 1
components = []
for kind in ("reference_image", "video_guide", "reference_audio"):
values = by_kind.get(kind)
if not values:
continue
components.append(
f"{labels[kind]} hit={values['hit']}/miss={values['miss']}"
f"/store={values['store']}/bypass={values['bypass']}"
)
state = inspect_ref_encode_cache()
resident_mib = int(state["resident_bytes"]) / float(1024**2)
limit_mib = int(state["max_bytes"]) / float(1024**2)
components.append(
"total "
f"hits={totals['hit']}, misses={totals['miss']}, "
f"stores={totals['store']}, bypasses={totals['bypass']}"
)
components.append(
f"resident={resident_mib:.1f} MiB/{limit_mib:.1f} MiB "
f"({int(state['resident_entries'])}/{int(state['max_entries'])} entries)"
)
return "Reference encode cache: " + "; ".join(components) + "."