Release V3.8X2 package 3.8.3
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
"""H3 Decode Cache Helper implementation; imports do not load ComfyUI models."""
|
||||
__version__ = "3.8.3"
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Content keys for already-selected decode entries; no model or storage mutation."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
import threading
|
||||
import weakref
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Iterator
|
||||
|
||||
import torch
|
||||
|
||||
CONTRACT = "h3-decode-helper-v1"
|
||||
BLOCK_BYTES = 8 * 1024 * 1024
|
||||
|
||||
|
||||
class CacheBypass(Exception):
|
||||
"""The native decoder may run, but cache identity is not safely available."""
|
||||
|
||||
|
||||
class ObjectTokens:
|
||||
"""Monotonic process-local identities, without retaining model objects."""
|
||||
|
||||
def __init__(self):
|
||||
self._refs = {}
|
||||
self._next = itertools.count(1)
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def token(self, obj):
|
||||
address = id(obj)
|
||||
with self._lock:
|
||||
record = self._refs.get(address)
|
||||
if record is not None and record[0]() is obj:
|
||||
return record[1]
|
||||
token = next(self._next)
|
||||
|
||||
def expired(ref, address=address):
|
||||
with self._lock:
|
||||
current = self._refs.get(address)
|
||||
if current is not None and current[0] is ref:
|
||||
self._refs.pop(address, None)
|
||||
|
||||
try:
|
||||
ref = weakref.ref(obj, expired)
|
||||
except TypeError as exc:
|
||||
raise CacheBypass("object does not support a weak identity") from exc
|
||||
self._refs[address] = (ref, token)
|
||||
return token
|
||||
|
||||
|
||||
TOKENS = ObjectTokens()
|
||||
|
||||
|
||||
def tensor_blocks(tensor: torch.Tensor, block_bytes=BLOCK_BYTES) -> Iterator[memoryview]:
|
||||
"""Yield C-order bytes, including BF16, with bounded contiguous scratch space."""
|
||||
if not isinstance(tensor, torch.Tensor) or tensor.layout != torch.strided:
|
||||
raise CacheBypass("only dense strided tensors are cacheable")
|
||||
if tensor.device.type != "cpu" or tensor.is_nested or tensor.is_quantized:
|
||||
raise CacheBypass("non-CPU/nested/quantized payload is delegated without cache")
|
||||
tensor = tensor.detach()
|
||||
if tensor.is_conj():
|
||||
tensor = tensor.resolve_conj()
|
||||
if tensor.is_neg():
|
||||
tensor = tensor.resolve_neg()
|
||||
size = tensor.numel() * tensor.element_size()
|
||||
if size == 0:
|
||||
return
|
||||
if size <= block_bytes:
|
||||
raw = tensor.contiguous().reshape(-1).view(torch.uint8).numpy()
|
||||
yield memoryview(raw).cast("B")
|
||||
return
|
||||
# All earlier dimensions are singleton, preserving logical C-order.
|
||||
for axis, length in enumerate(tensor.shape):
|
||||
if length > 1:
|
||||
per_slice = size // length
|
||||
step = max(1, block_bytes // max(1, per_slice))
|
||||
for start in range(0, length, step):
|
||||
yield from tensor_blocks(tensor.narrow(axis, start, min(step, length-start)), block_bytes)
|
||||
return
|
||||
raise CacheBypass("unrepresentable tensor block")
|
||||
|
||||
|
||||
def tensor_digest(tensor, interrupt=None):
|
||||
h = hashlib.sha256()
|
||||
for block in tensor_blocks(tensor):
|
||||
if interrupt is not None:
|
||||
interrupt()
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def canonical(value: Any, interrupt=None, depth=0):
|
||||
"""Reject opaque objects instead of using an unreliable repr or pointer key."""
|
||||
if depth > 16:
|
||||
raise CacheBypass("metadata nesting limit")
|
||||
if value is None or type(value) in (str, bool, int):
|
||||
return value
|
||||
if type(value) is float:
|
||||
if not math.isfinite(value):
|
||||
raise CacheBypass("non-finite metadata")
|
||||
return value
|
||||
if isinstance(value, (torch.dtype, torch.device)):
|
||||
return str(value)
|
||||
if isinstance(value, torch.Tensor):
|
||||
return ["tensor", str(value.dtype), list(value.shape), tensor_digest(value, interrupt)]
|
||||
if isinstance(value, Mapping):
|
||||
if len(value) > 256 or any(not isinstance(k, str) for k in value):
|
||||
raise CacheBypass("unsupported metadata mapping")
|
||||
return {k: canonical(value[k], interrupt, depth+1) for k in sorted(value)}
|
||||
if isinstance(value, (tuple, list)) and len(value) <= 4096:
|
||||
return [canonical(x, interrupt, depth+1) for x in value]
|
||||
raise CacheBypass(f"opaque metadata: {type(value).__name__}")
|
||||
|
||||
|
||||
def selected_latent(samples, stream):
|
||||
latent = samples["samples"]
|
||||
if getattr(latent, "is_nested", False):
|
||||
parts = latent.unbind()
|
||||
latent = parts[0 if stream == "video" else -1]
|
||||
if not isinstance(latent, torch.Tensor):
|
||||
raise CacheBypass("unknown latent payload")
|
||||
return latent
|
||||
|
||||
|
||||
def _callable_token(function):
|
||||
function = getattr(function, "__func__", function)
|
||||
return TOKENS.token(function)
|
||||
|
||||
|
||||
def _empty_option_tree(value):
|
||||
if value is None:
|
||||
return True
|
||||
if isinstance(value, Mapping):
|
||||
return all(_empty_option_tree(v) for v in value.values())
|
||||
if isinstance(value, (tuple, list)):
|
||||
return all(_empty_option_tree(v) for v in value)
|
||||
return False
|
||||
|
||||
|
||||
def vae_signature(vae, stream, delegate):
|
||||
"""Supported native H3 inference VAE only; unknown wrappers still decode normally.
|
||||
|
||||
This is not a model-file hash. Unsupported in-place .data changes require
|
||||
reset_token or a reloaded VAE. No weights are copied or read onto the CPU.
|
||||
"""
|
||||
model = getattr(vae, "first_stage_model", None)
|
||||
patcher = getattr(vae, "patcher", None)
|
||||
if type(vae).__module__ != "comfy.sd" or not type(model).__module__.startswith("comfy.ldm.minimax."):
|
||||
raise CacheBypass("unrecognized VAE identity; native decode only")
|
||||
if patcher is None or not hasattr(patcher, "patches_uuid"):
|
||||
raise CacheBypass("VAE patch version unavailable")
|
||||
if bool(getattr(model, "training", True)):
|
||||
raise CacheBypass("training-mode VAE")
|
||||
if "decode" in vars(vae) or "decode" in vars(model):
|
||||
raise CacheBypass("instance-patched decoder")
|
||||
if getattr(patcher, "object_patches", None):
|
||||
raise CacheBypass("VAE object patches require cache bypass")
|
||||
|
||||
if not _empty_option_tree(getattr(patcher, "model_options", None)):
|
||||
raise CacheBypass("non-default VAE patcher options")
|
||||
if any(getattr(patcher, field, None) for field in ("forced_hooks", "hook_patches")):
|
||||
raise CacheBypass("VAE hook state requires cache bypass")
|
||||
if any(getattr(part, "_forward_hooks", None) or getattr(part, "_forward_pre_hooks", None)
|
||||
for part in model.modules()):
|
||||
raise CacheBypass("VAE forward hooks require cache bypass")
|
||||
|
||||
config = {}
|
||||
for name in ("vae_dtype", "device", "output_device", "audio_sample_rate", "audio_sample_rate_output",
|
||||
"latent_channels", "disable_offload", "output_channels"):
|
||||
if hasattr(vae, name):
|
||||
config[name] = canonical(getattr(vae, name))
|
||||
# Native H3 tiling/temporal configuration is scalar, not model weights.
|
||||
model_config = {k: v for k, v in vars(model).items()
|
||||
if not k.startswith("_") and type(v) in (str, bool, int, float, type(None))}
|
||||
# Model residency can replace buffer objects without changing their values.
|
||||
# Logical names/versions + Core patch UUID are used, not buffer pointers.
|
||||
# Reloading weights through raw .data (outside Core) requires reset_token.
|
||||
versions = []
|
||||
for name, value in itertools.chain(model.named_parameters(), model.named_buffers()):
|
||||
try:
|
||||
version = value._version
|
||||
except RuntimeError: # inference tensors intentionally have no counter
|
||||
version = None
|
||||
versions.append((name, version, str(value.dtype), tuple(value.shape)))
|
||||
identity = (
|
||||
TOKENS.token(vae), TOKENS.token(model), TOKENS.token(patcher),
|
||||
str(patcher.patches_uuid), _callable_token(vae.decode), _callable_token(model.decode),
|
||||
_callable_token(delegate),
|
||||
_callable_token(vae.process_output) if callable(getattr(vae, "process_output", None)) else None,
|
||||
config, model_config, versions,
|
||||
str(torch.get_default_dtype()), torch.get_float32_matmul_precision(),
|
||||
bool(torch.backends.cuda.matmul.allow_tf32), bool(torch.backends.cudnn.allow_tf32),
|
||||
bool(torch.backends.cudnn.benchmark), bool(torch.backends.cudnn.deterministic),
|
||||
bool(torch.are_deterministic_algorithms_enabled()),
|
||||
bool(torch.is_grad_enabled()), bool(torch.is_inference_mode_enabled()),
|
||||
bool(torch.is_autocast_enabled("cpu")), str(torch.get_autocast_dtype("cpu")),
|
||||
bool(torch.is_autocast_enabled("cuda")), str(torch.get_autocast_dtype("cuda")),
|
||||
)
|
||||
payload = json.dumps(identity, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
||||
return hashlib.sha256(payload.encode()).hexdigest(), (weakref.ref(vae), weakref.ref(model), weakref.ref(patcher))
|
||||
|
||||
|
||||
def make_key(samples, stream, signature, reset_token=0, interrupt=None):
|
||||
latent = selected_latent(samples, stream)
|
||||
# Only the chosen AV stream is hashed. Other decode-relevant metadata,
|
||||
# notably samples['sample_rate'], is included, conservatively, as well.
|
||||
metadata = {k: v for k, v in samples.items() if k != "samples"}
|
||||
payload = [CONTRACT, stream, signature, int(reset_token), str(latent.dtype), list(latent.shape),
|
||||
tensor_digest(latent, interrupt), canonical(metadata, interrupt)]
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""One independent helper; existing Continuum node IDs and schemas are untouched."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from .service import DecodeService, reraise_control
|
||||
from .store import MiB, GiB
|
||||
|
||||
|
||||
class H3DecodeCacheHelper:
|
||||
CATEGORY = "MiniMax H3/Continuum/Helpers"
|
||||
FUNCTION = "decode"
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO", "STRING")
|
||||
RETURN_NAMES = ("images", "audio", "report")
|
||||
INPUT_IS_LIST = True
|
||||
OUTPUT_IS_LIST = (True, True, False)
|
||||
DESCRIPTION = (
|
||||
"Optional V3.8X2 helper: reuse complete physical VAE decode entries. "
|
||||
"Auto stores video in a private process-local disk cache, audio in bounded RAM. "
|
||||
"Connect the unchanged Assembly Plan directly to Finalize. No model unload or sampling changes."
|
||||
)
|
||||
SEARCH_ALIASES = ["Decode Cache Helper", "H3 cached decode", "video audio cache"]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"cache_mode": (["Auto", "RAM", "Off"], {"default": "Auto", "tooltip":
|
||||
"Auto: disk-backed video + small RAM audio. RAM: bounded RAM only. Off: native decoding, clear helper cache."}),
|
||||
"ram_budget_mb": ("INT", {"default": 256, "min": 0, "max": 8192, "step": 128, "advanced": True,
|
||||
"tooltip": "Additional private RAM retention limit (MiB), not total process memory. Low headroom disables RAM retention."}),
|
||||
"disk_budget_gb": ("INT", {"default": 8, "min": 0, "max": 128, "step": 1, "advanced": True,
|
||||
"tooltip": "Process-local cache quota in GiB. Set 0 for no cache file writes. Auto does not reuse files after restart."}),
|
||||
"reset_token": ("INT", {"default": 0, "min": 0, "max": 2147483647, "advanced": True,
|
||||
"tooltip": "Increase to discard this helper's cache, especially after unsupported in-place VAE weight edits."}),
|
||||
},
|
||||
"optional": {
|
||||
"video_samples": ("LATENT", {"tooltip": "Continuum video_latents LIST; each entry is decoded whole, before trim/seam."}),
|
||||
"video_vae": ("VAE", {"tooltip": "Same native Video VAE as the existing VAE Decode node."}),
|
||||
"audio_samples": ("LATENT", {"tooltip": "Continuum audio_latents LIST; optional, decoded independently."}),
|
||||
"audio_vae": ("VAE", {"tooltip": "Same native Audio VAE as VAE Decode Audio. Required only with audio_samples."}),
|
||||
},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, **kwargs):
|
||||
# Review must still execute; only this helper's content cache skips decode.
|
||||
return float("nan")
|
||||
|
||||
def __init__(self):
|
||||
self.service = DecodeService()
|
||||
self._lock = threading.RLock()
|
||||
|
||||
@staticmethod
|
||||
def _scalar(value, name):
|
||||
if isinstance(value, list):
|
||||
if len(value) != 1:
|
||||
raise ValueError(f"{name} expects one value, not a list batch")
|
||||
return value[0]
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _list(value):
|
||||
return value if isinstance(value, list) else ([] if value is None else [value])
|
||||
|
||||
def _stream(self, samples, vaes, stream, reset_token):
|
||||
entries = self._list(samples)
|
||||
vaes = self._list(vaes)
|
||||
if not entries:
|
||||
return [], []
|
||||
outputs, events = [], []
|
||||
for index, entry in enumerate(entries):
|
||||
if entry is None:
|
||||
# Same missing-output position; do not duplicate the previous entry.
|
||||
outputs.append(None)
|
||||
events.append({"stream": stream, "entry": index, "status": "unconnected"})
|
||||
continue
|
||||
vae = vaes[min(index, len(vaes)-1)] if vaes else None
|
||||
result, event = self.service.decode(entry, vae, stream, reset_token)
|
||||
outputs.append(result)
|
||||
events.append({"entry": index, **event})
|
||||
return outputs, events
|
||||
|
||||
def decode(self, cache_mode, ram_budget_mb, disk_budget_gb, reset_token,
|
||||
video_samples=None, video_vae=None, audio_samples=None, audio_vae=None):
|
||||
with self._lock:
|
||||
mode = self._scalar(cache_mode, "cache_mode")
|
||||
ram = int(self._scalar(ram_budget_mb, "ram_budget_mb"))
|
||||
disk = int(self._scalar(disk_budget_gb, "disk_budget_gb"))
|
||||
reset = int(self._scalar(reset_token, "reset_token"))
|
||||
if mode not in ("Auto", "RAM", "Off") or ram < 0 or disk < 0:
|
||||
raise ValueError("invalid cache settings")
|
||||
config_error = None
|
||||
try:
|
||||
self.service.store.configure(mode, ram*MiB, disk*GiB, reset)
|
||||
except Exception as exc:
|
||||
reraise_control(exc)
|
||||
self.service.store.mode = "Off"
|
||||
config_error = str(exc)[:160]
|
||||
logging.getLogger("h3_decode_cache_helper").warning("Cache unavailable; native decode: %s", exc)
|
||||
images, video_events = self._stream(video_samples, video_vae, "video", reset)
|
||||
audio, audio_events = self._stream(audio_samples, audio_vae, "audio", reset)
|
||||
try:
|
||||
stats = self.service.store.stats()
|
||||
except Exception as exc:
|
||||
reraise_control(exc)
|
||||
stats = {"diagnostic_error": str(exc)[:160]}
|
||||
report = {"helper_version": "3.8.3", "mode": mode,
|
||||
"video": video_events, "audio": audio_events, "cache": stats}
|
||||
if config_error:
|
||||
report["cache_disabled_reason"] = config_error
|
||||
return images, audio, json.dumps(report, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"H3DecodeCacheHelper": H3DecodeCacheHelper}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"H3DecodeCacheHelper": "Decode Cache Helper"}
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Native decoder delegation. Cache errors fail open; decoder/cancellation errors do not."""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
|
||||
from .identity import make_key, vae_signature
|
||||
from .store import DecodeStore
|
||||
|
||||
LOG = logging.getLogger("h3_decode_cache_helper")
|
||||
|
||||
|
||||
def interrupt_check():
|
||||
try:
|
||||
mm = importlib.import_module("comfy.model_management")
|
||||
except ImportError:
|
||||
return
|
||||
mm.throw_exception_if_processing_interrupted()
|
||||
|
||||
|
||||
def reraise_control(exc):
|
||||
# ComfyUI's interruption is an Exception, unlike KeyboardInterrupt.
|
||||
# Never convert an interrupted Queue into another Core Decode attempt.
|
||||
if any(cls.__name__ == "InterruptProcessingException" for cls in type(exc).__mro__):
|
||||
raise exc
|
||||
|
||||
|
||||
def native_delegate(stream):
|
||||
if stream == "video":
|
||||
core = importlib.import_module("nodes").VAEDecode
|
||||
method = core.decode
|
||||
|
||||
def execute(vae, samples):
|
||||
return core().decode(vae=vae, samples=samples)[0]
|
||||
return method, execute
|
||||
helper = importlib.import_module("comfy_extras.nodes_audio").vae_decode_audio
|
||||
return helper, lambda vae, samples: helper(vae, samples)
|
||||
|
||||
|
||||
class DecodeService:
|
||||
def __init__(self, store=None, delegate_resolver=native_delegate,
|
||||
signature_fn=vae_signature, interrupt=interrupt_check):
|
||||
self.store = store or DecodeStore(interrupt=interrupt)
|
||||
self.delegate_resolver = delegate_resolver
|
||||
self.signature_fn = signature_fn
|
||||
self.interrupt = interrupt
|
||||
|
||||
def decode(self, samples, vae, stream, reset_token=0):
|
||||
self.interrupt()
|
||||
delegate_identity, native = self.delegate_resolver(stream)
|
||||
event = {"stream": stream, "status": "miss", "hash_s": 0.0,
|
||||
"read_s": 0.0, "decode_s": 0.0, "write_s": 0.0}
|
||||
signature = key = guards = None
|
||||
epoch = self.store.epoch
|
||||
if self.store.mode != "Off":
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
signature, guards = self.signature_fn(vae, stream, delegate_identity)
|
||||
key = make_key(samples, stream, signature, reset_token, self.interrupt)
|
||||
except Exception as exc:
|
||||
reraise_control(exc)
|
||||
event["status"] = "bypass"
|
||||
event["reason"] = str(exc)[:160]
|
||||
event["hash_s"] = time.perf_counter() - start
|
||||
else:
|
||||
event["status"] = "off"
|
||||
if key is not None:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
cached = self.store.get(key)
|
||||
if cached is not None:
|
||||
tensor, extra = cached
|
||||
event.update(status="hit", read_s=time.perf_counter()-start)
|
||||
return (tensor if stream == "video" else {"waveform": tensor, **extra}), event
|
||||
except Exception as exc:
|
||||
reraise_control(exc)
|
||||
event["cache_read_error"] = str(exc)[:160]
|
||||
LOG.warning("Decode cache read failed; using native decoder: %s", exc)
|
||||
event["read_s"] = time.perf_counter() - start
|
||||
|
||||
# Outside cache exception handlers. No blind retry of OOM, invalid
|
||||
# inputs, VAE errors, or interrupted Sampling/Decode.
|
||||
self.interrupt()
|
||||
start = time.perf_counter()
|
||||
decoded = native(vae, samples)
|
||||
event["decode_s"] = time.perf_counter() - start
|
||||
self.interrupt()
|
||||
if key is not None:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
after, _ = self.signature_fn(vae, stream, delegate_identity)
|
||||
if after != signature:
|
||||
event["store_skipped"] = "VAE identity changed during native decode"
|
||||
else:
|
||||
if stream == "video":
|
||||
tensor, extra = decoded, {}
|
||||
else:
|
||||
tensor = decoded["waveform"]
|
||||
extra = {k: v for k, v in decoded.items() if k != "waveform"}
|
||||
# Only JSON-compatible metadata is retained; store defensively copies it.
|
||||
json.dumps(extra, allow_nan=False)
|
||||
replacement = self.store.put(key, tensor, stream, extra, guards, epoch)
|
||||
event["stored"] = self.store.contains(key)
|
||||
if replacement is not None:
|
||||
tensor, extra = replacement
|
||||
decoded = tensor if stream == "video" else {"waveform": tensor, **extra}
|
||||
except Exception as exc:
|
||||
reraise_control(exc)
|
||||
event["cache_write_error"] = str(exc)[:160]
|
||||
LOG.warning("Decode cache write failed; returning native result: %s", exc)
|
||||
event["write_s"] = time.perf_counter() - start
|
||||
return decoded, event
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Private, disposable decoded tensors. No Run Storage or global GPU lifecycle access."""
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from .identity import CacheBypass, tensor_blocks, tensor_digest
|
||||
|
||||
MiB = 1024**2
|
||||
GiB = 1024**3
|
||||
|
||||
|
||||
def available_ram():
|
||||
try:
|
||||
import psutil # already provided by ComfyUI; optional for cache operation
|
||||
return int(psutil.virtual_memory().available)
|
||||
except (ImportError, OSError, AttributeError):
|
||||
return 0 # unknown headroom must not trigger RAM retention
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entry:
|
||||
shape: tuple
|
||||
dtype: str
|
||||
digest: str
|
||||
payload_bytes: int
|
||||
guards: tuple
|
||||
extra: dict
|
||||
tensor: torch.Tensor | None = None
|
||||
path: Path | None = None
|
||||
file_bytes: int = 0
|
||||
leases: list = field(default_factory=list)
|
||||
|
||||
def live(self):
|
||||
self.leases[:] = [ref for ref in self.leases if ref() is not None]
|
||||
return bool(self.leases)
|
||||
|
||||
|
||||
def _cleanup_private_files(state):
|
||||
"""Best effort at object/process exit; never close arrays still owned downstream."""
|
||||
for path, entry in list(state["entries"].items()):
|
||||
if not entry.live():
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
state["entries"].pop(path, None)
|
||||
except OSError:
|
||||
pass
|
||||
if state["root"] is not None:
|
||||
try:
|
||||
state["root"].rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class DecodeStore:
|
||||
"""One store per Helper instance. Auto uses disk for video and small RAM for audio.
|
||||
|
||||
RAM hits return a clone. Disk hits return a new copy-on-write mmap, never
|
||||
a writable shared backing file. Already-returned results outlive eviction.
|
||||
"""
|
||||
|
||||
def __init__(self, temp_parent=None, memory_probe=available_ram, disk_probe=shutil.disk_usage,
|
||||
headroom_bytes=2*GiB, interrupt=None):
|
||||
self._lock = threading.RLock()
|
||||
self.entries = OrderedDict()
|
||||
self.retired = []
|
||||
self.root = None
|
||||
self.temp_parent = temp_parent if temp_parent is not None else (os.environ.get("H3_DECODE_CACHE_TEMP_DIR") or None)
|
||||
self.memory_probe = memory_probe
|
||||
self.disk_probe = disk_probe
|
||||
self.headroom_bytes = headroom_bytes
|
||||
self.interrupt = interrupt or (lambda: None)
|
||||
self.ram_limit = 512*MiB
|
||||
self.disk_limit = 16*GiB
|
||||
self.mode = "Auto"
|
||||
self.reset_token = 0
|
||||
self.epoch = 0
|
||||
self.max_entries = 256
|
||||
# Finalizer state has only paths, disk-entry metadata and weak leases;
|
||||
# no decoded RAM tensor, VAE, store, node, or GPU model is retained.
|
||||
self._cleanup_state = {"root": None, "entries": {}}
|
||||
weakref.finalize(self, _cleanup_private_files, self._cleanup_state)
|
||||
|
||||
def configure(self, mode, ram_limit, disk_limit, reset_token):
|
||||
with self._lock:
|
||||
settings = (mode, int(ram_limit), int(disk_limit), int(reset_token))
|
||||
if settings != (self.mode, self.ram_limit, self.disk_limit, self.reset_token):
|
||||
self.clear()
|
||||
self.mode, self.ram_limit, self.disk_limit, self.reset_token = settings
|
||||
self._reap()
|
||||
if self.memory_probe() < self.headroom_bytes:
|
||||
for key, entry in list(self.entries.items()):
|
||||
if entry.tensor is not None:
|
||||
self._drop(key)
|
||||
|
||||
@property
|
||||
def ram_bytes(self):
|
||||
return sum(e.payload_bytes for e in self.entries.values() if e.tensor is not None)
|
||||
|
||||
@property
|
||||
def disk_bytes(self):
|
||||
return sum(e.file_bytes for e in self.entries.values() if e.path is not None) + sum(e.file_bytes for e in self.retired)
|
||||
|
||||
def _delete(self, entry):
|
||||
if entry.path is None:
|
||||
return True
|
||||
if entry.live():
|
||||
return False
|
||||
try:
|
||||
entry.path.unlink(missing_ok=True)
|
||||
self._cleanup_state["entries"].pop(entry.path, None)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
def _drop(self, key):
|
||||
entry = self.entries.pop(key)
|
||||
if entry.path is not None and not self._delete(entry):
|
||||
self.retired.append(entry)
|
||||
|
||||
def _reap(self):
|
||||
self.retired[:] = [e for e in self.retired if not self._delete(e)]
|
||||
for key, entry in list(self.entries.items()):
|
||||
if not all(ref() is not None for ref in entry.guards):
|
||||
self._drop(key)
|
||||
|
||||
def clear(self):
|
||||
with self._lock:
|
||||
self.epoch += 1
|
||||
for key in list(self.entries):
|
||||
self._drop(key)
|
||||
self._reap()
|
||||
|
||||
def close(self):
|
||||
self.clear()
|
||||
if self.root is not None:
|
||||
try:
|
||||
self.root.rmdir() # only an empty helper-owned directory
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _directory(self):
|
||||
if self.root is None:
|
||||
self.root = Path(tempfile.mkdtemp(prefix=f"h3-decode-cache-{os.getpid()}-", dir=self.temp_parent))
|
||||
self._cleanup_state["root"] = self.root
|
||||
return self.root
|
||||
|
||||
def _ram_room(self, size):
|
||||
while self.ram_bytes + size > self.ram_limit:
|
||||
victim = next((k for k, e in self.entries.items() if e.tensor is not None), None)
|
||||
if victim is None:
|
||||
return False
|
||||
self._drop(victim)
|
||||
# Budget is additional retention only, not a claim about whole-process RSS.
|
||||
return self.memory_probe() >= self.headroom_bytes + 2*size
|
||||
|
||||
def _disk_room(self, size):
|
||||
self._reap()
|
||||
while self.disk_bytes + size > self.disk_limit:
|
||||
victim = next((k for k, e in self.entries.items() if e.path is not None and not e.live()), None)
|
||||
if victim is None:
|
||||
return False
|
||||
self._drop(victim)
|
||||
root = self._directory()
|
||||
return self.disk_probe(root).free >= size + 256*MiB
|
||||
|
||||
def _read_disk(self, entry):
|
||||
path = entry.path
|
||||
if path is None or path.is_symlink() or path.stat().st_size != entry.file_bytes:
|
||||
raise CacheBypass("disk cache file missing/changed")
|
||||
# Verify the exact serialized payload before exposing a mapping.
|
||||
# Header/size are also checked; pickle is never allowed.
|
||||
array = np.load(path, mmap_mode="c", allow_pickle=False, max_header_size=4096)
|
||||
if tuple(array.shape) != entry.shape or array.dtype.str != entry.dtype or not array.flags.c_contiguous:
|
||||
raise CacheBypass("disk cache header mismatch")
|
||||
if array.nbytes != entry.payload_bytes:
|
||||
raise CacheBypass("disk cache size mismatch")
|
||||
raw = memoryview(array).cast("B")
|
||||
h = hashlib.sha256()
|
||||
for start in range(0, len(raw), 8*MiB):
|
||||
self.interrupt()
|
||||
h.update(raw[start:start+8*MiB])
|
||||
if h.hexdigest() != entry.digest:
|
||||
raise CacheBypass("disk cache checksum mismatch")
|
||||
entry.leases.append(weakref.ref(array))
|
||||
# mode='c' makes writable, private pages. mode='r' is NOT used.
|
||||
return torch.from_numpy(array)
|
||||
|
||||
def get(self, key):
|
||||
with self._lock:
|
||||
self._reap()
|
||||
entry = self.entries.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
if not all(ref() is not None for ref in entry.guards):
|
||||
self._drop(key)
|
||||
return None
|
||||
self.entries.move_to_end(key)
|
||||
if entry.tensor is not None:
|
||||
if self.memory_probe() < self.headroom_bytes + entry.payload_bytes:
|
||||
self._drop(key)
|
||||
return None
|
||||
if (tuple(entry.tensor.shape) != entry.shape
|
||||
or entry.tensor.numpy().dtype.str != entry.dtype
|
||||
or tensor_digest(entry.tensor, self.interrupt) != entry.digest):
|
||||
self._drop(key)
|
||||
raise CacheBypass("RAM cache checksum/header mismatch")
|
||||
# Independent storage protects the cache from in-place seam/color edits.
|
||||
return entry.tensor.clone(), deepcopy(entry.extra)
|
||||
try:
|
||||
tensor = self._read_disk(entry)
|
||||
except Exception:
|
||||
self._drop(key)
|
||||
raise
|
||||
return tensor, deepcopy(entry.extra)
|
||||
|
||||
def put(self, key, tensor, stream, extra, guards, expected_epoch):
|
||||
with self._lock:
|
||||
if expected_epoch != self.epoch or self.mode == "Off":
|
||||
return None
|
||||
if (not isinstance(tensor, torch.Tensor) or tensor.device.type != "cpu"
|
||||
or tensor.layout != torch.strided or tensor.is_nested or tensor.numel() == 0):
|
||||
return None
|
||||
if tensor.dtype not in (torch.float16, torch.float32, torch.float64):
|
||||
return None # never silently downcast decoded output
|
||||
size = tensor.numel() * tensor.element_size()
|
||||
if not all(ref() is not None for ref in guards):
|
||||
return None
|
||||
if key in self.entries:
|
||||
self._drop(key)
|
||||
while len(self.entries) >= self.max_entries:
|
||||
self._drop(next(iter(self.entries)))
|
||||
use_ram = self.mode == "RAM" or (stream == "audio" and size <= 16*MiB)
|
||||
if use_ram and self._ram_room(size):
|
||||
stored = tensor.detach().clone(memory_format=torch.contiguous_format)
|
||||
self.entries[key] = Entry(tuple(stored.shape), stored.numpy().dtype.str,
|
||||
tensor_digest(stored, self.interrupt), size, guards, deepcopy(extra), tensor=stored)
|
||||
return None # original MISS output is independently owned
|
||||
if self.mode != "Auto" or self.disk_limit == 0 or size+4096 > self.disk_limit:
|
||||
return None
|
||||
if not self._disk_room(size+4096):
|
||||
return None
|
||||
root = self._directory()
|
||||
filename = uuid.uuid4().hex
|
||||
temp = root / (filename + ".tmp")
|
||||
final = root / (filename + ".npy")
|
||||
dtype = {torch.float16: np.dtype("float16"), torch.float32: np.dtype("float32"),
|
||||
torch.float64: np.dtype("float64")}[tensor.dtype]
|
||||
h = hashlib.sha256()
|
||||
try:
|
||||
with open(temp, "xb") as f:
|
||||
np.lib.format.write_array_header_1_0(f, {
|
||||
"descr": dtype.str, "fortran_order": False, "shape": tuple(tensor.shape)})
|
||||
for block in tensor_blocks(tensor):
|
||||
self.interrupt()
|
||||
h.update(block)
|
||||
if f.write(block) != len(block):
|
||||
raise OSError("short cache write")
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(temp, final)
|
||||
entry = Entry(tuple(tensor.shape), dtype.str, h.hexdigest(), size, guards,
|
||||
deepcopy(extra), path=final, file_bytes=final.stat().st_size)
|
||||
self.entries[key] = entry
|
||||
self._cleanup_state["entries"][final] = entry
|
||||
# Returning a COW mapping releases the large anonymous MISS output
|
||||
# after this call rather than keeping an extra private RAM copy.
|
||||
return self.get(key)
|
||||
except BaseException:
|
||||
if key in self.entries:
|
||||
self._drop(key)
|
||||
for path in (temp, final):
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def contains(self, key):
|
||||
with self._lock:
|
||||
return key in self.entries
|
||||
|
||||
def stats(self):
|
||||
with self._lock:
|
||||
self._reap()
|
||||
return {"entries": len(self.entries), "ram_bytes": self.ram_bytes,
|
||||
"disk_bytes": self.disk_bytes, "pending_delete_files": len(self.retired)}
|
||||
Reference in New Issue
Block a user