feat: release V3.5.0
This commit is contained in:
@@ -0,0 +1,960 @@
|
||||
"""Phase 2A P2-0 probe for a Windows-safe file-backed IMAGE buffer.
|
||||
|
||||
The accepted storage/lifetime implementation remains here as the single tested
|
||||
source and is re-exported through ``v3.file_backed_buffer`` for the V3.5
|
||||
assembler. The standalone probes below continue to exercise the same code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import asdict, dataclass
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import queue
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable, Iterator
|
||||
import uuid
|
||||
import weakref
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
SCHEMA = "h3-continuum-p20-file-backed-tensor"
|
||||
SCHEMA_VERSION = 1
|
||||
FILE_PREFIX = "h3c-p20-"
|
||||
_ID_PATTERN = re.compile(r"^h3c-p20-([0-9a-f]{32})\.(bin|json)$")
|
||||
_PARTIAL_PATTERN = re.compile(r"^h3c-p20-([0-9a-f]{32})\.json\.partial$")
|
||||
_LOCK_FILE_NAME = ".h3c-p20.lock"
|
||||
_ROOT_THREAD_LOCKS: dict[str, threading.RLock] = {}
|
||||
_ROOT_THREAD_LOCKS_GUARD = threading.Lock()
|
||||
_ROOT_LOCK_STATE = threading.local()
|
||||
_PROCESS_MANAGERS: dict[str, "StorageLifetimeManager"] = {}
|
||||
_PROCESS_MANAGERS_GUARD = threading.Lock()
|
||||
|
||||
|
||||
class FileBackedPoCError(RuntimeError):
|
||||
"""Raised when the P2-0 backend cannot uphold its safety contract."""
|
||||
|
||||
|
||||
class StorageTrackingUnavailable(FileBackedPoCError):
|
||||
"""Raised when the current PyTorch cannot weakly track Storage lifetime."""
|
||||
|
||||
|
||||
class ProbeCancelled(BaseException):
|
||||
"""BaseException used to verify cancellation cleanup in the standalone PoC."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackingRecord:
|
||||
allocation_id: str
|
||||
data_path: Path
|
||||
metadata_path: Path
|
||||
shape: tuple[int, ...]
|
||||
dtype: torch.dtype
|
||||
numel: int
|
||||
nbytes: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CollectionResult:
|
||||
reclaimed: int
|
||||
deferred: int
|
||||
errors: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StaleCleanupResult:
|
||||
removed_pairs: int
|
||||
removed_orphans: int
|
||||
retained_live: int
|
||||
retained_young: int
|
||||
invalid_metadata: int
|
||||
deferred: int
|
||||
errors: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProbeResult:
|
||||
torch_version: str
|
||||
bit_exact: bool
|
||||
file_size: int
|
||||
expected_bytes: int
|
||||
root_released_while_alias_alive: bool
|
||||
deferred_while_alias_alive: int
|
||||
reclaimed_after_last_alias: int
|
||||
files_remaining: int
|
||||
|
||||
|
||||
def _storage_weak_ref(tensor: torch.Tensor) -> Any:
|
||||
"""Create an alias-aware weak reference to the underlying Storage.
|
||||
|
||||
``weakref.finalize(tensor, ...)`` is not sufficient: a Tensor view can keep
|
||||
the Storage alive after that particular Tensor object has died. The PoC
|
||||
feature-detects PyTorch's StorageWeakRef and fails closed when unavailable.
|
||||
"""
|
||||
|
||||
try:
|
||||
from torch.multiprocessing.reductions import StorageWeakRef
|
||||
|
||||
reference = StorageWeakRef(tensor.untyped_storage())
|
||||
if not callable(getattr(reference, "expired", None)):
|
||||
raise TypeError("StorageWeakRef.expired is unavailable")
|
||||
return reference
|
||||
except Exception as exc:
|
||||
raise StorageTrackingUnavailable(
|
||||
"this PyTorch build cannot track the lifetime of file-backed "
|
||||
"Tensor storage aliases"
|
||||
) from exc
|
||||
|
||||
|
||||
def _process_create_time(pid: int) -> float | None:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
return float(psutil.Process(int(pid)).create_time())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
_CURRENT_PROCESS_CREATE_TIME = _process_create_time(os.getpid())
|
||||
|
||||
|
||||
def _same_live_process(pid: int, expected_create_time: float | None) -> bool:
|
||||
"""Return True conservatively when a sidecar may belong to a live process."""
|
||||
|
||||
pid = int(pid)
|
||||
if pid <= 0:
|
||||
return False
|
||||
try:
|
||||
import psutil
|
||||
|
||||
process = psutil.Process(pid)
|
||||
actual_create_time = float(process.create_time())
|
||||
if expected_create_time is None:
|
||||
return bool(process.is_running())
|
||||
return bool(process.is_running()) and abs(
|
||||
actual_create_time - float(expected_create_time)
|
||||
) <= 1.0
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
if isinstance(exc, psutil.NoSuchProcess):
|
||||
return False
|
||||
if isinstance(exc, psutil.AccessDenied):
|
||||
return True
|
||||
except Exception:
|
||||
return True
|
||||
return True
|
||||
|
||||
if pid == os.getpid():
|
||||
if expected_create_time is None or _CURRENT_PROCESS_CREATE_TIME is None:
|
||||
return True
|
||||
return abs(
|
||||
float(expected_create_time) - _CURRENT_PROCESS_CREATE_TIME
|
||||
) <= 1.0
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _atomic_write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
partial_path = path.with_suffix(path.suffix + ".partial")
|
||||
with partial_path.open("x", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, sort_keys=True)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(partial_path, path)
|
||||
|
||||
|
||||
def _dtype_name(dtype: torch.dtype) -> str:
|
||||
value = str(dtype)
|
||||
return value[6:] if value.startswith("torch.") else value
|
||||
|
||||
|
||||
def _validate_shape(shape: tuple[int, ...]) -> tuple[int, ...]:
|
||||
normalized_values: list[int] = []
|
||||
for value in shape:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise TypeError("file-backed Tensor dimensions must be integers")
|
||||
normalized_values.append(value)
|
||||
normalized = tuple(normalized_values)
|
||||
if not normalized or any(value <= 0 for value in normalized):
|
||||
raise ValueError("file-backed Tensor shape must contain positive dimensions")
|
||||
return normalized
|
||||
|
||||
|
||||
def _known_path(root: Path, path: Path) -> bool:
|
||||
try:
|
||||
return path.parent.resolve() == root.resolve() and bool(
|
||||
_ID_PATTERN.fullmatch(path.name) or _PARTIAL_PATTERN.fullmatch(path.name)
|
||||
)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _unlink_known(root: Path, path: Path) -> bool:
|
||||
if not _known_path(root, path):
|
||||
raise FileBackedPoCError(f"refusing to unlink an unknown path: {path}")
|
||||
try:
|
||||
path.unlink()
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def _thread_lock_for_root(root: Path) -> threading.RLock:
|
||||
key = os.path.normcase(str(root.resolve()))
|
||||
with _ROOT_THREAD_LOCKS_GUARD:
|
||||
lock = _ROOT_THREAD_LOCKS.get(key)
|
||||
if lock is None:
|
||||
lock = threading.RLock()
|
||||
_ROOT_THREAD_LOCKS[key] = lock
|
||||
return lock
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _root_lock(root: Path):
|
||||
"""Serialize allocation and cleanup across threads and ComfyUI processes."""
|
||||
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
key = os.path.normcase(str(root.resolve()))
|
||||
thread_lock = _thread_lock_for_root(root)
|
||||
with thread_lock:
|
||||
depths = getattr(_ROOT_LOCK_STATE, "depths", None)
|
||||
if depths is None:
|
||||
depths = {}
|
||||
_ROOT_LOCK_STATE.depths = depths
|
||||
depth = int(depths.get(key, 0))
|
||||
if depth > 0:
|
||||
depths[key] = depth + 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
remaining = int(depths[key]) - 1
|
||||
if remaining > 0:
|
||||
depths[key] = remaining
|
||||
else:
|
||||
depths.pop(key, None)
|
||||
return
|
||||
|
||||
lock_path = root / _LOCK_FILE_NAME
|
||||
try:
|
||||
with lock_path.open("a+b") as handle:
|
||||
handle.seek(0, os.SEEK_END)
|
||||
if handle.tell() == 0:
|
||||
handle.write(b"\0")
|
||||
handle.flush()
|
||||
handle.seek(0)
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
|
||||
depths[key] = 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
depths[key] = 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
finally:
|
||||
depths.pop(key, None)
|
||||
|
||||
|
||||
class _TrackedStorage:
|
||||
__slots__ = ("record", "storage_reference")
|
||||
|
||||
def __init__(self, record: BackingRecord, tensor: torch.Tensor):
|
||||
self.record = record
|
||||
self.storage_reference = _storage_weak_ref(tensor)
|
||||
|
||||
def expired(self) -> bool:
|
||||
return bool(self.storage_reference.expired())
|
||||
|
||||
|
||||
class FileBackedAllocation:
|
||||
"""Provisional allocation that is deleted unless explicitly published."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
manager: "StorageLifetimeManager",
|
||||
tensor: torch.Tensor,
|
||||
record: BackingRecord,
|
||||
):
|
||||
self._manager = manager
|
||||
self._tensor: torch.Tensor | None = tensor
|
||||
self.record = record
|
||||
self._published = False
|
||||
self._cleanup_finalizer = weakref.finalize(
|
||||
self,
|
||||
manager._discard_unpublished,
|
||||
record,
|
||||
)
|
||||
|
||||
@property
|
||||
def tensor(self) -> torch.Tensor:
|
||||
if self._tensor is None:
|
||||
raise FileBackedPoCError("file-backed allocation is no longer available")
|
||||
return self._tensor
|
||||
|
||||
def publish(self) -> torch.Tensor:
|
||||
if self._published:
|
||||
raise FileBackedPoCError("file-backed allocation was already published")
|
||||
tensor = self.tensor
|
||||
try:
|
||||
self._manager._register(self.record, tensor)
|
||||
except BaseException:
|
||||
self.abort()
|
||||
raise
|
||||
self._published = True
|
||||
self._cleanup_finalizer.detach()
|
||||
self._tensor = None
|
||||
return tensor
|
||||
|
||||
def abort(self) -> None:
|
||||
if self._published:
|
||||
raise FileBackedPoCError("cannot abort a published file-backed Tensor")
|
||||
self._tensor = None
|
||||
if self._cleanup_finalizer.alive:
|
||||
self._cleanup_finalizer()
|
||||
# Explicit cancellation should attempt cleanup now. The finalizer only
|
||||
# enqueues an intent so GC never performs filesystem I/O or waits on a
|
||||
# Windows byte lock. Cleanup failures remain queued for a later sweep
|
||||
# and must not replace the exception that caused __exit__ to abort.
|
||||
try:
|
||||
self._manager.collect_ready()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def __enter__(self) -> "FileBackedAllocation":
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback) -> bool:
|
||||
if not self._published:
|
||||
self.abort()
|
||||
return False
|
||||
|
||||
|
||||
class StorageLifetimeManager:
|
||||
"""Track backing files without retaining their Tensor/storage strongly."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
root: str | os.PathLike[str],
|
||||
*,
|
||||
allocation_hook: Callable[[Path], None] | None = None,
|
||||
):
|
||||
self.root = Path(root).expanduser().resolve()
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
self._tracked: dict[str, _TrackedStorage] = {}
|
||||
self._pending: dict[str, tuple[Path, ...]] = {}
|
||||
self._abandoned: queue.SimpleQueue[BackingRecord] = queue.SimpleQueue()
|
||||
self._lock = threading.RLock()
|
||||
self._allocation_hook = allocation_hook
|
||||
|
||||
def allocate(
|
||||
self,
|
||||
shape: tuple[int, ...],
|
||||
*,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
) -> FileBackedAllocation:
|
||||
with _root_lock(self.root):
|
||||
with self._lock:
|
||||
self._drain_abandoned_locked()
|
||||
self._collect_ready_locked()
|
||||
return self._allocate_locked(shape, dtype=dtype)
|
||||
|
||||
def _allocate_locked(
|
||||
self,
|
||||
shape: tuple[int, ...],
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
) -> FileBackedAllocation:
|
||||
normalized_shape = _validate_shape(shape)
|
||||
if not isinstance(dtype, torch.dtype):
|
||||
raise TypeError("dtype must be a torch.dtype")
|
||||
numel = math.prod(normalized_shape)
|
||||
element_size = torch.empty((), dtype=dtype).element_size()
|
||||
allocation_id = uuid.uuid4().hex
|
||||
data_path = self.root / f"{FILE_PREFIX}{allocation_id}.bin"
|
||||
metadata_path = self.root / f"{FILE_PREFIX}{allocation_id}.json"
|
||||
descriptor: int | None = None
|
||||
tensor: torch.Tensor | None = None
|
||||
try:
|
||||
descriptor = os.open(
|
||||
data_path,
|
||||
os.O_CREAT | os.O_EXCL | os.O_RDWR,
|
||||
0o600,
|
||||
)
|
||||
try:
|
||||
os.close(descriptor)
|
||||
finally:
|
||||
# close() may raise after the OS already released the handle.
|
||||
# Never retry that numeric fd: another thread could already
|
||||
# have reused it for an unrelated file.
|
||||
descriptor = None
|
||||
flat = torch.from_file(
|
||||
str(data_path),
|
||||
shared=True,
|
||||
size=numel,
|
||||
dtype=dtype,
|
||||
)
|
||||
tensor = flat.reshape(normalized_shape)
|
||||
del flat
|
||||
if tensor.device.type != "cpu" or not tensor.is_contiguous():
|
||||
raise FileBackedPoCError(
|
||||
"torch.from_file did not create a contiguous CPU Tensor"
|
||||
)
|
||||
if self._allocation_hook is not None:
|
||||
self._allocation_hook(data_path)
|
||||
record = BackingRecord(
|
||||
allocation_id=allocation_id,
|
||||
data_path=data_path,
|
||||
metadata_path=metadata_path,
|
||||
shape=normalized_shape,
|
||||
dtype=dtype,
|
||||
numel=numel,
|
||||
nbytes=numel * element_size,
|
||||
)
|
||||
_atomic_write_json(
|
||||
metadata_path,
|
||||
{
|
||||
"schema": SCHEMA,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"allocation_id": allocation_id,
|
||||
"data_file": data_path.name,
|
||||
"shape": list(normalized_shape),
|
||||
"dtype": _dtype_name(dtype),
|
||||
"numel": numel,
|
||||
"nbytes": record.nbytes,
|
||||
"owner_pid": os.getpid(),
|
||||
"owner_create_time": _CURRENT_PROCESS_CREATE_TIME,
|
||||
"created_at": time.time(),
|
||||
},
|
||||
)
|
||||
return FileBackedAllocation(self, tensor, record)
|
||||
except BaseException:
|
||||
tensor = None
|
||||
if descriptor is not None:
|
||||
try:
|
||||
os.close(descriptor)
|
||||
except BaseException:
|
||||
# Preserve the original cancellation/error. On Windows an
|
||||
# actually-live descriptor will also make unlink fail, so
|
||||
# the backing paths remain in the retryable pending set.
|
||||
pass
|
||||
paths = (
|
||||
metadata_path.with_suffix(metadata_path.suffix + ".partial"),
|
||||
metadata_path,
|
||||
data_path,
|
||||
)
|
||||
failed = False
|
||||
for path in paths:
|
||||
try:
|
||||
_unlink_known(self.root, path)
|
||||
except BaseException:
|
||||
failed = True
|
||||
if failed:
|
||||
self._pending[allocation_id] = paths
|
||||
raise
|
||||
|
||||
def _register(self, record: BackingRecord, tensor: torch.Tensor) -> None:
|
||||
with _root_lock(self.root):
|
||||
with self._lock:
|
||||
if record.allocation_id in self._tracked:
|
||||
raise FileBackedPoCError("duplicate file-backed allocation ID")
|
||||
if not record.data_path.is_file() or not record.metadata_path.is_file():
|
||||
raise FileBackedPoCError("file-backed allocation files are missing")
|
||||
self._tracked[record.allocation_id] = _TrackedStorage(record, tensor)
|
||||
|
||||
def _discard_unpublished(self, record: BackingRecord) -> None:
|
||||
"""Queue GC cleanup without taking locks or touching the filesystem."""
|
||||
|
||||
self._abandoned.put(record)
|
||||
|
||||
def _drain_abandoned_locked(self) -> None:
|
||||
"""Move finalizer intents into the retryable pending set."""
|
||||
|
||||
while True:
|
||||
try:
|
||||
record = self._abandoned.get_nowait()
|
||||
except queue.Empty:
|
||||
return
|
||||
paths = (
|
||||
record.metadata_path.with_suffix(
|
||||
record.metadata_path.suffix + ".partial"
|
||||
),
|
||||
record.metadata_path,
|
||||
record.data_path,
|
||||
)
|
||||
if record.allocation_id not in self._tracked:
|
||||
self._pending[record.allocation_id] = paths
|
||||
|
||||
def collect_ready(self) -> CollectionResult:
|
||||
with _root_lock(self.root):
|
||||
with self._lock:
|
||||
self._drain_abandoned_locked()
|
||||
return self._collect_ready_locked()
|
||||
|
||||
def _collect_ready_locked(self) -> CollectionResult:
|
||||
reclaimed = 0
|
||||
deferred = 0
|
||||
errors: list[str] = []
|
||||
for allocation_id, paths in list(self._pending.items()):
|
||||
failed = False
|
||||
for path in paths:
|
||||
try:
|
||||
_unlink_known(self.root, path)
|
||||
except OSError as exc:
|
||||
failed = True
|
||||
errors.append(f"{path.name}: {type(exc).__name__}: {exc}")
|
||||
if failed:
|
||||
deferred += 1
|
||||
continue
|
||||
del self._pending[allocation_id]
|
||||
reclaimed += 1
|
||||
for allocation_id, tracked in list(self._tracked.items()):
|
||||
try:
|
||||
expired = tracked.expired()
|
||||
except Exception as exc:
|
||||
deferred += 1
|
||||
errors.append(
|
||||
f"{allocation_id}: storage lifetime unavailable: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
continue
|
||||
if not expired:
|
||||
deferred += 1
|
||||
continue
|
||||
failed = False
|
||||
for path in (tracked.record.data_path, tracked.record.metadata_path):
|
||||
try:
|
||||
_unlink_known(self.root, path)
|
||||
except OSError as exc:
|
||||
failed = True
|
||||
errors.append(
|
||||
f"{path.name}: {type(exc).__name__}: {exc}"
|
||||
)
|
||||
if failed:
|
||||
deferred += 1
|
||||
continue
|
||||
del self._tracked[allocation_id]
|
||||
reclaimed += 1
|
||||
return CollectionResult(reclaimed, deferred, tuple(errors))
|
||||
|
||||
def tracked_count(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._tracked)
|
||||
|
||||
def pending_count(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._pending)
|
||||
|
||||
|
||||
def get_process_lifetime_manager(
|
||||
root: str | os.PathLike[str],
|
||||
) -> StorageLifetimeManager:
|
||||
"""Return a process-global manager so cache lifetime outlives one node call."""
|
||||
|
||||
resolved = Path(root).expanduser().resolve()
|
||||
key = os.path.normcase(str(resolved))
|
||||
with _PROCESS_MANAGERS_GUARD:
|
||||
manager = _PROCESS_MANAGERS.get(key)
|
||||
if manager is None:
|
||||
manager = StorageLifetimeManager(resolved)
|
||||
_PROCESS_MANAGERS[key] = manager
|
||||
return manager
|
||||
|
||||
|
||||
def _sidecar_age(metadata_path: Path, payload: dict[str, Any], now: float) -> float:
|
||||
created_at = payload.get("created_at")
|
||||
if isinstance(created_at, (int, float)) and math.isfinite(float(created_at)):
|
||||
return max(0.0, now - float(created_at))
|
||||
return max(0.0, now - metadata_path.stat().st_mtime)
|
||||
|
||||
|
||||
def _valid_sidecar(
|
||||
metadata_path: Path,
|
||||
payload: dict[str, Any],
|
||||
) -> tuple[bool, Path]:
|
||||
match = _ID_PATTERN.fullmatch(metadata_path.name)
|
||||
if match is None or match.group(2) != "json":
|
||||
return False, metadata_path.with_suffix(".bin")
|
||||
expected_data = metadata_path.with_suffix(".bin")
|
||||
shape = payload.get("shape")
|
||||
owner_pid = payload.get("owner_pid")
|
||||
owner_create_time = payload.get("owner_create_time")
|
||||
created_at = payload.get("created_at")
|
||||
numel = payload.get("numel")
|
||||
nbytes = payload.get("nbytes")
|
||||
dtype = payload.get("dtype")
|
||||
shape_valid = (
|
||||
isinstance(shape, list)
|
||||
and bool(shape)
|
||||
and all(type(value) is int and value > 0 for value in shape)
|
||||
)
|
||||
ownership_valid = (
|
||||
type(owner_pid) is int
|
||||
and owner_pid > 0
|
||||
and "owner_create_time" in payload
|
||||
and (
|
||||
owner_create_time is None
|
||||
or (
|
||||
isinstance(owner_create_time, (int, float))
|
||||
and not isinstance(owner_create_time, bool)
|
||||
and math.isfinite(float(owner_create_time))
|
||||
and float(owner_create_time) > 0.0
|
||||
)
|
||||
)
|
||||
and isinstance(created_at, (int, float))
|
||||
and not isinstance(created_at, bool)
|
||||
and math.isfinite(float(created_at))
|
||||
and float(created_at) > 0.0
|
||||
)
|
||||
size_valid = (
|
||||
type(numel) is int
|
||||
and numel > 0
|
||||
and shape_valid
|
||||
and numel == math.prod(shape)
|
||||
and type(nbytes) is int
|
||||
and nbytes > 0
|
||||
and isinstance(dtype, str)
|
||||
and bool(dtype)
|
||||
)
|
||||
valid = (
|
||||
payload.get("schema") == SCHEMA
|
||||
and payload.get("schema_version") == SCHEMA_VERSION
|
||||
and payload.get("allocation_id") == match.group(1)
|
||||
and payload.get("data_file") == expected_data.name
|
||||
and ownership_valid
|
||||
and size_valid
|
||||
)
|
||||
return bool(valid), expected_data
|
||||
|
||||
|
||||
def cleanup_stale_files(
|
||||
root: str | os.PathLike[str],
|
||||
*,
|
||||
grace_seconds: float,
|
||||
now: float | None = None,
|
||||
process_is_live: Callable[[int, float | None], bool] = _same_live_process,
|
||||
) -> StaleCleanupResult:
|
||||
"""Remove only dead-process artifacts inside the dedicated P2-0 root."""
|
||||
|
||||
root_path = Path(root).expanduser().resolve()
|
||||
root_path.mkdir(parents=True, exist_ok=True)
|
||||
with _root_lock(root_path):
|
||||
return _cleanup_stale_files_locked(
|
||||
root_path,
|
||||
grace_seconds=grace_seconds,
|
||||
now=now,
|
||||
process_is_live=process_is_live,
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_stale_files_locked(
|
||||
root_path: Path,
|
||||
*,
|
||||
grace_seconds: float,
|
||||
now: float | None,
|
||||
process_is_live: Callable[[int, float | None], bool],
|
||||
) -> StaleCleanupResult:
|
||||
grace_seconds = max(0.0, float(grace_seconds))
|
||||
current_time = time.time() if now is None else float(now)
|
||||
removed_pairs = 0
|
||||
removed_orphans = 0
|
||||
retained_live = 0
|
||||
retained_young = 0
|
||||
invalid_metadata = 0
|
||||
deferred = 0
|
||||
errors: list[str] = []
|
||||
referenced_data: set[str] = set()
|
||||
protected_data: set[str] = set()
|
||||
|
||||
for metadata_path in sorted(root_path.glob(f"{FILE_PREFIX}*.json")):
|
||||
if not _known_path(root_path, metadata_path):
|
||||
continue
|
||||
expected_data = metadata_path.with_suffix(".bin")
|
||||
protected_data.add(expected_data.name)
|
||||
try:
|
||||
payload = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("metadata root is not an object")
|
||||
except Exception as exc:
|
||||
invalid_metadata += 1
|
||||
errors.append(
|
||||
f"{metadata_path.name}: invalid metadata: {type(exc).__name__}: {exc}"
|
||||
)
|
||||
continue
|
||||
valid, data_path = _valid_sidecar(metadata_path, payload)
|
||||
if not valid:
|
||||
invalid_metadata += 1
|
||||
errors.append(f"{metadata_path.name}: metadata contract mismatch")
|
||||
continue
|
||||
referenced_data.add(data_path.name)
|
||||
try:
|
||||
age = _sidecar_age(metadata_path, payload, current_time)
|
||||
except OSError as exc:
|
||||
deferred += 1
|
||||
errors.append(f"{metadata_path.name}: {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
try:
|
||||
pid = int(payload.get("owner_pid"))
|
||||
except (TypeError, ValueError):
|
||||
pid = -1
|
||||
create_time_value = payload.get("owner_create_time")
|
||||
create_time = (
|
||||
float(create_time_value)
|
||||
if isinstance(create_time_value, (int, float))
|
||||
else None
|
||||
)
|
||||
if process_is_live(pid, create_time):
|
||||
retained_live += 1
|
||||
continue
|
||||
if age < grace_seconds:
|
||||
retained_young += 1
|
||||
continue
|
||||
pair_failed = False
|
||||
for path in (data_path, metadata_path):
|
||||
try:
|
||||
_unlink_known(root_path, path)
|
||||
except OSError as exc:
|
||||
pair_failed = True
|
||||
errors.append(f"{path.name}: {type(exc).__name__}: {exc}")
|
||||
if pair_failed:
|
||||
deferred += 1
|
||||
else:
|
||||
removed_pairs += 1
|
||||
|
||||
for data_path in sorted(root_path.glob(f"{FILE_PREFIX}*.bin")):
|
||||
if not _known_path(root_path, data_path):
|
||||
continue
|
||||
if data_path.name in referenced_data or data_path.name in protected_data:
|
||||
continue
|
||||
try:
|
||||
age = max(0.0, current_time - data_path.stat().st_mtime)
|
||||
if age < grace_seconds:
|
||||
retained_young += 1
|
||||
continue
|
||||
_unlink_known(root_path, data_path)
|
||||
removed_orphans += 1
|
||||
except OSError as exc:
|
||||
deferred += 1
|
||||
errors.append(f"{data_path.name}: {type(exc).__name__}: {exc}")
|
||||
|
||||
for partial_path in sorted(root_path.glob(f"{FILE_PREFIX}*.json.partial")):
|
||||
if not _known_path(root_path, partial_path):
|
||||
continue
|
||||
try:
|
||||
age = max(0.0, current_time - partial_path.stat().st_mtime)
|
||||
if age < grace_seconds:
|
||||
retained_young += 1
|
||||
continue
|
||||
_unlink_known(root_path, partial_path)
|
||||
removed_orphans += 1
|
||||
except OSError as exc:
|
||||
deferred += 1
|
||||
errors.append(f"{partial_path.name}: {type(exc).__name__}: {exc}")
|
||||
|
||||
return StaleCleanupResult(
|
||||
removed_pairs=removed_pairs,
|
||||
removed_orphans=removed_orphans,
|
||||
retained_live=retained_live,
|
||||
retained_young=retained_young,
|
||||
invalid_metadata=invalid_metadata,
|
||||
deferred=deferred,
|
||||
errors=tuple(errors),
|
||||
)
|
||||
|
||||
|
||||
def _raw_float32_bytes(tensor: torch.Tensor) -> bytes:
|
||||
value = tensor.detach().to(device="cpu", dtype=torch.float32)
|
||||
if not value.is_contiguous():
|
||||
value = value.contiguous()
|
||||
return value.numpy().tobytes(order="C")
|
||||
|
||||
|
||||
def _remaining_known_files(root: Path) -> Iterator[Path]:
|
||||
for path in root.iterdir():
|
||||
if _known_path(root, path):
|
||||
yield path
|
||||
|
||||
|
||||
def run_probe(root: str | os.PathLike[str]) -> ProbeResult:
|
||||
manager = get_process_lifetime_manager(root)
|
||||
cleanup_stale_files(manager.root, grace_seconds=0.0)
|
||||
reference = torch.arange(2 * 3 * 4 * 5, dtype=torch.float32).reshape(2, 3, 4, 5)
|
||||
with manager.allocate(tuple(reference.shape), dtype=reference.dtype) as allocation:
|
||||
allocation.tensor.copy_(reference)
|
||||
record = allocation.record
|
||||
output = allocation.publish()
|
||||
|
||||
bit_exact = torch.equal(output, reference) and (
|
||||
record.data_path.read_bytes() == _raw_float32_bytes(reference)
|
||||
)
|
||||
actual_file_size = record.data_path.stat().st_size
|
||||
alias = output[1]
|
||||
cache = {"output": alias}
|
||||
output_reference = weakref.ref(output)
|
||||
del output
|
||||
gc.collect()
|
||||
deferred = manager.collect_ready()
|
||||
root_released = output_reference() is None and float(cache["output"].sum()) >= 0.0
|
||||
del alias
|
||||
cache.clear()
|
||||
gc.collect()
|
||||
reclaimed = manager.collect_ready()
|
||||
return ProbeResult(
|
||||
torch_version=str(torch.__version__),
|
||||
bit_exact=bool(bit_exact),
|
||||
file_size=actual_file_size,
|
||||
expected_bytes=reference.numel() * reference.element_size(),
|
||||
root_released_while_alias_alive=root_released,
|
||||
deferred_while_alias_alive=deferred.deferred,
|
||||
reclaimed_after_last_alias=reclaimed.reclaimed,
|
||||
files_remaining=sum(1 for _ in _remaining_known_files(manager.root)),
|
||||
)
|
||||
|
||||
|
||||
def _crash_child(root: str | os.PathLike[str]) -> None:
|
||||
manager = StorageLifetimeManager(root)
|
||||
with manager.allocate((2, 3, 4), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.fill_(1.0)
|
||||
output = allocation.publish()
|
||||
if float(output.sum()) <= 0.0:
|
||||
os._exit(3)
|
||||
os._exit(0)
|
||||
|
||||
|
||||
def _lock_order_child(root: str | os.PathLike[str]) -> None:
|
||||
"""Regression child for root/manager lock order and GC finalizer safety."""
|
||||
|
||||
root_path = Path(root).expanduser().resolve()
|
||||
manager = StorageLifetimeManager(root_path)
|
||||
gc.disable()
|
||||
allocation = manager.allocate((2, 2), dtype=torch.float32)
|
||||
abandoned_cycle: list[Any] = [allocation]
|
||||
abandoned_cycle.append(abandoned_cycle)
|
||||
del allocation
|
||||
del abandoned_cycle
|
||||
|
||||
manager_lock_acquired = threading.Event()
|
||||
cleanup_callback_entered = threading.Event()
|
||||
collector_started = threading.Event()
|
||||
failures: list[BaseException] = []
|
||||
original_manager_lock = manager._lock
|
||||
|
||||
class SignalingLock:
|
||||
def __enter__(self):
|
||||
original_manager_lock.acquire()
|
||||
manager_lock_acquired.set()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
original_manager_lock.release()
|
||||
return False
|
||||
|
||||
manager._lock = SignalingLock() # type: ignore[assignment]
|
||||
|
||||
def process_is_live(_pid: int, _create_time: float | None) -> bool:
|
||||
cleanup_callback_entered.set()
|
||||
if not collector_started.wait(timeout=2.0):
|
||||
raise RuntimeError("collector did not start")
|
||||
# With root-before-manager ordering the collector must still be waiting
|
||||
# for the root lock here. The GC finalizer must only enqueue cleanup.
|
||||
manager_lock_acquired.wait(timeout=0.25)
|
||||
gc.collect()
|
||||
return True
|
||||
|
||||
def run_cleanup() -> None:
|
||||
try:
|
||||
cleanup_stale_files(
|
||||
root_path,
|
||||
grace_seconds=0.0,
|
||||
process_is_live=process_is_live,
|
||||
)
|
||||
except BaseException as exc:
|
||||
failures.append(exc)
|
||||
|
||||
def run_collector() -> None:
|
||||
collector_started.set()
|
||||
try:
|
||||
manager.collect_ready()
|
||||
except BaseException as exc:
|
||||
failures.append(exc)
|
||||
|
||||
cleanup_thread = threading.Thread(target=run_cleanup, daemon=True)
|
||||
cleanup_thread.start()
|
||||
if not cleanup_callback_entered.wait(timeout=2.0):
|
||||
os._exit(90)
|
||||
collector_thread = threading.Thread(target=run_collector, daemon=True)
|
||||
collector_thread.start()
|
||||
cleanup_thread.join(timeout=3.0)
|
||||
collector_thread.join(timeout=3.0)
|
||||
if cleanup_thread.is_alive() or collector_thread.is_alive():
|
||||
os._exit(91)
|
||||
if failures:
|
||||
os._exit(92)
|
||||
manager.collect_ready()
|
||||
if any(_remaining_known_files(root_path)):
|
||||
os._exit(93)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
parser.add_argument("--crash-child", action="store_true", help=argparse.SUPPRESS)
|
||||
parser.add_argument(
|
||||
"--lock-order-child",
|
||||
action="store_true",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
arguments = parser.parse_args(argv)
|
||||
if arguments.crash_child:
|
||||
_crash_child(arguments.root)
|
||||
return 0
|
||||
if arguments.lock_order_child:
|
||||
_lock_order_child(arguments.root)
|
||||
return 0
|
||||
result = run_probe(arguments.root)
|
||||
print(json.dumps(asdict(result), ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0 if all(
|
||||
(
|
||||
result.bit_exact,
|
||||
result.file_size == result.expected_bytes,
|
||||
result.root_released_while_alias_alive,
|
||||
result.deferred_while_alias_alive == 1,
|
||||
result.reclaimed_after_last_alias == 1,
|
||||
result.files_remaining == 0,
|
||||
)
|
||||
) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,676 @@
|
||||
"""Phase 2A P2-4 probe against the installed ComfyUI execution/cache stack.
|
||||
|
||||
The probe uses deterministic decoded CPU IMAGE/AUDIO fixtures. It does not
|
||||
load a model, run a VAE, or use CUDA. Its purpose is to exercise the real
|
||||
ComfyUI cache, Core Preview/Save nodes, installed VHS Video Combine, Core ESC
|
||||
interrupt propagation, and restart-style stale cleanup around the V3.5
|
||||
Disk-backed assembler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class ProbeFailure(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class _Server:
|
||||
def __init__(self):
|
||||
self.client_id = None
|
||||
self.last_node_id = None
|
||||
self.messages: list[tuple[str, dict[str, Any], Any]] = []
|
||||
|
||||
def send_sync(self, event, data, client_id):
|
||||
self.messages.append((event, data, client_id))
|
||||
|
||||
|
||||
def _load_package(package_root: Path):
|
||||
name = "h3_continuum_p24_probe"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
name,
|
||||
package_root / "__init__.py",
|
||||
submodule_search_locations=[str(package_root)],
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ProbeFailure("cannot create Continuum package import specification")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _install_paths(comfy_root: Path) -> None:
|
||||
for value in (
|
||||
comfy_root,
|
||||
comfy_root / "custom_nodes" / "comfyui-videohelpersuite",
|
||||
):
|
||||
text = str(value)
|
||||
if text not in sys.path:
|
||||
sys.path.insert(0, text)
|
||||
|
||||
|
||||
def _single_plan(package_name: str) -> dict[str, Any]:
|
||||
hardening = importlib.import_module(f"{package_name}.hardening")
|
||||
temporal = importlib.import_module(f"{package_name}.temporal")
|
||||
plan_module = importlib.import_module(f"{package_name}.v3.plan")
|
||||
total_frames = 124
|
||||
chunk = {
|
||||
"sequence_index": 1,
|
||||
"chunk_index": 1,
|
||||
"total_frames": total_frames,
|
||||
"trim_frames": 0,
|
||||
"net_frames": total_frames,
|
||||
"context_frames": 0,
|
||||
"expected_video_latent_t": temporal.video_latent_t(total_frames),
|
||||
"expected_audio_latent_t": temporal.audio_latent_t(total_frames),
|
||||
}
|
||||
return hardening.enrich_assembly_plan(
|
||||
{
|
||||
"magic": plan_module.ASSEMBLY_PLAN_MAGIC,
|
||||
"schema_version": 1,
|
||||
"fps": 24,
|
||||
"width": 16,
|
||||
"height": 16,
|
||||
"chunk_seconds": 5.0,
|
||||
"target_frames": 120,
|
||||
"preserve_final_frame": True,
|
||||
"chunks": [chunk],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _register_probe_nodes(core_nodes, package_module, video_combine):
|
||||
package_name = package_module.__name__
|
||||
|
||||
class P24DecodedFixture:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"nonce": ("INT", {"default": 0}),
|
||||
"interrupt_before_assembly": (
|
||||
"BOOLEAN",
|
||||
{"default": False},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = (
|
||||
"IMAGE",
|
||||
"AUDIO",
|
||||
"H3_CONTINUUM_ASSEMBLY_PLAN",
|
||||
)
|
||||
RETURN_NAMES = ("images", "audio", "assembly_plan")
|
||||
FUNCTION = "build"
|
||||
CATEGORY = "P2-4 Probe"
|
||||
|
||||
def build(self, nonce, interrupt_before_assembly):
|
||||
total_frames = 124
|
||||
values = (
|
||||
torch.arange(total_frames, dtype=torch.float32)
|
||||
+ float(int(nonce) % 7)
|
||||
) / float(total_frames + 7)
|
||||
images = values.reshape(-1, 1, 1, 1).expand(
|
||||
-1,
|
||||
16,
|
||||
16,
|
||||
3,
|
||||
)
|
||||
sample_rate = 32_000
|
||||
samples = int(round(total_frames / 24 * sample_rate))
|
||||
audio = {
|
||||
"waveform": torch.linspace(
|
||||
-0.05,
|
||||
0.05,
|
||||
samples,
|
||||
dtype=torch.float32,
|
||||
).reshape(1, 1, -1),
|
||||
"sample_rate": sample_rate,
|
||||
}
|
||||
plan = _single_plan(package_name)
|
||||
if bool(interrupt_before_assembly):
|
||||
core_nodes.interrupt_processing(True)
|
||||
return images, audio, plan
|
||||
|
||||
class P24FailOrPass:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
"fail": ("BOOLEAN", {"default": True}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "consume"
|
||||
OUTPUT_NODE = True
|
||||
CATEGORY = "P2-4 Probe"
|
||||
|
||||
def consume(self, images, fail):
|
||||
if bool(fail):
|
||||
raise RuntimeError("P2-4 intentional downstream failure")
|
||||
return {
|
||||
"ui": {"p24": ["downstream pass"]},
|
||||
"result": (images,),
|
||||
}
|
||||
|
||||
class P24ClearCacheOutput:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {"required": {"token": ("INT", {"default": 0})}}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "finish"
|
||||
OUTPUT_NODE = True
|
||||
CATEGORY = "P2-4 Probe"
|
||||
|
||||
def finish(self, token):
|
||||
return (f"clear-{int(token)}",)
|
||||
|
||||
core_nodes.NODE_CLASS_MAPPINGS.update(package_module.NODE_CLASS_MAPPINGS)
|
||||
core_nodes.NODE_CLASS_MAPPINGS.update(
|
||||
{
|
||||
"P24DecodedFixture": P24DecodedFixture,
|
||||
"P24FailOrPass": P24FailOrPass,
|
||||
"P24ClearCacheOutput": P24ClearCacheOutput,
|
||||
"P24VHSVideoCombine": video_combine,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _base_prompt(*, nonce: int, interrupt_before_assembly: bool) -> dict[str, Any]:
|
||||
return {
|
||||
"1": {
|
||||
"class_type": "P24DecodedFixture",
|
||||
"inputs": {
|
||||
"nonce": int(nonce),
|
||||
"interrupt_before_assembly": bool(interrupt_before_assembly),
|
||||
},
|
||||
},
|
||||
"2": {
|
||||
"class_type": "H3ContinuumAssembleSeamV35",
|
||||
"inputs": {
|
||||
"images": ["1", 0],
|
||||
"audio": ["1", 1],
|
||||
"assembly_plan": ["1", 2],
|
||||
"exact_total_duration": True,
|
||||
"audio_seam": "Off",
|
||||
"video_seam": "Off",
|
||||
"buffer_backend": "Disk-backed",
|
||||
"diagnostics": "Basic",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _failure_prompt(*, fail: bool, nonce: int = 0) -> dict[str, Any]:
|
||||
prompt = _base_prompt(nonce=nonce, interrupt_before_assembly=False)
|
||||
prompt["3"] = {
|
||||
"class_type": "P24FailOrPass",
|
||||
"inputs": {"images": ["2", 0], "fail": bool(fail)},
|
||||
}
|
||||
return prompt
|
||||
|
||||
|
||||
def _compatibility_prompt() -> dict[str, Any]:
|
||||
prompt = _base_prompt(nonce=0, interrupt_before_assembly=False)
|
||||
prompt.update(
|
||||
{
|
||||
"4": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {
|
||||
"images": ["2", 0],
|
||||
"filename_prefix": "P24/Save",
|
||||
},
|
||||
},
|
||||
"5": {
|
||||
"class_type": "PreviewImage",
|
||||
"inputs": {"images": ["2", 0]},
|
||||
},
|
||||
"6": {
|
||||
"class_type": "P24VHSVideoCombine",
|
||||
"inputs": {
|
||||
"images": ["2", 0],
|
||||
"audio": ["2", 1],
|
||||
"frame_rate": 24,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "P24/VHS",
|
||||
"format": "video/h264-mp4",
|
||||
"pingpong": False,
|
||||
"save_output": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
return prompt
|
||||
|
||||
|
||||
def _interrupt_prompt() -> dict[str, Any]:
|
||||
prompt = _base_prompt(nonce=99, interrupt_before_assembly=True)
|
||||
prompt["3"] = {
|
||||
"class_type": "P24FailOrPass",
|
||||
"inputs": {"images": ["2", 0], "fail": False},
|
||||
}
|
||||
return prompt
|
||||
|
||||
|
||||
def _clear_prompt(token: int) -> dict[str, Any]:
|
||||
return {
|
||||
"9": {
|
||||
"class_type": "P24ClearCacheOutput",
|
||||
"inputs": {"token": int(token)},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _cached_nodes(executor) -> set[str]:
|
||||
for event, data in executor.status_messages:
|
||||
if event == "execution_cached":
|
||||
return {str(value) for value in data.get("nodes", [])}
|
||||
return set()
|
||||
|
||||
|
||||
def _has_event(executor, wanted: str) -> bool:
|
||||
return any(event == wanted for event, _ in executor.status_messages)
|
||||
|
||||
|
||||
def _execute(executor, prompt, prompt_id: str, outputs: list[str]) -> None:
|
||||
executor.execute(
|
||||
prompt,
|
||||
prompt_id,
|
||||
extra_data={},
|
||||
execute_outputs=outputs,
|
||||
)
|
||||
|
||||
|
||||
def _managed_files(backing_root: Path, suffix: str) -> list[Path]:
|
||||
return sorted(backing_root.glob(f"h3c-p20-*.{suffix}"))
|
||||
|
||||
|
||||
def _all_output_files(root: Path) -> set[str]:
|
||||
if not root.exists():
|
||||
return set()
|
||||
return {
|
||||
str(path.relative_to(root))
|
||||
for path in root.rglob("*")
|
||||
if path.is_file()
|
||||
}
|
||||
|
||||
|
||||
def _assert_output_paths(output_root: Path, temp_root: Path, history: dict) -> dict:
|
||||
outputs = history.get("outputs", {})
|
||||
save_items = outputs.get("4", {}).get("images", [])
|
||||
preview_items = outputs.get("5", {}).get("images", [])
|
||||
gifs = outputs.get("6", {}).get("gifs", [])
|
||||
if len(save_items) != 120:
|
||||
raise ProbeFailure(f"Core SaveImage wrote {len(save_items)} frames, expected 120")
|
||||
if len(preview_items) != 120:
|
||||
raise ProbeFailure(
|
||||
f"Core PreviewImage wrote {len(preview_items)} frames, expected 120"
|
||||
)
|
||||
if len(gifs) != 1:
|
||||
raise ProbeFailure(f"VHS returned {len(gifs)} previews, expected 1")
|
||||
|
||||
for item in save_items:
|
||||
path = output_root / item.get("subfolder", "") / item["filename"]
|
||||
if not path.is_file() or path.stat().st_size <= 0:
|
||||
raise ProbeFailure(f"Core SaveImage output missing: {path}")
|
||||
for item in preview_items:
|
||||
path = temp_root / item.get("subfolder", "") / item["filename"]
|
||||
if not path.is_file() or path.stat().st_size <= 0:
|
||||
raise ProbeFailure(f"Core PreviewImage output missing: {path}")
|
||||
video_path = Path(gifs[0]["fullpath"])
|
||||
if not video_path.is_file() or video_path.stat().st_size <= 0:
|
||||
raise ProbeFailure(f"VHS output missing: {video_path}")
|
||||
ffprobe = shutil.which("ffprobe")
|
||||
if ffprobe is None:
|
||||
raise ProbeFailure("ffprobe is required for P2-4 VHS acceptance")
|
||||
inspected = subprocess.run(
|
||||
[
|
||||
ffprobe,
|
||||
"-v",
|
||||
"error",
|
||||
"-count_frames",
|
||||
"-show_entries",
|
||||
(
|
||||
"stream=codec_type,codec_name,width,height,r_frame_rate,"
|
||||
"nb_read_frames,sample_rate,channels,duration:"
|
||||
"format=duration,size"
|
||||
),
|
||||
"-of",
|
||||
"json",
|
||||
str(video_path),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
media = json.loads(inspected.stdout)
|
||||
video_stream = next(
|
||||
(item for item in media.get("streams", []) if item.get("codec_type") == "video"),
|
||||
None,
|
||||
)
|
||||
audio_stream = next(
|
||||
(item for item in media.get("streams", []) if item.get("codec_type") == "audio"),
|
||||
None,
|
||||
)
|
||||
if video_stream is None or audio_stream is None:
|
||||
raise ProbeFailure("VHS output does not contain both video and audio")
|
||||
if (
|
||||
video_stream.get("codec_name") != "h264"
|
||||
or int(video_stream.get("nb_read_frames", 0)) != 120
|
||||
or video_stream.get("r_frame_rate") != "24/1"
|
||||
or int(video_stream.get("width", 0)) != 16
|
||||
or int(video_stream.get("height", 0)) != 16
|
||||
or int(audio_stream.get("sample_rate", 0)) != 32_000
|
||||
):
|
||||
raise ProbeFailure(f"unexpected VHS media contract: {media}")
|
||||
return {
|
||||
"save_frames": len(save_items),
|
||||
"preview_frames": len(preview_items),
|
||||
"vhs_file": str(video_path),
|
||||
"vhs_bytes": int(video_path.stat().st_size),
|
||||
"vhs_video_frames": int(video_stream["nb_read_frames"]),
|
||||
"vhs_video_fps": video_stream["r_frame_rate"],
|
||||
"vhs_video_duration": float(video_stream["duration"]),
|
||||
"vhs_audio_sample_rate": int(audio_stream["sample_rate"]),
|
||||
"vhs_container_duration": float(media["format"]["duration"]),
|
||||
}
|
||||
|
||||
|
||||
def _collect_until_empty(manager, backing_root: Path) -> None:
|
||||
for _ in range(8):
|
||||
gc.collect()
|
||||
manager.collect_ready()
|
||||
if not _managed_files(backing_root, "bin") and not _managed_files(
|
||||
backing_root,
|
||||
"json",
|
||||
):
|
||||
return
|
||||
time.sleep(0.05)
|
||||
raise ProbeFailure("released cache storage did not become reclaimable")
|
||||
|
||||
|
||||
def _run_child_mode(args, package_root: Path) -> int:
|
||||
_install_paths(args.comfy_root)
|
||||
package = _load_package(package_root)
|
||||
buffers = importlib.import_module(f"{package.__name__}.v3.file_backed_buffer")
|
||||
root = args.backing_root.resolve()
|
||||
if args.child_mode == "create-stale":
|
||||
manager = buffers.get_file_backed_image_manager(root)
|
||||
allocation = manager.allocate((2, 8, 8, 3), dtype=torch.float32)
|
||||
tensor = allocation.publish()
|
||||
tensor.fill_(0.25)
|
||||
os._exit(0)
|
||||
if args.child_mode == "clean-stale":
|
||||
buffers.get_file_backed_image_manager(root)
|
||||
payload = {
|
||||
"bin": len(_managed_files(root, "bin")),
|
||||
"json": len(_managed_files(root, "json")),
|
||||
}
|
||||
print(json.dumps(payload, sort_keys=True))
|
||||
return 0
|
||||
raise ProbeFailure(f"unknown child mode: {args.child_mode}")
|
||||
|
||||
|
||||
def _restart_stale_probe(args, package_root: Path, stale_root: Path) -> dict:
|
||||
common = [
|
||||
sys.executable,
|
||||
str(Path(__file__).resolve()),
|
||||
"--comfy-root",
|
||||
str(args.comfy_root),
|
||||
"--package-root",
|
||||
str(package_root),
|
||||
"--backing-root",
|
||||
str(stale_root),
|
||||
]
|
||||
subprocess.run(
|
||||
[*common, "--child-mode", "create-stale"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
created = (
|
||||
len(_managed_files(stale_root, "bin")),
|
||||
len(_managed_files(stale_root, "json")),
|
||||
)
|
||||
if created != (1, 1):
|
||||
raise ProbeFailure(f"crash child left unexpected files: {created}")
|
||||
cleaned = subprocess.run(
|
||||
[*common, "--child-mode", "clean-stale"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
line = next(
|
||||
(value for value in reversed(cleaned.stdout.splitlines()) if value.startswith("{")),
|
||||
"",
|
||||
)
|
||||
payload = json.loads(line)
|
||||
if payload != {"bin": 0, "json": 0}:
|
||||
raise ProbeFailure(f"restart stale cleanup failed: {payload}")
|
||||
return {"created_pairs": 1, "remaining_pairs_after_restart": 0}
|
||||
|
||||
|
||||
def run_probe(args, package_root: Path) -> dict:
|
||||
_install_paths(args.comfy_root)
|
||||
import folder_paths
|
||||
import nodes as core_nodes
|
||||
import server as comfy_server
|
||||
from execution import CacheType, PromptExecutor
|
||||
|
||||
# A normal ComfyUI process installs this singleton before loading custom
|
||||
# nodes. The headless probe supplies only the queue fields VHS imports;
|
||||
# Video Combine itself does not use them without a meta-batch.
|
||||
if not hasattr(comfy_server.PromptServer, "instance"):
|
||||
comfy_server.PromptServer.instance = SimpleNamespace(
|
||||
prompt_queue=SimpleNamespace(
|
||||
currently_running={},
|
||||
put=lambda _item: None,
|
||||
),
|
||||
number=0,
|
||||
)
|
||||
from videohelpersuite.nodes import VideoCombine
|
||||
|
||||
package = _load_package(package_root)
|
||||
_register_probe_nodes(core_nodes, package, VideoCombine)
|
||||
buffers = importlib.import_module(f"{package.__name__}.v3.file_backed_buffer")
|
||||
|
||||
work_root = args.work_root.resolve()
|
||||
output_root = work_root / "output"
|
||||
temp_root = work_root / "temp"
|
||||
backing_root = temp_root / "h3-continuum-v35-file-backed"
|
||||
stale_root = work_root / "restart-stale"
|
||||
for path in (output_root, temp_root, backing_root, stale_root):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
old_output = folder_paths.get_output_directory()
|
||||
old_temp = folder_paths.get_temp_directory()
|
||||
folder_paths.set_output_directory(str(output_root))
|
||||
folder_paths.set_temp_directory(str(temp_root))
|
||||
prompt_graphs = {
|
||||
"downstream_error": _failure_prompt(fail=True),
|
||||
"recovery": _failure_prompt(fail=False),
|
||||
"compatibility": _compatibility_prompt(),
|
||||
"interrupt": _interrupt_prompt(),
|
||||
"clear": _clear_prompt(1),
|
||||
}
|
||||
(work_root / "p24_prompt_graphs.json").write_text(
|
||||
json.dumps(prompt_graphs, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
server = _Server()
|
||||
executor = PromptExecutor(
|
||||
server,
|
||||
cache_type=CacheType.CLASSIC,
|
||||
cache_args={"ram": 16.0, "ram_inactive": 0.0},
|
||||
)
|
||||
manager = buffers.get_file_backed_image_manager(backing_root)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
_execute(executor, prompt_graphs["downstream_error"], "p24-error", ["3"])
|
||||
if executor.success or not _has_event(executor, "execution_error"):
|
||||
raise ProbeFailure("intentional downstream error was not reported")
|
||||
if (len(_managed_files(backing_root, "bin")), len(_managed_files(backing_root, "json"))) != (1, 1):
|
||||
raise ProbeFailure("completed assembler output was not retained by Core cache")
|
||||
retained = manager.collect_ready()
|
||||
if retained.deferred < 1:
|
||||
raise ProbeFailure("Core cache did not keep mapped storage alive after error")
|
||||
|
||||
_execute(executor, prompt_graphs["recovery"], "p24-recovery", ["3"])
|
||||
recovery_cached = _cached_nodes(executor)
|
||||
if not executor.success or "2" not in recovery_cached:
|
||||
raise ProbeFailure("requeue did not reuse cached V3.5 assembler output")
|
||||
if len(_managed_files(backing_root, "bin")) != 1:
|
||||
raise ProbeFailure("same-input requeue created another backing file")
|
||||
|
||||
_execute(
|
||||
executor,
|
||||
prompt_graphs["compatibility"],
|
||||
"p24-compatibility",
|
||||
["4", "5", "6"],
|
||||
)
|
||||
if not executor.success or "2" not in _cached_nodes(executor):
|
||||
raise ProbeFailure("downstream compatibility run did not reuse assembler cache")
|
||||
downstream = _assert_output_paths(
|
||||
output_root,
|
||||
temp_root,
|
||||
executor.history_result,
|
||||
)
|
||||
output_files_before = _all_output_files(output_root)
|
||||
temp_files_before = _all_output_files(temp_root)
|
||||
|
||||
_execute(
|
||||
executor,
|
||||
prompt_graphs["compatibility"],
|
||||
"p24-compatibility-requeue",
|
||||
["4", "5", "6"],
|
||||
)
|
||||
second_cached = _cached_nodes(executor)
|
||||
if not {"2", "4", "5", "6"}.issubset(second_cached):
|
||||
raise ProbeFailure(
|
||||
f"compatibility requeue cache miss: {sorted(second_cached)}"
|
||||
)
|
||||
if output_files_before != _all_output_files(output_root):
|
||||
raise ProbeFailure("cached requeue unexpectedly rewrote output files")
|
||||
if temp_files_before != _all_output_files(temp_root):
|
||||
raise ProbeFailure("cached requeue unexpectedly rewrote temp files")
|
||||
|
||||
_execute(executor, prompt_graphs["clear"], "p24-clear", ["9"])
|
||||
gc.collect()
|
||||
if manager.tracked_count() != 1:
|
||||
raise ProbeFailure("cache eviction lost lifetime-manager tracking state")
|
||||
|
||||
replacement = _failure_prompt(fail=False, nonce=2)
|
||||
_execute(executor, replacement, "p24-replacement", ["3"])
|
||||
if not executor.success or len(_managed_files(backing_root, "bin")) != 1:
|
||||
raise ProbeFailure("post-eviction allocation did not keep backing pairs bounded")
|
||||
_execute(executor, _clear_prompt(2), "p24-final-clear", ["9"])
|
||||
_collect_until_empty(manager, backing_root)
|
||||
|
||||
_execute(executor, prompt_graphs["interrupt"], "p24-interrupt", ["3"])
|
||||
if executor.success or not _has_event(executor, "execution_interrupted"):
|
||||
raise ProbeFailure("Core interrupt was not propagated as execution_interrupted")
|
||||
if _managed_files(backing_root, "bin") or _managed_files(backing_root, "json"):
|
||||
raise ProbeFailure("interrupted unpublished allocation left backing files")
|
||||
_execute(executor, _clear_prompt(3), "p24-interrupt-clear", ["9"])
|
||||
_collect_until_empty(manager, backing_root)
|
||||
|
||||
stale = _restart_stale_probe(args, package_root, stale_root)
|
||||
elapsed = time.perf_counter() - started
|
||||
result = {
|
||||
"phase": "P2-4",
|
||||
"status": "PASS",
|
||||
"gpu_used": False,
|
||||
"fixture": {
|
||||
"source": "deterministic decoded CPU IMAGE/AUDIO",
|
||||
"frames": 124,
|
||||
"assembled_frames": 120,
|
||||
"width": 16,
|
||||
"height": 16,
|
||||
"fps": 24,
|
||||
"audio_sample_rate": 32_000,
|
||||
"model": "not applicable",
|
||||
"lora": "not applicable",
|
||||
"sampler": "not applicable",
|
||||
"prompt": "not applicable",
|
||||
"media_inputs": "none",
|
||||
},
|
||||
"cache": {
|
||||
"implementation": "ComfyUI PromptExecutor CLASSIC / HierarchicalCache",
|
||||
"downstream_error_retained": True,
|
||||
"same_input_requeue_cached": True,
|
||||
"recovery_cached_nodes": sorted(recovery_cached),
|
||||
"compatibility_requeue_cached_nodes": sorted(second_cached),
|
||||
"post_eviction_pair_count_bounded": True,
|
||||
"final_pairs": 0,
|
||||
},
|
||||
"downstream": downstream,
|
||||
"interrupt": {
|
||||
"core_execution_interrupted": True,
|
||||
"unpublished_pairs_remaining": 0,
|
||||
},
|
||||
"restart_stale_cleanup": stale,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"work_root": str(work_root),
|
||||
}
|
||||
(work_root / "p24_result.json").write_text(
|
||||
json.dumps(result, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return result
|
||||
finally:
|
||||
core_nodes.interrupt_processing(False)
|
||||
folder_paths.set_output_directory(old_output)
|
||||
folder_paths.set_temp_directory(old_temp)
|
||||
|
||||
|
||||
def _parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--comfy-root", type=Path, required=True)
|
||||
parser.add_argument("--package-root", type=Path)
|
||||
parser.add_argument("--work-root", type=Path)
|
||||
parser.add_argument("--child-mode", choices=("create-stale", "clean-stale"))
|
||||
parser.add_argument("--backing-root", type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parse_args()
|
||||
args.comfy_root = args.comfy_root.resolve()
|
||||
package_root = (
|
||||
args.package_root.resolve()
|
||||
if args.package_root is not None
|
||||
else Path(__file__).resolve().parents[1]
|
||||
)
|
||||
if args.child_mode is not None:
|
||||
if args.backing_root is None:
|
||||
raise ProbeFailure("--backing-root is required in child mode")
|
||||
return _run_child_mode(args, package_root)
|
||||
if args.work_root is None:
|
||||
raise ProbeFailure("--work-root is required")
|
||||
result = run_probe(args, package_root)
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,963 @@
|
||||
"""Phase 2A P2-5 GPU/RSS acceptance probe for the V3.5 assembler.
|
||||
|
||||
The probe reuses a completed three-chunk Run Storage revision. It restores
|
||||
the logical AV latents, losslessly reconstructs the accepted Long Terminal
|
||||
Merge physical decode groups, runs the installed Core Video/Audio VAE Decode
|
||||
nodes, and compares independent RAM and Disk-backed assembler processes.
|
||||
|
||||
No sampler/model/CLIP execution is performed and no browser interaction is
|
||||
required. Each backend runs in a fresh process so its RSS peak is not
|
||||
contaminated by the other backend's ComfyUI cache.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import hashlib
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import psutil
|
||||
import torch
|
||||
from safetensors.torch import load_file
|
||||
|
||||
|
||||
class ProbeFailure(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class _Server:
|
||||
def __init__(self):
|
||||
self.client_id = None
|
||||
self.last_node_id = None
|
||||
self.messages: list[tuple[str, dict[str, Any], Any]] = []
|
||||
|
||||
def send_sync(self, event, data, client_id):
|
||||
self.messages.append((event, data, client_id))
|
||||
|
||||
|
||||
class _ResourceMonitor:
|
||||
def __init__(self, interval: float = 0.05):
|
||||
self.interval = float(interval)
|
||||
self.process = psutil.Process()
|
||||
self.stop_event = threading.Event()
|
||||
self.thread: threading.Thread | None = None
|
||||
self.samples = 0
|
||||
self.peak_rss = 0
|
||||
self.minimum_system_available = 1 << 63
|
||||
self.minimum_cuda_free = 1 << 63
|
||||
|
||||
def _sample(self) -> None:
|
||||
try:
|
||||
self.peak_rss = max(self.peak_rss, int(self.process.memory_info().rss))
|
||||
self.minimum_system_available = min(
|
||||
self.minimum_system_available,
|
||||
int(psutil.virtual_memory().available),
|
||||
)
|
||||
if torch.cuda.is_available():
|
||||
free, _total = torch.cuda.mem_get_info()
|
||||
self.minimum_cuda_free = min(self.minimum_cuda_free, int(free))
|
||||
self.samples += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self.stop_event.wait(self.interval):
|
||||
self._sample()
|
||||
|
||||
def start(self) -> None:
|
||||
self._sample()
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def stop(self) -> dict[str, Any]:
|
||||
self._sample()
|
||||
self.stop_event.set()
|
||||
if self.thread is not None:
|
||||
self.thread.join(timeout=5.0)
|
||||
return {
|
||||
"samples": int(self.samples),
|
||||
"peak_rss_bytes": int(self.peak_rss),
|
||||
"minimum_system_available_bytes": int(self.minimum_system_available),
|
||||
"minimum_cuda_free_bytes": (
|
||||
int(self.minimum_cuda_free)
|
||||
if self.minimum_cuda_free != 1 << 63
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _install_paths(comfy_root: Path) -> None:
|
||||
for value in (
|
||||
comfy_root,
|
||||
comfy_root / "custom_nodes" / "comfyui-videohelpersuite",
|
||||
):
|
||||
text = str(value)
|
||||
if text not in sys.path:
|
||||
sys.path.insert(0, text)
|
||||
|
||||
|
||||
def _load_package(package_root: Path):
|
||||
name = "h3_continuum_p25_probe"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
name,
|
||||
package_root / "__init__.py",
|
||||
submodule_search_locations=[str(package_root)],
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ProbeFailure("cannot create Continuum package import specification")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _hash_tensor(tensor: torch.Tensor, batch: int = 8) -> str:
|
||||
digest = hashlib.sha256()
|
||||
value = tensor.detach().cpu()
|
||||
if value.ndim == 0:
|
||||
digest.update(value.numpy().tobytes(order="C"))
|
||||
return digest.hexdigest()
|
||||
for start in range(0, int(value.shape[0]), int(batch)):
|
||||
block = value[start : start + int(batch)].contiguous()
|
||||
digest.update(block.numpy().tobytes(order="C"))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _tensor_is_finite_batched(tensor: torch.Tensor, batch: int = 8) -> bool:
|
||||
value = tensor.detach().cpu()
|
||||
if value.ndim == 0:
|
||||
return bool(torch.isfinite(value).item())
|
||||
for start in range(0, int(value.shape[0]), int(batch)):
|
||||
if not bool(torch.isfinite(value[start : start + int(batch)]).all().item()):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _process_memory_snapshot() -> dict[str, int]:
|
||||
values = psutil.Process().memory_full_info()._asdict()
|
||||
return {key: int(value) for key, value in values.items()}
|
||||
|
||||
|
||||
def _managed_files(root: Path, suffix: str) -> list[Path]:
|
||||
return sorted(root.glob(f"h3c-p20-*.{suffix}"))
|
||||
|
||||
|
||||
def _collect_until_empty(manager, root: Path) -> None:
|
||||
for _ in range(20):
|
||||
gc.collect()
|
||||
manager.collect_ready()
|
||||
if not _managed_files(root, "bin") and not _managed_files(root, "json"):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise ProbeFailure("released Disk-backed output was not reclaimed")
|
||||
|
||||
|
||||
def _register_probe_nodes(
|
||||
core_nodes,
|
||||
package_module,
|
||||
revision_root: Path,
|
||||
results,
|
||||
video_combine=None,
|
||||
stress_size: int = 0,
|
||||
):
|
||||
plan_module = importlib.import_module(f"{package_module.__name__}.v3.plan")
|
||||
|
||||
class P25StoredLongTerminalFixture:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {"required": {"nonce": ("INT", {"default": 0})}}
|
||||
|
||||
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN")
|
||||
RETURN_NAMES = ("video_latents", "audio_latents", "assembly_plan")
|
||||
OUTPUT_IS_LIST = (True, True, False)
|
||||
FUNCTION = "load"
|
||||
CATEGORY = "P2-5 Probe"
|
||||
|
||||
def load(self, nonce):
|
||||
del nonce
|
||||
manifest_path = revision_root / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
records = sorted(
|
||||
manifest["chunks"],
|
||||
key=lambda item: int(item["sequence_index"]),
|
||||
)
|
||||
entries = []
|
||||
for record in records:
|
||||
tensors = load_file(
|
||||
str(revision_root / "chunks" / record["filename"]),
|
||||
device="cpu",
|
||||
)
|
||||
entry = dict(record["entry"])
|
||||
entry["video"] = tensors["video"]
|
||||
entry["audio"] = tensors["audio"]
|
||||
entry["prompt"] = f"stored:{entry.get('prompt_hash', '')}"
|
||||
entries.append(entry)
|
||||
decode_entries, plan = plan_module.prepare_physical_decode_entries(
|
||||
entries,
|
||||
chunk_seconds=float(manifest["contract"]["global"]["chunk_seconds"]),
|
||||
preserve_final_frame=manifest.get("last_frame_hash") not in (None, "none"),
|
||||
terminal_merged=True,
|
||||
)
|
||||
results["fixture"] = {
|
||||
"revision_id": manifest["revision_id"],
|
||||
"logical_chunks": len(entries),
|
||||
"physical_decode_groups": len(decode_entries),
|
||||
"video_shapes": [list(item["video"].shape) for item in decode_entries],
|
||||
"audio_shapes": [list(item["audio"].shape) for item in decode_entries],
|
||||
"width": int(plan["width"]),
|
||||
"height": int(plan["height"]),
|
||||
"target_frames": int(plan["target_frames"]),
|
||||
"prompt_mode": manifest["contract"]["prompt_mode"],
|
||||
"conditioning_mode": manifest["contract"]["global"]["conditioning_mode"],
|
||||
"sampler": manifest["contract"]["global"]["sampler"]["function"],
|
||||
"sigma_count": int(manifest["contract"]["global"]["sigmas"]["shape"][0]),
|
||||
"base_seed": int(manifest["contract"]["global"]["base_seed"]),
|
||||
}
|
||||
return (
|
||||
[{"samples": item["video"]} for item in decode_entries],
|
||||
[{"samples": item["audio"]} for item in decode_entries],
|
||||
plan,
|
||||
)
|
||||
|
||||
class P25DigestOutput:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
"audio": ("AUDIO",),
|
||||
"report": ("STRING",),
|
||||
"backend": (["Auto", "RAM", "Disk-backed"],),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "consume"
|
||||
OUTPUT_NODE = True
|
||||
CATEGORY = "P2-5 Probe"
|
||||
|
||||
def consume(self, images, audio, report, backend):
|
||||
waveform = audio["waveform"]
|
||||
hash_batch = 1 if results.get("stress") else 8
|
||||
results["pre_digest_memory"] = _process_memory_snapshot()
|
||||
results["digest"] = {
|
||||
"backend": str(backend),
|
||||
"video_shape": list(images.shape),
|
||||
"video_dtype": str(images.dtype),
|
||||
"video_sha256": _hash_tensor(images, batch=hash_batch),
|
||||
"video_finite": _tensor_is_finite_batched(
|
||||
images, batch=hash_batch
|
||||
),
|
||||
"audio_shape": list(waveform.shape),
|
||||
"audio_dtype": str(waveform.dtype),
|
||||
"audio_sha256": _hash_tensor(waveform),
|
||||
"audio_finite": _tensor_is_finite_batched(waveform),
|
||||
"audio_sample_rate": int(audio["sample_rate"]),
|
||||
"report": str(report),
|
||||
}
|
||||
return {"ui": {"p25": [str(backend)]}, "result": (str(backend),)}
|
||||
|
||||
class P25SyntheticDecodedStressFixture:
|
||||
"""Large decoded IMAGE fixture with O(H*W), not O(T*H*W), source RAM."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {"required": {"nonce": ("INT", {"default": 0})}}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_ASSEMBLY_PLAN")
|
||||
RETURN_NAMES = ("images", "audio", "assembly_plan")
|
||||
OUTPUT_IS_LIST = (True, True, False)
|
||||
FUNCTION = "load"
|
||||
CATEGORY = "P2-5 Probe"
|
||||
|
||||
def load(self, nonce):
|
||||
del nonce
|
||||
if stress_size <= 0:
|
||||
raise ProbeFailure("synthetic stress fixture requires --stress-size")
|
||||
|
||||
# Reuse the accepted Long Terminal Merge plan instead of inventing a
|
||||
# synthetic grouping contract. Only the already-decoded pixel frames
|
||||
# are replaced. Expanded frame views keep source RAM bounded while
|
||||
# the assembler must still write every byte of the final IMAGE.
|
||||
_video, _audio, plan = P25StoredLongTerminalFixture().load(0)
|
||||
del _video, _audio
|
||||
gc.collect()
|
||||
plan = dict(plan)
|
||||
plan["width"] = int(stress_size)
|
||||
plan["height"] = int(stress_size)
|
||||
units = list(plan.get("decode_groups") or plan["chunks"])
|
||||
|
||||
axis = torch.linspace(0.0, 1.0, stress_size, dtype=torch.float32)
|
||||
yy = axis.view(stress_size, 1)
|
||||
xx = axis.view(1, stress_size)
|
||||
frames = []
|
||||
waveforms = []
|
||||
sample_rate = 32000
|
||||
for index, unit in enumerate(units):
|
||||
base = torch.empty(
|
||||
(1, stress_size, stress_size, 3), dtype=torch.float32
|
||||
)
|
||||
base[0, ..., 0].copy_((xx + index * 0.07).remainder(1.0))
|
||||
base[0, ..., 1].copy_((yy + index * 0.11).remainder(1.0))
|
||||
base[0, ..., 2].copy_(((xx + yy) * 0.5 + index * 0.13).remainder(1.0))
|
||||
total_frames = int(unit["total_frames"])
|
||||
frames.append(base.expand(total_frames, -1, -1, -1))
|
||||
|
||||
total_samples = int(round(total_frames / 24.0 * sample_rate))
|
||||
phase = torch.linspace(
|
||||
0.0,
|
||||
(index + 1) * 24.0,
|
||||
total_samples,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
wave = (0.05 * torch.sin(phase)).reshape(1, 1, -1)
|
||||
waveforms.append(
|
||||
{
|
||||
"waveform": wave.expand(1, 2, -1),
|
||||
"sample_rate": sample_rate,
|
||||
}
|
||||
)
|
||||
|
||||
final_bytes = (
|
||||
int(plan["target_frames"])
|
||||
* stress_size
|
||||
* stress_size
|
||||
* 3
|
||||
* torch.tensor([], dtype=torch.float32).element_size()
|
||||
)
|
||||
results["stress"] = {
|
||||
"size": int(stress_size),
|
||||
"target_frames": int(plan["target_frames"]),
|
||||
"final_image_bytes": int(final_bytes),
|
||||
"final_image_gib": round(final_bytes / (1024**3), 3),
|
||||
"source_frame_storage_bytes": int(
|
||||
sum(frame.untyped_storage().nbytes() for frame in frames)
|
||||
),
|
||||
"source_frames_are_expanded_views": all(
|
||||
int(frame.stride(0)) == 0 for frame in frames
|
||||
),
|
||||
}
|
||||
return frames, waveforms, plan
|
||||
|
||||
class P25ClearOutput:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {"required": {"token": ("INT", {"default": 0})}}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "finish"
|
||||
OUTPUT_NODE = True
|
||||
CATEGORY = "P2-5 Probe"
|
||||
|
||||
def finish(self, token):
|
||||
return (f"clear-{int(token)}",)
|
||||
|
||||
class P25InterruptAfterAllocation:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
"audio": ("AUDIO",),
|
||||
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_ASSEMBLY_PLAN")
|
||||
RETURN_NAMES = ("images", "audio", "assembly_plan")
|
||||
INPUT_IS_LIST = True
|
||||
OUTPUT_IS_LIST = (True, True, False)
|
||||
FUNCTION = "arm"
|
||||
CATEGORY = "P2-5 Probe"
|
||||
|
||||
def arm(self, images, audio, assembly_plan):
|
||||
backing_root = Path(results["backing_root"])
|
||||
|
||||
def trigger():
|
||||
deadline = time.monotonic() + 30.0
|
||||
while time.monotonic() < deadline:
|
||||
if _managed_files(backing_root, "bin"):
|
||||
results["interrupt_observed_allocation"] = True
|
||||
time.sleep(0.05)
|
||||
core_nodes.interrupt_processing(True)
|
||||
return
|
||||
time.sleep(0.005)
|
||||
results["interrupt_observed_allocation"] = False
|
||||
core_nodes.interrupt_processing(True)
|
||||
|
||||
threading.Thread(target=trigger, daemon=True).start()
|
||||
return images, audio, assembly_plan[0]
|
||||
|
||||
from comfy_extras.nodes_audio import VAEDecodeAudio
|
||||
|
||||
core_nodes.NODE_CLASS_MAPPINGS.update(package_module.NODE_CLASS_MAPPINGS)
|
||||
core_nodes.NODE_CLASS_MAPPINGS.update(
|
||||
{
|
||||
"VAEDecodeAudio": VAEDecodeAudio,
|
||||
"P25StoredLongTerminalFixture": P25StoredLongTerminalFixture,
|
||||
"P25SyntheticDecodedStressFixture": P25SyntheticDecodedStressFixture,
|
||||
"P25DigestOutput": P25DigestOutput,
|
||||
"P25ClearOutput": P25ClearOutput,
|
||||
"P25InterruptAfterAllocation": P25InterruptAfterAllocation,
|
||||
}
|
||||
)
|
||||
if video_combine is not None:
|
||||
core_nodes.NODE_CLASS_MAPPINGS["P25VHSVideoCombine"] = video_combine
|
||||
|
||||
|
||||
def _prompt(
|
||||
backend: str,
|
||||
*,
|
||||
downstream: bool = False,
|
||||
interrupt_after_allocation: bool = False,
|
||||
stress: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
if stress:
|
||||
return {
|
||||
"1": {
|
||||
"class_type": "P25SyntheticDecodedStressFixture",
|
||||
"inputs": {"nonce": 0},
|
||||
},
|
||||
"6": {
|
||||
"class_type": "H3ContinuumAssembleSeamV35",
|
||||
"inputs": {
|
||||
"images": ["1", 0],
|
||||
"audio": ["1", 1],
|
||||
"assembly_plan": ["1", 2],
|
||||
"exact_total_duration": True,
|
||||
"audio_seam": "Auto",
|
||||
"video_seam": "Auto",
|
||||
"buffer_backend": str(backend),
|
||||
"diagnostics": "Detailed Report",
|
||||
},
|
||||
},
|
||||
"7": {
|
||||
"class_type": "P25DigestOutput",
|
||||
"inputs": {
|
||||
"images": ["6", 0],
|
||||
"audio": ["6", 1],
|
||||
"report": ["6", 2],
|
||||
"backend": str(backend),
|
||||
},
|
||||
},
|
||||
}
|
||||
prompt = {
|
||||
"1": {
|
||||
"class_type": "P25StoredLongTerminalFixture",
|
||||
"inputs": {"nonce": 0},
|
||||
},
|
||||
"2": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "minimax_h3_video_vae_fp16.safetensors"},
|
||||
},
|
||||
"3": {
|
||||
"class_type": "VAELoader",
|
||||
"inputs": {"vae_name": "minimax_h3_audio_vae_fp32.safetensors"},
|
||||
},
|
||||
"4": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["1", 0], "vae": ["2", 0]},
|
||||
},
|
||||
"5": {
|
||||
"class_type": "VAEDecodeAudio",
|
||||
"inputs": {"samples": ["1", 1], "vae": ["3", 0]},
|
||||
},
|
||||
"6": {
|
||||
"class_type": "H3ContinuumAssembleSeamV35",
|
||||
"inputs": {
|
||||
"images": ["4", 0],
|
||||
"audio": ["5", 0],
|
||||
"assembly_plan": ["1", 2],
|
||||
"exact_total_duration": True,
|
||||
"audio_seam": "Auto",
|
||||
"video_seam": "Auto",
|
||||
"buffer_backend": str(backend),
|
||||
"diagnostics": "Detailed Report",
|
||||
},
|
||||
},
|
||||
"7": {
|
||||
"class_type": "P25DigestOutput",
|
||||
"inputs": {
|
||||
"images": ["6", 0],
|
||||
"audio": ["6", 1],
|
||||
"report": ["6", 2],
|
||||
"backend": str(backend),
|
||||
},
|
||||
},
|
||||
}
|
||||
if interrupt_after_allocation:
|
||||
prompt["11"] = {
|
||||
"class_type": "P25InterruptAfterAllocation",
|
||||
"inputs": {
|
||||
"images": ["4", 0],
|
||||
"audio": ["5", 0],
|
||||
"assembly_plan": ["1", 2],
|
||||
},
|
||||
}
|
||||
prompt["6"]["inputs"].update(
|
||||
{
|
||||
"images": ["11", 0],
|
||||
"audio": ["11", 1],
|
||||
"assembly_plan": ["11", 2],
|
||||
}
|
||||
)
|
||||
if downstream:
|
||||
prompt.update(
|
||||
{
|
||||
"8": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {
|
||||
"images": ["6", 0],
|
||||
"filename_prefix": "P25/Save",
|
||||
},
|
||||
},
|
||||
"9": {
|
||||
"class_type": "PreviewImage",
|
||||
"inputs": {"images": ["6", 0]},
|
||||
},
|
||||
"10": {
|
||||
"class_type": "P25VHSVideoCombine",
|
||||
"inputs": {
|
||||
"images": ["6", 0],
|
||||
"audio": ["6", 1],
|
||||
"frame_rate": 24,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "P25/VHS",
|
||||
"format": "video/h264-mp4",
|
||||
"pingpong": False,
|
||||
"save_output": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
return prompt
|
||||
|
||||
|
||||
def _downstream_result(history: dict[str, Any]) -> dict[str, Any]:
|
||||
outputs = history.get("outputs", {})
|
||||
save_items = outputs.get("8", {}).get("images", [])
|
||||
preview_items = outputs.get("9", {}).get("images", [])
|
||||
video_items = outputs.get("10", {}).get("gifs", [])
|
||||
if len(save_items) != 360 or len(preview_items) != 360 or len(video_items) != 1:
|
||||
raise ProbeFailure(
|
||||
"unexpected downstream output counts: "
|
||||
f"save={len(save_items)} preview={len(preview_items)} vhs={len(video_items)}"
|
||||
)
|
||||
video_path = Path(video_items[0]["fullpath"])
|
||||
if not video_path.is_file() or video_path.stat().st_size <= 0:
|
||||
raise ProbeFailure(f"VHS output is missing: {video_path}")
|
||||
ffprobe = shutil.which("ffprobe")
|
||||
if ffprobe is None:
|
||||
raise ProbeFailure("ffprobe is required for P2-5 downstream acceptance")
|
||||
inspected = subprocess.run(
|
||||
[
|
||||
ffprobe,
|
||||
"-v",
|
||||
"error",
|
||||
"-count_frames",
|
||||
"-show_entries",
|
||||
(
|
||||
"stream=codec_type,codec_name,width,height,r_frame_rate,"
|
||||
"nb_read_frames,sample_rate,channels,duration:format=duration,size"
|
||||
),
|
||||
"-of",
|
||||
"json",
|
||||
str(video_path),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
media = json.loads(inspected.stdout)
|
||||
video_stream = next(
|
||||
(item for item in media.get("streams", []) if item.get("codec_type") == "video"),
|
||||
None,
|
||||
)
|
||||
audio_stream = next(
|
||||
(item for item in media.get("streams", []) if item.get("codec_type") == "audio"),
|
||||
None,
|
||||
)
|
||||
if video_stream is None or audio_stream is None:
|
||||
raise ProbeFailure("VHS output does not contain both video and audio")
|
||||
expected = (
|
||||
int(video_stream.get("width", 0)) == 640
|
||||
and int(video_stream.get("height", 0)) == 640
|
||||
and int(video_stream.get("nb_read_frames", 0)) == 360
|
||||
and video_stream.get("r_frame_rate") == "24/1"
|
||||
and int(audio_stream.get("sample_rate", 0)) == 32_000
|
||||
)
|
||||
if not expected:
|
||||
raise ProbeFailure(f"unexpected VHS media contract: {media}")
|
||||
return {
|
||||
"save_frames": len(save_items),
|
||||
"preview_frames": len(preview_items),
|
||||
"vhs_file": str(video_path),
|
||||
"vhs_bytes": int(video_path.stat().st_size),
|
||||
"video_codec": video_stream.get("codec_name"),
|
||||
"video_frames": int(video_stream["nb_read_frames"]),
|
||||
"video_fps": video_stream["r_frame_rate"],
|
||||
"video_duration": float(video_stream["duration"]),
|
||||
"audio_codec": audio_stream.get("codec_name"),
|
||||
"audio_sample_rate": int(audio_stream["sample_rate"]),
|
||||
"container_duration": float(media["format"]["duration"]),
|
||||
}
|
||||
|
||||
|
||||
def _clear_prompt() -> dict[str, Any]:
|
||||
return {"9": {"class_type": "P25ClearOutput", "inputs": {"token": 1}}}
|
||||
|
||||
|
||||
def _has_event(executor, wanted: str) -> bool:
|
||||
return any(event == wanted for event, _data in executor.status_messages)
|
||||
|
||||
|
||||
def _cached_nodes(executor) -> set[str]:
|
||||
for event, data in executor.status_messages:
|
||||
if event == "execution_cached":
|
||||
return {str(value) for value in data.get("nodes", [])}
|
||||
return set()
|
||||
|
||||
|
||||
def _run_backend(args, package_root: Path) -> dict[str, Any]:
|
||||
_install_paths(args.comfy_root)
|
||||
import folder_paths
|
||||
import nodes as core_nodes
|
||||
import server as comfy_server
|
||||
from execution import CacheType, PromptExecutor
|
||||
|
||||
if not hasattr(comfy_server.PromptServer, "instance"):
|
||||
comfy_server.PromptServer.instance = SimpleNamespace(
|
||||
prompt_queue=SimpleNamespace(currently_running={}, put=lambda _item: None),
|
||||
number=0,
|
||||
)
|
||||
|
||||
work_root = args.work_root.resolve()
|
||||
output_root = work_root / "output"
|
||||
temp_root = work_root / "temp"
|
||||
backing_root = temp_root / "h3-continuum-v35-file-backed"
|
||||
for path in (output_root, temp_root, backing_root):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
package = _load_package(package_root)
|
||||
results: dict[str, Any] = {"backing_root": str(backing_root)}
|
||||
video_combine = None
|
||||
if args.downstream:
|
||||
from videohelpersuite.nodes import VideoCombine
|
||||
|
||||
video_combine = VideoCombine
|
||||
_register_probe_nodes(
|
||||
core_nodes,
|
||||
package,
|
||||
args.revision_root,
|
||||
results,
|
||||
video_combine=video_combine,
|
||||
stress_size=int(args.stress_size),
|
||||
)
|
||||
buffers = importlib.import_module(f"{package.__name__}.v3.file_backed_buffer")
|
||||
|
||||
folder_paths.set_output_directory(str(output_root))
|
||||
folder_paths.set_temp_directory(str(temp_root))
|
||||
|
||||
prompt = _prompt(
|
||||
args.backend,
|
||||
downstream=bool(args.downstream),
|
||||
interrupt_after_allocation=bool(args.interrupt_after_allocation),
|
||||
stress=bool(args.stress_size),
|
||||
)
|
||||
(work_root / "p25_prompt_graph.json").write_text(
|
||||
json.dumps(prompt, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
executor = PromptExecutor(
|
||||
_Server(),
|
||||
cache_type=CacheType.CLASSIC,
|
||||
cache_args={"ram": 64.0, "ram_inactive": 0.0},
|
||||
)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
monitor = _ResourceMonitor()
|
||||
monitor.start()
|
||||
started = time.perf_counter()
|
||||
execute_outputs = ["7", "8", "9", "10"] if args.downstream else ["7"]
|
||||
executor.execute(
|
||||
prompt,
|
||||
f"p25-{args.backend.lower()}",
|
||||
extra_data={},
|
||||
execute_outputs=execute_outputs,
|
||||
)
|
||||
cache_requeue = None
|
||||
if args.cache_requeue and executor.success:
|
||||
first_pairs = (
|
||||
len(_managed_files(backing_root, "bin")),
|
||||
len(_managed_files(backing_root, "json")),
|
||||
)
|
||||
executor.execute(
|
||||
prompt,
|
||||
f"p25-{args.backend.lower()}-requeue",
|
||||
extra_data={},
|
||||
execute_outputs=execute_outputs,
|
||||
)
|
||||
cached = _cached_nodes(executor)
|
||||
second_pairs = (
|
||||
len(_managed_files(backing_root, "bin")),
|
||||
len(_managed_files(backing_root, "json")),
|
||||
)
|
||||
cache_requeue = {
|
||||
"cached_nodes": sorted(cached),
|
||||
"assembler_cached": "6" in cached,
|
||||
"first_pair_count": list(first_pairs),
|
||||
"second_pair_count": list(second_pairs),
|
||||
"pair_count_bounded": first_pairs == second_pairs,
|
||||
}
|
||||
if not executor.success or not cache_requeue["assembler_cached"]:
|
||||
raise ProbeFailure(f"large-output requeue cache miss: {cache_requeue}")
|
||||
if not cache_requeue["pair_count_bounded"]:
|
||||
raise ProbeFailure(f"large-output requeue grew backing files: {cache_requeue}")
|
||||
elapsed = time.perf_counter() - started
|
||||
resources = monitor.stop()
|
||||
manager = buffers.get_file_backed_image_manager(backing_root)
|
||||
if args.interrupt_after_allocation:
|
||||
interrupted = _has_event(executor, "execution_interrupted")
|
||||
core_nodes.interrupt_processing(False)
|
||||
executor.execute(_clear_prompt(), "p25-clear", extra_data={}, execute_outputs=["9"])
|
||||
del executor
|
||||
_collect_until_empty(manager, backing_root)
|
||||
result = {
|
||||
"backend": args.backend,
|
||||
"stage": "interrupt-after-allocation",
|
||||
"status": "PASS" if interrupted else "FAIL",
|
||||
"execution_interrupted": bool(interrupted),
|
||||
"allocation_observed_before_interrupt": bool(
|
||||
results.get("interrupt_observed_allocation", False)
|
||||
),
|
||||
"fixture": results.get("fixture"),
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"managed_bin_after_interrupt": len(_managed_files(backing_root, "bin")),
|
||||
"managed_json_after_interrupt": len(_managed_files(backing_root, "json")),
|
||||
"resources": resources,
|
||||
"work_root": str(work_root),
|
||||
}
|
||||
if not interrupted or not result["allocation_observed_before_interrupt"]:
|
||||
raise ProbeFailure(f"post-allocation interrupt was not proven: {result}")
|
||||
(work_root / "p25_interrupt_result.json").write_text(
|
||||
json.dumps(result, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return result
|
||||
if not executor.success or _has_event(executor, "execution_error"):
|
||||
raise ProbeFailure(f"{args.backend} PromptExecutor failed")
|
||||
if "digest" not in results or "fixture" not in results:
|
||||
raise ProbeFailure(f"{args.backend} probe outputs were not captured")
|
||||
|
||||
bin_files = _managed_files(backing_root, "bin")
|
||||
json_files = _managed_files(backing_root, "json")
|
||||
resources.update(
|
||||
{
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"cuda_peak_allocated_bytes": (
|
||||
int(torch.cuda.max_memory_allocated()) if torch.cuda.is_available() else 0
|
||||
),
|
||||
"cuda_peak_reserved_bytes": (
|
||||
int(torch.cuda.max_memory_reserved()) if torch.cuda.is_available() else 0
|
||||
),
|
||||
"backing_bin_count_while_cached": len(bin_files),
|
||||
"backing_json_count_while_cached": len(json_files),
|
||||
"backing_bytes_while_cached": sum(item.stat().st_size for item in bin_files),
|
||||
"pre_digest_memory": results.get("pre_digest_memory"),
|
||||
}
|
||||
)
|
||||
downstream = _downstream_result(executor.history_result) if args.downstream else None
|
||||
executor.execute(_clear_prompt(), "p25-clear", extra_data={}, execute_outputs=["9"])
|
||||
del executor
|
||||
_collect_until_empty(manager, backing_root)
|
||||
resources["managed_bin_after_cache_release"] = len(
|
||||
_managed_files(backing_root, "bin")
|
||||
)
|
||||
resources["managed_json_after_cache_release"] = len(
|
||||
_managed_files(backing_root, "json")
|
||||
)
|
||||
resources["nonmanaged_root_entries_after_cache_release"] = len(
|
||||
[
|
||||
item
|
||||
for item in backing_root.iterdir()
|
||||
if item.name != ".h3c-p20.lock"
|
||||
]
|
||||
)
|
||||
result = {
|
||||
"backend": args.backend,
|
||||
"status": "PASS",
|
||||
"fixture": results["fixture"],
|
||||
"stress": results.get("stress"),
|
||||
"digest": results["digest"],
|
||||
"resources": resources,
|
||||
"cache_requeue": cache_requeue,
|
||||
"downstream": downstream,
|
||||
"work_root": str(work_root),
|
||||
}
|
||||
(work_root / "p25_backend_result.json").write_text(
|
||||
json.dumps(result, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _last_json(stdout: str) -> dict[str, Any]:
|
||||
marker = "P25_RESULT_JSON="
|
||||
line = next(
|
||||
(value for value in reversed(stdout.splitlines()) if value.startswith(marker)),
|
||||
"",
|
||||
)
|
||||
if not line:
|
||||
raise ProbeFailure("backend child did not emit a result marker")
|
||||
return json.loads(line[len(marker) :])
|
||||
|
||||
|
||||
def _summarize_results(work_root: Path, results: dict[str, Any] | None = None):
|
||||
work_root = work_root.resolve()
|
||||
if results is None:
|
||||
results = {
|
||||
"RAM": json.loads(
|
||||
(work_root / "ram" / "p25_backend_result.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
),
|
||||
"Disk-backed": json.loads(
|
||||
(work_root / "disk" / "p25_backend_result.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
),
|
||||
}
|
||||
ram = results["RAM"]
|
||||
disk = results["Disk-backed"]
|
||||
disk_backing_root = (
|
||||
work_root / "disk" / "temp" / "h3-continuum-v35-file-backed"
|
||||
)
|
||||
managed_bin_after = len(_managed_files(disk_backing_root, "bin"))
|
||||
managed_json_after = len(_managed_files(disk_backing_root, "json"))
|
||||
disk["resources"]["managed_bin_after_cache_release"] = managed_bin_after
|
||||
disk["resources"]["managed_json_after_cache_release"] = managed_json_after
|
||||
comparisons = {
|
||||
"fixture_equal": ram["fixture"] == disk["fixture"],
|
||||
"video_shape_equal": ram["digest"]["video_shape"] == disk["digest"]["video_shape"],
|
||||
"video_sha256_equal": ram["digest"]["video_sha256"] == disk["digest"]["video_sha256"],
|
||||
"audio_shape_equal": ram["digest"]["audio_shape"] == disk["digest"]["audio_shape"],
|
||||
"audio_sha256_equal": ram["digest"]["audio_sha256"] == disk["digest"]["audio_sha256"],
|
||||
"audio_sample_rate_equal": ram["digest"]["audio_sample_rate"] == disk["digest"]["audio_sample_rate"],
|
||||
"both_finite": bool(
|
||||
ram["digest"]["video_finite"]
|
||||
and disk["digest"]["video_finite"]
|
||||
and ram["digest"]["audio_finite"]
|
||||
and disk["digest"]["audio_finite"]
|
||||
),
|
||||
"disk_backing_reclaimed": managed_bin_after == 0 and managed_json_after == 0,
|
||||
}
|
||||
if not all(comparisons.values()):
|
||||
raise ProbeFailure(f"RAM/Disk-backed comparison failed: {comparisons}")
|
||||
peak_delta = (
|
||||
int(ram["resources"]["peak_rss_bytes"])
|
||||
- int(disk["resources"]["peak_rss_bytes"])
|
||||
)
|
||||
result = {
|
||||
"phase": "P2-5",
|
||||
"stage": "small-long-terminal-ab",
|
||||
"status": "PASS",
|
||||
"gpu_used": True,
|
||||
"comparisons": comparisons,
|
||||
"rss_peak_reduction_bytes": peak_delta,
|
||||
"rss_peak_reduction_percent": round(
|
||||
peak_delta / max(1, int(ram["resources"]["peak_rss_bytes"])) * 100.0,
|
||||
3,
|
||||
),
|
||||
"backends": results,
|
||||
"work_root": str(work_root),
|
||||
}
|
||||
(work_root / "p25_result.json").write_text(
|
||||
json.dumps(result, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _run_parent(args, package_root: Path) -> dict[str, Any]:
|
||||
work_root = args.work_root.resolve()
|
||||
work_root.mkdir(parents=True, exist_ok=True)
|
||||
common = [
|
||||
sys.executable,
|
||||
str(Path(__file__).resolve()),
|
||||
"--comfy-root",
|
||||
str(args.comfy_root),
|
||||
"--package-root",
|
||||
str(package_root),
|
||||
"--revision-root",
|
||||
str(args.revision_root),
|
||||
]
|
||||
results = {}
|
||||
for backend, folder in (("RAM", "ram"), ("Disk-backed", "disk")):
|
||||
child = subprocess.run(
|
||||
[*common, "--work-root", str(work_root / folder), "--backend", backend],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
(work_root / f"{folder}_stdout.txt").write_text(child.stdout, encoding="utf-8")
|
||||
(work_root / f"{folder}_stderr.txt").write_text(child.stderr, encoding="utf-8")
|
||||
results[backend] = _last_json(child.stdout)
|
||||
return _summarize_results(work_root, results)
|
||||
|
||||
|
||||
def _parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--comfy-root", type=Path, required=True)
|
||||
parser.add_argument("--package-root", type=Path)
|
||||
parser.add_argument("--revision-root", type=Path, required=True)
|
||||
parser.add_argument("--work-root", type=Path, required=True)
|
||||
parser.add_argument("--backend", choices=("Auto", "RAM", "Disk-backed"))
|
||||
parser.add_argument("--summarize-existing", action="store_true")
|
||||
parser.add_argument("--downstream", action="store_true")
|
||||
parser.add_argument("--interrupt-after-allocation", action="store_true")
|
||||
parser.add_argument("--cache-requeue", action="store_true")
|
||||
parser.add_argument(
|
||||
"--stress-size",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Use a synthetic decoded square fixture of this size; 1536 is 9.49 GiB.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parse_args()
|
||||
args.comfy_root = args.comfy_root.resolve()
|
||||
args.revision_root = args.revision_root.resolve()
|
||||
args.work_root = args.work_root.resolve()
|
||||
package_root = (
|
||||
args.package_root.resolve()
|
||||
if args.package_root is not None
|
||||
else Path(__file__).resolve().parents[1]
|
||||
)
|
||||
if args.backend is not None:
|
||||
result = _run_backend(args, package_root)
|
||||
print("P25_RESULT_JSON=" + json.dumps(result, sort_keys=True))
|
||||
return 0
|
||||
if args.summarize_existing:
|
||||
result = _summarize_results(args.work_root)
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
result = _run_parent(args, package_root)
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -67,7 +67,12 @@ def main() -> int:
|
||||
|
||||
current_required = {
|
||||
"H3ContinuumSamplerV34",
|
||||
"H3ContinuumSamplerV35",
|
||||
"H3ContinuumAssembleSeamV34",
|
||||
"H3ContinuumAssembleSeamV35",
|
||||
"H3ContinuumHiResFixV35",
|
||||
"H3ContinuumLatentResizeV35",
|
||||
"H3ContinuumSecondPassV35",
|
||||
"H3ContinuumClipOverrides",
|
||||
"H3ContinuumResult",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user