git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
964 lines
35 KiB
Python
964 lines
35 KiB
Python
"""Phase 2A P2-5 GPU/RSS acceptance probe for the V3.5 assembler.
|
|
|
|
The probe reuses a completed three-chunk Run Storage revision. It restores
|
|
the logical AV latents, losslessly reconstructs the accepted Long Terminal
|
|
Merge physical decode groups, runs the installed Core Video/Audio VAE Decode
|
|
nodes, and compares independent RAM and Disk-backed assembler processes.
|
|
|
|
No sampler/model/CLIP execution is performed and no browser interaction is
|
|
required. Each backend runs in a fresh process so its RSS peak is not
|
|
contaminated by the other backend's ComfyUI cache.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gc
|
|
import hashlib
|
|
import importlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
import psutil
|
|
import torch
|
|
from safetensors.torch import load_file
|
|
|
|
|
|
class ProbeFailure(RuntimeError):
|
|
pass
|
|
|
|
|
|
class _Server:
|
|
def __init__(self):
|
|
self.client_id = None
|
|
self.last_node_id = None
|
|
self.messages: list[tuple[str, dict[str, Any], Any]] = []
|
|
|
|
def send_sync(self, event, data, client_id):
|
|
self.messages.append((event, data, client_id))
|
|
|
|
|
|
class _ResourceMonitor:
|
|
def __init__(self, interval: float = 0.05):
|
|
self.interval = float(interval)
|
|
self.process = psutil.Process()
|
|
self.stop_event = threading.Event()
|
|
self.thread: threading.Thread | None = None
|
|
self.samples = 0
|
|
self.peak_rss = 0
|
|
self.minimum_system_available = 1 << 63
|
|
self.minimum_cuda_free = 1 << 63
|
|
|
|
def _sample(self) -> None:
|
|
try:
|
|
self.peak_rss = max(self.peak_rss, int(self.process.memory_info().rss))
|
|
self.minimum_system_available = min(
|
|
self.minimum_system_available,
|
|
int(psutil.virtual_memory().available),
|
|
)
|
|
if torch.cuda.is_available():
|
|
free, _total = torch.cuda.mem_get_info()
|
|
self.minimum_cuda_free = min(self.minimum_cuda_free, int(free))
|
|
self.samples += 1
|
|
except Exception:
|
|
pass
|
|
|
|
def _run(self) -> None:
|
|
while not self.stop_event.wait(self.interval):
|
|
self._sample()
|
|
|
|
def start(self) -> None:
|
|
self._sample()
|
|
self.thread = threading.Thread(target=self._run, daemon=True)
|
|
self.thread.start()
|
|
|
|
def stop(self) -> dict[str, Any]:
|
|
self._sample()
|
|
self.stop_event.set()
|
|
if self.thread is not None:
|
|
self.thread.join(timeout=5.0)
|
|
return {
|
|
"samples": int(self.samples),
|
|
"peak_rss_bytes": int(self.peak_rss),
|
|
"minimum_system_available_bytes": int(self.minimum_system_available),
|
|
"minimum_cuda_free_bytes": (
|
|
int(self.minimum_cuda_free)
|
|
if self.minimum_cuda_free != 1 << 63
|
|
else None
|
|
),
|
|
}
|
|
|
|
|
|
def _install_paths(comfy_root: Path) -> None:
|
|
for value in (
|
|
comfy_root,
|
|
comfy_root / "custom_nodes" / "comfyui-videohelpersuite",
|
|
):
|
|
text = str(value)
|
|
if text not in sys.path:
|
|
sys.path.insert(0, text)
|
|
|
|
|
|
def _load_package(package_root: Path):
|
|
name = "h3_continuum_p25_probe"
|
|
spec = importlib.util.spec_from_file_location(
|
|
name,
|
|
package_root / "__init__.py",
|
|
submodule_search_locations=[str(package_root)],
|
|
)
|
|
if spec is None or spec.loader is None:
|
|
raise ProbeFailure("cannot create Continuum package import specification")
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _hash_tensor(tensor: torch.Tensor, batch: int = 8) -> str:
|
|
digest = hashlib.sha256()
|
|
value = tensor.detach().cpu()
|
|
if value.ndim == 0:
|
|
digest.update(value.numpy().tobytes(order="C"))
|
|
return digest.hexdigest()
|
|
for start in range(0, int(value.shape[0]), int(batch)):
|
|
block = value[start : start + int(batch)].contiguous()
|
|
digest.update(block.numpy().tobytes(order="C"))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _tensor_is_finite_batched(tensor: torch.Tensor, batch: int = 8) -> bool:
|
|
value = tensor.detach().cpu()
|
|
if value.ndim == 0:
|
|
return bool(torch.isfinite(value).item())
|
|
for start in range(0, int(value.shape[0]), int(batch)):
|
|
if not bool(torch.isfinite(value[start : start + int(batch)]).all().item()):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _process_memory_snapshot() -> dict[str, int]:
|
|
values = psutil.Process().memory_full_info()._asdict()
|
|
return {key: int(value) for key, value in values.items()}
|
|
|
|
|
|
def _managed_files(root: Path, suffix: str) -> list[Path]:
|
|
return sorted(root.glob(f"h3c-p20-*.{suffix}"))
|
|
|
|
|
|
def _collect_until_empty(manager, root: Path) -> None:
|
|
for _ in range(20):
|
|
gc.collect()
|
|
manager.collect_ready()
|
|
if not _managed_files(root, "bin") and not _managed_files(root, "json"):
|
|
return
|
|
time.sleep(0.1)
|
|
raise ProbeFailure("released Disk-backed output was not reclaimed")
|
|
|
|
|
|
def _register_probe_nodes(
|
|
core_nodes,
|
|
package_module,
|
|
revision_root: Path,
|
|
results,
|
|
video_combine=None,
|
|
stress_size: int = 0,
|
|
):
|
|
plan_module = importlib.import_module(f"{package_module.__name__}.v3.plan")
|
|
|
|
class P25StoredLongTerminalFixture:
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {"required": {"nonce": ("INT", {"default": 0})}}
|
|
|
|
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN")
|
|
RETURN_NAMES = ("video_latents", "audio_latents", "assembly_plan")
|
|
OUTPUT_IS_LIST = (True, True, False)
|
|
FUNCTION = "load"
|
|
CATEGORY = "P2-5 Probe"
|
|
|
|
def load(self, nonce):
|
|
del nonce
|
|
manifest_path = revision_root / "manifest.json"
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
records = sorted(
|
|
manifest["chunks"],
|
|
key=lambda item: int(item["sequence_index"]),
|
|
)
|
|
entries = []
|
|
for record in records:
|
|
tensors = load_file(
|
|
str(revision_root / "chunks" / record["filename"]),
|
|
device="cpu",
|
|
)
|
|
entry = dict(record["entry"])
|
|
entry["video"] = tensors["video"]
|
|
entry["audio"] = tensors["audio"]
|
|
entry["prompt"] = f"stored:{entry.get('prompt_hash', '')}"
|
|
entries.append(entry)
|
|
decode_entries, plan = plan_module.prepare_physical_decode_entries(
|
|
entries,
|
|
chunk_seconds=float(manifest["contract"]["global"]["chunk_seconds"]),
|
|
preserve_final_frame=manifest.get("last_frame_hash") not in (None, "none"),
|
|
terminal_merged=True,
|
|
)
|
|
results["fixture"] = {
|
|
"revision_id": manifest["revision_id"],
|
|
"logical_chunks": len(entries),
|
|
"physical_decode_groups": len(decode_entries),
|
|
"video_shapes": [list(item["video"].shape) for item in decode_entries],
|
|
"audio_shapes": [list(item["audio"].shape) for item in decode_entries],
|
|
"width": int(plan["width"]),
|
|
"height": int(plan["height"]),
|
|
"target_frames": int(plan["target_frames"]),
|
|
"prompt_mode": manifest["contract"]["prompt_mode"],
|
|
"conditioning_mode": manifest["contract"]["global"]["conditioning_mode"],
|
|
"sampler": manifest["contract"]["global"]["sampler"]["function"],
|
|
"sigma_count": int(manifest["contract"]["global"]["sigmas"]["shape"][0]),
|
|
"base_seed": int(manifest["contract"]["global"]["base_seed"]),
|
|
}
|
|
return (
|
|
[{"samples": item["video"]} for item in decode_entries],
|
|
[{"samples": item["audio"]} for item in decode_entries],
|
|
plan,
|
|
)
|
|
|
|
class P25DigestOutput:
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {
|
|
"required": {
|
|
"images": ("IMAGE",),
|
|
"audio": ("AUDIO",),
|
|
"report": ("STRING",),
|
|
"backend": (["Auto", "RAM", "Disk-backed"],),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("STRING",)
|
|
FUNCTION = "consume"
|
|
OUTPUT_NODE = True
|
|
CATEGORY = "P2-5 Probe"
|
|
|
|
def consume(self, images, audio, report, backend):
|
|
waveform = audio["waveform"]
|
|
hash_batch = 1 if results.get("stress") else 8
|
|
results["pre_digest_memory"] = _process_memory_snapshot()
|
|
results["digest"] = {
|
|
"backend": str(backend),
|
|
"video_shape": list(images.shape),
|
|
"video_dtype": str(images.dtype),
|
|
"video_sha256": _hash_tensor(images, batch=hash_batch),
|
|
"video_finite": _tensor_is_finite_batched(
|
|
images, batch=hash_batch
|
|
),
|
|
"audio_shape": list(waveform.shape),
|
|
"audio_dtype": str(waveform.dtype),
|
|
"audio_sha256": _hash_tensor(waveform),
|
|
"audio_finite": _tensor_is_finite_batched(waveform),
|
|
"audio_sample_rate": int(audio["sample_rate"]),
|
|
"report": str(report),
|
|
}
|
|
return {"ui": {"p25": [str(backend)]}, "result": (str(backend),)}
|
|
|
|
class P25SyntheticDecodedStressFixture:
|
|
"""Large decoded IMAGE fixture with O(H*W), not O(T*H*W), source RAM."""
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {"required": {"nonce": ("INT", {"default": 0})}}
|
|
|
|
RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_ASSEMBLY_PLAN")
|
|
RETURN_NAMES = ("images", "audio", "assembly_plan")
|
|
OUTPUT_IS_LIST = (True, True, False)
|
|
FUNCTION = "load"
|
|
CATEGORY = "P2-5 Probe"
|
|
|
|
def load(self, nonce):
|
|
del nonce
|
|
if stress_size <= 0:
|
|
raise ProbeFailure("synthetic stress fixture requires --stress-size")
|
|
|
|
# Reuse the accepted Long Terminal Merge plan instead of inventing a
|
|
# synthetic grouping contract. Only the already-decoded pixel frames
|
|
# are replaced. Expanded frame views keep source RAM bounded while
|
|
# the assembler must still write every byte of the final IMAGE.
|
|
_video, _audio, plan = P25StoredLongTerminalFixture().load(0)
|
|
del _video, _audio
|
|
gc.collect()
|
|
plan = dict(plan)
|
|
plan["width"] = int(stress_size)
|
|
plan["height"] = int(stress_size)
|
|
units = list(plan.get("decode_groups") or plan["chunks"])
|
|
|
|
axis = torch.linspace(0.0, 1.0, stress_size, dtype=torch.float32)
|
|
yy = axis.view(stress_size, 1)
|
|
xx = axis.view(1, stress_size)
|
|
frames = []
|
|
waveforms = []
|
|
sample_rate = 32000
|
|
for index, unit in enumerate(units):
|
|
base = torch.empty(
|
|
(1, stress_size, stress_size, 3), dtype=torch.float32
|
|
)
|
|
base[0, ..., 0].copy_((xx + index * 0.07).remainder(1.0))
|
|
base[0, ..., 1].copy_((yy + index * 0.11).remainder(1.0))
|
|
base[0, ..., 2].copy_(((xx + yy) * 0.5 + index * 0.13).remainder(1.0))
|
|
total_frames = int(unit["total_frames"])
|
|
frames.append(base.expand(total_frames, -1, -1, -1))
|
|
|
|
total_samples = int(round(total_frames / 24.0 * sample_rate))
|
|
phase = torch.linspace(
|
|
0.0,
|
|
(index + 1) * 24.0,
|
|
total_samples,
|
|
dtype=torch.float32,
|
|
)
|
|
wave = (0.05 * torch.sin(phase)).reshape(1, 1, -1)
|
|
waveforms.append(
|
|
{
|
|
"waveform": wave.expand(1, 2, -1),
|
|
"sample_rate": sample_rate,
|
|
}
|
|
)
|
|
|
|
final_bytes = (
|
|
int(plan["target_frames"])
|
|
* stress_size
|
|
* stress_size
|
|
* 3
|
|
* torch.tensor([], dtype=torch.float32).element_size()
|
|
)
|
|
results["stress"] = {
|
|
"size": int(stress_size),
|
|
"target_frames": int(plan["target_frames"]),
|
|
"final_image_bytes": int(final_bytes),
|
|
"final_image_gib": round(final_bytes / (1024**3), 3),
|
|
"source_frame_storage_bytes": int(
|
|
sum(frame.untyped_storage().nbytes() for frame in frames)
|
|
),
|
|
"source_frames_are_expanded_views": all(
|
|
int(frame.stride(0)) == 0 for frame in frames
|
|
),
|
|
}
|
|
return frames, waveforms, plan
|
|
|
|
class P25ClearOutput:
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {"required": {"token": ("INT", {"default": 0})}}
|
|
|
|
RETURN_TYPES = ("STRING",)
|
|
FUNCTION = "finish"
|
|
OUTPUT_NODE = True
|
|
CATEGORY = "P2-5 Probe"
|
|
|
|
def finish(self, token):
|
|
return (f"clear-{int(token)}",)
|
|
|
|
class P25InterruptAfterAllocation:
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {
|
|
"required": {
|
|
"images": ("IMAGE",),
|
|
"audio": ("AUDIO",),
|
|
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_ASSEMBLY_PLAN")
|
|
RETURN_NAMES = ("images", "audio", "assembly_plan")
|
|
INPUT_IS_LIST = True
|
|
OUTPUT_IS_LIST = (True, True, False)
|
|
FUNCTION = "arm"
|
|
CATEGORY = "P2-5 Probe"
|
|
|
|
def arm(self, images, audio, assembly_plan):
|
|
backing_root = Path(results["backing_root"])
|
|
|
|
def trigger():
|
|
deadline = time.monotonic() + 30.0
|
|
while time.monotonic() < deadline:
|
|
if _managed_files(backing_root, "bin"):
|
|
results["interrupt_observed_allocation"] = True
|
|
time.sleep(0.05)
|
|
core_nodes.interrupt_processing(True)
|
|
return
|
|
time.sleep(0.005)
|
|
results["interrupt_observed_allocation"] = False
|
|
core_nodes.interrupt_processing(True)
|
|
|
|
threading.Thread(target=trigger, daemon=True).start()
|
|
return images, audio, assembly_plan[0]
|
|
|
|
from comfy_extras.nodes_audio import VAEDecodeAudio
|
|
|
|
core_nodes.NODE_CLASS_MAPPINGS.update(package_module.NODE_CLASS_MAPPINGS)
|
|
core_nodes.NODE_CLASS_MAPPINGS.update(
|
|
{
|
|
"VAEDecodeAudio": VAEDecodeAudio,
|
|
"P25StoredLongTerminalFixture": P25StoredLongTerminalFixture,
|
|
"P25SyntheticDecodedStressFixture": P25SyntheticDecodedStressFixture,
|
|
"P25DigestOutput": P25DigestOutput,
|
|
"P25ClearOutput": P25ClearOutput,
|
|
"P25InterruptAfterAllocation": P25InterruptAfterAllocation,
|
|
}
|
|
)
|
|
if video_combine is not None:
|
|
core_nodes.NODE_CLASS_MAPPINGS["P25VHSVideoCombine"] = video_combine
|
|
|
|
|
|
def _prompt(
|
|
backend: str,
|
|
*,
|
|
downstream: bool = False,
|
|
interrupt_after_allocation: bool = False,
|
|
stress: bool = False,
|
|
) -> dict[str, Any]:
|
|
if stress:
|
|
return {
|
|
"1": {
|
|
"class_type": "P25SyntheticDecodedStressFixture",
|
|
"inputs": {"nonce": 0},
|
|
},
|
|
"6": {
|
|
"class_type": "H3ContinuumAssembleSeamV35",
|
|
"inputs": {
|
|
"images": ["1", 0],
|
|
"audio": ["1", 1],
|
|
"assembly_plan": ["1", 2],
|
|
"exact_total_duration": True,
|
|
"audio_seam": "Auto",
|
|
"video_seam": "Auto",
|
|
"buffer_backend": str(backend),
|
|
"diagnostics": "Detailed Report",
|
|
},
|
|
},
|
|
"7": {
|
|
"class_type": "P25DigestOutput",
|
|
"inputs": {
|
|
"images": ["6", 0],
|
|
"audio": ["6", 1],
|
|
"report": ["6", 2],
|
|
"backend": str(backend),
|
|
},
|
|
},
|
|
}
|
|
prompt = {
|
|
"1": {
|
|
"class_type": "P25StoredLongTerminalFixture",
|
|
"inputs": {"nonce": 0},
|
|
},
|
|
"2": {
|
|
"class_type": "VAELoader",
|
|
"inputs": {"vae_name": "minimax_h3_video_vae_fp16.safetensors"},
|
|
},
|
|
"3": {
|
|
"class_type": "VAELoader",
|
|
"inputs": {"vae_name": "minimax_h3_audio_vae_fp32.safetensors"},
|
|
},
|
|
"4": {
|
|
"class_type": "VAEDecode",
|
|
"inputs": {"samples": ["1", 0], "vae": ["2", 0]},
|
|
},
|
|
"5": {
|
|
"class_type": "VAEDecodeAudio",
|
|
"inputs": {"samples": ["1", 1], "vae": ["3", 0]},
|
|
},
|
|
"6": {
|
|
"class_type": "H3ContinuumAssembleSeamV35",
|
|
"inputs": {
|
|
"images": ["4", 0],
|
|
"audio": ["5", 0],
|
|
"assembly_plan": ["1", 2],
|
|
"exact_total_duration": True,
|
|
"audio_seam": "Auto",
|
|
"video_seam": "Auto",
|
|
"buffer_backend": str(backend),
|
|
"diagnostics": "Detailed Report",
|
|
},
|
|
},
|
|
"7": {
|
|
"class_type": "P25DigestOutput",
|
|
"inputs": {
|
|
"images": ["6", 0],
|
|
"audio": ["6", 1],
|
|
"report": ["6", 2],
|
|
"backend": str(backend),
|
|
},
|
|
},
|
|
}
|
|
if interrupt_after_allocation:
|
|
prompt["11"] = {
|
|
"class_type": "P25InterruptAfterAllocation",
|
|
"inputs": {
|
|
"images": ["4", 0],
|
|
"audio": ["5", 0],
|
|
"assembly_plan": ["1", 2],
|
|
},
|
|
}
|
|
prompt["6"]["inputs"].update(
|
|
{
|
|
"images": ["11", 0],
|
|
"audio": ["11", 1],
|
|
"assembly_plan": ["11", 2],
|
|
}
|
|
)
|
|
if downstream:
|
|
prompt.update(
|
|
{
|
|
"8": {
|
|
"class_type": "SaveImage",
|
|
"inputs": {
|
|
"images": ["6", 0],
|
|
"filename_prefix": "P25/Save",
|
|
},
|
|
},
|
|
"9": {
|
|
"class_type": "PreviewImage",
|
|
"inputs": {"images": ["6", 0]},
|
|
},
|
|
"10": {
|
|
"class_type": "P25VHSVideoCombine",
|
|
"inputs": {
|
|
"images": ["6", 0],
|
|
"audio": ["6", 1],
|
|
"frame_rate": 24,
|
|
"loop_count": 0,
|
|
"filename_prefix": "P25/VHS",
|
|
"format": "video/h264-mp4",
|
|
"pingpong": False,
|
|
"save_output": True,
|
|
},
|
|
},
|
|
}
|
|
)
|
|
return prompt
|
|
|
|
|
|
def _downstream_result(history: dict[str, Any]) -> dict[str, Any]:
|
|
outputs = history.get("outputs", {})
|
|
save_items = outputs.get("8", {}).get("images", [])
|
|
preview_items = outputs.get("9", {}).get("images", [])
|
|
video_items = outputs.get("10", {}).get("gifs", [])
|
|
if len(save_items) != 360 or len(preview_items) != 360 or len(video_items) != 1:
|
|
raise ProbeFailure(
|
|
"unexpected downstream output counts: "
|
|
f"save={len(save_items)} preview={len(preview_items)} vhs={len(video_items)}"
|
|
)
|
|
video_path = Path(video_items[0]["fullpath"])
|
|
if not video_path.is_file() or video_path.stat().st_size <= 0:
|
|
raise ProbeFailure(f"VHS output is missing: {video_path}")
|
|
ffprobe = shutil.which("ffprobe")
|
|
if ffprobe is None:
|
|
raise ProbeFailure("ffprobe is required for P2-5 downstream acceptance")
|
|
inspected = subprocess.run(
|
|
[
|
|
ffprobe,
|
|
"-v",
|
|
"error",
|
|
"-count_frames",
|
|
"-show_entries",
|
|
(
|
|
"stream=codec_type,codec_name,width,height,r_frame_rate,"
|
|
"nb_read_frames,sample_rate,channels,duration:format=duration,size"
|
|
),
|
|
"-of",
|
|
"json",
|
|
str(video_path),
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
media = json.loads(inspected.stdout)
|
|
video_stream = next(
|
|
(item for item in media.get("streams", []) if item.get("codec_type") == "video"),
|
|
None,
|
|
)
|
|
audio_stream = next(
|
|
(item for item in media.get("streams", []) if item.get("codec_type") == "audio"),
|
|
None,
|
|
)
|
|
if video_stream is None or audio_stream is None:
|
|
raise ProbeFailure("VHS output does not contain both video and audio")
|
|
expected = (
|
|
int(video_stream.get("width", 0)) == 640
|
|
and int(video_stream.get("height", 0)) == 640
|
|
and int(video_stream.get("nb_read_frames", 0)) == 360
|
|
and video_stream.get("r_frame_rate") == "24/1"
|
|
and int(audio_stream.get("sample_rate", 0)) == 32_000
|
|
)
|
|
if not expected:
|
|
raise ProbeFailure(f"unexpected VHS media contract: {media}")
|
|
return {
|
|
"save_frames": len(save_items),
|
|
"preview_frames": len(preview_items),
|
|
"vhs_file": str(video_path),
|
|
"vhs_bytes": int(video_path.stat().st_size),
|
|
"video_codec": video_stream.get("codec_name"),
|
|
"video_frames": int(video_stream["nb_read_frames"]),
|
|
"video_fps": video_stream["r_frame_rate"],
|
|
"video_duration": float(video_stream["duration"]),
|
|
"audio_codec": audio_stream.get("codec_name"),
|
|
"audio_sample_rate": int(audio_stream["sample_rate"]),
|
|
"container_duration": float(media["format"]["duration"]),
|
|
}
|
|
|
|
|
|
def _clear_prompt() -> dict[str, Any]:
|
|
return {"9": {"class_type": "P25ClearOutput", "inputs": {"token": 1}}}
|
|
|
|
|
|
def _has_event(executor, wanted: str) -> bool:
|
|
return any(event == wanted for event, _data in executor.status_messages)
|
|
|
|
|
|
def _cached_nodes(executor) -> set[str]:
|
|
for event, data in executor.status_messages:
|
|
if event == "execution_cached":
|
|
return {str(value) for value in data.get("nodes", [])}
|
|
return set()
|
|
|
|
|
|
def _run_backend(args, package_root: Path) -> dict[str, Any]:
|
|
_install_paths(args.comfy_root)
|
|
import folder_paths
|
|
import nodes as core_nodes
|
|
import server as comfy_server
|
|
from execution import CacheType, PromptExecutor
|
|
|
|
if not hasattr(comfy_server.PromptServer, "instance"):
|
|
comfy_server.PromptServer.instance = SimpleNamespace(
|
|
prompt_queue=SimpleNamespace(currently_running={}, put=lambda _item: None),
|
|
number=0,
|
|
)
|
|
|
|
work_root = args.work_root.resolve()
|
|
output_root = work_root / "output"
|
|
temp_root = work_root / "temp"
|
|
backing_root = temp_root / "h3-continuum-v35-file-backed"
|
|
for path in (output_root, temp_root, backing_root):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
|
|
package = _load_package(package_root)
|
|
results: dict[str, Any] = {"backing_root": str(backing_root)}
|
|
video_combine = None
|
|
if args.downstream:
|
|
from videohelpersuite.nodes import VideoCombine
|
|
|
|
video_combine = VideoCombine
|
|
_register_probe_nodes(
|
|
core_nodes,
|
|
package,
|
|
args.revision_root,
|
|
results,
|
|
video_combine=video_combine,
|
|
stress_size=int(args.stress_size),
|
|
)
|
|
buffers = importlib.import_module(f"{package.__name__}.v3.file_backed_buffer")
|
|
|
|
folder_paths.set_output_directory(str(output_root))
|
|
folder_paths.set_temp_directory(str(temp_root))
|
|
|
|
prompt = _prompt(
|
|
args.backend,
|
|
downstream=bool(args.downstream),
|
|
interrupt_after_allocation=bool(args.interrupt_after_allocation),
|
|
stress=bool(args.stress_size),
|
|
)
|
|
(work_root / "p25_prompt_graph.json").write_text(
|
|
json.dumps(prompt, indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
executor = PromptExecutor(
|
|
_Server(),
|
|
cache_type=CacheType.CLASSIC,
|
|
cache_args={"ram": 64.0, "ram_inactive": 0.0},
|
|
)
|
|
if torch.cuda.is_available():
|
|
torch.cuda.reset_peak_memory_stats()
|
|
monitor = _ResourceMonitor()
|
|
monitor.start()
|
|
started = time.perf_counter()
|
|
execute_outputs = ["7", "8", "9", "10"] if args.downstream else ["7"]
|
|
executor.execute(
|
|
prompt,
|
|
f"p25-{args.backend.lower()}",
|
|
extra_data={},
|
|
execute_outputs=execute_outputs,
|
|
)
|
|
cache_requeue = None
|
|
if args.cache_requeue and executor.success:
|
|
first_pairs = (
|
|
len(_managed_files(backing_root, "bin")),
|
|
len(_managed_files(backing_root, "json")),
|
|
)
|
|
executor.execute(
|
|
prompt,
|
|
f"p25-{args.backend.lower()}-requeue",
|
|
extra_data={},
|
|
execute_outputs=execute_outputs,
|
|
)
|
|
cached = _cached_nodes(executor)
|
|
second_pairs = (
|
|
len(_managed_files(backing_root, "bin")),
|
|
len(_managed_files(backing_root, "json")),
|
|
)
|
|
cache_requeue = {
|
|
"cached_nodes": sorted(cached),
|
|
"assembler_cached": "6" in cached,
|
|
"first_pair_count": list(first_pairs),
|
|
"second_pair_count": list(second_pairs),
|
|
"pair_count_bounded": first_pairs == second_pairs,
|
|
}
|
|
if not executor.success or not cache_requeue["assembler_cached"]:
|
|
raise ProbeFailure(f"large-output requeue cache miss: {cache_requeue}")
|
|
if not cache_requeue["pair_count_bounded"]:
|
|
raise ProbeFailure(f"large-output requeue grew backing files: {cache_requeue}")
|
|
elapsed = time.perf_counter() - started
|
|
resources = monitor.stop()
|
|
manager = buffers.get_file_backed_image_manager(backing_root)
|
|
if args.interrupt_after_allocation:
|
|
interrupted = _has_event(executor, "execution_interrupted")
|
|
core_nodes.interrupt_processing(False)
|
|
executor.execute(_clear_prompt(), "p25-clear", extra_data={}, execute_outputs=["9"])
|
|
del executor
|
|
_collect_until_empty(manager, backing_root)
|
|
result = {
|
|
"backend": args.backend,
|
|
"stage": "interrupt-after-allocation",
|
|
"status": "PASS" if interrupted else "FAIL",
|
|
"execution_interrupted": bool(interrupted),
|
|
"allocation_observed_before_interrupt": bool(
|
|
results.get("interrupt_observed_allocation", False)
|
|
),
|
|
"fixture": results.get("fixture"),
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"managed_bin_after_interrupt": len(_managed_files(backing_root, "bin")),
|
|
"managed_json_after_interrupt": len(_managed_files(backing_root, "json")),
|
|
"resources": resources,
|
|
"work_root": str(work_root),
|
|
}
|
|
if not interrupted or not result["allocation_observed_before_interrupt"]:
|
|
raise ProbeFailure(f"post-allocation interrupt was not proven: {result}")
|
|
(work_root / "p25_interrupt_result.json").write_text(
|
|
json.dumps(result, indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
return result
|
|
if not executor.success or _has_event(executor, "execution_error"):
|
|
raise ProbeFailure(f"{args.backend} PromptExecutor failed")
|
|
if "digest" not in results or "fixture" not in results:
|
|
raise ProbeFailure(f"{args.backend} probe outputs were not captured")
|
|
|
|
bin_files = _managed_files(backing_root, "bin")
|
|
json_files = _managed_files(backing_root, "json")
|
|
resources.update(
|
|
{
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"cuda_peak_allocated_bytes": (
|
|
int(torch.cuda.max_memory_allocated()) if torch.cuda.is_available() else 0
|
|
),
|
|
"cuda_peak_reserved_bytes": (
|
|
int(torch.cuda.max_memory_reserved()) if torch.cuda.is_available() else 0
|
|
),
|
|
"backing_bin_count_while_cached": len(bin_files),
|
|
"backing_json_count_while_cached": len(json_files),
|
|
"backing_bytes_while_cached": sum(item.stat().st_size for item in bin_files),
|
|
"pre_digest_memory": results.get("pre_digest_memory"),
|
|
}
|
|
)
|
|
downstream = _downstream_result(executor.history_result) if args.downstream else None
|
|
executor.execute(_clear_prompt(), "p25-clear", extra_data={}, execute_outputs=["9"])
|
|
del executor
|
|
_collect_until_empty(manager, backing_root)
|
|
resources["managed_bin_after_cache_release"] = len(
|
|
_managed_files(backing_root, "bin")
|
|
)
|
|
resources["managed_json_after_cache_release"] = len(
|
|
_managed_files(backing_root, "json")
|
|
)
|
|
resources["nonmanaged_root_entries_after_cache_release"] = len(
|
|
[
|
|
item
|
|
for item in backing_root.iterdir()
|
|
if item.name != ".h3c-p20.lock"
|
|
]
|
|
)
|
|
result = {
|
|
"backend": args.backend,
|
|
"status": "PASS",
|
|
"fixture": results["fixture"],
|
|
"stress": results.get("stress"),
|
|
"digest": results["digest"],
|
|
"resources": resources,
|
|
"cache_requeue": cache_requeue,
|
|
"downstream": downstream,
|
|
"work_root": str(work_root),
|
|
}
|
|
(work_root / "p25_backend_result.json").write_text(
|
|
json.dumps(result, indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
return result
|
|
|
|
|
|
def _last_json(stdout: str) -> dict[str, Any]:
|
|
marker = "P25_RESULT_JSON="
|
|
line = next(
|
|
(value for value in reversed(stdout.splitlines()) if value.startswith(marker)),
|
|
"",
|
|
)
|
|
if not line:
|
|
raise ProbeFailure("backend child did not emit a result marker")
|
|
return json.loads(line[len(marker) :])
|
|
|
|
|
|
def _summarize_results(work_root: Path, results: dict[str, Any] | None = None):
|
|
work_root = work_root.resolve()
|
|
if results is None:
|
|
results = {
|
|
"RAM": json.loads(
|
|
(work_root / "ram" / "p25_backend_result.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
),
|
|
"Disk-backed": json.loads(
|
|
(work_root / "disk" / "p25_backend_result.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
),
|
|
}
|
|
ram = results["RAM"]
|
|
disk = results["Disk-backed"]
|
|
disk_backing_root = (
|
|
work_root / "disk" / "temp" / "h3-continuum-v35-file-backed"
|
|
)
|
|
managed_bin_after = len(_managed_files(disk_backing_root, "bin"))
|
|
managed_json_after = len(_managed_files(disk_backing_root, "json"))
|
|
disk["resources"]["managed_bin_after_cache_release"] = managed_bin_after
|
|
disk["resources"]["managed_json_after_cache_release"] = managed_json_after
|
|
comparisons = {
|
|
"fixture_equal": ram["fixture"] == disk["fixture"],
|
|
"video_shape_equal": ram["digest"]["video_shape"] == disk["digest"]["video_shape"],
|
|
"video_sha256_equal": ram["digest"]["video_sha256"] == disk["digest"]["video_sha256"],
|
|
"audio_shape_equal": ram["digest"]["audio_shape"] == disk["digest"]["audio_shape"],
|
|
"audio_sha256_equal": ram["digest"]["audio_sha256"] == disk["digest"]["audio_sha256"],
|
|
"audio_sample_rate_equal": ram["digest"]["audio_sample_rate"] == disk["digest"]["audio_sample_rate"],
|
|
"both_finite": bool(
|
|
ram["digest"]["video_finite"]
|
|
and disk["digest"]["video_finite"]
|
|
and ram["digest"]["audio_finite"]
|
|
and disk["digest"]["audio_finite"]
|
|
),
|
|
"disk_backing_reclaimed": managed_bin_after == 0 and managed_json_after == 0,
|
|
}
|
|
if not all(comparisons.values()):
|
|
raise ProbeFailure(f"RAM/Disk-backed comparison failed: {comparisons}")
|
|
peak_delta = (
|
|
int(ram["resources"]["peak_rss_bytes"])
|
|
- int(disk["resources"]["peak_rss_bytes"])
|
|
)
|
|
result = {
|
|
"phase": "P2-5",
|
|
"stage": "small-long-terminal-ab",
|
|
"status": "PASS",
|
|
"gpu_used": True,
|
|
"comparisons": comparisons,
|
|
"rss_peak_reduction_bytes": peak_delta,
|
|
"rss_peak_reduction_percent": round(
|
|
peak_delta / max(1, int(ram["resources"]["peak_rss_bytes"])) * 100.0,
|
|
3,
|
|
),
|
|
"backends": results,
|
|
"work_root": str(work_root),
|
|
}
|
|
(work_root / "p25_result.json").write_text(
|
|
json.dumps(result, indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
return result
|
|
|
|
|
|
def _run_parent(args, package_root: Path) -> dict[str, Any]:
|
|
work_root = args.work_root.resolve()
|
|
work_root.mkdir(parents=True, exist_ok=True)
|
|
common = [
|
|
sys.executable,
|
|
str(Path(__file__).resolve()),
|
|
"--comfy-root",
|
|
str(args.comfy_root),
|
|
"--package-root",
|
|
str(package_root),
|
|
"--revision-root",
|
|
str(args.revision_root),
|
|
]
|
|
results = {}
|
|
for backend, folder in (("RAM", "ram"), ("Disk-backed", "disk")):
|
|
child = subprocess.run(
|
|
[*common, "--work-root", str(work_root / folder), "--backend", backend],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
(work_root / f"{folder}_stdout.txt").write_text(child.stdout, encoding="utf-8")
|
|
(work_root / f"{folder}_stderr.txt").write_text(child.stderr, encoding="utf-8")
|
|
results[backend] = _last_json(child.stdout)
|
|
return _summarize_results(work_root, results)
|
|
|
|
|
|
def _parse_args():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--comfy-root", type=Path, required=True)
|
|
parser.add_argument("--package-root", type=Path)
|
|
parser.add_argument("--revision-root", type=Path, required=True)
|
|
parser.add_argument("--work-root", type=Path, required=True)
|
|
parser.add_argument("--backend", choices=("Auto", "RAM", "Disk-backed"))
|
|
parser.add_argument("--summarize-existing", action="store_true")
|
|
parser.add_argument("--downstream", action="store_true")
|
|
parser.add_argument("--interrupt-after-allocation", action="store_true")
|
|
parser.add_argument("--cache-requeue", action="store_true")
|
|
parser.add_argument(
|
|
"--stress-size",
|
|
type=int,
|
|
default=0,
|
|
help="Use a synthetic decoded square fixture of this size; 1536 is 9.49 GiB.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = _parse_args()
|
|
args.comfy_root = args.comfy_root.resolve()
|
|
args.revision_root = args.revision_root.resolve()
|
|
args.work_root = args.work_root.resolve()
|
|
package_root = (
|
|
args.package_root.resolve()
|
|
if args.package_root is not None
|
|
else Path(__file__).resolve().parents[1]
|
|
)
|
|
if args.backend is not None:
|
|
result = _run_backend(args, package_root)
|
|
print("P25_RESULT_JSON=" + json.dumps(result, sort_keys=True))
|
|
return 0
|
|
if args.summarize_existing:
|
|
result = _summarize_results(args.work_root)
|
|
print(json.dumps(result, indent=2, sort_keys=True))
|
|
return 0
|
|
result = _run_parent(args, package_root)
|
|
print(json.dumps(result, indent=2, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|