Files
ComfyUI-H3-Continuum-Tiled/tools/v38_review_phase_f_gpu_gate_runner.py
T

416 lines
15 KiB
Python

"""Run the Phase F Full-vs-Review GPU parity checkpoint.
This first checkpoint intentionally submits identical V3.8 sampler inputs for
successive Review/Continue queues. It therefore exercises the same ComfyUI
cache contract as a fixed-seed workflow instead of adding a test-only cache
nonce that could hide a Production integration defect.
"""
from __future__ import annotations
import argparse
import copy
import hashlib
import json
from pathlib import Path
import shutil
import subprocess
import sys
from typing import Any, Mapping
import uuid
TOOLS_ROOT = Path(__file__).resolve().parent
if str(TOOLS_ROOT) not in sys.path:
sys.path.insert(0, str(TOOLS_ROOT))
from p32_api_runner import submit_and_wait
from v36_r1_stress_gate_runner import ResourceMonitor
from v38_easy_gpu_gate_runner import _output_media, _probe_media
class GateFailure(RuntimeError):
pass
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def _find_one(prompt: Mapping[str, Any], class_type: str) -> tuple[str, dict[str, Any]]:
matches = [
(str(node_id), node)
for node_id, node in prompt.items()
if str(node.get("class_type")) == class_type
]
if len(matches) != 1:
raise GateFailure(f"expected one {class_type}, found {len(matches)}")
return matches[0]
def _build_prompt(
template: Mapping[str, Any],
*,
run_name: str,
generation_mode: str,
review_action: str,
output_prefix: str,
seed: int,
) -> dict[str, Any]:
prompt = copy.deepcopy(dict(template))
_, sampler = _find_one(prompt, "H3ContinuumSamplerV38")
inputs = sampler["inputs"]
inputs.update(
{
"aspect": "Landscape 16:9",
"preset": "Draft — 0.30 MP",
"custom_mp": 0.30,
"chunks": 3,
"chunk_seconds": 10.0,
"base_seed": int(seed),
"diagnostics": "Detailed Report",
"reroll_from_chunk": "Auto",
"reroll_nonce": 0,
"strict_compatibility": False,
"debug": True,
"show_preview": False,
"run_storage": "Save + Auto Resume",
"run_name": run_name,
"project_id": str(uuid.uuid5(uuid.NAMESPACE_URL, run_name)),
"generation_mode": generation_mode,
"review_action": review_action,
}
)
_, save_video = _find_one(prompt, "SaveVideo")
save_video["inputs"]["filename_prefix"] = output_prefix
return prompt
def _latest_manifest(run_root: Path) -> tuple[Path, dict[str, Any]]:
candidates = list((run_root / "revisions").glob("*/manifest.json"))
if not candidates:
raise GateFailure(f"Run Storage manifest is missing: {run_root}")
candidates.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True)
path = candidates[0]
return path, json.loads(path.read_text(encoding="utf-8"))
def _tensor_sha256(path: Path, key: str) -> str:
from safetensors import safe_open
with safe_open(path, framework="pt", device="cpu") as handle:
tensor = handle.get_tensor(key).contiguous()
digest = hashlib.sha256()
digest.update(str(tensor.dtype).encode("ascii"))
digest.update(json.dumps(list(tensor.shape), separators=(",", ":")).encode("ascii"))
digest.update(tensor.numpy().tobytes(order="C"))
return digest.hexdigest()
def _manifest_snapshot(run_root: Path) -> dict[str, Any]:
manifest_path, manifest = _latest_manifest(run_root)
revision_root = manifest_path.parent
chunks = []
for stored in manifest.get("chunks") or []:
chunk_path = revision_root / "chunks" / str(stored["filename"])
entry = dict(stored.get("entry") or {})
chunks.append(
{
"sequence_index": int(stored["sequence_index"]),
"filename": str(stored["filename"]),
"file_sha256": _sha256_file(chunk_path),
"manifest_file_sha256": str(stored["file_sha256"]),
"video_latent_sha256": _tensor_sha256(chunk_path, "video"),
"audio_latent_sha256": _tensor_sha256(chunk_path, "audio"),
"seed": int(entry["seed"]),
"prompt_hash": str(entry["prompt_hash"]),
"plan": entry.get("plan"),
}
)
return {
"path": str(manifest_path),
"mtime_ns": manifest_path.stat().st_mtime_ns,
"status": str(manifest.get("status")),
"review_control_version": manifest.get("review_control_version"),
"review_pause_reason": manifest.get("review_pause_reason"),
"review_unit": manifest.get("review_unit"),
"branch_regenerate_from": manifest.get("branch_regenerate_from"),
"effective_reroll_nonce": manifest.get("effective_reroll_nonce"),
"chunks": chunks,
"report_summary": str(manifest.get("report_summary") or ""),
"manifest": manifest,
}
def _stream_sha256(command: list[str]) -> tuple[str, int]:
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.stdout is None:
raise GateFailure("failed to open ffmpeg stdout")
digest = hashlib.sha256()
size = 0
while True:
block = process.stdout.read(1024 * 1024)
if not block:
break
digest.update(block)
size += len(block)
stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else ""
code = process.wait()
if code:
raise GateFailure(f"ffmpeg raw decode failed ({code}): {stderr[-2000:]}")
return digest.hexdigest(), size
def _decoded_hashes(media_path: Path, ffmpeg: str, probe: Mapping[str, Any]) -> dict[str, Any]:
video_sha, video_bytes = _stream_sha256(
[
ffmpeg,
"-v",
"error",
"-i",
str(media_path),
"-map",
"0:v:0",
"-pix_fmt",
"rgb24",
"-f",
"rawvideo",
"-",
]
)
pcm_sha, pcm_bytes = _stream_sha256(
[
ffmpeg,
"-v",
"error",
"-i",
str(media_path),
"-map",
"0:a:0",
"-ac",
"2",
"-ar",
"32000",
"-c:a",
"pcm_f32le",
"-f",
"f32le",
"-",
]
)
return {
"raw_rgb24_sha256": video_sha,
"raw_rgb24_bytes": video_bytes,
"pcm_f32le_32k_stereo_sha256": pcm_sha,
"pcm_f32le_32k_stereo_bytes": pcm_bytes,
"decoded_frames": int(probe["frames"]),
"width": int(probe["width"]),
"height": int(probe["height"]),
"fps": str(probe["fps"]),
"audio_sample_rate": int(probe["audio_sample_rate"]),
"audio_channels": int(probe["audio_channels"]),
"format_duration": float(probe["format_duration"]),
}
def _run_queue(
args: argparse.Namespace,
*,
name: str,
prompt: dict[str, Any],
run_root: Path,
) -> dict[str, Any]:
case_root = args.output_root / name
case_root.mkdir(parents=True, exist_ok=True)
prompt_path = case_root / "prompt.json"
prompt_path.write_text(json.dumps(prompt, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"Phase F queue start: {name}", flush=True)
with ResourceMonitor(args.backend_pid, args.monitor_interval) as monitor:
prompt_id, history, elapsed = submit_and_wait(
server=args.server,
prompt=prompt,
client_id=f"v38-phase-f-{uuid.uuid4().hex}",
timeout_seconds=args.timeout_seconds,
poll_seconds=args.poll_seconds,
)
history_path = case_root / "history.json"
history_path.write_text(json.dumps(history, indent=2, ensure_ascii=False), encoding="utf-8")
media_path = _output_media(history, args.media_root)
ffprobe = shutil.which("ffprobe")
ffmpeg = shutil.which("ffmpeg")
if not ffprobe or not ffmpeg:
raise GateFailure("ffmpeg and ffprobe are required")
media = _probe_media(media_path, ffprobe)
decoded = _decoded_hashes(media_path, ffmpeg, media)
storage = _manifest_snapshot(run_root)
result = {
"name": name,
"prompt_id": prompt_id,
"api_elapsed_seconds": float(elapsed),
"prompt": str(prompt_path),
"history": str(history_path),
"media_path": str(media_path),
"media_file_sha256": _sha256_file(media_path),
"media": media,
"decoded": decoded,
"resource": monitor.as_dict(),
"run_storage": storage,
}
(case_root / "summary.json").write_text(
json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8"
)
print(
f"Phase F queue end: {name}; chunks={len(storage['chunks'])}; "
f"status={storage['status']}; {elapsed:.3f}s",
flush=True,
)
return result
def _parity(full: Mapping[str, Any], review: Mapping[str, Any]) -> dict[str, Any]:
full_chunks = full["run_storage"]["chunks"]
review_chunks = review["run_storage"]["chunks"]
fields = ("seed", "prompt_hash", "video_latent_sha256", "audio_latent_sha256", "plan")
chunk_checks = []
for index in range(max(len(full_chunks), len(review_chunks))):
if index >= len(full_chunks) or index >= len(review_chunks):
chunk_checks.append({"index": index + 1, "present": False})
continue
checks = {field: full_chunks[index][field] == review_chunks[index][field] for field in fields}
chunk_checks.append({"index": index + 1, "present": True, "checks": checks})
decoded_fields = (
"raw_rgb24_sha256",
"pcm_f32le_32k_stereo_sha256",
"decoded_frames",
"width",
"height",
"fps",
"audio_sample_rate",
"audio_channels",
"format_duration",
)
decoded_checks = {
field: full["decoded"][field] == review["decoded"][field]
for field in decoded_fields
}
passed = (
len(full_chunks) == len(review_chunks) == 3
and all(item.get("present") and all(item["checks"].values()) for item in chunk_checks)
and all(decoded_checks.values())
)
return {"pass": passed, "chunks": chunk_checks, "decoded": decoded_checks}
def run(args: argparse.Namespace) -> dict[str, Any]:
args.output_root.mkdir(parents=True, exist_ok=True)
template = json.loads(args.template_prompt.read_text(encoding="utf-8"))
stamp = args.run_tag
full_run_name = f"v38_phase_f_g0_full_{stamp}"
review_run_name = f"v38_phase_f_g1_review_{stamp}"
runs_root = args.run_storage_root
summary: dict[str, Any] = {
"format": "h3-continuum-v38-review-phase-f-g0-g1-v1",
"server": args.server,
"backend_pid": args.backend_pid,
"template_prompt": str(args.template_prompt),
"template_prompt_sha256": _sha256_file(args.template_prompt),
"seed": int(args.seed),
"run_names": {"g0": full_run_name, "g1": review_run_name},
"pass": False,
"stopped": False,
"results": {},
}
summary_path = args.output_root / "gate_summary.json"
try:
g0_prompt = _build_prompt(
template,
run_name=full_run_name,
generation_mode="Full Run",
review_action="Continue / Next",
output_prefix=f"v38_phase_f/{stamp}/g0_full",
seed=args.seed,
)
g0 = _run_queue(
args,
name="g0_full",
prompt=g0_prompt,
run_root=runs_root / full_run_name,
)
summary["results"]["g0"] = g0
if len(g0["run_storage"]["chunks"]) != 3 or g0["run_storage"]["status"] != "complete":
raise GateFailure("G0 Full baseline did not complete exactly three chunks")
review_results = []
previous_mtime = None
for queue_index in range(1, 4):
prompt = _build_prompt(
template,
run_name=review_run_name,
generation_mode="Review Each Chunk",
review_action="Continue / Next",
output_prefix=f"v38_phase_f/{stamp}/g1_review_q{queue_index}",
seed=args.seed,
)
result = _run_queue(
args,
name=f"g1_review_q{queue_index}",
prompt=prompt,
run_root=runs_root / review_run_name,
)
review_results.append(result)
summary["results"]["g1"] = review_results
actual_chunks = len(result["run_storage"]["chunks"])
expected_status = "complete" if queue_index == 3 else "review_ready"
if actual_chunks != queue_index or result["run_storage"]["status"] != expected_status:
unchanged = previous_mtime == result["run_storage"]["mtime_ns"]
raise GateFailure(
"G1 Review queue progression failed: "
f"queue={queue_index}, chunks={actual_chunks}, "
f"status={result['run_storage']['status']}, "
f"manifest_unchanged={unchanged}"
)
previous_mtime = result["run_storage"]["mtime_ns"]
parity = _parity(g0, review_results[-1])
summary["parity"] = parity
if not parity["pass"]:
raise GateFailure("G0 Full and G1 Review exact parity failed")
summary["pass"] = True
return summary
except Exception as exc:
summary["stopped"] = True
summary["failure_type"] = type(exc).__name__
summary["failure"] = str(exc)
return summary
finally:
summary_path.write_text(
json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8"
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--server", default="http://127.0.0.6:8190")
parser.add_argument("--template-prompt", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--media-root", type=Path, required=True)
parser.add_argument("--run-storage-root", type=Path, required=True)
parser.add_argument("--run-tag", required=True)
parser.add_argument("--backend-pid", type=int, default=None)
parser.add_argument("--seed", type=int, default=927966463230733)
parser.add_argument("--timeout-seconds", type=float, default=2400.0)
parser.add_argument("--poll-seconds", type=float, default=2.0)
parser.add_argument("--monitor-interval", type=float, default=1.0)
args = parser.parse_args()
summary = run(args)
print(json.dumps(summary, indent=2, ensure_ascii=False), flush=True)
return 0 if summary["pass"] else 2
if __name__ == "__main__":
raise SystemExit(main())