"""Run the Phase F Full-vs-Review GPU parity checkpoint. This first checkpoint intentionally submits identical V3.8 sampler inputs for successive Review/Continue queues. It therefore exercises the same ComfyUI cache contract as a fixed-seed workflow instead of adding a test-only cache nonce that could hide a Production integration defect. """ from __future__ import annotations import argparse import copy import hashlib import json from pathlib import Path import shutil import subprocess import sys from typing import Any, Mapping import uuid TOOLS_ROOT = Path(__file__).resolve().parent if str(TOOLS_ROOT) not in sys.path: sys.path.insert(0, str(TOOLS_ROOT)) from p32_api_runner import submit_and_wait from v36_r1_stress_gate_runner import ResourceMonitor from v38_easy_gpu_gate_runner import _output_media, _probe_media class GateFailure(RuntimeError): pass def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for block in iter(lambda: stream.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def _find_one(prompt: Mapping[str, Any], class_type: str) -> tuple[str, dict[str, Any]]: matches = [ (str(node_id), node) for node_id, node in prompt.items() if str(node.get("class_type")) == class_type ] if len(matches) != 1: raise GateFailure(f"expected one {class_type}, found {len(matches)}") return matches[0] def _build_prompt( template: Mapping[str, Any], *, run_name: str, generation_mode: str, review_action: str, output_prefix: str, seed: int, ) -> dict[str, Any]: prompt = copy.deepcopy(dict(template)) _, sampler = _find_one(prompt, "H3ContinuumSamplerV38") inputs = sampler["inputs"] inputs.update( { "aspect": "Landscape 16:9", "preset": "Draft — 0.30 MP", "custom_mp": 0.30, "chunks": 3, "chunk_seconds": 10.0, "base_seed": int(seed), "diagnostics": "Detailed Report", "reroll_from_chunk": "Auto", "reroll_nonce": 0, "strict_compatibility": False, "debug": True, "show_preview": False, "run_storage": "Save + Auto Resume", "run_name": run_name, "project_id": str(uuid.uuid5(uuid.NAMESPACE_URL, run_name)), "generation_mode": generation_mode, "review_action": review_action, } ) _, save_video = _find_one(prompt, "SaveVideo") save_video["inputs"]["filename_prefix"] = output_prefix return prompt def _latest_manifest(run_root: Path) -> tuple[Path, dict[str, Any]]: candidates = list((run_root / "revisions").glob("*/manifest.json")) if not candidates: raise GateFailure(f"Run Storage manifest is missing: {run_root}") candidates.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True) path = candidates[0] return path, json.loads(path.read_text(encoding="utf-8")) def _tensor_sha256(path: Path, key: str) -> str: from safetensors import safe_open with safe_open(path, framework="pt", device="cpu") as handle: tensor = handle.get_tensor(key).contiguous() digest = hashlib.sha256() digest.update(str(tensor.dtype).encode("ascii")) digest.update(json.dumps(list(tensor.shape), separators=(",", ":")).encode("ascii")) digest.update(tensor.numpy().tobytes(order="C")) return digest.hexdigest() def _manifest_snapshot(run_root: Path) -> dict[str, Any]: manifest_path, manifest = _latest_manifest(run_root) revision_root = manifest_path.parent chunks = [] for stored in manifest.get("chunks") or []: chunk_path = revision_root / "chunks" / str(stored["filename"]) entry = dict(stored.get("entry") or {}) chunks.append( { "sequence_index": int(stored["sequence_index"]), "filename": str(stored["filename"]), "file_sha256": _sha256_file(chunk_path), "manifest_file_sha256": str(stored["file_sha256"]), "video_latent_sha256": _tensor_sha256(chunk_path, "video"), "audio_latent_sha256": _tensor_sha256(chunk_path, "audio"), "seed": int(entry["seed"]), "prompt_hash": str(entry["prompt_hash"]), "plan": entry.get("plan"), } ) return { "path": str(manifest_path), "mtime_ns": manifest_path.stat().st_mtime_ns, "status": str(manifest.get("status")), "review_control_version": manifest.get("review_control_version"), "review_pause_reason": manifest.get("review_pause_reason"), "review_unit": manifest.get("review_unit"), "branch_regenerate_from": manifest.get("branch_regenerate_from"), "effective_reroll_nonce": manifest.get("effective_reroll_nonce"), "chunks": chunks, "report_summary": str(manifest.get("report_summary") or ""), "manifest": manifest, } def _stream_sha256(command: list[str]) -> tuple[str, int]: process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.stdout is None: raise GateFailure("failed to open ffmpeg stdout") digest = hashlib.sha256() size = 0 while True: block = process.stdout.read(1024 * 1024) if not block: break digest.update(block) size += len(block) stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else "" code = process.wait() if code: raise GateFailure(f"ffmpeg raw decode failed ({code}): {stderr[-2000:]}") return digest.hexdigest(), size def _decoded_hashes(media_path: Path, ffmpeg: str, probe: Mapping[str, Any]) -> dict[str, Any]: video_sha, video_bytes = _stream_sha256( [ ffmpeg, "-v", "error", "-i", str(media_path), "-map", "0:v:0", "-pix_fmt", "rgb24", "-f", "rawvideo", "-", ] ) pcm_sha, pcm_bytes = _stream_sha256( [ ffmpeg, "-v", "error", "-i", str(media_path), "-map", "0:a:0", "-ac", "2", "-ar", "32000", "-c:a", "pcm_f32le", "-f", "f32le", "-", ] ) return { "raw_rgb24_sha256": video_sha, "raw_rgb24_bytes": video_bytes, "pcm_f32le_32k_stereo_sha256": pcm_sha, "pcm_f32le_32k_stereo_bytes": pcm_bytes, "decoded_frames": int(probe["frames"]), "width": int(probe["width"]), "height": int(probe["height"]), "fps": str(probe["fps"]), "audio_sample_rate": int(probe["audio_sample_rate"]), "audio_channels": int(probe["audio_channels"]), "format_duration": float(probe["format_duration"]), } def _run_queue( args: argparse.Namespace, *, name: str, prompt: dict[str, Any], run_root: Path, ) -> dict[str, Any]: case_root = args.output_root / name case_root.mkdir(parents=True, exist_ok=True) prompt_path = case_root / "prompt.json" prompt_path.write_text(json.dumps(prompt, indent=2, ensure_ascii=False), encoding="utf-8") print(f"Phase F queue start: {name}", flush=True) with ResourceMonitor(args.backend_pid, args.monitor_interval) as monitor: prompt_id, history, elapsed = submit_and_wait( server=args.server, prompt=prompt, client_id=f"v38-phase-f-{uuid.uuid4().hex}", timeout_seconds=args.timeout_seconds, poll_seconds=args.poll_seconds, ) history_path = case_root / "history.json" history_path.write_text(json.dumps(history, indent=2, ensure_ascii=False), encoding="utf-8") media_path = _output_media(history, args.media_root) ffprobe = shutil.which("ffprobe") ffmpeg = shutil.which("ffmpeg") if not ffprobe or not ffmpeg: raise GateFailure("ffmpeg and ffprobe are required") media = _probe_media(media_path, ffprobe) decoded = _decoded_hashes(media_path, ffmpeg, media) storage = _manifest_snapshot(run_root) result = { "name": name, "prompt_id": prompt_id, "api_elapsed_seconds": float(elapsed), "prompt": str(prompt_path), "history": str(history_path), "media_path": str(media_path), "media_file_sha256": _sha256_file(media_path), "media": media, "decoded": decoded, "resource": monitor.as_dict(), "run_storage": storage, } (case_root / "summary.json").write_text( json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8" ) print( f"Phase F queue end: {name}; chunks={len(storage['chunks'])}; " f"status={storage['status']}; {elapsed:.3f}s", flush=True, ) return result def _parity(full: Mapping[str, Any], review: Mapping[str, Any]) -> dict[str, Any]: full_chunks = full["run_storage"]["chunks"] review_chunks = review["run_storage"]["chunks"] fields = ("seed", "prompt_hash", "video_latent_sha256", "audio_latent_sha256", "plan") chunk_checks = [] for index in range(max(len(full_chunks), len(review_chunks))): if index >= len(full_chunks) or index >= len(review_chunks): chunk_checks.append({"index": index + 1, "present": False}) continue checks = {field: full_chunks[index][field] == review_chunks[index][field] for field in fields} chunk_checks.append({"index": index + 1, "present": True, "checks": checks}) decoded_fields = ( "raw_rgb24_sha256", "pcm_f32le_32k_stereo_sha256", "decoded_frames", "width", "height", "fps", "audio_sample_rate", "audio_channels", "format_duration", ) decoded_checks = { field: full["decoded"][field] == review["decoded"][field] for field in decoded_fields } passed = ( len(full_chunks) == len(review_chunks) == 3 and all(item.get("present") and all(item["checks"].values()) for item in chunk_checks) and all(decoded_checks.values()) ) return {"pass": passed, "chunks": chunk_checks, "decoded": decoded_checks} def run(args: argparse.Namespace) -> dict[str, Any]: args.output_root.mkdir(parents=True, exist_ok=True) template = json.loads(args.template_prompt.read_text(encoding="utf-8")) stamp = args.run_tag full_run_name = f"v38_phase_f_g0_full_{stamp}" review_run_name = f"v38_phase_f_g1_review_{stamp}" runs_root = args.run_storage_root summary: dict[str, Any] = { "format": "h3-continuum-v38-review-phase-f-g0-g1-v1", "server": args.server, "backend_pid": args.backend_pid, "template_prompt": str(args.template_prompt), "template_prompt_sha256": _sha256_file(args.template_prompt), "seed": int(args.seed), "run_names": {"g0": full_run_name, "g1": review_run_name}, "pass": False, "stopped": False, "results": {}, } summary_path = args.output_root / "gate_summary.json" try: g0_prompt = _build_prompt( template, run_name=full_run_name, generation_mode="Full Run", review_action="Continue / Next", output_prefix=f"v38_phase_f/{stamp}/g0_full", seed=args.seed, ) g0 = _run_queue( args, name="g0_full", prompt=g0_prompt, run_root=runs_root / full_run_name, ) summary["results"]["g0"] = g0 if len(g0["run_storage"]["chunks"]) != 3 or g0["run_storage"]["status"] != "complete": raise GateFailure("G0 Full baseline did not complete exactly three chunks") review_results = [] previous_mtime = None for queue_index in range(1, 4): prompt = _build_prompt( template, run_name=review_run_name, generation_mode="Review Each Chunk", review_action="Continue / Next", output_prefix=f"v38_phase_f/{stamp}/g1_review_q{queue_index}", seed=args.seed, ) result = _run_queue( args, name=f"g1_review_q{queue_index}", prompt=prompt, run_root=runs_root / review_run_name, ) review_results.append(result) summary["results"]["g1"] = review_results actual_chunks = len(result["run_storage"]["chunks"]) expected_status = "complete" if queue_index == 3 else "review_ready" if actual_chunks != queue_index or result["run_storage"]["status"] != expected_status: unchanged = previous_mtime == result["run_storage"]["mtime_ns"] raise GateFailure( "G1 Review queue progression failed: " f"queue={queue_index}, chunks={actual_chunks}, " f"status={result['run_storage']['status']}, " f"manifest_unchanged={unchanged}" ) previous_mtime = result["run_storage"]["mtime_ns"] parity = _parity(g0, review_results[-1]) summary["parity"] = parity if not parity["pass"]: raise GateFailure("G0 Full and G1 Review exact parity failed") summary["pass"] = True return summary except Exception as exc: summary["stopped"] = True summary["failure_type"] = type(exc).__name__ summary["failure"] = str(exc) return summary finally: summary_path.write_text( json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8" ) def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--server", default="http://127.0.0.6:8190") parser.add_argument("--template-prompt", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument("--media-root", type=Path, required=True) parser.add_argument("--run-storage-root", type=Path, required=True) parser.add_argument("--run-tag", required=True) parser.add_argument("--backend-pid", type=int, default=None) parser.add_argument("--seed", type=int, default=927966463230733) parser.add_argument("--timeout-seconds", type=float, default=2400.0) parser.add_argument("--poll-seconds", type=float, default=2.0) parser.add_argument("--monitor-interval", type=float, default=1.0) args = parser.parse_args() summary = run(args) print(json.dumps(summary, indent=2, ensure_ascii=False), flush=True) return 0 if summary["pass"] else 2 if __name__ == "__main__": raise SystemExit(main())