Files
ComfyUI-H3-Continuum-Tiled/tools/v38_review_phase_f_recovery_runner.py

198 lines
7.3 KiB
Python

"""Resume the stopped Phase F G1 run after the Review cache fix.
The failed Queue 2 prompt is submitted unchanged for both recovery queues. This
keeps the sampler inputs (and the complete API graph) identical while proving
that Review mode no longer reuses the sampler's ComfyUI output cache.
"""
from __future__ import annotations
import argparse
import copy
import json
from pathlib import Path
from typing import Any, Mapping
from v38_review_phase_f_gpu_gate_runner import (
GateFailure,
_find_one,
_manifest_snapshot,
_parity,
_run_queue,
_sha256_file,
)
def _cached_nodes(history: Mapping[str, Any]) -> set[str]:
nodes: set[str] = set()
for prompt_history in history.values():
for message in (prompt_history.get("status") or {}).get("messages") or []:
if len(message) != 2 or message[0] != "execution_cached":
continue
nodes.update(str(node_id) for node_id in (message[1].get("nodes") or []))
return nodes
def _chunk_identity(chunk: Mapping[str, Any]) -> dict[str, Any]:
fields = (
"sequence_index",
"file_sha256",
"video_latent_sha256",
"audio_latent_sha256",
"seed",
"prompt_hash",
"plan",
)
return {field: copy.deepcopy(chunk[field]) for field in fields}
def _preserved_prefix(before: Mapping[str, Any], after: Mapping[str, Any]) -> bool:
before_chunks = before["chunks"]
after_chunks = after["chunks"]
return len(after_chunks) >= len(before_chunks) and all(
_chunk_identity(old) == _chunk_identity(after_chunks[index])
for index, old in enumerate(before_chunks)
)
def _sampler_inputs(prompt: Mapping[str, Any]) -> tuple[str, Mapping[str, Any]]:
node_id, sampler = _find_one(prompt, "H3ContinuumSamplerV38")
return node_id, sampler["inputs"]
def run(args: argparse.Namespace) -> dict[str, Any]:
args.output_root.mkdir(parents=True, exist_ok=True)
prior = json.loads(args.prior_summary.read_text(encoding="utf-8"))
g0 = prior["results"]["g0"]
failed_q2 = prior["results"]["g1"][-1]
prompt_path = Path(failed_q2["prompt"])
prompt = json.loads(prompt_path.read_text(encoding="utf-8"))
sampler_id, sampler_inputs = _sampler_inputs(prompt)
review_run_name = str(prior["run_names"]["g1"])
run_root = args.run_storage_root / review_run_name
before = _manifest_snapshot(run_root)
summary: dict[str, Any] = {
"format": "h3-continuum-v38-review-phase-f-r1-recovery-v1",
"server": args.server,
"backend_pid": args.backend_pid,
"prior_summary": str(args.prior_summary),
"failed_queue_prompt": str(prompt_path),
"failed_queue_prompt_sha256": _sha256_file(prompt_path),
"sampler_node_id": sampler_id,
"sampler_inputs": sampler_inputs,
"run_name": review_run_name,
"initial_run_storage": before,
"pass": False,
"stopped": False,
"results": {},
}
summary_path = args.output_root / "recovery_summary.json"
try:
if len(before["chunks"]) != 1 or before["status"] != "review_ready":
raise GateFailure(
"Recovery must start at Chunk 1 review_ready: "
f"chunks={len(before['chunks'])}, status={before['status']}"
)
q2 = _run_queue(
args,
name="g1_recovery_q2",
prompt=prompt,
run_root=run_root,
)
q2_cached = sorted(_cached_nodes(json.loads(Path(q2["history"]).read_text(encoding="utf-8"))))
q2_storage = q2["run_storage"]
q2_checks = {
"exact_failed_prompt_reused": _sha256_file(Path(q2["prompt"]))
== summary["failed_queue_prompt_sha256"],
"sampler_not_cached": sampler_id not in q2_cached,
"status_review_ready": q2_storage["status"] == "review_ready",
"exactly_two_chunks": len(q2_storage["chunks"]) == 2,
"exactly_one_new_physical_chunk": len(q2_storage["chunks"])
== len(before["chunks"]) + 1,
"chunk_1_exactly_reused": _preserved_prefix(before, q2_storage),
"manifest_updated": q2_storage["mtime_ns"] != before["mtime_ns"],
}
summary["results"]["q2"] = {
**q2,
"cached_nodes": q2_cached,
"checks": q2_checks,
}
if not all(q2_checks.values()):
raise GateFailure(f"Recovery Queue 2 failed: {q2_checks}")
q3 = _run_queue(
args,
name="g1_recovery_q3",
prompt=prompt,
run_root=run_root,
)
q3_cached = sorted(_cached_nodes(json.loads(Path(q3["history"]).read_text(encoding="utf-8"))))
q3_storage = q3["run_storage"]
q3_checks = {
"same_prompt_as_queue_2": _sha256_file(Path(q3["prompt"]))
== _sha256_file(Path(q2["prompt"])),
"sampler_not_cached": sampler_id not in q3_cached,
"status_complete": q3_storage["status"] == "complete",
"exactly_three_chunks": len(q3_storage["chunks"]) == 3,
"exactly_one_new_physical_chunk": len(q3_storage["chunks"])
== len(q2_storage["chunks"]) + 1,
"chunks_1_2_exactly_reused": _preserved_prefix(q2_storage, q3_storage),
"manifest_updated": q3_storage["mtime_ns"] != q2_storage["mtime_ns"],
}
parity = _parity(g0, q3)
summary["results"]["q3"] = {
**q3,
"cached_nodes": q3_cached,
"checks": q3_checks,
}
summary["parity"] = parity
if not all(q3_checks.values()):
raise GateFailure(f"Recovery Queue 3 failed: {q3_checks}")
if not parity["pass"]:
raise GateFailure("Recovered G1 does not exactly match the Phase F G0 baseline")
summary["pass"] = True
return summary
except Exception as exc:
summary["stopped"] = True
summary["failure_type"] = type(exc).__name__
summary["failure"] = str(exc)
return summary
finally:
summary_path.write_text(
json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8"
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--server", default="http://127.0.0.6:8190")
parser.add_argument("--prior-summary", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--media-root", type=Path, required=True)
parser.add_argument("--run-storage-root", type=Path, required=True)
parser.add_argument("--backend-pid", type=int, default=None)
parser.add_argument("--timeout-seconds", type=float, default=2400.0)
parser.add_argument("--poll-seconds", type=float, default=2.0)
parser.add_argument("--monitor-interval", type=float, default=1.0)
args = parser.parse_args()
summary = run(args)
print(
json.dumps(
{
"pass": summary["pass"],
"stopped": summary["stopped"],
"failure": summary.get("failure"),
"summary": str(args.output_root / "recovery_summary.json"),
},
indent=2,
ensure_ascii=False,
),
flush=True,
)
return 0 if summary["pass"] else 2
if __name__ == "__main__":
raise SystemExit(main())