198 lines
7.3 KiB
Python
198 lines
7.3 KiB
Python
"""Resume the stopped Phase F G1 run after the Review cache fix.
|
|
|
|
The failed Queue 2 prompt is submitted unchanged for both recovery queues. This
|
|
keeps the sampler inputs (and the complete API graph) identical while proving
|
|
that Review mode no longer reuses the sampler's ComfyUI output cache.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
|
|
from v38_review_phase_f_gpu_gate_runner import (
|
|
GateFailure,
|
|
_find_one,
|
|
_manifest_snapshot,
|
|
_parity,
|
|
_run_queue,
|
|
_sha256_file,
|
|
)
|
|
|
|
|
|
def _cached_nodes(history: Mapping[str, Any]) -> set[str]:
|
|
nodes: set[str] = set()
|
|
for prompt_history in history.values():
|
|
for message in (prompt_history.get("status") or {}).get("messages") or []:
|
|
if len(message) != 2 or message[0] != "execution_cached":
|
|
continue
|
|
nodes.update(str(node_id) for node_id in (message[1].get("nodes") or []))
|
|
return nodes
|
|
|
|
|
|
def _chunk_identity(chunk: Mapping[str, Any]) -> dict[str, Any]:
|
|
fields = (
|
|
"sequence_index",
|
|
"file_sha256",
|
|
"video_latent_sha256",
|
|
"audio_latent_sha256",
|
|
"seed",
|
|
"prompt_hash",
|
|
"plan",
|
|
)
|
|
return {field: copy.deepcopy(chunk[field]) for field in fields}
|
|
|
|
|
|
def _preserved_prefix(before: Mapping[str, Any], after: Mapping[str, Any]) -> bool:
|
|
before_chunks = before["chunks"]
|
|
after_chunks = after["chunks"]
|
|
return len(after_chunks) >= len(before_chunks) and all(
|
|
_chunk_identity(old) == _chunk_identity(after_chunks[index])
|
|
for index, old in enumerate(before_chunks)
|
|
)
|
|
|
|
|
|
def _sampler_inputs(prompt: Mapping[str, Any]) -> tuple[str, Mapping[str, Any]]:
|
|
node_id, sampler = _find_one(prompt, "H3ContinuumSamplerV38")
|
|
return node_id, sampler["inputs"]
|
|
|
|
|
|
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
args.output_root.mkdir(parents=True, exist_ok=True)
|
|
prior = json.loads(args.prior_summary.read_text(encoding="utf-8"))
|
|
g0 = prior["results"]["g0"]
|
|
failed_q2 = prior["results"]["g1"][-1]
|
|
prompt_path = Path(failed_q2["prompt"])
|
|
prompt = json.loads(prompt_path.read_text(encoding="utf-8"))
|
|
sampler_id, sampler_inputs = _sampler_inputs(prompt)
|
|
review_run_name = str(prior["run_names"]["g1"])
|
|
run_root = args.run_storage_root / review_run_name
|
|
before = _manifest_snapshot(run_root)
|
|
summary: dict[str, Any] = {
|
|
"format": "h3-continuum-v38-review-phase-f-r1-recovery-v1",
|
|
"server": args.server,
|
|
"backend_pid": args.backend_pid,
|
|
"prior_summary": str(args.prior_summary),
|
|
"failed_queue_prompt": str(prompt_path),
|
|
"failed_queue_prompt_sha256": _sha256_file(prompt_path),
|
|
"sampler_node_id": sampler_id,
|
|
"sampler_inputs": sampler_inputs,
|
|
"run_name": review_run_name,
|
|
"initial_run_storage": before,
|
|
"pass": False,
|
|
"stopped": False,
|
|
"results": {},
|
|
}
|
|
summary_path = args.output_root / "recovery_summary.json"
|
|
try:
|
|
if len(before["chunks"]) != 1 or before["status"] != "review_ready":
|
|
raise GateFailure(
|
|
"Recovery must start at Chunk 1 review_ready: "
|
|
f"chunks={len(before['chunks'])}, status={before['status']}"
|
|
)
|
|
|
|
q2 = _run_queue(
|
|
args,
|
|
name="g1_recovery_q2",
|
|
prompt=prompt,
|
|
run_root=run_root,
|
|
)
|
|
q2_cached = sorted(_cached_nodes(json.loads(Path(q2["history"]).read_text(encoding="utf-8"))))
|
|
q2_storage = q2["run_storage"]
|
|
q2_checks = {
|
|
"exact_failed_prompt_reused": _sha256_file(Path(q2["prompt"]))
|
|
== summary["failed_queue_prompt_sha256"],
|
|
"sampler_not_cached": sampler_id not in q2_cached,
|
|
"status_review_ready": q2_storage["status"] == "review_ready",
|
|
"exactly_two_chunks": len(q2_storage["chunks"]) == 2,
|
|
"exactly_one_new_physical_chunk": len(q2_storage["chunks"])
|
|
== len(before["chunks"]) + 1,
|
|
"chunk_1_exactly_reused": _preserved_prefix(before, q2_storage),
|
|
"manifest_updated": q2_storage["mtime_ns"] != before["mtime_ns"],
|
|
}
|
|
summary["results"]["q2"] = {
|
|
**q2,
|
|
"cached_nodes": q2_cached,
|
|
"checks": q2_checks,
|
|
}
|
|
if not all(q2_checks.values()):
|
|
raise GateFailure(f"Recovery Queue 2 failed: {q2_checks}")
|
|
|
|
q3 = _run_queue(
|
|
args,
|
|
name="g1_recovery_q3",
|
|
prompt=prompt,
|
|
run_root=run_root,
|
|
)
|
|
q3_cached = sorted(_cached_nodes(json.loads(Path(q3["history"]).read_text(encoding="utf-8"))))
|
|
q3_storage = q3["run_storage"]
|
|
q3_checks = {
|
|
"same_prompt_as_queue_2": _sha256_file(Path(q3["prompt"]))
|
|
== _sha256_file(Path(q2["prompt"])),
|
|
"sampler_not_cached": sampler_id not in q3_cached,
|
|
"status_complete": q3_storage["status"] == "complete",
|
|
"exactly_three_chunks": len(q3_storage["chunks"]) == 3,
|
|
"exactly_one_new_physical_chunk": len(q3_storage["chunks"])
|
|
== len(q2_storage["chunks"]) + 1,
|
|
"chunks_1_2_exactly_reused": _preserved_prefix(q2_storage, q3_storage),
|
|
"manifest_updated": q3_storage["mtime_ns"] != q2_storage["mtime_ns"],
|
|
}
|
|
parity = _parity(g0, q3)
|
|
summary["results"]["q3"] = {
|
|
**q3,
|
|
"cached_nodes": q3_cached,
|
|
"checks": q3_checks,
|
|
}
|
|
summary["parity"] = parity
|
|
if not all(q3_checks.values()):
|
|
raise GateFailure(f"Recovery Queue 3 failed: {q3_checks}")
|
|
if not parity["pass"]:
|
|
raise GateFailure("Recovered G1 does not exactly match the Phase F G0 baseline")
|
|
summary["pass"] = True
|
|
return summary
|
|
except Exception as exc:
|
|
summary["stopped"] = True
|
|
summary["failure_type"] = type(exc).__name__
|
|
summary["failure"] = str(exc)
|
|
return summary
|
|
finally:
|
|
summary_path.write_text(
|
|
json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--server", default="http://127.0.0.6:8190")
|
|
parser.add_argument("--prior-summary", type=Path, required=True)
|
|
parser.add_argument("--output-root", type=Path, required=True)
|
|
parser.add_argument("--media-root", type=Path, required=True)
|
|
parser.add_argument("--run-storage-root", type=Path, required=True)
|
|
parser.add_argument("--backend-pid", type=int, default=None)
|
|
parser.add_argument("--timeout-seconds", type=float, default=2400.0)
|
|
parser.add_argument("--poll-seconds", type=float, default=2.0)
|
|
parser.add_argument("--monitor-interval", type=float, default=1.0)
|
|
args = parser.parse_args()
|
|
summary = run(args)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"pass": summary["pass"],
|
|
"stopped": summary["stopped"],
|
|
"failure": summary.get("failure"),
|
|
"summary": str(args.output_root / "recovery_summary.json"),
|
|
},
|
|
indent=2,
|
|
ensure_ascii=False,
|
|
),
|
|
flush=True,
|
|
)
|
|
return 0 if summary["pass"] else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|