Release V3.8 source, illustrated manual and Spectrum workflow
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
"""Resume the stopped Phase F G1 run after the Review cache fix.
|
||||
|
||||
The failed Queue 2 prompt is submitted unchanged for both recovery queues. This
|
||||
keeps the sampler inputs (and the complete API graph) identical while proving
|
||||
that Review mode no longer reuses the sampler's ComfyUI output cache.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from v38_review_phase_f_gpu_gate_runner import (
|
||||
GateFailure,
|
||||
_find_one,
|
||||
_manifest_snapshot,
|
||||
_parity,
|
||||
_run_queue,
|
||||
_sha256_file,
|
||||
)
|
||||
|
||||
|
||||
def _cached_nodes(history: Mapping[str, Any]) -> set[str]:
|
||||
nodes: set[str] = set()
|
||||
for prompt_history in history.values():
|
||||
for message in (prompt_history.get("status") or {}).get("messages") or []:
|
||||
if len(message) != 2 or message[0] != "execution_cached":
|
||||
continue
|
||||
nodes.update(str(node_id) for node_id in (message[1].get("nodes") or []))
|
||||
return nodes
|
||||
|
||||
|
||||
def _chunk_identity(chunk: Mapping[str, Any]) -> dict[str, Any]:
|
||||
fields = (
|
||||
"sequence_index",
|
||||
"file_sha256",
|
||||
"video_latent_sha256",
|
||||
"audio_latent_sha256",
|
||||
"seed",
|
||||
"prompt_hash",
|
||||
"plan",
|
||||
)
|
||||
return {field: copy.deepcopy(chunk[field]) for field in fields}
|
||||
|
||||
|
||||
def _preserved_prefix(before: Mapping[str, Any], after: Mapping[str, Any]) -> bool:
|
||||
before_chunks = before["chunks"]
|
||||
after_chunks = after["chunks"]
|
||||
return len(after_chunks) >= len(before_chunks) and all(
|
||||
_chunk_identity(old) == _chunk_identity(after_chunks[index])
|
||||
for index, old in enumerate(before_chunks)
|
||||
)
|
||||
|
||||
|
||||
def _sampler_inputs(prompt: Mapping[str, Any]) -> tuple[str, Mapping[str, Any]]:
|
||||
node_id, sampler = _find_one(prompt, "H3ContinuumSamplerV38")
|
||||
return node_id, sampler["inputs"]
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
args.output_root.mkdir(parents=True, exist_ok=True)
|
||||
prior = json.loads(args.prior_summary.read_text(encoding="utf-8"))
|
||||
g0 = prior["results"]["g0"]
|
||||
failed_q2 = prior["results"]["g1"][-1]
|
||||
prompt_path = Path(failed_q2["prompt"])
|
||||
prompt = json.loads(prompt_path.read_text(encoding="utf-8"))
|
||||
sampler_id, sampler_inputs = _sampler_inputs(prompt)
|
||||
review_run_name = str(prior["run_names"]["g1"])
|
||||
run_root = args.run_storage_root / review_run_name
|
||||
before = _manifest_snapshot(run_root)
|
||||
summary: dict[str, Any] = {
|
||||
"format": "h3-continuum-v38-review-phase-f-r1-recovery-v1",
|
||||
"server": args.server,
|
||||
"backend_pid": args.backend_pid,
|
||||
"prior_summary": str(args.prior_summary),
|
||||
"failed_queue_prompt": str(prompt_path),
|
||||
"failed_queue_prompt_sha256": _sha256_file(prompt_path),
|
||||
"sampler_node_id": sampler_id,
|
||||
"sampler_inputs": sampler_inputs,
|
||||
"run_name": review_run_name,
|
||||
"initial_run_storage": before,
|
||||
"pass": False,
|
||||
"stopped": False,
|
||||
"results": {},
|
||||
}
|
||||
summary_path = args.output_root / "recovery_summary.json"
|
||||
try:
|
||||
if len(before["chunks"]) != 1 or before["status"] != "review_ready":
|
||||
raise GateFailure(
|
||||
"Recovery must start at Chunk 1 review_ready: "
|
||||
f"chunks={len(before['chunks'])}, status={before['status']}"
|
||||
)
|
||||
|
||||
q2 = _run_queue(
|
||||
args,
|
||||
name="g1_recovery_q2",
|
||||
prompt=prompt,
|
||||
run_root=run_root,
|
||||
)
|
||||
q2_cached = sorted(_cached_nodes(json.loads(Path(q2["history"]).read_text(encoding="utf-8"))))
|
||||
q2_storage = q2["run_storage"]
|
||||
q2_checks = {
|
||||
"exact_failed_prompt_reused": _sha256_file(Path(q2["prompt"]))
|
||||
== summary["failed_queue_prompt_sha256"],
|
||||
"sampler_not_cached": sampler_id not in q2_cached,
|
||||
"status_review_ready": q2_storage["status"] == "review_ready",
|
||||
"exactly_two_chunks": len(q2_storage["chunks"]) == 2,
|
||||
"exactly_one_new_physical_chunk": len(q2_storage["chunks"])
|
||||
== len(before["chunks"]) + 1,
|
||||
"chunk_1_exactly_reused": _preserved_prefix(before, q2_storage),
|
||||
"manifest_updated": q2_storage["mtime_ns"] != before["mtime_ns"],
|
||||
}
|
||||
summary["results"]["q2"] = {
|
||||
**q2,
|
||||
"cached_nodes": q2_cached,
|
||||
"checks": q2_checks,
|
||||
}
|
||||
if not all(q2_checks.values()):
|
||||
raise GateFailure(f"Recovery Queue 2 failed: {q2_checks}")
|
||||
|
||||
q3 = _run_queue(
|
||||
args,
|
||||
name="g1_recovery_q3",
|
||||
prompt=prompt,
|
||||
run_root=run_root,
|
||||
)
|
||||
q3_cached = sorted(_cached_nodes(json.loads(Path(q3["history"]).read_text(encoding="utf-8"))))
|
||||
q3_storage = q3["run_storage"]
|
||||
q3_checks = {
|
||||
"same_prompt_as_queue_2": _sha256_file(Path(q3["prompt"]))
|
||||
== _sha256_file(Path(q2["prompt"])),
|
||||
"sampler_not_cached": sampler_id not in q3_cached,
|
||||
"status_complete": q3_storage["status"] == "complete",
|
||||
"exactly_three_chunks": len(q3_storage["chunks"]) == 3,
|
||||
"exactly_one_new_physical_chunk": len(q3_storage["chunks"])
|
||||
== len(q2_storage["chunks"]) + 1,
|
||||
"chunks_1_2_exactly_reused": _preserved_prefix(q2_storage, q3_storage),
|
||||
"manifest_updated": q3_storage["mtime_ns"] != q2_storage["mtime_ns"],
|
||||
}
|
||||
parity = _parity(g0, q3)
|
||||
summary["results"]["q3"] = {
|
||||
**q3,
|
||||
"cached_nodes": q3_cached,
|
||||
"checks": q3_checks,
|
||||
}
|
||||
summary["parity"] = parity
|
||||
if not all(q3_checks.values()):
|
||||
raise GateFailure(f"Recovery Queue 3 failed: {q3_checks}")
|
||||
if not parity["pass"]:
|
||||
raise GateFailure("Recovered G1 does not exactly match the Phase F G0 baseline")
|
||||
summary["pass"] = True
|
||||
return summary
|
||||
except Exception as exc:
|
||||
summary["stopped"] = True
|
||||
summary["failure_type"] = type(exc).__name__
|
||||
summary["failure"] = str(exc)
|
||||
return summary
|
||||
finally:
|
||||
summary_path.write_text(
|
||||
json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--server", default="http://127.0.0.6:8190")
|
||||
parser.add_argument("--prior-summary", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
parser.add_argument("--media-root", type=Path, required=True)
|
||||
parser.add_argument("--run-storage-root", type=Path, required=True)
|
||||
parser.add_argument("--backend-pid", type=int, default=None)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=2400.0)
|
||||
parser.add_argument("--poll-seconds", type=float, default=2.0)
|
||||
parser.add_argument("--monitor-interval", type=float, default=1.0)
|
||||
args = parser.parse_args()
|
||||
summary = run(args)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"pass": summary["pass"],
|
||||
"stopped": summary["stopped"],
|
||||
"failure": summary.get("failure"),
|
||||
"summary": str(args.output_root / "recovery_summary.json"),
|
||||
},
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return 0 if summary["pass"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user