525 lines
20 KiB
Python
525 lines
20 KiB
Python
"""Run the A4 Chunk-scoped Refine GPU Matrix against the public node."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import copy
|
|
from dataclasses import asdict, dataclass
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import sys
|
|
from typing import Any, Mapping
|
|
|
|
|
|
TOOLS_ROOT = Path(__file__).resolve().parent
|
|
if str(TOOLS_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(TOOLS_ROOT))
|
|
|
|
from v38_a2_audio_only_gpu_canary import _output_latents, _output_text
|
|
from v38_a3c_selective_refine_gpu_matrix import (
|
|
GateFailure,
|
|
MatrixCase,
|
|
_configure_selective,
|
|
_execute,
|
|
_marker_json,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ScopeCase:
|
|
name: str
|
|
scope: str
|
|
scope_index: int
|
|
terminal_merge: bool
|
|
selected_group_index: int | None
|
|
selected_logical_chunks: tuple[int, ...]
|
|
|
|
|
|
CASES = (
|
|
ScopeCase("g1_all_video_audio_3x5", "All", 1, False, None, (1, 2, 3)),
|
|
ScopeCase("g2_physical_group_2", "Physical Group", 2, False, 1, (2,)),
|
|
ScopeCase("g3_logical_chunk_2", "Logical Chunk", 2, False, 1, (2,)),
|
|
ScopeCase("g4_terminal_logical_chunk_2", "Logical Chunk", 2, True, 1, (2, 3)),
|
|
ScopeCase("g5_terminal_logical_chunk_3", "Logical Chunk", 3, True, 1, (2, 3)),
|
|
)
|
|
|
|
|
|
def _key_paths(value: Any, prefix: str = "$") -> list[str]:
|
|
paths: list[str] = []
|
|
if isinstance(value, Mapping):
|
|
for key, item in sorted(value.items(), key=lambda pair: str(pair[0])):
|
|
path = f"{prefix}.{key}"
|
|
paths.append(path)
|
|
paths.extend(_key_paths(item, path))
|
|
elif isinstance(value, list):
|
|
for index, item in enumerate(value):
|
|
paths.extend(_key_paths(item, f"{prefix}[]"))
|
|
return paths
|
|
|
|
|
|
def _cross_run_plan_identity(value: Any) -> Any:
|
|
"""Ignore only the pre-existing per-execution monotonic start marker."""
|
|
|
|
normalized = copy.deepcopy(value)
|
|
if isinstance(normalized, dict):
|
|
normalized.pop("_runtime_started_at", None)
|
|
return normalized
|
|
|
|
|
|
def _configure(
|
|
template: Mapping[str, Any],
|
|
*,
|
|
case: ScopeCase,
|
|
output_prefix: str,
|
|
) -> dict[str, Any]:
|
|
base_case = MatrixCase(
|
|
name=case.name,
|
|
target="Video + Audio",
|
|
chunks=3,
|
|
terminal_merge=case.terminal_merge,
|
|
expected_physical_groups=2 if case.terminal_merge else 3,
|
|
boundary_seconds=(5.0, 10.0),
|
|
)
|
|
prompt = _configure_selective(
|
|
template,
|
|
case=base_case,
|
|
output_prefix=output_prefix,
|
|
)
|
|
selective = prompt["302"]["inputs"]
|
|
selective["refine_scope"] = case.scope
|
|
selective["refine_scope_index"] = int(case.scope_index)
|
|
prompt["302"]["_meta"]["title"] = (
|
|
f"Selective Video + Audio / {case.scope} {case.scope_index}"
|
|
)
|
|
return prompt
|
|
|
|
|
|
def _load_a3c_baseline(path: Path, result_name: str) -> dict[str, Any]:
|
|
summary = json.loads(path.read_text(encoding="utf-8"))
|
|
if not bool(summary.get("pass")):
|
|
raise GateFailure("A3c baseline summary is not PASS")
|
|
result = (summary.get("results") or {}).get(result_name)
|
|
if not isinstance(result, dict) or not bool(result.get("pass")):
|
|
raise GateFailure(f"A3c baseline {result_name!r} is missing or not PASS")
|
|
return result
|
|
|
|
|
|
def _identity_checks(
|
|
probe: Mapping[str, Any],
|
|
*,
|
|
selected_group_index: int,
|
|
) -> dict[str, bool]:
|
|
count = len(probe.get("physical_groups") or [])
|
|
expected_selected = [index == selected_group_index for index in range(count)]
|
|
video_object = list(probe.get("input_video_object_identity") or [])
|
|
video_tensor = list(probe.get("input_video_tensor_identity") or [])
|
|
audio_object = list(probe.get("input_audio_object_identity") or [])
|
|
audio_tensor = list(probe.get("input_audio_tensor_identity") or [])
|
|
input_video = list(probe.get("input_video_sha256") or [])
|
|
output_video = list(probe.get("output_video_sha256") or [])
|
|
input_audio = list(probe.get("input_audio_sha256") or [])
|
|
output_audio = list(probe.get("output_audio_sha256") or [])
|
|
return {
|
|
"identity_vector_lengths": all(
|
|
len(values) == count
|
|
for values in (
|
|
video_object,
|
|
video_tensor,
|
|
audio_object,
|
|
audio_tensor,
|
|
input_video,
|
|
output_video,
|
|
input_audio,
|
|
output_audio,
|
|
)
|
|
),
|
|
"video_object_passthrough": all(
|
|
identity == (not selected)
|
|
for identity, selected in zip(
|
|
video_object,
|
|
expected_selected,
|
|
strict=True,
|
|
)
|
|
),
|
|
"video_tensor_passthrough": all(
|
|
identity == (not selected)
|
|
for identity, selected in zip(
|
|
video_tensor,
|
|
expected_selected,
|
|
strict=True,
|
|
)
|
|
),
|
|
"audio_object_passthrough": all(
|
|
identity == (not selected)
|
|
for identity, selected in zip(
|
|
audio_object,
|
|
expected_selected,
|
|
strict=True,
|
|
)
|
|
),
|
|
"audio_tensor_passthrough": all(
|
|
identity == (not selected)
|
|
for identity, selected in zip(
|
|
audio_tensor,
|
|
expected_selected,
|
|
strict=True,
|
|
)
|
|
),
|
|
"video_sha_selection": all(
|
|
(before != after) is selected
|
|
for before, after, selected in zip(
|
|
input_video,
|
|
output_video,
|
|
expected_selected,
|
|
strict=True,
|
|
)
|
|
),
|
|
"audio_sha_selection": all(
|
|
(before != after) is selected
|
|
for before, after, selected in zip(
|
|
input_audio,
|
|
output_audio,
|
|
expected_selected,
|
|
strict=True,
|
|
)
|
|
),
|
|
}
|
|
|
|
|
|
def _run_case(
|
|
args: argparse.Namespace,
|
|
*,
|
|
case: ScopeCase,
|
|
template: Mapping[str, Any],
|
|
baseline_probe: Mapping[str, Any],
|
|
all_result: Mapping[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
output_prefix = f"v38_a4/{args.run_tag}/{case.name}"
|
|
prompt = _configure(template, case=case, output_prefix=output_prefix)
|
|
result, entry = _execute(
|
|
args,
|
|
name=case.name,
|
|
prompt=prompt,
|
|
expected_seconds=15.0,
|
|
boundaries=(5.0, 10.0),
|
|
)
|
|
probe_status = _output_text(entry, "908")
|
|
probe = _marker_json(probe_status, "A3C_PROBE_JSON=")
|
|
first_videos = _output_latents(entry, "902", args.output_root)
|
|
first_audios = _output_latents(entry, "903", args.output_root)
|
|
refined_videos = _output_latents(entry, "904", args.output_root)
|
|
refined_audios = _output_latents(entry, "905", args.output_root)
|
|
groups = list(probe.get("physical_groups") or [])
|
|
expected_groups = 2 if case.terminal_merge else 3
|
|
contract = (probe.get("output_plan") or {}).get("normalized", {}).get(
|
|
"second_pass_contract",
|
|
{},
|
|
)
|
|
checks = {
|
|
"physical_group_count": len(groups) == expected_groups,
|
|
"latent_counts": len(first_videos)
|
|
== len(first_audios)
|
|
== len(refined_videos)
|
|
== len(refined_audios)
|
|
== expected_groups,
|
|
"target_contract": probe.get("target_mode") == "video_audio"
|
|
and probe.get("seed_namespace") == "h3-continuum-refine-av-v1",
|
|
"contract_version": probe.get("second_pass_contract_version") == 1,
|
|
"schedule_hash": probe.get("schedule_sigma_hash")
|
|
== baseline_probe.get("schedule_sigma_hash"),
|
|
"plan_scope_field_absent": "refine_scope_contract" not in contract,
|
|
"media": int(result["media"]["frames"]) == 360
|
|
and abs(float(result["media"]["format_duration"]) - 15.0) <= 0.05
|
|
and int(result["media"]["audio_sample_rate"]) == 32000
|
|
and int(result["media"]["audio_channels"]) == 2,
|
|
"audio_health": bool(result["audio_health"]["pass"]),
|
|
}
|
|
if case.selected_group_index is None:
|
|
current_plan = (probe.get("output_plan") or {}).get("normalized")
|
|
baseline_plan = (baseline_probe.get("output_plan") or {}).get("normalized")
|
|
checks.update(
|
|
{
|
|
"all_video_input_parity": probe.get("input_video_sha256")
|
|
== baseline_probe.get("input_video_sha256"),
|
|
"all_audio_input_parity": probe.get("input_audio_sha256")
|
|
== baseline_probe.get("input_audio_sha256"),
|
|
"all_video_output_parity": probe.get("output_video_sha256")
|
|
== baseline_probe.get("output_video_sha256"),
|
|
"all_audio_output_parity": probe.get("output_audio_sha256")
|
|
== baseline_probe.get("output_audio_sha256"),
|
|
"all_seed_parity": probe.get("refine_group_seeds")
|
|
== baseline_probe.get("refine_group_seeds"),
|
|
"all_sampling_parity": probe.get("sampling_passes_reported")
|
|
== baseline_probe.get("sampling_passes_reported")
|
|
== expected_groups,
|
|
"all_plan_field_set": _key_paths(current_plan)
|
|
== _key_paths(baseline_plan),
|
|
"all_plan_parity": _cross_run_plan_identity(current_plan)
|
|
== _cross_run_plan_identity(baseline_plan),
|
|
"all_decoded_video_parity": result["decoded"]["video_sha256"]
|
|
== args.a3c_decoded_video_sha,
|
|
"all_decoded_audio_parity": result["decoded"]["audio_pcm_sha256"]
|
|
== args.a3c_decoded_audio_sha,
|
|
"all_scope_status": "scope=All." in probe_status,
|
|
}
|
|
)
|
|
else:
|
|
selected_index = int(case.selected_group_index)
|
|
expected_seed = list(baseline_probe.get("refine_group_seeds") or [])[selected_index]
|
|
checks.update(
|
|
{
|
|
"one_sampling": probe.get("sampling_passes_reported") == 1,
|
|
"one_seed": probe.get("refine_group_seeds") == [expected_seed],
|
|
"scope_status": f"scope={case.scope} {case.scope_index}." in probe_status,
|
|
"resolved_groups_status": f"selected_physical_groups=[{selected_index + 1}]"
|
|
in probe_status,
|
|
"resolved_logical_status": (
|
|
"selected_logical_chunks="
|
|
+ str(list(case.selected_logical_chunks))
|
|
)
|
|
in probe_status,
|
|
**_identity_checks(
|
|
probe,
|
|
selected_group_index=selected_index,
|
|
),
|
|
}
|
|
)
|
|
if all_result is not None:
|
|
all_plan = (all_result.get("probe") or {}).get("output_plan") or {}
|
|
checks["assembly_field_set_parity"] = _key_paths(
|
|
(probe.get("output_plan") or {}).get("normalized")
|
|
) == _key_paths(all_plan.get("normalized"))
|
|
result.update(
|
|
{
|
|
"case": asdict(case),
|
|
"probe_status": probe_status,
|
|
"probe": probe,
|
|
"first_video_latents": first_videos,
|
|
"first_audio_latents": first_audios,
|
|
"refined_video_latents": refined_videos,
|
|
"refined_audio_latents": refined_audios,
|
|
"checks": checks,
|
|
"pass": all(checks.values()),
|
|
}
|
|
)
|
|
case_root = args.evidence_root / case.name
|
|
(case_root / "result.json").write_text(
|
|
json.dumps(result, indent=2, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
print(f"A4 end: {case.name}; pass={result['pass']}", flush=True)
|
|
if not result["pass"]:
|
|
failed = [name for name, passed in checks.items() if not passed]
|
|
raise GateFailure(f"{case.name} failed: {failed}")
|
|
return result
|
|
|
|
|
|
def _load_resumed_all_result(
|
|
path: Path,
|
|
*,
|
|
baseline_probe: Mapping[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Revalidate one completed G1 without repeating its GPU Sampling."""
|
|
|
|
result = json.loads(path.read_text(encoding="utf-8"))
|
|
probe = result.get("probe") or {}
|
|
checks = dict(result.get("checks") or {})
|
|
current_plan = (probe.get("output_plan") or {}).get("normalized")
|
|
baseline_plan = (baseline_probe.get("output_plan") or {}).get("normalized")
|
|
checks["all_plan_field_set"] = _key_paths(current_plan) == _key_paths(
|
|
baseline_plan
|
|
)
|
|
checks["all_plan_parity"] = _cross_run_plan_identity(
|
|
current_plan
|
|
) == _cross_run_plan_identity(baseline_plan)
|
|
result["checks"] = checks
|
|
result["pass"] = all(checks.values())
|
|
result["resumed_without_resampling"] = True
|
|
if not result["pass"]:
|
|
failed = [name for name, passed in checks.items() if not passed]
|
|
raise GateFailure(f"resumed G1 validation failed: {failed}")
|
|
path.with_name("result.revalidated.json").write_text(
|
|
json.dumps(result, indent=2, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
return result
|
|
|
|
|
|
def _load_resumed_scoped_result(
|
|
path: Path,
|
|
*,
|
|
assembly_reference_probe: Mapping[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Revalidate one completed scoped case without repeating GPU Sampling."""
|
|
|
|
result = json.loads(path.read_text(encoding="utf-8"))
|
|
probe = result.get("probe") or {}
|
|
checks = dict(result.get("checks") or {})
|
|
current_plan = (probe.get("output_plan") or {}).get("normalized")
|
|
reference_plan = (assembly_reference_probe.get("output_plan") or {}).get(
|
|
"normalized"
|
|
)
|
|
checks["assembly_field_set_parity"] = _key_paths(current_plan) == _key_paths(
|
|
reference_plan
|
|
)
|
|
result["checks"] = checks
|
|
result["pass"] = all(checks.values())
|
|
result["resumed_without_resampling"] = True
|
|
if not result["pass"]:
|
|
failed = [name for name, passed in checks.items() if not passed]
|
|
raise GateFailure(f"resumed scoped validation failed: {failed}")
|
|
path.with_name("result.revalidated.json").write_text(
|
|
json.dumps(result, indent=2, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
return result
|
|
|
|
|
|
def _cross_case_checks(results: Mapping[str, Mapping[str, Any]]) -> dict[str, bool]:
|
|
g2 = results[CASES[1].name]
|
|
g3 = results[CASES[2].name]
|
|
g4 = results[CASES[3].name]
|
|
g5 = results[CASES[4].name]
|
|
return {
|
|
"physical_vs_logical_nonterminal_video": g2["probe"]["output_video_sha256"]
|
|
== g3["probe"]["output_video_sha256"],
|
|
"physical_vs_logical_nonterminal_audio": g2["probe"]["output_audio_sha256"]
|
|
== g3["probe"]["output_audio_sha256"],
|
|
"physical_vs_logical_nonterminal_seed": g2["probe"]["refine_group_seeds"]
|
|
== g3["probe"]["refine_group_seeds"],
|
|
"terminal_2_vs_3_video": g4["probe"]["output_video_sha256"]
|
|
== g5["probe"]["output_video_sha256"],
|
|
"terminal_2_vs_3_audio": g4["probe"]["output_audio_sha256"]
|
|
== g5["probe"]["output_audio_sha256"],
|
|
"terminal_2_vs_3_seed": g4["probe"]["refine_group_seeds"]
|
|
== g5["probe"]["refine_group_seeds"],
|
|
"terminal_2_vs_3_plan": g4["probe"]["output_plan"]["sha256"]
|
|
== g5["probe"]["output_plan"]["sha256"],
|
|
"terminal_2_vs_3_decoded_video": g4["decoded"]["video_sha256"]
|
|
== g5["decoded"]["video_sha256"],
|
|
"terminal_2_vs_3_decoded_audio": g4["decoded"]["audio_pcm_sha256"]
|
|
== g5["decoded"]["audio_pcm_sha256"],
|
|
}
|
|
|
|
|
|
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
args.evidence_root.mkdir(parents=True, exist_ok=True)
|
|
t2_template = json.loads(args.t2_template.read_text(encoding="utf-8"))
|
|
terminal_template = json.loads(args.terminal_template.read_text(encoding="utf-8"))
|
|
baseline_result = _load_a3c_baseline(
|
|
args.a3c_baseline,
|
|
"g4_video_audio_3x5",
|
|
)
|
|
terminal_baseline_result = _load_a3c_baseline(
|
|
args.a3c_baseline,
|
|
"g5_video_audio_terminal",
|
|
)
|
|
baseline_probe = baseline_result["probe"]
|
|
args.a3c_decoded_video_sha = baseline_result["decoded"]["video_sha256"]
|
|
args.a3c_decoded_audio_sha = baseline_result["decoded"]["audio_pcm_sha256"]
|
|
summary: dict[str, Any] = {
|
|
"format": "h3-continuum-v38-a4-chunk-scoped-refine-gpu-matrix-v1",
|
|
"server": args.server,
|
|
"backend_pid": args.backend_pid,
|
|
"run_tag": args.run_tag,
|
|
"a3c_baseline": str(args.a3c_baseline),
|
|
"results": {},
|
|
"pass": False,
|
|
"production_status": "HOLD for audio-targeted modes",
|
|
}
|
|
summary_path = args.evidence_root / "summary.json"
|
|
try:
|
|
resume_results: dict[str, Path] = {}
|
|
for path in args.resume_result or ():
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
name = str((payload.get("case") or {}).get("name") or "")
|
|
if not name or name in resume_results:
|
|
raise GateFailure(f"invalid or duplicate resumed result: {path}")
|
|
resume_results[name] = path
|
|
all_result: Mapping[str, Any] | None = None
|
|
for case in CASES:
|
|
assembly_reference_result = (
|
|
terminal_baseline_result if case.terminal_merge else all_result
|
|
)
|
|
if case.name in resume_results:
|
|
if case.selected_group_index is None:
|
|
result = _load_resumed_all_result(
|
|
resume_results[case.name],
|
|
baseline_probe=baseline_probe,
|
|
)
|
|
else:
|
|
if assembly_reference_result is None:
|
|
raise GateFailure("scoped resume has no Assembly reference")
|
|
result = _load_resumed_scoped_result(
|
|
resume_results[case.name],
|
|
assembly_reference_probe=assembly_reference_result["probe"],
|
|
)
|
|
else:
|
|
template = terminal_template if case.terminal_merge else t2_template
|
|
result = _run_case(
|
|
args,
|
|
case=case,
|
|
template=template,
|
|
baseline_probe=(
|
|
terminal_baseline_result["probe"]
|
|
if case.terminal_merge
|
|
else baseline_probe
|
|
),
|
|
all_result=assembly_reference_result,
|
|
)
|
|
summary["results"][case.name] = result
|
|
if case.selected_group_index is None:
|
|
all_result = result
|
|
summary["cross_case_checks"] = _cross_case_checks(summary["results"])
|
|
if not all(summary["cross_case_checks"].values()):
|
|
failed = [
|
|
name
|
|
for name, passed in summary["cross_case_checks"].items()
|
|
if not passed
|
|
]
|
|
raise GateFailure(f"cross-case parity failed: {failed}")
|
|
summary["pass"] = True
|
|
summary["status"] = "A4 Chunk-scoped Refine GPU Experimental PASS"
|
|
return summary
|
|
except Exception as exc:
|
|
summary["failure_type"] = type(exc).__name__
|
|
summary["failure"] = str(exc)
|
|
summary["status"] = "A4 STOP / Production HOLD"
|
|
return summary
|
|
finally:
|
|
summary_path.write_text(
|
|
json.dumps(summary, indent=2, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--server", default="http://127.0.0.6:8190")
|
|
parser.add_argument("--backend-pid", type=int, required=True)
|
|
parser.add_argument("--run-tag", required=True)
|
|
parser.add_argument("--evidence-root", type=Path, required=True)
|
|
parser.add_argument("--t2-template", type=Path, required=True)
|
|
parser.add_argument("--terminal-template", type=Path, required=True)
|
|
parser.add_argument("--a3c-baseline", type=Path, required=True)
|
|
parser.add_argument("--resume-result", type=Path, action="append")
|
|
parser.add_argument(
|
|
"--output-root",
|
|
type=Path,
|
|
default=Path(r"D:\output\video\comfy_video"),
|
|
)
|
|
parser.add_argument("--ffmpeg", default=shutil.which("ffmpeg"))
|
|
parser.add_argument("--ffprobe", default=shutil.which("ffprobe"))
|
|
parser.add_argument("--timeout-seconds", type=float, default=3600.0)
|
|
args = parser.parse_args()
|
|
if not args.ffmpeg or not args.ffprobe:
|
|
raise SystemExit("ffmpeg and ffprobe are required")
|
|
summary = run(args)
|
|
print(json.dumps(summary, indent=2, ensure_ascii=False), flush=True)
|
|
return 0 if summary["pass"] else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|