Files
ComfyUI-H3-Continuum-Tiled/tools/v36_r1_stress_gate_analyze.py

257 lines
10 KiB
Python

#!/usr/bin/env python3
"""Aggregate V3.6-R1 Stress Gate timings and boundary video metrics."""
from __future__ import annotations
import argparse
import json
import math
import re
from pathlib import Path
import cv2
import numpy as np
SAMPLING_RE = re.compile(r"sampling=([0-9.]+)s")
def _median(values: list[float]) -> float:
values = sorted(values)
middle = len(values) // 2
if len(values) % 2:
return values[middle]
return (values[middle - 1] + values[middle]) / 2.0
def _read_summary(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
def _sampling_seconds(run: dict) -> list[float]:
values: list[float] = []
for line in run["performance_lines"]:
if "physical group" not in line:
continue
match = SAMPLING_RE.search(line)
if match:
values.append(float(match.group(1)))
return values
def _video_path(output_prefix: str, output_dir: Path) -> Path:
stem = Path(output_prefix).name
matches = sorted(output_dir.glob(f"{stem}_*.mp4"), key=lambda path: path.stat().st_mtime)
if not matches:
raise FileNotFoundError(f"No output video for {stem}")
return matches[-1]
def _read_frames(path: Path) -> tuple[list[np.ndarray], float]:
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
raise RuntimeError(f"Unable to open {path}")
fps = float(capture.get(cv2.CAP_PROP_FPS))
frames: list[np.ndarray] = []
while True:
ok, frame = capture.read()
if not ok:
break
frames.append(frame)
capture.release()
return frames, fps
def _safe_cosine(left: np.ndarray, right: np.ndarray) -> float:
denominator = float(np.linalg.norm(left) * np.linalg.norm(right))
if denominator <= 1e-12:
return 1.0
return float(np.dot(left, right) / denominator)
def _boundary_metrics(frames: list[np.ndarray], boundary: int) -> dict:
first = max(0, boundary - 12)
last = min(len(frames) - 1, boundary + 24)
magnitudes: list[float] = []
mean_vectors: list[np.ndarray] = []
for index in range(first, last):
previous = cv2.cvtColor(frames[index], cv2.COLOR_BGR2GRAY)
current = cv2.cvtColor(frames[index + 1], cv2.COLOR_BGR2GRAY)
previous = cv2.resize(previous, (320, 320), interpolation=cv2.INTER_AREA)
current = cv2.resize(current, (320, 320), interpolation=cv2.INTER_AREA)
flow = cv2.calcOpticalFlowFarneback(
previous,
current,
None,
0.5,
3,
21,
3,
5,
1.2,
0,
)
magnitude = np.linalg.norm(flow, axis=2)
magnitudes.append(float(np.median(magnitude)))
mean_vectors.append(np.mean(flow.reshape(-1, 2), axis=0))
transition_index = boundary - first - 1
pre = magnitudes[max(0, transition_index - 6) : max(0, transition_index)]
post = magnitudes[transition_index + 1 : transition_index + 7]
pre_magnitude = float(np.median(pre)) if pre else 0.0
post_magnitude = float(np.median(post)) if post else 0.0
magnitude_ratio = post_magnitude / pre_magnitude if pre_magnitude > 1e-12 else math.inf
pre_vectors = mean_vectors[max(0, transition_index - 6) : max(0, transition_index)]
post_vectors = mean_vectors[transition_index + 1 : transition_index + 7]
pre_vector = np.mean(pre_vectors, axis=0) if pre_vectors else np.zeros(2, dtype=np.float32)
post_vector = np.mean(post_vectors, axis=0) if post_vectors else np.zeros(2, dtype=np.float32)
accelerations = np.diff(np.asarray(magnitudes, dtype=np.float64))
jerks = np.diff(accelerations)
local_jerk = jerks[max(0, transition_index - 3) : transition_index + 4]
before_frames = frames[max(0, boundary - 12) : boundary]
after_frames = frames[boundary : min(len(frames), boundary + 24)]
def color_stats(items: list[np.ndarray]) -> tuple[float, float, float]:
if not items:
return 0.0, 0.0, 0.0
luma: list[float] = []
contrast: list[float] = []
saturation: list[float] = []
for item in items:
gray = cv2.cvtColor(item, cv2.COLOR_BGR2GRAY)
hsv = cv2.cvtColor(item, cv2.COLOR_BGR2HSV)
luma.append(float(gray.mean()))
contrast.append(float(gray.std()))
saturation.append(float(hsv[:, :, 1].mean()))
return float(np.mean(luma)), float(np.mean(contrast)), float(np.mean(saturation))
before_luma, before_contrast, before_saturation = color_stats(before_frames)
after_luma, after_contrast, after_saturation = color_stats(after_frames)
return {
"boundary_frame": boundary,
"flow_magnitude_pre": pre_magnitude,
"flow_magnitude_post": post_magnitude,
"flow_magnitude_ratio_post_over_pre": magnitude_ratio,
"flow_direction_cosine": _safe_cosine(pre_vector, post_vector),
"flow_peak_abs_jerk": float(np.max(np.abs(local_jerk))) if len(local_jerk) else 0.0,
"luma_delta": after_luma - before_luma,
"contrast_delta": after_contrast - before_contrast,
"saturation_delta": after_saturation - before_saturation,
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--normal-root", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--result", type=Path, required=True)
args = parser.parse_args()
runs = []
for summary_path in sorted(args.normal_root.glob("normal_*/summary.json")):
run = _read_summary(summary_path)
sampling = _sampling_seconds(run)
diagnostics = run["diagnostic"]
calls = diagnostics["sample_calls"]
continuation_calls = calls[1:]
video_path = _video_path(run["output_prefix"], args.output_dir)
frames, fps = _read_frames(video_path)
layouts = [call["packed_layouts"][0] for call in continuation_calls]
runs.append(
{
"label": run["label"],
"seed": run["seed"],
"transport": run["transport"],
"api_elapsed_seconds": run["api_elapsed_seconds"],
"sampling_seconds": sampling,
"continuation_sampling_seconds": sum(sampling[1:]),
"sample_host_continuation_seconds": sum(
float(call["sample_host_elapsed_seconds"]) for call in continuation_calls
),
"packed_seq_len": [layout["seq_len"] for layout in layouts],
"packed_rows_by_kind": [layout["rows_by_kind"] for layout in layouts],
"final_prefix_bit_exact": [
pair["bit_exact"] for pair in diagnostics["final_prefix_pairs"]
],
"final_prefix_max_abs_diff": [
pair["max_abs_diff"] for pair in diagnostics["final_prefix_pairs"]
],
"peak_process_rss_bytes": run["resources"]["peak_process_rss_bytes"],
"peak_process_uss_bytes": run["resources"]["peak_process_uss_bytes"],
"peak_device_used_bytes": run["resources"]["peak_device_used_bytes"],
"video": str(video_path),
"video_width": int(frames[0].shape[1]),
"video_height": int(frames[0].shape[0]),
"video_frames": len(frames),
"video_fps": fps,
"video_duration_seconds": len(frames) / fps,
"boundaries": [_boundary_metrics(frames, boundary) for boundary in (120, 240)],
}
)
pairs = []
for seed in sorted({run["seed"] for run in runs}):
by_transport = {run["transport"]: run for run in runs if run["seed"] == seed}
reference = by_transport["reference_context_v1"]
masked = by_transport["masked_video_prefix_v1"]
pairs.append(
{
"seed": seed,
"reference_api_seconds": reference["api_elapsed_seconds"],
"masked_api_seconds": masked["api_elapsed_seconds"],
"api_masked_faster_seconds": reference["api_elapsed_seconds"]
- masked["api_elapsed_seconds"],
"api_masked_faster_percent": 100.0
* (reference["api_elapsed_seconds"] - masked["api_elapsed_seconds"])
/ reference["api_elapsed_seconds"],
"reference_continuation_sampling_seconds": reference[
"continuation_sampling_seconds"
],
"masked_continuation_sampling_seconds": masked[
"continuation_sampling_seconds"
],
"continuation_masked_faster_percent": 100.0
* (
reference["continuation_sampling_seconds"]
- masked["continuation_sampling_seconds"]
)
/ reference["continuation_sampling_seconds"],
}
)
result = {
"format": "h3-continuum-v36-r1-stress-gate-analysis-v1",
"runs": runs,
"pairs": pairs,
"medians": {
"reference_api_seconds": _median(
[run["api_elapsed_seconds"] for run in runs if run["transport"] == "reference_context_v1"]
),
"masked_api_seconds": _median(
[run["api_elapsed_seconds"] for run in runs if run["transport"] == "masked_video_prefix_v1"]
),
"api_masked_faster_percent": _median(
[pair["api_masked_faster_percent"] for pair in pairs]
),
"reference_continuation_sampling_seconds": _median(
[pair["reference_continuation_sampling_seconds"] for pair in pairs]
),
"masked_continuation_sampling_seconds": _median(
[pair["masked_continuation_sampling_seconds"] for pair in pairs]
),
"continuation_masked_faster_percent": _median(
[pair["continuation_masked_faster_percent"] for pair in pairs]
),
},
}
args.result.parent.mkdir(parents=True, exist_ok=True)
args.result.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
print(json.dumps({"pairs": pairs, "medians": result["medians"]}, indent=2))
if __name__ == "__main__":
main()