Files
ComfyUI-H3-Continuum-Tiled/tools/issue13_r2_analyze.py
T

463 lines
16 KiB
Python

"""Analyze Issue #13 R2 decoded video and continuation-latent trends."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
import cv2
import numpy as np
from PIL import Image, ImageDraw, ImageFont
METRIC_KEYS = (
"luma_mean",
"luma_std",
"saturation_mean",
"saturation_p95",
"contrast_p95_p05",
"sharpness_laplacian_variance",
"gradient_mean",
"highlight_clip_fraction",
"skin_highlight_proxy_fraction",
"shadow_clip_fraction",
)
LATENT_KEYS = (
"mean",
"std",
"rms",
"minimum",
"maximum",
"temporal_delta_rms",
)
def _video_path(record: dict, video_root: Path) -> Path:
item = record.get("output_video") or {}
return video_root / str(item.get("subfolder", "")) / str(item.get("filename", ""))
def _read_frames(path: Path) -> tuple[list[np.ndarray], float]:
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
raise RuntimeError(f"cannot open video: {path}")
fps = float(capture.get(cv2.CAP_PROP_FPS) or 24.0)
frames = []
while True:
ok, frame = capture.read()
if not ok:
break
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
capture.release()
if not frames:
raise RuntimeError(f"video contains no frames: {path}")
return frames, fps
def _frame_metrics(frame: np.ndarray) -> dict[str, float]:
rgb = frame.astype(np.float32) / 255.0
y = 0.2126 * rgb[..., 0] + 0.7152 * rgb[..., 1] + 0.0722 * rgb[..., 2]
hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV).astype(np.float32)
saturation = hsv[..., 1] / 255.0
gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
laplacian = cv2.Laplacian(gray, cv2.CV_64F)
sobel_x = cv2.Sobel(gray, cv2.CV_32F, 1, 0, ksize=3)
sobel_y = cv2.Sobel(gray, cv2.CV_32F, 0, 1, ksize=3)
gradient = cv2.magnitude(sobel_x, sobel_y) / 255.0
ycrcb = cv2.cvtColor(frame, cv2.COLOR_RGB2YCrCb)
skin_mask = (
(ycrcb[..., 1] >= 133)
& (ycrcb[..., 1] <= 173)
& (ycrcb[..., 2] >= 77)
& (ycrcb[..., 2] <= 127)
)
skin_count = int(np.count_nonzero(skin_mask))
skin_highlight = (
float(np.count_nonzero(skin_mask & (y >= (220.0 / 255.0))))
/ float(skin_count)
if skin_count
else 0.0
)
return {
"luma_mean": float(np.mean(y)),
"luma_std": float(np.std(y)),
"red_mean": float(np.mean(rgb[..., 0])),
"green_mean": float(np.mean(rgb[..., 1])),
"blue_mean": float(np.mean(rgb[..., 2])),
"saturation_mean": float(np.mean(saturation)),
"saturation_p95": float(np.percentile(saturation, 95)),
"contrast_p95_p05": float(np.percentile(y, 95) - np.percentile(y, 5)),
"sharpness_laplacian_variance": float(np.var(laplacian)),
"gradient_mean": float(np.mean(gradient)),
"highlight_clip_fraction": float(np.mean(y >= (250.0 / 255.0))),
"skin_highlight_proxy_fraction": skin_highlight,
"shadow_clip_fraction": float(np.mean(y <= (5.0 / 255.0))),
}
def _chunk_metrics(
frames: list[np.ndarray], *, chunks: int, radius: int
) -> tuple[list[dict], list[np.ndarray]]:
frame_count = len(frames)
rows = []
representatives = []
for index in range(chunks):
center = int(round((index + 0.5) * frame_count / chunks))
center = min(max(center, 0), frame_count - 1)
start = max(0, center - radius)
end = min(frame_count, center + radius + 1)
values = [_frame_metrics(frame) for frame in frames[start:end]]
merged = {
key: float(np.mean([item[key] for item in values])) for key in values[0]
}
merged.update(
{
"chunk": index + 1,
"center_frame": center,
"sample_start": start,
"sample_end_exclusive": end,
"sample_count": end - start,
}
)
rows.append(merged)
representatives.append(frames[center])
return rows, representatives
def _normalized_histogram(values: np.ndarray, *, bins: int = 16) -> list[float]:
counts, _ = np.histogram(values, bins=bins, range=(0.0, 1.0))
total = int(counts.sum())
if total <= 0:
return [0.0] * int(bins)
return [float(value) / float(total) for value in counts]
def _continuation_input_rgb_proxy(
frames: list[np.ndarray], *, chunks: int, context_frames: int
) -> dict:
"""Measure decoded source frames immediately before each chunk boundary.
The R2 diagnostic separately proves that each source latent tail is copied
bit-exactly into the following target prefix. This read-only decoded window
is therefore a visual proxy; it does not add a VAE decode to Sampling.
"""
boundaries = []
frame_count = len(frames)
for target_index in range(1, int(chunks)):
boundary = int(round(target_index * frame_count / int(chunks)))
boundary = min(max(boundary, 1), frame_count)
start = max(0, boundary - int(context_frames))
window = frames[start:boundary]
values = [_frame_metrics(frame) for frame in window]
merged = {
key: float(np.mean([item[key] for item in values])) for key in values[0]
}
rgb = np.concatenate(
[frame.astype(np.float32).reshape(-1, 3) / 255.0 for frame in window],
axis=0,
)
luma = 0.2126 * rgb[:, 0] + 0.7152 * rgb[:, 1] + 0.0722 * rgb[:, 2]
saturation = np.concatenate(
[
cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)[..., 1]
.reshape(-1)
.astype(np.float32)
/ 255.0
for frame in window
]
)
merged.update(
{
"source_group": target_index,
"target_group": target_index + 1,
"boundary_frame": boundary,
"window_start": start,
"window_end_exclusive": boundary,
"window_frames": len(window),
"histogram_bins": 16,
"red_histogram": _normalized_histogram(rgb[:, 0]),
"green_histogram": _normalized_histogram(rgb[:, 1]),
"blue_histogram": _normalized_histogram(rgb[:, 2]),
"luma_histogram": _normalized_histogram(luma),
"saturation_histogram": _normalized_histogram(saturation),
}
)
boundaries.append(merged)
return {
"contract": "decoded source-tail proxy; no extra VAE decode during Sampling",
"context_frames": int(context_frames),
"boundaries": boundaries,
"trend": _trend(boundaries, METRIC_KEYS) if boundaries else {},
}
def _trend(rows: list[dict], keys: tuple[str, ...]) -> dict:
x = np.arange(1, len(rows) + 1, dtype=np.float64)
result = {}
for key in keys:
values = np.asarray([float(row[key]) for row in rows], dtype=np.float64)
result[key] = {
"first": float(values[0]),
"last": float(values[-1]),
"last_minus_first": float(values[-1] - values[0]),
"linear_slope_per_chunk": float(np.polyfit(x, values, 1)[0])
if len(values) > 1
else 0.0,
"increase_steps": int(np.count_nonzero(np.diff(values) > 0.0)),
"decrease_steps": int(np.count_nonzero(np.diff(values) < 0.0)),
}
return result
def _flow_mean(first: np.ndarray, second: np.ndarray) -> float:
first_gray = cv2.cvtColor(first, cv2.COLOR_RGB2GRAY)
second_gray = cv2.cvtColor(second, cv2.COLOR_RGB2GRAY)
flow = cv2.calcOpticalFlowFarneback(
first_gray,
second_gray,
None,
0.5,
3,
15,
3,
5,
1.2,
0,
)
return float(np.linalg.norm(flow, axis=-1).mean())
def _boundary_metrics(frames: list[np.ndarray], *, chunks: int) -> list[dict]:
rows = []
frame_count = len(frames)
for target_index in range(1, int(chunks)):
boundary = int(round(target_index * frame_count / int(chunks)))
boundary = min(max(boundary, 2), frame_count - 2)
before2, before, after, after2 = frames[
boundary - 2
], frames[boundary - 1], frames[boundary], frames[boundary + 1]
before_float = before.astype(np.float32) / 255.0
after_float = after.astype(np.float32) / 255.0
before_metrics = _frame_metrics(before)
after_metrics = _frame_metrics(after)
flow_before = _flow_mean(before2, before)
flow_boundary = _flow_mean(before, after)
flow_after = _flow_mean(after, after2)
local_flow = 0.5 * (flow_before + flow_after)
rows.append(
{
"source_group": target_index,
"target_group": target_index + 1,
"boundary_frame": boundary,
"frame_mae": float(np.abs(after_float - before_float).mean()),
"luma_jump_abs": abs(
after_metrics["luma_mean"] - before_metrics["luma_mean"]
),
"saturation_jump_abs": abs(
after_metrics["saturation_mean"]
- before_metrics["saturation_mean"]
),
"gradient_jump_abs": abs(
after_metrics["gradient_mean"]
- before_metrics["gradient_mean"]
),
"flow_before": flow_before,
"flow_boundary": flow_boundary,
"flow_after": flow_after,
"trajectory_flow_discontinuity": abs(flow_boundary - local_flow)
/ max(local_flow, 1.0e-6),
}
)
return rows
def _latent_trend(diagnostic: dict) -> dict:
calls = list(diagnostic.get("sample_calls") or [])
def rows_for(name: str) -> list[dict]:
rows = []
for call in calls:
value = call.get(name)
if not isinstance(value, dict):
continue
row = {"sample_number": int(call.get("sample_number", len(rows) + 1))}
for key in LATENT_KEYS:
if value.get(key) is not None:
row[key] = float(value[key])
rows.append(row)
return rows
streams = {}
for name in ("input_video", "output_video", "output_tail", "source_tail"):
rows = rows_for(name)
available = tuple(key for key in LATENT_KEYS if rows and key in rows[0])
streams[name] = {
"rows": rows,
"trend": _trend(rows, available) if rows and available else {},
}
prefix_pairs = list(diagnostic.get("final_prefix_pairs") or [])
return {
"streams": streams,
"prefix_pair_count": len(prefix_pairs),
"all_video_prefix_pairs_bit_exact": bool(prefix_pairs)
and all(bool(pair.get("bit_exact")) for pair in prefix_pairs),
"all_audio_prefix_pairs_bit_exact": bool(prefix_pairs)
and all(bool(pair.get("audio_bit_exact")) for pair in prefix_pairs),
"maximum_video_prefix_abs_diff": max(
(float(pair.get("max_abs_diff", 0.0)) for pair in prefix_pairs),
default=0.0,
),
"maximum_audio_prefix_abs_diff": max(
(float(pair.get("audio_max_abs_diff", 0.0)) for pair in prefix_pairs),
default=0.0,
),
}
def _contact_sheet(
rows: list[tuple[str, list[np.ndarray]]],
output: Path,
*,
column_labels: list[str] | None = None,
) -> None:
thumb_w, thumb_h = 224, 224
label_w, top_h = 240, 42
columns = max((len(frames) for _, frames in rows), default=0)
width = label_w + thumb_w * columns
height = top_h + thumb_h * len(rows)
sheet = Image.new("RGB", (width, height), (9, 15, 25))
draw = ImageDraw.Draw(sheet)
font = ImageFont.load_default()
for column in range(columns):
draw.text(
(label_w + column * thumb_w + 8, 14),
column_labels[column]
if column_labels is not None and column < len(column_labels)
else f"Chunk {column + 1}",
fill=(210, 225, 245),
font=font,
)
for row_index, (label, frames) in enumerate(rows):
y = top_h + row_index * thumb_h
draw.text((10, y + 16), label, fill=(115, 210, 255), font=font)
for column, frame in enumerate(frames):
image = Image.fromarray(frame).resize(
(thumb_w, thumb_h), Image.Resampling.LANCZOS
)
sheet.paste(image, (label_w + column * thumb_w, y))
output.parent.mkdir(parents=True, exist_ok=True)
sheet.save(output)
def _milestone_frames(
frames: list[np.ndarray], *, chunks: int
) -> tuple[list[str], list[np.ndarray]]:
frame_count = len(frames)
chunk_frames = frame_count / float(chunks)
indices = [
min(frame_count - 1, 12),
min(frame_count - 1, max(0, int(round(3 * chunk_frames)) - 1)),
min(frame_count - 1, int(round(3 * chunk_frames)) + 12),
min(frame_count - 1, int(round(4.5 * chunk_frames))),
frame_count - 1,
]
return (
["Near 0s", "G3 end", "G4 post", "G5", "G6 end"],
[frames[index] for index in indices],
)
def run(args: argparse.Namespace) -> dict:
summary = json.loads((args.output_root / "summary.json").read_text(encoding="utf-8"))
all_rows = []
sheets = []
milestone_sheets = []
milestone_labels = None
videos = {}
for record in summary["cases"]:
case = str(record["case"])
chunks = int(record["chunks"])
path = _video_path(record, args.video_root)
frames, fps = _read_frames(path)
metrics, representatives = _chunk_metrics(
frames, chunks=chunks, radius=args.radius
)
for row in metrics:
row["case"] = case
row["video"] = str(path)
row["fps"] = fps
row["frame_count"] = len(frames)
all_rows.append(row)
sheets.append((case, representatives))
labels, milestones = _milestone_frames(frames, chunks=chunks)
milestone_labels = labels
milestone_sheets.append((case, milestones))
videos[case] = {
"path": str(path),
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"fps": fps,
"frame_count": len(frames),
"duration_seconds": len(frames) / fps,
"chunks": chunks,
"chunk_seconds": float(record["chunk_seconds"]),
"metrics": metrics,
"decoded_trend": _trend(metrics, METRIC_KEYS),
"continuation_input_rgb_proxy": _continuation_input_rgb_proxy(
frames,
chunks=chunks,
context_frames=int(
(record.get("diagnostic") or {}).get("context_frames", 22)
),
),
"boundary_metrics": _boundary_metrics(frames, chunks=chunks),
"latent_trend": _latent_trend(record.get("diagnostic") or {}),
}
if not all_rows:
raise RuntimeError("Issue #13 R2 summary contains no completed cases")
csv_path = args.output_root / "chunk_metrics.csv"
with csv_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=list(all_rows[0]))
writer.writeheader()
writer.writerows(all_rows)
contact_path = args.output_root / "contact_sheet.png"
_contact_sheet(sheets, contact_path)
milestone_contact_path = args.output_root / "milestone_contact_sheet.png"
_contact_sheet(
milestone_sheets,
milestone_contact_path,
column_labels=milestone_labels,
)
result = {
"format": "h3-continuum-issue13-r2-analysis-v2",
"sample_radius_frames": args.radius,
"videos": videos,
"csv": str(csv_path),
"contact_sheet": str(contact_path),
"milestone_contact_sheet": str(milestone_contact_path),
}
(args.output_root / "analysis.json").write_text(
json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8"
)
print(json.dumps(result, ensure_ascii=False))
return result
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument(
"--video-root", type=Path, default=Path(r"D:\output\video\comfy_video")
)
parser.add_argument("--radius", type=int, default=12)
return parser
if __name__ == "__main__":
run(_parser().parse_args())