463 lines
16 KiB
Python
463 lines
16 KiB
Python
"""Analyze Issue #13 R2 decoded video and continuation-latent trends."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import cv2
|
|
import numpy as np
|
|
from PIL import Image, ImageDraw, ImageFont
|
|
|
|
|
|
METRIC_KEYS = (
|
|
"luma_mean",
|
|
"luma_std",
|
|
"saturation_mean",
|
|
"saturation_p95",
|
|
"contrast_p95_p05",
|
|
"sharpness_laplacian_variance",
|
|
"gradient_mean",
|
|
"highlight_clip_fraction",
|
|
"skin_highlight_proxy_fraction",
|
|
"shadow_clip_fraction",
|
|
)
|
|
|
|
LATENT_KEYS = (
|
|
"mean",
|
|
"std",
|
|
"rms",
|
|
"minimum",
|
|
"maximum",
|
|
"temporal_delta_rms",
|
|
)
|
|
|
|
|
|
def _video_path(record: dict, video_root: Path) -> Path:
|
|
item = record.get("output_video") or {}
|
|
return video_root / str(item.get("subfolder", "")) / str(item.get("filename", ""))
|
|
|
|
|
|
def _read_frames(path: Path) -> tuple[list[np.ndarray], float]:
|
|
capture = cv2.VideoCapture(str(path))
|
|
if not capture.isOpened():
|
|
raise RuntimeError(f"cannot open video: {path}")
|
|
fps = float(capture.get(cv2.CAP_PROP_FPS) or 24.0)
|
|
frames = []
|
|
while True:
|
|
ok, frame = capture.read()
|
|
if not ok:
|
|
break
|
|
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
|
capture.release()
|
|
if not frames:
|
|
raise RuntimeError(f"video contains no frames: {path}")
|
|
return frames, fps
|
|
|
|
|
|
def _frame_metrics(frame: np.ndarray) -> dict[str, float]:
|
|
rgb = frame.astype(np.float32) / 255.0
|
|
y = 0.2126 * rgb[..., 0] + 0.7152 * rgb[..., 1] + 0.0722 * rgb[..., 2]
|
|
hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV).astype(np.float32)
|
|
saturation = hsv[..., 1] / 255.0
|
|
gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
|
|
laplacian = cv2.Laplacian(gray, cv2.CV_64F)
|
|
sobel_x = cv2.Sobel(gray, cv2.CV_32F, 1, 0, ksize=3)
|
|
sobel_y = cv2.Sobel(gray, cv2.CV_32F, 0, 1, ksize=3)
|
|
gradient = cv2.magnitude(sobel_x, sobel_y) / 255.0
|
|
ycrcb = cv2.cvtColor(frame, cv2.COLOR_RGB2YCrCb)
|
|
skin_mask = (
|
|
(ycrcb[..., 1] >= 133)
|
|
& (ycrcb[..., 1] <= 173)
|
|
& (ycrcb[..., 2] >= 77)
|
|
& (ycrcb[..., 2] <= 127)
|
|
)
|
|
skin_count = int(np.count_nonzero(skin_mask))
|
|
skin_highlight = (
|
|
float(np.count_nonzero(skin_mask & (y >= (220.0 / 255.0))))
|
|
/ float(skin_count)
|
|
if skin_count
|
|
else 0.0
|
|
)
|
|
return {
|
|
"luma_mean": float(np.mean(y)),
|
|
"luma_std": float(np.std(y)),
|
|
"red_mean": float(np.mean(rgb[..., 0])),
|
|
"green_mean": float(np.mean(rgb[..., 1])),
|
|
"blue_mean": float(np.mean(rgb[..., 2])),
|
|
"saturation_mean": float(np.mean(saturation)),
|
|
"saturation_p95": float(np.percentile(saturation, 95)),
|
|
"contrast_p95_p05": float(np.percentile(y, 95) - np.percentile(y, 5)),
|
|
"sharpness_laplacian_variance": float(np.var(laplacian)),
|
|
"gradient_mean": float(np.mean(gradient)),
|
|
"highlight_clip_fraction": float(np.mean(y >= (250.0 / 255.0))),
|
|
"skin_highlight_proxy_fraction": skin_highlight,
|
|
"shadow_clip_fraction": float(np.mean(y <= (5.0 / 255.0))),
|
|
}
|
|
|
|
|
|
def _chunk_metrics(
|
|
frames: list[np.ndarray], *, chunks: int, radius: int
|
|
) -> tuple[list[dict], list[np.ndarray]]:
|
|
frame_count = len(frames)
|
|
rows = []
|
|
representatives = []
|
|
for index in range(chunks):
|
|
center = int(round((index + 0.5) * frame_count / chunks))
|
|
center = min(max(center, 0), frame_count - 1)
|
|
start = max(0, center - radius)
|
|
end = min(frame_count, center + radius + 1)
|
|
values = [_frame_metrics(frame) for frame in frames[start:end]]
|
|
merged = {
|
|
key: float(np.mean([item[key] for item in values])) for key in values[0]
|
|
}
|
|
merged.update(
|
|
{
|
|
"chunk": index + 1,
|
|
"center_frame": center,
|
|
"sample_start": start,
|
|
"sample_end_exclusive": end,
|
|
"sample_count": end - start,
|
|
}
|
|
)
|
|
rows.append(merged)
|
|
representatives.append(frames[center])
|
|
return rows, representatives
|
|
|
|
|
|
def _normalized_histogram(values: np.ndarray, *, bins: int = 16) -> list[float]:
|
|
counts, _ = np.histogram(values, bins=bins, range=(0.0, 1.0))
|
|
total = int(counts.sum())
|
|
if total <= 0:
|
|
return [0.0] * int(bins)
|
|
return [float(value) / float(total) for value in counts]
|
|
|
|
|
|
def _continuation_input_rgb_proxy(
|
|
frames: list[np.ndarray], *, chunks: int, context_frames: int
|
|
) -> dict:
|
|
"""Measure decoded source frames immediately before each chunk boundary.
|
|
|
|
The R2 diagnostic separately proves that each source latent tail is copied
|
|
bit-exactly into the following target prefix. This read-only decoded window
|
|
is therefore a visual proxy; it does not add a VAE decode to Sampling.
|
|
"""
|
|
|
|
boundaries = []
|
|
frame_count = len(frames)
|
|
for target_index in range(1, int(chunks)):
|
|
boundary = int(round(target_index * frame_count / int(chunks)))
|
|
boundary = min(max(boundary, 1), frame_count)
|
|
start = max(0, boundary - int(context_frames))
|
|
window = frames[start:boundary]
|
|
values = [_frame_metrics(frame) for frame in window]
|
|
merged = {
|
|
key: float(np.mean([item[key] for item in values])) for key in values[0]
|
|
}
|
|
rgb = np.concatenate(
|
|
[frame.astype(np.float32).reshape(-1, 3) / 255.0 for frame in window],
|
|
axis=0,
|
|
)
|
|
luma = 0.2126 * rgb[:, 0] + 0.7152 * rgb[:, 1] + 0.0722 * rgb[:, 2]
|
|
saturation = np.concatenate(
|
|
[
|
|
cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)[..., 1]
|
|
.reshape(-1)
|
|
.astype(np.float32)
|
|
/ 255.0
|
|
for frame in window
|
|
]
|
|
)
|
|
merged.update(
|
|
{
|
|
"source_group": target_index,
|
|
"target_group": target_index + 1,
|
|
"boundary_frame": boundary,
|
|
"window_start": start,
|
|
"window_end_exclusive": boundary,
|
|
"window_frames": len(window),
|
|
"histogram_bins": 16,
|
|
"red_histogram": _normalized_histogram(rgb[:, 0]),
|
|
"green_histogram": _normalized_histogram(rgb[:, 1]),
|
|
"blue_histogram": _normalized_histogram(rgb[:, 2]),
|
|
"luma_histogram": _normalized_histogram(luma),
|
|
"saturation_histogram": _normalized_histogram(saturation),
|
|
}
|
|
)
|
|
boundaries.append(merged)
|
|
return {
|
|
"contract": "decoded source-tail proxy; no extra VAE decode during Sampling",
|
|
"context_frames": int(context_frames),
|
|
"boundaries": boundaries,
|
|
"trend": _trend(boundaries, METRIC_KEYS) if boundaries else {},
|
|
}
|
|
|
|
|
|
def _trend(rows: list[dict], keys: tuple[str, ...]) -> dict:
|
|
x = np.arange(1, len(rows) + 1, dtype=np.float64)
|
|
result = {}
|
|
for key in keys:
|
|
values = np.asarray([float(row[key]) for row in rows], dtype=np.float64)
|
|
result[key] = {
|
|
"first": float(values[0]),
|
|
"last": float(values[-1]),
|
|
"last_minus_first": float(values[-1] - values[0]),
|
|
"linear_slope_per_chunk": float(np.polyfit(x, values, 1)[0])
|
|
if len(values) > 1
|
|
else 0.0,
|
|
"increase_steps": int(np.count_nonzero(np.diff(values) > 0.0)),
|
|
"decrease_steps": int(np.count_nonzero(np.diff(values) < 0.0)),
|
|
}
|
|
return result
|
|
|
|
|
|
def _flow_mean(first: np.ndarray, second: np.ndarray) -> float:
|
|
first_gray = cv2.cvtColor(first, cv2.COLOR_RGB2GRAY)
|
|
second_gray = cv2.cvtColor(second, cv2.COLOR_RGB2GRAY)
|
|
flow = cv2.calcOpticalFlowFarneback(
|
|
first_gray,
|
|
second_gray,
|
|
None,
|
|
0.5,
|
|
3,
|
|
15,
|
|
3,
|
|
5,
|
|
1.2,
|
|
0,
|
|
)
|
|
return float(np.linalg.norm(flow, axis=-1).mean())
|
|
|
|
|
|
def _boundary_metrics(frames: list[np.ndarray], *, chunks: int) -> list[dict]:
|
|
rows = []
|
|
frame_count = len(frames)
|
|
for target_index in range(1, int(chunks)):
|
|
boundary = int(round(target_index * frame_count / int(chunks)))
|
|
boundary = min(max(boundary, 2), frame_count - 2)
|
|
before2, before, after, after2 = frames[
|
|
boundary - 2
|
|
], frames[boundary - 1], frames[boundary], frames[boundary + 1]
|
|
before_float = before.astype(np.float32) / 255.0
|
|
after_float = after.astype(np.float32) / 255.0
|
|
before_metrics = _frame_metrics(before)
|
|
after_metrics = _frame_metrics(after)
|
|
flow_before = _flow_mean(before2, before)
|
|
flow_boundary = _flow_mean(before, after)
|
|
flow_after = _flow_mean(after, after2)
|
|
local_flow = 0.5 * (flow_before + flow_after)
|
|
rows.append(
|
|
{
|
|
"source_group": target_index,
|
|
"target_group": target_index + 1,
|
|
"boundary_frame": boundary,
|
|
"frame_mae": float(np.abs(after_float - before_float).mean()),
|
|
"luma_jump_abs": abs(
|
|
after_metrics["luma_mean"] - before_metrics["luma_mean"]
|
|
),
|
|
"saturation_jump_abs": abs(
|
|
after_metrics["saturation_mean"]
|
|
- before_metrics["saturation_mean"]
|
|
),
|
|
"gradient_jump_abs": abs(
|
|
after_metrics["gradient_mean"]
|
|
- before_metrics["gradient_mean"]
|
|
),
|
|
"flow_before": flow_before,
|
|
"flow_boundary": flow_boundary,
|
|
"flow_after": flow_after,
|
|
"trajectory_flow_discontinuity": abs(flow_boundary - local_flow)
|
|
/ max(local_flow, 1.0e-6),
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def _latent_trend(diagnostic: dict) -> dict:
|
|
calls = list(diagnostic.get("sample_calls") or [])
|
|
|
|
def rows_for(name: str) -> list[dict]:
|
|
rows = []
|
|
for call in calls:
|
|
value = call.get(name)
|
|
if not isinstance(value, dict):
|
|
continue
|
|
row = {"sample_number": int(call.get("sample_number", len(rows) + 1))}
|
|
for key in LATENT_KEYS:
|
|
if value.get(key) is not None:
|
|
row[key] = float(value[key])
|
|
rows.append(row)
|
|
return rows
|
|
|
|
streams = {}
|
|
for name in ("input_video", "output_video", "output_tail", "source_tail"):
|
|
rows = rows_for(name)
|
|
available = tuple(key for key in LATENT_KEYS if rows and key in rows[0])
|
|
streams[name] = {
|
|
"rows": rows,
|
|
"trend": _trend(rows, available) if rows and available else {},
|
|
}
|
|
prefix_pairs = list(diagnostic.get("final_prefix_pairs") or [])
|
|
return {
|
|
"streams": streams,
|
|
"prefix_pair_count": len(prefix_pairs),
|
|
"all_video_prefix_pairs_bit_exact": bool(prefix_pairs)
|
|
and all(bool(pair.get("bit_exact")) for pair in prefix_pairs),
|
|
"all_audio_prefix_pairs_bit_exact": bool(prefix_pairs)
|
|
and all(bool(pair.get("audio_bit_exact")) for pair in prefix_pairs),
|
|
"maximum_video_prefix_abs_diff": max(
|
|
(float(pair.get("max_abs_diff", 0.0)) for pair in prefix_pairs),
|
|
default=0.0,
|
|
),
|
|
"maximum_audio_prefix_abs_diff": max(
|
|
(float(pair.get("audio_max_abs_diff", 0.0)) for pair in prefix_pairs),
|
|
default=0.0,
|
|
),
|
|
}
|
|
|
|
|
|
def _contact_sheet(
|
|
rows: list[tuple[str, list[np.ndarray]]],
|
|
output: Path,
|
|
*,
|
|
column_labels: list[str] | None = None,
|
|
) -> None:
|
|
thumb_w, thumb_h = 224, 224
|
|
label_w, top_h = 240, 42
|
|
columns = max((len(frames) for _, frames in rows), default=0)
|
|
width = label_w + thumb_w * columns
|
|
height = top_h + thumb_h * len(rows)
|
|
sheet = Image.new("RGB", (width, height), (9, 15, 25))
|
|
draw = ImageDraw.Draw(sheet)
|
|
font = ImageFont.load_default()
|
|
for column in range(columns):
|
|
draw.text(
|
|
(label_w + column * thumb_w + 8, 14),
|
|
column_labels[column]
|
|
if column_labels is not None and column < len(column_labels)
|
|
else f"Chunk {column + 1}",
|
|
fill=(210, 225, 245),
|
|
font=font,
|
|
)
|
|
for row_index, (label, frames) in enumerate(rows):
|
|
y = top_h + row_index * thumb_h
|
|
draw.text((10, y + 16), label, fill=(115, 210, 255), font=font)
|
|
for column, frame in enumerate(frames):
|
|
image = Image.fromarray(frame).resize(
|
|
(thumb_w, thumb_h), Image.Resampling.LANCZOS
|
|
)
|
|
sheet.paste(image, (label_w + column * thumb_w, y))
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
sheet.save(output)
|
|
|
|
|
|
def _milestone_frames(
|
|
frames: list[np.ndarray], *, chunks: int
|
|
) -> tuple[list[str], list[np.ndarray]]:
|
|
frame_count = len(frames)
|
|
chunk_frames = frame_count / float(chunks)
|
|
indices = [
|
|
min(frame_count - 1, 12),
|
|
min(frame_count - 1, max(0, int(round(3 * chunk_frames)) - 1)),
|
|
min(frame_count - 1, int(round(3 * chunk_frames)) + 12),
|
|
min(frame_count - 1, int(round(4.5 * chunk_frames))),
|
|
frame_count - 1,
|
|
]
|
|
return (
|
|
["Near 0s", "G3 end", "G4 post", "G5", "G6 end"],
|
|
[frames[index] for index in indices],
|
|
)
|
|
|
|
|
|
def run(args: argparse.Namespace) -> dict:
|
|
summary = json.loads((args.output_root / "summary.json").read_text(encoding="utf-8"))
|
|
all_rows = []
|
|
sheets = []
|
|
milestone_sheets = []
|
|
milestone_labels = None
|
|
videos = {}
|
|
for record in summary["cases"]:
|
|
case = str(record["case"])
|
|
chunks = int(record["chunks"])
|
|
path = _video_path(record, args.video_root)
|
|
frames, fps = _read_frames(path)
|
|
metrics, representatives = _chunk_metrics(
|
|
frames, chunks=chunks, radius=args.radius
|
|
)
|
|
for row in metrics:
|
|
row["case"] = case
|
|
row["video"] = str(path)
|
|
row["fps"] = fps
|
|
row["frame_count"] = len(frames)
|
|
all_rows.append(row)
|
|
sheets.append((case, representatives))
|
|
labels, milestones = _milestone_frames(frames, chunks=chunks)
|
|
milestone_labels = labels
|
|
milestone_sheets.append((case, milestones))
|
|
videos[case] = {
|
|
"path": str(path),
|
|
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
"fps": fps,
|
|
"frame_count": len(frames),
|
|
"duration_seconds": len(frames) / fps,
|
|
"chunks": chunks,
|
|
"chunk_seconds": float(record["chunk_seconds"]),
|
|
"metrics": metrics,
|
|
"decoded_trend": _trend(metrics, METRIC_KEYS),
|
|
"continuation_input_rgb_proxy": _continuation_input_rgb_proxy(
|
|
frames,
|
|
chunks=chunks,
|
|
context_frames=int(
|
|
(record.get("diagnostic") or {}).get("context_frames", 22)
|
|
),
|
|
),
|
|
"boundary_metrics": _boundary_metrics(frames, chunks=chunks),
|
|
"latent_trend": _latent_trend(record.get("diagnostic") or {}),
|
|
}
|
|
|
|
if not all_rows:
|
|
raise RuntimeError("Issue #13 R2 summary contains no completed cases")
|
|
csv_path = args.output_root / "chunk_metrics.csv"
|
|
with csv_path.open("w", encoding="utf-8", newline="") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=list(all_rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(all_rows)
|
|
contact_path = args.output_root / "contact_sheet.png"
|
|
_contact_sheet(sheets, contact_path)
|
|
milestone_contact_path = args.output_root / "milestone_contact_sheet.png"
|
|
_contact_sheet(
|
|
milestone_sheets,
|
|
milestone_contact_path,
|
|
column_labels=milestone_labels,
|
|
)
|
|
result = {
|
|
"format": "h3-continuum-issue13-r2-analysis-v2",
|
|
"sample_radius_frames": args.radius,
|
|
"videos": videos,
|
|
"csv": str(csv_path),
|
|
"contact_sheet": str(contact_path),
|
|
"milestone_contact_sheet": str(milestone_contact_path),
|
|
}
|
|
(args.output_root / "analysis.json").write_text(
|
|
json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
return result
|
|
|
|
|
|
def _parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output-root", type=Path, required=True)
|
|
parser.add_argument(
|
|
"--video-root", type=Path, default=Path(r"D:\output\video\comfy_video")
|
|
)
|
|
parser.add_argument("--radius", type=int, default=12)
|
|
return parser
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(_parser().parse_args())
|