Squashed 'vendor/dumas_h3_continuum/' content from commit d07ed31
git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
This commit is contained in:
@@ -0,0 +1,462 @@
|
||||
"""Analyze Issue #13 R2 decoded video and continuation-latent trends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
|
||||
METRIC_KEYS = (
|
||||
"luma_mean",
|
||||
"luma_std",
|
||||
"saturation_mean",
|
||||
"saturation_p95",
|
||||
"contrast_p95_p05",
|
||||
"sharpness_laplacian_variance",
|
||||
"gradient_mean",
|
||||
"highlight_clip_fraction",
|
||||
"skin_highlight_proxy_fraction",
|
||||
"shadow_clip_fraction",
|
||||
)
|
||||
|
||||
LATENT_KEYS = (
|
||||
"mean",
|
||||
"std",
|
||||
"rms",
|
||||
"minimum",
|
||||
"maximum",
|
||||
"temporal_delta_rms",
|
||||
)
|
||||
|
||||
|
||||
def _video_path(record: dict, video_root: Path) -> Path:
|
||||
item = record.get("output_video") or {}
|
||||
return video_root / str(item.get("subfolder", "")) / str(item.get("filename", ""))
|
||||
|
||||
|
||||
def _read_frames(path: Path) -> tuple[list[np.ndarray], float]:
|
||||
capture = cv2.VideoCapture(str(path))
|
||||
if not capture.isOpened():
|
||||
raise RuntimeError(f"cannot open video: {path}")
|
||||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 24.0)
|
||||
frames = []
|
||||
while True:
|
||||
ok, frame = capture.read()
|
||||
if not ok:
|
||||
break
|
||||
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
|
||||
capture.release()
|
||||
if not frames:
|
||||
raise RuntimeError(f"video contains no frames: {path}")
|
||||
return frames, fps
|
||||
|
||||
|
||||
def _frame_metrics(frame: np.ndarray) -> dict[str, float]:
|
||||
rgb = frame.astype(np.float32) / 255.0
|
||||
y = 0.2126 * rgb[..., 0] + 0.7152 * rgb[..., 1] + 0.0722 * rgb[..., 2]
|
||||
hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV).astype(np.float32)
|
||||
saturation = hsv[..., 1] / 255.0
|
||||
gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
|
||||
laplacian = cv2.Laplacian(gray, cv2.CV_64F)
|
||||
sobel_x = cv2.Sobel(gray, cv2.CV_32F, 1, 0, ksize=3)
|
||||
sobel_y = cv2.Sobel(gray, cv2.CV_32F, 0, 1, ksize=3)
|
||||
gradient = cv2.magnitude(sobel_x, sobel_y) / 255.0
|
||||
ycrcb = cv2.cvtColor(frame, cv2.COLOR_RGB2YCrCb)
|
||||
skin_mask = (
|
||||
(ycrcb[..., 1] >= 133)
|
||||
& (ycrcb[..., 1] <= 173)
|
||||
& (ycrcb[..., 2] >= 77)
|
||||
& (ycrcb[..., 2] <= 127)
|
||||
)
|
||||
skin_count = int(np.count_nonzero(skin_mask))
|
||||
skin_highlight = (
|
||||
float(np.count_nonzero(skin_mask & (y >= (220.0 / 255.0))))
|
||||
/ float(skin_count)
|
||||
if skin_count
|
||||
else 0.0
|
||||
)
|
||||
return {
|
||||
"luma_mean": float(np.mean(y)),
|
||||
"luma_std": float(np.std(y)),
|
||||
"red_mean": float(np.mean(rgb[..., 0])),
|
||||
"green_mean": float(np.mean(rgb[..., 1])),
|
||||
"blue_mean": float(np.mean(rgb[..., 2])),
|
||||
"saturation_mean": float(np.mean(saturation)),
|
||||
"saturation_p95": float(np.percentile(saturation, 95)),
|
||||
"contrast_p95_p05": float(np.percentile(y, 95) - np.percentile(y, 5)),
|
||||
"sharpness_laplacian_variance": float(np.var(laplacian)),
|
||||
"gradient_mean": float(np.mean(gradient)),
|
||||
"highlight_clip_fraction": float(np.mean(y >= (250.0 / 255.0))),
|
||||
"skin_highlight_proxy_fraction": skin_highlight,
|
||||
"shadow_clip_fraction": float(np.mean(y <= (5.0 / 255.0))),
|
||||
}
|
||||
|
||||
|
||||
def _chunk_metrics(
|
||||
frames: list[np.ndarray], *, chunks: int, radius: int
|
||||
) -> tuple[list[dict], list[np.ndarray]]:
|
||||
frame_count = len(frames)
|
||||
rows = []
|
||||
representatives = []
|
||||
for index in range(chunks):
|
||||
center = int(round((index + 0.5) * frame_count / chunks))
|
||||
center = min(max(center, 0), frame_count - 1)
|
||||
start = max(0, center - radius)
|
||||
end = min(frame_count, center + radius + 1)
|
||||
values = [_frame_metrics(frame) for frame in frames[start:end]]
|
||||
merged = {
|
||||
key: float(np.mean([item[key] for item in values])) for key in values[0]
|
||||
}
|
||||
merged.update(
|
||||
{
|
||||
"chunk": index + 1,
|
||||
"center_frame": center,
|
||||
"sample_start": start,
|
||||
"sample_end_exclusive": end,
|
||||
"sample_count": end - start,
|
||||
}
|
||||
)
|
||||
rows.append(merged)
|
||||
representatives.append(frames[center])
|
||||
return rows, representatives
|
||||
|
||||
|
||||
def _normalized_histogram(values: np.ndarray, *, bins: int = 16) -> list[float]:
|
||||
counts, _ = np.histogram(values, bins=bins, range=(0.0, 1.0))
|
||||
total = int(counts.sum())
|
||||
if total <= 0:
|
||||
return [0.0] * int(bins)
|
||||
return [float(value) / float(total) for value in counts]
|
||||
|
||||
|
||||
def _continuation_input_rgb_proxy(
|
||||
frames: list[np.ndarray], *, chunks: int, context_frames: int
|
||||
) -> dict:
|
||||
"""Measure decoded source frames immediately before each chunk boundary.
|
||||
|
||||
The R2 diagnostic separately proves that each source latent tail is copied
|
||||
bit-exactly into the following target prefix. This read-only decoded window
|
||||
is therefore a visual proxy; it does not add a VAE decode to Sampling.
|
||||
"""
|
||||
|
||||
boundaries = []
|
||||
frame_count = len(frames)
|
||||
for target_index in range(1, int(chunks)):
|
||||
boundary = int(round(target_index * frame_count / int(chunks)))
|
||||
boundary = min(max(boundary, 1), frame_count)
|
||||
start = max(0, boundary - int(context_frames))
|
||||
window = frames[start:boundary]
|
||||
values = [_frame_metrics(frame) for frame in window]
|
||||
merged = {
|
||||
key: float(np.mean([item[key] for item in values])) for key in values[0]
|
||||
}
|
||||
rgb = np.concatenate(
|
||||
[frame.astype(np.float32).reshape(-1, 3) / 255.0 for frame in window],
|
||||
axis=0,
|
||||
)
|
||||
luma = 0.2126 * rgb[:, 0] + 0.7152 * rgb[:, 1] + 0.0722 * rgb[:, 2]
|
||||
saturation = np.concatenate(
|
||||
[
|
||||
cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)[..., 1]
|
||||
.reshape(-1)
|
||||
.astype(np.float32)
|
||||
/ 255.0
|
||||
for frame in window
|
||||
]
|
||||
)
|
||||
merged.update(
|
||||
{
|
||||
"source_group": target_index,
|
||||
"target_group": target_index + 1,
|
||||
"boundary_frame": boundary,
|
||||
"window_start": start,
|
||||
"window_end_exclusive": boundary,
|
||||
"window_frames": len(window),
|
||||
"histogram_bins": 16,
|
||||
"red_histogram": _normalized_histogram(rgb[:, 0]),
|
||||
"green_histogram": _normalized_histogram(rgb[:, 1]),
|
||||
"blue_histogram": _normalized_histogram(rgb[:, 2]),
|
||||
"luma_histogram": _normalized_histogram(luma),
|
||||
"saturation_histogram": _normalized_histogram(saturation),
|
||||
}
|
||||
)
|
||||
boundaries.append(merged)
|
||||
return {
|
||||
"contract": "decoded source-tail proxy; no extra VAE decode during Sampling",
|
||||
"context_frames": int(context_frames),
|
||||
"boundaries": boundaries,
|
||||
"trend": _trend(boundaries, METRIC_KEYS) if boundaries else {},
|
||||
}
|
||||
|
||||
|
||||
def _trend(rows: list[dict], keys: tuple[str, ...]) -> dict:
|
||||
x = np.arange(1, len(rows) + 1, dtype=np.float64)
|
||||
result = {}
|
||||
for key in keys:
|
||||
values = np.asarray([float(row[key]) for row in rows], dtype=np.float64)
|
||||
result[key] = {
|
||||
"first": float(values[0]),
|
||||
"last": float(values[-1]),
|
||||
"last_minus_first": float(values[-1] - values[0]),
|
||||
"linear_slope_per_chunk": float(np.polyfit(x, values, 1)[0])
|
||||
if len(values) > 1
|
||||
else 0.0,
|
||||
"increase_steps": int(np.count_nonzero(np.diff(values) > 0.0)),
|
||||
"decrease_steps": int(np.count_nonzero(np.diff(values) < 0.0)),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def _flow_mean(first: np.ndarray, second: np.ndarray) -> float:
|
||||
first_gray = cv2.cvtColor(first, cv2.COLOR_RGB2GRAY)
|
||||
second_gray = cv2.cvtColor(second, cv2.COLOR_RGB2GRAY)
|
||||
flow = cv2.calcOpticalFlowFarneback(
|
||||
first_gray,
|
||||
second_gray,
|
||||
None,
|
||||
0.5,
|
||||
3,
|
||||
15,
|
||||
3,
|
||||
5,
|
||||
1.2,
|
||||
0,
|
||||
)
|
||||
return float(np.linalg.norm(flow, axis=-1).mean())
|
||||
|
||||
|
||||
def _boundary_metrics(frames: list[np.ndarray], *, chunks: int) -> list[dict]:
|
||||
rows = []
|
||||
frame_count = len(frames)
|
||||
for target_index in range(1, int(chunks)):
|
||||
boundary = int(round(target_index * frame_count / int(chunks)))
|
||||
boundary = min(max(boundary, 2), frame_count - 2)
|
||||
before2, before, after, after2 = frames[
|
||||
boundary - 2
|
||||
], frames[boundary - 1], frames[boundary], frames[boundary + 1]
|
||||
before_float = before.astype(np.float32) / 255.0
|
||||
after_float = after.astype(np.float32) / 255.0
|
||||
before_metrics = _frame_metrics(before)
|
||||
after_metrics = _frame_metrics(after)
|
||||
flow_before = _flow_mean(before2, before)
|
||||
flow_boundary = _flow_mean(before, after)
|
||||
flow_after = _flow_mean(after, after2)
|
||||
local_flow = 0.5 * (flow_before + flow_after)
|
||||
rows.append(
|
||||
{
|
||||
"source_group": target_index,
|
||||
"target_group": target_index + 1,
|
||||
"boundary_frame": boundary,
|
||||
"frame_mae": float(np.abs(after_float - before_float).mean()),
|
||||
"luma_jump_abs": abs(
|
||||
after_metrics["luma_mean"] - before_metrics["luma_mean"]
|
||||
),
|
||||
"saturation_jump_abs": abs(
|
||||
after_metrics["saturation_mean"]
|
||||
- before_metrics["saturation_mean"]
|
||||
),
|
||||
"gradient_jump_abs": abs(
|
||||
after_metrics["gradient_mean"]
|
||||
- before_metrics["gradient_mean"]
|
||||
),
|
||||
"flow_before": flow_before,
|
||||
"flow_boundary": flow_boundary,
|
||||
"flow_after": flow_after,
|
||||
"trajectory_flow_discontinuity": abs(flow_boundary - local_flow)
|
||||
/ max(local_flow, 1.0e-6),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _latent_trend(diagnostic: dict) -> dict:
|
||||
calls = list(diagnostic.get("sample_calls") or [])
|
||||
|
||||
def rows_for(name: str) -> list[dict]:
|
||||
rows = []
|
||||
for call in calls:
|
||||
value = call.get(name)
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
row = {"sample_number": int(call.get("sample_number", len(rows) + 1))}
|
||||
for key in LATENT_KEYS:
|
||||
if value.get(key) is not None:
|
||||
row[key] = float(value[key])
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
streams = {}
|
||||
for name in ("input_video", "output_video", "output_tail", "source_tail"):
|
||||
rows = rows_for(name)
|
||||
available = tuple(key for key in LATENT_KEYS if rows and key in rows[0])
|
||||
streams[name] = {
|
||||
"rows": rows,
|
||||
"trend": _trend(rows, available) if rows and available else {},
|
||||
}
|
||||
prefix_pairs = list(diagnostic.get("final_prefix_pairs") or [])
|
||||
return {
|
||||
"streams": streams,
|
||||
"prefix_pair_count": len(prefix_pairs),
|
||||
"all_video_prefix_pairs_bit_exact": bool(prefix_pairs)
|
||||
and all(bool(pair.get("bit_exact")) for pair in prefix_pairs),
|
||||
"all_audio_prefix_pairs_bit_exact": bool(prefix_pairs)
|
||||
and all(bool(pair.get("audio_bit_exact")) for pair in prefix_pairs),
|
||||
"maximum_video_prefix_abs_diff": max(
|
||||
(float(pair.get("max_abs_diff", 0.0)) for pair in prefix_pairs),
|
||||
default=0.0,
|
||||
),
|
||||
"maximum_audio_prefix_abs_diff": max(
|
||||
(float(pair.get("audio_max_abs_diff", 0.0)) for pair in prefix_pairs),
|
||||
default=0.0,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _contact_sheet(
|
||||
rows: list[tuple[str, list[np.ndarray]]],
|
||||
output: Path,
|
||||
*,
|
||||
column_labels: list[str] | None = None,
|
||||
) -> None:
|
||||
thumb_w, thumb_h = 224, 224
|
||||
label_w, top_h = 240, 42
|
||||
columns = max((len(frames) for _, frames in rows), default=0)
|
||||
width = label_w + thumb_w * columns
|
||||
height = top_h + thumb_h * len(rows)
|
||||
sheet = Image.new("RGB", (width, height), (9, 15, 25))
|
||||
draw = ImageDraw.Draw(sheet)
|
||||
font = ImageFont.load_default()
|
||||
for column in range(columns):
|
||||
draw.text(
|
||||
(label_w + column * thumb_w + 8, 14),
|
||||
column_labels[column]
|
||||
if column_labels is not None and column < len(column_labels)
|
||||
else f"Chunk {column + 1}",
|
||||
fill=(210, 225, 245),
|
||||
font=font,
|
||||
)
|
||||
for row_index, (label, frames) in enumerate(rows):
|
||||
y = top_h + row_index * thumb_h
|
||||
draw.text((10, y + 16), label, fill=(115, 210, 255), font=font)
|
||||
for column, frame in enumerate(frames):
|
||||
image = Image.fromarray(frame).resize(
|
||||
(thumb_w, thumb_h), Image.Resampling.LANCZOS
|
||||
)
|
||||
sheet.paste(image, (label_w + column * thumb_w, y))
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
sheet.save(output)
|
||||
|
||||
|
||||
def _milestone_frames(
|
||||
frames: list[np.ndarray], *, chunks: int
|
||||
) -> tuple[list[str], list[np.ndarray]]:
|
||||
frame_count = len(frames)
|
||||
chunk_frames = frame_count / float(chunks)
|
||||
indices = [
|
||||
min(frame_count - 1, 12),
|
||||
min(frame_count - 1, max(0, int(round(3 * chunk_frames)) - 1)),
|
||||
min(frame_count - 1, int(round(3 * chunk_frames)) + 12),
|
||||
min(frame_count - 1, int(round(4.5 * chunk_frames))),
|
||||
frame_count - 1,
|
||||
]
|
||||
return (
|
||||
["Near 0s", "G3 end", "G4 post", "G5", "G6 end"],
|
||||
[frames[index] for index in indices],
|
||||
)
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> dict:
|
||||
summary = json.loads((args.output_root / "summary.json").read_text(encoding="utf-8"))
|
||||
all_rows = []
|
||||
sheets = []
|
||||
milestone_sheets = []
|
||||
milestone_labels = None
|
||||
videos = {}
|
||||
for record in summary["cases"]:
|
||||
case = str(record["case"])
|
||||
chunks = int(record["chunks"])
|
||||
path = _video_path(record, args.video_root)
|
||||
frames, fps = _read_frames(path)
|
||||
metrics, representatives = _chunk_metrics(
|
||||
frames, chunks=chunks, radius=args.radius
|
||||
)
|
||||
for row in metrics:
|
||||
row["case"] = case
|
||||
row["video"] = str(path)
|
||||
row["fps"] = fps
|
||||
row["frame_count"] = len(frames)
|
||||
all_rows.append(row)
|
||||
sheets.append((case, representatives))
|
||||
labels, milestones = _milestone_frames(frames, chunks=chunks)
|
||||
milestone_labels = labels
|
||||
milestone_sheets.append((case, milestones))
|
||||
videos[case] = {
|
||||
"path": str(path),
|
||||
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
"fps": fps,
|
||||
"frame_count": len(frames),
|
||||
"duration_seconds": len(frames) / fps,
|
||||
"chunks": chunks,
|
||||
"chunk_seconds": float(record["chunk_seconds"]),
|
||||
"metrics": metrics,
|
||||
"decoded_trend": _trend(metrics, METRIC_KEYS),
|
||||
"continuation_input_rgb_proxy": _continuation_input_rgb_proxy(
|
||||
frames,
|
||||
chunks=chunks,
|
||||
context_frames=int(
|
||||
(record.get("diagnostic") or {}).get("context_frames", 22)
|
||||
),
|
||||
),
|
||||
"boundary_metrics": _boundary_metrics(frames, chunks=chunks),
|
||||
"latent_trend": _latent_trend(record.get("diagnostic") or {}),
|
||||
}
|
||||
|
||||
if not all_rows:
|
||||
raise RuntimeError("Issue #13 R2 summary contains no completed cases")
|
||||
csv_path = args.output_root / "chunk_metrics.csv"
|
||||
with csv_path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(all_rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(all_rows)
|
||||
contact_path = args.output_root / "contact_sheet.png"
|
||||
_contact_sheet(sheets, contact_path)
|
||||
milestone_contact_path = args.output_root / "milestone_contact_sheet.png"
|
||||
_contact_sheet(
|
||||
milestone_sheets,
|
||||
milestone_contact_path,
|
||||
column_labels=milestone_labels,
|
||||
)
|
||||
result = {
|
||||
"format": "h3-continuum-issue13-r2-analysis-v2",
|
||||
"sample_radius_frames": args.radius,
|
||||
"videos": videos,
|
||||
"csv": str(csv_path),
|
||||
"contact_sheet": str(contact_path),
|
||||
"milestone_contact_sheet": str(milestone_contact_path),
|
||||
}
|
||||
(args.output_root / "analysis.json").write_text(
|
||||
json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return result
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--video-root", type=Path, default=Path(r"D:\output\video\comfy_video")
|
||||
)
|
||||
parser.add_argument("--radius", type=int, default=12)
|
||||
return parser
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(_parser().parse_args())
|
||||
Reference in New Issue
Block a user