Files
ComfyUI-H3-Continuum-Tiled/v3/video_seam.py
T

529 lines
21 KiB
Python

"""Read-only decoded video-boundary analysis for H3 Continuum V3."""
from __future__ import annotations
from dataclasses import dataclass
import math
from typing import Any
import torch
import torch.nn.functional as F
from .plan import validate_assembly_plan
COMPARE_LONG_SIDE = 192
MAX_REWIND_FRAMES = 2
FLASH_WINDOW_FRAMES = 6
FLASH_TILE_GRID = 4
MIN_FLASH_LUMA_SHIFT = 0.01
MIN_FLASH_REVERSAL = 0.35
MIN_FLASH_GLOBAL_FRACTION = 0.75
MIN_ANOMALY_RATIO = 1.35
MIN_RELATIVE_IMPROVEMENT = 0.06
MIN_AUTO_FLASH_REVERSAL = 0.70
MIN_AUTO_FLASH_IMPROVEMENT = 0.70
SMALLER_SHIFT_TOLERANCE = 0.03
MIN_MICRO_FLASH_LUMA_SHIFT = 0.003
MIN_MICRO_FLASH_REVERSAL = 0.25
MIN_MICRO_FLASH_GLOBAL_FRACTION = 0.70
MAX_EXPOSURE_RAMP_REVERSAL = 0.25
MIN_MOTION_CROSS_DELTA = 0.006
MIN_MOTION_JUMP_RATIO = 1.60
@dataclass(frozen=True)
class VideoBoundaryAnalysis:
boundary_index: int
nominal_error: float
baseline_error: float
anomaly_ratio: float
recommended_rewind: int
best_error: float
improvement: float
luma_delta: float
chroma_delta: float
edge_delta: float
scene_cut_score: float
scene_cut: bool
flash_luma_shift: float
flash_reversal: float
flash_global_fraction: float
transient_flash_candidate: bool
cross_frame_delta: float
motion_baseline_delta: float
motion_jump_ratio: float
micro_flash_candidate: bool
exposure_ramp_candidate: bool
motion_hitch_candidate: bool
classification: str
reason: str
def _validate_frames(frames: Any, label: str) -> None:
if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3:
raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]")
if int(frames.shape[0]) < 1 or int(frames.shape[1]) < 1 or int(frames.shape[2]) < 1:
raise ValueError(f"{label} must not be empty")
def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor:
_validate_frames(frames, "video seam frames")
rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32)
rgb = rgb.permute(0, 3, 1, 2)
height, width = int(rgb.shape[-2]), int(rgb.shape[-1])
scale = min(1.0, COMPARE_LONG_SIDE / float(max(height, width)))
target_height = max(1, int(round(height * scale)))
target_width = max(1, int(round(width * scale)))
if (target_height, target_width) != (height, width):
rgb = F.interpolate(
rgb,
size=(target_height, target_width),
mode="bilinear",
align_corners=False,
)
rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0)
if not bool(torch.isfinite(rgb).all().item()):
raise ValueError("video seam comparison contains NaN or Inf")
return rgb
def _luma(frame: torch.Tensor) -> torch.Tensor:
weights = frame.new_tensor((0.2126, 0.7152, 0.0722))
return torch.sum(frame * weights, dim=-1)
def _frame_delta(left: torch.Tensor, right: torch.Tensor) -> float:
return float(torch.mean(torch.abs(left - right)).item())
def _prediction_error(history: torch.Tensor, candidate: torch.Tensor) -> float:
if int(history.shape[0]) < 3:
raise ValueError("temporal prediction requires three history frames")
recent_velocity = history[-1] - history[-2]
prior_velocity = history[-2] - history[-3]
predicted = (
history[-1] + recent_velocity * 0.65 + prior_velocity * 0.35
).clamp(0.0, 1.0)
motion = torch.mean(torch.abs(recent_velocity), dim=-1)
motion_scale = motion / motion.mean().clamp_min(1.0e-6)
weights = (1.0 + motion_scale).clamp(1.0, 4.0).unsqueeze(-1)
value = float(torch.mean((predicted - candidate).square() * weights).item())
if not math.isfinite(value):
raise ValueError("video seam prediction score is not finite")
return max(0.0, value)
def _baseline_prediction_error(previous: torch.Tensor) -> float:
recent = previous[-min(8, int(previous.shape[0])) :]
errors = [
_prediction_error(recent[:index], recent[index])
for index in range(3, int(recent.shape[0]))
]
if not errors:
return 0.0
return float(torch.tensor(errors, dtype=torch.float64).median().item())
def _edge_delta(left: torch.Tensor, right: torch.Tensor) -> float:
left_y = _luma(left)
right_y = _luma(right)
left_x = left_y[:, 1:] - left_y[:, :-1]
right_x = right_y[:, 1:] - right_y[:, :-1]
left_v = left_y[1:, :] - left_y[:-1, :]
right_v = right_y[1:, :] - right_y[:-1, :]
horizontal = torch.mean(torch.abs(left_x - right_x)) if left_x.numel() else left_y.new_tensor(0.0)
vertical = torch.mean(torch.abs(left_v - right_v)) if left_v.numel() else left_y.new_tensor(0.0)
return float(((horizontal + vertical) * 0.5).item())
def _scene_cut_metrics(previous: torch.Tensor, current: torch.Tensor) -> tuple[float, bool]:
cross_delta = _frame_delta(previous[-1], current[0])
previous_internal = _frame_delta(previous[-2], previous[-1])
if int(current.shape[0]) >= 3:
current_internal = 0.5 * (
_frame_delta(current[0], current[1])
+ _frame_delta(current[1], current[2])
)
elif int(current.shape[0]) == 2:
current_internal = _frame_delta(current[0], current[1])
else:
current_internal = cross_delta
denominator = cross_delta + previous_internal + current_internal + 1.0e-8
score = cross_delta / denominator
persistent_new_scene = (
int(current.shape[0]) >= 3
and cross_delta >= 0.12
and current_internal <= cross_delta * 0.35
and cross_delta >= max(0.12, previous_internal * 3.0)
)
return max(0.0, min(1.0, score)), persistent_new_scene
def _transient_flash_metrics(
previous: torch.Tensor,
current: torch.Tensor,
) -> tuple[float, float, float]:
"""Measure a broad boundary exposure pulse that quickly returns."""
previous_y = _luma(previous[-1])
current_y = _luma(current[:FLASH_WINDOW_FRAMES])
shifts = torch.mean(current_y - previous_y.unsqueeze(0), dim=(1, 2))
initial_shift = float(torch.abs(shifts[0]).item())
if int(shifts.shape[0]) > 1 and initial_shift > 1.0e-8:
closest_later = float(torch.min(torch.abs(shifts[1:])).item())
reversal = max(0.0, min(1.0, 1.0 - closest_later / initial_shift))
else:
reversal = 0.0
height, width = int(previous_y.shape[0]), int(previous_y.shape[1])
tile_height = max(1, height // FLASH_TILE_GRID)
tile_width = max(1, width // FLASH_TILE_GRID)
delta = current_y[0] - previous_y
sign = 1.0 if float(shifts[0].item()) >= 0.0 else -1.0
tile_matches = []
for top in range(0, height, tile_height):
for left in range(0, width, tile_width):
tile = delta[
top : min(height, top + tile_height),
left : min(width, left + tile_width),
]
tile_shift = float(torch.mean(tile).item())
tile_matches.append(
tile_shift * sign >= min(MIN_FLASH_LUMA_SHIFT * 0.5, initial_shift * 0.5)
)
global_fraction = sum(tile_matches) / max(1, len(tile_matches))
return initial_shift, reversal, float(global_fraction)
def _temporal_motion_metrics(
previous: torch.Tensor,
current: torch.Tensor,
) -> tuple[float, float, float]:
"""Compare the boundary transition with nearby native frame motion."""
cross_delta = _frame_delta(previous[-1], current[0])
internal_deltas: list[float] = []
previous_start = max(1, int(previous.shape[0]) - 5)
for index in range(previous_start, int(previous.shape[0])):
internal_deltas.append(_frame_delta(previous[index - 1], previous[index]))
current_limit = min(5, int(current.shape[0]))
for index in range(1, current_limit):
internal_deltas.append(_frame_delta(current[index - 1], current[index]))
if internal_deltas:
baseline = float(
torch.tensor(internal_deltas, dtype=torch.float64).median().item()
)
else:
baseline = 0.0
if cross_delta <= 1.0e-8:
ratio = 0.0
else:
ratio = cross_delta / max(baseline, 1.0e-5)
return cross_delta, baseline, ratio
def analyze_video_boundary(
previous_images: torch.Tensor,
current_raw_images: torch.Tensor,
*,
trim_frames: int,
boundary_index: int = 1,
) -> VideoBoundaryAnalysis:
"""Analyze one boundary without modifying either input tensor."""
_validate_frames(previous_images, "previous decoded image chunk")
_validate_frames(current_raw_images, "current decoded image chunk")
trim_frames = int(trim_frames)
if int(previous_images.shape[0]) < 4:
raise ValueError("video seam analysis requires four previous frames")
if trim_frames < 0 or trim_frames >= int(current_raw_images.shape[0]):
raise ValueError("video seam trim position is outside the current chunk")
if trim_frames + 3 > int(current_raw_images.shape[0]):
raise ValueError("video seam analysis requires three retained current frames")
previous = _downsample_rgb(previous_images[-8:])
candidate_start = max(0, trim_frames - MAX_REWIND_FRAMES)
current = _downsample_rgb(
current_raw_images[
candidate_start : trim_frames + FLASH_WINDOW_FRAMES
]
)
nominal_index = trim_frames - candidate_start
nominal = current[nominal_index]
candidate_errors: dict[int, float] = {}
maximum_rewind = min(MAX_REWIND_FRAMES, trim_frames)
for rewind in range(maximum_rewind + 1):
candidate = current[nominal_index - rewind]
candidate_errors[rewind] = _prediction_error(previous, candidate)
nominal_error = candidate_errors[0]
minimum_error = min(candidate_errors.values())
tolerance = minimum_error * SMALLER_SHIFT_TOLERANCE + 1.0e-10
best_rewind = min(
rewind
for rewind, error in candidate_errors.items()
if error <= minimum_error + tolerance
)
best_error = candidate_errors[best_rewind]
baseline_error = _baseline_prediction_error(previous)
anomaly_ratio = nominal_error / max(baseline_error, 1.0e-5)
improvement = (nominal_error - best_error) / max(nominal_error, 1.0e-12)
previous_edge = previous[-1]
previous_y = _luma(previous_edge)
nominal_y = _luma(nominal)
luma_delta = float(torch.mean(torch.abs(previous_y - nominal_y)).item())
previous_chroma = previous_edge - previous_y.unsqueeze(-1)
nominal_chroma = nominal - nominal_y.unsqueeze(-1)
chroma_delta = float(
torch.mean(torch.abs(previous_chroma - nominal_chroma)).item()
)
edge_delta = _edge_delta(previous_edge, nominal)
scene_score, scene_cut = _scene_cut_metrics(
previous,
current[nominal_index : nominal_index + 3],
)
flash_luma_shift, flash_reversal, flash_global_fraction = (
_transient_flash_metrics(previous, current[nominal_index:])
)
cross_frame_delta, motion_baseline_delta, motion_jump_ratio = (
_temporal_motion_metrics(previous, current[nominal_index:])
)
transient_flash_candidate = (
not scene_cut
and flash_luma_shift >= MIN_FLASH_LUMA_SHIFT
and flash_reversal >= MIN_FLASH_REVERSAL
and flash_global_fraction >= MIN_FLASH_GLOBAL_FRACTION
)
micro_flash_candidate = (
not scene_cut
and not transient_flash_candidate
and flash_luma_shift >= MIN_MICRO_FLASH_LUMA_SHIFT
and flash_reversal >= MIN_MICRO_FLASH_REVERSAL
and flash_global_fraction >= MIN_MICRO_FLASH_GLOBAL_FRACTION
)
exposure_ramp_candidate = (
not scene_cut
and not transient_flash_candidate
and not micro_flash_candidate
and anomaly_ratio >= MIN_ANOMALY_RATIO
and flash_luma_shift >= MIN_FLASH_LUMA_SHIFT
and flash_reversal < MAX_EXPOSURE_RAMP_REVERSAL
and flash_global_fraction >= MIN_FLASH_GLOBAL_FRACTION
)
motion_hitch_candidate = (
not scene_cut
and not transient_flash_candidate
and not micro_flash_candidate
and not exposure_ramp_candidate
and cross_frame_delta >= MIN_MOTION_CROSS_DELTA
and motion_jump_ratio >= MIN_MOTION_JUMP_RATIO
)
if scene_cut:
classification = "scene_cut"
elif transient_flash_candidate:
classification = "transient_flash"
elif micro_flash_candidate:
classification = "micro_flash"
elif exposure_ramp_candidate:
classification = "exposure_ramp"
elif motion_hitch_candidate:
classification = "motion_hitch"
else:
classification = "clean_boundary"
recommended_rewind = 0
if scene_cut:
reason = "kept nominal boundary; persistent content change resembles a scene cut"
elif best_rewind == 0:
reason = "kept nominal boundary; it has the lowest prediction error"
elif (
transient_flash_candidate
and flash_reversal >= MIN_AUTO_FLASH_REVERSAL
and improvement >= MIN_AUTO_FLASH_IMPROVEMENT
):
recommended_rewind = best_rewind
reason = f"mild transient flash recommends correcting {best_rewind} frame(s)"
elif anomaly_ratio < MIN_ANOMALY_RATIO:
reason = "kept nominal boundary; anomaly ratio is below the analysis threshold"
elif improvement < MIN_RELATIVE_IMPROVEMENT:
reason = "kept nominal boundary; candidate improvement is below 6%"
else:
recommended_rewind = best_rewind
reason = f"analysis recommends shift {-best_rewind:+d} frame(s)"
return VideoBoundaryAnalysis(
boundary_index=int(boundary_index),
nominal_error=nominal_error,
baseline_error=baseline_error,
anomaly_ratio=anomaly_ratio,
recommended_rewind=recommended_rewind,
best_error=best_error,
improvement=improvement,
luma_delta=luma_delta,
chroma_delta=chroma_delta,
edge_delta=edge_delta,
scene_cut_score=scene_score,
scene_cut=scene_cut,
flash_luma_shift=flash_luma_shift,
flash_reversal=flash_reversal,
flash_global_fraction=flash_global_fraction,
transient_flash_candidate=transient_flash_candidate,
cross_frame_delta=cross_frame_delta,
motion_baseline_delta=motion_baseline_delta,
motion_jump_ratio=motion_jump_ratio,
micro_flash_candidate=micro_flash_candidate,
exposure_ramp_candidate=exposure_ramp_candidate,
motion_hitch_candidate=motion_hitch_candidate,
classification=classification,
reason=reason,
)
def analyze_decoded_boundaries(
*,
images: list[Any],
assembly_plan: dict[str, Any],
) -> list[VideoBoundaryAnalysis]:
"""Analyze every decoded chunk boundary without changing decoded content."""
plan = validate_assembly_plan(assembly_plan)
chunks = list(plan["chunks"])
if len(images) != len(chunks):
raise ValueError("decoded image count does not match the assembly plan")
analyses: list[VideoBoundaryAnalysis] = []
for index in range(1, len(chunks)):
previous_raw = images[index - 1]
current_raw = images[index]
previous_plan = chunks[index - 1]
current_plan = chunks[index]
_validate_frames(previous_raw, f"decoded image chunk {index}")
_validate_frames(current_raw, f"decoded image chunk {index + 1}")
previous_total = int(previous_plan["total_frames"])
previous_trim = int(previous_plan["trim_frames"])
current_total = int(current_plan["total_frames"])
current_trim = int(current_plan["trim_frames"])
previous_segment = previous_raw[:previous_total][previous_trim:]
current_segment = current_raw[:current_total]
analyses.append(
analyze_video_boundary(
previous_segment,
current_segment,
trim_frames=current_trim,
boundary_index=index,
)
)
return analyses
def correct_decoded_boundaries(
*,
images: list[Any],
assembly_plan: dict[str, Any],
analyses: list[VideoBoundaryAnalysis],
enable_exposure_ramp: bool = False,
) -> tuple[list[Any], dict[int, str]]:
"""Normalize qualified transient exposure shifts without changing motion."""
plan = validate_assembly_plan(assembly_plan)
chunks = list(plan["chunks"])
if len(images) != len(chunks):
raise ValueError("decoded image count does not match the assembly plan")
corrected = list(images)
actions: dict[int, str] = {}
for analysis in analyses:
boundary = int(analysis.boundary_index)
correction_kind = "transient flash"
replace_frames = int(analysis.recommended_rewind)
if analysis.micro_flash_candidate:
correction_kind = "micro-flash"
replace_frames = 1
elif enable_exposure_ramp and analysis.exposure_ramp_candidate:
correction_kind = "exposure ramp"
replace_frames = 4
if replace_frames <= 0:
actions[boundary] = "kept native boundary"
continue
if analysis.scene_cut or not (
analysis.transient_flash_candidate
or analysis.micro_flash_candidate
or (enable_exposure_ramp and analysis.exposure_ramp_candidate)
):
actions[boundary] = "kept native boundary; correction guard rejected candidate"
continue
previous_index = boundary - 1
current_index = boundary
if previous_index < 0 or current_index >= len(images):
raise ValueError("video seam boundary index is outside decoded chunks")
previous_raw = images[previous_index]
current_raw = images[current_index]
_validate_frames(previous_raw, f"decoded image chunk {previous_index + 1}")
_validate_frames(current_raw, f"decoded image chunk {current_index + 1}")
previous_plan = chunks[previous_index]
current_plan = chunks[current_index]
previous_total = int(previous_plan["total_frames"])
previous_trim = int(previous_plan["trim_frames"])
current_total = int(current_plan["total_frames"])
current_trim = int(current_plan["trim_frames"])
if current_trim + replace_frames >= current_total:
raise ValueError("video seam correction has no retained recovery frame")
previous_segment = previous_raw[:previous_total][previous_trim:]
if int(previous_segment.shape[0]) < 1:
raise ValueError("video seam correction has no previous retained frame")
current_cpu = current_raw.detach().to(device="cpu").clone()
previous_anchor = previous_segment[-1].detach().to(
device="cpu", dtype=torch.float32
)
recovery_anchor = current_cpu[current_trim + replace_frames].to(
dtype=torch.float32
)
if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape):
raise ValueError("decoded image geometry changed at video seam")
for offset in range(replace_frames):
alpha = float(offset + 1) / float(replace_frames + 1)
original = current_cpu[current_trim + offset].to(dtype=torch.float32)
target = torch.lerp(previous_anchor, recovery_anchor, alpha)
original_mean = original.mean(dim=(0, 1), keepdim=True)
target_mean = target.mean(dim=(0, 1), keepdim=True)
normalized = (original + target_mean - original_mean).clamp(0.0, 1.0)
current_cpu[current_trim + offset].copy_(
normalized.to(dtype=current_cpu.dtype)
)
corrected[current_index] = current_cpu
actions[boundary] = (
f"normalized exposure/color on {replace_frames} {correction_kind} "
"boundary frame(s)"
)
return corrected, actions
def format_video_boundary_analysis(
analysis: VideoBoundaryAnalysis,
*,
action: str = "analysis only",
) -> str:
shift = -int(analysis.recommended_rewind)
return (
f"video seam {analysis.boundary_index}->{analysis.boundary_index + 1}: "
f"anomaly={analysis.anomaly_ratio:.3f}, "
f"prediction={analysis.nominal_error:.6f}->{analysis.best_error:.6f}, "
f"improvement={analysis.improvement:.1%}, shift={shift:+d}, "
f"luma={analysis.luma_delta:.6f}, chroma={analysis.chroma_delta:.6f}, "
f"edge={analysis.edge_delta:.6f}, "
f"scene_cut={'yes' if analysis.scene_cut else 'no'} "
f"flash_shift={analysis.flash_luma_shift:.6f}, "
f"flash_reversal={analysis.flash_reversal:.1%}, "
f"flash_global={analysis.flash_global_fraction:.1%}, "
f"flash_candidate={'yes' if analysis.transient_flash_candidate else 'no'} "
f"class={analysis.classification}, "
f"cross={analysis.cross_frame_delta:.6f}, "
f"motion_base={analysis.motion_baseline_delta:.6f}, "
f"motion_jump={analysis.motion_jump_ratio:.2f}x, "
f"micro_flash={'yes' if analysis.micro_flash_candidate else 'no'}, "
f"exposure_ramp={'yes' if analysis.exposure_ramp_candidate else 'no'}, "
f"motion_hitch={'yes' if analysis.motion_hitch_candidate else 'no'} "
f"({analysis.reason}); action={action}"
)