"""Conservative CPU-only decoded seam correction for H3 Continuum V2.1.""" from __future__ import annotations import math import torch import torch.nn.functional as F from .seam_types import AudioSeamMetrics, SeamDecision, VideoSeamMetrics COMPARE_SIZE = 96 MAX_CUT_REWIND = 3 MIN_RELATIVE_IMPROVEMENT = 0.10 MIN_ABSOLUTE_IMPROVEMENT = 0.003 SAFE_SCORE_LIMIT = 0.35 WEIGHT_PIXEL = 0.35 WEIGHT_LUMA = 0.20 WEIGHT_CHROMA = 0.10 WEIGHT_EDGE = 0.15 WEIGHT_TEMPORAL = 0.20 def _video_shape(frames: torch.Tensor, label: str) -> None: if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3: raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]") def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor: _video_shape(frames, "video seam frames") rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32) rgb = rgb.permute(0, 3, 1, 2) if tuple(rgb.shape[-2:]) != (COMPARE_SIZE, COMPARE_SIZE): rgb = F.interpolate( rgb, size=(COMPARE_SIZE, COMPARE_SIZE), mode="bilinear", align_corners=False, ) rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0) if not bool(torch.isfinite(rgb).all().item()): raise ValueError("video seam comparison contains NaN or Inf") return rgb def _luma(frame: torch.Tensor) -> torch.Tensor: weights = frame.new_tensor((0.2126, 0.7152, 0.0722)) return torch.sum(frame * weights, dim=-1) def _boundary_metrics( previous_images: torch.Tensor, next_images: torch.Tensor, ) -> VideoSeamMetrics: if previous_images.shape[0] < 2 or next_images.shape[0] < 2: raise ValueError("video seam scoring needs two frames on each side") prev = _downsample_rgb(previous_images[-2:]) nxt = _downsample_rgb(next_images[:2]) prev_before, prev_edge = prev[0], prev[1] next_edge, next_after = nxt[0], nxt[1] pixel = torch.mean(torch.abs(prev_edge - next_edge)).clamp(0.0, 1.0) prev_y = _luma(prev_edge) next_y = _luma(next_edge) luma = torch.mean(torch.abs(prev_y - next_y)).clamp(0.0, 1.0) prev_chroma = prev_edge - prev_y.unsqueeze(-1) next_chroma = next_edge - next_y.unsqueeze(-1) chroma = ( torch.mean(torch.abs(prev_chroma - next_chroma)) / 2.0 ).clamp(0.0, 1.0) prev_gx = prev_y[:, 1:] - prev_y[:, :-1] next_gx = next_y[:, 1:] - next_y[:, :-1] prev_gy = prev_y[1:, :] - prev_y[:-1, :] next_gy = next_y[1:, :] - next_y[:-1, :] edge = ( ( torch.mean(torch.abs(prev_gx - next_gx)) + torch.mean(torch.abs(prev_gy - next_gy)) ) / 4.0 ).clamp(0.0, 1.0) prev_delta = prev_edge - prev_before next_delta = next_after - next_edge temporal = ( torch.mean(torch.abs(prev_delta - next_delta)) / 2.0 ).clamp(0.0, 1.0) motion = max( float(torch.mean(torch.abs(prev_delta)).item()), float(torch.mean(torch.abs(next_delta)).item()), ) score = ( WEIGHT_PIXEL * float(pixel.item()) + WEIGHT_LUMA * float(luma.item()) + WEIGHT_CHROMA * float(chroma.item()) + WEIGHT_EDGE * float(edge.item()) + WEIGHT_TEMPORAL * float(temporal.item()) ) if not math.isfinite(score): raise ValueError("video seam score is not finite") return VideoSeamMetrics( pixel_mae_norm=float(pixel.item()), luma_norm=float(luma.item()), chroma_norm=float(chroma.item()), edge_norm=float(edge.item()), temporal_norm=float(temporal.item()), motion=motion, score=max(0.0, min(1.0, score)), ) def analyze_video_seam( previous_images: torch.Tensor, next_raw_images: torch.Tensor, *, context_frames: int, rewind_frames: int = 0, ) -> VideoSeamMetrics: context_frames = int(context_frames) rewind_frames = int(rewind_frames) cut_index = context_frames - rewind_frames previous_stop = int(previous_images.shape[0]) - rewind_frames if rewind_frames < 0 or rewind_frames > MAX_CUT_REWIND: raise ValueError("rewind_frames is outside the V2.1 safe range") if cut_index < 0 or previous_stop < 2 or cut_index + 2 > next_raw_images.shape[0]: raise ValueError("insufficient frames for the requested adaptive cut") return _boundary_metrics( previous_images[previous_stop - 2 : previous_stop], next_raw_images[cut_index : cut_index + 2], ) def choose_adaptive_cut( previous_images: torch.Tensor, next_raw_images: torch.Tensor, *, context_frames: int, ) -> tuple[int, VideoSeamMetrics, VideoSeamMetrics]: legacy = analyze_video_seam( previous_images, next_raw_images, context_frames=context_frames, rewind_frames=0, ) best_rewind = 0 best = legacy maximum = min( MAX_CUT_REWIND, max(0, int(context_frames)), max(0, int(previous_images.shape[0]) - 2), ) for rewind in range(1, maximum + 1): try: candidate = analyze_video_seam( previous_images, next_raw_images, context_frames=context_frames, rewind_frames=rewind, ) except ValueError: continue if candidate.score < best.score: best_rewind, best = rewind, candidate absolute = legacy.score - best.score relative = absolute / max(legacy.score, 1e-12) accepted = ( best_rewind > 0 and relative >= MIN_RELATIVE_IMPROVEMENT and absolute >= MIN_ABSOLUTE_IMPROVEMENT and best.score <= SAFE_SCORE_LIMIT ) return (best_rewind, legacy, best) if accepted else (0, legacy, legacy) def apply_color_guard( previous_images: torch.Tensor, current_images: torch.Tensor, ) -> tuple[ torch.Tensor, float, tuple[float, float, float], VideoSeamMetrics, ]: before = _boundary_metrics(previous_images[-2:], current_images[:2]) count = min(4, int(current_images.shape[0])) if count < 2: return current_images, 1.0, (0.0, 0.0, 0.0), before prev_rgb = _downsample_rgb(previous_images[-2:]) current_rgb = _downsample_rgb(current_images[:2]) prev_luma = float(_luma(prev_rgb).mean().item()) current_luma = float(_luma(current_rgb).mean().item()) requested_gain = prev_luma / max(current_luma, 1e-6) if not 0.85 <= requested_gain <= 1.15: return current_images, 1.0, (0.0, 0.0, 0.0), before gain = max(0.94, min(1.06, requested_gain)) prev_y = _luma(prev_rgb).unsqueeze(-1) current_y = _luma(current_rgb).unsqueeze(-1) requested_bias = ( (prev_rgb - prev_y).mean(dim=(0, 1, 2)) - (current_rgb - current_y).mean(dim=(0, 1, 2)) ) if float(torch.max(torch.abs(requested_bias)).item()) > 0.08: return current_images, 1.0, (0.0, 0.0, 0.0), before bias = requested_bias.clamp(-0.02, 0.02) result = current_images.clone().contiguous() original = result[:count].clone() fade = result.new_tensor((1.0, 0.65, 0.30, 0.0))[:count] luma_weights = result.new_tensor((0.2126, 0.7152, 0.0722)) for index in range(count): weight = fade[index] rgb = result[index, ..., :3] y = torch.sum(rgb * luma_weights, dim=-1, keepdim=True) chroma = rgb - y frame_gain = 1.0 + (gain - 1.0) * weight frame_bias = bias.to(dtype=rgb.dtype) * weight result[index, ..., :3] = ( y * frame_gain + chroma + frame_bias ).clamp(0.0, 1.0) after = _boundary_metrics(previous_images[-2:], result[:2]) if after.score > before.score * 1.01: result[:count].copy_(original) return result, 1.0, (0.0, 0.0, 0.0), before return ( result, gain, tuple(float(value) for value in bias.tolist()), after, ) def choose_video_blend(metrics: VideoSeamMetrics) -> int: if metrics.motion <= 0.015: return 2 if metrics.motion <= 0.040: return 1 return 0 def apply_video_blend( previous_frames: torch.Tensor, aligned_next_frames: torch.Tensor, *, blend_frames: int, ) -> torch.Tensor: blend_frames = min( int(blend_frames), int(previous_frames.shape[0]), int(aligned_next_frames.shape[0]), 2, ) if blend_frames <= 0: return previous_frames[:0].clone() previous = previous_frames[-blend_frames:].clone() current = aligned_next_frames[-blend_frames:].to(dtype=previous.dtype) for index in range(blend_frames): progress = float(index + 1) / float(blend_frames + 1) alpha = 0.5 - 0.5 * math.cos(math.pi * progress) previous[index].mul_(1.0 - alpha).add_(current[index], alpha=alpha) return previous def correct_seam( previous_images: torch.Tensor, next_raw_images: torch.Tensor, *, context_frames: int, automatic: bool = False, ) -> tuple[torch.Tensor, SeamDecision, torch.Tensor | None]: rewind, legacy, selected = choose_adaptive_cut( previous_images, next_raw_images, context_frames=context_frames, ) cut_index = int(context_frames) - rewind previous_stop = int(previous_images.shape[0]) - rewind if previous_stop < 2 or cut_index < 0: raise ValueError("adaptive cut would exhaust a seam side") current = next_raw_images[cut_index:].clone().contiguous() kept_previous = previous_images[:previous_stop] current, gain, bias, corrected = apply_color_guard(kept_previous, current) blend_frames = min( choose_video_blend(selected), cut_index, previous_stop, ) video_patch: torch.Tensor | None = None if blend_frames > 0: video_patch = apply_video_blend( kept_previous[-blend_frames:], next_raw_images[cut_index - blend_frames : cut_index], blend_frames=blend_frames, ) score_previous = kept_previous[-2:].clone() replaced = min(blend_frames, 2) score_previous[-replaced:].copy_(video_patch[-replaced:]) corrected = _boundary_metrics(score_previous, current[:2]) improvement = (legacy.score - corrected.score) / max(legacy.score, 1e-12) if corrected.score > legacy.score * 1.01: fallback = next_raw_images[int(context_frames) :].clone().contiguous() return ( fallback, SeamDecision( legacy_score=legacy.score, cut_score=legacy.score, corrected_score=legacy.score, fallback_reason="corrected score was worse than legacy", ), None, ) absolute_improvement = legacy.score - corrected.score if automatic and ( improvement < MIN_RELATIVE_IMPROVEMENT or absolute_improvement < MIN_ABSOLUTE_IMPROVEMENT ): fallback = next_raw_images[int(context_frames) :].clone().contiguous() return ( fallback, SeamDecision( legacy_score=legacy.score, cut_score=selected.score, corrected_score=legacy.score, fallback_reason="Auto kept the legacy video seam; improvement was below threshold", ), None, ) decision = SeamDecision( cut_rewind_frames=rewind, legacy_score=legacy.score, cut_score=selected.score, corrected_score=corrected.score, improvement=improvement, video_blend_frames=blend_frames, luma_gain=gain, chroma_bias=bias, ) return current, decision, video_patch def _audio_mid(waveform: torch.Tensor) -> torch.Tensor: if not torch.is_tensor(waveform) or waveform.ndim != 3: raise ValueError("audio seam waveform must be [B,C,S]") value = waveform.detach().to(device="cpu", dtype=torch.float32) if not bool(torch.isfinite(value).all().item()): raise ValueError("audio seam waveform contains NaN or Inf") return value.mean(dim=(0, 1)) def _correlation(a: torch.Tensor, b: torch.Tensor) -> float: length = min(int(a.numel()), int(b.numel())) if length < 8: return 0.0 a = a[:length] - a[:length].mean() b = b[:length] - b[:length].mean() denominator = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12) value = float((torch.sum(a * b) / denominator).item()) return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0 def analyze_audio_seam( previous_waveform: torch.Tensor, next_raw_waveform: torch.Tensor, *, sample_rate: int, cut_sample: int, ) -> AudioSeamMetrics: sample_rate = int(sample_rate) cut_sample = int(cut_sample) previous = _audio_mid(previous_waveform) current = _audio_mid(next_raw_waveform) max_lag = max(1, int(round(sample_rate * 0.020))) window = min( max(8, int(round(sample_rate * 0.080))), int(previous.shape[-1]), max(0, cut_sample - max_lag), ) if sample_rate <= 0 or window < 8: raise ValueError("insufficient audio for seam alignment") previous_tail = previous[-window:] def correlation_at(lag: int) -> float | None: stop = cut_sample + int(lag) start = stop - window if start < 0 or stop > current.shape[-1]: return None return _correlation(previous_tail, current[start:stop]) baseline = correlation_at(0) if baseline is None: raise ValueError("audio seam cut is outside the decoded waveform") best_lag, best_correlation = 0, baseline for lag in range(-max_lag, max_lag + 1): correlation = correlation_at(lag) if correlation is not None and correlation > best_correlation: best_lag, best_correlation = lag, correlation if best_correlation < 0.20 or best_correlation - baseline < 0.02: best_lag, best_correlation = 0, baseline else: center = max(0, min(int(current.shape[-1]) - 1, cut_sample + best_lag)) radius = max(1, int(round(sample_rate * 0.001))) start = max(0, center - radius) stop = min(int(current.shape[-1]), center + radius + 1) if stop > start: zero = int(torch.argmin(torch.abs(current[start:stop])).item()) + start best_lag = zero - cut_sample before_jump = float( torch.mean( torch.abs(previous_waveform[..., -1] - next_raw_waveform[..., cut_sample]) ).item() ) return AudioSeamMetrics( correlation_before=baseline, correlation_after=best_correlation, boundary_jump_before=before_jump, boundary_jump_after=before_jump, offset_samples=int(best_lag), ) def _audio_crossfade_ms( previous_waveform: torch.Tensor, next_raw_waveform: torch.Tensor, *, cut_sample: int, sample_rate: int, ) -> int: probe = max(8, int(round(sample_rate * 0.020))) if previous_waveform.shape[-1] < probe or cut_sample < probe: return 10 previous = _audio_mid(previous_waveform)[-probe:] current = _audio_mid(next_raw_waveform)[cut_sample - probe : cut_sample] rms = torch.sqrt(torch.mean(torch.cat((previous, current)).square())).clamp_min(1e-6) transient = max( float(torch.mean(torch.abs(torch.diff(previous))).item()), float(torch.mean(torch.abs(torch.diff(current))).item()), ) / float(rms.item()) if transient > 0.80: return 10 if transient > 0.35: return 20 if transient > 0.15: return 40 return 60 def apply_audio_seam( previous_waveform: torch.Tensor, next_raw_waveform: torch.Tensor, metrics: AudioSeamMetrics, *, sample_rate: int, cut_sample: int, ) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]: if metrics.correlation_after < 0.20: return None, metrics, 0, 1.0, 0.0 fade_ms = _audio_crossfade_ms( previous_waveform, next_raw_waveform, cut_sample=cut_sample, sample_rate=sample_rate, ) fade_samples = min( int(round(sample_rate * fade_ms / 1000.0)), int(previous_waveform.shape[-1]), int(cut_sample), ) if fade_samples < 2: return None, metrics, 0, 1.0, 0.0 progress = torch.linspace( 0.0, 1.0, fade_samples, dtype=torch.float32, device="cpu", ) base_indices = torch.arange(cut_sample - fade_samples, cut_sample, dtype=torch.long) offsets = torch.round( float(metrics.offset_samples) * (1.0 - progress) ).to(dtype=torch.long) indices = (base_indices + offsets).clamp(0, int(next_raw_waveform.shape[-1]) - 1) current = next_raw_waveform.detach().to("cpu").index_select(-1, indices) previous = previous_waveform[..., -fade_samples:].detach().to("cpu") previous_rms = torch.sqrt(torch.mean(previous.float().square())).clamp_min(1e-6) current_rms = torch.sqrt(torch.mean(current.float().square())).clamp_min(1e-6) requested_db = 20.0 * math.log10(float((previous_rms / current_rms).item())) level_db = max(-1.5, min(1.5, requested_db)) if abs(requested_db) <= 6.0 else 0.0 level_gain = 10.0 ** (level_db / 20.0) requested_dc = float((previous.float().mean() - current.float().mean()).item()) dc_bias = max(-0.02, min(0.02, requested_dc)) if abs(requested_dc) <= 0.10 else 0.0 current = current * level_gain + dc_bias shape = [1] * (previous.ndim - 1) + [fade_samples] fade_in = torch.sin(progress * (math.pi / 2.0)).reshape(shape) fade_out = torch.cos(progress * (math.pi / 2.0)).reshape(shape) patch = previous * fade_out + current.to(dtype=previous.dtype) * fade_in input_peak = max( float(previous.abs().max().item()), float(current.abs().max().item()), 1e-6, ) patch_peak = float(patch.abs().max().item()) if patch_peak > input_peak * 1.05: patch = patch * ((input_peak * 1.05) / patch_peak) after_jump = float( torch.mean(torch.abs(patch[..., -1] - next_raw_waveform[..., cut_sample])).item() ) updated = AudioSeamMetrics( correlation_before=metrics.correlation_before, correlation_after=metrics.correlation_after, boundary_jump_before=metrics.boundary_jump_before, boundary_jump_after=after_jump, offset_samples=metrics.offset_samples, ) return patch.contiguous(), updated, fade_samples, level_gain, dc_bias def correct_audio_seam( previous_waveform: torch.Tensor, next_raw_waveform: torch.Tensor, *, sample_rate: int, cut_sample: int, ) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]: metrics = analyze_audio_seam( previous_waveform, next_raw_waveform, sample_rate=sample_rate, cut_sample=cut_sample, ) return apply_audio_seam( previous_waveform, next_raw_waveform, metrics, sample_rate=sample_rate, cut_sample=cut_sample, )