Files
ComfyUI-H3-Continuum-Tiled/v2/decoder.py
T

466 lines
20 KiB
Python

"""Deferred VAE decoding, overlap trimming, assembly, and exact-duration correction."""
from __future__ import annotations
from typing import Any
import torch
import torch.nn.functional as F
from ..constants import DIAGNOSTICS_FULL, DIAGNOSTICS_OFF, FPS
from ..media import validate_audio
from ..state import validate_plan
from .diagnostics import format_join_metrics, format_seam_decision, overlap_diagnostics
from .seam_guard import correct_audio_seam, correct_seam
from .seam_types import SeamDecision
from .session import entry_to_latent, validate_chunk_entry
def decode_video(video_vae: Any, latent: dict[str, Any]) -> torch.Tensor:
samples = latent["samples"]
if hasattr(samples, "unbind"):
video = list(samples.unbind())[0]
else:
video = samples[0]
images = video_vae.decode(video)
if images.ndim == 5:
images = images.reshape(-1, images.shape[-3], images.shape[-2], images.shape[-1])
if images.ndim != 4:
raise ValueError(f"video VAE returned unexpected shape {tuple(images.shape)}")
# Long sequences must not leave decoded frames resident on the GPU. ComfyUI
# IMAGE tensors are conventionally CPU float tensors, so transfer each chunk
# immediately and release its raw decode before moving to the next chunk.
return images.detach().to("cpu").contiguous()
def decode_audio(audio_vae: Any, latent: dict[str, Any]) -> dict[str, Any]:
samples = latent["samples"]
if hasattr(samples, "unbind"):
audio_latent = list(samples.unbind())[-1]
else:
audio_latent = samples[-1]
audio = audio_vae.decode(audio_latent).movedim(-1, 1)
std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0
std[std < 1.0] = 1.0
audio = (audio / std).detach().to("cpu").contiguous()
sample_rate = getattr(
audio_vae,
"audio_sample_rate_output",
getattr(audio_vae, "audio_sample_rate", 32000),
)
return {"waveform": audio, "sample_rate": int(sample_rate)}
def _trim_images(images: torch.Tensor, plan: dict[str, Any]) -> torch.Tensor:
plan = validate_plan(plan)
total_frames = int(plan["total_frames"])
trim_frames = int(plan["trim_frames"])
if images.shape[0] < total_frames:
raise ValueError(
f"decoded batch has {images.shape[0]} frames; expected at least {total_frames}"
)
return images[:total_frames][trim_frames:].contiguous()
def _slice_audio_for_timeline(
audio: dict[str, Any],
*,
trim_frames: int,
frame_start: int,
frame_stop: int,
) -> torch.Tensor:
"""Return one segment at its exact cumulative 24fps sample interval.
Segment-local rounding drifts by one sample on common 119-frame Continuum
chunks. Deriving every boundary from cumulative frame counts guarantees that
the assembled audio length is always ``round(total_frames / 24 * rate)``.
"""
waveform, sample_rate = validate_audio(audio)
trim_samples = int(round(float(trim_frames) / FPS * sample_rate))
target_start = int(round(float(frame_start) / FPS * sample_rate))
target_stop = int(round(float(frame_stop) / FPS * sample_rate))
wanted = target_stop - target_start
if wanted < 0:
raise ValueError("audio timeline interval is negative")
result = waveform[..., trim_samples : trim_samples + wanted]
if result.shape[-1] < wanted:
missing = wanted - int(result.shape[-1])
if result.shape[-1] > 0:
tail = result[..., -1:].expand(*result.shape[:-1], missing).clone()
result = torch.cat((result, tail), dim=-1)
else:
result = F.pad(result, (0, missing))
return result.contiguous()
def decode_sequence(
*,
entries: list[dict[str, Any]],
video_vae: Any,
audio_vae: Any,
diagnostics_full: bool,
) -> tuple[torch.Tensor, dict[str, Any], list[str]]:
"""Decode and assemble all chunks with bounded transient memory.
The final IMAGE tensor is preallocated once. Each raw VAE decode is copied
into its destination range and released before the next chunk, avoiding the
repeated full-sequence copies produced by progressive ``torch.cat``.
"""
if not entries:
raise ValueError("cannot decode an empty Continuum sequence")
checked = [validate_chunk_entry(entry) for entry in entries]
plans = [validate_plan(entry["plan"]) for entry in checked]
total_retained_frames = sum(int(plan["net_frames"]) for plan in plans)
if total_retained_frames < 1:
raise ValueError("Continuum sequence contains no retained frames")
image_buffer: torch.Tensor | None = None
audio_buffer: torch.Tensor | None = None
audio_rate: int | None = None
frame_cursor = 0
previous_segment_images: torch.Tensor | None = None
previous_segment_audio: dict[str, Any] | None = None
reports: list[str] = []
for index, (entry, plan) in enumerate(zip(checked, plans), start=1):
latent = entry_to_latent(entry)
raw_images = decode_video(video_vae, latent)
raw_audio = decode_audio(audio_vae, latent)
if diagnostics_full and previous_segment_images is not None:
metrics = overlap_diagnostics(
previous_images=previous_segment_images,
previous_audio=previous_segment_audio,
next_raw_images=raw_images,
next_raw_audio=raw_audio,
context_frames=int(plan["trim_frames"]),
)
reports.append(format_join_metrics(index, metrics))
images = _trim_images(raw_images, plan)
segment_frames = int(images.shape[0])
if segment_frames != int(plan["net_frames"]):
raise ValueError(
f"chunk {index} retained {segment_frames} frames; plan expects {plan['net_frames']}"
)
frame_stop = frame_cursor + segment_frames
waveform, sample_rate = validate_audio(raw_audio)
if audio_rate is None:
audio_rate = sample_rate
elif sample_rate != audio_rate:
raise ValueError(
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
)
segment_waveform = _slice_audio_for_timeline(
raw_audio,
trim_frames=int(plan["trim_frames"]),
frame_start=frame_cursor,
frame_stop=frame_stop,
)
if image_buffer is None:
image_buffer = torch.empty(
(total_retained_frames, *images.shape[1:]),
dtype=images.dtype,
device="cpu",
)
elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]):
raise ValueError(
f"decoded image geometry changed at chunk {index}: "
f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}"
)
image_buffer[frame_cursor:frame_stop].copy_(images)
if audio_buffer is None:
total_samples = int(round(total_retained_frames / FPS * sample_rate))
audio_buffer = torch.empty(
(*waveform.shape[:-1], total_samples),
dtype=waveform.dtype,
device="cpu",
)
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
raise ValueError("decoded audio batch/channel structure changed between chunks")
sample_start = int(round(frame_cursor / FPS * sample_rate))
sample_stop = int(round(frame_stop / FPS * sample_rate))
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
# Retain only what Full diagnostics needs for the next seam. Audio is
# small, while video is bounded to the next overlap instead of a full clip.
next_context = int(plans[index]["trim_frames"]) if index < len(plans) else 0
if next_context > 0:
previous_segment_images = images[-next_context:].contiguous()
previous_segment_audio = {
"waveform": segment_waveform,
"sample_rate": sample_rate,
}
else:
previous_segment_images = None
previous_segment_audio = None
reports.append(
f"decoded chunk {index}: {segment_frames} retained frames, "
f"cumulative {frame_stop}/{total_retained_frames}"
)
frame_cursor = frame_stop
del latent, raw_images, raw_audio, images, segment_waveform
if image_buffer is None or audio_buffer is None or audio_rate is None:
raise RuntimeError("Continuum decoder failed to allocate output buffers")
if frame_cursor != total_retained_frames:
raise RuntimeError(
f"Continuum decode cursor mismatch: {frame_cursor} != {total_retained_frames}"
)
audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate}
reports.append(
f"assembled {len(entries)} chunks into {total_retained_frames} frames with "
"cumulative sample-boundary alignment"
)
return image_buffer.contiguous(), audio, reports
def decode_sequence_with_seam(
*,
entries: list[dict[str, Any]],
video_vae: Any,
audio_vae: Any,
diagnostics_mode: str,
automatic: bool = False,
) -> tuple[torch.Tensor, dict[str, Any], list[str]]:
"""Decode with guarded V2.1 correction while preserving legacy output length."""
if not entries:
raise ValueError("cannot decode an empty Continuum sequence")
checked = [validate_chunk_entry(entry) for entry in entries]
plans = [validate_plan(entry["plan"]) for entry in checked]
total_retained_frames = sum(int(plan["net_frames"]) for plan in plans)
if total_retained_frames < 1:
raise ValueError("Continuum sequence contains no retained frames")
image_buffer: torch.Tensor | None = None
audio_buffer: torch.Tensor | None = None
audio_rate: int | None = None
frame_cursor = 0
reports: list[str] = []
for index, (entry, plan) in enumerate(zip(checked, plans), start=1):
latent = entry_to_latent(entry)
raw_images = decode_video(video_vae, latent)
raw_audio = decode_audio(audio_vae, latent)
waveform, sample_rate = validate_audio(raw_audio)
if audio_rate is None:
audio_rate = sample_rate
elif sample_rate != audio_rate:
raise ValueError(
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
)
legacy_frames = int(plan["net_frames"])
legacy_frame_stop = frame_cursor + legacy_frames
frame_start = frame_cursor
trim_frames = int(plan["trim_frames"])
decision: SeamDecision | None = None
video_patch: torch.Tensor | None = None
audio_patch: torch.Tensor | None = None
if index == 1:
images = _trim_images(raw_images, plan)
else:
try:
if image_buffer is None or audio_buffer is None:
raise RuntimeError("seam buffers are unavailable")
limited_raw_images = raw_images[: int(plan["total_frames"])]
images, decision, video_patch = correct_seam(
image_buffer[:frame_cursor],
limited_raw_images,
context_frames=trim_frames,
automatic=automatic,
)
frame_start = frame_cursor - int(decision.cut_rewind_frames)
trim_frames -= int(decision.cut_rewind_frames)
if frame_start < 0 or trim_frames < 0:
raise ValueError("adaptive cut produced a negative boundary")
if frame_start + int(images.shape[0]) != legacy_frame_stop:
raise ValueError("adaptive cut changed the cumulative frame boundary")
sample_start = int(round(frame_start / FPS * sample_rate))
cut_sample = int(round(trim_frames / FPS * sample_rate))
(
audio_patch,
audio_metrics,
crossfade_samples,
level_gain,
dc_bias,
) = correct_audio_seam(
audio_buffer[..., :sample_start],
waveform,
sample_rate=sample_rate,
cut_sample=cut_sample,
)
decision.audio_metrics = audio_metrics
decision.audio_crossfade_samples = crossfade_samples
decision.audio_level_gain = level_gain
decision.audio_dc_bias = dc_bias
except Exception as exc:
images = _trim_images(raw_images, plan)
frame_start = frame_cursor
trim_frames = int(plan["trim_frames"])
video_patch = None
audio_patch = None
decision = SeamDecision(
fallback_reason=f"{type(exc).__name__}: {exc}",
)
segment_frames = int(images.shape[0])
frame_stop = frame_start + segment_frames
if frame_stop != legacy_frame_stop:
raise ValueError(
f"chunk {index} corrected boundary ends at {frame_stop}; "
f"expected {legacy_frame_stop}"
)
segment_waveform = _slice_audio_for_timeline(
raw_audio,
trim_frames=trim_frames,
frame_start=frame_start,
frame_stop=frame_stop,
)
if image_buffer is None:
image_buffer = torch.empty(
(total_retained_frames, *images.shape[1:]),
dtype=images.dtype,
device="cpu",
)
elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]):
raise ValueError(
f"decoded image geometry changed at chunk {index}: "
f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}"
)
if video_patch is not None and video_patch.shape[0] > 0:
patch_start = frame_start - int(video_patch.shape[0])
if patch_start < 0:
raise ValueError("video seam patch starts before the output")
image_buffer[patch_start:frame_start].copy_(video_patch)
image_buffer[frame_start:frame_stop].copy_(images)
if audio_buffer is None:
total_samples = int(round(total_retained_frames / FPS * sample_rate))
audio_buffer = torch.empty(
(*waveform.shape[:-1], total_samples),
dtype=waveform.dtype,
device="cpu",
)
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
raise ValueError("decoded audio batch/channel structure changed between chunks")
sample_start = int(round(frame_start / FPS * sample_rate))
sample_stop = int(round(frame_stop / FPS * sample_rate))
if audio_patch is not None and audio_patch.shape[-1] > 0:
patch_start = sample_start - int(audio_patch.shape[-1])
if patch_start < 0:
raise ValueError("audio seam patch starts before the output")
audio_buffer[..., patch_start:sample_start].copy_(audio_patch)
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
if decision is not None and diagnostics_mode != DIAGNOSTICS_OFF:
reports.append(
format_seam_decision(
index,
decision,
full=diagnostics_mode == DIAGNOSTICS_FULL,
)
)
reports.append(
f"decoded chunk {index}: {segment_frames} retained/corrected frames, "
f"cumulative {frame_stop}/{total_retained_frames}"
)
frame_cursor = frame_stop
del latent, raw_images, raw_audio, images, segment_waveform
if image_buffer is None or audio_buffer is None or audio_rate is None:
raise RuntimeError("Continuum seam decoder failed to allocate output buffers")
if frame_cursor != total_retained_frames:
raise RuntimeError(
f"Continuum seam decode cursor mismatch: {frame_cursor} != {total_retained_frames}"
)
audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate}
reports.append(
f"assembled {len(entries)} chunks into {total_retained_frames} frames with "
f"V2.1 {'Auto' if automatic else 'Basic'} seam correction and "
"cumulative sample-boundary alignment"
)
return image_buffer.contiguous(), audio, reports
def enforce_total_frames(
images: torch.Tensor,
audio: dict[str, Any],
*,
target_frames: int,
preserve_final_frame: bool = False,
) -> tuple[torch.Tensor, dict[str, Any], str]:
target_frames = int(target_frames)
if target_frames < 1:
raise ValueError("target_frames must be positive")
current = int(images.shape[0])
adjustment = target_frames - current
anchor_preserved = bool(preserve_final_frame and adjustment < 0 and target_frames >= 2)
if adjustment < 0:
if anchor_preserved:
# H3's valid temporal grid rarely lands on an exact whole-second total.
# When a user supplied a final FL2VA frame, trimming only from the end
# would discard the very anchor they requested. Remove the small excess
# immediately before the final decoded frame instead.
images = torch.cat((images[: target_frames - 1], images[-1:]), dim=0).contiguous()
else:
images = images[:target_frames].contiguous()
elif adjustment > 0:
if current == 0:
raise ValueError("cannot pad an empty image sequence")
pad = images[-1:].expand(adjustment, *images.shape[1:]).clone()
images = torch.cat((images, pad), dim=0).contiguous()
waveform = audio["waveform"]
sample_rate = int(audio["sample_rate"])
original_samples = int(waveform.shape[-1])
target_samples = int(round(target_frames / FPS * sample_rate))
delta_samples = target_samples - original_samples
if delta_samples < 0:
if anchor_preserved and target_samples > 1:
# Preserve the final video-frame interval of audio as well. A short
# 5 ms interpolation at the deletion boundary suppresses a hard click
# without changing the exact output length.
tail_samples = min(
max(1, int(round(sample_rate / FPS))),
target_samples,
original_samples,
)
head_samples = target_samples - tail_samples
head = waveform[..., :head_samples]
tail = waveform[..., -tail_samples:].clone()
fade = min(int(round(sample_rate * 0.005)), head_samples, tail_samples)
if fade > 0:
start = head[..., -1:].to(dtype=tail.dtype)
alpha = torch.linspace(
0.0, 1.0, fade, dtype=tail.dtype, device=tail.device
).reshape(*([1] * (tail.ndim - 1)), fade)
tail[..., :fade] = start * (1.0 - alpha) + tail[..., :fade] * alpha
waveform = torch.cat((head, tail), dim=-1).contiguous()
else:
waveform = waveform[..., :target_samples].contiguous()
elif delta_samples > 0:
# Repeat the final sample instead of introducing a zero click.
if waveform.shape[-1] == 0:
waveform = F.pad(waveform, (0, delta_samples))
else:
tail = waveform[..., -1:].expand(*waveform.shape[:-1], delta_samples).clone()
waveform = torch.cat((waveform, tail), dim=-1).contiguous()
audio = {**audio, "waveform": waveform, "sample_rate": sample_rate}
mode = "final anchor preserved; pre-tail trim" if anchor_preserved else "tail trim/pad"
report = (
f"Exact-duration adjustment: frames {current}->{target_frames} ({adjustment:+d}), "
f"audio samples correction {delta_samples:+d}; {mode}."
)
return images, audio, report