463 lines
20 KiB
Python
463 lines
20 KiB
Python
"""Deferred VAE decoding, overlap trimming, assembly, and exact-duration correction."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
from ..constants import DIAGNOSTICS_FULL, DIAGNOSTICS_OFF, FPS
|
|
from ..media import validate_audio
|
|
from ..state import validate_plan
|
|
from .diagnostics import format_join_metrics, format_seam_decision, overlap_diagnostics
|
|
from .seam_guard import correct_audio_seam, correct_seam
|
|
from .seam_types import SeamDecision
|
|
from .session import entry_to_latent, validate_chunk_entry
|
|
|
|
|
|
def decode_video(video_vae: Any, latent: dict[str, Any]) -> torch.Tensor:
|
|
samples = latent["samples"]
|
|
if hasattr(samples, "unbind"):
|
|
video = list(samples.unbind())[0]
|
|
else:
|
|
video = samples[0]
|
|
images = video_vae.decode(video)
|
|
if images.ndim == 5:
|
|
images = images.reshape(-1, images.shape[-3], images.shape[-2], images.shape[-1])
|
|
if images.ndim != 4:
|
|
raise ValueError(f"video VAE returned unexpected shape {tuple(images.shape)}")
|
|
# Long sequences must not leave decoded frames resident on the GPU. ComfyUI
|
|
# IMAGE tensors are conventionally CPU float tensors, so transfer each chunk
|
|
# immediately and release its raw decode before moving to the next chunk.
|
|
return images.detach().to("cpu").contiguous()
|
|
|
|
|
|
def decode_audio(audio_vae: Any, latent: dict[str, Any]) -> dict[str, Any]:
|
|
samples = latent["samples"]
|
|
if hasattr(samples, "unbind"):
|
|
audio_latent = list(samples.unbind())[-1]
|
|
else:
|
|
audio_latent = samples[-1]
|
|
audio = audio_vae.decode(audio_latent).movedim(-1, 1)
|
|
std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0
|
|
std[std < 1.0] = 1.0
|
|
audio = (audio / std).detach().to("cpu").contiguous()
|
|
sample_rate = getattr(
|
|
audio_vae,
|
|
"audio_sample_rate_output",
|
|
getattr(audio_vae, "audio_sample_rate", 32000),
|
|
)
|
|
return {"waveform": audio, "sample_rate": int(sample_rate)}
|
|
|
|
|
|
def _trim_images(images: torch.Tensor, plan: dict[str, Any]) -> torch.Tensor:
|
|
plan = validate_plan(plan)
|
|
total_frames = int(plan["total_frames"])
|
|
trim_frames = int(plan["trim_frames"])
|
|
if images.shape[0] < total_frames:
|
|
raise ValueError(
|
|
f"decoded batch has {images.shape[0]} frames; expected at least {total_frames}"
|
|
)
|
|
return images[:total_frames][trim_frames:].contiguous()
|
|
|
|
|
|
def _slice_audio_for_timeline(
|
|
audio: dict[str, Any],
|
|
*,
|
|
trim_frames: int,
|
|
frame_start: int,
|
|
frame_stop: int,
|
|
) -> torch.Tensor:
|
|
"""Return one segment at its exact cumulative 24fps sample interval.
|
|
|
|
Segment-local rounding drifts by one sample on common 119-frame Continuum
|
|
chunks. Deriving every boundary from cumulative frame counts guarantees that
|
|
the assembled audio length is always ``round(total_frames / 24 * rate)``.
|
|
"""
|
|
|
|
waveform, sample_rate = validate_audio(audio)
|
|
trim_samples = int(round(float(trim_frames) / FPS * sample_rate))
|
|
target_start = int(round(float(frame_start) / FPS * sample_rate))
|
|
target_stop = int(round(float(frame_stop) / FPS * sample_rate))
|
|
wanted = target_stop - target_start
|
|
if wanted < 0:
|
|
raise ValueError("audio timeline interval is negative")
|
|
result = waveform[..., trim_samples : trim_samples + wanted]
|
|
if result.shape[-1] < wanted:
|
|
missing = wanted - int(result.shape[-1])
|
|
if result.shape[-1] > 0:
|
|
tail = result[..., -1:].expand(*result.shape[:-1], missing).clone()
|
|
result = torch.cat((result, tail), dim=-1)
|
|
else:
|
|
result = F.pad(result, (0, missing))
|
|
return result.contiguous()
|
|
|
|
|
|
def decode_sequence(
|
|
*,
|
|
entries: list[dict[str, Any]],
|
|
video_vae: Any,
|
|
audio_vae: Any,
|
|
diagnostics_full: bool,
|
|
) -> tuple[torch.Tensor, dict[str, Any], list[str]]:
|
|
"""Decode and assemble all chunks with bounded transient memory.
|
|
|
|
The final IMAGE tensor is preallocated once. Each raw VAE decode is copied
|
|
into its destination range and released before the next chunk, avoiding the
|
|
repeated full-sequence copies produced by progressive ``torch.cat``.
|
|
"""
|
|
|
|
if not entries:
|
|
raise ValueError("cannot decode an empty Continuum sequence")
|
|
checked = [validate_chunk_entry(entry) for entry in entries]
|
|
plans = [validate_plan(entry["plan"]) for entry in checked]
|
|
total_retained_frames = sum(int(plan["net_frames"]) for plan in plans)
|
|
if total_retained_frames < 1:
|
|
raise ValueError("Continuum sequence contains no retained frames")
|
|
|
|
image_buffer: torch.Tensor | None = None
|
|
audio_buffer: torch.Tensor | None = None
|
|
audio_rate: int | None = None
|
|
frame_cursor = 0
|
|
previous_segment_images: torch.Tensor | None = None
|
|
previous_segment_audio: dict[str, Any] | None = None
|
|
reports: list[str] = []
|
|
|
|
for index, (entry, plan) in enumerate(zip(checked, plans), start=1):
|
|
latent = entry_to_latent(entry)
|
|
raw_images = decode_video(video_vae, latent)
|
|
raw_audio = decode_audio(audio_vae, latent)
|
|
|
|
if diagnostics_full and previous_segment_images is not None:
|
|
metrics = overlap_diagnostics(
|
|
previous_images=previous_segment_images,
|
|
previous_audio=previous_segment_audio,
|
|
next_raw_images=raw_images,
|
|
next_raw_audio=raw_audio,
|
|
context_frames=int(plan["trim_frames"]),
|
|
)
|
|
reports.append(format_join_metrics(index, metrics))
|
|
|
|
images = _trim_images(raw_images, plan)
|
|
segment_frames = int(images.shape[0])
|
|
if segment_frames != int(plan["net_frames"]):
|
|
raise ValueError(
|
|
f"chunk {index} retained {segment_frames} frames; plan expects {plan['net_frames']}"
|
|
)
|
|
frame_stop = frame_cursor + segment_frames
|
|
|
|
waveform, sample_rate = validate_audio(raw_audio)
|
|
if audio_rate is None:
|
|
audio_rate = sample_rate
|
|
elif sample_rate != audio_rate:
|
|
raise ValueError(
|
|
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
|
|
)
|
|
segment_waveform = _slice_audio_for_timeline(
|
|
raw_audio,
|
|
trim_frames=int(plan["trim_frames"]),
|
|
frame_start=frame_cursor,
|
|
frame_stop=frame_stop,
|
|
)
|
|
|
|
if image_buffer is None:
|
|
image_buffer = torch.empty(
|
|
(total_retained_frames, *images.shape[1:]),
|
|
dtype=images.dtype,
|
|
device="cpu",
|
|
)
|
|
elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]):
|
|
raise ValueError(
|
|
f"decoded image geometry changed at chunk {index}: "
|
|
f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}"
|
|
)
|
|
image_buffer[frame_cursor:frame_stop].copy_(images)
|
|
|
|
if audio_buffer is None:
|
|
total_samples = int(round(total_retained_frames / FPS * sample_rate))
|
|
audio_buffer = torch.empty(
|
|
(*waveform.shape[:-1], total_samples),
|
|
dtype=waveform.dtype,
|
|
device="cpu",
|
|
)
|
|
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
|
|
raise ValueError("decoded audio batch/channel structure changed between chunks")
|
|
sample_start = int(round(frame_cursor / FPS * sample_rate))
|
|
sample_stop = int(round(frame_stop / FPS * sample_rate))
|
|
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
|
|
|
|
# Retain only what Full diagnostics needs for the next seam. Audio is
|
|
# small, while video is bounded to the next overlap instead of a full clip.
|
|
next_context = int(plans[index]["trim_frames"]) if index < len(plans) else 0
|
|
if next_context > 0:
|
|
previous_segment_images = images[-next_context:].contiguous()
|
|
previous_segment_audio = {
|
|
"waveform": segment_waveform,
|
|
"sample_rate": sample_rate,
|
|
}
|
|
else:
|
|
previous_segment_images = None
|
|
previous_segment_audio = None
|
|
|
|
reports.append(
|
|
f"decoded chunk {index}: {segment_frames} retained frames, "
|
|
f"cumulative {frame_stop}/{total_retained_frames}"
|
|
)
|
|
frame_cursor = frame_stop
|
|
del latent, raw_images, raw_audio, images, segment_waveform
|
|
|
|
if image_buffer is None or audio_buffer is None or audio_rate is None:
|
|
raise RuntimeError("Continuum decoder failed to allocate output buffers")
|
|
if frame_cursor != total_retained_frames:
|
|
raise RuntimeError(
|
|
f"Continuum decode cursor mismatch: {frame_cursor} != {total_retained_frames}"
|
|
)
|
|
audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate}
|
|
reports.append(
|
|
f"assembled {len(entries)} chunks into {total_retained_frames} frames with "
|
|
"cumulative sample-boundary alignment"
|
|
)
|
|
return image_buffer.contiguous(), audio, reports
|
|
|
|
|
|
def decode_sequence_with_seam(
|
|
*,
|
|
entries: list[dict[str, Any]],
|
|
video_vae: Any,
|
|
audio_vae: Any,
|
|
diagnostics_mode: str,
|
|
) -> tuple[torch.Tensor, dict[str, Any], list[str]]:
|
|
"""Decode with V2.1 Basic correction while preserving legacy output length."""
|
|
|
|
if not entries:
|
|
raise ValueError("cannot decode an empty Continuum sequence")
|
|
checked = [validate_chunk_entry(entry) for entry in entries]
|
|
plans = [validate_plan(entry["plan"]) for entry in checked]
|
|
total_retained_frames = sum(int(plan["net_frames"]) for plan in plans)
|
|
if total_retained_frames < 1:
|
|
raise ValueError("Continuum sequence contains no retained frames")
|
|
|
|
image_buffer: torch.Tensor | None = None
|
|
audio_buffer: torch.Tensor | None = None
|
|
audio_rate: int | None = None
|
|
frame_cursor = 0
|
|
reports: list[str] = []
|
|
|
|
for index, (entry, plan) in enumerate(zip(checked, plans), start=1):
|
|
latent = entry_to_latent(entry)
|
|
raw_images = decode_video(video_vae, latent)
|
|
raw_audio = decode_audio(audio_vae, latent)
|
|
waveform, sample_rate = validate_audio(raw_audio)
|
|
if audio_rate is None:
|
|
audio_rate = sample_rate
|
|
elif sample_rate != audio_rate:
|
|
raise ValueError(
|
|
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
|
|
)
|
|
|
|
legacy_frames = int(plan["net_frames"])
|
|
legacy_frame_stop = frame_cursor + legacy_frames
|
|
frame_start = frame_cursor
|
|
trim_frames = int(plan["trim_frames"])
|
|
decision: SeamDecision | None = None
|
|
video_patch: torch.Tensor | None = None
|
|
audio_patch: torch.Tensor | None = None
|
|
|
|
if index == 1:
|
|
images = _trim_images(raw_images, plan)
|
|
else:
|
|
try:
|
|
if image_buffer is None or audio_buffer is None:
|
|
raise RuntimeError("seam buffers are unavailable")
|
|
limited_raw_images = raw_images[: int(plan["total_frames"])]
|
|
images, decision, video_patch = correct_seam(
|
|
image_buffer[:frame_cursor],
|
|
limited_raw_images,
|
|
context_frames=trim_frames,
|
|
)
|
|
frame_start = frame_cursor - int(decision.cut_rewind_frames)
|
|
trim_frames -= int(decision.cut_rewind_frames)
|
|
if frame_start < 0 or trim_frames < 0:
|
|
raise ValueError("adaptive cut produced a negative boundary")
|
|
if frame_start + int(images.shape[0]) != legacy_frame_stop:
|
|
raise ValueError("adaptive cut changed the cumulative frame boundary")
|
|
|
|
sample_start = int(round(frame_start / FPS * sample_rate))
|
|
cut_sample = int(round(trim_frames / FPS * sample_rate))
|
|
(
|
|
audio_patch,
|
|
audio_metrics,
|
|
crossfade_samples,
|
|
level_gain,
|
|
dc_bias,
|
|
) = correct_audio_seam(
|
|
audio_buffer[..., :sample_start],
|
|
waveform,
|
|
sample_rate=sample_rate,
|
|
cut_sample=cut_sample,
|
|
)
|
|
decision.audio_metrics = audio_metrics
|
|
decision.audio_crossfade_samples = crossfade_samples
|
|
decision.audio_level_gain = level_gain
|
|
decision.audio_dc_bias = dc_bias
|
|
except Exception as exc:
|
|
images = _trim_images(raw_images, plan)
|
|
frame_start = frame_cursor
|
|
trim_frames = int(plan["trim_frames"])
|
|
video_patch = None
|
|
audio_patch = None
|
|
decision = SeamDecision(
|
|
fallback_reason=f"{type(exc).__name__}: {exc}",
|
|
)
|
|
|
|
segment_frames = int(images.shape[0])
|
|
frame_stop = frame_start + segment_frames
|
|
if frame_stop != legacy_frame_stop:
|
|
raise ValueError(
|
|
f"chunk {index} corrected boundary ends at {frame_stop}; "
|
|
f"expected {legacy_frame_stop}"
|
|
)
|
|
segment_waveform = _slice_audio_for_timeline(
|
|
raw_audio,
|
|
trim_frames=trim_frames,
|
|
frame_start=frame_start,
|
|
frame_stop=frame_stop,
|
|
)
|
|
|
|
if image_buffer is None:
|
|
image_buffer = torch.empty(
|
|
(total_retained_frames, *images.shape[1:]),
|
|
dtype=images.dtype,
|
|
device="cpu",
|
|
)
|
|
elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]):
|
|
raise ValueError(
|
|
f"decoded image geometry changed at chunk {index}: "
|
|
f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}"
|
|
)
|
|
if video_patch is not None and video_patch.shape[0] > 0:
|
|
patch_start = frame_start - int(video_patch.shape[0])
|
|
if patch_start < 0:
|
|
raise ValueError("video seam patch starts before the output")
|
|
image_buffer[patch_start:frame_start].copy_(video_patch)
|
|
image_buffer[frame_start:frame_stop].copy_(images)
|
|
|
|
if audio_buffer is None:
|
|
total_samples = int(round(total_retained_frames / FPS * sample_rate))
|
|
audio_buffer = torch.empty(
|
|
(*waveform.shape[:-1], total_samples),
|
|
dtype=waveform.dtype,
|
|
device="cpu",
|
|
)
|
|
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
|
|
raise ValueError("decoded audio batch/channel structure changed between chunks")
|
|
sample_start = int(round(frame_start / FPS * sample_rate))
|
|
sample_stop = int(round(frame_stop / FPS * sample_rate))
|
|
if audio_patch is not None and audio_patch.shape[-1] > 0:
|
|
patch_start = sample_start - int(audio_patch.shape[-1])
|
|
if patch_start < 0:
|
|
raise ValueError("audio seam patch starts before the output")
|
|
audio_buffer[..., patch_start:sample_start].copy_(audio_patch)
|
|
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
|
|
|
|
if decision is not None and diagnostics_mode != DIAGNOSTICS_OFF:
|
|
reports.append(
|
|
format_seam_decision(
|
|
index,
|
|
decision,
|
|
full=diagnostics_mode == DIAGNOSTICS_FULL,
|
|
)
|
|
)
|
|
reports.append(
|
|
f"decoded chunk {index}: {segment_frames} retained/corrected frames, "
|
|
f"cumulative {frame_stop}/{total_retained_frames}"
|
|
)
|
|
frame_cursor = frame_stop
|
|
del latent, raw_images, raw_audio, images, segment_waveform
|
|
|
|
if image_buffer is None or audio_buffer is None or audio_rate is None:
|
|
raise RuntimeError("Continuum seam decoder failed to allocate output buffers")
|
|
if frame_cursor != total_retained_frames:
|
|
raise RuntimeError(
|
|
f"Continuum seam decode cursor mismatch: {frame_cursor} != {total_retained_frames}"
|
|
)
|
|
audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate}
|
|
reports.append(
|
|
f"assembled {len(entries)} chunks into {total_retained_frames} frames with "
|
|
"V2.1 Basic seam correction and cumulative sample-boundary alignment"
|
|
)
|
|
return image_buffer.contiguous(), audio, reports
|
|
|
|
|
|
def enforce_total_frames(
|
|
images: torch.Tensor,
|
|
audio: dict[str, Any],
|
|
*,
|
|
target_frames: int,
|
|
preserve_final_frame: bool = False,
|
|
) -> tuple[torch.Tensor, dict[str, Any], str]:
|
|
target_frames = int(target_frames)
|
|
if target_frames < 1:
|
|
raise ValueError("target_frames must be positive")
|
|
current = int(images.shape[0])
|
|
adjustment = target_frames - current
|
|
anchor_preserved = bool(preserve_final_frame and adjustment < 0 and target_frames >= 2)
|
|
if adjustment < 0:
|
|
if anchor_preserved:
|
|
# H3's valid temporal grid rarely lands on an exact whole-second total.
|
|
# When a user supplied a final FL2VA frame, trimming only from the end
|
|
# would discard the very anchor they requested. Remove the small excess
|
|
# immediately before the final decoded frame instead.
|
|
images = torch.cat((images[: target_frames - 1], images[-1:]), dim=0).contiguous()
|
|
else:
|
|
images = images[:target_frames].contiguous()
|
|
elif adjustment > 0:
|
|
if current == 0:
|
|
raise ValueError("cannot pad an empty image sequence")
|
|
pad = images[-1:].expand(adjustment, *images.shape[1:]).clone()
|
|
images = torch.cat((images, pad), dim=0).contiguous()
|
|
|
|
waveform = audio["waveform"]
|
|
sample_rate = int(audio["sample_rate"])
|
|
original_samples = int(waveform.shape[-1])
|
|
target_samples = int(round(target_frames / FPS * sample_rate))
|
|
delta_samples = target_samples - original_samples
|
|
if delta_samples < 0:
|
|
if anchor_preserved and target_samples > 1:
|
|
# Preserve the final video-frame interval of audio as well. A short
|
|
# 5 ms interpolation at the deletion boundary suppresses a hard click
|
|
# without changing the exact output length.
|
|
tail_samples = min(
|
|
max(1, int(round(sample_rate / FPS))),
|
|
target_samples,
|
|
original_samples,
|
|
)
|
|
head_samples = target_samples - tail_samples
|
|
head = waveform[..., :head_samples]
|
|
tail = waveform[..., -tail_samples:].clone()
|
|
fade = min(int(round(sample_rate * 0.005)), head_samples, tail_samples)
|
|
if fade > 0:
|
|
start = head[..., -1:].to(dtype=tail.dtype)
|
|
alpha = torch.linspace(
|
|
0.0, 1.0, fade, dtype=tail.dtype, device=tail.device
|
|
).reshape(*([1] * (tail.ndim - 1)), fade)
|
|
tail[..., :fade] = start * (1.0 - alpha) + tail[..., :fade] * alpha
|
|
waveform = torch.cat((head, tail), dim=-1).contiguous()
|
|
else:
|
|
waveform = waveform[..., :target_samples].contiguous()
|
|
elif delta_samples > 0:
|
|
# Repeat the final sample instead of introducing a zero click.
|
|
if waveform.shape[-1] == 0:
|
|
waveform = F.pad(waveform, (0, delta_samples))
|
|
else:
|
|
tail = waveform[..., -1:].expand(*waveform.shape[:-1], delta_samples).clone()
|
|
waveform = torch.cat((waveform, tail), dim=-1).contiguous()
|
|
audio = {**audio, "waveform": waveform, "sample_rate": sample_rate}
|
|
mode = "final anchor preserved; pre-tail trim" if anchor_preserved else "tail trim/pad"
|
|
report = (
|
|
f"Exact-duration adjustment: frames {current}->{target_frames} ({adjustment:+d}), "
|
|
f"audio samples correction {delta_samples:+d}; {mode}."
|
|
)
|
|
return images, audio, report
|