430 lines
16 KiB
Python
430 lines
16 KiB
Python
"""Decoded chunk assembly for latent-first H3 Continuum V3."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
import time
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..media import validate_audio
|
|
from ..v2.decoder import _slice_audio_for_timeline, enforce_total_frames
|
|
from ..constants import (
|
|
DIAGNOSTICS_FULL,
|
|
DIAGNOSTICS_OFF,
|
|
DIAGNOSTICS_OPTIONS,
|
|
normalize_diagnostics_mode,
|
|
)
|
|
from ..v2.seam_guard import correct_audio_seam
|
|
from .plan import FPS, validate_assembly_plan
|
|
|
|
AUDIO_SEAM_OFF = "Off"
|
|
AUDIO_SEAM_AUTO = "Auto"
|
|
AUDIO_SEAM_OPTIONS = (AUDIO_SEAM_OFF, AUDIO_SEAM_AUTO)
|
|
VIDEO_SEAM_OFF = "Off"
|
|
VIDEO_SEAM_ANALYZE = "Analyze Only"
|
|
VIDEO_SEAM_AUTO = "Auto"
|
|
VIDEO_SEAM_AUTO_2 = "Auto 2"
|
|
VIDEO_SEAM_ANALYSIS_OPTIONS = (
|
|
VIDEO_SEAM_AUTO,
|
|
VIDEO_SEAM_AUTO_2,
|
|
VIDEO_SEAM_ANALYZE,
|
|
VIDEO_SEAM_OFF,
|
|
)
|
|
|
|
|
|
def _singleton(value: Any, name: str) -> Any:
|
|
while isinstance(value, list):
|
|
if len(value) != 1:
|
|
raise ValueError(f"{name} must contain exactly one value")
|
|
value = value[0]
|
|
return value
|
|
|
|
|
|
def _chunk_list(value: Any) -> list[Any]:
|
|
if isinstance(value, list):
|
|
if len(value) == 1 and isinstance(value[0], list):
|
|
return list(value[0])
|
|
return list(value)
|
|
return [value]
|
|
|
|
|
|
def _format_elapsed(seconds: float) -> str:
|
|
total = max(0.0, float(seconds))
|
|
hours, remainder = divmod(total, 3600.0)
|
|
minutes, seconds = divmod(remainder, 60.0)
|
|
return f"{int(hours):02d}:{int(minutes):02d}:{seconds:05.2f}"
|
|
|
|
|
|
def assemble_decoded_chunks(
|
|
*,
|
|
images: list[Any],
|
|
audio: list[Any],
|
|
assembly_plan: dict[str, Any],
|
|
exact_total_duration: bool,
|
|
audio_seam: str,
|
|
diagnostics: str,
|
|
):
|
|
plan = validate_assembly_plan(assembly_plan)
|
|
diagnostics_mode = normalize_diagnostics_mode(diagnostics)
|
|
if audio_seam not in AUDIO_SEAM_OPTIONS:
|
|
raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}")
|
|
|
|
chunk_plans = list(plan["chunks"])
|
|
if len(images) != len(chunk_plans) or len(audio) != len(chunk_plans):
|
|
raise ValueError(
|
|
"decoded chunk count does not match the V3 assembly plan: "
|
|
f"images={len(images)}, audio={len(audio)}, plan={len(chunk_plans)}"
|
|
)
|
|
|
|
total_retained_frames = sum(int(item["net_frames"]) for item in chunk_plans)
|
|
image_buffer = None
|
|
audio_buffer = None
|
|
audio_rate = None
|
|
frame_cursor = 0
|
|
reports = [
|
|
"H3 Continuum Assemble V3",
|
|
f"Audio Seam: {audio_seam}. Video seam correction is disabled.",
|
|
]
|
|
|
|
for index, (raw_images, raw_audio, chunk) in enumerate(
|
|
zip(images, audio, chunk_plans), start=1
|
|
):
|
|
if not torch.is_tensor(raw_images) or raw_images.ndim != 4:
|
|
raise ValueError(f"decoded image chunk {index} must be [frames,H,W,C]")
|
|
raw_images = raw_images.detach().to("cpu")
|
|
total_frames = int(chunk["total_frames"])
|
|
trim_frames = int(chunk["trim_frames"])
|
|
net_frames = int(chunk["net_frames"])
|
|
if int(raw_images.shape[0]) < total_frames:
|
|
raise ValueError(
|
|
f"decoded image chunk {index} has {raw_images.shape[0]} frames; "
|
|
f"expected at least {total_frames}"
|
|
)
|
|
segment_images = raw_images[:total_frames][trim_frames:].contiguous()
|
|
if int(segment_images.shape[0]) != net_frames:
|
|
raise ValueError(
|
|
f"decoded image chunk {index} retained {segment_images.shape[0]} frames; "
|
|
f"plan expects {net_frames}"
|
|
)
|
|
|
|
waveform, sample_rate = validate_audio(raw_audio)
|
|
waveform = waveform.detach().to("cpu")
|
|
sample_rate = int(sample_rate)
|
|
raw_audio_cpu = {"waveform": waveform, "sample_rate": sample_rate}
|
|
if audio_rate is None:
|
|
audio_rate = sample_rate
|
|
elif sample_rate != audio_rate:
|
|
raise ValueError(
|
|
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
|
|
)
|
|
|
|
frame_stop = frame_cursor + net_frames
|
|
segment_waveform = _slice_audio_for_timeline(
|
|
raw_audio_cpu,
|
|
trim_frames=trim_frames,
|
|
frame_start=frame_cursor,
|
|
frame_stop=frame_stop,
|
|
)
|
|
|
|
if image_buffer is None:
|
|
image_buffer = torch.empty(
|
|
(total_retained_frames, *segment_images.shape[1:]),
|
|
dtype=segment_images.dtype,
|
|
device="cpu",
|
|
)
|
|
elif tuple(segment_images.shape[1:]) != tuple(image_buffer.shape[1:]):
|
|
raise ValueError(f"decoded image geometry changed at chunk {index}")
|
|
image_buffer[frame_cursor:frame_stop].copy_(segment_images)
|
|
|
|
if audio_buffer is None:
|
|
audio_buffer = torch.empty(
|
|
(
|
|
*waveform.shape[:-1],
|
|
int(round(total_retained_frames / FPS * sample_rate)),
|
|
),
|
|
dtype=waveform.dtype,
|
|
device="cpu",
|
|
)
|
|
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
|
|
raise ValueError(
|
|
"decoded audio batch/channel structure changed between chunks"
|
|
)
|
|
|
|
sample_start = int(round(frame_cursor / FPS * sample_rate))
|
|
sample_stop = int(round(frame_stop / FPS * sample_rate))
|
|
seam_report = None
|
|
if index > 1 and audio_seam == AUDIO_SEAM_AUTO:
|
|
try:
|
|
cut_sample = int(round(trim_frames / FPS * sample_rate))
|
|
patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam(
|
|
audio_buffer[..., :sample_start],
|
|
waveform,
|
|
sample_rate=sample_rate,
|
|
cut_sample=cut_sample,
|
|
)
|
|
if patch is not None and int(patch.shape[-1]) > 0:
|
|
patch_start = sample_start - int(patch.shape[-1])
|
|
if patch_start < 0:
|
|
raise ValueError("audio seam patch starts before the output")
|
|
audio_buffer[..., patch_start:sample_start].copy_(patch)
|
|
seam_report = (
|
|
f"audio seam {index-1}->{index}: "
|
|
f"corr={metrics.correlation_before:.4f}->{metrics.correlation_after:.4f}, "
|
|
f"jump={metrics.boundary_jump_before:.6f}->{metrics.boundary_jump_after:.6f}, "
|
|
f"fade={fade_samples} samples"
|
|
)
|
|
if diagnostics_mode == DIAGNOSTICS_FULL:
|
|
seam_report += (
|
|
f", offset={metrics.offset_samples:+d}, "
|
|
f"gain={level_gain:.4f}, dc={dc_bias:+.6f}"
|
|
)
|
|
except Exception as exc:
|
|
seam_report = (
|
|
f"audio seam {index-1}->{index}: fallback to native boundary "
|
|
f"({type(exc).__name__}: {exc})"
|
|
)
|
|
|
|
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
|
|
if diagnostics_mode != DIAGNOSTICS_OFF:
|
|
reports.append(
|
|
f"assembled decoded chunk {index}: {net_frames} retained frames, "
|
|
f"cumulative {frame_stop}/{total_retained_frames}"
|
|
)
|
|
if seam_report:
|
|
reports.append(seam_report)
|
|
frame_cursor = frame_stop
|
|
|
|
if image_buffer is None or audio_buffer is None or audio_rate is None:
|
|
raise RuntimeError("V3 assembler failed to allocate output buffers")
|
|
if frame_cursor != total_retained_frames:
|
|
raise RuntimeError(
|
|
f"V3 assembly cursor mismatch: {frame_cursor} != {total_retained_frames}"
|
|
)
|
|
|
|
result_audio = {
|
|
"waveform": audio_buffer.contiguous(),
|
|
"sample_rate": audio_rate,
|
|
}
|
|
duration_report = ""
|
|
if exact_total_duration:
|
|
image_buffer, result_audio, duration_report = enforce_total_frames(
|
|
image_buffer.contiguous(),
|
|
result_audio,
|
|
target_frames=int(plan["target_frames"]),
|
|
preserve_final_frame=bool(plan.get("preserve_final_frame")),
|
|
)
|
|
|
|
reports.append(
|
|
f"Assembled {len(chunk_plans)} externally decoded chunks into "
|
|
f"{image_buffer.shape[0]} frames with cumulative sample-boundary alignment."
|
|
)
|
|
if duration_report:
|
|
reports.append(duration_report)
|
|
runtime_started_at = plan.get("_runtime_started_at")
|
|
if isinstance(runtime_started_at, (int, float)):
|
|
elapsed = time.perf_counter() - float(runtime_started_at)
|
|
if math.isfinite(elapsed) and elapsed >= 0.0:
|
|
reports.append(f"Total workflow elapsed: {_format_elapsed(elapsed)}")
|
|
return image_buffer.contiguous(), result_audio, "\n".join(reports)
|
|
|
|
|
|
class H3ContinuumAssembleV3:
|
|
DESCRIPTION = (
|
|
"Assemble full AV chunks decoded by ComfyUI Core. Trims decoded context, "
|
|
"aligns audio on cumulative frame boundaries, and optionally applies Audio Seam Auto."
|
|
)
|
|
SEARCH_ALIASES = ["H3 latent assemble", "H3 external VAE decode"]
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {
|
|
"required": {
|
|
"images": ("IMAGE",),
|
|
"audio": ("AUDIO",),
|
|
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
|
"exact_total_duration": (
|
|
"BOOLEAN",
|
|
{
|
|
"default": True,
|
|
"tooltip": "Adjust the final output to the exact requested total duration.",
|
|
},
|
|
),
|
|
"audio_seam": (
|
|
AUDIO_SEAM_OPTIONS,
|
|
{
|
|
"default": AUDIO_SEAM_AUTO,
|
|
"display_name": "Audio Seam",
|
|
"tooltip": (
|
|
"Auto corrects only decoded audio boundaries; "
|
|
"video frames are never altered."
|
|
),
|
|
},
|
|
),
|
|
"diagnostics": (
|
|
DIAGNOSTICS_OPTIONS,
|
|
{
|
|
"default": DIAGNOSTICS_OPTIONS[0],
|
|
"display_name": "Report Detail",
|
|
},
|
|
),
|
|
}
|
|
}
|
|
|
|
INPUT_IS_LIST = True
|
|
RETURN_TYPES = ("IMAGE", "AUDIO", "STRING")
|
|
RETURN_NAMES = ("images", "audio", "report")
|
|
FUNCTION = "assemble"
|
|
CATEGORY = "MiniMax H3/Continuum"
|
|
|
|
def assemble(
|
|
self,
|
|
images,
|
|
audio,
|
|
assembly_plan,
|
|
exact_total_duration,
|
|
audio_seam,
|
|
diagnostics,
|
|
):
|
|
return assemble_decoded_chunks(
|
|
images=_chunk_list(images),
|
|
audio=_chunk_list(audio),
|
|
assembly_plan=_singleton(assembly_plan, "assembly_plan"),
|
|
exact_total_duration=bool(
|
|
_singleton(exact_total_duration, "exact_total_duration")
|
|
),
|
|
audio_seam=str(_singleton(audio_seam, "audio_seam")),
|
|
diagnostics=str(_singleton(diagnostics, "diagnostics")),
|
|
)
|
|
# V3.0.1 hardening integration: preflight before allocation; Detailed Report only.
|
|
from ..hardening import assemble_with_hardening as _assemble_with_hardening
|
|
|
|
_assemble_decoded_chunks_v300 = assemble_decoded_chunks
|
|
|
|
|
|
def assemble_decoded_chunks(*args, **kwargs):
|
|
return _assemble_with_hardening(_assemble_decoded_chunks_v300, args, kwargs)
|
|
|
|
|
|
class H3ContinuumAssembleSeamExperimental(H3ContinuumAssembleV3):
|
|
DEPRECATED = False
|
|
CATEGORY = "MiniMax H3/Continuum"
|
|
DESCRIPTION = (
|
|
"Analyze decoded chunk boundaries and apply guarded video seam correction."
|
|
)
|
|
SEARCH_ALIASES = ["H3 seam analysis", "H3 boundary flicker analysis"]
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
schema = super().INPUT_TYPES()
|
|
required = dict(schema["required"])
|
|
diagnostics = required.pop("diagnostics")
|
|
required["video_seam"] = (
|
|
VIDEO_SEAM_ANALYSIS_OPTIONS,
|
|
{
|
|
"default": VIDEO_SEAM_AUTO,
|
|
"display_name": "Video Seam",
|
|
"tooltip": (
|
|
"Analyze Only reports decoded video boundaries without changing them. "
|
|
"Auto applies the validated transient and micro-flash correction. "
|
|
"Auto 2 experimentally adds qualified exposure-ramp smoothing."
|
|
),
|
|
},
|
|
)
|
|
required["diagnostics"] = diagnostics
|
|
schema["required"] = required
|
|
return schema
|
|
|
|
def assemble(
|
|
self,
|
|
images,
|
|
audio,
|
|
assembly_plan,
|
|
exact_total_duration,
|
|
audio_seam,
|
|
video_seam,
|
|
diagnostics,
|
|
):
|
|
mode = str(_singleton(video_seam, "video_seam"))
|
|
if mode not in VIDEO_SEAM_ANALYSIS_OPTIONS:
|
|
raise ValueError(f"unknown Video Seam mode: {mode!r}")
|
|
if mode == VIDEO_SEAM_OFF:
|
|
return super().assemble(
|
|
images,
|
|
audio,
|
|
assembly_plan,
|
|
exact_total_duration,
|
|
audio_seam,
|
|
diagnostics,
|
|
)
|
|
|
|
image_chunks = _chunk_list(images)
|
|
plan = _singleton(assembly_plan, "assembly_plan")
|
|
analyses = []
|
|
analysis_error = None
|
|
actions = {}
|
|
try:
|
|
from .video_seam import (
|
|
analyze_decoded_boundaries,
|
|
correct_decoded_boundaries,
|
|
format_video_boundary_analysis,
|
|
)
|
|
|
|
analyses = analyze_decoded_boundaries(
|
|
images=image_chunks,
|
|
assembly_plan=plan,
|
|
)
|
|
if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2):
|
|
image_chunks, actions = correct_decoded_boundaries(
|
|
images=image_chunks,
|
|
assembly_plan=plan,
|
|
analyses=analyses,
|
|
enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2,
|
|
)
|
|
except Exception as exc:
|
|
analysis_error = exc
|
|
|
|
result_images, result_audio, report = assemble_decoded_chunks(
|
|
images=image_chunks,
|
|
audio=_chunk_list(audio),
|
|
assembly_plan=plan,
|
|
exact_total_duration=bool(
|
|
_singleton(exact_total_duration, "exact_total_duration")
|
|
),
|
|
audio_seam=str(_singleton(audio_seam, "audio_seam")),
|
|
diagnostics=str(_singleton(diagnostics, "diagnostics")),
|
|
)
|
|
if mode == VIDEO_SEAM_ANALYZE:
|
|
status = "Video Seam: Analyze Only; decoded frames and audio are unchanged."
|
|
elif mode == VIDEO_SEAM_AUTO:
|
|
status = (
|
|
"Video Seam: Auto; guarded transient and micro-flash correction enabled."
|
|
)
|
|
else:
|
|
status = (
|
|
"Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, "
|
|
"and exposure-ramp correction enabled."
|
|
)
|
|
report = report.replace("Video seam correction is disabled.", status, 1)
|
|
if analysis_error is None:
|
|
lines = [
|
|
format_video_boundary_analysis(
|
|
item,
|
|
action=(
|
|
"analysis only"
|
|
if mode == VIDEO_SEAM_ANALYZE
|
|
else actions.get(item.boundary_index, "kept native boundary")
|
|
),
|
|
)
|
|
for item in analyses
|
|
]
|
|
if not lines:
|
|
lines = [f"Video Seam {mode}: no decoded chunk boundary to analyze."]
|
|
else:
|
|
lines = [
|
|
f"Video Seam {mode}: native output preserved; analysis unavailable "
|
|
f"({type(analysis_error).__name__}: {analysis_error})"
|
|
]
|
|
return result_images, result_audio, report.rstrip() + "\n" + "\n".join(lines)
|