Release latent-first H3 Continuum V3

This commit is contained in:
ukr8b3g-cmyk
2026-08-12 01:58:29 +09:00
parent 8c7ed4e8bd
commit 6afc4152c7
12 changed files with 1154 additions and 533 deletions
+5
View File
@@ -0,0 +1,5 @@
"""Latent-first H3 Continuum V3 nodes."""
from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
+269
View File
@@ -0,0 +1,269 @@
"""Decoded chunk assembly for latent-first H3 Continuum V3."""
from __future__ import annotations
from typing import Any
import torch
from ..media import validate_audio
from ..v2.decoder import _slice_audio_for_timeline, enforce_total_frames
from ..constants import (
DIAGNOSTICS_FULL,
DIAGNOSTICS_OFF,
DIAGNOSTICS_OPTIONS,
normalize_diagnostics_mode,
)
from ..v2.seam_guard import correct_audio_seam
from .plan import FPS, validate_assembly_plan
AUDIO_SEAM_OFF = "Off"
AUDIO_SEAM_AUTO = "Auto"
AUDIO_SEAM_OPTIONS = (AUDIO_SEAM_OFF, AUDIO_SEAM_AUTO)
def _singleton(value: Any, name: str) -> Any:
while isinstance(value, list):
if len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
value = value[0]
return value
def _chunk_list(value: Any) -> list[Any]:
if isinstance(value, list):
if len(value) == 1 and isinstance(value[0], list):
return list(value[0])
return list(value)
return [value]
def assemble_decoded_chunks(
*,
images: list[Any],
audio: list[Any],
assembly_plan: dict[str, Any],
exact_total_duration: bool,
audio_seam: str,
diagnostics: str,
):
plan = validate_assembly_plan(assembly_plan)
diagnostics_mode = normalize_diagnostics_mode(diagnostics)
if audio_seam not in AUDIO_SEAM_OPTIONS:
raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}")
chunk_plans = list(plan["chunks"])
if len(images) != len(chunk_plans) or len(audio) != len(chunk_plans):
raise ValueError(
"decoded chunk count does not match the V3 assembly plan: "
f"images={len(images)}, audio={len(audio)}, plan={len(chunk_plans)}"
)
total_retained_frames = sum(int(item["net_frames"]) for item in chunk_plans)
image_buffer = None
audio_buffer = None
audio_rate = None
frame_cursor = 0
reports = [
"H3 Continuum Assemble V3",
f"Audio Seam: {audio_seam}. Video seam correction is disabled.",
]
for index, (raw_images, raw_audio, chunk) in enumerate(
zip(images, audio, chunk_plans), start=1
):
if not torch.is_tensor(raw_images) or raw_images.ndim != 4:
raise ValueError(f"decoded image chunk {index} must be [frames,H,W,C]")
raw_images = raw_images.detach().to("cpu")
total_frames = int(chunk["total_frames"])
trim_frames = int(chunk["trim_frames"])
net_frames = int(chunk["net_frames"])
if int(raw_images.shape[0]) < total_frames:
raise ValueError(
f"decoded image chunk {index} has {raw_images.shape[0]} frames; "
f"expected at least {total_frames}"
)
segment_images = raw_images[:total_frames][trim_frames:].contiguous()
if int(segment_images.shape[0]) != net_frames:
raise ValueError(
f"decoded image chunk {index} retained {segment_images.shape[0]} frames; "
f"plan expects {net_frames}"
)
waveform, sample_rate = validate_audio(raw_audio)
waveform = waveform.detach().to("cpu")
sample_rate = int(sample_rate)
raw_audio_cpu = {"waveform": waveform, "sample_rate": sample_rate}
if audio_rate is None:
audio_rate = sample_rate
elif sample_rate != audio_rate:
raise ValueError(
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
)
frame_stop = frame_cursor + net_frames
segment_waveform = _slice_audio_for_timeline(
raw_audio_cpu,
trim_frames=trim_frames,
frame_start=frame_cursor,
frame_stop=frame_stop,
)
if image_buffer is None:
image_buffer = torch.empty(
(total_retained_frames, *segment_images.shape[1:]),
dtype=segment_images.dtype,
device="cpu",
)
elif tuple(segment_images.shape[1:]) != tuple(image_buffer.shape[1:]):
raise ValueError(f"decoded image geometry changed at chunk {index}")
image_buffer[frame_cursor:frame_stop].copy_(segment_images)
if audio_buffer is None:
audio_buffer = torch.empty(
(
*waveform.shape[:-1],
int(round(total_retained_frames / FPS * sample_rate)),
),
dtype=waveform.dtype,
device="cpu",
)
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
raise ValueError(
"decoded audio batch/channel structure changed between chunks"
)
sample_start = int(round(frame_cursor / FPS * sample_rate))
sample_stop = int(round(frame_stop / FPS * sample_rate))
seam_report = None
if index > 1 and audio_seam == AUDIO_SEAM_AUTO:
try:
cut_sample = int(round(trim_frames / FPS * sample_rate))
patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam(
audio_buffer[..., :sample_start],
waveform,
sample_rate=sample_rate,
cut_sample=cut_sample,
)
if patch is not None and int(patch.shape[-1]) > 0:
patch_start = sample_start - int(patch.shape[-1])
if patch_start < 0:
raise ValueError("audio seam patch starts before the output")
audio_buffer[..., patch_start:sample_start].copy_(patch)
seam_report = (
f"audio seam {index-1}->{index}: "
f"corr={metrics.correlation_before:.4f}->{metrics.correlation_after:.4f}, "
f"jump={metrics.boundary_jump_before:.6f}->{metrics.boundary_jump_after:.6f}, "
f"fade={fade_samples} samples"
)
if diagnostics_mode == DIAGNOSTICS_FULL:
seam_report += (
f", offset={metrics.offset_samples:+d}, "
f"gain={level_gain:.4f}, dc={dc_bias:+.6f}"
)
except Exception as exc:
seam_report = (
f"audio seam {index-1}->{index}: fallback to native boundary "
f"({type(exc).__name__}: {exc})"
)
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
if diagnostics_mode != DIAGNOSTICS_OFF:
reports.append(
f"assembled decoded chunk {index}: {net_frames} retained frames, "
f"cumulative {frame_stop}/{total_retained_frames}"
)
if seam_report:
reports.append(seam_report)
frame_cursor = frame_stop
if image_buffer is None or audio_buffer is None or audio_rate is None:
raise RuntimeError("V3 assembler failed to allocate output buffers")
if frame_cursor != total_retained_frames:
raise RuntimeError(
f"V3 assembly cursor mismatch: {frame_cursor} != {total_retained_frames}"
)
result_audio = {
"waveform": audio_buffer.contiguous(),
"sample_rate": audio_rate,
}
duration_report = ""
if exact_total_duration:
image_buffer, result_audio, duration_report = enforce_total_frames(
image_buffer.contiguous(),
result_audio,
target_frames=int(plan["target_frames"]),
preserve_final_frame=bool(plan.get("preserve_final_frame")),
)
reports.append(
f"Assembled {len(chunk_plans)} externally decoded chunks into "
f"{image_buffer.shape[0]} frames with cumulative sample-boundary alignment."
)
if duration_report:
reports.append(duration_report)
return image_buffer.contiguous(), result_audio, "\n".join(reports)
class H3ContinuumAssembleV3:
DESCRIPTION = (
"Assemble full AV chunks decoded by ComfyUI Core. Trims decoded context, "
"aligns audio on cumulative frame boundaries, and optionally applies Audio Seam Auto."
)
SEARCH_ALIASES = ["H3 latent assemble", "H3 external VAE decode"]
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"images": ("IMAGE",),
"audio": ("AUDIO",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"exact_total_duration": ("BOOLEAN", {"default": True}),
"audio_seam": (
AUDIO_SEAM_OPTIONS,
{
"default": AUDIO_SEAM_OFF,
"display_name": "Audio Seam",
"tooltip": (
"Auto corrects only decoded audio boundaries; "
"video frames are never altered."
),
},
),
"diagnostics": (
DIAGNOSTICS_OPTIONS,
{
"default": DIAGNOSTICS_OPTIONS[0],
"display_name": "Report Detail",
},
),
}
}
INPUT_IS_LIST = True
RETURN_TYPES = ("IMAGE", "AUDIO", "STRING")
RETURN_NAMES = ("images", "audio", "report")
FUNCTION = "assemble"
CATEGORY = "MiniMax H3/Continuum"
def assemble(
self,
images,
audio,
assembly_plan,
exact_total_duration,
audio_seam,
diagnostics,
):
return assemble_decoded_chunks(
images=_chunk_list(images),
audio=_chunk_list(audio),
assembly_plan=_singleton(assembly_plan, "assembly_plan"),
exact_total_duration=bool(
_singleton(exact_total_duration, "exact_total_duration")
),
audio_seam=str(_singleton(audio_seam, "audio_seam")),
diagnostics=str(_singleton(diagnostics, "diagnostics")),
)
+308
View File
@@ -0,0 +1,308 @@
"""Public latent-first facade nodes for H3 Continuum V3."""
from __future__ import annotations
from ..v2.nodes import (
CATEGORY,
DIAGNOSTICS_OPTIONS,
H3ContinuumSamplerV2,
PROMPT_FORMAT_OPTIONS,
SEAM_CORRECTION_OFF,
SPARSE_OVERRIDE_SCHEMA_VERSION,
V2_CONTINUITY_OPTIONS,
validate_sparse_prompt_overrides,
)
from .assembly import H3ContinuumAssembleV3
from .plan import make_assembly_plan
class H3ContinuumAdvancedV3:
DESCRIPTION = (
"Optional continuation, session, reroll, and diagnostics settings for V3."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"audio_continuity": ("BOOLEAN", {"default": True}),
"diagnostics": (
DIAGNOSTICS_OPTIONS,
{
"default": DIAGNOSTICS_OPTIONS[0],
"display_name": "Report Detail",
},
),
"reroll_from_chunk": (
"INT",
{"default": 0, "min": 0, "max": 16, "step": 1},
),
"reroll_nonce": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFF,
"step": 1,
},
),
"strict_compatibility": ("BOOLEAN", {"default": True}),
"debug": ("BOOLEAN", {"default": False}),
"show_preview": ("BOOLEAN", {"default": True}),
},
"optional": {
"last_frame": ("IMAGE",),
"session": ("H3_CONTINUUM_SESSION",),
"initial_state": ("H3_CONTINUUM_STATE",),
"prompt_plan": ("H3_CONTINUUM_PROMPT_PLAN",),
},
}
RETURN_TYPES = ("H3_CONTINUUM_ADVANCED_V3",)
RETURN_NAMES = ("advanced",)
FUNCTION = "pack"
CATEGORY = CATEGORY
def pack(
self,
audio_continuity,
diagnostics,
reroll_from_chunk,
reroll_nonce,
strict_compatibility,
debug,
show_preview,
last_frame=None,
session=None,
initial_state=None,
prompt_plan=None,
):
return (
{
"audio_continuity": bool(audio_continuity),
"diagnostics": diagnostics,
"reroll_from_chunk": int(reroll_from_chunk),
"reroll_nonce": int(reroll_nonce),
"strict_compatibility": bool(strict_compatibility),
"debug": bool(debug),
"show_preview": bool(show_preview),
"last_frame": last_frame,
"session": session,
"initial_state": initial_state,
"prompt_plan": prompt_plan,
},
)
class H3ContinuumSamplerV3:
DESCRIPTION = (
"Latent-first N-chunk MiniMax H3 sampler. Sampling, native continuation, "
"State/Session, and Spectrum interop stay inside Continuum; ComfyUI Core "
"performs video and audio VAE decoding."
)
SEARCH_ALIASES = [
"H3 Continuum V3",
"H3 latent first",
"MiniMax H3 external VAE decode",
]
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"video_vae": (
"VAE",
{
"tooltip": (
"Used only to encode first/last-frame conditioning; "
"V3 never decodes with it."
),
},
),
"sampler": ("SAMPLER",),
"sigmas": ("SIGMAS",),
"sequence_prompt": (
"STRING",
{
"forceInput": True,
"display_name": "Sequence Prompt",
"tooltip": (
"Connect one Text (Multiline) for the complete sequence."
),
},
),
"prompt_mode": (
PROMPT_FORMAT_OPTIONS,
{
"default": PROMPT_FORMAT_OPTIONS[0],
"display_name": "Prompt Format",
},
),
"chunks": ("INT", {"default": 3, "min": 1, "max": 16, "step": 1}),
"chunk_seconds": (
"FLOAT",
{"default": 5.0, "min": 4.0, "max": 15.0, "step": 0.1},
),
"width": (
"INT",
{"default": 1344, "min": 32, "max": 16384, "step": 32},
),
"height": (
"INT",
{"default": 768, "min": 32, "max": 16384, "step": 32},
),
"continuity": (
V2_CONTINUITY_OPTIONS,
{"default": V2_CONTINUITY_OPTIONS[1]},
),
"base_seed": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
},
),
},
"optional": {
"first_frame": ("IMAGE",),
"prompt_overrides": ("H3_CONTINUUM_CLIP_OVERRIDES",),
"advanced": ("H3_CONTINUUM_ADVANCED_V3",),
},
}
RETURN_TYPES = (
"LATENT",
"LATENT",
"H3_CONTINUUM_ASSEMBLY_PLAN",
"H3_CONTINUUM_RESULT",
)
RETURN_NAMES = (
"video_latents",
"audio_latents",
"assembly_plan",
"result",
)
OUTPUT_IS_LIST = (True, True, False, False)
FUNCTION = "run"
CATEGORY = CATEGORY
def run(
self,
model,
clip,
video_vae,
sampler,
sigmas,
sequence_prompt,
prompt_mode,
chunks,
chunk_seconds,
width,
height,
continuity,
base_seed,
first_frame=None,
prompt_overrides=None,
advanced=None,
):
if prompt_overrides is not None and not isinstance(prompt_overrides, dict):
raise TypeError(
"prompt_overrides must be an H3_CONTINUUM_CLIP_OVERRIDES pack"
)
if advanced is not None and not isinstance(advanced, dict):
raise TypeError("advanced must be an H3_CONTINUUM_ADVANCED_V3 pack")
advanced_values = {
"audio_continuity": True,
"diagnostics": DIAGNOSTICS_OPTIONS[0],
"reroll_from_chunk": 0,
"reroll_nonce": 0,
"strict_compatibility": True,
"debug": False,
"show_preview": True,
"last_frame": None,
"session": None,
"initial_state": None,
"prompt_plan": None,
}
if advanced:
advanced_values.update(advanced)
clip_prompt_inputs = {}
if prompt_overrides:
if (
type(prompt_overrides.get("schema_version")) is not int
or prompt_overrides.get("schema_version")
!= SPARSE_OVERRIDE_SCHEMA_VERSION
):
raise ValueError("unsupported H3_CONTINUUM_CLIP_OVERRIDES schema")
sparse_overrides = validate_sparse_prompt_overrides(
prompt_overrides.get("overrides"), chunks=int(chunks)
)
for index, prompt in sparse_overrides.items():
clip_prompt_inputs[f"clip_{index}_prompt"] = prompt
entries, last_state, session, report = H3ContinuumSamplerV2().run(
model=model,
clip=clip,
video_vae=video_vae,
audio_vae=None,
sampler=sampler,
sigmas=sigmas,
prompt_mode=prompt_mode,
prompt_script=sequence_prompt,
chunks=chunks,
chunk_seconds=chunk_seconds,
width=width,
height=height,
continuity=continuity,
base_seed=base_seed,
audio_continuity=advanced_values["audio_continuity"],
exact_total_duration=False,
diagnostics=advanced_values["diagnostics"],
reroll_from_chunk=advanced_values["reroll_from_chunk"],
reroll_nonce=advanced_values["reroll_nonce"],
strict_compatibility=advanced_values["strict_compatibility"],
debug=advanced_values["debug"],
seam_correction=SEAM_CORRECTION_OFF,
first_frame=first_frame,
last_frame=advanced_values["last_frame"],
session=advanced_values["session"],
initial_state=advanced_values["initial_state"],
prompt_plan=advanced_values["prompt_plan"],
sequence_prompt=sequence_prompt,
show_preview=advanced_values["show_preview"],
latent_only=True,
**clip_prompt_inputs,
)
assembly_plan = make_assembly_plan(
entries,
chunk_seconds=float(chunk_seconds),
preserve_final_frame=advanced_values["last_frame"] is not None,
)
video_latents = [{"samples": entry["video"]} for entry in entries]
audio_latents = [{"samples": entry["audio"]} for entry in entries]
result = {
"last_state": last_state,
"session": session,
"report": report,
}
return video_latents, audio_latents, assembly_plan, result
NODE_CLASS_MAPPINGS = {
"H3ContinuumSamplerV3": H3ContinuumSamplerV3,
"H3ContinuumAdvancedV3": H3ContinuumAdvancedV3,
"H3ContinuumAssembleV3": H3ContinuumAssembleV3,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumSamplerV3": "H3 Continuum Sampler V3",
"H3ContinuumAdvancedV3": "H3 Continuum Advanced V3",
"H3ContinuumAssembleV3": "H3 Continuum Assemble V3",
}
+122
View File
@@ -0,0 +1,122 @@
"""Assembly-plan contract for latent-first H3 Continuum V3."""
from __future__ import annotations
from typing import Any
import torch
from ..state import validate_plan
ASSEMBLY_PLAN_MAGIC = "H3_CONTINUUM_ASSEMBLY_PLAN"
ASSEMBLY_PLAN_SCHEMA_VERSION = 1
FPS = 24
class AssemblyPlanError(ValueError):
pass
def make_assembly_plan(
entries: list[dict[str, Any]],
*,
chunk_seconds: float,
preserve_final_frame: bool,
) -> dict[str, Any]:
if not entries:
raise AssemblyPlanError("cannot create an assembly plan for an empty sequence")
chunks: list[dict[str, Any]] = []
width = height = None
for sequence_index, entry in enumerate(entries, start=1):
if not isinstance(entry, dict):
raise AssemblyPlanError(f"chunk {sequence_index} entry is not a dictionary")
plan = validate_plan(entry.get("plan"))
video = entry.get("video")
audio = entry.get("audio")
if not torch.is_tensor(video) or video.ndim < 3:
raise AssemblyPlanError(f"chunk {sequence_index} video latent is invalid")
if not torch.is_tensor(audio) or audio.ndim < 3:
raise AssemblyPlanError(f"chunk {sequence_index} audio latent is invalid")
plan_width = int(plan["width"])
plan_height = int(plan["height"])
if width is None:
width, height = plan_width, plan_height
elif (plan_width, plan_height) != (width, height):
raise AssemblyPlanError("chunk dimensions changed within the sequence")
chunks.append(
{
"sequence_index": sequence_index,
"chunk_index": int(plan["clip_index"]),
"total_frames": int(plan["total_frames"]),
"trim_frames": int(plan["trim_frames"]),
"net_frames": int(plan["net_frames"]),
"context_frames": int(plan["context_frames"]),
"expected_video_latent_t": int(video.shape[2]),
"expected_audio_latent_t": int(audio.shape[2]),
}
)
result = {
"magic": ASSEMBLY_PLAN_MAGIC,
"schema_version": ASSEMBLY_PLAN_SCHEMA_VERSION,
"fps": FPS,
"width": int(width),
"height": int(height),
"chunk_seconds": float(chunk_seconds),
"target_frames": int(round(len(chunks) * float(chunk_seconds) * FPS)),
"preserve_final_frame": bool(preserve_final_frame),
"chunks": chunks,
}
return validate_assembly_plan(result)
def validate_assembly_plan(plan: Any) -> dict[str, Any]:
if not isinstance(plan, dict) or plan.get("magic") != ASSEMBLY_PLAN_MAGIC:
raise AssemblyPlanError("invalid H3 Continuum V3 assembly plan")
if (
type(plan.get("schema_version")) is not int
or plan["schema_version"] != ASSEMBLY_PLAN_SCHEMA_VERSION
):
raise AssemblyPlanError(
f"unsupported assembly plan schema {plan.get('schema_version')!r}"
)
if int(plan.get("fps", 0)) != FPS:
raise AssemblyPlanError("assembly plan FPS must be 24")
if int(plan.get("width", 0)) <= 0 or int(plan.get("height", 0)) <= 0:
raise AssemblyPlanError("assembly plan dimensions are invalid")
if float(plan.get("chunk_seconds", 0.0)) <= 0:
raise AssemblyPlanError("assembly plan chunk duration is invalid")
if int(plan.get("target_frames", 0)) <= 0:
raise AssemblyPlanError("assembly plan target frame count is invalid")
chunks = plan.get("chunks")
if not isinstance(chunks, list) or not chunks:
raise AssemblyPlanError("assembly plan contains no chunks")
for expected_index, chunk in enumerate(chunks, start=1):
if not isinstance(chunk, dict):
raise AssemblyPlanError(f"assembly chunk {expected_index} is invalid")
if int(chunk.get("sequence_index", 0)) != expected_index:
raise AssemblyPlanError("assembly chunk order is not contiguous")
total = int(chunk.get("total_frames", 0))
trim = int(chunk.get("trim_frames", -1))
net = int(chunk.get("net_frames", 0))
if total <= 0 or trim < 0 or net <= 0 or total - trim != net:
raise AssemblyPlanError(
f"assembly chunk {expected_index} frame plan is invalid"
)
if int(chunk.get("context_frames", -1)) < 0:
raise AssemblyPlanError(
f"assembly chunk {expected_index} context is invalid"
)
if int(chunk.get("expected_video_latent_t", 0)) <= 0:
raise AssemblyPlanError(
f"assembly chunk {expected_index} video latent length is invalid"
)
if int(chunk.get("expected_audio_latent_t", 0)) <= 0:
raise AssemblyPlanError(
f"assembly chunk {expected_index} audio latent length is invalid"
)
return plan