Release v3.6.0 masked AV continuation
This commit is contained in:
@@ -265,6 +265,80 @@ class H3ContinuumSamplerV35(H3ContinuumSamplerV34):
|
||||
)
|
||||
|
||||
|
||||
CONTINUATION_BACKEND_STANDARD = "Standard"
|
||||
CONTINUATION_BACKEND_COMPATIBILITY = "Compatibility"
|
||||
CONTINUATION_BACKEND_OPTIONS = (
|
||||
CONTINUATION_BACKEND_STANDARD,
|
||||
CONTINUATION_BACKEND_COMPATIBILITY,
|
||||
)
|
||||
|
||||
|
||||
class H3ContinuumSamplerV36(H3ContinuumSamplerV35):
|
||||
"""V3.6 sampler with Masked continuation and a legacy fallback."""
|
||||
|
||||
DEPRECATED = False
|
||||
CATEGORY = CONTINUUM_CATEGORY
|
||||
DESCRIPTION = (
|
||||
"H3 Continuum V3.6 sampler. Standard continuation preserves prior Video/Audio "
|
||||
"inside the next H3 target with native noise masks; Compatibility retains the "
|
||||
"V3.5 Reference Context transport."
|
||||
)
|
||||
SEARCH_ALIASES = [
|
||||
"H3 Continuum Sampler V3.6",
|
||||
"MiniMax H3 masked continuation",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
schema = super().INPUT_TYPES()
|
||||
required = dict(schema.get("required", {}))
|
||||
required["continuation_backend"] = (
|
||||
CONTINUATION_BACKEND_OPTIONS,
|
||||
{
|
||||
"default": CONTINUATION_BACKEND_STANDARD,
|
||||
"display_name": "Continuation Backend",
|
||||
"advanced": True,
|
||||
"tooltip": (
|
||||
"Standard uses the V3.6 target-preserving continuation path. "
|
||||
"Compatibility restores the V3.5 Reference Context path for "
|
||||
"older workflows or comparison. Run Storage keeps the two "
|
||||
"backends in separate revisions."
|
||||
),
|
||||
},
|
||||
)
|
||||
schema["required"] = required
|
||||
return schema
|
||||
|
||||
def run(
|
||||
self,
|
||||
*args,
|
||||
continuation_backend=CONTINUATION_BACKEND_STANDARD,
|
||||
**kwargs,
|
||||
):
|
||||
from ..v2.sequence import (
|
||||
MASKED_AV_PREFIX_22_V1,
|
||||
MASKED_VIDEO_PREFIX_V1,
|
||||
REFERENCE_CONTEXT_V1,
|
||||
)
|
||||
|
||||
backend = str(continuation_backend)
|
||||
if backend == CONTINUATION_BACKEND_COMPATIBILITY:
|
||||
transport = REFERENCE_CONTEXT_V1
|
||||
else:
|
||||
audio_continuity = kwargs.get("audio_continuity")
|
||||
if audio_continuity is None and len(args) > 13:
|
||||
audio_continuity = args[13]
|
||||
if audio_continuity is None:
|
||||
audio_continuity = True
|
||||
transport = (
|
||||
MASKED_AV_PREFIX_22_V1
|
||||
if bool(audio_continuity)
|
||||
else MASKED_VIDEO_PREFIX_V1
|
||||
)
|
||||
kwargs["continuation_transport"] = transport
|
||||
return super().run(*args, **kwargs)
|
||||
|
||||
|
||||
class H3ContinuumAssembleSeamV34(H3ContinuumAssembleSeamExperimental):
|
||||
"""Assemble decoded chunks and select original Driving Audio when connected."""
|
||||
|
||||
@@ -444,6 +518,7 @@ class H3ContinuumAssembleSeamV35(H3ContinuumAssembleSeamV34):
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumSamplerV34": H3ContinuumSamplerV34,
|
||||
"H3ContinuumSamplerV35": H3ContinuumSamplerV35,
|
||||
"H3ContinuumSamplerV36": H3ContinuumSamplerV36,
|
||||
"H3ContinuumAssembleSeamV34": H3ContinuumAssembleSeamV34,
|
||||
"H3ContinuumAssembleSeamV35": H3ContinuumAssembleSeamV35,
|
||||
}
|
||||
@@ -451,6 +526,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumSamplerV34": "H3 Continuum Sampler V3.4",
|
||||
"H3ContinuumSamplerV35": "H3 Continuum Sampler V3.5",
|
||||
"H3ContinuumSamplerV36": "H3 Continuum Sampler V3.6",
|
||||
"H3ContinuumAssembleSeamV34": "H3 Continuum Assemble + Seam V3.4",
|
||||
"H3ContinuumAssembleSeamV35": "H3 Continuum Assemble + Seam V3.5",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Private V3.6 masked-prefix continuation experiments.
|
||||
|
||||
This module is intentionally not registered as a public node contract. It
|
||||
builds H3 AV targets whose protected prefixes are preserved by Core's denoise
|
||||
mask. R1 protects video only; R2-0 can protect matching video and audio tails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import extract_av_streams
|
||||
from ..temporal import audio_latent_t, context_slots
|
||||
|
||||
|
||||
REFERENCE_CONTEXT_V1 = "reference_context_v1"
|
||||
MASKED_VIDEO_PREFIX_V1 = "masked_video_prefix_v1"
|
||||
MASKED_AV_PREFIX_22_V1 = "masked_av_prefix_22_v1"
|
||||
MASKED_AV_PREFIX_39_V1 = "masked_av_prefix_39_v1"
|
||||
CONTINUATION_TRANSPORTS = (
|
||||
REFERENCE_CONTEXT_V1,
|
||||
MASKED_VIDEO_PREFIX_V1,
|
||||
MASKED_AV_PREFIX_22_V1,
|
||||
MASKED_AV_PREFIX_39_V1,
|
||||
)
|
||||
|
||||
|
||||
def _build_masked_prefix_latent(
|
||||
target_latent: dict[str, Any],
|
||||
video_context: torch.Tensor,
|
||||
context_frames: int,
|
||||
*,
|
||||
audio_context: torch.Tensor | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Copy normalized AV tails into a target and attach stream masks."""
|
||||
|
||||
try:
|
||||
import comfy.nested_tensor
|
||||
except Exception as exc: # pragma: no cover - ComfyUI runtime only
|
||||
raise RuntimeError(f"ComfyUI NestedTensor helper unavailable: {exc}") from exc
|
||||
|
||||
target_video, target_audio = extract_av_streams(target_latent)
|
||||
frames = int(context_frames)
|
||||
slots = context_slots(frames)
|
||||
if not torch.is_tensor(video_context) or video_context.ndim != 5:
|
||||
raise ValueError("masked video context must be a five-dimensional Tensor")
|
||||
if tuple(video_context.shape[:2]) != tuple(target_video.shape[:2]):
|
||||
raise ValueError(
|
||||
"masked video context batch/channel shape does not match the target: "
|
||||
f"context={tuple(video_context.shape)}, target={tuple(target_video.shape)}"
|
||||
)
|
||||
if int(video_context.shape[2]) != slots:
|
||||
raise ValueError(
|
||||
f"masked video context has T={video_context.shape[2]}; expected {slots} "
|
||||
f"for {frames} frames"
|
||||
)
|
||||
if tuple(video_context.shape[-2:]) != tuple(target_video.shape[-2:]):
|
||||
raise ValueError(
|
||||
"masked video context geometry does not match the target: "
|
||||
f"context={tuple(video_context.shape[-2:])}, "
|
||||
f"target={tuple(target_video.shape[-2:])}"
|
||||
)
|
||||
if int(target_video.shape[2]) <= slots:
|
||||
raise ValueError("masked video target must contain generated slots after the prefix")
|
||||
|
||||
normalized_video = video_context.detach().to(
|
||||
device=target_video.device,
|
||||
dtype=target_video.dtype,
|
||||
).contiguous()
|
||||
if not bool(torch.isfinite(normalized_video.float()).all().item()):
|
||||
raise ValueError("masked video context contains NaN or Inf")
|
||||
|
||||
normalized_audio = None
|
||||
audio_steps = 0
|
||||
if audio_context is not None:
|
||||
audio_steps = audio_latent_t(frames)
|
||||
if not torch.is_tensor(audio_context) or audio_context.ndim != 4:
|
||||
raise ValueError("masked audio context must be a four-dimensional Tensor")
|
||||
if tuple(audio_context.shape[:3]) != tuple(target_audio.shape[:3]):
|
||||
raise ValueError(
|
||||
"masked audio context batch/channel shape does not match the target: "
|
||||
f"context={tuple(audio_context.shape)}, target={tuple(target_audio.shape)}"
|
||||
)
|
||||
if int(audio_context.shape[-1]) != audio_steps:
|
||||
raise ValueError(
|
||||
f"masked audio context has T={audio_context.shape[-1]}; expected "
|
||||
f"{audio_steps} for {frames} frames"
|
||||
)
|
||||
if int(target_audio.shape[-1]) <= audio_steps:
|
||||
raise ValueError("masked audio target must contain generated ticks after the prefix")
|
||||
normalized_audio = audio_context.detach().to(
|
||||
device=target_audio.device,
|
||||
dtype=target_audio.dtype,
|
||||
).contiguous()
|
||||
if not bool(torch.isfinite(normalized_audio.float()).all().item()):
|
||||
raise ValueError("masked audio context contains NaN or Inf")
|
||||
|
||||
video = target_video.clone()
|
||||
audio = target_audio.clone()
|
||||
video[:, :, :slots].copy_(normalized_video)
|
||||
if normalized_audio is not None:
|
||||
audio[..., :audio_steps].copy_(normalized_audio)
|
||||
|
||||
video_mask = torch.ones(
|
||||
(
|
||||
int(video.shape[0]),
|
||||
1,
|
||||
int(video.shape[2]),
|
||||
int(video.shape[3]),
|
||||
int(video.shape[4]),
|
||||
),
|
||||
device=video.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
video_mask[:, :, :slots] = 0.0
|
||||
audio_mask = torch.ones(
|
||||
(
|
||||
int(audio.shape[0]),
|
||||
1,
|
||||
int(audio.shape[2]),
|
||||
int(audio.shape[3]),
|
||||
),
|
||||
device=audio.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if normalized_audio is not None:
|
||||
audio_mask[..., :audio_steps] = 0.0
|
||||
|
||||
result = dict(target_latent)
|
||||
result["samples"] = comfy.nested_tensor.NestedTensor((video, audio))
|
||||
result["noise_mask"] = comfy.nested_tensor.NestedTensor(
|
||||
(video_mask, audio_mask)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def build_masked_video_prefix_latent(
|
||||
target_latent: dict[str, Any],
|
||||
video_context: torch.Tensor,
|
||||
context_frames: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Copy a normalized video tail into the target and attach AV noise masks."""
|
||||
return _build_masked_prefix_latent(
|
||||
target_latent,
|
||||
video_context,
|
||||
context_frames,
|
||||
audio_context=None,
|
||||
)
|
||||
|
||||
|
||||
def build_masked_av_prefix_latent(
|
||||
target_latent: dict[str, Any],
|
||||
video_context: torch.Tensor,
|
||||
audio_context: torch.Tensor,
|
||||
context_frames: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Copy matching video/audio tails and preserve both with denoise masks."""
|
||||
|
||||
return _build_masked_prefix_latent(
|
||||
target_latent,
|
||||
video_context,
|
||||
context_frames,
|
||||
audio_context=audio_context,
|
||||
)
|
||||
|
||||
|
||||
def _restore_masked_prefix(
|
||||
sampled_latent: dict[str, Any],
|
||||
target_latent: dict[str, Any],
|
||||
context_frames: int,
|
||||
*,
|
||||
restore_audio: bool,
|
||||
) -> dict[str, Any]:
|
||||
"""Reapply only protected prefix regions after sampler integration."""
|
||||
|
||||
try:
|
||||
import comfy.nested_tensor
|
||||
except Exception as exc: # pragma: no cover - ComfyUI runtime only
|
||||
raise RuntimeError(f"ComfyUI NestedTensor helper unavailable: {exc}") from exc
|
||||
|
||||
sampled_video, sampled_audio = extract_av_streams(sampled_latent)
|
||||
target_video, target_audio = extract_av_streams(target_latent)
|
||||
slots = context_slots(int(context_frames))
|
||||
if tuple(sampled_video.shape) != tuple(target_video.shape):
|
||||
raise ValueError(
|
||||
"sampled video shape does not match the masked target: "
|
||||
f"sampled={tuple(sampled_video.shape)}, target={tuple(target_video.shape)}"
|
||||
)
|
||||
restored_video = sampled_video.clone()
|
||||
restored_video[:, :, :slots].copy_(target_video[:, :, :slots])
|
||||
restored_audio = sampled_audio
|
||||
if restore_audio:
|
||||
if tuple(sampled_audio.shape) != tuple(target_audio.shape):
|
||||
raise ValueError(
|
||||
"sampled audio shape does not match the masked target: "
|
||||
f"sampled={tuple(sampled_audio.shape)}, target={tuple(target_audio.shape)}"
|
||||
)
|
||||
audio_steps = audio_latent_t(int(context_frames))
|
||||
restored_audio = sampled_audio.clone()
|
||||
restored_audio[..., :audio_steps].copy_(target_audio[..., :audio_steps])
|
||||
result = dict(sampled_latent)
|
||||
result["samples"] = comfy.nested_tensor.NestedTensor(
|
||||
(restored_video, restored_audio)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def restore_masked_video_prefix(
|
||||
sampled_latent: dict[str, Any],
|
||||
target_latent: dict[str, Any],
|
||||
context_frames: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Reapply only the preserved prefix after sampler integration rounding."""
|
||||
return _restore_masked_prefix(
|
||||
sampled_latent,
|
||||
target_latent,
|
||||
context_frames,
|
||||
restore_audio=False,
|
||||
)
|
||||
|
||||
|
||||
def restore_masked_av_prefix(
|
||||
sampled_latent: dict[str, Any],
|
||||
target_latent: dict[str, Any],
|
||||
context_frames: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Reapply only protected video and audio prefixes after Sampling."""
|
||||
|
||||
return _restore_masked_prefix(
|
||||
sampled_latent,
|
||||
target_latent,
|
||||
context_frames,
|
||||
restore_audio=True,
|
||||
)
|
||||
+24
-7
@@ -4,7 +4,14 @@ from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from ..constants import DIAGNOSTICS_FULL
|
||||
from ..constants import (
|
||||
CHUNK_SECONDS_DEFAULT,
|
||||
CHUNK_SECONDS_MAX,
|
||||
CHUNK_SECONDS_MIN,
|
||||
CHUNK_SECONDS_STEP,
|
||||
CHUNK_SECONDS_TOOLTIP,
|
||||
DIAGNOSTICS_FULL,
|
||||
)
|
||||
from ..v2.nodes import (
|
||||
CATEGORY,
|
||||
DIAGNOSTICS_OPTIONS,
|
||||
@@ -181,7 +188,13 @@ class H3ContinuumSamplerV3:
|
||||
"chunks": ("INT", {"default": 3, "min": 1, "max": 16, "step": 1}),
|
||||
"chunk_seconds": (
|
||||
"FLOAT",
|
||||
{"default": 5.0, "min": 4.0, "max": 15.0, "step": 0.1},
|
||||
{
|
||||
"default": CHUNK_SECONDS_DEFAULT,
|
||||
"min": CHUNK_SECONDS_MIN,
|
||||
"max": CHUNK_SECONDS_MAX,
|
||||
"step": CHUNK_SECONDS_STEP,
|
||||
"tooltip": CHUNK_SECONDS_TOOLTIP,
|
||||
},
|
||||
),
|
||||
"width": (
|
||||
"INT",
|
||||
@@ -259,6 +272,7 @@ class H3ContinuumSamplerV3:
|
||||
capture_refine_context=False,
|
||||
memory_attribution=False,
|
||||
prompt_conditioning_cache=False,
|
||||
continuation_transport="reference_context_v1",
|
||||
):
|
||||
if prompt_overrides is not None and not isinstance(prompt_overrides, dict):
|
||||
prompt_overrides = None
|
||||
@@ -341,6 +355,7 @@ class H3ContinuumSamplerV3:
|
||||
capture_refine_context=bool(capture_refine_context),
|
||||
memory_attribution=bool(memory_attribution),
|
||||
prompt_conditioning_cache=bool(prompt_conditioning_cache),
|
||||
continuation_transport=str(continuation_transport),
|
||||
**clip_prompt_inputs,
|
||||
)
|
||||
if bool(capture_refine_context):
|
||||
@@ -465,11 +480,11 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
"chunk_seconds": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 5.0,
|
||||
"min": 4.0,
|
||||
"max": 15.0,
|
||||
"step": 0.1,
|
||||
"tooltip": "Target duration per chunk. Five seconds is the validated default.",
|
||||
"default": CHUNK_SECONDS_DEFAULT,
|
||||
"min": CHUNK_SECONDS_MIN,
|
||||
"max": CHUNK_SECONDS_MAX,
|
||||
"step": CHUNK_SECONDS_STEP,
|
||||
"tooltip": CHUNK_SECONDS_TOOLTIP,
|
||||
},
|
||||
),
|
||||
"width": (
|
||||
@@ -666,6 +681,7 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
capture_refine_context=False,
|
||||
memory_attribution=False,
|
||||
prompt_conditioning_cache=False,
|
||||
continuation_transport="reference_context_v1",
|
||||
):
|
||||
runtime_started_at = time.perf_counter()
|
||||
from ..reference import prepare_reference_assets
|
||||
@@ -718,6 +734,7 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
capture_refine_context=bool(capture_refine_context),
|
||||
memory_attribution=bool(memory_attribution),
|
||||
prompt_conditioning_cache=bool(prompt_conditioning_cache),
|
||||
continuation_transport=str(continuation_transport),
|
||||
advanced={
|
||||
"audio_continuity": bool(audio_continuity),
|
||||
"diagnostics": diagnostics,
|
||||
|
||||
Reference in New Issue
Block a user