feat: release V3.5.0

This commit is contained in:
ukr8b3g-cmyk
2026-08-24 15:00:45 +09:00
parent 7e24cbe6f6
commit 0d1950b109
54 changed files with 12038 additions and 94 deletions
+720
View File
@@ -0,0 +1,720 @@
"""V3.5 direct-write assembly with a manual RAM or file-backed IMAGE buffer."""
from __future__ import annotations
from contextlib import AbstractContextManager
from dataclasses import dataclass
import math
from pathlib import Path
import time
from typing import Any, Callable, Mapping
import torch
from ..constants import (
DIAGNOSTICS_FULL,
DIAGNOSTICS_OFF,
normalize_diagnostics_mode,
)
from ..hardening import assemble_with_hardening
from ..media import validate_audio
from ..v2.seam_guard import correct_audio_seam
from .assembly import (
AUDIO_SEAM_AUTO,
AUDIO_SEAM_OPTIONS,
VIDEO_SEAM_ANALYZE,
VIDEO_SEAM_ANALYSIS_OPTIONS,
VIDEO_SEAM_AUTO,
VIDEO_SEAM_AUTO_2,
VIDEO_SEAM_OFF,
)
from .file_backed_buffer import (
BUFFER_BACKEND_AUTO,
BUFFER_BACKEND_DISK,
BUFFER_BACKEND_OPTIONS,
BUFFER_BACKEND_RAM,
allocate_file_backed_image,
format_auto_buffer_decision,
select_auto_buffer_backend,
)
from .plan import FPS, validate_assembly_plan
_COPY_BATCH_FRAMES = 8
def _check_interrupted(
interrupt_check: Callable[[], None] | None,
) -> None:
"""Honor ComfyUI's normal ESC interrupt without requiring Core in tests."""
if interrupt_check is not None:
interrupt_check()
return
try:
from comfy.model_management import throw_exception_if_processing_interrupted
except ModuleNotFoundError:
return
throw_exception_if_processing_interrupted()
@dataclass(frozen=True)
class CopySpan:
"""Map a half-open natural-timeline interval into the output buffer."""
source_start: int
source_stop: int
destination_start: int
class _RamAllocation(AbstractContextManager):
"""Match the provisional file-backed allocation interface for RAM."""
def __init__(self, shape: tuple[int, ...], dtype: torch.dtype):
self.tensor = torch.empty(shape, dtype=dtype, device="cpu")
self.record = None
self._published = False
def publish(self) -> torch.Tensor:
if self._published:
raise RuntimeError("RAM assembly buffer was already published")
self._published = True
return self.tensor
def __exit__(self, exc_type, exc, traceback) -> bool:
return False
def _video_copy_spans(
*,
natural_frames: int,
output_frames: int,
preserve_final_frame: bool,
) -> tuple[CopySpan, ...]:
if output_frames >= natural_frames:
return (CopySpan(0, natural_frames, 0),)
if preserve_final_frame and output_frames >= 2:
return (
CopySpan(0, output_frames - 1, 0),
CopySpan(natural_frames - 1, natural_frames, output_frames - 1),
)
return (CopySpan(0, output_frames, 0),)
def _copy_natural_interval(
destination: torch.Tensor,
source: torch.Tensor,
*,
natural_start: int,
spans: tuple[CopySpan, ...],
interrupt_check: Callable[[], None] | None = None,
) -> None:
natural_stop = natural_start + int(source.shape[0])
for span in spans:
overlap_start = max(natural_start, int(span.source_start))
overlap_stop = min(natural_stop, int(span.source_stop))
if overlap_stop <= overlap_start:
continue
source_start = overlap_start - natural_start
destination_start = int(span.destination_start) + (
overlap_start - int(span.source_start)
)
frame_count = overlap_stop - overlap_start
for offset in range(0, frame_count, _COPY_BATCH_FRAMES):
_check_interrupted(interrupt_check)
batch_frames = min(_COPY_BATCH_FRAMES, frame_count - offset)
source_batch_start = source_start + offset
destination_batch_start = destination_start + offset
destination[
destination_batch_start : destination_batch_start + batch_frames
].copy_(
source[source_batch_start : source_batch_start + batch_frames]
)
def _repeat_final_video_frame(
destination: torch.Tensor,
*,
natural_frames: int,
output_frames: int,
interrupt_check: Callable[[], None] | None,
) -> None:
for destination_start in range(
natural_frames,
output_frames,
_COPY_BATCH_FRAMES,
):
_check_interrupted(interrupt_check)
destination_stop = min(
output_frames,
destination_start + _COPY_BATCH_FRAMES,
)
destination[destination_start:destination_stop].copy_(
destination[natural_frames - 1 : natural_frames].expand(
destination_stop - destination_start,
*destination.shape[1:],
)
)
def _copy_audio_segment(
destination: torch.Tensor,
waveform: torch.Tensor,
*,
trim_samples: int,
) -> None:
"""Reproduce timeline slicing/padding without concatenating a waveform."""
wanted = int(destination.shape[-1])
trim_samples = max(0, int(trim_samples))
available = max(0, min(wanted, int(waveform.shape[-1]) - trim_samples))
if available > 0:
destination[..., :available].copy_(
waveform[..., trim_samples : trim_samples + available]
)
if available >= wanted:
return
if available > 0:
destination[..., available:].copy_(
destination[..., available - 1 : available].expand(
*destination.shape[:-1],
wanted - available,
)
)
else:
destination.zero_()
def adjust_audio_duration_direct(
audio: Mapping[str, Any],
*,
target_frames: int,
preserve_final_audio_anchor: bool,
) -> tuple[dict[str, Any], int]:
"""Apply the legacy audio trim/pad values using direct allocation and copy."""
waveform, sample_rate = validate_audio(audio)
waveform = waveform.detach().to(device="cpu")
sample_rate = int(sample_rate)
target_samples = int(round(int(target_frames) / FPS * sample_rate))
original_samples = int(waveform.shape[-1])
delta_samples = target_samples - original_samples
if delta_samples == 0:
return {"waveform": waveform, "sample_rate": sample_rate}, 0
output = torch.empty(
(*waveform.shape[:-1], target_samples),
dtype=waveform.dtype,
device="cpu",
)
if delta_samples < 0:
if preserve_final_audio_anchor and target_samples > 1:
tail_samples = min(
max(1, int(round(sample_rate / FPS))),
target_samples,
original_samples,
)
head_samples = target_samples - tail_samples
if head_samples > 0:
output[..., :head_samples].copy_(waveform[..., :head_samples])
output[..., head_samples:].copy_(waveform[..., -tail_samples:])
fade = min(
int(round(sample_rate * 0.005)),
head_samples,
tail_samples,
)
if fade > 0:
start = output[..., head_samples - 1 : head_samples].to(
dtype=output.dtype
)
alpha = torch.linspace(
0.0,
1.0,
fade,
dtype=output.dtype,
device=output.device,
).reshape(*([1] * (output.ndim - 1)), fade)
tail_head = output[..., head_samples : head_samples + fade]
tail_head.copy_(start * (1.0 - alpha) + tail_head * alpha)
else:
output.copy_(waveform[..., :target_samples])
else:
if original_samples > 0:
output[..., :original_samples].copy_(waveform)
output[..., original_samples:].copy_(
waveform[..., -1:].expand(
*waveform.shape[:-1],
delta_samples,
)
)
else:
output.zero_()
return {"waveform": output, "sample_rate": sample_rate}, delta_samples
def _prepare_video_seam(
*,
images: list[Any],
plan: dict[str, Any],
mode: str,
) -> tuple[list[Any], dict[int, torch.Tensor], dict[int, str], Exception | None]:
if mode == VIDEO_SEAM_OFF:
return [], {}, {}, None
analyses: list[Any] = []
try:
from .video_seam import analyze_decoded_boundaries
analyses = analyze_decoded_boundaries(
images=images,
assembly_plan=plan,
)
patches: dict[int, torch.Tensor] = {}
actions: dict[int, str] = {}
if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2):
from .video_seam_v35 import build_boundary_patches
patches, actions = build_boundary_patches(
images=images,
assembly_plan=plan,
analyses=analyses,
enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2,
)
return analyses, patches, actions, None
except Exception as exc:
# Match V3.4's all-native fallback: no partial patch is retained.
return analyses, {}, {}, exc
def _video_seam_status(mode: str) -> str:
if mode == VIDEO_SEAM_OFF:
return "Video seam correction is disabled."
if mode == VIDEO_SEAM_ANALYZE:
return "Video Seam: Analyze Only; decoded frames and audio are unchanged."
if mode == VIDEO_SEAM_AUTO:
return "Video Seam: Auto; guarded transient and micro-flash correction enabled."
return (
"Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, "
"and exposure-ramp correction enabled."
)
def _format_video_seam_lines(
*,
mode: str,
analyses: list[Any],
actions: dict[int, str],
error: Exception | None,
) -> list[str]:
if mode == VIDEO_SEAM_OFF:
return []
if error is not None:
return [
f"Video Seam {mode}: native output preserved; analysis unavailable "
f"({type(error).__name__}: {error})"
]
from .video_seam import format_video_boundary_analysis
lines = [
format_video_boundary_analysis(
item,
action=(
"analysis only"
if mode == VIDEO_SEAM_ANALYZE
else actions.get(item.boundary_index, "kept native boundary")
),
)
for item in analyses
]
if not lines:
lines.append(f"Video Seam {mode}: no decoded chunk boundary to analyze.")
return lines
def _format_elapsed(seconds: float) -> str:
total = max(0.0, float(seconds))
hours, remainder = divmod(total, 3600.0)
minutes, seconds = divmod(remainder, 60.0)
return f"{int(hours):02d}:{int(minutes):02d}:{seconds:05.2f}"
def _write_assembly(
*,
image_buffer: torch.Tensor,
images: list[Any],
audio: list[Any],
plan: dict[str, Any],
exact_total_duration: bool,
audio_seam: str,
video_seam: str,
diagnostics: str,
natural_frames: int,
output_frames: int,
video_spans: tuple[CopySpan, ...],
seam_patches: dict[int, torch.Tensor],
seam_analyses: list[Any],
seam_actions: dict[int, str],
seam_error: Exception | None,
buffer_backend: str,
auto_decision_report: str | None,
final_audio_override: Mapping[str, Any] | None,
final_audio_source: str | None,
interrupt_check: Callable[[], None] | None,
) -> tuple[dict[str, Any], str]:
diagnostics_mode = normalize_diagnostics_mode(diagnostics)
units = list(plan.get("decode_groups") or plan["chunks"])
first_waveform, audio_rate = validate_audio(audio[0])
first_waveform = first_waveform.detach().to(device="cpu")
audio_rate = int(audio_rate)
natural_samples = int(round(natural_frames / FPS * audio_rate))
audio_buffer = torch.empty(
(*first_waveform.shape[:-1], natural_samples),
dtype=first_waveform.dtype,
device="cpu",
)
reports = [
"H3 Continuum Assemble V3.5",
f"Audio Seam: {audio_seam}. {_video_seam_status(video_seam)}",
f"Buffer Backend: {buffer_backend}; IMAGE direct-write "
f"frames={output_frames}, bytes={image_buffer.numel() * image_buffer.element_size()}; "
"no final full IMAGE copy.",
]
if auto_decision_report is not None:
reports.append(auto_decision_report)
frame_cursor = 0
for index, (raw_images, raw_audio, unit) in enumerate(
zip(images, audio, units),
start=1,
):
_check_interrupted(interrupt_check)
total_frames = int(unit["total_frames"])
trim_frames = int(unit["trim_frames"])
net_frames = int(unit["net_frames"])
raw_images_cpu = raw_images.detach().to(device="cpu")
retained_images = raw_images_cpu[trim_frames:total_frames]
if int(retained_images.shape[0]) != net_frames:
raise ValueError(
f"decoded image chunk {index} retained {retained_images.shape[0]} "
f"frames; plan expects {net_frames}"
)
_copy_natural_interval(
image_buffer,
retained_images,
natural_start=frame_cursor,
spans=video_spans,
interrupt_check=interrupt_check,
)
_check_interrupted(interrupt_check)
waveform, sample_rate = validate_audio(raw_audio)
waveform = waveform.detach().to(device="cpu")
sample_rate = int(sample_rate)
if sample_rate != audio_rate:
raise ValueError(
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
)
frame_stop = frame_cursor + net_frames
sample_start = int(round(frame_cursor / FPS * sample_rate))
sample_stop = int(round(frame_stop / FPS * sample_rate))
seam_report = None
if index > 1 and audio_seam == AUDIO_SEAM_AUTO:
try:
cut_sample = int(round(trim_frames / FPS * sample_rate))
patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam(
audio_buffer[..., :sample_start],
waveform,
sample_rate=sample_rate,
cut_sample=cut_sample,
)
if patch is not None and int(patch.shape[-1]) > 0:
patch_start = sample_start - int(patch.shape[-1])
if patch_start < 0:
raise ValueError("audio seam patch starts before the output")
audio_buffer[..., patch_start:sample_start].copy_(patch)
seam_report = (
f"audio seam {index-1}->{index}: "
f"corr={metrics.correlation_before:.4f}->"
f"{metrics.correlation_after:.4f}, "
f"jump={metrics.boundary_jump_before:.6f}->"
f"{metrics.boundary_jump_after:.6f}, "
f"fade={fade_samples} samples"
)
if diagnostics_mode == DIAGNOSTICS_FULL:
seam_report += (
f", offset={metrics.offset_samples:+d}, "
f"gain={level_gain:.4f}, dc={dc_bias:+.6f}"
)
except Exception as exc:
seam_report = (
f"audio seam {index-1}->{index}: fallback to native boundary "
f"({type(exc).__name__}: {exc})"
)
trim_samples = int(round(trim_frames / FPS * sample_rate))
_copy_audio_segment(
audio_buffer[..., sample_start:sample_stop],
waveform,
trim_samples=trim_samples,
)
if diagnostics_mode != DIAGNOSTICS_OFF:
reports.append(
f"assembled decoded chunk {index}: {net_frames} retained frames, "
f"cumulative {frame_stop}/{natural_frames}"
)
if seam_report:
reports.append(seam_report)
frame_cursor = frame_stop
if frame_cursor != natural_frames:
raise RuntimeError(
f"V3.5 assembly cursor mismatch: {frame_cursor} != {natural_frames}"
)
unit_starts: list[int] = []
cursor = 0
for unit in units:
unit_starts.append(cursor)
cursor += int(unit["net_frames"])
for boundary, patch in seam_patches.items():
_check_interrupted(interrupt_check)
if boundary <= 0 or boundary >= len(unit_starts):
raise ValueError("V3.5 video seam patch boundary is outside decode groups")
_copy_natural_interval(
image_buffer,
patch,
natural_start=unit_starts[boundary],
spans=video_spans,
interrupt_check=interrupt_check,
)
if output_frames > natural_frames:
_repeat_final_video_frame(
image_buffer,
natural_frames=natural_frames,
output_frames=output_frames,
interrupt_check=interrupt_check,
)
_check_interrupted(interrupt_check)
result_audio: dict[str, Any] = {
"waveform": audio_buffer,
"sample_rate": audio_rate,
}
duration_report = ""
if exact_total_duration:
preserve_anchor = bool(
plan.get("preserve_final_frame", False)
and output_frames < natural_frames
and output_frames >= 2
)
result_audio, delta_samples = adjust_audio_duration_direct(
result_audio,
target_frames=output_frames,
preserve_final_audio_anchor=preserve_anchor,
)
adjustment = output_frames - natural_frames
mode = (
"final anchor preserved; pre-tail trim"
if preserve_anchor
else "tail trim/pad"
)
duration_report = (
f"Exact-duration adjustment: frames {natural_frames}->{output_frames} "
f"({adjustment:+d}), audio samples correction {delta_samples:+d}; {mode}."
)
driving_audio_report = ""
if final_audio_override is not None:
result_audio = dict(final_audio_override)
if exact_total_duration:
result_audio, _ = adjust_audio_duration_direct(
result_audio,
target_frames=output_frames,
preserve_final_audio_anchor=False,
)
waveform, selected_rate = validate_audio(result_audio)
result_audio = {
"waveform": waveform,
"sample_rate": int(selected_rate),
}
driving_audio_report = (
"Driving Audio: preserved source selected from "
f"{final_audio_source or 'direct input'}; "
f"sample_rate={int(selected_rate)}, samples={int(waveform.shape[-1])}; "
"generated audio and Audio Seam bypassed."
)
reports.append(
f"Assembled {len(units)} physical decoded group(s) into "
f"{output_frames} frames with cumulative sample-boundary alignment."
)
if duration_report:
reports.append(duration_report)
if driving_audio_report:
reports.append(driving_audio_report)
runtime_started_at = plan.get("_runtime_started_at")
if isinstance(runtime_started_at, (int, float)):
elapsed = time.perf_counter() - float(runtime_started_at)
if math.isfinite(elapsed) and elapsed >= 0.0:
reports.append(f"Total workflow elapsed: {_format_elapsed(elapsed)}")
reports.extend(
_format_video_seam_lines(
mode=video_seam,
analyses=seam_analyses,
actions=seam_actions,
error=seam_error,
)
)
return result_audio, "\n".join(reports)
def _assemble_decoded_chunks_v35_core(
*,
images: list[Any],
audio: list[Any],
assembly_plan: dict[str, Any],
exact_total_duration: bool,
audio_seam: str,
video_seam: str,
diagnostics: str,
buffer_backend: str,
backing_root: str | Path | None = None,
final_audio_override: Mapping[str, Any] | None = None,
final_audio_source: str | None = None,
interrupt_check: Callable[[], None] | None = None,
):
plan = validate_assembly_plan(assembly_plan)
if audio_seam not in AUDIO_SEAM_OPTIONS:
raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}")
if video_seam not in VIDEO_SEAM_ANALYSIS_OPTIONS:
raise ValueError(f"unknown Video Seam mode: {video_seam!r}")
if buffer_backend not in BUFFER_BACKEND_OPTIONS:
raise ValueError(f"unknown Buffer Backend: {buffer_backend!r}")
units = list(plan.get("decode_groups") or plan["chunks"])
natural_frames = sum(int(unit["net_frames"]) for unit in units)
output_frames = (
int(plan["target_frames"])
if bool(exact_total_duration)
else natural_frames
)
if natural_frames <= 0 or output_frames <= 0:
raise ValueError("V3.5 assembly frame counts must be positive")
first_image = images[0]
shape = (output_frames, *tuple(int(value) for value in first_image.shape[1:]))
video_spans = _video_copy_spans(
natural_frames=natural_frames,
output_frames=output_frames,
preserve_final_frame=bool(plan.get("preserve_final_frame", False)),
)
analyses, patches, actions, analysis_error = _prepare_video_seam(
images=images,
plan=plan,
mode=video_seam,
)
resolved_backend = buffer_backend
resolved_backing_root = backing_root
auto_decision_report = None
if buffer_backend == BUFFER_BACKEND_AUTO:
final_image_bytes = (
math.prod(shape) * first_image.element_size()
)
decision = select_auto_buffer_backend(
final_image_bytes,
backing_root=backing_root,
)
resolved_backend = decision.selected_backend
resolved_backing_root = decision.backing_root
auto_decision_report = format_auto_buffer_decision(decision)
allocation: Any
if resolved_backend == BUFFER_BACKEND_RAM:
allocation = _RamAllocation(shape, first_image.dtype)
else:
allocation = allocate_file_backed_image(
shape,
dtype=first_image.dtype,
backing_root=resolved_backing_root,
)
with allocation:
# The file exists at this point, so an ESC received immediately before
# assembly exercises the same unpublished cleanup path as a mid-copy
# cancellation.
_check_interrupted(interrupt_check)
result_audio, report = _write_assembly(
image_buffer=allocation.tensor,
images=images,
audio=audio,
plan=plan,
exact_total_duration=bool(exact_total_duration),
audio_seam=audio_seam,
video_seam=video_seam,
diagnostics=diagnostics,
natural_frames=natural_frames,
output_frames=output_frames,
video_spans=video_spans,
seam_patches=patches,
seam_analyses=analyses,
seam_actions=actions,
seam_error=analysis_error,
buffer_backend=(
resolved_backend
if buffer_backend != BUFFER_BACKEND_AUTO
else f"Auto -> {resolved_backend}"
),
auto_decision_report=auto_decision_report,
final_audio_override=final_audio_override,
final_audio_source=final_audio_source,
interrupt_check=interrupt_check,
)
result_images = allocation.publish()
return result_images, result_audio, report
def assemble_decoded_chunks_v35(
*,
images: list[Any],
audio: list[Any],
assembly_plan: dict[str, Any],
exact_total_duration: bool,
audio_seam: str,
video_seam: str,
diagnostics: str,
buffer_backend: str,
backing_root: str | Path | None = None,
final_audio_override: Mapping[str, Any] | None = None,
final_audio_source: str | None = None,
interrupt_check: Callable[[], None] | None = None,
):
"""Preflight all inputs, then execute the independent V3.5 writer."""
kwargs = {
"images": images,
"audio": audio,
"assembly_plan": assembly_plan,
"exact_total_duration": bool(exact_total_duration),
"audio_seam": str(audio_seam),
"video_seam": str(video_seam),
"diagnostics": str(diagnostics),
"buffer_backend": str(buffer_backend),
"backing_root": backing_root,
"final_audio_override": final_audio_override,
"final_audio_source": final_audio_source,
"interrupt_check": interrupt_check,
}
return assemble_with_hardening(
_assemble_decoded_chunks_v35_core,
(),
kwargs,
)
__all__ = [
"BUFFER_BACKEND_DISK",
"BUFFER_BACKEND_AUTO",
"BUFFER_BACKEND_OPTIONS",
"BUFFER_BACKEND_RAM",
"CopySpan",
"adjust_audio_duration_direct",
"assemble_decoded_chunks_v35",
]
+170 -1
View File
@@ -6,12 +6,26 @@ import torch
from ..constants import FPS
from ..driving_audio import prepare_driving_audio_source
from ..hardening import diagnostics_is_full
from ..reference_video import (
REFERENCE_VIDEO_SIZE_EFFICIENT,
REFERENCE_VIDEO_SIZE_OPTIONS,
)
from ..v2.decoder import enforce_total_frames
from .assembly import H3ContinuumAssembleSeamExperimental
from .assembly import (
H3ContinuumAssembleSeamExperimental,
_chunk_list,
_singleton,
)
from .assembly_v35 import (
BUFFER_BACKEND_AUTO,
BUFFER_BACKEND_OPTIONS,
assemble_decoded_chunks_v35,
)
from .memory_attribution import (
format_assembly_projection,
project_assembly_buffers,
)
from .nodes import CATEGORY as CONTINUUM_CATEGORY, H3ContinuumSamplerProduction
@@ -136,6 +150,7 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction):
audio_vae=None,
reference_video_1=None,
video_reference_size=REFERENCE_VIDEO_SIZE_EFFICIENT,
capture_refine_context=False,
**kwargs,
):
from ..reference_video import prepare_reference_video_source
@@ -165,6 +180,7 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction):
driving_audio_source=source,
driving_audio_vae=audio_vae,
reference_video_source=reference_video_source,
capture_refine_context=bool(capture_refine_context),
**kwargs,
)
selected_audio = _copy_audio(source.source_audio) if source is not None else None
@@ -172,9 +188,46 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction):
assembly_plan = dict(outputs[2])
assembly_plan[_DRIVING_AUDIO_PLAN_KEY] = _copy_audio(selected_audio)
outputs = (*outputs[:2], assembly_plan, *outputs[3:])
if bool(capture_refine_context):
return (*outputs[:4], selected_audio, outputs[4])
return (*outputs, selected_audio)
class H3ContinuumSamplerV35(H3ContinuumSamplerV34):
"""V3.5 sampler with a physical-group First Pass refine context output."""
DEPRECATED = False
CATEGORY = CONTINUUM_CATEGORY
DESCRIPTION = (
"H3 Continuum V3.5 sampler. It preserves the V3.4 outputs and adds one "
"physical-group refine context for context-aware Hi-Res Fix or an "
"external latent Second Pass."
)
SEARCH_ALIASES = [
"H3 Continuum Sampler V3.5",
"MiniMax H3 context-aware Hi-Res Fix",
]
RETURN_TYPES = (
*H3ContinuumSamplerV34.RETURN_TYPES,
"H3_CONTINUUM_REFINE_CONTEXT",
)
RETURN_NAMES = (
*H3ContinuumSamplerV34.RETURN_NAMES,
"refine_context",
)
OUTPUT_IS_LIST = (*H3ContinuumSamplerV34.OUTPUT_IS_LIST, False)
def run(self, *args, **kwargs):
kwargs.pop("capture_refine_context", None)
kwargs.pop("memory_attribution", None)
return super().run(
*args,
capture_refine_context=True,
memory_attribution=True,
**kwargs,
)
class H3ContinuumAssembleSeamV34(H3ContinuumAssembleSeamExperimental):
"""Assemble decoded chunks and select original Driving Audio when connected."""
@@ -239,12 +292,128 @@ class H3ContinuumAssembleSeamV34(H3ContinuumAssembleSeamExperimental):
return images, selected, report
def _assembly_argument(args, kwargs, name, index):
if name in kwargs:
return kwargs[name]
if len(args) > index:
return args[index]
return None
def _insert_projection_report(report, line):
lines = str(report).splitlines()
for index, value in enumerate(lines):
if value.startswith("Memory [assemble preflight]"):
lines.insert(index + 1, line)
return "\n".join(lines)
lines.append(line)
return "\n".join(lines)
class H3ContinuumAssembleSeamV35(H3ContinuumAssembleSeamV34):
"""Independent V3.5 direct-write assembly with selectable IMAGE storage."""
DEPRECATED = False
CATEGORY = CONTINUUM_CATEGORY
DESCRIPTION = (
"V3.5 decoded-chunk assembler with Auto, RAM, or Disk-backed IMAGE "
"storage, direct Exact Duration writes, and small-patch Video Seam."
)
@classmethod
def INPUT_TYPES(cls):
schema = super().INPUT_TYPES()
required = dict(schema["required"])
diagnostics = required.pop("diagnostics")
required["buffer_backend"] = (
BUFFER_BACKEND_OPTIONS,
{
"default": BUFFER_BACKEND_AUTO,
"display_name": "Buffer Backend",
"tooltip": (
"Auto keeps outputs up to 4 GiB in RAM only when physical "
"memory has a safety reserve; otherwise it maps the final "
"IMAGE to disk. Manual RAM and Disk-backed remain available."
),
},
)
required["diagnostics"] = diagnostics
schema["required"] = required
return schema
def assemble(
self,
images,
audio,
assembly_plan,
exact_total_duration,
audio_seam,
video_seam,
buffer_backend,
diagnostics,
driving_audio=None,
):
image_chunks = _chunk_list(images)
audio_chunks = _chunk_list(audio)
plan = _singleton(assembly_plan, "assembly_plan")
exact = bool(_singleton(exact_total_duration, "exact_total_duration"))
audio_seam_mode = str(_singleton(audio_seam, "audio_seam"))
video_seam_mode = str(_singleton(video_seam, "video_seam"))
backend = str(_singleton(buffer_backend, "buffer_backend"))
diagnostics_mode = str(_singleton(diagnostics, "diagnostics"))
preserved_audio = _copy_audio(plan.get(_DRIVING_AUDIO_PLAN_KEY))
selected_audio = preserved_audio or _copy_audio(driving_audio)
selected_audio_source = None
if selected_audio is not None:
selected_audio_source = (
"assembly plan" if preserved_audio is not None else "direct input"
)
projection_line = None
if diagnostics_is_full(diagnostics_mode):
try:
projection = project_assembly_buffers(
images=image_chunks,
audio=audio_chunks,
assembly_plan=plan,
exact_total_duration=exact,
)
projection_line = format_assembly_projection(projection) + (
"; V3.5 Exact Duration IMAGE write=direct; "
"additional final IMAGE copy=no"
)
except Exception as exc:
projection_line = (
"Projected assembly buffers [V3.5]: unavailable "
f"({type(exc).__name__}: {exc})"
)
result_images, result_audio, report = assemble_decoded_chunks_v35(
images=image_chunks,
audio=audio_chunks,
assembly_plan=plan,
exact_total_duration=exact,
audio_seam=audio_seam_mode,
video_seam=video_seam_mode,
diagnostics=diagnostics_mode,
buffer_backend=backend,
final_audio_override=selected_audio,
final_audio_source=selected_audio_source,
)
if projection_line is not None:
report = _insert_projection_report(report, projection_line)
return result_images, result_audio, report
NODE_CLASS_MAPPINGS = {
"H3ContinuumSamplerV34": H3ContinuumSamplerV34,
"H3ContinuumSamplerV35": H3ContinuumSamplerV35,
"H3ContinuumAssembleSeamV34": H3ContinuumAssembleSeamV34,
"H3ContinuumAssembleSeamV35": H3ContinuumAssembleSeamV35,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumSamplerV34": "H3 Continuum Sampler V3.4",
"H3ContinuumSamplerV35": "H3 Continuum Sampler V3.5",
"H3ContinuumAssembleSeamV34": "H3 Continuum Assemble + Seam V3.4",
"H3ContinuumAssembleSeamV35": "H3 Continuum Assemble + Seam V3.5",
}
+257
View File
@@ -0,0 +1,257 @@
"""V3.5 adapter for the accepted P2-0 file-backed Tensor backend.
The storage and lifetime implementation remains in the P2-0 module so the
Windows-tested locking, sidecar, cancellation, and alias-lifetime contracts
have one implementation. This module only exposes the small Production API
needed by the V3.5 assembler.
"""
from __future__ import annotations
from dataclasses import dataclass
import os
from pathlib import Path
import shutil
import tempfile
import threading
import torch
from ..tools.p20_file_backed_tensor_poc import (
FileBackedAllocation,
StorageLifetimeManager,
cleanup_stale_files,
get_process_lifetime_manager,
)
BUFFER_BACKEND_AUTO = "Auto"
BUFFER_BACKEND_RAM = "RAM"
BUFFER_BACKEND_DISK = "Disk-backed"
BUFFER_BACKEND_OPTIONS = (
BUFFER_BACKEND_AUTO,
BUFFER_BACKEND_RAM,
BUFFER_BACKEND_DISK,
)
AUTO_RAM_IMAGE_LIMIT_BYTES = 4 * 1024**3
AUTO_RAM_RESERVE_MIN_BYTES = 4 * 1024**3
AUTO_RAM_RESERVE_FRACTION = 0.10
AUTO_DISK_RESERVE_BYTES = 2 * 1024**3
_BACKING_DIRECTORY_NAME = "h3-continuum-v35-file-backed"
_CLEANED_ROOTS: set[str] = set()
_CLEANED_ROOTS_LOCK = threading.Lock()
def validate_buffer_backend(value: str) -> str:
"""Validate the explicit or automatic V3.5 IMAGE backend request."""
if value not in BUFFER_BACKEND_OPTIONS:
raise ValueError(f"unknown Buffer Backend: {value!r}")
return value
@dataclass(frozen=True)
class AutoBufferDecision:
selected_backend: str
final_image_bytes: int
backing_root: Path
available_memory_bytes: int | None
total_memory_bytes: int | None
memory_reserve_bytes: int | None
ram_image_limit_bytes: int
disk_free_bytes: int | None
disk_reserve_bytes: int
reason: str
def _system_memory_bytes() -> tuple[int | None, int | None]:
try:
import psutil
counters = psutil.virtual_memory()
return int(counters.available), int(counters.total)
except Exception:
return None, None
def _existing_disk_anchor(path: Path) -> Path:
candidate = path
while not candidate.exists() and candidate != candidate.parent:
candidate = candidate.parent
if not candidate.exists():
raise OSError(f"no existing filesystem anchor for backing root: {path}")
return candidate
def _disk_free_bytes(path: Path) -> int:
return int(shutil.disk_usage(_existing_disk_anchor(path)).free)
def select_auto_buffer_backend(
final_image_bytes: int,
*,
backing_root: str | os.PathLike[str] | None = None,
) -> AutoBufferDecision:
"""Choose RAM only when both output size and physical headroom are safe.
Auto never converts a Disk-backed decision into RAM after a disk-space or
allocation failure. Users can still explicitly select either manual path.
"""
final_bytes = int(final_image_bytes)
if final_bytes <= 0:
raise ValueError("Auto Buffer Backend requires a positive final IMAGE size")
root = resolve_backing_root(backing_root)
available, total = _system_memory_bytes()
reserve = None
if available is not None and total is not None and total > 0:
reserve = max(
AUTO_RAM_RESERVE_MIN_BYTES,
int(total * AUTO_RAM_RESERVE_FRACTION),
)
if (
reserve is not None
and final_bytes <= AUTO_RAM_IMAGE_LIMIT_BYTES
and available >= final_bytes + reserve
):
return AutoBufferDecision(
selected_backend=BUFFER_BACKEND_RAM,
final_image_bytes=final_bytes,
backing_root=root,
available_memory_bytes=available,
total_memory_bytes=total,
memory_reserve_bytes=reserve,
ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES,
disk_free_bytes=None,
disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES,
reason="final IMAGE fits the Auto RAM size limit and safety reserve",
)
if reserve is None:
reason = "physical-memory counters are unavailable"
elif final_bytes > AUTO_RAM_IMAGE_LIMIT_BYTES:
reason = "final IMAGE exceeds the Auto RAM size limit"
else:
reason = "available physical memory is below final IMAGE plus reserve"
try:
disk_free = _disk_free_bytes(root)
except Exception as exc:
raise RuntimeError(
"Auto Buffer Backend selected Disk-backed but disk capacity could "
f"not be measured for {root}: {type(exc).__name__}: {exc}. "
"Select RAM manually only if sufficient memory is known to be available."
) from exc
required_disk = final_bytes + AUTO_DISK_RESERVE_BYTES
if disk_free < required_disk:
raise RuntimeError(
"Auto Buffer Backend selected Disk-backed but the backing volume "
f"has {disk_free} free bytes; {required_disk} bytes are required "
f"for the final IMAGE plus the {AUTO_DISK_RESERVE_BYTES}-byte safety "
"reserve. Free disk space or select RAM manually only if sufficient "
"memory is known to be available."
)
return AutoBufferDecision(
selected_backend=BUFFER_BACKEND_DISK,
final_image_bytes=final_bytes,
backing_root=root,
available_memory_bytes=available,
total_memory_bytes=total,
memory_reserve_bytes=reserve,
ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES,
disk_free_bytes=disk_free,
disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES,
reason=reason,
)
def format_auto_buffer_decision(decision: AutoBufferDecision) -> str:
def value(number: int | None) -> str:
return "n/a" if number is None else str(int(number))
return (
"Auto Buffer Decision: "
f"selected={decision.selected_backend}, "
f"final_IMAGE={decision.final_image_bytes} bytes, "
f"system_available={value(decision.available_memory_bytes)} bytes, "
f"system_total={value(decision.total_memory_bytes)} bytes, "
f"RAM_reserve={value(decision.memory_reserve_bytes)} bytes, "
f"RAM_IMAGE_limit={decision.ram_image_limit_bytes} bytes, "
f"disk_free={value(decision.disk_free_bytes)} bytes, "
f"disk_reserve={decision.disk_reserve_bytes} bytes; "
f"reason={decision.reason}."
)
def resolve_backing_root(
backing_root: str | os.PathLike[str] | None = None,
) -> Path:
"""Resolve the dedicated backing root without importing ComfyUI eagerly."""
if backing_root is not None:
return Path(backing_root).expanduser().resolve()
try:
import folder_paths
except ImportError:
base = Path(tempfile.gettempdir())
else:
value = folder_paths.get_temp_directory()
if value is None or not str(value).strip():
raise RuntimeError("ComfyUI returned an empty temporary directory")
base = Path(value)
return (base.expanduser().resolve() / _BACKING_DIRECTORY_NAME).resolve()
def get_file_backed_image_manager(
backing_root: str | os.PathLike[str] | None = None,
) -> StorageLifetimeManager:
"""Return the accepted process-global manager for one resolved root."""
root = resolve_backing_root(backing_root)
key = os.path.normcase(str(root))
with _CLEANED_ROOTS_LOCK:
if key not in _CLEANED_ROOTS:
# The P2-0 root lock prevents cleanup from observing an allocation
# between its data-file and sidecar publication. A live owner is
# always retained; only prior abnormal-exit artifacts are removed.
cleanup_stale_files(root, grace_seconds=0.0)
_CLEANED_ROOTS.add(key)
return get_process_lifetime_manager(root)
def allocate_file_backed_image(
shape: tuple[int, ...],
*,
dtype: torch.dtype,
backing_root: str | os.PathLike[str] | None = None,
) -> FileBackedAllocation:
"""Create a provisional Disk-backed IMAGE allocation.
The caller must use the returned context manager and call ``publish()``
only after every output element has been written. Backend failures are
propagated; this manual Disk-backed path never falls back to RAM.
"""
manager = get_file_backed_image_manager(backing_root)
return manager.allocate(shape, dtype=dtype)
__all__ = [
"AUTO_DISK_RESERVE_BYTES",
"AUTO_RAM_IMAGE_LIMIT_BYTES",
"AUTO_RAM_RESERVE_FRACTION",
"AUTO_RAM_RESERVE_MIN_BYTES",
"AutoBufferDecision",
"BUFFER_BACKEND_AUTO",
"BUFFER_BACKEND_DISK",
"BUFFER_BACKEND_OPTIONS",
"BUFFER_BACKEND_RAM",
"allocate_file_backed_image",
"format_auto_buffer_decision",
"get_file_backed_image_manager",
"resolve_backing_root",
"select_auto_buffer_backend",
"validate_buffer_backend",
]
+463
View File
@@ -0,0 +1,463 @@
"""V3.5 Continuum-aware Hi-res fix convenience node."""
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from typing import Any
from .pixel_vae_resize import (
resize_video_latents_via_vae,
resolve_pixel_latent_scale,
)
from .second_pass import SecondPassContractError, run_second_pass_groups
_MISSING = object()
HIRES_UPSCALE_METHODS = (
"lanczos",
"nearest-exact",
"bilinear",
"area",
"bicubic",
"bislerp",
)
_CONDITIONING_UPSCALE_METHOD = "bilinear"
def _single_list_input(name: str, value: Any) -> Any:
if not isinstance(value, list) or len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
return value[0]
def _enabled_value(value: Any) -> bool:
if isinstance(value, list):
value = _single_list_input("enabled", value)
return bool(value)
def _lazy_missing(value: Any) -> bool:
return value is None or (
isinstance(value, (list, tuple))
and len(value) == 1
and value[0] is None
)
def h3_core_native_canvas(
width: int,
height: int,
*,
adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None,
) -> tuple[int, int]:
"""Return the native canvas selected by ComfyUI Core for this aspect ratio."""
if width < 1 or height < 1:
raise SecondPassContractError("source pixel width and height must be positive")
if adapt_canvas_fn is None:
from comfy_extras.nodes_minimax_h3 import adapt_canvas
adapt_canvas_fn = adapt_canvas
target = adapt_canvas_fn(int(width), int(height))
if not isinstance(target, Sequence) or len(target) != 2:
raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry")
target_width, target_height = (int(value) for value in target)
if target_width < 1 or target_height < 1:
raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry")
return target_width, target_height
def resolve_h3_native_latent_target(
video_latents: Sequence[Mapping[str, Any]],
assembly_plan: Mapping[str, Any],
*,
adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None,
) -> dict[str, int | bool]:
"""Resolve Core's native canvas without shrinking an existing video latent."""
contract = assembly_plan.get("second_pass_contract")
if not isinstance(contract, Mapping):
raise SecondPassContractError("assembly_plan has no second_pass_contract")
groups = contract.get("physical_groups")
if not isinstance(groups, list) or not groups:
raise SecondPassContractError(
"second_pass_contract.physical_groups must be non-empty"
)
if not isinstance(video_latents, Sequence) or not video_latents:
raise SecondPassContractError("video_latents must be a non-empty sequence")
if len(video_latents) != len(groups):
raise SecondPassContractError("video latent group count changed")
first_group = groups[0]
if not isinstance(first_group, Mapping):
raise SecondPassContractError("physical group metadata must be a mapping")
source_width = int(first_group.get("source_width", -1))
source_height = int(first_group.get("source_height", -1))
source_latent_width = int(first_group.get("source_latent_w", -1))
source_latent_height = int(first_group.get("source_latent_h", -1))
if min(source_width, source_height, source_latent_width, source_latent_height) < 1:
raise SecondPassContractError("source pixel/latent geometry is invalid")
if source_width % source_latent_width or source_height % source_latent_height:
raise SecondPassContractError("source pixel/latent geometry is not integral")
pixel_scale_width = source_width // source_latent_width
pixel_scale_height = source_height // source_latent_height
current_geometry: tuple[int, int] | None = None
for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)):
if not isinstance(group, Mapping):
raise SecondPassContractError("physical group metadata must be a mapping")
if int(group.get("source_width", -1)) != source_width or int(
group.get("source_height", -1)
) != source_height:
raise SecondPassContractError("source geometry differs between physical groups")
if int(group.get("source_latent_w", -1)) != source_latent_width or int(
group.get("source_latent_h", -1)
) != source_latent_height:
raise SecondPassContractError(
"source latent geometry differs between physical groups"
)
if not isinstance(latent, Mapping) or "samples" not in latent:
raise SecondPassContractError(
f"video_latents[{position}] must be a LATENT mapping with samples"
)
samples = latent["samples"]
if not hasattr(samples, "shape") or len(samples.shape) != 5:
raise SecondPassContractError("video latent must have shape [B,C,T,H,W]")
geometry = (int(samples.shape[-2]), int(samples.shape[-1]))
if current_geometry is None:
current_geometry = geometry
elif geometry != current_geometry:
raise SecondPassContractError(
"all physical groups must use the same current H/W"
)
assert current_geometry is not None
native_width, native_height = h3_core_native_canvas(
source_width,
source_height,
adapt_canvas_fn=adapt_canvas_fn,
)
if native_width % pixel_scale_width or native_height % pixel_scale_height:
raise SecondPassContractError(
"ComfyUI Core native canvas is not aligned to the H3 latent scale"
)
native_latent_width = native_width // pixel_scale_width
native_latent_height = native_height // pixel_scale_height
current_height, current_width = current_geometry
preserve_current = (
native_latent_width < current_width
or native_latent_height < current_height
)
if preserve_current:
target_latent_width = current_width
target_latent_height = current_height
else:
target_latent_width = native_latent_width
target_latent_height = native_latent_height
return {
"source_width": source_width,
"source_height": source_height,
"native_width": native_width,
"native_height": native_height,
"target_width": target_latent_width * pixel_scale_width,
"target_height": target_latent_height * pixel_scale_height,
"target_latent_width": target_latent_width,
"target_latent_height": target_latent_height,
"preserved_current": preserve_current,
}
def _prepend_resize_status(
second_pass_status: str,
*,
backend: str,
mode: str,
upscale_method: str,
scale_by: float,
source_width: int,
source_height: int,
target_width: int,
target_height: int,
native_width: int,
native_height: int,
) -> str:
lines = [
"H3 Continuum Hi-Res Fix V3.5",
"Pixel/VAE resize: "
f"backend={backend}, mode={mode}, method={upscale_method}, "
f"scale_by={scale_by:g}, "
f"source={source_width}x{source_height}, "
f"target={target_width}x{target_height}, "
f"core_native={native_width}x{native_height}.",
]
if target_width > native_width or target_height > native_height:
lines.append(
"INFO: target exceeds the ComfyUI Core H3 base canvas; the Pixel/VAE "
"roundtrip is preserved and resource use may increase."
)
lines.append(second_pass_status)
return "\n".join(lines)
class H3ContinuumHiResFixV35:
"""Pixel/VAE resize and low-sigma refine complete physical video groups."""
DEPRECATED = False
DESCRIPTION = (
"Continuum-aware Hi-res fix. When enabled, each first-pass physical video "
"LATENT is decoded with the connected H3 Video VAE, resized in pixel space, "
"re-encoded with the same VAE, and refined once with the workflow SIGMAS. "
"scale_by 0 uses ComfyUI Core's H3 native canvas; positive values use the "
"manual multiplier. Connect the optional Continuum refine context to preserve "
"First/Last/Reference conditioning at the target geometry. "
"When disabled, first-pass video/audio LATENT objects and the original assembly "
"plan are returned unchanged."
)
@classmethod
def INPUT_TYPES(cls):
lazy = {"lazy": True}
return {
"required": {
"model": ("MODEL", lazy),
"clip": ("CLIP", lazy),
"sampler": ("SAMPLER", lazy),
"sigmas": ("SIGMAS", lazy),
"video_latents": ("LATENT",),
"audio_latents": ("LATENT",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"enabled": (
"BOOLEAN",
{
"default": True,
"tooltip": (
"Enable Resize + Second Pass. Off returns the first-pass "
"video/audio LATENT objects and assembly plan unchanged."
),
},
),
"upscale_method": (
HIRES_UPSCALE_METHODS,
{"default": "lanczos"},
),
"scale_by": (
"FLOAT",
{
"default": 2.0,
"min": 0.0,
"max": 4.0,
"step": 0.01,
"tooltip": (
"2.0 = validated 2x Pixel/VAE Hi-res Fix. 0 uses the "
"ComfyUI Core H3 native canvas automatically. Manual "
"values follow the existing Second Pass contract and must "
"be 1.0 or greater."
),
},
),
"refine_seed": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
"tooltip": "Base seed independently derived once per physical group.",
},
),
},
"optional": {
"refine_context": (
"H3_CONTINUUM_REFINE_CONTEXT",
{"lazy": True},
),
"video_vae": (
"VAE",
{
"lazy": True,
"tooltip": (
"Required while enabled for the safe Pixel/VAE roundtrip; "
"not evaluated while disabled."
),
},
),
},
}
INPUT_IS_LIST = True
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
RETURN_NAMES = (
"video_latents",
"audio_latents",
"updated_assembly_plan",
"status",
)
OUTPUT_IS_LIST = (True, True, False, False)
FUNCTION = "apply"
CATEGORY = "MiniMax H3/Continuum"
def check_lazy_status(
self,
enabled,
model=None,
clip=None,
sampler=None,
sigmas=None,
refine_context=_MISSING,
video_vae=_MISSING,
**_kwargs,
):
if not _enabled_value(enabled):
return []
required = [
name
for name, value in (
("model", model),
("clip", clip),
("sampler", sampler),
("sigmas", sigmas),
)
if _lazy_missing(value)
]
required.extend(
name
for name, value in (
("refine_context", refine_context),
("video_vae", video_vae),
)
if value is not _MISSING and _lazy_missing(value)
)
return required
def apply(
self,
model,
clip,
sampler,
sigmas,
video_latents,
audio_latents,
assembly_plan,
enabled,
upscale_method,
scale_by,
refine_seed,
refine_context=None,
video_vae=None,
):
plan = _single_list_input("assembly_plan", assembly_plan)
if not _enabled_value(enabled):
return (
video_latents,
audio_latents,
plan,
"H3 Continuum Hi-Res Fix V3.5: disabled; first-pass video/audio "
"LATENT objects and assembly_plan returned unchanged.",
)
method = str(_single_list_input("upscale_method", upscale_method))
scale = float(_single_list_input("scale_by", scale_by))
if 0.0 < scale < 1.0:
raise SecondPassContractError(
"scale_by must be 0 for H3 native Auto or at least 1.0; the "
"existing Second Pass contract preserves or enlarges spatial size"
)
vae = (
None
if video_vae is None
else _single_list_input("video_vae", video_vae)
)
if vae is None:
raise SecondPassContractError(
"Hi-Res Fix enabled requires video_vae for the safe Pixel/VAE "
"roundtrip; legacy direct latent interpolation is disabled"
)
source_width = int(plan["width"])
source_height = int(plan["height"])
native_width, native_height = h3_core_native_canvas(
source_width,
source_height,
)
first_samples = video_latents[0]["samples"]
current_geometry = (
int(first_samples.shape[-2]),
int(first_samples.shape[-1]),
)
if scale == 0.0:
native_target = resolve_h3_native_latent_target(video_latents, plan)
target_latent_width = int(native_target["target_latent_width"])
target_latent_height = int(native_target["target_latent_height"])
target_width = int(native_target["target_width"])
target_height = int(native_target["target_height"])
resize_mode = "h3_native_canvas_auto"
else:
pixel_scale_width, pixel_scale_height = resolve_pixel_latent_scale(plan)
target_latent_width = round(current_geometry[1] * scale)
target_latent_height = round(current_geometry[0] * scale)
if target_latent_width < 1 or target_latent_height < 1:
raise SecondPassContractError(
"scale_by produced a non-positive target geometry"
)
target_width = target_latent_width * pixel_scale_width
target_height = target_latent_height * pixel_scale_height
resize_mode = "manual_scale_by"
if current_geometry == (target_latent_height, target_latent_width):
resized_video_latents = list(video_latents)
resize_backend = "direct_second_pass_no_resize"
else:
resized_video_latents = resize_video_latents_via_vae(
video_latents,
plan,
video_vae=vae,
target_width=target_width,
target_height=target_height,
upscale_method=method,
)
resize_backend = "sequential_pixel_vae_roundtrip"
refined_video, output_audio, updated_plan, status = run_second_pass_groups(
model=_single_list_input("model", model),
clip=_single_list_input("clip", clip),
sampler=_single_list_input("sampler", sampler),
sigmas=_single_list_input("sigmas", sigmas),
video_latents=resized_video_latents,
audio_latents=audio_latents,
assembly_plan=plan,
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
refine_context=(
None
if refine_context is None
else _single_list_input("refine_context", refine_context)
),
video_vae=vae,
conditioning_upscale_method=_CONDITIONING_UPSCALE_METHOD,
)
return (
refined_video,
output_audio,
updated_plan,
_prepend_resize_status(
status,
backend=resize_backend,
mode=resize_mode,
upscale_method=method,
scale_by=scale,
source_width=source_width,
source_height=source_height,
target_width=int(updated_plan["width"]),
target_height=int(updated_plan["height"]),
native_width=native_width,
native_height=native_height,
),
)
NODE_CLASS_MAPPINGS = {
"H3ContinuumHiResFixV35": H3ContinuumHiResFixV35,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumHiResFixV35": "H3 Continuum Hi-Res Fix V3.5",
}
+140
View File
@@ -0,0 +1,140 @@
"""V3.5 Core-compatible spatial resize for physical video latents."""
from __future__ import annotations
from collections.abc import Callable
from typing import Any
UPSCALE_METHODS = (
"nearest-exact",
"bilinear",
"area",
"bicubic",
"bislerp",
)
def _single_list_input(name: str, value: Any) -> Any:
if not isinstance(value, list) or len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
return value[0]
def resize_video_latents(
video_latents: list[dict[str, Any]],
*,
upscale_method: str,
scale_by: float,
upscale_fn: Callable[..., Any] | None = None,
) -> list[dict[str, Any]]:
"""Resize only spatial H/W through the ComfyUI Core latent implementation."""
if upscale_fn is None:
from comfy.utils import common_upscale
upscale_fn = common_upscale
resized: list[dict[str, Any]] = []
for latent in video_latents:
samples = latent["samples"]
width = round(samples.shape[-1] * scale_by)
height = round(samples.shape[-2] * scale_by)
output = latent.copy()
output["samples"] = upscale_fn(
samples,
width,
height,
upscale_method,
"disabled",
)
resized.append(output)
return resized
def resize_video_latents_to_size(
video_latents: list[dict[str, Any]],
*,
upscale_method: str,
target_width: int,
target_height: int,
upscale_fn: Callable[..., Any] | None = None,
) -> list[dict[str, Any]]:
"""Resize every physical video group to one exact latent H/W."""
if target_width < 1 or target_height < 1:
raise ValueError("target latent width and height must be positive")
if upscale_fn is None:
from comfy.utils import common_upscale
upscale_fn = common_upscale
resized: list[dict[str, Any]] = []
for latent in video_latents:
samples = latent["samples"]
output = latent.copy()
output["samples"] = upscale_fn(
samples,
int(target_width),
int(target_height),
upscale_method,
"disabled",
)
resized.append(output)
return resized
class H3ContinuumLatentResizeV35:
"""List-aware Core spatial resize for Continuum physical video groups."""
DEPRECATED = False
DESCRIPTION = (
"V3.5 Hi-res fix resize. Spatially resizes every Continuum "
"physical video LATENT with the selected ComfyUI Core interpolation method. "
"B/C/T, physical group order, and LATENT metadata are preserved."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"video_latents": ("LATENT",),
"upscale_method": (UPSCALE_METHODS,),
"scale_by": (
"FLOAT",
{
"default": 2.0,
"min": 0.01,
"max": 8.0,
"step": 0.01,
},
),
}
}
INPUT_IS_LIST = True
RETURN_TYPES = ("LATENT",)
RETURN_NAMES = ("resized_video_latents",)
OUTPUT_IS_LIST = (True,)
FUNCTION = "resize"
CATEGORY = "MiniMax H3/Continuum/Utilities"
def resize(self, video_latents, upscale_method, scale_by):
return (
resize_video_latents(
video_latents,
upscale_method=str(
_single_list_input("upscale_method", upscale_method)
),
scale_by=float(_single_list_input("scale_by", scale_by)),
),
)
NODE_CLASS_MAPPINGS = {
"H3ContinuumLatentResizeV35": H3ContinuumLatentResizeV35,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumLatentResizeV35": "H3 Continuum Latent Resize V3.5",
}
+294
View File
@@ -0,0 +1,294 @@
"""V3.5-only, non-synchronizing memory attribution helpers."""
from __future__ import annotations
from dataclasses import dataclass
import math
from typing import Any, Mapping
import torch
from ..constants import FPS
from ..hardening import tensor_storage_stats
from ..state import extract_av_streams
from .plan import validate_assembly_plan
_MIB = 1024**2
def _mib(value: int | None) -> str:
if value is None:
return "n/a"
return f"{int(value) / _MIB:.1f} MiB"
def _bytes_and_mib(value: int) -> str:
return f"{int(value)} bytes ({int(value) / _MIB:.1f} MiB)"
def _process_counters() -> tuple[int | None, int | None]:
try:
import psutil
return (
int(psutil.Process().memory_info().rss),
int(psutil.virtual_memory().available),
)
except Exception:
return None, None
def _cuda_counters() -> tuple[int | None, int | None, int | None]:
"""Read counters only; never synchronize, clear caches, or reset peaks."""
try:
if torch.cuda.is_available():
return (
int(torch.cuda.memory_allocated()),
int(torch.cuda.memory_reserved()),
int(torch.cuda.max_memory_allocated()),
)
except Exception:
pass
return None, None, None
def _retained_stream_bytes(
entries: Any,
stream: str,
) -> int:
values: list[Any] = []
if isinstance(entries, (list, tuple)):
for entry in entries:
if isinstance(entry, Mapping):
values.append(entry.get(stream))
stats = tensor_storage_stats(values)
return int(stats["cpu_bytes"]) + int(stats["gpu_bytes"])
def _observed_streams(latent: Any) -> tuple[Any, ...]:
if latent is None:
return ()
if isinstance(latent, Mapping) and "samples" in latent:
try:
return tuple(extract_av_streams(latent))
except Exception:
return ()
return (latent,)
def format_attribution_snapshot(
label: str,
*,
retained_entries: Any = None,
observed_latent: Any = None,
retained_refine_context: Any = None,
) -> str:
"""Format one fail-soft snapshot without changing allocator state."""
try:
rss, available = _process_counters()
cuda_allocated, cuda_reserved, cuda_peak = _cuda_counters()
retained_video = _retained_stream_bytes(retained_entries, "video")
retained_audio = _retained_stream_bytes(retained_entries, "audio")
observed = tensor_storage_stats(_observed_streams(observed_latent))
refine_context = tensor_storage_stats(retained_refine_context)
return (
f"Memory [{label}]: RSS={_mib(rss)}, "
f"system_available={_mib(available)}, "
f"CUDA_allocated={_mib(cuda_allocated)}, "
f"CUDA_reserved={_mib(cuda_reserved)}, "
f"CUDA_peak_allocated_since_external_reset={_mib(cuda_peak)}, "
f"retained_video={_bytes_and_mib(retained_video)}, "
f"retained_audio={_bytes_and_mib(retained_audio)}, "
"retained_refine_context_CPU="
f"{_bytes_and_mib(int(refine_context['cpu_bytes']))}, "
"retained_refine_context_GPU="
f"{_bytes_and_mib(int(refine_context['gpu_bytes']))}, "
f"observed_CPU_storage={_bytes_and_mib(int(observed['cpu_bytes']))}, "
f"observed_GPU_storage={_bytes_and_mib(int(observed['gpu_bytes']))}"
)
except Exception as exc:
return (
f"Memory [{label}]: unavailable "
f"({type(exc).__name__}: {exc})"
)
class MemoryAttributionCollector:
"""Keep V3.5 memory events in chronological order until report assembly."""
def __init__(self):
self.lines: list[str] = []
def capture_sequence_start(self) -> None:
self.lines.append(format_attribution_snapshot("V3.5 sequence start"))
def capture_group(
self,
*,
physical_group: int,
logical_chunks: tuple[int, ...],
stage: str,
retained_entries: Any,
observed_latent: Any = None,
retained_refine_context: Any = None,
) -> None:
logical = ",".join(str(int(value)) for value in logical_chunks)
self.lines.append(
format_attribution_snapshot(
f"V3.5 physical group {int(physical_group)} "
f"logical=[{logical}] {stage}",
retained_entries=retained_entries,
observed_latent=observed_latent,
retained_refine_context=retained_refine_context,
)
)
def capture_sequence_complete(
self,
entries: Any,
retained_refine_context: Any = None,
) -> None:
self.lines.append(
format_attribution_snapshot(
"V3.5 sequence complete",
retained_entries=entries,
retained_refine_context=retained_refine_context,
)
)
def capture_attribution_fail_soft(
collector: Any,
method: str,
**kwargs: Any,
) -> None:
"""A diagnostics failure must never replace a sampling result."""
try:
getattr(collector, method)(**kwargs)
except Exception as exc:
try:
collector.lines.append(
f"Memory attribution [{method}]: unavailable "
f"({type(exc).__name__}: {exc})"
)
except Exception:
pass
@dataclass(frozen=True)
class AssemblyBufferProjection:
natural_frames: int
target_frames: int
natural_image_bytes: int
natural_audio_bytes: int
final_image_bytes: int
final_audio_bytes: int
frame_count_adjustment_required: bool
def _as_list(value: Any) -> list[Any]:
if isinstance(value, list):
if len(value) == 1 and isinstance(value[0], list):
return list(value[0])
return list(value)
if isinstance(value, tuple):
return list(value)
return [value]
def _singleton(value: Any, name: str) -> Any:
while isinstance(value, list):
if len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
value = value[0]
return value
def project_assembly_buffers(
*,
images: Any,
audio: Any,
assembly_plan: Any,
exact_total_duration: Any,
) -> AssemblyBufferProjection:
"""Calculate natural and final buffer sizes without allocating a buffer."""
plan = validate_assembly_plan(_singleton(assembly_plan, "assembly_plan"))
image_chunks = _as_list(images)
audio_chunks = _as_list(audio)
if not image_chunks or not audio_chunks:
raise ValueError("decoded image/audio chunks are required for projection")
first_image = _singleton(image_chunks[0], "decoded image chunk")
if not torch.is_tensor(first_image) or first_image.ndim != 4:
raise ValueError("decoded image chunk must be [frames,H,W,C]")
frame_shape = tuple(int(value) for value in first_image.shape[1:])
for index, value in enumerate(image_chunks, start=1):
tensor = _singleton(value, f"decoded image chunk {index}")
if (
not torch.is_tensor(tensor)
or tensor.ndim != 4
or tuple(int(item) for item in tensor.shape[1:]) != frame_shape
or tensor.dtype != first_image.dtype
):
raise ValueError("decoded image geometry or dtype changed between chunks")
first_audio = _singleton(audio_chunks[0], "decoded audio chunk")
if not isinstance(first_audio, Mapping):
raise ValueError("decoded audio chunk must be an AUDIO mapping")
waveform = first_audio.get("waveform")
sample_rate = first_audio.get("sample_rate")
if not torch.is_tensor(waveform) or waveform.ndim < 1 or sample_rate is None:
raise ValueError("decoded audio chunk has an invalid waveform or sample rate")
sample_rate = int(sample_rate)
if sample_rate <= 0:
raise ValueError("decoded audio sample rate must be positive")
decode_units = list(plan.get("decode_groups") or plan["chunks"])
natural_frames = sum(int(unit["net_frames"]) for unit in decode_units)
exact = bool(_singleton(exact_total_duration, "exact_total_duration"))
target_frames = int(plan["target_frames"]) if exact else natural_frames
if natural_frames <= 0 or target_frames <= 0:
raise ValueError("assembly frame counts must be positive")
frame_elements = math.prod(frame_shape)
audio_streams = math.prod(int(value) for value in waveform.shape[:-1])
natural_samples = int(round(natural_frames / FPS * sample_rate))
target_samples = int(round(target_frames / FPS * sample_rate))
return AssemblyBufferProjection(
natural_frames=natural_frames,
target_frames=target_frames,
natural_image_bytes=(
natural_frames * frame_elements * first_image.element_size()
),
natural_audio_bytes=(
natural_samples * audio_streams * waveform.element_size()
),
final_image_bytes=(
target_frames * frame_elements * first_image.element_size()
),
final_audio_bytes=(
target_samples * audio_streams * waveform.element_size()
),
frame_count_adjustment_required=bool(
exact and target_frames != natural_frames
),
)
def format_assembly_projection(projection: AssemblyBufferProjection) -> str:
return (
"Projected assembly buffers [V3.5]: "
f"natural_frames={projection.natural_frames}, "
f"projected_natural_IMAGE={_bytes_and_mib(projection.natural_image_bytes)}, "
"projected_natural_generated_AUDIO="
f"{_bytes_and_mib(projection.natural_audio_bytes)}, "
f"target_frames={projection.target_frames}, "
f"final_IMAGE={_bytes_and_mib(projection.final_image_bytes)}, "
f"final_generated_AUDIO={_bytes_and_mib(projection.final_audio_bytes)}, "
"frame_count_adjustment_required="
f"{'yes' if projection.frame_count_adjustment_required else 'no'}"
)
+40 -6
View File
@@ -256,6 +256,8 @@ class H3ContinuumSamplerV3:
driving_audio_vae=None,
reference_video_source=None,
timeline_video_source=None,
capture_refine_context=False,
memory_attribution=False,
):
if prompt_overrides is not None and not isinstance(prompt_overrides, dict):
prompt_overrides = None
@@ -297,7 +299,7 @@ class H3ContinuumSamplerV3:
for index, prompt in sparse_overrides.items():
clip_prompt_inputs[f"clip_{index}_prompt"] = prompt
entries, last_state, session, report = H3ContinuumSamplerV2().run(
sequence_outputs = H3ContinuumSamplerV2().run(
model=model,
clip=clip,
video_vae=video_vae,
@@ -335,8 +337,14 @@ class H3ContinuumSamplerV3:
driving_audio_vae=driving_audio_vae,
reference_video_source=reference_video_source,
timeline_video_source=timeline_video_source,
capture_refine_context=bool(capture_refine_context),
memory_attribution=bool(memory_attribution),
**clip_prompt_inputs,
)
if bool(capture_refine_context):
entries, last_state, session, report, refine_context = sequence_outputs
else:
entries, last_state, session, report = sequence_outputs
from ..v2.sequence import _terminal_flf_merge_enabled
from .plan import prepare_physical_decode_entries
@@ -370,7 +378,10 @@ class H3ContinuumSamplerV3:
"session": session,
"report": report,
}
return video_latents, audio_latents, assembly_plan, result
outputs = (video_latents, audio_latents, assembly_plan, result)
if bool(capture_refine_context):
return (*outputs, refine_context)
return outputs
class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
@@ -650,6 +661,8 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
driving_audio_vae=None,
reference_video_source=None,
timeline_video_source=None,
capture_refine_context=False,
memory_attribution=False,
):
runtime_started_at = time.perf_counter()
from ..reference import prepare_reference_assets
@@ -699,6 +712,8 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
driving_audio_vae=driving_audio_vae,
reference_video_source=reference_video_source,
timeline_video_source=timeline_video_source,
capture_refine_context=bool(capture_refine_context),
memory_attribution=bool(memory_attribution),
advanced={
"audio_continuity": bool(audio_continuity),
"diagnostics": diagnostics,
@@ -713,13 +728,20 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
if run_storage == "Off":
_validate_regenerate_storage(run_storage, regenerate_from)
video_latents, audio_latents, assembly_plan, result = execute()
return (
execute_outputs = execute()
if bool(capture_refine_context):
video_latents, audio_latents, assembly_plan, result, refine_context = execute_outputs
else:
video_latents, audio_latents, assembly_plan, result = execute_outputs
outputs = (
video_latents,
audio_latents,
mark_runtime_start(assembly_plan),
str(result["report"]),
)
if bool(capture_refine_context):
return (*outputs, refine_context)
return outputs
if run_storage != "Save + Auto Resume":
raise ValueError(f"unknown Run Storage mode: {run_storage!r}")
from ..run_storage import (
@@ -735,14 +757,26 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
with run_storage_scope(
storage_name, prompt=prompt, unique_id=unique_id
) as storage:
video_latents, audio_latents, assembly_plan, result = execute()
execute_outputs = execute()
if bool(capture_refine_context):
video_latents, audio_latents, assembly_plan, result, refine_context = execute_outputs
else:
video_latents, audio_latents, assembly_plan, result = execute_outputs
result = dict(result)
report = str(result["report"]) + "\n" + storage.summary(
detailed=diagnostics == DIAGNOSTICS_FULL
)
result["report"] = report
storage.finalize(session=result["session"], report=report)
return video_latents, audio_latents, mark_runtime_start(assembly_plan), report
outputs = (
video_latents,
audio_latents,
mark_runtime_start(assembly_plan),
report,
)
if bool(capture_refine_context):
return (*outputs, refine_context)
return outputs
class H3ContinuumSamplerTimelineVideo(H3ContinuumSamplerProduction):
+367
View File
@@ -0,0 +1,367 @@
"""Sequential pixel/VAE resize for Continuum physical video groups."""
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from typing import Any
import torch
from .plan import SECOND_PASS_CONTRACT_VERSION
PIXEL_UPSCALE_METHODS = (
"nearest-exact",
"bilinear",
"area",
"bicubic",
"bislerp",
"lanczos",
)
_PIXEL_RESIZE_FRAME_BATCH = 8
class PixelVaeResizeError(ValueError):
"""Raised when a pixel/VAE resize would violate the physical-group contract."""
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
if not isinstance(assembly_plan, Mapping):
raise PixelVaeResizeError("assembly_plan must be a mapping")
contract = assembly_plan.get("second_pass_contract")
if not isinstance(contract, Mapping):
raise PixelVaeResizeError("assembly_plan has no second_pass_contract")
if contract.get("version") != SECOND_PASS_CONTRACT_VERSION:
raise PixelVaeResizeError("unsupported second_pass_contract version")
groups = contract.get("physical_groups")
if (
not isinstance(groups, list)
or not groups
or not all(isinstance(group, Mapping) for group in groups)
):
raise PixelVaeResizeError(
"second_pass_contract.physical_groups must be non-empty"
)
return groups
def resolve_pixel_latent_scale(
assembly_plan: Mapping[str, Any],
) -> tuple[int, int]:
"""Return plan-derived pixel-per-latent ratios as ``(width, height)``."""
groups = _contract_groups(assembly_plan)
scale: tuple[int, int] | None = None
for position, group in enumerate(groups):
if int(group.get("group_id", -1)) != position:
raise PixelVaeResizeError("physical group order or group_id changed")
source_width = int(group.get("source_width", -1))
source_height = int(group.get("source_height", -1))
source_latent_width = int(group.get("source_latent_w", -1))
source_latent_height = int(group.get("source_latent_h", -1))
if min(
source_width,
source_height,
source_latent_width,
source_latent_height,
) < 1:
raise PixelVaeResizeError("source pixel/latent geometry is invalid")
if (
source_width % source_latent_width
or source_height % source_latent_height
):
raise PixelVaeResizeError(
"source pixel/latent geometry is not integral"
)
group_scale = (
source_width // source_latent_width,
source_height // source_latent_height,
)
if scale is None:
scale = group_scale
elif scale != group_scale:
raise PixelVaeResizeError(
"source pixel/latent scale differs between physical groups"
)
assert scale is not None
return scale
def _latent_samples(
latent: Any,
*,
position: int,
group: Mapping[str, Any],
) -> torch.Tensor:
if not isinstance(latent, Mapping) or "samples" not in latent:
raise PixelVaeResizeError(
f"video_latents[{position}] must be a LATENT mapping with samples"
)
samples = latent["samples"]
if not isinstance(samples, torch.Tensor) or samples.ndim != 5:
raise PixelVaeResizeError("video latent must have shape [B,C,T,H,W]")
batch, channels, temporal, height, width = (
int(value) for value in samples.shape
)
expected = (
int(group.get("source_batch", -1)),
int(group.get("latent_channels", -1)),
int(group.get("source_latent_t", -1)),
int(group.get("source_latent_h", -1)),
int(group.get("source_latent_w", -1)),
)
if (batch, channels, temporal, height, width) != expected:
raise PixelVaeResizeError(
"video latent shape or physical group order changed"
)
if channels != 24:
raise PixelVaeResizeError("MiniMax H3 video latent must have 24 channels")
if not bool(torch.isfinite(samples).all()):
raise PixelVaeResizeError("input video latent contains NaN or Inf")
return samples
def _resize_one_group(
samples: torch.Tensor,
*,
video_vae: Any,
source_width: int,
source_height: int,
target_width: int,
target_height: int,
target_latent_width: int,
target_latent_height: int,
upscale_method: str,
upscale_fn: Callable[..., Any],
) -> torch.Tensor:
"""Decode, resize, and encode one group so its pixel buffer dies per call."""
decoded_pixels = video_vae.decode(samples)
if not isinstance(decoded_pixels, torch.Tensor) or decoded_pixels.ndim != 5:
raise PixelVaeResizeError(
"Video VAE decode must return pixels with shape [B,F,H,W,C]"
)
batch, frames, pixel_height, pixel_width, channels = (
int(value) for value in decoded_pixels.shape
)
if batch != int(samples.shape[0]):
raise PixelVaeResizeError("Video VAE decode changed the batch size")
if frames < 1 or channels != 3:
raise PixelVaeResizeError(
"Video VAE decode returned invalid frame or channel geometry"
)
if (pixel_width, pixel_height) != (source_width, source_height):
raise PixelVaeResizeError(
"Video VAE decode geometry differs from the assembly plan"
)
for frame_start in range(0, frames, _PIXEL_RESIZE_FRAME_BATCH):
if not bool(
torch.isfinite(
decoded_pixels[
:,
frame_start : frame_start + _PIXEL_RESIZE_FRAME_BATCH,
]
).all()
):
raise PixelVaeResizeError("Video VAE decode contains NaN or Inf")
# Pixel resizing is intentionally CPU-owned. Core's Lanczos implementation
# is CPU/PIL-based already, and this also avoids retaining a full video-sized
# resize workspace in VRAM under --gpu-only configurations.
pixels = decoded_pixels.detach().to("cpu").contiguous()
del decoded_pixels
# Core common_upscale is a 4D image boundary. Make B*F explicit so video
# frame order is unambiguous. Resize bounded frame batches into one CPU
# [B,F,H,W,C] buffer required by the temporal H3 VAE encoder.
source_flat = pixels.reshape(
batch * frames, pixel_height, pixel_width, channels
)
resized_pixels = torch.empty(
(batch, frames, target_height, target_width, channels),
dtype=pixels.dtype,
device="cpu",
)
target_flat = resized_pixels.reshape(
batch * frames, target_height, target_width, channels
)
for frame_start in range(
0,
batch * frames,
_PIXEL_RESIZE_FRAME_BATCH,
):
frame_stop = min(
frame_start + _PIXEL_RESIZE_FRAME_BATCH,
batch * frames,
)
pixel_nchw = (
source_flat[frame_start:frame_stop]
.movedim(-1, 1)
.contiguous()
)
resized_nchw = upscale_fn(
pixel_nchw,
int(target_width),
int(target_height),
upscale_method,
"disabled",
)
expected_resize_shape = (
frame_stop - frame_start,
channels,
target_height,
target_width,
)
if not isinstance(resized_nchw, torch.Tensor) or tuple(
resized_nchw.shape
) != expected_resize_shape:
raise PixelVaeResizeError("Core pixel resize returned invalid geometry")
resized_nchw = resized_nchw.detach().to("cpu")
if not bool(torch.isfinite(resized_nchw).all()):
raise PixelVaeResizeError("Core pixel resize contains NaN or Inf")
target_flat[frame_start:frame_stop].copy_(
resized_nchw.movedim(1, -1)
)
del pixel_nchw, resized_nchw
del pixels, source_flat, target_flat
# Match Core VAEEncode exactly: H3's VAE wrapper expects a 4D IMAGE batch
# and promotes its frame axis to one temporal video batch. Passing the 5D
# [B,F,H,W,C] tensor directly makes Core crop F as if it were spatial.
# Encode each Continuum batch independently, then restore B explicitly.
encoded_batches: list[torch.Tensor] = []
expected_batch_shape = (
1,
int(samples.shape[1]),
int(samples.shape[2]),
int(target_latent_height),
int(target_latent_width),
)
for batch_index in range(batch):
encoded_batch = video_vae.encode(resized_pixels[batch_index])
if not isinstance(encoded_batch, torch.Tensor) or encoded_batch.ndim != 5:
raise PixelVaeResizeError(
"Video VAE encode must return a latent with shape [B,C,T,H,W]"
)
if int(encoded_batch.shape[0]) != 1:
raise PixelVaeResizeError(
"Core Video VAE Encode must return one batch per IMAGE sequence"
)
encoded_batch_shape = tuple(int(value) for value in encoded_batch.shape)
if encoded_batch_shape != expected_batch_shape:
raise PixelVaeResizeError(
"Video VAE encode changed B/C/T or returned the wrong target H/W "
f"for batch {batch_index}: expected {expected_batch_shape}, "
f"got {encoded_batch_shape}"
)
encoded_batches.append(encoded_batch)
encoded = (
encoded_batches[0]
if len(encoded_batches) == 1
else torch.cat(encoded_batches, dim=0)
)
if not isinstance(encoded, torch.Tensor) or encoded.ndim != 5:
raise PixelVaeResizeError(
"Video VAE encode must return a latent with shape [B,C,T,H,W]"
)
expected_shape = (
int(samples.shape[0]),
int(samples.shape[1]),
int(samples.shape[2]),
int(target_latent_height),
int(target_latent_width),
)
if tuple(int(value) for value in encoded.shape) != expected_shape:
raise PixelVaeResizeError(
"Video VAE encode changed B/C/T or returned the wrong target H/W: "
f"expected {expected_shape}, got {tuple(int(value) for value in encoded.shape)}"
)
if not bool(torch.isfinite(encoded).all()):
raise PixelVaeResizeError("Video VAE encode contains NaN or Inf")
# Continuum stores accepted physical latents on CPU; do not retain one GPU
# allocation per physical group while later groups are being converted.
output = encoded.detach().to("cpu").contiguous()
del resized_pixels, encoded_batches, encoded
return output
@torch.inference_mode()
def resize_video_latents_via_vae(
video_latents: Sequence[Mapping[str, Any]],
assembly_plan: Mapping[str, Any],
*,
video_vae: Any,
target_width: int,
target_height: int,
upscale_method: str = "lanczos",
upscale_fn: Callable[..., Any] | None = None,
) -> list[dict[str, Any]]:
"""Resize physical groups sequentially through the connected H3 Video VAE.
``target_width`` and ``target_height`` are pixel dimensions. The expected
encoded latent geometry is derived exclusively from assembly-plan metadata.
VAE failures are intentionally not caught: Core owns its OOM/tiled fallback,
and this path must never fall back to direct latent interpolation.
"""
groups = _contract_groups(assembly_plan)
if (
not isinstance(video_latents, Sequence)
or isinstance(video_latents, (str, bytes))
or len(video_latents) != len(groups)
):
raise PixelVaeResizeError("video latent group count changed")
if (
video_vae is None
or not callable(getattr(video_vae, "decode", None))
or not callable(getattr(video_vae, "encode", None))
):
raise PixelVaeResizeError("a usable H3 Video VAE is required")
if upscale_method not in PIXEL_UPSCALE_METHODS:
raise PixelVaeResizeError(
f"unsupported pixel upscale method {upscale_method!r}"
)
target_width = int(target_width)
target_height = int(target_height)
if target_width < 1 or target_height < 1:
raise PixelVaeResizeError("target pixel width and height must be positive")
scale_width, scale_height = resolve_pixel_latent_scale(assembly_plan)
if target_width % scale_width or target_height % scale_height:
raise PixelVaeResizeError(
"target pixel geometry is not aligned to the plan-derived latent scale"
)
target_latent_width = target_width // scale_width
target_latent_height = target_height // scale_height
if upscale_fn is None:
from comfy.utils import common_upscale
upscale_fn = common_upscale
outputs: list[dict[str, Any]] = []
for position, (latent, group) in enumerate(
zip(video_latents, groups, strict=True)
):
samples = _latent_samples(latent, position=position, group=group)
source_width = int(group["source_width"])
source_height = int(group["source_height"])
resized_samples = _resize_one_group(
samples,
video_vae=video_vae,
source_width=source_width,
source_height=source_height,
target_width=target_width,
target_height=target_height,
target_latent_width=target_latent_width,
target_latent_height=target_latent_height,
upscale_method=upscale_method,
upscale_fn=upscale_fn,
)
output = dict(latent)
output["samples"] = resized_samples
outputs.append(output)
return outputs
+96
View File
@@ -11,6 +11,7 @@ from ..state import validate_plan
ASSEMBLY_PLAN_MAGIC = "H3_CONTINUUM_ASSEMBLY_PLAN"
ASSEMBLY_PLAN_SCHEMA_VERSION = 1
FPS = 24
SECOND_PASS_CONTRACT_VERSION = 1
class AssemblyPlanError(ValueError):
@@ -199,6 +200,89 @@ def _recombine_terminal_tensor(
return output.contiguous()
def _terminal_physical_prompt(
prompts: list[str],
*,
chunk_seconds: float,
) -> tuple[str, str]:
"""Reproduce the established terminal-pair prompt without reparsing it later."""
first, second = prompts
if first == second:
return first, "shared_prompt_v1"
split = f"{float(chunk_seconds):g}"
stop = f"{float(chunk_seconds) * 2:g}"
return (
f"[0-{split}s]\n{first}\n\n[{split}-{stop}s]\n{second}",
"paired_timeline_v1",
)
def _attach_second_pass_contract(
plan: dict[str, Any],
*,
logical_entries: list[dict[str, Any]],
physical_entries: list[dict[str, Any]],
chunk_seconds: float,
) -> None:
"""Attach additive V3.5 metadata without changing V3.4 generation semantics."""
decode_groups = plan.get("decode_groups")
if not isinstance(decode_groups, list):
decode_groups = [
{
"logical_chunk_indices": [index + 1],
"terminal_merged": False,
"total_frames": plan["chunks"][index]["total_frames"],
"trim_frames": plan["chunks"][index]["trim_frames"],
}
for index in range(len(physical_entries))
]
groups: list[dict[str, Any]] = []
for group_id, (entry, decode_group) in enumerate(
zip(physical_entries, decode_groups, strict=True)
):
video = entry["video"]
audio = entry["audio"]
logical_chunks = [int(value) for value in decode_group["logical_chunk_indices"]]
prompts = [str(logical_entries[index - 1].get("prompt", "")) for index in logical_chunks]
if len(prompts) == 1:
physical_prompt = prompts[0]
prompt_policy = "single"
elif len(prompts) == 2 and bool(decode_group.get("terminal_merged")):
physical_prompt, prompt_policy = _terminal_physical_prompt(
prompts,
chunk_seconds=chunk_seconds,
)
else:
physical_prompt = "\n\n".join(prompts)
prompt_policy = "physical_group_v1"
groups.append(
{
"group_id": group_id,
"logical_chunks": logical_chunks,
"physical_prompt": physical_prompt,
"prompt_policy": prompt_policy,
"physical_frames": int(decode_group["total_frames"]),
"trim_prefix_frames": int(decode_group["trim_frames"]),
"terminal_merged": bool(decode_group.get("terminal_merged", False)),
"source_width": int(plan["width"]),
"source_height": int(plan["height"]),
"source_batch": int(video.shape[0]),
"latent_channels": int(video.shape[1]),
"source_latent_t": int(video.shape[2]),
"source_latent_h": int(video.shape[3]),
"source_latent_w": int(video.shape[4]),
"source_audio_shape": [int(value) for value in audio.shape],
}
)
plan["second_pass_contract"] = {
"version": SECOND_PASS_CONTRACT_VERSION,
"physical_groups": groups,
}
def prepare_physical_decode_entries(
entries,
*,
@@ -215,6 +299,12 @@ def prepare_physical_decode_entries(
preserve_final_frame=bool(preserve_final_frame),
)
if not terminal_merged:
_attach_second_pass_contract(
plan,
logical_entries=logical_entries,
physical_entries=logical_entries,
chunk_seconds=float(chunk_seconds),
)
return logical_entries, plan
if len(logical_entries) < 2:
raise ValueError("terminal physical decode requires two logical entries")
@@ -287,5 +377,11 @@ def prepare_physical_decode_entries(
plan["logical_chunk_count"] = len(logical_chunks)
plan["physical_decode_group_count"] = len(decode_groups)
plan["decode_groups"] = decode_groups
_attach_second_pass_contract(
plan,
logical_entries=logical_entries,
physical_entries=decode_entries,
chunk_seconds=float(chunk_seconds),
)
validate_assembly_plan(plan)
return decode_entries, plan
+546
View File
@@ -0,0 +1,546 @@
"""Runtime-only First Pass conditioning for Continuum-aware refinement.
The context deliberately remains outside the persisted Assembly Plan contract. It
owns CPU copies of the First Pass conditioning tensors and records them in physical
sampling order. Second Pass callers may then derive a target-canvas conditioning
without mutating the captured source context.
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any
import torch
from ..constants import MARK_VIDEO_CONTEXT
MAGIC = "H3_CONTINUUM_REFINE_CONTEXT"
VERSION = 1
class RefineContextError(ValueError):
"""Raised when the runtime-only refine context is structurally invalid."""
class RefineConditioningAdaptationError(RefineContextError):
"""Raised when a valid captured conditioning cannot be adapted safely."""
def _cpu_tensor(value: torch.Tensor) -> torch.Tensor:
return value.detach().to(device="cpu").clone().contiguous()
def _freeze(value: Any) -> Any:
if torch.is_tensor(value):
return _cpu_tensor(value)
if isinstance(value, Mapping):
return MappingProxyType({key: _freeze(item) for key, item in value.items()})
if isinstance(value, (list, tuple)):
return tuple(_freeze(item) for item in value)
if isinstance(value, (set, frozenset)):
return frozenset(_freeze(item) for item in value)
return value
def freeze_conditioning(conditioning: Any) -> tuple[tuple[torch.Tensor, Mapping], ...]:
"""Return a CPU-owned conditioning clone with immutable containers.
Tensors are detached and cloned, so later mutation or device movement of the
original First Pass conditioning cannot affect the refine context. Container
immutability makes the result safe to share until target adaptation performs
copy-on-write replacement of visual fields.
"""
if not isinstance(conditioning, Sequence) or isinstance(
conditioning, (str, bytes, bytearray)
):
raise RefineContextError("conditioning must be a non-empty sequence")
output: list[tuple[torch.Tensor, Mapping]] = []
for index, entry in enumerate(conditioning):
if not isinstance(entry, Sequence) or isinstance(
entry, (str, bytes, bytearray)
) or len(entry) != 2:
raise RefineContextError(
f"conditioning entry {index} is not [tensor, metadata]"
)
tensor, metadata = entry
if not torch.is_tensor(tensor):
raise RefineContextError(
f"conditioning entry {index} text value is not a tensor"
)
if not isinstance(metadata, Mapping):
raise RefineContextError(
f"conditioning entry {index} metadata is not a mapping"
)
output.append((_cpu_tensor(tensor), _freeze(metadata)))
if not output:
raise RefineContextError("conditioning is empty")
return tuple(output)
def _optional_image(value: Any, name: str) -> torch.Tensor | None:
if value is None:
return None
if not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3:
raise RefineContextError(
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
)
return _cpu_tensor(value)
def _validate_optional_image(value: Any, name: str) -> None:
if value is not None and (
not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3
):
raise RefineContextError(
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
)
def make_refine_group(
*,
conditioning: Any,
group_id: int,
logical_chunks: Sequence[int],
physical_frames: int,
prompt_policy: str,
physical_prompt: str,
source_video_shape: Sequence[int],
physical_clip_index: int,
context_frames: int,
first_image: torch.Tensor | None = None,
last_image: torch.Tensor | None = None,
) -> dict[str, Any]:
"""Capture one physical First Pass sampling group's effective conditioning."""
group_id = int(group_id)
physical_frames = int(physical_frames)
physical_clip_index = int(physical_clip_index)
context_frames = int(context_frames)
chunks = tuple(int(value) for value in logical_chunks)
shape = tuple(int(value) for value in source_video_shape)
if group_id < 0:
raise RefineContextError("refine group id must be non-negative")
if not chunks or any(value <= 0 for value in chunks):
raise RefineContextError("refine group logical chunks are invalid")
if tuple(sorted(chunks)) != chunks or len(set(chunks)) != len(chunks):
raise RefineContextError("refine group logical chunks must be ordered and unique")
if physical_frames <= 0:
raise RefineContextError("refine group physical frame count is invalid")
if not str(prompt_policy):
raise RefineContextError("refine group prompt policy is empty")
if len(shape) != 5 or any(value <= 0 for value in shape):
raise RefineContextError("refine group source video shape must be [B,C,T,H,W]")
if physical_clip_index <= 0:
raise RefineContextError("refine group physical clip index is invalid")
if context_frames < 0:
raise RefineContextError("refine group context frame count is invalid")
return {
"group_id": group_id,
"logical_chunks": chunks,
"physical_frames": physical_frames,
"prompt_policy": str(prompt_policy),
"physical_prompt": str(physical_prompt),
"source_video_shape": shape,
"physical_clip_index": physical_clip_index,
"context_frames": context_frames,
"conditioning": freeze_conditioning(conditioning),
"first_image": _optional_image(first_image, "first_image"),
"last_image": _optional_image(last_image, "last_image"),
}
def make_refine_context(
groups: Sequence[Mapping[str, Any]],
*,
source_width: int,
source_height: int,
conditioning_mode: str,
complete: bool = True,
notes: Sequence[str] = (),
) -> dict[str, Any]:
"""Build and validate a physical-order runtime refine context."""
context = {
"magic": MAGIC,
"version": VERSION,
"source_width": int(source_width),
"source_height": int(source_height),
"conditioning_mode": str(conditioning_mode),
"complete": bool(complete),
"groups": tuple(dict(group) for group in groups),
"notes": tuple(str(note) for note in notes),
}
return validate_refine_context(context)
def _validate_frozen_conditioning(value: Any, *, group_id: int) -> None:
if not isinstance(value, tuple) or not value:
raise RefineContextError(f"refine group {group_id} conditioning is invalid")
for index, entry in enumerate(value):
if not isinstance(entry, tuple) or len(entry) != 2:
raise RefineContextError(
f"refine group {group_id} conditioning entry {index} is invalid"
)
tensor, metadata = entry
if not torch.is_tensor(tensor) or tensor.device.type != "cpu":
raise RefineContextError(
f"refine group {group_id} conditioning tensor {index} is not CPU-owned"
)
if not isinstance(metadata, Mapping):
raise RefineContextError(
f"refine group {group_id} conditioning metadata {index} is invalid"
)
def _group_field_tuple(group: Mapping[str, Any], name: str) -> tuple[int, ...]:
value = group.get(name)
if not isinstance(value, (tuple, list)):
raise RefineContextError(f"refine group {group.get('group_id')} {name} is invalid")
try:
return tuple(int(item) for item in value)
except (TypeError, ValueError) as exc:
raise RefineContextError(
f"refine group {group.get('group_id')} {name} is invalid"
) from exc
def validate_refine_context(
context: Any, assembly_plan: Mapping[str, Any] | None = None
) -> dict[str, Any]:
"""Validate a runtime context and, when supplied, its Assembly Plan pairing."""
if not isinstance(context, dict) or context.get("magic") != MAGIC:
raise RefineContextError("invalid H3 Continuum refine context")
if type(context.get("version")) is not int or context["version"] != VERSION:
raise RefineContextError(
f"unsupported refine context version {context.get('version')!r}"
)
if int(context.get("source_width", 0)) <= 0 or int(
context.get("source_height", 0)
) <= 0:
raise RefineContextError("refine context source dimensions are invalid")
if not str(context.get("conditioning_mode", "")):
raise RefineContextError("refine context conditioning mode is empty")
if type(context.get("complete")) is not bool:
raise RefineContextError("refine context complete flag is invalid")
groups = context.get("groups")
if not isinstance(groups, tuple):
raise RefineContextError("refine context groups must be an immutable tuple")
if context["complete"] and not groups:
raise RefineContextError("complete refine context contains no physical groups")
notes = context.get("notes")
if not isinstance(notes, tuple) or any(not isinstance(note, str) for note in notes):
raise RefineContextError("refine context notes must be an immutable string tuple")
previous_clip_index = 0
for expected_id, group in enumerate(groups):
if not isinstance(group, Mapping):
raise RefineContextError(f"refine group {expected_id} is not a mapping")
if type(group.get("group_id")) is not int or group["group_id"] != expected_id:
raise RefineContextError("refine context physical group order is not contiguous")
logical_chunks = _group_field_tuple(group, "logical_chunks")
if not logical_chunks or any(value <= 0 for value in logical_chunks):
raise RefineContextError(f"refine group {expected_id} logical chunks are invalid")
if tuple(sorted(logical_chunks)) != logical_chunks or len(
set(logical_chunks)
) != len(logical_chunks):
raise RefineContextError(
f"refine group {expected_id} logical chunks are not ordered and unique"
)
if int(group.get("physical_frames", 0)) <= 0:
raise RefineContextError(f"refine group {expected_id} physical frames are invalid")
if not str(group.get("prompt_policy", "")):
raise RefineContextError(f"refine group {expected_id} prompt policy is empty")
if "physical_prompt" not in group:
raise RefineContextError(f"refine group {expected_id} physical prompt is absent")
shape = _group_field_tuple(group, "source_video_shape")
if len(shape) != 5 or any(value <= 0 for value in shape):
raise RefineContextError(f"refine group {expected_id} source shape is invalid")
clip_index = int(group.get("physical_clip_index", 0))
if clip_index <= previous_clip_index:
raise RefineContextError(
"refine context physical clip indices are not strictly increasing"
)
previous_clip_index = clip_index
if int(group.get("context_frames", -1)) < 0:
raise RefineContextError(f"refine group {expected_id} context frames are invalid")
_validate_frozen_conditioning(group.get("conditioning"), group_id=expected_id)
_validate_optional_image(group.get("first_image"), "first_image")
_validate_optional_image(group.get("last_image"), "last_image")
if assembly_plan is not None:
if not isinstance(assembly_plan, Mapping):
raise RefineContextError("assembly plan paired with refine context is invalid")
if int(assembly_plan.get("width", 0)) != int(context["source_width"]) or int(
assembly_plan.get("height", 0)
) != int(context["source_height"]):
raise RefineContextError("refine context source dimensions differ from assembly plan")
contract = assembly_plan.get("second_pass_contract")
plan_groups = contract.get("physical_groups") if isinstance(contract, Mapping) else None
if not isinstance(plan_groups, Sequence):
raise RefineContextError("assembly plan has no Second Pass physical-group contract")
# An incomplete context is diagnostic-only (for example, Run Storage may
# have reused an earlier group that was never sampled in this execution).
# Its local group ids describe capture order, not Assembly Plan position;
# validating those partial entries by index would incorrectly turn the
# intended prompt-only fallback into a broken typed-contract hard error.
if not context["complete"]:
return context
if context["complete"] and len(plan_groups) != len(groups):
raise RefineContextError(
"complete refine context physical-group count differs from assembly plan"
)
for group in groups:
group_id = int(group["group_id"])
if group_id >= len(plan_groups) or not isinstance(plan_groups[group_id], Mapping):
raise RefineContextError(
f"refine group {group_id} has no matching assembly-plan group"
)
planned = plan_groups[group_id]
comparisons = (
("logical_chunks", tuple(int(v) for v in group["logical_chunks"]),
tuple(int(v) for v in planned.get("logical_chunks", ()))),
("physical_frames", int(group["physical_frames"]),
int(planned.get("physical_frames", 0))),
("prompt_policy", str(group["prompt_policy"]),
str(planned.get("prompt_policy", ""))),
("physical_prompt", str(group["physical_prompt"]),
str(planned.get("physical_prompt", ""))),
("context_frames", int(group["context_frames"]),
int(planned.get("trim_prefix_frames", -1))),
)
for name, actual, expected in comparisons:
if actual != expected:
raise RefineContextError(
f"refine group {group_id} {name} differs from assembly plan"
)
expected_shape = (
int(planned.get("source_batch", 0)),
int(planned.get("latent_channels", 0)),
int(planned.get("source_latent_t", 0)),
int(planned.get("source_latent_h", 0)),
int(planned.get("source_latent_w", 0)),
)
if tuple(group["source_video_shape"]) != expected_shape:
raise RefineContextError(
f"refine group {group_id} source shape differs from assembly plan"
)
return context
def _common_upscale(
samples: torch.Tensor,
width: int,
height: int,
method: str,
crop: str,
) -> torch.Tensor:
try:
import comfy.utils
except Exception as exc: # pragma: no cover - ComfyUI runtime only
raise RefineConditioningAdaptationError(
f"ComfyUI Core resize support is unavailable: {exc}"
) from exc
return comfy.utils.common_upscale(samples, width, height, method, crop)
def _resize_latent(
latent: Any,
*,
target_h: int,
target_w: int,
method: str,
) -> torch.Tensor:
if not torch.is_tensor(latent) or latent.ndim != 5:
raise RefineConditioningAdaptationError(
"H3 visual conditioning latent must be [B,C,T,H,W]"
)
batch, channels, temporal, _, _ = (int(value) for value in latent.shape)
flat = latent.permute(0, 2, 1, 3, 4).reshape(
batch * temporal, channels, int(latent.shape[-2]), int(latent.shape[-1])
)
try:
resized = _common_upscale(flat, target_w, target_h, method, "disabled")
except RefineConditioningAdaptationError:
raise
except Exception as exc:
raise RefineConditioningAdaptationError(
f"visual conditioning latent resize failed: {exc}"
) from exc
if not torch.is_tensor(resized) or tuple(resized.shape[-2:]) != (
target_h,
target_w,
):
raise RefineConditioningAdaptationError(
"Core resize returned an incompatible visual conditioning shape"
)
return (
resized.reshape(batch, temporal, channels, target_h, target_w)
.permute(0, 2, 1, 3, 4)
.contiguous()
)
def _encode_keyframe_image(
image: torch.Tensor,
*,
target_h: int,
target_w: int,
crop: str,
video_vae: Any,
) -> torch.Tensor:
pixels_h = target_h * 16
pixels_w = target_w * 16
samples = image[:1, ..., :3].movedim(-1, 1)
try:
resized = _common_upscale(samples, pixels_w, pixels_h, "lanczos", crop)
latent = video_vae.encode(resized.movedim(1, -1).contiguous())
except RefineConditioningAdaptationError:
raise
except Exception as exc:
raise RefineConditioningAdaptationError(
f"target keyframe VAE encode failed: {exc}"
) from exc
if not torch.is_tensor(latent) or latent.ndim != 5 or tuple(
int(value) for value in latent.shape[-2:]
) != (target_h, target_w):
raise RefineConditioningAdaptationError(
"target keyframe VAE encode returned an incompatible latent shape"
)
return latent.contiguous()
def adapt_group_conditioning(
group: Mapping[str, Any],
target_latent_h: int,
target_latent_w: int,
video_vae: Any = None,
upscale_method: str = "bilinear",
) -> tuple[list[list[Any]], dict[str, Any]]:
"""Copy-on-write target adaptation for keyframes and Continuum video context.
Ordinary MiniMax references, all audio latents, and text conditioning tensors
remain shared with the frozen source. Only keyframe video latents and
``MARK_VIDEO_CONTEXT`` reference video latents are target-sized.
"""
if not isinstance(group, Mapping):
raise RefineConditioningAdaptationError("refine group is not a mapping")
group_id = int(group.get("group_id", -1))
_validate_frozen_conditioning(group.get("conditioning"), group_id=group_id)
target_h = int(target_latent_h)
target_w = int(target_latent_w)
if target_h <= 0 or target_w <= 0:
raise RefineConditioningAdaptationError("target latent dimensions are invalid")
physical_frames = int(group.get("physical_frames", 0))
if physical_frames <= 0:
raise RefineConditioningAdaptationError("refine group physical frames are invalid")
warnings: list[str] = []
stats: dict[str, Any] = {
"group_id": group_id,
"target_latent_h": target_h,
"target_latent_w": target_w,
"keyframes_resized": 0,
"keyframes_reencoded": 0,
"keyframes_latent_fallback": 0,
"context_refs_resized": 0,
}
output: list[list[Any]] = []
for text_tensor, frozen_metadata in group["conditioning"]:
metadata = dict(frozen_metadata)
keyframes = frozen_metadata.get("minimax_keyframes")
if keyframes is not None:
adapted_keyframes = []
for frozen_keyframe in keyframes:
if not isinstance(frozen_keyframe, Mapping):
raise RefineConditioningAdaptationError(
f"refine group {group_id} contains an invalid keyframe"
)
keyframe = dict(frozen_keyframe)
latent = keyframe.get("latent")
if not torch.is_tensor(latent) or latent.ndim != 5:
raise RefineConditioningAdaptationError(
f"refine group {group_id} keyframe latent is invalid"
)
if tuple(int(value) for value in latent.shape[-2:]) != (
target_h,
target_w,
):
frame_index = int(keyframe.get("resolved_frame_index", -1))
image = None
crop = "disabled"
if frame_index == 0:
image = group.get("first_image")
elif frame_index == physical_frames - 1:
image = group.get("last_image")
crop = "center"
if video_vae is not None and torch.is_tensor(image):
keyframe["latent"] = _encode_keyframe_image(
image,
target_h=target_h,
target_w=target_w,
crop=crop,
video_vae=video_vae,
)
stats["keyframes_reencoded"] += 1
else:
keyframe["latent"] = _resize_latent(
latent,
target_h=target_h,
target_w=target_w,
method=str(upscale_method),
)
stats["keyframes_latent_fallback"] += 1
warning = (
f"group {group_id} keyframe {frame_index} used latent resize "
"because its source image and video VAE were not both available"
)
warnings.append(warning)
stats["keyframes_resized"] += 1
adapted_keyframes.append(keyframe)
metadata["minimax_keyframes"] = adapted_keyframes
refs = frozen_metadata.get("minimax_refs")
if refs is not None:
adapted_refs = []
refs_changed = False
for frozen_ref in refs:
if not isinstance(frozen_ref, Mapping):
raise RefineConditioningAdaptationError(
f"refine group {group_id} contains an invalid MiniMax reference"
)
if bool(frozen_ref.get(MARK_VIDEO_CONTEXT)):
latent = frozen_ref.get("latent")
if not torch.is_tensor(latent) or latent.ndim != 5:
raise RefineConditioningAdaptationError(
f"refine group {group_id} Continuum video context is invalid"
)
if tuple(int(value) for value in latent.shape[-2:]) != (
target_h,
target_w,
):
adapted_ref = dict(frozen_ref)
adapted_ref["latent"] = _resize_latent(
latent,
target_h=target_h,
target_w=target_w,
method=str(upscale_method),
)
adapted_ref["latent_h"] = target_h
adapted_ref["latent_w"] = target_w
adapted_refs.append(adapted_ref)
refs_changed = True
stats["context_refs_resized"] += 1
continue
adapted_refs.append(frozen_ref)
if refs_changed:
metadata["minimax_refs"] = adapted_refs
output.append([text_tensor, metadata])
stats["warnings"] = tuple(warnings)
return output, stats
+158
View File
@@ -0,0 +1,158 @@
"""V3.5 Second Pass sampling with the first-pass audio stream locked."""
from __future__ import annotations
from typing import Any, Callable
import torch
from ..state import extract_av_streams
class RefineSamplingRuntimeError(RuntimeError):
"""Raised when the ComfyUI sampling runtime cannot satisfy the refine contract."""
def _make_basic_guider(model: Any, conditioning: list):
try:
import comfy.samplers
except Exception as exc: # pragma: no cover
raise RefineSamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc
class _BasicGuider(comfy.samplers.CFGGuider):
def set_positive(self, positive):
self.inner_set_conds({"positive": positive})
guider = _BasicGuider(model)
guider.set_positive(conditioning)
return guider
def _load_runtime_modules():
try:
import comfy.model_management
import comfy.nested_tensor
import comfy.sample
import comfy.utils
import latent_preview
except Exception as exc: # pragma: no cover
raise RefineSamplingRuntimeError(
f"ComfyUI refine sampling runtime unavailable: {exc}"
) from exc
return (
comfy.model_management,
comfy.nested_tensor,
comfy.sample,
comfy.utils,
latent_preview,
)
def _prepare_refine_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
) -> tuple[Any, Any]:
"""Build random video noise and a Core-expandable audio-lock mask."""
if video.ndim != 5 or audio.ndim != 4:
raise RefineSamplingRuntimeError(
"refine latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
)
if int(video.shape[0]) != int(audio.shape[0]):
raise RefineSamplingRuntimeError("refine video/audio batch sizes differ")
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
raise RefineSamplingRuntimeError("ComfyUI returned invalid video noise geometry")
# Core prepare_noise creates CPU noise. Keep both streams on that same handoff
# device so NestedTensor packing remains valid before Core moves it to the model.
audio_noise = torch.zeros_like(audio, device="cpu")
# These masks deliberately use the smallest shapes accepted by Core reshape_mask.
# Core expands them to the full stream geometry on the sampling device.
video_mask = torch.ones(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.zeros(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
noise = nested_builder((video_noise, audio_noise))
noise_mask = nested_builder((video_mask, audio_mask))
return noise, noise_mask
def sample_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Refine video while Core carries the first-pass audio as a locked AV stream."""
if not isinstance(latent, dict) or "samples" not in latent:
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
_load_runtime_modules()
)
working = latent.copy()
latent_image = comfy_sample.fix_empty_latent_channels(
model,
working["samples"],
working.get("downscale_ratio_spacial"),
working.get("downscale_ratio_temporal"),
)
working["samples"] = latent_image
video, audio = extract_av_streams(working)
noise, noise_mask = _prepare_refine_noise_and_mask(
video,
audio,
seed=int(seed),
batch_inds=working.get("batch_index"),
prepare_noise_fn=comfy_sample.prepare_noise,
nested_builder=nested_tensor.NestedTensor,
)
guider = _make_basic_guider(model, conditioning)
x0_output: dict[str, Any] = {}
callback = None
if enable_preview:
callback = preview.prepare_callback(
model,
int(sigmas.shape[-1]) - 1,
x0_output,
)
disable_pbar = not comfy_utils.PROGRESS_BAR_ENABLED
samples = guider.sample(
noise,
latent_image,
sampler,
sigmas,
denoise_mask=noise_mask,
callback=callback,
disable_pbar=disable_pbar,
seed=int(seed),
)
samples = samples.to(model_management.intermediate_device())
output = working.copy()
output.pop("downscale_ratio_spacial", None)
output.pop("downscale_ratio_temporal", None)
output["samples"] = samples
extract_av_streams(output)
return output
+395
View File
@@ -0,0 +1,395 @@
"""V3.5 Second Pass contracts and the physical-group sampling runtime."""
from __future__ import annotations
import copy
import hashlib
from collections.abc import Mapping, Sequence
from typing import Any
import torch
from ..model_patch import clone_model_for_chunk
from ..state import extract_av_streams
from ..v2.h3_builder import encode_prompt_conditioning
from ..v2.sampling import latent_from_cpu
from .plan import SECOND_PASS_CONTRACT_VERSION
from .refine_context import (
MAGIC as REFINE_CONTEXT_MAGIC,
RefineConditioningAdaptationError,
RefineContextError,
adapt_group_conditioning,
validate_refine_context,
)
from .refine_sampling import sample_refine_chunk
class SecondPassContractError(ValueError):
"""Raised only when a Second Pass structural contract is impossible."""
def derive_refine_seed(refine_seed: int, physical_group_index: int) -> int:
"""Derive one deterministic seed per physical group in an independent namespace."""
if type(refine_seed) is not int or refine_seed < 0:
raise SecondPassContractError("refine_seed must be a non-negative integer")
if type(physical_group_index) is not int or physical_group_index < 0:
raise SecondPassContractError("physical_group_index must be a non-negative integer")
payload = f"h3-continuum-refine-v1:{refine_seed}:{physical_group_index}".encode("ascii")
return int.from_bytes(hashlib.blake2b(payload, digest_size=8).digest(), "big")
def _latent_samples(value: Any, *, name: str) -> torch.Tensor:
if not isinstance(value, Mapping) or "samples" not in value:
raise SecondPassContractError(f"{name} must be a LATENT mapping with samples")
samples = value["samples"]
if not isinstance(samples, torch.Tensor):
raise SecondPassContractError(f"{name}.samples must be a tensor")
return samples
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
if not isinstance(assembly_plan, Mapping):
raise SecondPassContractError("assembly_plan must be a mapping")
contract = assembly_plan.get("second_pass_contract")
if not isinstance(contract, Mapping):
raise SecondPassContractError("assembly_plan has no second_pass_contract")
if contract.get("version") != SECOND_PASS_CONTRACT_VERSION:
raise SecondPassContractError("unsupported second_pass_contract version")
groups = contract.get("physical_groups")
if not isinstance(groups, list) or not groups or not all(isinstance(g, Mapping) for g in groups):
raise SecondPassContractError("second_pass_contract.physical_groups must be non-empty")
return groups
def _validate_video_groups(
video_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
) -> tuple[list[Mapping[str, Any]], tuple[int, int]]:
groups = _contract_groups(assembly_plan)
if not isinstance(video_latents, Sequence) or isinstance(video_latents, (str, bytes)):
raise SecondPassContractError("video_latents must be a sequence")
if len(video_latents) != len(groups):
raise SecondPassContractError("video latent group count changed")
target_geometry: tuple[int, int] | None = None
for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)):
if group.get("group_id") != position:
raise SecondPassContractError("physical group order or group_id changed")
samples = _latent_samples(latent, name=f"video_latents[{position}]")
if samples.ndim != 5:
raise SecondPassContractError("video latent must have shape [B,C,T,H,W]")
batch, channels, temporal, height, width = (int(value) for value in samples.shape)
if channels != 24:
raise SecondPassContractError("MiniMax H3 video latent must have 24 channels")
if batch != int(group.get("source_batch", -1)):
raise SecondPassContractError("video latent batch changed")
if temporal != int(group.get("source_latent_t", -1)):
raise SecondPassContractError("video latent temporal length changed")
source_h = int(group.get("source_latent_h", -1))
source_w = int(group.get("source_latent_w", -1))
if source_h < 1 or source_w < 1 or height < source_h or width < source_w:
raise SecondPassContractError("video latent spatial size must be preserved or enlarged")
if target_geometry is None:
target_geometry = (height, width)
elif target_geometry != (height, width):
raise SecondPassContractError("all physical groups must use the same target H/W")
assert target_geometry is not None
return groups, target_geometry
def validate_second_pass_inputs(
video_latents: Sequence[Any],
audio_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
) -> dict[str, Any]:
"""Validate only structural invariants required by a future Second Pass node."""
groups, target_geometry = _validate_video_groups(video_latents, assembly_plan)
if not isinstance(audio_latents, Sequence) or isinstance(audio_latents, (str, bytes)):
raise SecondPassContractError("audio_latents must be a sequence")
if len(audio_latents) != len(groups):
raise SecondPassContractError("audio latent group count changed")
for position, (latent, group) in enumerate(zip(audio_latents, groups, strict=True)):
samples = _latent_samples(latent, name=f"audio_latents[{position}]")
expected_shape = group.get("source_audio_shape")
if not isinstance(expected_shape, list) or tuple(expected_shape) != tuple(samples.shape):
raise SecondPassContractError("audio latent shape or physical group order changed")
return {
"physical_group_count": len(groups),
"target_latent_h": target_geometry[0],
"target_latent_w": target_geometry[1],
"audio_passthrough": True,
}
def passthrough_audio_latents(audio_latents: Sequence[Any]) -> list[Any]:
"""Return original LATENT objects; temporary refined audio is not adopted in V1."""
return list(audio_latents)
def update_second_pass_geometry(
assembly_plan: Mapping[str, Any],
video_latents: Sequence[Any],
) -> dict[str, Any]:
"""Return a copied plan whose output geometry follows the upscaled physical latents."""
groups, (target_h, target_w) = _validate_video_groups(video_latents, assembly_plan)
scale_h: int | None = None
scale_w: int | None = None
for group in groups:
source_h = int(group["source_latent_h"])
source_w = int(group["source_latent_w"])
source_height = int(group["source_height"])
source_width = int(group["source_width"])
if source_height % source_h or source_width % source_w:
raise SecondPassContractError("source pixel/latent geometry is not integral")
group_scale_h = source_height // source_h
group_scale_w = source_width // source_w
if scale_h is None:
scale_h, scale_w = group_scale_h, group_scale_w
elif (scale_h, scale_w) != (group_scale_h, group_scale_w):
raise SecondPassContractError("source geometry scale differs between groups")
assert scale_h is not None and scale_w is not None
target_height = target_h * scale_h
target_width = target_w * scale_w
updated = copy.deepcopy(dict(assembly_plan))
updated["width"] = target_width
updated["height"] = target_height
contract = updated["second_pass_contract"]
contract["target_width"] = target_width
contract["target_height"] = target_height
for group in contract["physical_groups"]:
group["target_latent_h"] = target_h
group["target_latent_w"] = target_w
group["target_width"] = target_width
group["target_height"] = target_height
return updated
def run_second_pass_groups(
*,
model: Any,
clip: Any,
sampler: Any,
sigmas: torch.Tensor,
video_latents: Sequence[Any],
audio_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
refine_seed: int,
refine_context: Any = None,
video_vae: Any = None,
conditioning_upscale_method: str = "bilinear",
enable_preview: bool = True,
encode_prompt_fn=None,
latent_builder=None,
sample_fn=None,
stream_extractor=None,
clone_model_fn=None,
validate_refine_context_fn=None,
adapt_group_conditioning_fn=None,
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
"""Refine complete physical groups while preserving first-pass audio exactly."""
validation = validate_second_pass_inputs(video_latents, audio_latents, assembly_plan)
groups = _contract_groups(assembly_plan)
encode_prompt_fn = encode_prompt_fn or encode_prompt_conditioning
latent_builder = latent_builder or latent_from_cpu
sample_fn = sample_fn or sample_refine_chunk
stream_extractor = stream_extractor or extract_av_streams
clone_model_fn = clone_model_fn or clone_model_for_chunk
validate_refine_context_fn = validate_refine_context_fn or validate_refine_context
adapt_group_conditioning_fn = (
adapt_group_conditioning_fn or adapt_group_conditioning
)
captured_groups: Sequence[Mapping[str, Any]] | None = None
context_warnings: list[str] = []
if refine_context is None:
context_warnings.append(
"WARNING: refine_context is not connected; using legacy prompt-only "
"conditioning."
)
else:
typed_context = (
isinstance(refine_context, Mapping)
and refine_context.get("magic") == REFINE_CONTEXT_MAGIC
)
try:
validated_context = validate_refine_context_fn(
refine_context,
assembly_plan=assembly_plan,
)
except RefineContextError as exc:
if typed_context:
raise SecondPassContractError(
f"refine_context contract is invalid: {exc}"
) from exc
context_warnings.append(
"WARNING: optional refine_context was not a typed Continuum context; "
f"using legacy prompt-only conditioning ({exc})."
)
else:
if bool(validated_context.get("complete")):
captured_groups = validated_context["groups"]
else:
context_warnings.append(
"WARNING: refine_context capture is incomplete; using legacy "
"prompt-only conditioning."
)
refined_videos: list[dict[str, Any]] = []
group_seeds: list[int] = []
conditioning_sources: list[str] = []
report_lines = [
"H3 Continuum Second Pass V3.5",
"Mode: context-aware physical-group refine when a complete First Pass "
"refine_context is available.",
f"Physical groups: {validation['physical_group_count']}.",
"Sampling contract: video random noise/mask=1; audio zero noise/mask=0; "
"workflow SIGMAS are applied directly by Core.",
*context_warnings,
]
for group_index, (video_latent, audio_latent, group) in enumerate(
zip(video_latents, audio_latents, groups, strict=True)
):
video_samples = _latent_samples(video_latent, name=f"video_latents[{group_index}]")
audio_samples = _latent_samples(audio_latent, name=f"audio_latents[{group_index}]")
captured_group = (
captured_groups[group_index] if captured_groups is not None else None
)
model_group = captured_group if captured_group is not None else group
adaptation_stats: Mapping[str, Any] | None = None
context_conditioning_available = False
if captured_group is not None:
try:
conditioning, adaptation_stats = adapt_group_conditioning_fn(
captured_group,
target_latent_h=validation["target_latent_h"],
target_latent_w=validation["target_latent_w"],
video_vae=video_vae,
upscale_method=str(conditioning_upscale_method),
)
conditioning_source = "refine_context"
context_conditioning_available = True
except RefineConditioningAdaptationError as exc:
report_lines.append(
f"WARNING: group {group_index + 1} refine_context target "
f"adaptation was unavailable; using legacy prompt-only "
f"conditioning ({exc})."
)
except RefineContextError as exc:
raise SecondPassContractError(
f"refine_context group {group_index} is invalid: {exc}"
) from exc
if not context_conditioning_available:
prompt = str(group.get("physical_prompt", ""))
conditioning = encode_prompt_fn(
clip,
prompt,
first_image=None,
last_image=None,
reference_audio_assets=None,
timeline_video_assets=None,
)
conditioning_source = "prompt_only_fallback"
logical_chunks = model_group.get("logical_chunks")
if not isinstance(logical_chunks, Sequence) or not logical_chunks:
logical_chunks = group.get("logical_chunks")
if not isinstance(logical_chunks, Sequence) or not logical_chunks:
raise SecondPassContractError(
f"physical group {group_index} has no logical chunk identity"
)
physical_clip_index = int(
model_group.get("physical_clip_index", logical_chunks[0])
)
context_frames = int(
model_group.get(
"context_frames",
group.get("trim_prefix_frames", 0),
)
)
if physical_clip_index <= 0 or context_frames < 0:
raise SecondPassContractError(
f"physical group {group_index} MODEL context is invalid"
)
chunk_model = clone_model_fn(
model,
strict=False,
debug=False,
chunk_index=physical_clip_index,
context_frames=context_frames if context_frames > 0 else None,
)
physical_seed = derive_refine_seed(int(refine_seed), group_index)
group_seeds.append(physical_seed)
conditioning_sources.append(conditioning_source)
nested_latent = latent_builder(video_samples, audio_samples)
sampled = sample_fn(
model=chunk_model,
conditioning=conditioning,
latent=nested_latent,
sampler=sampler,
sigmas=sigmas,
seed=physical_seed,
enable_preview=bool(enable_preview),
)
refined_video, _temporary_audio = stream_extractor(sampled)
if tuple(refined_video.shape) != tuple(video_samples.shape):
raise SecondPassContractError(
f"refined video group {group_index} changed B/C/T/H/W geometry"
)
if not bool(torch.isfinite(refined_video.float()).all().item()):
raise SecondPassContractError(
f"refined video group {group_index} contains NaN or Inf"
)
output_latent = dict(video_latent)
output_latent["samples"] = refined_video
refined_videos.append(output_latent)
report_lines.append(
"group "
f"{group_index + 1}: logical_chunks={group.get('logical_chunks')}, "
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
f"conditioning_source={conditioning_source}, "
f"physical_clip_index={physical_clip_index}, "
f"context_frames={context_frames}, "
f"refine_seed={physical_seed}, shape={tuple(refined_video.shape)}, "
"sampling_passes=1, temporary_audio_discarded=true"
)
if adaptation_stats is not None:
report_lines.append(
"group "
f"{group_index + 1} target conditioning: "
f"keyframes_resized={int(adaptation_stats.get('keyframes_resized', 0))}, "
f"keyframes_reencoded={int(adaptation_stats.get('keyframes_reencoded', 0))}, "
f"context_refs_resized={int(adaptation_stats.get('context_refs_resized', 0))}"
)
report_lines.extend(
f"WARNING: {warning}"
for warning in adaptation_stats.get("warnings", ())
)
passthrough_audio = passthrough_audio_latents(audio_latents)
updated_plan = update_second_pass_geometry(assembly_plan, refined_videos)
contract = updated_plan["second_pass_contract"]
if conditioning_sources and all(
source == "refine_context" for source in conditioning_sources
):
contract["execution"] = "context_aware_physical_groups_audio_locked_v3"
elif any(source == "refine_context" for source in conditioning_sources):
contract["execution"] = "mixed_context_physical_groups_audio_locked_v3"
else:
contract["execution"] = "t2va_physical_groups_audio_locked_v2"
contract["conditioning_sources"] = conditioning_sources
contract["refine_seed_base"] = int(refine_seed)
contract["refine_group_seeds"] = group_seeds
contract["audio_output"] = "bit_exact_first_pass_passthrough"
contract["audio_sampling"] = "zero_noise_mask_locked"
report_lines.append(
"Audio output: first-pass physical audio LATENT objects returned bit-exact; "
"temporary refined audio discarded."
)
return refined_videos, passthrough_audio, updated_plan, "\n".join(report_lines)
+110
View File
@@ -0,0 +1,110 @@
"""V3.5 Second Pass public node."""
from __future__ import annotations
from typing import Any
from .second_pass import run_second_pass_groups
def _single_list_input(name: str, value: Any) -> Any:
if not isinstance(value, list) or len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
return value[0]
class H3ContinuumSecondPassV35:
"""Low-sigma refinement over complete physical decode groups."""
DEPRECATED = False
DESCRIPTION = (
"V3.5 context-aware Second Pass. Connect externally upscaled H3 video "
"latents, the matching first-pass audio latents, and the original assembly plan. "
"Connect the optional Continuum refine context to preserve the physical group's "
"First/Last/Reference conditioning. "
"SIGMAS controls the low-denoise refine schedule; audio output is preserved "
"bit-exact from the first pass."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"sampler": ("SAMPLER",),
"sigmas": ("SIGMAS",),
"video_latents": ("LATENT",),
"audio_latents": ("LATENT",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"refine_seed": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
"tooltip": "Base seed independently derived once per physical group.",
},
),
},
"optional": {
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
"video_vae": ("VAE",),
},
}
INPUT_IS_LIST = True
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
RETURN_NAMES = (
"refined_video_latents",
"audio_latents",
"updated_assembly_plan",
"status",
)
OUTPUT_IS_LIST = (True, True, False, False)
FUNCTION = "refine"
CATEGORY = "MiniMax H3/Continuum/Advanced"
def refine(
self,
model,
clip,
sampler,
sigmas,
video_latents,
audio_latents,
assembly_plan,
refine_seed,
refine_context=None,
video_vae=None,
):
return run_second_pass_groups(
model=_single_list_input("model", model),
clip=_single_list_input("clip", clip),
sampler=_single_list_input("sampler", sampler),
sigmas=_single_list_input("sigmas", sigmas),
video_latents=video_latents,
audio_latents=audio_latents,
assembly_plan=_single_list_input("assembly_plan", assembly_plan),
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
refine_context=(
None
if refine_context is None
else _single_list_input("refine_context", refine_context)
),
video_vae=(
None
if video_vae is None
else _single_list_input("video_vae", video_vae)
),
)
NODE_CLASS_MAPPINGS = {
"H3ContinuumSecondPassV35": H3ContinuumSecondPassV35,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumSecondPassV35": "H3 Continuum Second Pass V3.5",
}
+131
View File
@@ -0,0 +1,131 @@
"""Small decoded-boundary patches for the V3.5 direct-write assembler.
The established V3.4 seam analyzer remains the source of all boundary
decisions. This module reproduces only the correction step while avoiding the
full physical-group clone performed by ``correct_decoded_boundaries``.
"""
from __future__ import annotations
from typing import Any
import torch
from .plan import validate_assembly_plan
from .video_seam import VideoBoundaryAnalysis, _validate_frames
def build_boundary_patches(
images: list[Any],
assembly_plan: dict[str, Any],
analyses: list[VideoBoundaryAnalysis],
*,
enable_exposure_ramp: bool = False,
) -> tuple[dict[int, torch.Tensor], dict[int, str]]:
"""Return qualified 1--4 frame corrections without changing ``images``.
Patch keys are the existing one-based ``boundary_index`` values. They are
also the zero-based index of the current physical decode group, so the
direct writer can add its natural-frame cursor without this module knowing
the final buffer layout.
The result is transactional: all patches and actions are built in local
containers and are returned only after every requested boundary succeeds.
Any exception therefore leaves the caller with no partial correction set.
"""
plan = validate_assembly_plan(assembly_plan)
groups = list(
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
)
if len(images) != len(groups):
raise ValueError("decoded image count does not match the assembly plan")
patches: dict[int, torch.Tensor] = {}
actions: dict[int, str] = {}
for analysis in analyses:
boundary = int(analysis.boundary_index)
correction_kind = "transient flash"
replace_frames = int(analysis.recommended_rewind)
if analysis.micro_flash_candidate:
correction_kind = "micro-flash"
replace_frames = 1
elif enable_exposure_ramp and analysis.exposure_ramp_candidate:
correction_kind = "exposure ramp"
replace_frames = 4
if replace_frames <= 0:
actions[boundary] = "kept native boundary"
continue
if analysis.scene_cut or not (
analysis.transient_flash_candidate
or analysis.micro_flash_candidate
or (enable_exposure_ramp and analysis.exposure_ramp_candidate)
):
actions[boundary] = (
"kept native boundary; correction guard rejected candidate"
)
continue
previous_index = boundary - 1
current_index = boundary
if previous_index < 0 or current_index >= len(images):
raise ValueError("video seam boundary index is outside decoded chunks")
previous_raw = images[previous_index]
current_raw = images[current_index]
_validate_frames(
previous_raw,
f"decoded image chunk {previous_index + 1}",
)
_validate_frames(
current_raw,
f"decoded image chunk {current_index + 1}",
)
previous_plan = groups[previous_index]
current_plan = groups[current_index]
previous_total = int(previous_plan["total_frames"])
previous_trim = int(previous_plan["trim_frames"])
current_total = int(current_plan["total_frames"])
current_trim = int(current_plan["trim_frames"])
if current_trim + replace_frames >= current_total:
raise ValueError("video seam correction has no retained recovery frame")
previous_segment = previous_raw[:previous_total][previous_trim:]
if int(previous_segment.shape[0]) < 1:
raise ValueError("video seam correction has no previous retained frame")
# Only the frames that may be changed are cloned. The recovery frame
# and previous anchor remain read-only views of their physical groups.
patch = current_raw[
current_trim : current_trim + replace_frames
].detach().to(device="cpu").clone()
if int(patch.shape[0]) != replace_frames:
raise ValueError("video seam correction frames are outside the current chunk")
previous_anchor = previous_segment[-1].detach().to(
device="cpu",
dtype=torch.float32,
)
recovery_anchor = current_raw[current_trim + replace_frames].detach().to(
device="cpu",
dtype=torch.float32,
)
if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape):
raise ValueError("decoded image geometry changed at video seam")
for offset in range(replace_frames):
alpha = float(offset + 1) / float(replace_frames + 1)
original = patch[offset].to(dtype=torch.float32)
target = torch.lerp(previous_anchor, recovery_anchor, alpha)
original_mean = original.mean(dim=(0, 1), keepdim=True)
target_mean = target.mean(dim=(0, 1), keepdim=True)
normalized = (original + target_mean - original_mean).clamp(0.0, 1.0)
patch[offset].copy_(normalized.to(dtype=patch.dtype))
patches[boundary] = patch
actions[boundary] = (
f"normalized exposure/color on {replace_frames} {correction_kind} "
"boundary frame(s)"
)
return patches, actions