feat: release V3.5.0
This commit is contained in:
@@ -0,0 +1,720 @@
|
||||
"""V3.5 direct-write assembly with a manual RAM or file-backed IMAGE buffer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
from ..constants import (
|
||||
DIAGNOSTICS_FULL,
|
||||
DIAGNOSTICS_OFF,
|
||||
normalize_diagnostics_mode,
|
||||
)
|
||||
from ..hardening import assemble_with_hardening
|
||||
from ..media import validate_audio
|
||||
from ..v2.seam_guard import correct_audio_seam
|
||||
from .assembly import (
|
||||
AUDIO_SEAM_AUTO,
|
||||
AUDIO_SEAM_OPTIONS,
|
||||
VIDEO_SEAM_ANALYZE,
|
||||
VIDEO_SEAM_ANALYSIS_OPTIONS,
|
||||
VIDEO_SEAM_AUTO,
|
||||
VIDEO_SEAM_AUTO_2,
|
||||
VIDEO_SEAM_OFF,
|
||||
)
|
||||
from .file_backed_buffer import (
|
||||
BUFFER_BACKEND_AUTO,
|
||||
BUFFER_BACKEND_DISK,
|
||||
BUFFER_BACKEND_OPTIONS,
|
||||
BUFFER_BACKEND_RAM,
|
||||
allocate_file_backed_image,
|
||||
format_auto_buffer_decision,
|
||||
select_auto_buffer_backend,
|
||||
)
|
||||
from .plan import FPS, validate_assembly_plan
|
||||
|
||||
|
||||
_COPY_BATCH_FRAMES = 8
|
||||
|
||||
|
||||
def _check_interrupted(
|
||||
interrupt_check: Callable[[], None] | None,
|
||||
) -> None:
|
||||
"""Honor ComfyUI's normal ESC interrupt without requiring Core in tests."""
|
||||
|
||||
if interrupt_check is not None:
|
||||
interrupt_check()
|
||||
return
|
||||
try:
|
||||
from comfy.model_management import throw_exception_if_processing_interrupted
|
||||
except ModuleNotFoundError:
|
||||
return
|
||||
throw_exception_if_processing_interrupted()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CopySpan:
|
||||
"""Map a half-open natural-timeline interval into the output buffer."""
|
||||
|
||||
source_start: int
|
||||
source_stop: int
|
||||
destination_start: int
|
||||
|
||||
|
||||
class _RamAllocation(AbstractContextManager):
|
||||
"""Match the provisional file-backed allocation interface for RAM."""
|
||||
|
||||
def __init__(self, shape: tuple[int, ...], dtype: torch.dtype):
|
||||
self.tensor = torch.empty(shape, dtype=dtype, device="cpu")
|
||||
self.record = None
|
||||
self._published = False
|
||||
|
||||
def publish(self) -> torch.Tensor:
|
||||
if self._published:
|
||||
raise RuntimeError("RAM assembly buffer was already published")
|
||||
self._published = True
|
||||
return self.tensor
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _video_copy_spans(
|
||||
*,
|
||||
natural_frames: int,
|
||||
output_frames: int,
|
||||
preserve_final_frame: bool,
|
||||
) -> tuple[CopySpan, ...]:
|
||||
if output_frames >= natural_frames:
|
||||
return (CopySpan(0, natural_frames, 0),)
|
||||
if preserve_final_frame and output_frames >= 2:
|
||||
return (
|
||||
CopySpan(0, output_frames - 1, 0),
|
||||
CopySpan(natural_frames - 1, natural_frames, output_frames - 1),
|
||||
)
|
||||
return (CopySpan(0, output_frames, 0),)
|
||||
|
||||
|
||||
def _copy_natural_interval(
|
||||
destination: torch.Tensor,
|
||||
source: torch.Tensor,
|
||||
*,
|
||||
natural_start: int,
|
||||
spans: tuple[CopySpan, ...],
|
||||
interrupt_check: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
natural_stop = natural_start + int(source.shape[0])
|
||||
for span in spans:
|
||||
overlap_start = max(natural_start, int(span.source_start))
|
||||
overlap_stop = min(natural_stop, int(span.source_stop))
|
||||
if overlap_stop <= overlap_start:
|
||||
continue
|
||||
source_start = overlap_start - natural_start
|
||||
destination_start = int(span.destination_start) + (
|
||||
overlap_start - int(span.source_start)
|
||||
)
|
||||
frame_count = overlap_stop - overlap_start
|
||||
for offset in range(0, frame_count, _COPY_BATCH_FRAMES):
|
||||
_check_interrupted(interrupt_check)
|
||||
batch_frames = min(_COPY_BATCH_FRAMES, frame_count - offset)
|
||||
source_batch_start = source_start + offset
|
||||
destination_batch_start = destination_start + offset
|
||||
destination[
|
||||
destination_batch_start : destination_batch_start + batch_frames
|
||||
].copy_(
|
||||
source[source_batch_start : source_batch_start + batch_frames]
|
||||
)
|
||||
|
||||
|
||||
def _repeat_final_video_frame(
|
||||
destination: torch.Tensor,
|
||||
*,
|
||||
natural_frames: int,
|
||||
output_frames: int,
|
||||
interrupt_check: Callable[[], None] | None,
|
||||
) -> None:
|
||||
for destination_start in range(
|
||||
natural_frames,
|
||||
output_frames,
|
||||
_COPY_BATCH_FRAMES,
|
||||
):
|
||||
_check_interrupted(interrupt_check)
|
||||
destination_stop = min(
|
||||
output_frames,
|
||||
destination_start + _COPY_BATCH_FRAMES,
|
||||
)
|
||||
destination[destination_start:destination_stop].copy_(
|
||||
destination[natural_frames - 1 : natural_frames].expand(
|
||||
destination_stop - destination_start,
|
||||
*destination.shape[1:],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _copy_audio_segment(
|
||||
destination: torch.Tensor,
|
||||
waveform: torch.Tensor,
|
||||
*,
|
||||
trim_samples: int,
|
||||
) -> None:
|
||||
"""Reproduce timeline slicing/padding without concatenating a waveform."""
|
||||
|
||||
wanted = int(destination.shape[-1])
|
||||
trim_samples = max(0, int(trim_samples))
|
||||
available = max(0, min(wanted, int(waveform.shape[-1]) - trim_samples))
|
||||
if available > 0:
|
||||
destination[..., :available].copy_(
|
||||
waveform[..., trim_samples : trim_samples + available]
|
||||
)
|
||||
if available >= wanted:
|
||||
return
|
||||
if available > 0:
|
||||
destination[..., available:].copy_(
|
||||
destination[..., available - 1 : available].expand(
|
||||
*destination.shape[:-1],
|
||||
wanted - available,
|
||||
)
|
||||
)
|
||||
else:
|
||||
destination.zero_()
|
||||
|
||||
|
||||
def adjust_audio_duration_direct(
|
||||
audio: Mapping[str, Any],
|
||||
*,
|
||||
target_frames: int,
|
||||
preserve_final_audio_anchor: bool,
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
"""Apply the legacy audio trim/pad values using direct allocation and copy."""
|
||||
|
||||
waveform, sample_rate = validate_audio(audio)
|
||||
waveform = waveform.detach().to(device="cpu")
|
||||
sample_rate = int(sample_rate)
|
||||
target_samples = int(round(int(target_frames) / FPS * sample_rate))
|
||||
original_samples = int(waveform.shape[-1])
|
||||
delta_samples = target_samples - original_samples
|
||||
if delta_samples == 0:
|
||||
return {"waveform": waveform, "sample_rate": sample_rate}, 0
|
||||
|
||||
output = torch.empty(
|
||||
(*waveform.shape[:-1], target_samples),
|
||||
dtype=waveform.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
if delta_samples < 0:
|
||||
if preserve_final_audio_anchor and target_samples > 1:
|
||||
tail_samples = min(
|
||||
max(1, int(round(sample_rate / FPS))),
|
||||
target_samples,
|
||||
original_samples,
|
||||
)
|
||||
head_samples = target_samples - tail_samples
|
||||
if head_samples > 0:
|
||||
output[..., :head_samples].copy_(waveform[..., :head_samples])
|
||||
output[..., head_samples:].copy_(waveform[..., -tail_samples:])
|
||||
fade = min(
|
||||
int(round(sample_rate * 0.005)),
|
||||
head_samples,
|
||||
tail_samples,
|
||||
)
|
||||
if fade > 0:
|
||||
start = output[..., head_samples - 1 : head_samples].to(
|
||||
dtype=output.dtype
|
||||
)
|
||||
alpha = torch.linspace(
|
||||
0.0,
|
||||
1.0,
|
||||
fade,
|
||||
dtype=output.dtype,
|
||||
device=output.device,
|
||||
).reshape(*([1] * (output.ndim - 1)), fade)
|
||||
tail_head = output[..., head_samples : head_samples + fade]
|
||||
tail_head.copy_(start * (1.0 - alpha) + tail_head * alpha)
|
||||
else:
|
||||
output.copy_(waveform[..., :target_samples])
|
||||
else:
|
||||
if original_samples > 0:
|
||||
output[..., :original_samples].copy_(waveform)
|
||||
output[..., original_samples:].copy_(
|
||||
waveform[..., -1:].expand(
|
||||
*waveform.shape[:-1],
|
||||
delta_samples,
|
||||
)
|
||||
)
|
||||
else:
|
||||
output.zero_()
|
||||
return {"waveform": output, "sample_rate": sample_rate}, delta_samples
|
||||
|
||||
|
||||
def _prepare_video_seam(
|
||||
*,
|
||||
images: list[Any],
|
||||
plan: dict[str, Any],
|
||||
mode: str,
|
||||
) -> tuple[list[Any], dict[int, torch.Tensor], dict[int, str], Exception | None]:
|
||||
if mode == VIDEO_SEAM_OFF:
|
||||
return [], {}, {}, None
|
||||
analyses: list[Any] = []
|
||||
try:
|
||||
from .video_seam import analyze_decoded_boundaries
|
||||
|
||||
analyses = analyze_decoded_boundaries(
|
||||
images=images,
|
||||
assembly_plan=plan,
|
||||
)
|
||||
patches: dict[int, torch.Tensor] = {}
|
||||
actions: dict[int, str] = {}
|
||||
if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2):
|
||||
from .video_seam_v35 import build_boundary_patches
|
||||
|
||||
patches, actions = build_boundary_patches(
|
||||
images=images,
|
||||
assembly_plan=plan,
|
||||
analyses=analyses,
|
||||
enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2,
|
||||
)
|
||||
return analyses, patches, actions, None
|
||||
except Exception as exc:
|
||||
# Match V3.4's all-native fallback: no partial patch is retained.
|
||||
return analyses, {}, {}, exc
|
||||
|
||||
|
||||
def _video_seam_status(mode: str) -> str:
|
||||
if mode == VIDEO_SEAM_OFF:
|
||||
return "Video seam correction is disabled."
|
||||
if mode == VIDEO_SEAM_ANALYZE:
|
||||
return "Video Seam: Analyze Only; decoded frames and audio are unchanged."
|
||||
if mode == VIDEO_SEAM_AUTO:
|
||||
return "Video Seam: Auto; guarded transient and micro-flash correction enabled."
|
||||
return (
|
||||
"Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, "
|
||||
"and exposure-ramp correction enabled."
|
||||
)
|
||||
|
||||
|
||||
def _format_video_seam_lines(
|
||||
*,
|
||||
mode: str,
|
||||
analyses: list[Any],
|
||||
actions: dict[int, str],
|
||||
error: Exception | None,
|
||||
) -> list[str]:
|
||||
if mode == VIDEO_SEAM_OFF:
|
||||
return []
|
||||
if error is not None:
|
||||
return [
|
||||
f"Video Seam {mode}: native output preserved; analysis unavailable "
|
||||
f"({type(error).__name__}: {error})"
|
||||
]
|
||||
from .video_seam import format_video_boundary_analysis
|
||||
|
||||
lines = [
|
||||
format_video_boundary_analysis(
|
||||
item,
|
||||
action=(
|
||||
"analysis only"
|
||||
if mode == VIDEO_SEAM_ANALYZE
|
||||
else actions.get(item.boundary_index, "kept native boundary")
|
||||
),
|
||||
)
|
||||
for item in analyses
|
||||
]
|
||||
if not lines:
|
||||
lines.append(f"Video Seam {mode}: no decoded chunk boundary to analyze.")
|
||||
return lines
|
||||
|
||||
|
||||
def _format_elapsed(seconds: float) -> str:
|
||||
total = max(0.0, float(seconds))
|
||||
hours, remainder = divmod(total, 3600.0)
|
||||
minutes, seconds = divmod(remainder, 60.0)
|
||||
return f"{int(hours):02d}:{int(minutes):02d}:{seconds:05.2f}"
|
||||
|
||||
|
||||
def _write_assembly(
|
||||
*,
|
||||
image_buffer: torch.Tensor,
|
||||
images: list[Any],
|
||||
audio: list[Any],
|
||||
plan: dict[str, Any],
|
||||
exact_total_duration: bool,
|
||||
audio_seam: str,
|
||||
video_seam: str,
|
||||
diagnostics: str,
|
||||
natural_frames: int,
|
||||
output_frames: int,
|
||||
video_spans: tuple[CopySpan, ...],
|
||||
seam_patches: dict[int, torch.Tensor],
|
||||
seam_analyses: list[Any],
|
||||
seam_actions: dict[int, str],
|
||||
seam_error: Exception | None,
|
||||
buffer_backend: str,
|
||||
auto_decision_report: str | None,
|
||||
final_audio_override: Mapping[str, Any] | None,
|
||||
final_audio_source: str | None,
|
||||
interrupt_check: Callable[[], None] | None,
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
diagnostics_mode = normalize_diagnostics_mode(diagnostics)
|
||||
units = list(plan.get("decode_groups") or plan["chunks"])
|
||||
first_waveform, audio_rate = validate_audio(audio[0])
|
||||
first_waveform = first_waveform.detach().to(device="cpu")
|
||||
audio_rate = int(audio_rate)
|
||||
natural_samples = int(round(natural_frames / FPS * audio_rate))
|
||||
audio_buffer = torch.empty(
|
||||
(*first_waveform.shape[:-1], natural_samples),
|
||||
dtype=first_waveform.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
reports = [
|
||||
"H3 Continuum Assemble V3.5",
|
||||
f"Audio Seam: {audio_seam}. {_video_seam_status(video_seam)}",
|
||||
f"Buffer Backend: {buffer_backend}; IMAGE direct-write "
|
||||
f"frames={output_frames}, bytes={image_buffer.numel() * image_buffer.element_size()}; "
|
||||
"no final full IMAGE copy.",
|
||||
]
|
||||
if auto_decision_report is not None:
|
||||
reports.append(auto_decision_report)
|
||||
|
||||
frame_cursor = 0
|
||||
for index, (raw_images, raw_audio, unit) in enumerate(
|
||||
zip(images, audio, units),
|
||||
start=1,
|
||||
):
|
||||
_check_interrupted(interrupt_check)
|
||||
total_frames = int(unit["total_frames"])
|
||||
trim_frames = int(unit["trim_frames"])
|
||||
net_frames = int(unit["net_frames"])
|
||||
raw_images_cpu = raw_images.detach().to(device="cpu")
|
||||
retained_images = raw_images_cpu[trim_frames:total_frames]
|
||||
if int(retained_images.shape[0]) != net_frames:
|
||||
raise ValueError(
|
||||
f"decoded image chunk {index} retained {retained_images.shape[0]} "
|
||||
f"frames; plan expects {net_frames}"
|
||||
)
|
||||
_copy_natural_interval(
|
||||
image_buffer,
|
||||
retained_images,
|
||||
natural_start=frame_cursor,
|
||||
spans=video_spans,
|
||||
interrupt_check=interrupt_check,
|
||||
)
|
||||
|
||||
_check_interrupted(interrupt_check)
|
||||
waveform, sample_rate = validate_audio(raw_audio)
|
||||
waveform = waveform.detach().to(device="cpu")
|
||||
sample_rate = int(sample_rate)
|
||||
if sample_rate != audio_rate:
|
||||
raise ValueError(
|
||||
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
|
||||
)
|
||||
frame_stop = frame_cursor + net_frames
|
||||
sample_start = int(round(frame_cursor / FPS * sample_rate))
|
||||
sample_stop = int(round(frame_stop / FPS * sample_rate))
|
||||
seam_report = None
|
||||
if index > 1 and audio_seam == AUDIO_SEAM_AUTO:
|
||||
try:
|
||||
cut_sample = int(round(trim_frames / FPS * sample_rate))
|
||||
patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam(
|
||||
audio_buffer[..., :sample_start],
|
||||
waveform,
|
||||
sample_rate=sample_rate,
|
||||
cut_sample=cut_sample,
|
||||
)
|
||||
if patch is not None and int(patch.shape[-1]) > 0:
|
||||
patch_start = sample_start - int(patch.shape[-1])
|
||||
if patch_start < 0:
|
||||
raise ValueError("audio seam patch starts before the output")
|
||||
audio_buffer[..., patch_start:sample_start].copy_(patch)
|
||||
seam_report = (
|
||||
f"audio seam {index-1}->{index}: "
|
||||
f"corr={metrics.correlation_before:.4f}->"
|
||||
f"{metrics.correlation_after:.4f}, "
|
||||
f"jump={metrics.boundary_jump_before:.6f}->"
|
||||
f"{metrics.boundary_jump_after:.6f}, "
|
||||
f"fade={fade_samples} samples"
|
||||
)
|
||||
if diagnostics_mode == DIAGNOSTICS_FULL:
|
||||
seam_report += (
|
||||
f", offset={metrics.offset_samples:+d}, "
|
||||
f"gain={level_gain:.4f}, dc={dc_bias:+.6f}"
|
||||
)
|
||||
except Exception as exc:
|
||||
seam_report = (
|
||||
f"audio seam {index-1}->{index}: fallback to native boundary "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
trim_samples = int(round(trim_frames / FPS * sample_rate))
|
||||
_copy_audio_segment(
|
||||
audio_buffer[..., sample_start:sample_stop],
|
||||
waveform,
|
||||
trim_samples=trim_samples,
|
||||
)
|
||||
if diagnostics_mode != DIAGNOSTICS_OFF:
|
||||
reports.append(
|
||||
f"assembled decoded chunk {index}: {net_frames} retained frames, "
|
||||
f"cumulative {frame_stop}/{natural_frames}"
|
||||
)
|
||||
if seam_report:
|
||||
reports.append(seam_report)
|
||||
frame_cursor = frame_stop
|
||||
|
||||
if frame_cursor != natural_frames:
|
||||
raise RuntimeError(
|
||||
f"V3.5 assembly cursor mismatch: {frame_cursor} != {natural_frames}"
|
||||
)
|
||||
|
||||
unit_starts: list[int] = []
|
||||
cursor = 0
|
||||
for unit in units:
|
||||
unit_starts.append(cursor)
|
||||
cursor += int(unit["net_frames"])
|
||||
for boundary, patch in seam_patches.items():
|
||||
_check_interrupted(interrupt_check)
|
||||
if boundary <= 0 or boundary >= len(unit_starts):
|
||||
raise ValueError("V3.5 video seam patch boundary is outside decode groups")
|
||||
_copy_natural_interval(
|
||||
image_buffer,
|
||||
patch,
|
||||
natural_start=unit_starts[boundary],
|
||||
spans=video_spans,
|
||||
interrupt_check=interrupt_check,
|
||||
)
|
||||
|
||||
if output_frames > natural_frames:
|
||||
_repeat_final_video_frame(
|
||||
image_buffer,
|
||||
natural_frames=natural_frames,
|
||||
output_frames=output_frames,
|
||||
interrupt_check=interrupt_check,
|
||||
)
|
||||
|
||||
_check_interrupted(interrupt_check)
|
||||
result_audio: dict[str, Any] = {
|
||||
"waveform": audio_buffer,
|
||||
"sample_rate": audio_rate,
|
||||
}
|
||||
duration_report = ""
|
||||
if exact_total_duration:
|
||||
preserve_anchor = bool(
|
||||
plan.get("preserve_final_frame", False)
|
||||
and output_frames < natural_frames
|
||||
and output_frames >= 2
|
||||
)
|
||||
result_audio, delta_samples = adjust_audio_duration_direct(
|
||||
result_audio,
|
||||
target_frames=output_frames,
|
||||
preserve_final_audio_anchor=preserve_anchor,
|
||||
)
|
||||
adjustment = output_frames - natural_frames
|
||||
mode = (
|
||||
"final anchor preserved; pre-tail trim"
|
||||
if preserve_anchor
|
||||
else "tail trim/pad"
|
||||
)
|
||||
duration_report = (
|
||||
f"Exact-duration adjustment: frames {natural_frames}->{output_frames} "
|
||||
f"({adjustment:+d}), audio samples correction {delta_samples:+d}; {mode}."
|
||||
)
|
||||
|
||||
driving_audio_report = ""
|
||||
if final_audio_override is not None:
|
||||
result_audio = dict(final_audio_override)
|
||||
if exact_total_duration:
|
||||
result_audio, _ = adjust_audio_duration_direct(
|
||||
result_audio,
|
||||
target_frames=output_frames,
|
||||
preserve_final_audio_anchor=False,
|
||||
)
|
||||
waveform, selected_rate = validate_audio(result_audio)
|
||||
result_audio = {
|
||||
"waveform": waveform,
|
||||
"sample_rate": int(selected_rate),
|
||||
}
|
||||
driving_audio_report = (
|
||||
"Driving Audio: preserved source selected from "
|
||||
f"{final_audio_source or 'direct input'}; "
|
||||
f"sample_rate={int(selected_rate)}, samples={int(waveform.shape[-1])}; "
|
||||
"generated audio and Audio Seam bypassed."
|
||||
)
|
||||
|
||||
reports.append(
|
||||
f"Assembled {len(units)} physical decoded group(s) into "
|
||||
f"{output_frames} frames with cumulative sample-boundary alignment."
|
||||
)
|
||||
if duration_report:
|
||||
reports.append(duration_report)
|
||||
if driving_audio_report:
|
||||
reports.append(driving_audio_report)
|
||||
runtime_started_at = plan.get("_runtime_started_at")
|
||||
if isinstance(runtime_started_at, (int, float)):
|
||||
elapsed = time.perf_counter() - float(runtime_started_at)
|
||||
if math.isfinite(elapsed) and elapsed >= 0.0:
|
||||
reports.append(f"Total workflow elapsed: {_format_elapsed(elapsed)}")
|
||||
reports.extend(
|
||||
_format_video_seam_lines(
|
||||
mode=video_seam,
|
||||
analyses=seam_analyses,
|
||||
actions=seam_actions,
|
||||
error=seam_error,
|
||||
)
|
||||
)
|
||||
return result_audio, "\n".join(reports)
|
||||
|
||||
|
||||
def _assemble_decoded_chunks_v35_core(
|
||||
*,
|
||||
images: list[Any],
|
||||
audio: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
exact_total_duration: bool,
|
||||
audio_seam: str,
|
||||
video_seam: str,
|
||||
diagnostics: str,
|
||||
buffer_backend: str,
|
||||
backing_root: str | Path | None = None,
|
||||
final_audio_override: Mapping[str, Any] | None = None,
|
||||
final_audio_source: str | None = None,
|
||||
interrupt_check: Callable[[], None] | None = None,
|
||||
):
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
if audio_seam not in AUDIO_SEAM_OPTIONS:
|
||||
raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}")
|
||||
if video_seam not in VIDEO_SEAM_ANALYSIS_OPTIONS:
|
||||
raise ValueError(f"unknown Video Seam mode: {video_seam!r}")
|
||||
if buffer_backend not in BUFFER_BACKEND_OPTIONS:
|
||||
raise ValueError(f"unknown Buffer Backend: {buffer_backend!r}")
|
||||
units = list(plan.get("decode_groups") or plan["chunks"])
|
||||
natural_frames = sum(int(unit["net_frames"]) for unit in units)
|
||||
output_frames = (
|
||||
int(plan["target_frames"])
|
||||
if bool(exact_total_duration)
|
||||
else natural_frames
|
||||
)
|
||||
if natural_frames <= 0 or output_frames <= 0:
|
||||
raise ValueError("V3.5 assembly frame counts must be positive")
|
||||
first_image = images[0]
|
||||
shape = (output_frames, *tuple(int(value) for value in first_image.shape[1:]))
|
||||
video_spans = _video_copy_spans(
|
||||
natural_frames=natural_frames,
|
||||
output_frames=output_frames,
|
||||
preserve_final_frame=bool(plan.get("preserve_final_frame", False)),
|
||||
)
|
||||
analyses, patches, actions, analysis_error = _prepare_video_seam(
|
||||
images=images,
|
||||
plan=plan,
|
||||
mode=video_seam,
|
||||
)
|
||||
|
||||
resolved_backend = buffer_backend
|
||||
resolved_backing_root = backing_root
|
||||
auto_decision_report = None
|
||||
if buffer_backend == BUFFER_BACKEND_AUTO:
|
||||
final_image_bytes = (
|
||||
math.prod(shape) * first_image.element_size()
|
||||
)
|
||||
decision = select_auto_buffer_backend(
|
||||
final_image_bytes,
|
||||
backing_root=backing_root,
|
||||
)
|
||||
resolved_backend = decision.selected_backend
|
||||
resolved_backing_root = decision.backing_root
|
||||
auto_decision_report = format_auto_buffer_decision(decision)
|
||||
|
||||
allocation: Any
|
||||
if resolved_backend == BUFFER_BACKEND_RAM:
|
||||
allocation = _RamAllocation(shape, first_image.dtype)
|
||||
else:
|
||||
allocation = allocate_file_backed_image(
|
||||
shape,
|
||||
dtype=first_image.dtype,
|
||||
backing_root=resolved_backing_root,
|
||||
)
|
||||
with allocation:
|
||||
# The file exists at this point, so an ESC received immediately before
|
||||
# assembly exercises the same unpublished cleanup path as a mid-copy
|
||||
# cancellation.
|
||||
_check_interrupted(interrupt_check)
|
||||
result_audio, report = _write_assembly(
|
||||
image_buffer=allocation.tensor,
|
||||
images=images,
|
||||
audio=audio,
|
||||
plan=plan,
|
||||
exact_total_duration=bool(exact_total_duration),
|
||||
audio_seam=audio_seam,
|
||||
video_seam=video_seam,
|
||||
diagnostics=diagnostics,
|
||||
natural_frames=natural_frames,
|
||||
output_frames=output_frames,
|
||||
video_spans=video_spans,
|
||||
seam_patches=patches,
|
||||
seam_analyses=analyses,
|
||||
seam_actions=actions,
|
||||
seam_error=analysis_error,
|
||||
buffer_backend=(
|
||||
resolved_backend
|
||||
if buffer_backend != BUFFER_BACKEND_AUTO
|
||||
else f"Auto -> {resolved_backend}"
|
||||
),
|
||||
auto_decision_report=auto_decision_report,
|
||||
final_audio_override=final_audio_override,
|
||||
final_audio_source=final_audio_source,
|
||||
interrupt_check=interrupt_check,
|
||||
)
|
||||
result_images = allocation.publish()
|
||||
return result_images, result_audio, report
|
||||
|
||||
|
||||
def assemble_decoded_chunks_v35(
|
||||
*,
|
||||
images: list[Any],
|
||||
audio: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
exact_total_duration: bool,
|
||||
audio_seam: str,
|
||||
video_seam: str,
|
||||
diagnostics: str,
|
||||
buffer_backend: str,
|
||||
backing_root: str | Path | None = None,
|
||||
final_audio_override: Mapping[str, Any] | None = None,
|
||||
final_audio_source: str | None = None,
|
||||
interrupt_check: Callable[[], None] | None = None,
|
||||
):
|
||||
"""Preflight all inputs, then execute the independent V3.5 writer."""
|
||||
|
||||
kwargs = {
|
||||
"images": images,
|
||||
"audio": audio,
|
||||
"assembly_plan": assembly_plan,
|
||||
"exact_total_duration": bool(exact_total_duration),
|
||||
"audio_seam": str(audio_seam),
|
||||
"video_seam": str(video_seam),
|
||||
"diagnostics": str(diagnostics),
|
||||
"buffer_backend": str(buffer_backend),
|
||||
"backing_root": backing_root,
|
||||
"final_audio_override": final_audio_override,
|
||||
"final_audio_source": final_audio_source,
|
||||
"interrupt_check": interrupt_check,
|
||||
}
|
||||
return assemble_with_hardening(
|
||||
_assemble_decoded_chunks_v35_core,
|
||||
(),
|
||||
kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BUFFER_BACKEND_DISK",
|
||||
"BUFFER_BACKEND_AUTO",
|
||||
"BUFFER_BACKEND_OPTIONS",
|
||||
"BUFFER_BACKEND_RAM",
|
||||
"CopySpan",
|
||||
"adjust_audio_duration_direct",
|
||||
"assemble_decoded_chunks_v35",
|
||||
]
|
||||
+170
-1
@@ -6,12 +6,26 @@ import torch
|
||||
|
||||
from ..constants import FPS
|
||||
from ..driving_audio import prepare_driving_audio_source
|
||||
from ..hardening import diagnostics_is_full
|
||||
from ..reference_video import (
|
||||
REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
REFERENCE_VIDEO_SIZE_OPTIONS,
|
||||
)
|
||||
from ..v2.decoder import enforce_total_frames
|
||||
from .assembly import H3ContinuumAssembleSeamExperimental
|
||||
from .assembly import (
|
||||
H3ContinuumAssembleSeamExperimental,
|
||||
_chunk_list,
|
||||
_singleton,
|
||||
)
|
||||
from .assembly_v35 import (
|
||||
BUFFER_BACKEND_AUTO,
|
||||
BUFFER_BACKEND_OPTIONS,
|
||||
assemble_decoded_chunks_v35,
|
||||
)
|
||||
from .memory_attribution import (
|
||||
format_assembly_projection,
|
||||
project_assembly_buffers,
|
||||
)
|
||||
from .nodes import CATEGORY as CONTINUUM_CATEGORY, H3ContinuumSamplerProduction
|
||||
|
||||
|
||||
@@ -136,6 +150,7 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction):
|
||||
audio_vae=None,
|
||||
reference_video_1=None,
|
||||
video_reference_size=REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
capture_refine_context=False,
|
||||
**kwargs,
|
||||
):
|
||||
from ..reference_video import prepare_reference_video_source
|
||||
@@ -165,6 +180,7 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction):
|
||||
driving_audio_source=source,
|
||||
driving_audio_vae=audio_vae,
|
||||
reference_video_source=reference_video_source,
|
||||
capture_refine_context=bool(capture_refine_context),
|
||||
**kwargs,
|
||||
)
|
||||
selected_audio = _copy_audio(source.source_audio) if source is not None else None
|
||||
@@ -172,9 +188,46 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction):
|
||||
assembly_plan = dict(outputs[2])
|
||||
assembly_plan[_DRIVING_AUDIO_PLAN_KEY] = _copy_audio(selected_audio)
|
||||
outputs = (*outputs[:2], assembly_plan, *outputs[3:])
|
||||
if bool(capture_refine_context):
|
||||
return (*outputs[:4], selected_audio, outputs[4])
|
||||
return (*outputs, selected_audio)
|
||||
|
||||
|
||||
class H3ContinuumSamplerV35(H3ContinuumSamplerV34):
|
||||
"""V3.5 sampler with a physical-group First Pass refine context output."""
|
||||
|
||||
DEPRECATED = False
|
||||
CATEGORY = CONTINUUM_CATEGORY
|
||||
DESCRIPTION = (
|
||||
"H3 Continuum V3.5 sampler. It preserves the V3.4 outputs and adds one "
|
||||
"physical-group refine context for context-aware Hi-Res Fix or an "
|
||||
"external latent Second Pass."
|
||||
)
|
||||
SEARCH_ALIASES = [
|
||||
"H3 Continuum Sampler V3.5",
|
||||
"MiniMax H3 context-aware Hi-Res Fix",
|
||||
]
|
||||
RETURN_TYPES = (
|
||||
*H3ContinuumSamplerV34.RETURN_TYPES,
|
||||
"H3_CONTINUUM_REFINE_CONTEXT",
|
||||
)
|
||||
RETURN_NAMES = (
|
||||
*H3ContinuumSamplerV34.RETURN_NAMES,
|
||||
"refine_context",
|
||||
)
|
||||
OUTPUT_IS_LIST = (*H3ContinuumSamplerV34.OUTPUT_IS_LIST, False)
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
kwargs.pop("capture_refine_context", None)
|
||||
kwargs.pop("memory_attribution", None)
|
||||
return super().run(
|
||||
*args,
|
||||
capture_refine_context=True,
|
||||
memory_attribution=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class H3ContinuumAssembleSeamV34(H3ContinuumAssembleSeamExperimental):
|
||||
"""Assemble decoded chunks and select original Driving Audio when connected."""
|
||||
|
||||
@@ -239,12 +292,128 @@ class H3ContinuumAssembleSeamV34(H3ContinuumAssembleSeamExperimental):
|
||||
return images, selected, report
|
||||
|
||||
|
||||
def _assembly_argument(args, kwargs, name, index):
|
||||
if name in kwargs:
|
||||
return kwargs[name]
|
||||
if len(args) > index:
|
||||
return args[index]
|
||||
return None
|
||||
|
||||
|
||||
def _insert_projection_report(report, line):
|
||||
lines = str(report).splitlines()
|
||||
for index, value in enumerate(lines):
|
||||
if value.startswith("Memory [assemble preflight]"):
|
||||
lines.insert(index + 1, line)
|
||||
return "\n".join(lines)
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class H3ContinuumAssembleSeamV35(H3ContinuumAssembleSeamV34):
|
||||
"""Independent V3.5 direct-write assembly with selectable IMAGE storage."""
|
||||
|
||||
DEPRECATED = False
|
||||
CATEGORY = CONTINUUM_CATEGORY
|
||||
DESCRIPTION = (
|
||||
"V3.5 decoded-chunk assembler with Auto, RAM, or Disk-backed IMAGE "
|
||||
"storage, direct Exact Duration writes, and small-patch Video Seam."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
schema = super().INPUT_TYPES()
|
||||
required = dict(schema["required"])
|
||||
diagnostics = required.pop("diagnostics")
|
||||
required["buffer_backend"] = (
|
||||
BUFFER_BACKEND_OPTIONS,
|
||||
{
|
||||
"default": BUFFER_BACKEND_AUTO,
|
||||
"display_name": "Buffer Backend",
|
||||
"tooltip": (
|
||||
"Auto keeps outputs up to 4 GiB in RAM only when physical "
|
||||
"memory has a safety reserve; otherwise it maps the final "
|
||||
"IMAGE to disk. Manual RAM and Disk-backed remain available."
|
||||
),
|
||||
},
|
||||
)
|
||||
required["diagnostics"] = diagnostics
|
||||
schema["required"] = required
|
||||
return schema
|
||||
|
||||
def assemble(
|
||||
self,
|
||||
images,
|
||||
audio,
|
||||
assembly_plan,
|
||||
exact_total_duration,
|
||||
audio_seam,
|
||||
video_seam,
|
||||
buffer_backend,
|
||||
diagnostics,
|
||||
driving_audio=None,
|
||||
):
|
||||
image_chunks = _chunk_list(images)
|
||||
audio_chunks = _chunk_list(audio)
|
||||
plan = _singleton(assembly_plan, "assembly_plan")
|
||||
exact = bool(_singleton(exact_total_duration, "exact_total_duration"))
|
||||
audio_seam_mode = str(_singleton(audio_seam, "audio_seam"))
|
||||
video_seam_mode = str(_singleton(video_seam, "video_seam"))
|
||||
backend = str(_singleton(buffer_backend, "buffer_backend"))
|
||||
diagnostics_mode = str(_singleton(diagnostics, "diagnostics"))
|
||||
preserved_audio = _copy_audio(plan.get(_DRIVING_AUDIO_PLAN_KEY))
|
||||
selected_audio = preserved_audio or _copy_audio(driving_audio)
|
||||
selected_audio_source = None
|
||||
if selected_audio is not None:
|
||||
selected_audio_source = (
|
||||
"assembly plan" if preserved_audio is not None else "direct input"
|
||||
)
|
||||
projection_line = None
|
||||
if diagnostics_is_full(diagnostics_mode):
|
||||
try:
|
||||
projection = project_assembly_buffers(
|
||||
images=image_chunks,
|
||||
audio=audio_chunks,
|
||||
assembly_plan=plan,
|
||||
exact_total_duration=exact,
|
||||
)
|
||||
projection_line = format_assembly_projection(projection) + (
|
||||
"; V3.5 Exact Duration IMAGE write=direct; "
|
||||
"additional final IMAGE copy=no"
|
||||
)
|
||||
except Exception as exc:
|
||||
projection_line = (
|
||||
"Projected assembly buffers [V3.5]: unavailable "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
result_images, result_audio, report = assemble_decoded_chunks_v35(
|
||||
images=image_chunks,
|
||||
audio=audio_chunks,
|
||||
assembly_plan=plan,
|
||||
exact_total_duration=exact,
|
||||
audio_seam=audio_seam_mode,
|
||||
video_seam=video_seam_mode,
|
||||
diagnostics=diagnostics_mode,
|
||||
buffer_backend=backend,
|
||||
final_audio_override=selected_audio,
|
||||
final_audio_source=selected_audio_source,
|
||||
)
|
||||
if projection_line is not None:
|
||||
report = _insert_projection_report(report, projection_line)
|
||||
return result_images, result_audio, report
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumSamplerV34": H3ContinuumSamplerV34,
|
||||
"H3ContinuumSamplerV35": H3ContinuumSamplerV35,
|
||||
"H3ContinuumAssembleSeamV34": H3ContinuumAssembleSeamV34,
|
||||
"H3ContinuumAssembleSeamV35": H3ContinuumAssembleSeamV35,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumSamplerV34": "H3 Continuum Sampler V3.4",
|
||||
"H3ContinuumSamplerV35": "H3 Continuum Sampler V3.5",
|
||||
"H3ContinuumAssembleSeamV34": "H3 Continuum Assemble + Seam V3.4",
|
||||
"H3ContinuumAssembleSeamV35": "H3 Continuum Assemble + Seam V3.5",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
"""V3.5 adapter for the accepted P2-0 file-backed Tensor backend.
|
||||
|
||||
The storage and lifetime implementation remains in the P2-0 module so the
|
||||
Windows-tested locking, sidecar, cancellation, and alias-lifetime contracts
|
||||
have one implementation. This module only exposes the small Production API
|
||||
needed by the V3.5 assembler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
import torch
|
||||
|
||||
from ..tools.p20_file_backed_tensor_poc import (
|
||||
FileBackedAllocation,
|
||||
StorageLifetimeManager,
|
||||
cleanup_stale_files,
|
||||
get_process_lifetime_manager,
|
||||
)
|
||||
|
||||
|
||||
BUFFER_BACKEND_AUTO = "Auto"
|
||||
BUFFER_BACKEND_RAM = "RAM"
|
||||
BUFFER_BACKEND_DISK = "Disk-backed"
|
||||
BUFFER_BACKEND_OPTIONS = (
|
||||
BUFFER_BACKEND_AUTO,
|
||||
BUFFER_BACKEND_RAM,
|
||||
BUFFER_BACKEND_DISK,
|
||||
)
|
||||
AUTO_RAM_IMAGE_LIMIT_BYTES = 4 * 1024**3
|
||||
AUTO_RAM_RESERVE_MIN_BYTES = 4 * 1024**3
|
||||
AUTO_RAM_RESERVE_FRACTION = 0.10
|
||||
AUTO_DISK_RESERVE_BYTES = 2 * 1024**3
|
||||
_BACKING_DIRECTORY_NAME = "h3-continuum-v35-file-backed"
|
||||
_CLEANED_ROOTS: set[str] = set()
|
||||
_CLEANED_ROOTS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def validate_buffer_backend(value: str) -> str:
|
||||
"""Validate the explicit or automatic V3.5 IMAGE backend request."""
|
||||
|
||||
if value not in BUFFER_BACKEND_OPTIONS:
|
||||
raise ValueError(f"unknown Buffer Backend: {value!r}")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutoBufferDecision:
|
||||
selected_backend: str
|
||||
final_image_bytes: int
|
||||
backing_root: Path
|
||||
available_memory_bytes: int | None
|
||||
total_memory_bytes: int | None
|
||||
memory_reserve_bytes: int | None
|
||||
ram_image_limit_bytes: int
|
||||
disk_free_bytes: int | None
|
||||
disk_reserve_bytes: int
|
||||
reason: str
|
||||
|
||||
|
||||
def _system_memory_bytes() -> tuple[int | None, int | None]:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
counters = psutil.virtual_memory()
|
||||
return int(counters.available), int(counters.total)
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
|
||||
def _existing_disk_anchor(path: Path) -> Path:
|
||||
candidate = path
|
||||
while not candidate.exists() and candidate != candidate.parent:
|
||||
candidate = candidate.parent
|
||||
if not candidate.exists():
|
||||
raise OSError(f"no existing filesystem anchor for backing root: {path}")
|
||||
return candidate
|
||||
|
||||
|
||||
def _disk_free_bytes(path: Path) -> int:
|
||||
return int(shutil.disk_usage(_existing_disk_anchor(path)).free)
|
||||
|
||||
|
||||
def select_auto_buffer_backend(
|
||||
final_image_bytes: int,
|
||||
*,
|
||||
backing_root: str | os.PathLike[str] | None = None,
|
||||
) -> AutoBufferDecision:
|
||||
"""Choose RAM only when both output size and physical headroom are safe.
|
||||
|
||||
Auto never converts a Disk-backed decision into RAM after a disk-space or
|
||||
allocation failure. Users can still explicitly select either manual path.
|
||||
"""
|
||||
|
||||
final_bytes = int(final_image_bytes)
|
||||
if final_bytes <= 0:
|
||||
raise ValueError("Auto Buffer Backend requires a positive final IMAGE size")
|
||||
root = resolve_backing_root(backing_root)
|
||||
available, total = _system_memory_bytes()
|
||||
reserve = None
|
||||
if available is not None and total is not None and total > 0:
|
||||
reserve = max(
|
||||
AUTO_RAM_RESERVE_MIN_BYTES,
|
||||
int(total * AUTO_RAM_RESERVE_FRACTION),
|
||||
)
|
||||
|
||||
if (
|
||||
reserve is not None
|
||||
and final_bytes <= AUTO_RAM_IMAGE_LIMIT_BYTES
|
||||
and available >= final_bytes + reserve
|
||||
):
|
||||
return AutoBufferDecision(
|
||||
selected_backend=BUFFER_BACKEND_RAM,
|
||||
final_image_bytes=final_bytes,
|
||||
backing_root=root,
|
||||
available_memory_bytes=available,
|
||||
total_memory_bytes=total,
|
||||
memory_reserve_bytes=reserve,
|
||||
ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES,
|
||||
disk_free_bytes=None,
|
||||
disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES,
|
||||
reason="final IMAGE fits the Auto RAM size limit and safety reserve",
|
||||
)
|
||||
|
||||
if reserve is None:
|
||||
reason = "physical-memory counters are unavailable"
|
||||
elif final_bytes > AUTO_RAM_IMAGE_LIMIT_BYTES:
|
||||
reason = "final IMAGE exceeds the Auto RAM size limit"
|
||||
else:
|
||||
reason = "available physical memory is below final IMAGE plus reserve"
|
||||
|
||||
try:
|
||||
disk_free = _disk_free_bytes(root)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"Auto Buffer Backend selected Disk-backed but disk capacity could "
|
||||
f"not be measured for {root}: {type(exc).__name__}: {exc}. "
|
||||
"Select RAM manually only if sufficient memory is known to be available."
|
||||
) from exc
|
||||
required_disk = final_bytes + AUTO_DISK_RESERVE_BYTES
|
||||
if disk_free < required_disk:
|
||||
raise RuntimeError(
|
||||
"Auto Buffer Backend selected Disk-backed but the backing volume "
|
||||
f"has {disk_free} free bytes; {required_disk} bytes are required "
|
||||
f"for the final IMAGE plus the {AUTO_DISK_RESERVE_BYTES}-byte safety "
|
||||
"reserve. Free disk space or select RAM manually only if sufficient "
|
||||
"memory is known to be available."
|
||||
)
|
||||
return AutoBufferDecision(
|
||||
selected_backend=BUFFER_BACKEND_DISK,
|
||||
final_image_bytes=final_bytes,
|
||||
backing_root=root,
|
||||
available_memory_bytes=available,
|
||||
total_memory_bytes=total,
|
||||
memory_reserve_bytes=reserve,
|
||||
ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES,
|
||||
disk_free_bytes=disk_free,
|
||||
disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def format_auto_buffer_decision(decision: AutoBufferDecision) -> str:
|
||||
def value(number: int | None) -> str:
|
||||
return "n/a" if number is None else str(int(number))
|
||||
|
||||
return (
|
||||
"Auto Buffer Decision: "
|
||||
f"selected={decision.selected_backend}, "
|
||||
f"final_IMAGE={decision.final_image_bytes} bytes, "
|
||||
f"system_available={value(decision.available_memory_bytes)} bytes, "
|
||||
f"system_total={value(decision.total_memory_bytes)} bytes, "
|
||||
f"RAM_reserve={value(decision.memory_reserve_bytes)} bytes, "
|
||||
f"RAM_IMAGE_limit={decision.ram_image_limit_bytes} bytes, "
|
||||
f"disk_free={value(decision.disk_free_bytes)} bytes, "
|
||||
f"disk_reserve={decision.disk_reserve_bytes} bytes; "
|
||||
f"reason={decision.reason}."
|
||||
)
|
||||
|
||||
|
||||
def resolve_backing_root(
|
||||
backing_root: str | os.PathLike[str] | None = None,
|
||||
) -> Path:
|
||||
"""Resolve the dedicated backing root without importing ComfyUI eagerly."""
|
||||
|
||||
if backing_root is not None:
|
||||
return Path(backing_root).expanduser().resolve()
|
||||
|
||||
try:
|
||||
import folder_paths
|
||||
except ImportError:
|
||||
base = Path(tempfile.gettempdir())
|
||||
else:
|
||||
value = folder_paths.get_temp_directory()
|
||||
if value is None or not str(value).strip():
|
||||
raise RuntimeError("ComfyUI returned an empty temporary directory")
|
||||
base = Path(value)
|
||||
return (base.expanduser().resolve() / _BACKING_DIRECTORY_NAME).resolve()
|
||||
|
||||
|
||||
def get_file_backed_image_manager(
|
||||
backing_root: str | os.PathLike[str] | None = None,
|
||||
) -> StorageLifetimeManager:
|
||||
"""Return the accepted process-global manager for one resolved root."""
|
||||
|
||||
root = resolve_backing_root(backing_root)
|
||||
key = os.path.normcase(str(root))
|
||||
with _CLEANED_ROOTS_LOCK:
|
||||
if key not in _CLEANED_ROOTS:
|
||||
# The P2-0 root lock prevents cleanup from observing an allocation
|
||||
# between its data-file and sidecar publication. A live owner is
|
||||
# always retained; only prior abnormal-exit artifacts are removed.
|
||||
cleanup_stale_files(root, grace_seconds=0.0)
|
||||
_CLEANED_ROOTS.add(key)
|
||||
return get_process_lifetime_manager(root)
|
||||
|
||||
|
||||
def allocate_file_backed_image(
|
||||
shape: tuple[int, ...],
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
backing_root: str | os.PathLike[str] | None = None,
|
||||
) -> FileBackedAllocation:
|
||||
"""Create a provisional Disk-backed IMAGE allocation.
|
||||
|
||||
The caller must use the returned context manager and call ``publish()``
|
||||
only after every output element has been written. Backend failures are
|
||||
propagated; this manual Disk-backed path never falls back to RAM.
|
||||
"""
|
||||
|
||||
manager = get_file_backed_image_manager(backing_root)
|
||||
return manager.allocate(shape, dtype=dtype)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AUTO_DISK_RESERVE_BYTES",
|
||||
"AUTO_RAM_IMAGE_LIMIT_BYTES",
|
||||
"AUTO_RAM_RESERVE_FRACTION",
|
||||
"AUTO_RAM_RESERVE_MIN_BYTES",
|
||||
"AutoBufferDecision",
|
||||
"BUFFER_BACKEND_AUTO",
|
||||
"BUFFER_BACKEND_DISK",
|
||||
"BUFFER_BACKEND_OPTIONS",
|
||||
"BUFFER_BACKEND_RAM",
|
||||
"allocate_file_backed_image",
|
||||
"format_auto_buffer_decision",
|
||||
"get_file_backed_image_manager",
|
||||
"resolve_backing_root",
|
||||
"select_auto_buffer_backend",
|
||||
"validate_buffer_backend",
|
||||
]
|
||||
@@ -0,0 +1,463 @@
|
||||
"""V3.5 Continuum-aware Hi-res fix convenience node."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
from .pixel_vae_resize import (
|
||||
resize_video_latents_via_vae,
|
||||
resolve_pixel_latent_scale,
|
||||
)
|
||||
from .second_pass import SecondPassContractError, run_second_pass_groups
|
||||
|
||||
|
||||
_MISSING = object()
|
||||
HIRES_UPSCALE_METHODS = (
|
||||
"lanczos",
|
||||
"nearest-exact",
|
||||
"bilinear",
|
||||
"area",
|
||||
"bicubic",
|
||||
"bislerp",
|
||||
)
|
||||
_CONDITIONING_UPSCALE_METHOD = "bilinear"
|
||||
|
||||
|
||||
def _single_list_input(name: str, value: Any) -> Any:
|
||||
if not isinstance(value, list) or len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
return value[0]
|
||||
|
||||
|
||||
def _enabled_value(value: Any) -> bool:
|
||||
if isinstance(value, list):
|
||||
value = _single_list_input("enabled", value)
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _lazy_missing(value: Any) -> bool:
|
||||
return value is None or (
|
||||
isinstance(value, (list, tuple))
|
||||
and len(value) == 1
|
||||
and value[0] is None
|
||||
)
|
||||
|
||||
|
||||
def h3_core_native_canvas(
|
||||
width: int,
|
||||
height: int,
|
||||
*,
|
||||
adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Return the native canvas selected by ComfyUI Core for this aspect ratio."""
|
||||
|
||||
if width < 1 or height < 1:
|
||||
raise SecondPassContractError("source pixel width and height must be positive")
|
||||
if adapt_canvas_fn is None:
|
||||
from comfy_extras.nodes_minimax_h3 import adapt_canvas
|
||||
|
||||
adapt_canvas_fn = adapt_canvas
|
||||
target = adapt_canvas_fn(int(width), int(height))
|
||||
if not isinstance(target, Sequence) or len(target) != 2:
|
||||
raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry")
|
||||
target_width, target_height = (int(value) for value in target)
|
||||
if target_width < 1 or target_height < 1:
|
||||
raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry")
|
||||
return target_width, target_height
|
||||
|
||||
|
||||
def resolve_h3_native_latent_target(
|
||||
video_latents: Sequence[Mapping[str, Any]],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
*,
|
||||
adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None,
|
||||
) -> dict[str, int | bool]:
|
||||
"""Resolve Core's native canvas without shrinking an existing video latent."""
|
||||
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, Mapping):
|
||||
raise SecondPassContractError("assembly_plan has no second_pass_contract")
|
||||
groups = contract.get("physical_groups")
|
||||
if not isinstance(groups, list) or not groups:
|
||||
raise SecondPassContractError(
|
||||
"second_pass_contract.physical_groups must be non-empty"
|
||||
)
|
||||
if not isinstance(video_latents, Sequence) or not video_latents:
|
||||
raise SecondPassContractError("video_latents must be a non-empty sequence")
|
||||
if len(video_latents) != len(groups):
|
||||
raise SecondPassContractError("video latent group count changed")
|
||||
|
||||
first_group = groups[0]
|
||||
if not isinstance(first_group, Mapping):
|
||||
raise SecondPassContractError("physical group metadata must be a mapping")
|
||||
source_width = int(first_group.get("source_width", -1))
|
||||
source_height = int(first_group.get("source_height", -1))
|
||||
source_latent_width = int(first_group.get("source_latent_w", -1))
|
||||
source_latent_height = int(first_group.get("source_latent_h", -1))
|
||||
if min(source_width, source_height, source_latent_width, source_latent_height) < 1:
|
||||
raise SecondPassContractError("source pixel/latent geometry is invalid")
|
||||
if source_width % source_latent_width or source_height % source_latent_height:
|
||||
raise SecondPassContractError("source pixel/latent geometry is not integral")
|
||||
pixel_scale_width = source_width // source_latent_width
|
||||
pixel_scale_height = source_height // source_latent_height
|
||||
|
||||
current_geometry: tuple[int, int] | None = None
|
||||
for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)):
|
||||
if not isinstance(group, Mapping):
|
||||
raise SecondPassContractError("physical group metadata must be a mapping")
|
||||
if int(group.get("source_width", -1)) != source_width or int(
|
||||
group.get("source_height", -1)
|
||||
) != source_height:
|
||||
raise SecondPassContractError("source geometry differs between physical groups")
|
||||
if int(group.get("source_latent_w", -1)) != source_latent_width or int(
|
||||
group.get("source_latent_h", -1)
|
||||
) != source_latent_height:
|
||||
raise SecondPassContractError(
|
||||
"source latent geometry differs between physical groups"
|
||||
)
|
||||
if not isinstance(latent, Mapping) or "samples" not in latent:
|
||||
raise SecondPassContractError(
|
||||
f"video_latents[{position}] must be a LATENT mapping with samples"
|
||||
)
|
||||
samples = latent["samples"]
|
||||
if not hasattr(samples, "shape") or len(samples.shape) != 5:
|
||||
raise SecondPassContractError("video latent must have shape [B,C,T,H,W]")
|
||||
geometry = (int(samples.shape[-2]), int(samples.shape[-1]))
|
||||
if current_geometry is None:
|
||||
current_geometry = geometry
|
||||
elif geometry != current_geometry:
|
||||
raise SecondPassContractError(
|
||||
"all physical groups must use the same current H/W"
|
||||
)
|
||||
|
||||
assert current_geometry is not None
|
||||
native_width, native_height = h3_core_native_canvas(
|
||||
source_width,
|
||||
source_height,
|
||||
adapt_canvas_fn=adapt_canvas_fn,
|
||||
)
|
||||
if native_width % pixel_scale_width or native_height % pixel_scale_height:
|
||||
raise SecondPassContractError(
|
||||
"ComfyUI Core native canvas is not aligned to the H3 latent scale"
|
||||
)
|
||||
native_latent_width = native_width // pixel_scale_width
|
||||
native_latent_height = native_height // pixel_scale_height
|
||||
current_height, current_width = current_geometry
|
||||
preserve_current = (
|
||||
native_latent_width < current_width
|
||||
or native_latent_height < current_height
|
||||
)
|
||||
if preserve_current:
|
||||
target_latent_width = current_width
|
||||
target_latent_height = current_height
|
||||
else:
|
||||
target_latent_width = native_latent_width
|
||||
target_latent_height = native_latent_height
|
||||
|
||||
return {
|
||||
"source_width": source_width,
|
||||
"source_height": source_height,
|
||||
"native_width": native_width,
|
||||
"native_height": native_height,
|
||||
"target_width": target_latent_width * pixel_scale_width,
|
||||
"target_height": target_latent_height * pixel_scale_height,
|
||||
"target_latent_width": target_latent_width,
|
||||
"target_latent_height": target_latent_height,
|
||||
"preserved_current": preserve_current,
|
||||
}
|
||||
|
||||
|
||||
def _prepend_resize_status(
|
||||
second_pass_status: str,
|
||||
*,
|
||||
backend: str,
|
||||
mode: str,
|
||||
upscale_method: str,
|
||||
scale_by: float,
|
||||
source_width: int,
|
||||
source_height: int,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
native_width: int,
|
||||
native_height: int,
|
||||
) -> str:
|
||||
lines = [
|
||||
"H3 Continuum Hi-Res Fix V3.5",
|
||||
"Pixel/VAE resize: "
|
||||
f"backend={backend}, mode={mode}, method={upscale_method}, "
|
||||
f"scale_by={scale_by:g}, "
|
||||
f"source={source_width}x{source_height}, "
|
||||
f"target={target_width}x{target_height}, "
|
||||
f"core_native={native_width}x{native_height}.",
|
||||
]
|
||||
if target_width > native_width or target_height > native_height:
|
||||
lines.append(
|
||||
"INFO: target exceeds the ComfyUI Core H3 base canvas; the Pixel/VAE "
|
||||
"roundtrip is preserved and resource use may increase."
|
||||
)
|
||||
lines.append(second_pass_status)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class H3ContinuumHiResFixV35:
|
||||
"""Pixel/VAE resize and low-sigma refine complete physical video groups."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"Continuum-aware Hi-res fix. When enabled, each first-pass physical video "
|
||||
"LATENT is decoded with the connected H3 Video VAE, resized in pixel space, "
|
||||
"re-encoded with the same VAE, and refined once with the workflow SIGMAS. "
|
||||
"scale_by 0 uses ComfyUI Core's H3 native canvas; positive values use the "
|
||||
"manual multiplier. Connect the optional Continuum refine context to preserve "
|
||||
"First/Last/Reference conditioning at the target geometry. "
|
||||
"When disabled, first-pass video/audio LATENT objects and the original assembly "
|
||||
"plan are returned unchanged."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
lazy = {"lazy": True}
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL", lazy),
|
||||
"clip": ("CLIP", lazy),
|
||||
"sampler": ("SAMPLER", lazy),
|
||||
"sigmas": ("SIGMAS", lazy),
|
||||
"video_latents": ("LATENT",),
|
||||
"audio_latents": ("LATENT",),
|
||||
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
||||
"enabled": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": (
|
||||
"Enable Resize + Second Pass. Off returns the first-pass "
|
||||
"video/audio LATENT objects and assembly plan unchanged."
|
||||
),
|
||||
},
|
||||
),
|
||||
"upscale_method": (
|
||||
HIRES_UPSCALE_METHODS,
|
||||
{"default": "lanczos"},
|
||||
),
|
||||
"scale_by": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 2.0,
|
||||
"min": 0.0,
|
||||
"max": 4.0,
|
||||
"step": 0.01,
|
||||
"tooltip": (
|
||||
"2.0 = validated 2x Pixel/VAE Hi-res Fix. 0 uses the "
|
||||
"ComfyUI Core H3 native canvas automatically. Manual "
|
||||
"values follow the existing Second Pass contract and must "
|
||||
"be 1.0 or greater."
|
||||
),
|
||||
},
|
||||
),
|
||||
"refine_seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": True,
|
||||
"tooltip": "Base seed independently derived once per physical group.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"refine_context": (
|
||||
"H3_CONTINUUM_REFINE_CONTEXT",
|
||||
{"lazy": True},
|
||||
),
|
||||
"video_vae": (
|
||||
"VAE",
|
||||
{
|
||||
"lazy": True,
|
||||
"tooltip": (
|
||||
"Required while enabled for the safe Pixel/VAE roundtrip; "
|
||||
"not evaluated while disabled."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
|
||||
RETURN_NAMES = (
|
||||
"video_latents",
|
||||
"audio_latents",
|
||||
"updated_assembly_plan",
|
||||
"status",
|
||||
)
|
||||
OUTPUT_IS_LIST = (True, True, False, False)
|
||||
FUNCTION = "apply"
|
||||
CATEGORY = "MiniMax H3/Continuum"
|
||||
|
||||
def check_lazy_status(
|
||||
self,
|
||||
enabled,
|
||||
model=None,
|
||||
clip=None,
|
||||
sampler=None,
|
||||
sigmas=None,
|
||||
refine_context=_MISSING,
|
||||
video_vae=_MISSING,
|
||||
**_kwargs,
|
||||
):
|
||||
if not _enabled_value(enabled):
|
||||
return []
|
||||
required = [
|
||||
name
|
||||
for name, value in (
|
||||
("model", model),
|
||||
("clip", clip),
|
||||
("sampler", sampler),
|
||||
("sigmas", sigmas),
|
||||
)
|
||||
if _lazy_missing(value)
|
||||
]
|
||||
required.extend(
|
||||
name
|
||||
for name, value in (
|
||||
("refine_context", refine_context),
|
||||
("video_vae", video_vae),
|
||||
)
|
||||
if value is not _MISSING and _lazy_missing(value)
|
||||
)
|
||||
return required
|
||||
|
||||
def apply(
|
||||
self,
|
||||
model,
|
||||
clip,
|
||||
sampler,
|
||||
sigmas,
|
||||
video_latents,
|
||||
audio_latents,
|
||||
assembly_plan,
|
||||
enabled,
|
||||
upscale_method,
|
||||
scale_by,
|
||||
refine_seed,
|
||||
refine_context=None,
|
||||
video_vae=None,
|
||||
):
|
||||
plan = _single_list_input("assembly_plan", assembly_plan)
|
||||
if not _enabled_value(enabled):
|
||||
return (
|
||||
video_latents,
|
||||
audio_latents,
|
||||
plan,
|
||||
"H3 Continuum Hi-Res Fix V3.5: disabled; first-pass video/audio "
|
||||
"LATENT objects and assembly_plan returned unchanged.",
|
||||
)
|
||||
|
||||
method = str(_single_list_input("upscale_method", upscale_method))
|
||||
scale = float(_single_list_input("scale_by", scale_by))
|
||||
if 0.0 < scale < 1.0:
|
||||
raise SecondPassContractError(
|
||||
"scale_by must be 0 for H3 native Auto or at least 1.0; the "
|
||||
"existing Second Pass contract preserves or enlarges spatial size"
|
||||
)
|
||||
vae = (
|
||||
None
|
||||
if video_vae is None
|
||||
else _single_list_input("video_vae", video_vae)
|
||||
)
|
||||
if vae is None:
|
||||
raise SecondPassContractError(
|
||||
"Hi-Res Fix enabled requires video_vae for the safe Pixel/VAE "
|
||||
"roundtrip; legacy direct latent interpolation is disabled"
|
||||
)
|
||||
source_width = int(plan["width"])
|
||||
source_height = int(plan["height"])
|
||||
native_width, native_height = h3_core_native_canvas(
|
||||
source_width,
|
||||
source_height,
|
||||
)
|
||||
first_samples = video_latents[0]["samples"]
|
||||
current_geometry = (
|
||||
int(first_samples.shape[-2]),
|
||||
int(first_samples.shape[-1]),
|
||||
)
|
||||
if scale == 0.0:
|
||||
native_target = resolve_h3_native_latent_target(video_latents, plan)
|
||||
target_latent_width = int(native_target["target_latent_width"])
|
||||
target_latent_height = int(native_target["target_latent_height"])
|
||||
target_width = int(native_target["target_width"])
|
||||
target_height = int(native_target["target_height"])
|
||||
resize_mode = "h3_native_canvas_auto"
|
||||
else:
|
||||
pixel_scale_width, pixel_scale_height = resolve_pixel_latent_scale(plan)
|
||||
target_latent_width = round(current_geometry[1] * scale)
|
||||
target_latent_height = round(current_geometry[0] * scale)
|
||||
if target_latent_width < 1 or target_latent_height < 1:
|
||||
raise SecondPassContractError(
|
||||
"scale_by produced a non-positive target geometry"
|
||||
)
|
||||
target_width = target_latent_width * pixel_scale_width
|
||||
target_height = target_latent_height * pixel_scale_height
|
||||
resize_mode = "manual_scale_by"
|
||||
|
||||
if current_geometry == (target_latent_height, target_latent_width):
|
||||
resized_video_latents = list(video_latents)
|
||||
resize_backend = "direct_second_pass_no_resize"
|
||||
else:
|
||||
resized_video_latents = resize_video_latents_via_vae(
|
||||
video_latents,
|
||||
plan,
|
||||
video_vae=vae,
|
||||
target_width=target_width,
|
||||
target_height=target_height,
|
||||
upscale_method=method,
|
||||
)
|
||||
resize_backend = "sequential_pixel_vae_roundtrip"
|
||||
|
||||
refined_video, output_audio, updated_plan, status = run_second_pass_groups(
|
||||
model=_single_list_input("model", model),
|
||||
clip=_single_list_input("clip", clip),
|
||||
sampler=_single_list_input("sampler", sampler),
|
||||
sigmas=_single_list_input("sigmas", sigmas),
|
||||
video_latents=resized_video_latents,
|
||||
audio_latents=audio_latents,
|
||||
assembly_plan=plan,
|
||||
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
|
||||
refine_context=(
|
||||
None
|
||||
if refine_context is None
|
||||
else _single_list_input("refine_context", refine_context)
|
||||
),
|
||||
video_vae=vae,
|
||||
conditioning_upscale_method=_CONDITIONING_UPSCALE_METHOD,
|
||||
)
|
||||
return (
|
||||
refined_video,
|
||||
output_audio,
|
||||
updated_plan,
|
||||
_prepend_resize_status(
|
||||
status,
|
||||
backend=resize_backend,
|
||||
mode=resize_mode,
|
||||
upscale_method=method,
|
||||
scale_by=scale,
|
||||
source_width=source_width,
|
||||
source_height=source_height,
|
||||
target_width=int(updated_plan["width"]),
|
||||
target_height=int(updated_plan["height"]),
|
||||
native_width=native_width,
|
||||
native_height=native_height,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumHiResFixV35": H3ContinuumHiResFixV35,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumHiResFixV35": "H3 Continuum Hi-Res Fix V3.5",
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
"""V3.5 Core-compatible spatial resize for physical video latents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
|
||||
UPSCALE_METHODS = (
|
||||
"nearest-exact",
|
||||
"bilinear",
|
||||
"area",
|
||||
"bicubic",
|
||||
"bislerp",
|
||||
)
|
||||
|
||||
|
||||
def _single_list_input(name: str, value: Any) -> Any:
|
||||
if not isinstance(value, list) or len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
return value[0]
|
||||
|
||||
|
||||
def resize_video_latents(
|
||||
video_latents: list[dict[str, Any]],
|
||||
*,
|
||||
upscale_method: str,
|
||||
scale_by: float,
|
||||
upscale_fn: Callable[..., Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Resize only spatial H/W through the ComfyUI Core latent implementation."""
|
||||
|
||||
if upscale_fn is None:
|
||||
from comfy.utils import common_upscale
|
||||
|
||||
upscale_fn = common_upscale
|
||||
|
||||
resized: list[dict[str, Any]] = []
|
||||
for latent in video_latents:
|
||||
samples = latent["samples"]
|
||||
width = round(samples.shape[-1] * scale_by)
|
||||
height = round(samples.shape[-2] * scale_by)
|
||||
output = latent.copy()
|
||||
output["samples"] = upscale_fn(
|
||||
samples,
|
||||
width,
|
||||
height,
|
||||
upscale_method,
|
||||
"disabled",
|
||||
)
|
||||
resized.append(output)
|
||||
return resized
|
||||
|
||||
|
||||
def resize_video_latents_to_size(
|
||||
video_latents: list[dict[str, Any]],
|
||||
*,
|
||||
upscale_method: str,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
upscale_fn: Callable[..., Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Resize every physical video group to one exact latent H/W."""
|
||||
|
||||
if target_width < 1 or target_height < 1:
|
||||
raise ValueError("target latent width and height must be positive")
|
||||
if upscale_fn is None:
|
||||
from comfy.utils import common_upscale
|
||||
|
||||
upscale_fn = common_upscale
|
||||
|
||||
resized: list[dict[str, Any]] = []
|
||||
for latent in video_latents:
|
||||
samples = latent["samples"]
|
||||
output = latent.copy()
|
||||
output["samples"] = upscale_fn(
|
||||
samples,
|
||||
int(target_width),
|
||||
int(target_height),
|
||||
upscale_method,
|
||||
"disabled",
|
||||
)
|
||||
resized.append(output)
|
||||
return resized
|
||||
|
||||
|
||||
class H3ContinuumLatentResizeV35:
|
||||
"""List-aware Core spatial resize for Continuum physical video groups."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"V3.5 Hi-res fix resize. Spatially resizes every Continuum "
|
||||
"physical video LATENT with the selected ComfyUI Core interpolation method. "
|
||||
"B/C/T, physical group order, and LATENT metadata are preserved."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"video_latents": ("LATENT",),
|
||||
"upscale_method": (UPSCALE_METHODS,),
|
||||
"scale_by": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 2.0,
|
||||
"min": 0.01,
|
||||
"max": 8.0,
|
||||
"step": 0.01,
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("LATENT",)
|
||||
RETURN_NAMES = ("resized_video_latents",)
|
||||
OUTPUT_IS_LIST = (True,)
|
||||
FUNCTION = "resize"
|
||||
CATEGORY = "MiniMax H3/Continuum/Utilities"
|
||||
|
||||
def resize(self, video_latents, upscale_method, scale_by):
|
||||
return (
|
||||
resize_video_latents(
|
||||
video_latents,
|
||||
upscale_method=str(
|
||||
_single_list_input("upscale_method", upscale_method)
|
||||
),
|
||||
scale_by=float(_single_list_input("scale_by", scale_by)),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumLatentResizeV35": H3ContinuumLatentResizeV35,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumLatentResizeV35": "H3 Continuum Latent Resize V3.5",
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
"""V3.5-only, non-synchronizing memory attribution helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
from typing import Any, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
from ..constants import FPS
|
||||
from ..hardening import tensor_storage_stats
|
||||
from ..state import extract_av_streams
|
||||
from .plan import validate_assembly_plan
|
||||
|
||||
|
||||
_MIB = 1024**2
|
||||
|
||||
|
||||
def _mib(value: int | None) -> str:
|
||||
if value is None:
|
||||
return "n/a"
|
||||
return f"{int(value) / _MIB:.1f} MiB"
|
||||
|
||||
|
||||
def _bytes_and_mib(value: int) -> str:
|
||||
return f"{int(value)} bytes ({int(value) / _MIB:.1f} MiB)"
|
||||
|
||||
|
||||
def _process_counters() -> tuple[int | None, int | None]:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
return (
|
||||
int(psutil.Process().memory_info().rss),
|
||||
int(psutil.virtual_memory().available),
|
||||
)
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
|
||||
def _cuda_counters() -> tuple[int | None, int | None, int | None]:
|
||||
"""Read counters only; never synchronize, clear caches, or reset peaks."""
|
||||
|
||||
try:
|
||||
if torch.cuda.is_available():
|
||||
return (
|
||||
int(torch.cuda.memory_allocated()),
|
||||
int(torch.cuda.memory_reserved()),
|
||||
int(torch.cuda.max_memory_allocated()),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return None, None, None
|
||||
|
||||
|
||||
def _retained_stream_bytes(
|
||||
entries: Any,
|
||||
stream: str,
|
||||
) -> int:
|
||||
values: list[Any] = []
|
||||
if isinstance(entries, (list, tuple)):
|
||||
for entry in entries:
|
||||
if isinstance(entry, Mapping):
|
||||
values.append(entry.get(stream))
|
||||
stats = tensor_storage_stats(values)
|
||||
return int(stats["cpu_bytes"]) + int(stats["gpu_bytes"])
|
||||
|
||||
|
||||
def _observed_streams(latent: Any) -> tuple[Any, ...]:
|
||||
if latent is None:
|
||||
return ()
|
||||
if isinstance(latent, Mapping) and "samples" in latent:
|
||||
try:
|
||||
return tuple(extract_av_streams(latent))
|
||||
except Exception:
|
||||
return ()
|
||||
return (latent,)
|
||||
|
||||
|
||||
def format_attribution_snapshot(
|
||||
label: str,
|
||||
*,
|
||||
retained_entries: Any = None,
|
||||
observed_latent: Any = None,
|
||||
retained_refine_context: Any = None,
|
||||
) -> str:
|
||||
"""Format one fail-soft snapshot without changing allocator state."""
|
||||
try:
|
||||
rss, available = _process_counters()
|
||||
cuda_allocated, cuda_reserved, cuda_peak = _cuda_counters()
|
||||
retained_video = _retained_stream_bytes(retained_entries, "video")
|
||||
retained_audio = _retained_stream_bytes(retained_entries, "audio")
|
||||
observed = tensor_storage_stats(_observed_streams(observed_latent))
|
||||
refine_context = tensor_storage_stats(retained_refine_context)
|
||||
return (
|
||||
f"Memory [{label}]: RSS={_mib(rss)}, "
|
||||
f"system_available={_mib(available)}, "
|
||||
f"CUDA_allocated={_mib(cuda_allocated)}, "
|
||||
f"CUDA_reserved={_mib(cuda_reserved)}, "
|
||||
f"CUDA_peak_allocated_since_external_reset={_mib(cuda_peak)}, "
|
||||
f"retained_video={_bytes_and_mib(retained_video)}, "
|
||||
f"retained_audio={_bytes_and_mib(retained_audio)}, "
|
||||
"retained_refine_context_CPU="
|
||||
f"{_bytes_and_mib(int(refine_context['cpu_bytes']))}, "
|
||||
"retained_refine_context_GPU="
|
||||
f"{_bytes_and_mib(int(refine_context['gpu_bytes']))}, "
|
||||
f"observed_CPU_storage={_bytes_and_mib(int(observed['cpu_bytes']))}, "
|
||||
f"observed_GPU_storage={_bytes_and_mib(int(observed['gpu_bytes']))}"
|
||||
)
|
||||
except Exception as exc:
|
||||
return (
|
||||
f"Memory [{label}]: unavailable "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
|
||||
class MemoryAttributionCollector:
|
||||
"""Keep V3.5 memory events in chronological order until report assembly."""
|
||||
|
||||
def __init__(self):
|
||||
self.lines: list[str] = []
|
||||
|
||||
def capture_sequence_start(self) -> None:
|
||||
self.lines.append(format_attribution_snapshot("V3.5 sequence start"))
|
||||
|
||||
def capture_group(
|
||||
self,
|
||||
*,
|
||||
physical_group: int,
|
||||
logical_chunks: tuple[int, ...],
|
||||
stage: str,
|
||||
retained_entries: Any,
|
||||
observed_latent: Any = None,
|
||||
retained_refine_context: Any = None,
|
||||
) -> None:
|
||||
logical = ",".join(str(int(value)) for value in logical_chunks)
|
||||
self.lines.append(
|
||||
format_attribution_snapshot(
|
||||
f"V3.5 physical group {int(physical_group)} "
|
||||
f"logical=[{logical}] {stage}",
|
||||
retained_entries=retained_entries,
|
||||
observed_latent=observed_latent,
|
||||
retained_refine_context=retained_refine_context,
|
||||
)
|
||||
)
|
||||
|
||||
def capture_sequence_complete(
|
||||
self,
|
||||
entries: Any,
|
||||
retained_refine_context: Any = None,
|
||||
) -> None:
|
||||
self.lines.append(
|
||||
format_attribution_snapshot(
|
||||
"V3.5 sequence complete",
|
||||
retained_entries=entries,
|
||||
retained_refine_context=retained_refine_context,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def capture_attribution_fail_soft(
|
||||
collector: Any,
|
||||
method: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""A diagnostics failure must never replace a sampling result."""
|
||||
|
||||
try:
|
||||
getattr(collector, method)(**kwargs)
|
||||
except Exception as exc:
|
||||
try:
|
||||
collector.lines.append(
|
||||
f"Memory attribution [{method}]: unavailable "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AssemblyBufferProjection:
|
||||
natural_frames: int
|
||||
target_frames: int
|
||||
natural_image_bytes: int
|
||||
natural_audio_bytes: int
|
||||
final_image_bytes: int
|
||||
final_audio_bytes: int
|
||||
frame_count_adjustment_required: bool
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list[Any]:
|
||||
if isinstance(value, list):
|
||||
if len(value) == 1 and isinstance(value[0], list):
|
||||
return list(value[0])
|
||||
return list(value)
|
||||
if isinstance(value, tuple):
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
def _singleton(value: Any, name: str) -> Any:
|
||||
while isinstance(value, list):
|
||||
if len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
value = value[0]
|
||||
return value
|
||||
|
||||
|
||||
def project_assembly_buffers(
|
||||
*,
|
||||
images: Any,
|
||||
audio: Any,
|
||||
assembly_plan: Any,
|
||||
exact_total_duration: Any,
|
||||
) -> AssemblyBufferProjection:
|
||||
"""Calculate natural and final buffer sizes without allocating a buffer."""
|
||||
|
||||
plan = validate_assembly_plan(_singleton(assembly_plan, "assembly_plan"))
|
||||
image_chunks = _as_list(images)
|
||||
audio_chunks = _as_list(audio)
|
||||
if not image_chunks or not audio_chunks:
|
||||
raise ValueError("decoded image/audio chunks are required for projection")
|
||||
|
||||
first_image = _singleton(image_chunks[0], "decoded image chunk")
|
||||
if not torch.is_tensor(first_image) or first_image.ndim != 4:
|
||||
raise ValueError("decoded image chunk must be [frames,H,W,C]")
|
||||
frame_shape = tuple(int(value) for value in first_image.shape[1:])
|
||||
for index, value in enumerate(image_chunks, start=1):
|
||||
tensor = _singleton(value, f"decoded image chunk {index}")
|
||||
if (
|
||||
not torch.is_tensor(tensor)
|
||||
or tensor.ndim != 4
|
||||
or tuple(int(item) for item in tensor.shape[1:]) != frame_shape
|
||||
or tensor.dtype != first_image.dtype
|
||||
):
|
||||
raise ValueError("decoded image geometry or dtype changed between chunks")
|
||||
|
||||
first_audio = _singleton(audio_chunks[0], "decoded audio chunk")
|
||||
if not isinstance(first_audio, Mapping):
|
||||
raise ValueError("decoded audio chunk must be an AUDIO mapping")
|
||||
waveform = first_audio.get("waveform")
|
||||
sample_rate = first_audio.get("sample_rate")
|
||||
if not torch.is_tensor(waveform) or waveform.ndim < 1 or sample_rate is None:
|
||||
raise ValueError("decoded audio chunk has an invalid waveform or sample rate")
|
||||
sample_rate = int(sample_rate)
|
||||
if sample_rate <= 0:
|
||||
raise ValueError("decoded audio sample rate must be positive")
|
||||
|
||||
decode_units = list(plan.get("decode_groups") or plan["chunks"])
|
||||
natural_frames = sum(int(unit["net_frames"]) for unit in decode_units)
|
||||
exact = bool(_singleton(exact_total_duration, "exact_total_duration"))
|
||||
target_frames = int(plan["target_frames"]) if exact else natural_frames
|
||||
if natural_frames <= 0 or target_frames <= 0:
|
||||
raise ValueError("assembly frame counts must be positive")
|
||||
|
||||
frame_elements = math.prod(frame_shape)
|
||||
audio_streams = math.prod(int(value) for value in waveform.shape[:-1])
|
||||
natural_samples = int(round(natural_frames / FPS * sample_rate))
|
||||
target_samples = int(round(target_frames / FPS * sample_rate))
|
||||
return AssemblyBufferProjection(
|
||||
natural_frames=natural_frames,
|
||||
target_frames=target_frames,
|
||||
natural_image_bytes=(
|
||||
natural_frames * frame_elements * first_image.element_size()
|
||||
),
|
||||
natural_audio_bytes=(
|
||||
natural_samples * audio_streams * waveform.element_size()
|
||||
),
|
||||
final_image_bytes=(
|
||||
target_frames * frame_elements * first_image.element_size()
|
||||
),
|
||||
final_audio_bytes=(
|
||||
target_samples * audio_streams * waveform.element_size()
|
||||
),
|
||||
frame_count_adjustment_required=bool(
|
||||
exact and target_frames != natural_frames
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def format_assembly_projection(projection: AssemblyBufferProjection) -> str:
|
||||
return (
|
||||
"Projected assembly buffers [V3.5]: "
|
||||
f"natural_frames={projection.natural_frames}, "
|
||||
f"projected_natural_IMAGE={_bytes_and_mib(projection.natural_image_bytes)}, "
|
||||
"projected_natural_generated_AUDIO="
|
||||
f"{_bytes_and_mib(projection.natural_audio_bytes)}, "
|
||||
f"target_frames={projection.target_frames}, "
|
||||
f"final_IMAGE={_bytes_and_mib(projection.final_image_bytes)}, "
|
||||
f"final_generated_AUDIO={_bytes_and_mib(projection.final_audio_bytes)}, "
|
||||
"frame_count_adjustment_required="
|
||||
f"{'yes' if projection.frame_count_adjustment_required else 'no'}"
|
||||
)
|
||||
+40
-6
@@ -256,6 +256,8 @@ class H3ContinuumSamplerV3:
|
||||
driving_audio_vae=None,
|
||||
reference_video_source=None,
|
||||
timeline_video_source=None,
|
||||
capture_refine_context=False,
|
||||
memory_attribution=False,
|
||||
):
|
||||
if prompt_overrides is not None and not isinstance(prompt_overrides, dict):
|
||||
prompt_overrides = None
|
||||
@@ -297,7 +299,7 @@ class H3ContinuumSamplerV3:
|
||||
for index, prompt in sparse_overrides.items():
|
||||
clip_prompt_inputs[f"clip_{index}_prompt"] = prompt
|
||||
|
||||
entries, last_state, session, report = H3ContinuumSamplerV2().run(
|
||||
sequence_outputs = H3ContinuumSamplerV2().run(
|
||||
model=model,
|
||||
clip=clip,
|
||||
video_vae=video_vae,
|
||||
@@ -335,8 +337,14 @@ class H3ContinuumSamplerV3:
|
||||
driving_audio_vae=driving_audio_vae,
|
||||
reference_video_source=reference_video_source,
|
||||
timeline_video_source=timeline_video_source,
|
||||
capture_refine_context=bool(capture_refine_context),
|
||||
memory_attribution=bool(memory_attribution),
|
||||
**clip_prompt_inputs,
|
||||
)
|
||||
if bool(capture_refine_context):
|
||||
entries, last_state, session, report, refine_context = sequence_outputs
|
||||
else:
|
||||
entries, last_state, session, report = sequence_outputs
|
||||
|
||||
from ..v2.sequence import _terminal_flf_merge_enabled
|
||||
from .plan import prepare_physical_decode_entries
|
||||
@@ -370,7 +378,10 @@ class H3ContinuumSamplerV3:
|
||||
"session": session,
|
||||
"report": report,
|
||||
}
|
||||
return video_latents, audio_latents, assembly_plan, result
|
||||
outputs = (video_latents, audio_latents, assembly_plan, result)
|
||||
if bool(capture_refine_context):
|
||||
return (*outputs, refine_context)
|
||||
return outputs
|
||||
|
||||
|
||||
class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
@@ -650,6 +661,8 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
driving_audio_vae=None,
|
||||
reference_video_source=None,
|
||||
timeline_video_source=None,
|
||||
capture_refine_context=False,
|
||||
memory_attribution=False,
|
||||
):
|
||||
runtime_started_at = time.perf_counter()
|
||||
from ..reference import prepare_reference_assets
|
||||
@@ -699,6 +712,8 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
driving_audio_vae=driving_audio_vae,
|
||||
reference_video_source=reference_video_source,
|
||||
timeline_video_source=timeline_video_source,
|
||||
capture_refine_context=bool(capture_refine_context),
|
||||
memory_attribution=bool(memory_attribution),
|
||||
advanced={
|
||||
"audio_continuity": bool(audio_continuity),
|
||||
"diagnostics": diagnostics,
|
||||
@@ -713,13 +728,20 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
|
||||
if run_storage == "Off":
|
||||
_validate_regenerate_storage(run_storage, regenerate_from)
|
||||
video_latents, audio_latents, assembly_plan, result = execute()
|
||||
return (
|
||||
execute_outputs = execute()
|
||||
if bool(capture_refine_context):
|
||||
video_latents, audio_latents, assembly_plan, result, refine_context = execute_outputs
|
||||
else:
|
||||
video_latents, audio_latents, assembly_plan, result = execute_outputs
|
||||
outputs = (
|
||||
video_latents,
|
||||
audio_latents,
|
||||
mark_runtime_start(assembly_plan),
|
||||
str(result["report"]),
|
||||
)
|
||||
if bool(capture_refine_context):
|
||||
return (*outputs, refine_context)
|
||||
return outputs
|
||||
if run_storage != "Save + Auto Resume":
|
||||
raise ValueError(f"unknown Run Storage mode: {run_storage!r}")
|
||||
from ..run_storage import (
|
||||
@@ -735,14 +757,26 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
with run_storage_scope(
|
||||
storage_name, prompt=prompt, unique_id=unique_id
|
||||
) as storage:
|
||||
video_latents, audio_latents, assembly_plan, result = execute()
|
||||
execute_outputs = execute()
|
||||
if bool(capture_refine_context):
|
||||
video_latents, audio_latents, assembly_plan, result, refine_context = execute_outputs
|
||||
else:
|
||||
video_latents, audio_latents, assembly_plan, result = execute_outputs
|
||||
result = dict(result)
|
||||
report = str(result["report"]) + "\n" + storage.summary(
|
||||
detailed=diagnostics == DIAGNOSTICS_FULL
|
||||
)
|
||||
result["report"] = report
|
||||
storage.finalize(session=result["session"], report=report)
|
||||
return video_latents, audio_latents, mark_runtime_start(assembly_plan), report
|
||||
outputs = (
|
||||
video_latents,
|
||||
audio_latents,
|
||||
mark_runtime_start(assembly_plan),
|
||||
report,
|
||||
)
|
||||
if bool(capture_refine_context):
|
||||
return (*outputs, refine_context)
|
||||
return outputs
|
||||
|
||||
|
||||
class H3ContinuumSamplerTimelineVideo(H3ContinuumSamplerProduction):
|
||||
|
||||
@@ -0,0 +1,367 @@
|
||||
"""Sequential pixel/VAE resize for Continuum physical video groups."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from .plan import SECOND_PASS_CONTRACT_VERSION
|
||||
|
||||
|
||||
PIXEL_UPSCALE_METHODS = (
|
||||
"nearest-exact",
|
||||
"bilinear",
|
||||
"area",
|
||||
"bicubic",
|
||||
"bislerp",
|
||||
"lanczos",
|
||||
)
|
||||
_PIXEL_RESIZE_FRAME_BATCH = 8
|
||||
|
||||
|
||||
class PixelVaeResizeError(ValueError):
|
||||
"""Raised when a pixel/VAE resize would violate the physical-group contract."""
|
||||
|
||||
|
||||
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
raise PixelVaeResizeError("assembly_plan must be a mapping")
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, Mapping):
|
||||
raise PixelVaeResizeError("assembly_plan has no second_pass_contract")
|
||||
if contract.get("version") != SECOND_PASS_CONTRACT_VERSION:
|
||||
raise PixelVaeResizeError("unsupported second_pass_contract version")
|
||||
groups = contract.get("physical_groups")
|
||||
if (
|
||||
not isinstance(groups, list)
|
||||
or not groups
|
||||
or not all(isinstance(group, Mapping) for group in groups)
|
||||
):
|
||||
raise PixelVaeResizeError(
|
||||
"second_pass_contract.physical_groups must be non-empty"
|
||||
)
|
||||
return groups
|
||||
|
||||
|
||||
def resolve_pixel_latent_scale(
|
||||
assembly_plan: Mapping[str, Any],
|
||||
) -> tuple[int, int]:
|
||||
"""Return plan-derived pixel-per-latent ratios as ``(width, height)``."""
|
||||
|
||||
groups = _contract_groups(assembly_plan)
|
||||
scale: tuple[int, int] | None = None
|
||||
for position, group in enumerate(groups):
|
||||
if int(group.get("group_id", -1)) != position:
|
||||
raise PixelVaeResizeError("physical group order or group_id changed")
|
||||
source_width = int(group.get("source_width", -1))
|
||||
source_height = int(group.get("source_height", -1))
|
||||
source_latent_width = int(group.get("source_latent_w", -1))
|
||||
source_latent_height = int(group.get("source_latent_h", -1))
|
||||
if min(
|
||||
source_width,
|
||||
source_height,
|
||||
source_latent_width,
|
||||
source_latent_height,
|
||||
) < 1:
|
||||
raise PixelVaeResizeError("source pixel/latent geometry is invalid")
|
||||
if (
|
||||
source_width % source_latent_width
|
||||
or source_height % source_latent_height
|
||||
):
|
||||
raise PixelVaeResizeError(
|
||||
"source pixel/latent geometry is not integral"
|
||||
)
|
||||
group_scale = (
|
||||
source_width // source_latent_width,
|
||||
source_height // source_latent_height,
|
||||
)
|
||||
if scale is None:
|
||||
scale = group_scale
|
||||
elif scale != group_scale:
|
||||
raise PixelVaeResizeError(
|
||||
"source pixel/latent scale differs between physical groups"
|
||||
)
|
||||
|
||||
assert scale is not None
|
||||
return scale
|
||||
|
||||
|
||||
def _latent_samples(
|
||||
latent: Any,
|
||||
*,
|
||||
position: int,
|
||||
group: Mapping[str, Any],
|
||||
) -> torch.Tensor:
|
||||
if not isinstance(latent, Mapping) or "samples" not in latent:
|
||||
raise PixelVaeResizeError(
|
||||
f"video_latents[{position}] must be a LATENT mapping with samples"
|
||||
)
|
||||
samples = latent["samples"]
|
||||
if not isinstance(samples, torch.Tensor) or samples.ndim != 5:
|
||||
raise PixelVaeResizeError("video latent must have shape [B,C,T,H,W]")
|
||||
batch, channels, temporal, height, width = (
|
||||
int(value) for value in samples.shape
|
||||
)
|
||||
expected = (
|
||||
int(group.get("source_batch", -1)),
|
||||
int(group.get("latent_channels", -1)),
|
||||
int(group.get("source_latent_t", -1)),
|
||||
int(group.get("source_latent_h", -1)),
|
||||
int(group.get("source_latent_w", -1)),
|
||||
)
|
||||
if (batch, channels, temporal, height, width) != expected:
|
||||
raise PixelVaeResizeError(
|
||||
"video latent shape or physical group order changed"
|
||||
)
|
||||
if channels != 24:
|
||||
raise PixelVaeResizeError("MiniMax H3 video latent must have 24 channels")
|
||||
if not bool(torch.isfinite(samples).all()):
|
||||
raise PixelVaeResizeError("input video latent contains NaN or Inf")
|
||||
return samples
|
||||
|
||||
|
||||
def _resize_one_group(
|
||||
samples: torch.Tensor,
|
||||
*,
|
||||
video_vae: Any,
|
||||
source_width: int,
|
||||
source_height: int,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
target_latent_width: int,
|
||||
target_latent_height: int,
|
||||
upscale_method: str,
|
||||
upscale_fn: Callable[..., Any],
|
||||
) -> torch.Tensor:
|
||||
"""Decode, resize, and encode one group so its pixel buffer dies per call."""
|
||||
|
||||
decoded_pixels = video_vae.decode(samples)
|
||||
if not isinstance(decoded_pixels, torch.Tensor) or decoded_pixels.ndim != 5:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE decode must return pixels with shape [B,F,H,W,C]"
|
||||
)
|
||||
batch, frames, pixel_height, pixel_width, channels = (
|
||||
int(value) for value in decoded_pixels.shape
|
||||
)
|
||||
if batch != int(samples.shape[0]):
|
||||
raise PixelVaeResizeError("Video VAE decode changed the batch size")
|
||||
if frames < 1 or channels != 3:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE decode returned invalid frame or channel geometry"
|
||||
)
|
||||
if (pixel_width, pixel_height) != (source_width, source_height):
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE decode geometry differs from the assembly plan"
|
||||
)
|
||||
for frame_start in range(0, frames, _PIXEL_RESIZE_FRAME_BATCH):
|
||||
if not bool(
|
||||
torch.isfinite(
|
||||
decoded_pixels[
|
||||
:,
|
||||
frame_start : frame_start + _PIXEL_RESIZE_FRAME_BATCH,
|
||||
]
|
||||
).all()
|
||||
):
|
||||
raise PixelVaeResizeError("Video VAE decode contains NaN or Inf")
|
||||
|
||||
# Pixel resizing is intentionally CPU-owned. Core's Lanczos implementation
|
||||
# is CPU/PIL-based already, and this also avoids retaining a full video-sized
|
||||
# resize workspace in VRAM under --gpu-only configurations.
|
||||
pixels = decoded_pixels.detach().to("cpu").contiguous()
|
||||
del decoded_pixels
|
||||
|
||||
# Core common_upscale is a 4D image boundary. Make B*F explicit so video
|
||||
# frame order is unambiguous. Resize bounded frame batches into one CPU
|
||||
# [B,F,H,W,C] buffer required by the temporal H3 VAE encoder.
|
||||
source_flat = pixels.reshape(
|
||||
batch * frames, pixel_height, pixel_width, channels
|
||||
)
|
||||
resized_pixels = torch.empty(
|
||||
(batch, frames, target_height, target_width, channels),
|
||||
dtype=pixels.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
target_flat = resized_pixels.reshape(
|
||||
batch * frames, target_height, target_width, channels
|
||||
)
|
||||
for frame_start in range(
|
||||
0,
|
||||
batch * frames,
|
||||
_PIXEL_RESIZE_FRAME_BATCH,
|
||||
):
|
||||
frame_stop = min(
|
||||
frame_start + _PIXEL_RESIZE_FRAME_BATCH,
|
||||
batch * frames,
|
||||
)
|
||||
pixel_nchw = (
|
||||
source_flat[frame_start:frame_stop]
|
||||
.movedim(-1, 1)
|
||||
.contiguous()
|
||||
)
|
||||
resized_nchw = upscale_fn(
|
||||
pixel_nchw,
|
||||
int(target_width),
|
||||
int(target_height),
|
||||
upscale_method,
|
||||
"disabled",
|
||||
)
|
||||
expected_resize_shape = (
|
||||
frame_stop - frame_start,
|
||||
channels,
|
||||
target_height,
|
||||
target_width,
|
||||
)
|
||||
if not isinstance(resized_nchw, torch.Tensor) or tuple(
|
||||
resized_nchw.shape
|
||||
) != expected_resize_shape:
|
||||
raise PixelVaeResizeError("Core pixel resize returned invalid geometry")
|
||||
resized_nchw = resized_nchw.detach().to("cpu")
|
||||
if not bool(torch.isfinite(resized_nchw).all()):
|
||||
raise PixelVaeResizeError("Core pixel resize contains NaN or Inf")
|
||||
target_flat[frame_start:frame_stop].copy_(
|
||||
resized_nchw.movedim(1, -1)
|
||||
)
|
||||
del pixel_nchw, resized_nchw
|
||||
|
||||
del pixels, source_flat, target_flat
|
||||
|
||||
# Match Core VAEEncode exactly: H3's VAE wrapper expects a 4D IMAGE batch
|
||||
# and promotes its frame axis to one temporal video batch. Passing the 5D
|
||||
# [B,F,H,W,C] tensor directly makes Core crop F as if it were spatial.
|
||||
# Encode each Continuum batch independently, then restore B explicitly.
|
||||
encoded_batches: list[torch.Tensor] = []
|
||||
expected_batch_shape = (
|
||||
1,
|
||||
int(samples.shape[1]),
|
||||
int(samples.shape[2]),
|
||||
int(target_latent_height),
|
||||
int(target_latent_width),
|
||||
)
|
||||
for batch_index in range(batch):
|
||||
encoded_batch = video_vae.encode(resized_pixels[batch_index])
|
||||
if not isinstance(encoded_batch, torch.Tensor) or encoded_batch.ndim != 5:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE encode must return a latent with shape [B,C,T,H,W]"
|
||||
)
|
||||
if int(encoded_batch.shape[0]) != 1:
|
||||
raise PixelVaeResizeError(
|
||||
"Core Video VAE Encode must return one batch per IMAGE sequence"
|
||||
)
|
||||
encoded_batch_shape = tuple(int(value) for value in encoded_batch.shape)
|
||||
if encoded_batch_shape != expected_batch_shape:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE encode changed B/C/T or returned the wrong target H/W "
|
||||
f"for batch {batch_index}: expected {expected_batch_shape}, "
|
||||
f"got {encoded_batch_shape}"
|
||||
)
|
||||
encoded_batches.append(encoded_batch)
|
||||
encoded = (
|
||||
encoded_batches[0]
|
||||
if len(encoded_batches) == 1
|
||||
else torch.cat(encoded_batches, dim=0)
|
||||
)
|
||||
if not isinstance(encoded, torch.Tensor) or encoded.ndim != 5:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE encode must return a latent with shape [B,C,T,H,W]"
|
||||
)
|
||||
expected_shape = (
|
||||
int(samples.shape[0]),
|
||||
int(samples.shape[1]),
|
||||
int(samples.shape[2]),
|
||||
int(target_latent_height),
|
||||
int(target_latent_width),
|
||||
)
|
||||
if tuple(int(value) for value in encoded.shape) != expected_shape:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE encode changed B/C/T or returned the wrong target H/W: "
|
||||
f"expected {expected_shape}, got {tuple(int(value) for value in encoded.shape)}"
|
||||
)
|
||||
if not bool(torch.isfinite(encoded).all()):
|
||||
raise PixelVaeResizeError("Video VAE encode contains NaN or Inf")
|
||||
|
||||
# Continuum stores accepted physical latents on CPU; do not retain one GPU
|
||||
# allocation per physical group while later groups are being converted.
|
||||
output = encoded.detach().to("cpu").contiguous()
|
||||
del resized_pixels, encoded_batches, encoded
|
||||
return output
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def resize_video_latents_via_vae(
|
||||
video_latents: Sequence[Mapping[str, Any]],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
*,
|
||||
video_vae: Any,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
upscale_method: str = "lanczos",
|
||||
upscale_fn: Callable[..., Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Resize physical groups sequentially through the connected H3 Video VAE.
|
||||
|
||||
``target_width`` and ``target_height`` are pixel dimensions. The expected
|
||||
encoded latent geometry is derived exclusively from assembly-plan metadata.
|
||||
VAE failures are intentionally not caught: Core owns its OOM/tiled fallback,
|
||||
and this path must never fall back to direct latent interpolation.
|
||||
"""
|
||||
|
||||
groups = _contract_groups(assembly_plan)
|
||||
if (
|
||||
not isinstance(video_latents, Sequence)
|
||||
or isinstance(video_latents, (str, bytes))
|
||||
or len(video_latents) != len(groups)
|
||||
):
|
||||
raise PixelVaeResizeError("video latent group count changed")
|
||||
if (
|
||||
video_vae is None
|
||||
or not callable(getattr(video_vae, "decode", None))
|
||||
or not callable(getattr(video_vae, "encode", None))
|
||||
):
|
||||
raise PixelVaeResizeError("a usable H3 Video VAE is required")
|
||||
if upscale_method not in PIXEL_UPSCALE_METHODS:
|
||||
raise PixelVaeResizeError(
|
||||
f"unsupported pixel upscale method {upscale_method!r}"
|
||||
)
|
||||
target_width = int(target_width)
|
||||
target_height = int(target_height)
|
||||
if target_width < 1 or target_height < 1:
|
||||
raise PixelVaeResizeError("target pixel width and height must be positive")
|
||||
|
||||
scale_width, scale_height = resolve_pixel_latent_scale(assembly_plan)
|
||||
if target_width % scale_width or target_height % scale_height:
|
||||
raise PixelVaeResizeError(
|
||||
"target pixel geometry is not aligned to the plan-derived latent scale"
|
||||
)
|
||||
target_latent_width = target_width // scale_width
|
||||
target_latent_height = target_height // scale_height
|
||||
if upscale_fn is None:
|
||||
from comfy.utils import common_upscale
|
||||
|
||||
upscale_fn = common_upscale
|
||||
|
||||
outputs: list[dict[str, Any]] = []
|
||||
for position, (latent, group) in enumerate(
|
||||
zip(video_latents, groups, strict=True)
|
||||
):
|
||||
samples = _latent_samples(latent, position=position, group=group)
|
||||
source_width = int(group["source_width"])
|
||||
source_height = int(group["source_height"])
|
||||
resized_samples = _resize_one_group(
|
||||
samples,
|
||||
video_vae=video_vae,
|
||||
source_width=source_width,
|
||||
source_height=source_height,
|
||||
target_width=target_width,
|
||||
target_height=target_height,
|
||||
target_latent_width=target_latent_width,
|
||||
target_latent_height=target_latent_height,
|
||||
upscale_method=upscale_method,
|
||||
upscale_fn=upscale_fn,
|
||||
)
|
||||
output = dict(latent)
|
||||
output["samples"] = resized_samples
|
||||
outputs.append(output)
|
||||
|
||||
return outputs
|
||||
+96
@@ -11,6 +11,7 @@ from ..state import validate_plan
|
||||
ASSEMBLY_PLAN_MAGIC = "H3_CONTINUUM_ASSEMBLY_PLAN"
|
||||
ASSEMBLY_PLAN_SCHEMA_VERSION = 1
|
||||
FPS = 24
|
||||
SECOND_PASS_CONTRACT_VERSION = 1
|
||||
|
||||
|
||||
class AssemblyPlanError(ValueError):
|
||||
@@ -199,6 +200,89 @@ def _recombine_terminal_tensor(
|
||||
return output.contiguous()
|
||||
|
||||
|
||||
def _terminal_physical_prompt(
|
||||
prompts: list[str],
|
||||
*,
|
||||
chunk_seconds: float,
|
||||
) -> tuple[str, str]:
|
||||
"""Reproduce the established terminal-pair prompt without reparsing it later."""
|
||||
first, second = prompts
|
||||
if first == second:
|
||||
return first, "shared_prompt_v1"
|
||||
split = f"{float(chunk_seconds):g}"
|
||||
stop = f"{float(chunk_seconds) * 2:g}"
|
||||
return (
|
||||
f"[0-{split}s]\n{first}\n\n[{split}-{stop}s]\n{second}",
|
||||
"paired_timeline_v1",
|
||||
)
|
||||
|
||||
|
||||
def _attach_second_pass_contract(
|
||||
plan: dict[str, Any],
|
||||
*,
|
||||
logical_entries: list[dict[str, Any]],
|
||||
physical_entries: list[dict[str, Any]],
|
||||
chunk_seconds: float,
|
||||
) -> None:
|
||||
"""Attach additive V3.5 metadata without changing V3.4 generation semantics."""
|
||||
decode_groups = plan.get("decode_groups")
|
||||
if not isinstance(decode_groups, list):
|
||||
decode_groups = [
|
||||
{
|
||||
"logical_chunk_indices": [index + 1],
|
||||
"terminal_merged": False,
|
||||
"total_frames": plan["chunks"][index]["total_frames"],
|
||||
"trim_frames": plan["chunks"][index]["trim_frames"],
|
||||
}
|
||||
for index in range(len(physical_entries))
|
||||
]
|
||||
|
||||
groups: list[dict[str, Any]] = []
|
||||
for group_id, (entry, decode_group) in enumerate(
|
||||
zip(physical_entries, decode_groups, strict=True)
|
||||
):
|
||||
video = entry["video"]
|
||||
audio = entry["audio"]
|
||||
logical_chunks = [int(value) for value in decode_group["logical_chunk_indices"]]
|
||||
prompts = [str(logical_entries[index - 1].get("prompt", "")) for index in logical_chunks]
|
||||
if len(prompts) == 1:
|
||||
physical_prompt = prompts[0]
|
||||
prompt_policy = "single"
|
||||
elif len(prompts) == 2 and bool(decode_group.get("terminal_merged")):
|
||||
physical_prompt, prompt_policy = _terminal_physical_prompt(
|
||||
prompts,
|
||||
chunk_seconds=chunk_seconds,
|
||||
)
|
||||
else:
|
||||
physical_prompt = "\n\n".join(prompts)
|
||||
prompt_policy = "physical_group_v1"
|
||||
|
||||
groups.append(
|
||||
{
|
||||
"group_id": group_id,
|
||||
"logical_chunks": logical_chunks,
|
||||
"physical_prompt": physical_prompt,
|
||||
"prompt_policy": prompt_policy,
|
||||
"physical_frames": int(decode_group["total_frames"]),
|
||||
"trim_prefix_frames": int(decode_group["trim_frames"]),
|
||||
"terminal_merged": bool(decode_group.get("terminal_merged", False)),
|
||||
"source_width": int(plan["width"]),
|
||||
"source_height": int(plan["height"]),
|
||||
"source_batch": int(video.shape[0]),
|
||||
"latent_channels": int(video.shape[1]),
|
||||
"source_latent_t": int(video.shape[2]),
|
||||
"source_latent_h": int(video.shape[3]),
|
||||
"source_latent_w": int(video.shape[4]),
|
||||
"source_audio_shape": [int(value) for value in audio.shape],
|
||||
}
|
||||
)
|
||||
|
||||
plan["second_pass_contract"] = {
|
||||
"version": SECOND_PASS_CONTRACT_VERSION,
|
||||
"physical_groups": groups,
|
||||
}
|
||||
|
||||
|
||||
def prepare_physical_decode_entries(
|
||||
entries,
|
||||
*,
|
||||
@@ -215,6 +299,12 @@ def prepare_physical_decode_entries(
|
||||
preserve_final_frame=bool(preserve_final_frame),
|
||||
)
|
||||
if not terminal_merged:
|
||||
_attach_second_pass_contract(
|
||||
plan,
|
||||
logical_entries=logical_entries,
|
||||
physical_entries=logical_entries,
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
)
|
||||
return logical_entries, plan
|
||||
if len(logical_entries) < 2:
|
||||
raise ValueError("terminal physical decode requires two logical entries")
|
||||
@@ -287,5 +377,11 @@ def prepare_physical_decode_entries(
|
||||
plan["logical_chunk_count"] = len(logical_chunks)
|
||||
plan["physical_decode_group_count"] = len(decode_groups)
|
||||
plan["decode_groups"] = decode_groups
|
||||
_attach_second_pass_contract(
|
||||
plan,
|
||||
logical_entries=logical_entries,
|
||||
physical_entries=decode_entries,
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
)
|
||||
validate_assembly_plan(plan)
|
||||
return decode_entries, plan
|
||||
|
||||
@@ -0,0 +1,546 @@
|
||||
"""Runtime-only First Pass conditioning for Continuum-aware refinement.
|
||||
|
||||
The context deliberately remains outside the persisted Assembly Plan contract. It
|
||||
owns CPU copies of the First Pass conditioning tensors and records them in physical
|
||||
sampling order. Second Pass callers may then derive a target-canvas conditioning
|
||||
without mutating the captured source context.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..constants import MARK_VIDEO_CONTEXT
|
||||
|
||||
|
||||
MAGIC = "H3_CONTINUUM_REFINE_CONTEXT"
|
||||
VERSION = 1
|
||||
|
||||
|
||||
class RefineContextError(ValueError):
|
||||
"""Raised when the runtime-only refine context is structurally invalid."""
|
||||
|
||||
|
||||
class RefineConditioningAdaptationError(RefineContextError):
|
||||
"""Raised when a valid captured conditioning cannot be adapted safely."""
|
||||
|
||||
|
||||
def _cpu_tensor(value: torch.Tensor) -> torch.Tensor:
|
||||
return value.detach().to(device="cpu").clone().contiguous()
|
||||
|
||||
|
||||
def _freeze(value: Any) -> Any:
|
||||
if torch.is_tensor(value):
|
||||
return _cpu_tensor(value)
|
||||
if isinstance(value, Mapping):
|
||||
return MappingProxyType({key: _freeze(item) for key, item in value.items()})
|
||||
if isinstance(value, (list, tuple)):
|
||||
return tuple(_freeze(item) for item in value)
|
||||
if isinstance(value, (set, frozenset)):
|
||||
return frozenset(_freeze(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def freeze_conditioning(conditioning: Any) -> tuple[tuple[torch.Tensor, Mapping], ...]:
|
||||
"""Return a CPU-owned conditioning clone with immutable containers.
|
||||
|
||||
Tensors are detached and cloned, so later mutation or device movement of the
|
||||
original First Pass conditioning cannot affect the refine context. Container
|
||||
immutability makes the result safe to share until target adaptation performs
|
||||
copy-on-write replacement of visual fields.
|
||||
"""
|
||||
|
||||
if not isinstance(conditioning, Sequence) or isinstance(
|
||||
conditioning, (str, bytes, bytearray)
|
||||
):
|
||||
raise RefineContextError("conditioning must be a non-empty sequence")
|
||||
output: list[tuple[torch.Tensor, Mapping]] = []
|
||||
for index, entry in enumerate(conditioning):
|
||||
if not isinstance(entry, Sequence) or isinstance(
|
||||
entry, (str, bytes, bytearray)
|
||||
) or len(entry) != 2:
|
||||
raise RefineContextError(
|
||||
f"conditioning entry {index} is not [tensor, metadata]"
|
||||
)
|
||||
tensor, metadata = entry
|
||||
if not torch.is_tensor(tensor):
|
||||
raise RefineContextError(
|
||||
f"conditioning entry {index} text value is not a tensor"
|
||||
)
|
||||
if not isinstance(metadata, Mapping):
|
||||
raise RefineContextError(
|
||||
f"conditioning entry {index} metadata is not a mapping"
|
||||
)
|
||||
output.append((_cpu_tensor(tensor), _freeze(metadata)))
|
||||
if not output:
|
||||
raise RefineContextError("conditioning is empty")
|
||||
return tuple(output)
|
||||
|
||||
|
||||
def _optional_image(value: Any, name: str) -> torch.Tensor | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3:
|
||||
raise RefineContextError(
|
||||
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
|
||||
)
|
||||
return _cpu_tensor(value)
|
||||
|
||||
|
||||
def _validate_optional_image(value: Any, name: str) -> None:
|
||||
if value is not None and (
|
||||
not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3
|
||||
):
|
||||
raise RefineContextError(
|
||||
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
|
||||
)
|
||||
|
||||
|
||||
def make_refine_group(
|
||||
*,
|
||||
conditioning: Any,
|
||||
group_id: int,
|
||||
logical_chunks: Sequence[int],
|
||||
physical_frames: int,
|
||||
prompt_policy: str,
|
||||
physical_prompt: str,
|
||||
source_video_shape: Sequence[int],
|
||||
physical_clip_index: int,
|
||||
context_frames: int,
|
||||
first_image: torch.Tensor | None = None,
|
||||
last_image: torch.Tensor | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Capture one physical First Pass sampling group's effective conditioning."""
|
||||
|
||||
group_id = int(group_id)
|
||||
physical_frames = int(physical_frames)
|
||||
physical_clip_index = int(physical_clip_index)
|
||||
context_frames = int(context_frames)
|
||||
chunks = tuple(int(value) for value in logical_chunks)
|
||||
shape = tuple(int(value) for value in source_video_shape)
|
||||
if group_id < 0:
|
||||
raise RefineContextError("refine group id must be non-negative")
|
||||
if not chunks or any(value <= 0 for value in chunks):
|
||||
raise RefineContextError("refine group logical chunks are invalid")
|
||||
if tuple(sorted(chunks)) != chunks or len(set(chunks)) != len(chunks):
|
||||
raise RefineContextError("refine group logical chunks must be ordered and unique")
|
||||
if physical_frames <= 0:
|
||||
raise RefineContextError("refine group physical frame count is invalid")
|
||||
if not str(prompt_policy):
|
||||
raise RefineContextError("refine group prompt policy is empty")
|
||||
if len(shape) != 5 or any(value <= 0 for value in shape):
|
||||
raise RefineContextError("refine group source video shape must be [B,C,T,H,W]")
|
||||
if physical_clip_index <= 0:
|
||||
raise RefineContextError("refine group physical clip index is invalid")
|
||||
if context_frames < 0:
|
||||
raise RefineContextError("refine group context frame count is invalid")
|
||||
return {
|
||||
"group_id": group_id,
|
||||
"logical_chunks": chunks,
|
||||
"physical_frames": physical_frames,
|
||||
"prompt_policy": str(prompt_policy),
|
||||
"physical_prompt": str(physical_prompt),
|
||||
"source_video_shape": shape,
|
||||
"physical_clip_index": physical_clip_index,
|
||||
"context_frames": context_frames,
|
||||
"conditioning": freeze_conditioning(conditioning),
|
||||
"first_image": _optional_image(first_image, "first_image"),
|
||||
"last_image": _optional_image(last_image, "last_image"),
|
||||
}
|
||||
|
||||
|
||||
def make_refine_context(
|
||||
groups: Sequence[Mapping[str, Any]],
|
||||
*,
|
||||
source_width: int,
|
||||
source_height: int,
|
||||
conditioning_mode: str,
|
||||
complete: bool = True,
|
||||
notes: Sequence[str] = (),
|
||||
) -> dict[str, Any]:
|
||||
"""Build and validate a physical-order runtime refine context."""
|
||||
|
||||
context = {
|
||||
"magic": MAGIC,
|
||||
"version": VERSION,
|
||||
"source_width": int(source_width),
|
||||
"source_height": int(source_height),
|
||||
"conditioning_mode": str(conditioning_mode),
|
||||
"complete": bool(complete),
|
||||
"groups": tuple(dict(group) for group in groups),
|
||||
"notes": tuple(str(note) for note in notes),
|
||||
}
|
||||
return validate_refine_context(context)
|
||||
|
||||
|
||||
def _validate_frozen_conditioning(value: Any, *, group_id: int) -> None:
|
||||
if not isinstance(value, tuple) or not value:
|
||||
raise RefineContextError(f"refine group {group_id} conditioning is invalid")
|
||||
for index, entry in enumerate(value):
|
||||
if not isinstance(entry, tuple) or len(entry) != 2:
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} conditioning entry {index} is invalid"
|
||||
)
|
||||
tensor, metadata = entry
|
||||
if not torch.is_tensor(tensor) or tensor.device.type != "cpu":
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} conditioning tensor {index} is not CPU-owned"
|
||||
)
|
||||
if not isinstance(metadata, Mapping):
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} conditioning metadata {index} is invalid"
|
||||
)
|
||||
|
||||
|
||||
def _group_field_tuple(group: Mapping[str, Any], name: str) -> tuple[int, ...]:
|
||||
value = group.get(name)
|
||||
if not isinstance(value, (tuple, list)):
|
||||
raise RefineContextError(f"refine group {group.get('group_id')} {name} is invalid")
|
||||
try:
|
||||
return tuple(int(item) for item in value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RefineContextError(
|
||||
f"refine group {group.get('group_id')} {name} is invalid"
|
||||
) from exc
|
||||
|
||||
|
||||
def validate_refine_context(
|
||||
context: Any, assembly_plan: Mapping[str, Any] | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Validate a runtime context and, when supplied, its Assembly Plan pairing."""
|
||||
|
||||
if not isinstance(context, dict) or context.get("magic") != MAGIC:
|
||||
raise RefineContextError("invalid H3 Continuum refine context")
|
||||
if type(context.get("version")) is not int or context["version"] != VERSION:
|
||||
raise RefineContextError(
|
||||
f"unsupported refine context version {context.get('version')!r}"
|
||||
)
|
||||
if int(context.get("source_width", 0)) <= 0 or int(
|
||||
context.get("source_height", 0)
|
||||
) <= 0:
|
||||
raise RefineContextError("refine context source dimensions are invalid")
|
||||
if not str(context.get("conditioning_mode", "")):
|
||||
raise RefineContextError("refine context conditioning mode is empty")
|
||||
if type(context.get("complete")) is not bool:
|
||||
raise RefineContextError("refine context complete flag is invalid")
|
||||
groups = context.get("groups")
|
||||
if not isinstance(groups, tuple):
|
||||
raise RefineContextError("refine context groups must be an immutable tuple")
|
||||
if context["complete"] and not groups:
|
||||
raise RefineContextError("complete refine context contains no physical groups")
|
||||
notes = context.get("notes")
|
||||
if not isinstance(notes, tuple) or any(not isinstance(note, str) for note in notes):
|
||||
raise RefineContextError("refine context notes must be an immutable string tuple")
|
||||
|
||||
previous_clip_index = 0
|
||||
for expected_id, group in enumerate(groups):
|
||||
if not isinstance(group, Mapping):
|
||||
raise RefineContextError(f"refine group {expected_id} is not a mapping")
|
||||
if type(group.get("group_id")) is not int or group["group_id"] != expected_id:
|
||||
raise RefineContextError("refine context physical group order is not contiguous")
|
||||
logical_chunks = _group_field_tuple(group, "logical_chunks")
|
||||
if not logical_chunks or any(value <= 0 for value in logical_chunks):
|
||||
raise RefineContextError(f"refine group {expected_id} logical chunks are invalid")
|
||||
if tuple(sorted(logical_chunks)) != logical_chunks or len(
|
||||
set(logical_chunks)
|
||||
) != len(logical_chunks):
|
||||
raise RefineContextError(
|
||||
f"refine group {expected_id} logical chunks are not ordered and unique"
|
||||
)
|
||||
if int(group.get("physical_frames", 0)) <= 0:
|
||||
raise RefineContextError(f"refine group {expected_id} physical frames are invalid")
|
||||
if not str(group.get("prompt_policy", "")):
|
||||
raise RefineContextError(f"refine group {expected_id} prompt policy is empty")
|
||||
if "physical_prompt" not in group:
|
||||
raise RefineContextError(f"refine group {expected_id} physical prompt is absent")
|
||||
shape = _group_field_tuple(group, "source_video_shape")
|
||||
if len(shape) != 5 or any(value <= 0 for value in shape):
|
||||
raise RefineContextError(f"refine group {expected_id} source shape is invalid")
|
||||
clip_index = int(group.get("physical_clip_index", 0))
|
||||
if clip_index <= previous_clip_index:
|
||||
raise RefineContextError(
|
||||
"refine context physical clip indices are not strictly increasing"
|
||||
)
|
||||
previous_clip_index = clip_index
|
||||
if int(group.get("context_frames", -1)) < 0:
|
||||
raise RefineContextError(f"refine group {expected_id} context frames are invalid")
|
||||
_validate_frozen_conditioning(group.get("conditioning"), group_id=expected_id)
|
||||
_validate_optional_image(group.get("first_image"), "first_image")
|
||||
_validate_optional_image(group.get("last_image"), "last_image")
|
||||
|
||||
if assembly_plan is not None:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
raise RefineContextError("assembly plan paired with refine context is invalid")
|
||||
if int(assembly_plan.get("width", 0)) != int(context["source_width"]) or int(
|
||||
assembly_plan.get("height", 0)
|
||||
) != int(context["source_height"]):
|
||||
raise RefineContextError("refine context source dimensions differ from assembly plan")
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
plan_groups = contract.get("physical_groups") if isinstance(contract, Mapping) else None
|
||||
if not isinstance(plan_groups, Sequence):
|
||||
raise RefineContextError("assembly plan has no Second Pass physical-group contract")
|
||||
# An incomplete context is diagnostic-only (for example, Run Storage may
|
||||
# have reused an earlier group that was never sampled in this execution).
|
||||
# Its local group ids describe capture order, not Assembly Plan position;
|
||||
# validating those partial entries by index would incorrectly turn the
|
||||
# intended prompt-only fallback into a broken typed-contract hard error.
|
||||
if not context["complete"]:
|
||||
return context
|
||||
if context["complete"] and len(plan_groups) != len(groups):
|
||||
raise RefineContextError(
|
||||
"complete refine context physical-group count differs from assembly plan"
|
||||
)
|
||||
for group in groups:
|
||||
group_id = int(group["group_id"])
|
||||
if group_id >= len(plan_groups) or not isinstance(plan_groups[group_id], Mapping):
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} has no matching assembly-plan group"
|
||||
)
|
||||
planned = plan_groups[group_id]
|
||||
comparisons = (
|
||||
("logical_chunks", tuple(int(v) for v in group["logical_chunks"]),
|
||||
tuple(int(v) for v in planned.get("logical_chunks", ()))),
|
||||
("physical_frames", int(group["physical_frames"]),
|
||||
int(planned.get("physical_frames", 0))),
|
||||
("prompt_policy", str(group["prompt_policy"]),
|
||||
str(planned.get("prompt_policy", ""))),
|
||||
("physical_prompt", str(group["physical_prompt"]),
|
||||
str(planned.get("physical_prompt", ""))),
|
||||
("context_frames", int(group["context_frames"]),
|
||||
int(planned.get("trim_prefix_frames", -1))),
|
||||
)
|
||||
for name, actual, expected in comparisons:
|
||||
if actual != expected:
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} {name} differs from assembly plan"
|
||||
)
|
||||
expected_shape = (
|
||||
int(planned.get("source_batch", 0)),
|
||||
int(planned.get("latent_channels", 0)),
|
||||
int(planned.get("source_latent_t", 0)),
|
||||
int(planned.get("source_latent_h", 0)),
|
||||
int(planned.get("source_latent_w", 0)),
|
||||
)
|
||||
if tuple(group["source_video_shape"]) != expected_shape:
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} source shape differs from assembly plan"
|
||||
)
|
||||
return context
|
||||
|
||||
|
||||
def _common_upscale(
|
||||
samples: torch.Tensor,
|
||||
width: int,
|
||||
height: int,
|
||||
method: str,
|
||||
crop: str,
|
||||
) -> torch.Tensor:
|
||||
try:
|
||||
import comfy.utils
|
||||
except Exception as exc: # pragma: no cover - ComfyUI runtime only
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"ComfyUI Core resize support is unavailable: {exc}"
|
||||
) from exc
|
||||
return comfy.utils.common_upscale(samples, width, height, method, crop)
|
||||
|
||||
|
||||
def _resize_latent(
|
||||
latent: Any,
|
||||
*,
|
||||
target_h: int,
|
||||
target_w: int,
|
||||
method: str,
|
||||
) -> torch.Tensor:
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5:
|
||||
raise RefineConditioningAdaptationError(
|
||||
"H3 visual conditioning latent must be [B,C,T,H,W]"
|
||||
)
|
||||
batch, channels, temporal, _, _ = (int(value) for value in latent.shape)
|
||||
flat = latent.permute(0, 2, 1, 3, 4).reshape(
|
||||
batch * temporal, channels, int(latent.shape[-2]), int(latent.shape[-1])
|
||||
)
|
||||
try:
|
||||
resized = _common_upscale(flat, target_w, target_h, method, "disabled")
|
||||
except RefineConditioningAdaptationError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"visual conditioning latent resize failed: {exc}"
|
||||
) from exc
|
||||
if not torch.is_tensor(resized) or tuple(resized.shape[-2:]) != (
|
||||
target_h,
|
||||
target_w,
|
||||
):
|
||||
raise RefineConditioningAdaptationError(
|
||||
"Core resize returned an incompatible visual conditioning shape"
|
||||
)
|
||||
return (
|
||||
resized.reshape(batch, temporal, channels, target_h, target_w)
|
||||
.permute(0, 2, 1, 3, 4)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _encode_keyframe_image(
|
||||
image: torch.Tensor,
|
||||
*,
|
||||
target_h: int,
|
||||
target_w: int,
|
||||
crop: str,
|
||||
video_vae: Any,
|
||||
) -> torch.Tensor:
|
||||
pixels_h = target_h * 16
|
||||
pixels_w = target_w * 16
|
||||
samples = image[:1, ..., :3].movedim(-1, 1)
|
||||
try:
|
||||
resized = _common_upscale(samples, pixels_w, pixels_h, "lanczos", crop)
|
||||
latent = video_vae.encode(resized.movedim(1, -1).contiguous())
|
||||
except RefineConditioningAdaptationError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"target keyframe VAE encode failed: {exc}"
|
||||
) from exc
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5 or tuple(
|
||||
int(value) for value in latent.shape[-2:]
|
||||
) != (target_h, target_w):
|
||||
raise RefineConditioningAdaptationError(
|
||||
"target keyframe VAE encode returned an incompatible latent shape"
|
||||
)
|
||||
return latent.contiguous()
|
||||
|
||||
|
||||
def adapt_group_conditioning(
|
||||
group: Mapping[str, Any],
|
||||
target_latent_h: int,
|
||||
target_latent_w: int,
|
||||
video_vae: Any = None,
|
||||
upscale_method: str = "bilinear",
|
||||
) -> tuple[list[list[Any]], dict[str, Any]]:
|
||||
"""Copy-on-write target adaptation for keyframes and Continuum video context.
|
||||
|
||||
Ordinary MiniMax references, all audio latents, and text conditioning tensors
|
||||
remain shared with the frozen source. Only keyframe video latents and
|
||||
``MARK_VIDEO_CONTEXT`` reference video latents are target-sized.
|
||||
"""
|
||||
|
||||
if not isinstance(group, Mapping):
|
||||
raise RefineConditioningAdaptationError("refine group is not a mapping")
|
||||
group_id = int(group.get("group_id", -1))
|
||||
_validate_frozen_conditioning(group.get("conditioning"), group_id=group_id)
|
||||
target_h = int(target_latent_h)
|
||||
target_w = int(target_latent_w)
|
||||
if target_h <= 0 or target_w <= 0:
|
||||
raise RefineConditioningAdaptationError("target latent dimensions are invalid")
|
||||
physical_frames = int(group.get("physical_frames", 0))
|
||||
if physical_frames <= 0:
|
||||
raise RefineConditioningAdaptationError("refine group physical frames are invalid")
|
||||
|
||||
warnings: list[str] = []
|
||||
stats: dict[str, Any] = {
|
||||
"group_id": group_id,
|
||||
"target_latent_h": target_h,
|
||||
"target_latent_w": target_w,
|
||||
"keyframes_resized": 0,
|
||||
"keyframes_reencoded": 0,
|
||||
"keyframes_latent_fallback": 0,
|
||||
"context_refs_resized": 0,
|
||||
}
|
||||
output: list[list[Any]] = []
|
||||
for text_tensor, frozen_metadata in group["conditioning"]:
|
||||
metadata = dict(frozen_metadata)
|
||||
keyframes = frozen_metadata.get("minimax_keyframes")
|
||||
if keyframes is not None:
|
||||
adapted_keyframes = []
|
||||
for frozen_keyframe in keyframes:
|
||||
if not isinstance(frozen_keyframe, Mapping):
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} contains an invalid keyframe"
|
||||
)
|
||||
keyframe = dict(frozen_keyframe)
|
||||
latent = keyframe.get("latent")
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5:
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} keyframe latent is invalid"
|
||||
)
|
||||
if tuple(int(value) for value in latent.shape[-2:]) != (
|
||||
target_h,
|
||||
target_w,
|
||||
):
|
||||
frame_index = int(keyframe.get("resolved_frame_index", -1))
|
||||
image = None
|
||||
crop = "disabled"
|
||||
if frame_index == 0:
|
||||
image = group.get("first_image")
|
||||
elif frame_index == physical_frames - 1:
|
||||
image = group.get("last_image")
|
||||
crop = "center"
|
||||
if video_vae is not None and torch.is_tensor(image):
|
||||
keyframe["latent"] = _encode_keyframe_image(
|
||||
image,
|
||||
target_h=target_h,
|
||||
target_w=target_w,
|
||||
crop=crop,
|
||||
video_vae=video_vae,
|
||||
)
|
||||
stats["keyframes_reencoded"] += 1
|
||||
else:
|
||||
keyframe["latent"] = _resize_latent(
|
||||
latent,
|
||||
target_h=target_h,
|
||||
target_w=target_w,
|
||||
method=str(upscale_method),
|
||||
)
|
||||
stats["keyframes_latent_fallback"] += 1
|
||||
warning = (
|
||||
f"group {group_id} keyframe {frame_index} used latent resize "
|
||||
"because its source image and video VAE were not both available"
|
||||
)
|
||||
warnings.append(warning)
|
||||
stats["keyframes_resized"] += 1
|
||||
adapted_keyframes.append(keyframe)
|
||||
metadata["minimax_keyframes"] = adapted_keyframes
|
||||
|
||||
refs = frozen_metadata.get("minimax_refs")
|
||||
if refs is not None:
|
||||
adapted_refs = []
|
||||
refs_changed = False
|
||||
for frozen_ref in refs:
|
||||
if not isinstance(frozen_ref, Mapping):
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} contains an invalid MiniMax reference"
|
||||
)
|
||||
if bool(frozen_ref.get(MARK_VIDEO_CONTEXT)):
|
||||
latent = frozen_ref.get("latent")
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5:
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} Continuum video context is invalid"
|
||||
)
|
||||
if tuple(int(value) for value in latent.shape[-2:]) != (
|
||||
target_h,
|
||||
target_w,
|
||||
):
|
||||
adapted_ref = dict(frozen_ref)
|
||||
adapted_ref["latent"] = _resize_latent(
|
||||
latent,
|
||||
target_h=target_h,
|
||||
target_w=target_w,
|
||||
method=str(upscale_method),
|
||||
)
|
||||
adapted_ref["latent_h"] = target_h
|
||||
adapted_ref["latent_w"] = target_w
|
||||
adapted_refs.append(adapted_ref)
|
||||
refs_changed = True
|
||||
stats["context_refs_resized"] += 1
|
||||
continue
|
||||
adapted_refs.append(frozen_ref)
|
||||
if refs_changed:
|
||||
metadata["minimax_refs"] = adapted_refs
|
||||
|
||||
output.append([text_tensor, metadata])
|
||||
stats["warnings"] = tuple(warnings)
|
||||
return output, stats
|
||||
@@ -0,0 +1,158 @@
|
||||
"""V3.5 Second Pass sampling with the first-pass audio stream locked."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import extract_av_streams
|
||||
|
||||
|
||||
class RefineSamplingRuntimeError(RuntimeError):
|
||||
"""Raised when the ComfyUI sampling runtime cannot satisfy the refine contract."""
|
||||
|
||||
|
||||
def _make_basic_guider(model: Any, conditioning: list):
|
||||
try:
|
||||
import comfy.samplers
|
||||
except Exception as exc: # pragma: no cover
|
||||
raise RefineSamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc
|
||||
|
||||
class _BasicGuider(comfy.samplers.CFGGuider):
|
||||
def set_positive(self, positive):
|
||||
self.inner_set_conds({"positive": positive})
|
||||
|
||||
guider = _BasicGuider(model)
|
||||
guider.set_positive(conditioning)
|
||||
return guider
|
||||
|
||||
|
||||
def _load_runtime_modules():
|
||||
try:
|
||||
import comfy.model_management
|
||||
import comfy.nested_tensor
|
||||
import comfy.sample
|
||||
import comfy.utils
|
||||
import latent_preview
|
||||
except Exception as exc: # pragma: no cover
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"ComfyUI refine sampling runtime unavailable: {exc}"
|
||||
) from exc
|
||||
return (
|
||||
comfy.model_management,
|
||||
comfy.nested_tensor,
|
||||
comfy.sample,
|
||||
comfy.utils,
|
||||
latent_preview,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_refine_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build random video noise and a Core-expandable audio-lock mask."""
|
||||
|
||||
if video.ndim != 5 or audio.ndim != 4:
|
||||
raise RefineSamplingRuntimeError(
|
||||
"refine latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
||||
)
|
||||
if int(video.shape[0]) != int(audio.shape[0]):
|
||||
raise RefineSamplingRuntimeError("refine video/audio batch sizes differ")
|
||||
|
||||
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
||||
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
||||
raise RefineSamplingRuntimeError("ComfyUI returned invalid video noise geometry")
|
||||
|
||||
# Core prepare_noise creates CPU noise. Keep both streams on that same handoff
|
||||
# device so NestedTensor packing remains valid before Core moves it to the model.
|
||||
audio_noise = torch.zeros_like(audio, device="cpu")
|
||||
|
||||
# These masks deliberately use the smallest shapes accepted by Core reshape_mask.
|
||||
# Core expands them to the full stream geometry on the sampling device.
|
||||
video_mask = torch.ones(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.zeros(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
noise = nested_builder((video_noise, audio_noise))
|
||||
noise_mask = nested_builder((video_mask, audio_mask))
|
||||
return noise, noise_mask
|
||||
|
||||
|
||||
def sample_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Refine video while Core carries the first-pass audio as a locked AV stream."""
|
||||
|
||||
if not isinstance(latent, dict) or "samples" not in latent:
|
||||
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
|
||||
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
|
||||
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
|
||||
|
||||
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
|
||||
_load_runtime_modules()
|
||||
)
|
||||
working = latent.copy()
|
||||
latent_image = comfy_sample.fix_empty_latent_channels(
|
||||
model,
|
||||
working["samples"],
|
||||
working.get("downscale_ratio_spacial"),
|
||||
working.get("downscale_ratio_temporal"),
|
||||
)
|
||||
working["samples"] = latent_image
|
||||
video, audio = extract_av_streams(working)
|
||||
noise, noise_mask = _prepare_refine_noise_and_mask(
|
||||
video,
|
||||
audio,
|
||||
seed=int(seed),
|
||||
batch_inds=working.get("batch_index"),
|
||||
prepare_noise_fn=comfy_sample.prepare_noise,
|
||||
nested_builder=nested_tensor.NestedTensor,
|
||||
)
|
||||
guider = _make_basic_guider(model, conditioning)
|
||||
|
||||
x0_output: dict[str, Any] = {}
|
||||
callback = None
|
||||
if enable_preview:
|
||||
callback = preview.prepare_callback(
|
||||
model,
|
||||
int(sigmas.shape[-1]) - 1,
|
||||
x0_output,
|
||||
)
|
||||
disable_pbar = not comfy_utils.PROGRESS_BAR_ENABLED
|
||||
samples = guider.sample(
|
||||
noise,
|
||||
latent_image,
|
||||
sampler,
|
||||
sigmas,
|
||||
denoise_mask=noise_mask,
|
||||
callback=callback,
|
||||
disable_pbar=disable_pbar,
|
||||
seed=int(seed),
|
||||
)
|
||||
samples = samples.to(model_management.intermediate_device())
|
||||
output = working.copy()
|
||||
output.pop("downscale_ratio_spacial", None)
|
||||
output.pop("downscale_ratio_temporal", None)
|
||||
output["samples"] = samples
|
||||
extract_av_streams(output)
|
||||
return output
|
||||
@@ -0,0 +1,395 @@
|
||||
"""V3.5 Second Pass contracts and the physical-group sampling runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..model_patch import clone_model_for_chunk
|
||||
from ..state import extract_av_streams
|
||||
from ..v2.h3_builder import encode_prompt_conditioning
|
||||
from ..v2.sampling import latent_from_cpu
|
||||
from .plan import SECOND_PASS_CONTRACT_VERSION
|
||||
from .refine_context import (
|
||||
MAGIC as REFINE_CONTEXT_MAGIC,
|
||||
RefineConditioningAdaptationError,
|
||||
RefineContextError,
|
||||
adapt_group_conditioning,
|
||||
validate_refine_context,
|
||||
)
|
||||
from .refine_sampling import sample_refine_chunk
|
||||
|
||||
|
||||
class SecondPassContractError(ValueError):
|
||||
"""Raised only when a Second Pass structural contract is impossible."""
|
||||
|
||||
|
||||
def derive_refine_seed(refine_seed: int, physical_group_index: int) -> int:
|
||||
"""Derive one deterministic seed per physical group in an independent namespace."""
|
||||
if type(refine_seed) is not int or refine_seed < 0:
|
||||
raise SecondPassContractError("refine_seed must be a non-negative integer")
|
||||
if type(physical_group_index) is not int or physical_group_index < 0:
|
||||
raise SecondPassContractError("physical_group_index must be a non-negative integer")
|
||||
payload = f"h3-continuum-refine-v1:{refine_seed}:{physical_group_index}".encode("ascii")
|
||||
return int.from_bytes(hashlib.blake2b(payload, digest_size=8).digest(), "big")
|
||||
|
||||
|
||||
def _latent_samples(value: Any, *, name: str) -> torch.Tensor:
|
||||
if not isinstance(value, Mapping) or "samples" not in value:
|
||||
raise SecondPassContractError(f"{name} must be a LATENT mapping with samples")
|
||||
samples = value["samples"]
|
||||
if not isinstance(samples, torch.Tensor):
|
||||
raise SecondPassContractError(f"{name}.samples must be a tensor")
|
||||
return samples
|
||||
|
||||
|
||||
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
raise SecondPassContractError("assembly_plan must be a mapping")
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, Mapping):
|
||||
raise SecondPassContractError("assembly_plan has no second_pass_contract")
|
||||
if contract.get("version") != SECOND_PASS_CONTRACT_VERSION:
|
||||
raise SecondPassContractError("unsupported second_pass_contract version")
|
||||
groups = contract.get("physical_groups")
|
||||
if not isinstance(groups, list) or not groups or not all(isinstance(g, Mapping) for g in groups):
|
||||
raise SecondPassContractError("second_pass_contract.physical_groups must be non-empty")
|
||||
return groups
|
||||
|
||||
|
||||
def _validate_video_groups(
|
||||
video_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
) -> tuple[list[Mapping[str, Any]], tuple[int, int]]:
|
||||
groups = _contract_groups(assembly_plan)
|
||||
if not isinstance(video_latents, Sequence) or isinstance(video_latents, (str, bytes)):
|
||||
raise SecondPassContractError("video_latents must be a sequence")
|
||||
if len(video_latents) != len(groups):
|
||||
raise SecondPassContractError("video latent group count changed")
|
||||
|
||||
target_geometry: tuple[int, int] | None = None
|
||||
for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)):
|
||||
if group.get("group_id") != position:
|
||||
raise SecondPassContractError("physical group order or group_id changed")
|
||||
samples = _latent_samples(latent, name=f"video_latents[{position}]")
|
||||
if samples.ndim != 5:
|
||||
raise SecondPassContractError("video latent must have shape [B,C,T,H,W]")
|
||||
batch, channels, temporal, height, width = (int(value) for value in samples.shape)
|
||||
if channels != 24:
|
||||
raise SecondPassContractError("MiniMax H3 video latent must have 24 channels")
|
||||
if batch != int(group.get("source_batch", -1)):
|
||||
raise SecondPassContractError("video latent batch changed")
|
||||
if temporal != int(group.get("source_latent_t", -1)):
|
||||
raise SecondPassContractError("video latent temporal length changed")
|
||||
source_h = int(group.get("source_latent_h", -1))
|
||||
source_w = int(group.get("source_latent_w", -1))
|
||||
if source_h < 1 or source_w < 1 or height < source_h or width < source_w:
|
||||
raise SecondPassContractError("video latent spatial size must be preserved or enlarged")
|
||||
if target_geometry is None:
|
||||
target_geometry = (height, width)
|
||||
elif target_geometry != (height, width):
|
||||
raise SecondPassContractError("all physical groups must use the same target H/W")
|
||||
|
||||
assert target_geometry is not None
|
||||
return groups, target_geometry
|
||||
|
||||
|
||||
def validate_second_pass_inputs(
|
||||
video_latents: Sequence[Any],
|
||||
audio_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Validate only structural invariants required by a future Second Pass node."""
|
||||
groups, target_geometry = _validate_video_groups(video_latents, assembly_plan)
|
||||
if not isinstance(audio_latents, Sequence) or isinstance(audio_latents, (str, bytes)):
|
||||
raise SecondPassContractError("audio_latents must be a sequence")
|
||||
if len(audio_latents) != len(groups):
|
||||
raise SecondPassContractError("audio latent group count changed")
|
||||
|
||||
for position, (latent, group) in enumerate(zip(audio_latents, groups, strict=True)):
|
||||
samples = _latent_samples(latent, name=f"audio_latents[{position}]")
|
||||
expected_shape = group.get("source_audio_shape")
|
||||
if not isinstance(expected_shape, list) or tuple(expected_shape) != tuple(samples.shape):
|
||||
raise SecondPassContractError("audio latent shape or physical group order changed")
|
||||
|
||||
return {
|
||||
"physical_group_count": len(groups),
|
||||
"target_latent_h": target_geometry[0],
|
||||
"target_latent_w": target_geometry[1],
|
||||
"audio_passthrough": True,
|
||||
}
|
||||
|
||||
|
||||
def passthrough_audio_latents(audio_latents: Sequence[Any]) -> list[Any]:
|
||||
"""Return original LATENT objects; temporary refined audio is not adopted in V1."""
|
||||
return list(audio_latents)
|
||||
|
||||
|
||||
def update_second_pass_geometry(
|
||||
assembly_plan: Mapping[str, Any],
|
||||
video_latents: Sequence[Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Return a copied plan whose output geometry follows the upscaled physical latents."""
|
||||
groups, (target_h, target_w) = _validate_video_groups(video_latents, assembly_plan)
|
||||
scale_h: int | None = None
|
||||
scale_w: int | None = None
|
||||
for group in groups:
|
||||
source_h = int(group["source_latent_h"])
|
||||
source_w = int(group["source_latent_w"])
|
||||
source_height = int(group["source_height"])
|
||||
source_width = int(group["source_width"])
|
||||
if source_height % source_h or source_width % source_w:
|
||||
raise SecondPassContractError("source pixel/latent geometry is not integral")
|
||||
group_scale_h = source_height // source_h
|
||||
group_scale_w = source_width // source_w
|
||||
if scale_h is None:
|
||||
scale_h, scale_w = group_scale_h, group_scale_w
|
||||
elif (scale_h, scale_w) != (group_scale_h, group_scale_w):
|
||||
raise SecondPassContractError("source geometry scale differs between groups")
|
||||
|
||||
assert scale_h is not None and scale_w is not None
|
||||
target_height = target_h * scale_h
|
||||
target_width = target_w * scale_w
|
||||
updated = copy.deepcopy(dict(assembly_plan))
|
||||
updated["width"] = target_width
|
||||
updated["height"] = target_height
|
||||
contract = updated["second_pass_contract"]
|
||||
contract["target_width"] = target_width
|
||||
contract["target_height"] = target_height
|
||||
for group in contract["physical_groups"]:
|
||||
group["target_latent_h"] = target_h
|
||||
group["target_latent_w"] = target_w
|
||||
group["target_width"] = target_width
|
||||
group["target_height"] = target_height
|
||||
return updated
|
||||
|
||||
|
||||
def run_second_pass_groups(
|
||||
*,
|
||||
model: Any,
|
||||
clip: Any,
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
video_latents: Sequence[Any],
|
||||
audio_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
refine_seed: int,
|
||||
refine_context: Any = None,
|
||||
video_vae: Any = None,
|
||||
conditioning_upscale_method: str = "bilinear",
|
||||
enable_preview: bool = True,
|
||||
encode_prompt_fn=None,
|
||||
latent_builder=None,
|
||||
sample_fn=None,
|
||||
stream_extractor=None,
|
||||
clone_model_fn=None,
|
||||
validate_refine_context_fn=None,
|
||||
adapt_group_conditioning_fn=None,
|
||||
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
|
||||
"""Refine complete physical groups while preserving first-pass audio exactly."""
|
||||
|
||||
validation = validate_second_pass_inputs(video_latents, audio_latents, assembly_plan)
|
||||
groups = _contract_groups(assembly_plan)
|
||||
encode_prompt_fn = encode_prompt_fn or encode_prompt_conditioning
|
||||
latent_builder = latent_builder or latent_from_cpu
|
||||
sample_fn = sample_fn or sample_refine_chunk
|
||||
stream_extractor = stream_extractor or extract_av_streams
|
||||
clone_model_fn = clone_model_fn or clone_model_for_chunk
|
||||
validate_refine_context_fn = validate_refine_context_fn or validate_refine_context
|
||||
adapt_group_conditioning_fn = (
|
||||
adapt_group_conditioning_fn or adapt_group_conditioning
|
||||
)
|
||||
|
||||
captured_groups: Sequence[Mapping[str, Any]] | None = None
|
||||
context_warnings: list[str] = []
|
||||
if refine_context is None:
|
||||
context_warnings.append(
|
||||
"WARNING: refine_context is not connected; using legacy prompt-only "
|
||||
"conditioning."
|
||||
)
|
||||
else:
|
||||
typed_context = (
|
||||
isinstance(refine_context, Mapping)
|
||||
and refine_context.get("magic") == REFINE_CONTEXT_MAGIC
|
||||
)
|
||||
try:
|
||||
validated_context = validate_refine_context_fn(
|
||||
refine_context,
|
||||
assembly_plan=assembly_plan,
|
||||
)
|
||||
except RefineContextError as exc:
|
||||
if typed_context:
|
||||
raise SecondPassContractError(
|
||||
f"refine_context contract is invalid: {exc}"
|
||||
) from exc
|
||||
context_warnings.append(
|
||||
"WARNING: optional refine_context was not a typed Continuum context; "
|
||||
f"using legacy prompt-only conditioning ({exc})."
|
||||
)
|
||||
else:
|
||||
if bool(validated_context.get("complete")):
|
||||
captured_groups = validated_context["groups"]
|
||||
else:
|
||||
context_warnings.append(
|
||||
"WARNING: refine_context capture is incomplete; using legacy "
|
||||
"prompt-only conditioning."
|
||||
)
|
||||
|
||||
refined_videos: list[dict[str, Any]] = []
|
||||
group_seeds: list[int] = []
|
||||
conditioning_sources: list[str] = []
|
||||
report_lines = [
|
||||
"H3 Continuum Second Pass V3.5",
|
||||
"Mode: context-aware physical-group refine when a complete First Pass "
|
||||
"refine_context is available.",
|
||||
f"Physical groups: {validation['physical_group_count']}.",
|
||||
"Sampling contract: video random noise/mask=1; audio zero noise/mask=0; "
|
||||
"workflow SIGMAS are applied directly by Core.",
|
||||
*context_warnings,
|
||||
]
|
||||
|
||||
for group_index, (video_latent, audio_latent, group) in enumerate(
|
||||
zip(video_latents, audio_latents, groups, strict=True)
|
||||
):
|
||||
video_samples = _latent_samples(video_latent, name=f"video_latents[{group_index}]")
|
||||
audio_samples = _latent_samples(audio_latent, name=f"audio_latents[{group_index}]")
|
||||
captured_group = (
|
||||
captured_groups[group_index] if captured_groups is not None else None
|
||||
)
|
||||
model_group = captured_group if captured_group is not None else group
|
||||
adaptation_stats: Mapping[str, Any] | None = None
|
||||
context_conditioning_available = False
|
||||
if captured_group is not None:
|
||||
try:
|
||||
conditioning, adaptation_stats = adapt_group_conditioning_fn(
|
||||
captured_group,
|
||||
target_latent_h=validation["target_latent_h"],
|
||||
target_latent_w=validation["target_latent_w"],
|
||||
video_vae=video_vae,
|
||||
upscale_method=str(conditioning_upscale_method),
|
||||
)
|
||||
conditioning_source = "refine_context"
|
||||
context_conditioning_available = True
|
||||
except RefineConditioningAdaptationError as exc:
|
||||
report_lines.append(
|
||||
f"WARNING: group {group_index + 1} refine_context target "
|
||||
f"adaptation was unavailable; using legacy prompt-only "
|
||||
f"conditioning ({exc})."
|
||||
)
|
||||
except RefineContextError as exc:
|
||||
raise SecondPassContractError(
|
||||
f"refine_context group {group_index} is invalid: {exc}"
|
||||
) from exc
|
||||
|
||||
if not context_conditioning_available:
|
||||
prompt = str(group.get("physical_prompt", ""))
|
||||
conditioning = encode_prompt_fn(
|
||||
clip,
|
||||
prompt,
|
||||
first_image=None,
|
||||
last_image=None,
|
||||
reference_audio_assets=None,
|
||||
timeline_video_assets=None,
|
||||
)
|
||||
conditioning_source = "prompt_only_fallback"
|
||||
|
||||
logical_chunks = model_group.get("logical_chunks")
|
||||
if not isinstance(logical_chunks, Sequence) or not logical_chunks:
|
||||
logical_chunks = group.get("logical_chunks")
|
||||
if not isinstance(logical_chunks, Sequence) or not logical_chunks:
|
||||
raise SecondPassContractError(
|
||||
f"physical group {group_index} has no logical chunk identity"
|
||||
)
|
||||
physical_clip_index = int(
|
||||
model_group.get("physical_clip_index", logical_chunks[0])
|
||||
)
|
||||
context_frames = int(
|
||||
model_group.get(
|
||||
"context_frames",
|
||||
group.get("trim_prefix_frames", 0),
|
||||
)
|
||||
)
|
||||
if physical_clip_index <= 0 or context_frames < 0:
|
||||
raise SecondPassContractError(
|
||||
f"physical group {group_index} MODEL context is invalid"
|
||||
)
|
||||
chunk_model = clone_model_fn(
|
||||
model,
|
||||
strict=False,
|
||||
debug=False,
|
||||
chunk_index=physical_clip_index,
|
||||
context_frames=context_frames if context_frames > 0 else None,
|
||||
)
|
||||
physical_seed = derive_refine_seed(int(refine_seed), group_index)
|
||||
group_seeds.append(physical_seed)
|
||||
conditioning_sources.append(conditioning_source)
|
||||
nested_latent = latent_builder(video_samples, audio_samples)
|
||||
sampled = sample_fn(
|
||||
model=chunk_model,
|
||||
conditioning=conditioning,
|
||||
latent=nested_latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=physical_seed,
|
||||
enable_preview=bool(enable_preview),
|
||||
)
|
||||
refined_video, _temporary_audio = stream_extractor(sampled)
|
||||
if tuple(refined_video.shape) != tuple(video_samples.shape):
|
||||
raise SecondPassContractError(
|
||||
f"refined video group {group_index} changed B/C/T/H/W geometry"
|
||||
)
|
||||
if not bool(torch.isfinite(refined_video.float()).all().item()):
|
||||
raise SecondPassContractError(
|
||||
f"refined video group {group_index} contains NaN or Inf"
|
||||
)
|
||||
output_latent = dict(video_latent)
|
||||
output_latent["samples"] = refined_video
|
||||
refined_videos.append(output_latent)
|
||||
report_lines.append(
|
||||
"group "
|
||||
f"{group_index + 1}: logical_chunks={group.get('logical_chunks')}, "
|
||||
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
|
||||
f"conditioning_source={conditioning_source}, "
|
||||
f"physical_clip_index={physical_clip_index}, "
|
||||
f"context_frames={context_frames}, "
|
||||
f"refine_seed={physical_seed}, shape={tuple(refined_video.shape)}, "
|
||||
"sampling_passes=1, temporary_audio_discarded=true"
|
||||
)
|
||||
if adaptation_stats is not None:
|
||||
report_lines.append(
|
||||
"group "
|
||||
f"{group_index + 1} target conditioning: "
|
||||
f"keyframes_resized={int(adaptation_stats.get('keyframes_resized', 0))}, "
|
||||
f"keyframes_reencoded={int(adaptation_stats.get('keyframes_reencoded', 0))}, "
|
||||
f"context_refs_resized={int(adaptation_stats.get('context_refs_resized', 0))}"
|
||||
)
|
||||
report_lines.extend(
|
||||
f"WARNING: {warning}"
|
||||
for warning in adaptation_stats.get("warnings", ())
|
||||
)
|
||||
|
||||
passthrough_audio = passthrough_audio_latents(audio_latents)
|
||||
updated_plan = update_second_pass_geometry(assembly_plan, refined_videos)
|
||||
contract = updated_plan["second_pass_contract"]
|
||||
if conditioning_sources and all(
|
||||
source == "refine_context" for source in conditioning_sources
|
||||
):
|
||||
contract["execution"] = "context_aware_physical_groups_audio_locked_v3"
|
||||
elif any(source == "refine_context" for source in conditioning_sources):
|
||||
contract["execution"] = "mixed_context_physical_groups_audio_locked_v3"
|
||||
else:
|
||||
contract["execution"] = "t2va_physical_groups_audio_locked_v2"
|
||||
contract["conditioning_sources"] = conditioning_sources
|
||||
contract["refine_seed_base"] = int(refine_seed)
|
||||
contract["refine_group_seeds"] = group_seeds
|
||||
contract["audio_output"] = "bit_exact_first_pass_passthrough"
|
||||
contract["audio_sampling"] = "zero_noise_mask_locked"
|
||||
report_lines.append(
|
||||
"Audio output: first-pass physical audio LATENT objects returned bit-exact; "
|
||||
"temporary refined audio discarded."
|
||||
)
|
||||
return refined_videos, passthrough_audio, updated_plan, "\n".join(report_lines)
|
||||
@@ -0,0 +1,110 @@
|
||||
"""V3.5 Second Pass public node."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .second_pass import run_second_pass_groups
|
||||
|
||||
|
||||
def _single_list_input(name: str, value: Any) -> Any:
|
||||
if not isinstance(value, list) or len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
return value[0]
|
||||
|
||||
|
||||
class H3ContinuumSecondPassV35:
|
||||
"""Low-sigma refinement over complete physical decode groups."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"V3.5 context-aware Second Pass. Connect externally upscaled H3 video "
|
||||
"latents, the matching first-pass audio latents, and the original assembly plan. "
|
||||
"Connect the optional Continuum refine context to preserve the physical group's "
|
||||
"First/Last/Reference conditioning. "
|
||||
"SIGMAS controls the low-denoise refine schedule; audio output is preserved "
|
||||
"bit-exact from the first pass."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"clip": ("CLIP",),
|
||||
"sampler": ("SAMPLER",),
|
||||
"sigmas": ("SIGMAS",),
|
||||
"video_latents": ("LATENT",),
|
||||
"audio_latents": ("LATENT",),
|
||||
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
||||
"refine_seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": True,
|
||||
"tooltip": "Base seed independently derived once per physical group.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
|
||||
"video_vae": ("VAE",),
|
||||
},
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
|
||||
RETURN_NAMES = (
|
||||
"refined_video_latents",
|
||||
"audio_latents",
|
||||
"updated_assembly_plan",
|
||||
"status",
|
||||
)
|
||||
OUTPUT_IS_LIST = (True, True, False, False)
|
||||
FUNCTION = "refine"
|
||||
CATEGORY = "MiniMax H3/Continuum/Advanced"
|
||||
|
||||
def refine(
|
||||
self,
|
||||
model,
|
||||
clip,
|
||||
sampler,
|
||||
sigmas,
|
||||
video_latents,
|
||||
audio_latents,
|
||||
assembly_plan,
|
||||
refine_seed,
|
||||
refine_context=None,
|
||||
video_vae=None,
|
||||
):
|
||||
return run_second_pass_groups(
|
||||
model=_single_list_input("model", model),
|
||||
clip=_single_list_input("clip", clip),
|
||||
sampler=_single_list_input("sampler", sampler),
|
||||
sigmas=_single_list_input("sigmas", sigmas),
|
||||
video_latents=video_latents,
|
||||
audio_latents=audio_latents,
|
||||
assembly_plan=_single_list_input("assembly_plan", assembly_plan),
|
||||
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
|
||||
refine_context=(
|
||||
None
|
||||
if refine_context is None
|
||||
else _single_list_input("refine_context", refine_context)
|
||||
),
|
||||
video_vae=(
|
||||
None
|
||||
if video_vae is None
|
||||
else _single_list_input("video_vae", video_vae)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumSecondPassV35": H3ContinuumSecondPassV35,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumSecondPassV35": "H3 Continuum Second Pass V3.5",
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Small decoded-boundary patches for the V3.5 direct-write assembler.
|
||||
|
||||
The established V3.4 seam analyzer remains the source of all boundary
|
||||
decisions. This module reproduces only the correction step while avoiding the
|
||||
full physical-group clone performed by ``correct_decoded_boundaries``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from .plan import validate_assembly_plan
|
||||
from .video_seam import VideoBoundaryAnalysis, _validate_frames
|
||||
|
||||
|
||||
def build_boundary_patches(
|
||||
images: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
analyses: list[VideoBoundaryAnalysis],
|
||||
*,
|
||||
enable_exposure_ramp: bool = False,
|
||||
) -> tuple[dict[int, torch.Tensor], dict[int, str]]:
|
||||
"""Return qualified 1--4 frame corrections without changing ``images``.
|
||||
|
||||
Patch keys are the existing one-based ``boundary_index`` values. They are
|
||||
also the zero-based index of the current physical decode group, so the
|
||||
direct writer can add its natural-frame cursor without this module knowing
|
||||
the final buffer layout.
|
||||
|
||||
The result is transactional: all patches and actions are built in local
|
||||
containers and are returned only after every requested boundary succeeds.
|
||||
Any exception therefore leaves the caller with no partial correction set.
|
||||
"""
|
||||
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
groups = list(
|
||||
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
|
||||
)
|
||||
if len(images) != len(groups):
|
||||
raise ValueError("decoded image count does not match the assembly plan")
|
||||
|
||||
patches: dict[int, torch.Tensor] = {}
|
||||
actions: dict[int, str] = {}
|
||||
for analysis in analyses:
|
||||
boundary = int(analysis.boundary_index)
|
||||
correction_kind = "transient flash"
|
||||
replace_frames = int(analysis.recommended_rewind)
|
||||
if analysis.micro_flash_candidate:
|
||||
correction_kind = "micro-flash"
|
||||
replace_frames = 1
|
||||
elif enable_exposure_ramp and analysis.exposure_ramp_candidate:
|
||||
correction_kind = "exposure ramp"
|
||||
replace_frames = 4
|
||||
|
||||
if replace_frames <= 0:
|
||||
actions[boundary] = "kept native boundary"
|
||||
continue
|
||||
if analysis.scene_cut or not (
|
||||
analysis.transient_flash_candidate
|
||||
or analysis.micro_flash_candidate
|
||||
or (enable_exposure_ramp and analysis.exposure_ramp_candidate)
|
||||
):
|
||||
actions[boundary] = (
|
||||
"kept native boundary; correction guard rejected candidate"
|
||||
)
|
||||
continue
|
||||
|
||||
previous_index = boundary - 1
|
||||
current_index = boundary
|
||||
if previous_index < 0 or current_index >= len(images):
|
||||
raise ValueError("video seam boundary index is outside decoded chunks")
|
||||
|
||||
previous_raw = images[previous_index]
|
||||
current_raw = images[current_index]
|
||||
_validate_frames(
|
||||
previous_raw,
|
||||
f"decoded image chunk {previous_index + 1}",
|
||||
)
|
||||
_validate_frames(
|
||||
current_raw,
|
||||
f"decoded image chunk {current_index + 1}",
|
||||
)
|
||||
previous_plan = groups[previous_index]
|
||||
current_plan = groups[current_index]
|
||||
previous_total = int(previous_plan["total_frames"])
|
||||
previous_trim = int(previous_plan["trim_frames"])
|
||||
current_total = int(current_plan["total_frames"])
|
||||
current_trim = int(current_plan["trim_frames"])
|
||||
if current_trim + replace_frames >= current_total:
|
||||
raise ValueError("video seam correction has no retained recovery frame")
|
||||
|
||||
previous_segment = previous_raw[:previous_total][previous_trim:]
|
||||
if int(previous_segment.shape[0]) < 1:
|
||||
raise ValueError("video seam correction has no previous retained frame")
|
||||
|
||||
# Only the frames that may be changed are cloned. The recovery frame
|
||||
# and previous anchor remain read-only views of their physical groups.
|
||||
patch = current_raw[
|
||||
current_trim : current_trim + replace_frames
|
||||
].detach().to(device="cpu").clone()
|
||||
if int(patch.shape[0]) != replace_frames:
|
||||
raise ValueError("video seam correction frames are outside the current chunk")
|
||||
previous_anchor = previous_segment[-1].detach().to(
|
||||
device="cpu",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
recovery_anchor = current_raw[current_trim + replace_frames].detach().to(
|
||||
device="cpu",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape):
|
||||
raise ValueError("decoded image geometry changed at video seam")
|
||||
|
||||
for offset in range(replace_frames):
|
||||
alpha = float(offset + 1) / float(replace_frames + 1)
|
||||
original = patch[offset].to(dtype=torch.float32)
|
||||
target = torch.lerp(previous_anchor, recovery_anchor, alpha)
|
||||
original_mean = original.mean(dim=(0, 1), keepdim=True)
|
||||
target_mean = target.mean(dim=(0, 1), keepdim=True)
|
||||
normalized = (original + target_mean - original_mean).clamp(0.0, 1.0)
|
||||
patch[offset].copy_(normalized.to(dtype=patch.dtype))
|
||||
|
||||
patches[boundary] = patch
|
||||
actions[boundary] = (
|
||||
f"normalized exposure/color on {replace_frames} {correction_kind} "
|
||||
"boundary frame(s)"
|
||||
)
|
||||
|
||||
return patches, actions
|
||||
Reference in New Issue
Block a user