Release V2.1.4 UI simplification

This commit is contained in:
ukr8b3g-cmyk
2026-08-10 16:28:20 +09:00
commit 83590a3935
76 changed files with 10956 additions and 0 deletions
+5
View File
@@ -0,0 +1,5 @@
"""Integrated V2 runtime and nodes."""
from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
+462
View File
@@ -0,0 +1,462 @@
"""Deferred VAE decoding, overlap trimming, assembly, and exact-duration correction."""
from __future__ import annotations
from typing import Any
import torch
import torch.nn.functional as F
from ..constants import DIAGNOSTICS_FULL, DIAGNOSTICS_OFF, FPS
from ..media import validate_audio
from ..state import validate_plan
from .diagnostics import format_join_metrics, format_seam_decision, overlap_diagnostics
from .seam_guard import correct_audio_seam, correct_seam
from .seam_types import SeamDecision
from .session import entry_to_latent, validate_chunk_entry
def decode_video(video_vae: Any, latent: dict[str, Any]) -> torch.Tensor:
samples = latent["samples"]
if hasattr(samples, "unbind"):
video = list(samples.unbind())[0]
else:
video = samples[0]
images = video_vae.decode(video)
if images.ndim == 5:
images = images.reshape(-1, images.shape[-3], images.shape[-2], images.shape[-1])
if images.ndim != 4:
raise ValueError(f"video VAE returned unexpected shape {tuple(images.shape)}")
# Long sequences must not leave decoded frames resident on the GPU. ComfyUI
# IMAGE tensors are conventionally CPU float tensors, so transfer each chunk
# immediately and release its raw decode before moving to the next chunk.
return images.detach().to("cpu").contiguous()
def decode_audio(audio_vae: Any, latent: dict[str, Any]) -> dict[str, Any]:
samples = latent["samples"]
if hasattr(samples, "unbind"):
audio_latent = list(samples.unbind())[-1]
else:
audio_latent = samples[-1]
audio = audio_vae.decode(audio_latent).movedim(-1, 1)
std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0
std[std < 1.0] = 1.0
audio = (audio / std).detach().to("cpu").contiguous()
sample_rate = getattr(
audio_vae,
"audio_sample_rate_output",
getattr(audio_vae, "audio_sample_rate", 32000),
)
return {"waveform": audio, "sample_rate": int(sample_rate)}
def _trim_images(images: torch.Tensor, plan: dict[str, Any]) -> torch.Tensor:
plan = validate_plan(plan)
total_frames = int(plan["total_frames"])
trim_frames = int(plan["trim_frames"])
if images.shape[0] < total_frames:
raise ValueError(
f"decoded batch has {images.shape[0]} frames; expected at least {total_frames}"
)
return images[:total_frames][trim_frames:].contiguous()
def _slice_audio_for_timeline(
audio: dict[str, Any],
*,
trim_frames: int,
frame_start: int,
frame_stop: int,
) -> torch.Tensor:
"""Return one segment at its exact cumulative 24fps sample interval.
Segment-local rounding drifts by one sample on common 119-frame Continuum
chunks. Deriving every boundary from cumulative frame counts guarantees that
the assembled audio length is always ``round(total_frames / 24 * rate)``.
"""
waveform, sample_rate = validate_audio(audio)
trim_samples = int(round(float(trim_frames) / FPS * sample_rate))
target_start = int(round(float(frame_start) / FPS * sample_rate))
target_stop = int(round(float(frame_stop) / FPS * sample_rate))
wanted = target_stop - target_start
if wanted < 0:
raise ValueError("audio timeline interval is negative")
result = waveform[..., trim_samples : trim_samples + wanted]
if result.shape[-1] < wanted:
missing = wanted - int(result.shape[-1])
if result.shape[-1] > 0:
tail = result[..., -1:].expand(*result.shape[:-1], missing).clone()
result = torch.cat((result, tail), dim=-1)
else:
result = F.pad(result, (0, missing))
return result.contiguous()
def decode_sequence(
*,
entries: list[dict[str, Any]],
video_vae: Any,
audio_vae: Any,
diagnostics_full: bool,
) -> tuple[torch.Tensor, dict[str, Any], list[str]]:
"""Decode and assemble all chunks with bounded transient memory.
The final IMAGE tensor is preallocated once. Each raw VAE decode is copied
into its destination range and released before the next chunk, avoiding the
repeated full-sequence copies produced by progressive ``torch.cat``.
"""
if not entries:
raise ValueError("cannot decode an empty Continuum sequence")
checked = [validate_chunk_entry(entry) for entry in entries]
plans = [validate_plan(entry["plan"]) for entry in checked]
total_retained_frames = sum(int(plan["net_frames"]) for plan in plans)
if total_retained_frames < 1:
raise ValueError("Continuum sequence contains no retained frames")
image_buffer: torch.Tensor | None = None
audio_buffer: torch.Tensor | None = None
audio_rate: int | None = None
frame_cursor = 0
previous_segment_images: torch.Tensor | None = None
previous_segment_audio: dict[str, Any] | None = None
reports: list[str] = []
for index, (entry, plan) in enumerate(zip(checked, plans), start=1):
latent = entry_to_latent(entry)
raw_images = decode_video(video_vae, latent)
raw_audio = decode_audio(audio_vae, latent)
if diagnostics_full and previous_segment_images is not None:
metrics = overlap_diagnostics(
previous_images=previous_segment_images,
previous_audio=previous_segment_audio,
next_raw_images=raw_images,
next_raw_audio=raw_audio,
context_frames=int(plan["trim_frames"]),
)
reports.append(format_join_metrics(index, metrics))
images = _trim_images(raw_images, plan)
segment_frames = int(images.shape[0])
if segment_frames != int(plan["net_frames"]):
raise ValueError(
f"chunk {index} retained {segment_frames} frames; plan expects {plan['net_frames']}"
)
frame_stop = frame_cursor + segment_frames
waveform, sample_rate = validate_audio(raw_audio)
if audio_rate is None:
audio_rate = sample_rate
elif sample_rate != audio_rate:
raise ValueError(
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
)
segment_waveform = _slice_audio_for_timeline(
raw_audio,
trim_frames=int(plan["trim_frames"]),
frame_start=frame_cursor,
frame_stop=frame_stop,
)
if image_buffer is None:
image_buffer = torch.empty(
(total_retained_frames, *images.shape[1:]),
dtype=images.dtype,
device="cpu",
)
elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]):
raise ValueError(
f"decoded image geometry changed at chunk {index}: "
f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}"
)
image_buffer[frame_cursor:frame_stop].copy_(images)
if audio_buffer is None:
total_samples = int(round(total_retained_frames / FPS * sample_rate))
audio_buffer = torch.empty(
(*waveform.shape[:-1], total_samples),
dtype=waveform.dtype,
device="cpu",
)
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
raise ValueError("decoded audio batch/channel structure changed between chunks")
sample_start = int(round(frame_cursor / FPS * sample_rate))
sample_stop = int(round(frame_stop / FPS * sample_rate))
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
# Retain only what Full diagnostics needs for the next seam. Audio is
# small, while video is bounded to the next overlap instead of a full clip.
next_context = int(plans[index]["trim_frames"]) if index < len(plans) else 0
if next_context > 0:
previous_segment_images = images[-next_context:].contiguous()
previous_segment_audio = {
"waveform": segment_waveform,
"sample_rate": sample_rate,
}
else:
previous_segment_images = None
previous_segment_audio = None
reports.append(
f"decoded chunk {index}: {segment_frames} retained frames, "
f"cumulative {frame_stop}/{total_retained_frames}"
)
frame_cursor = frame_stop
del latent, raw_images, raw_audio, images, segment_waveform
if image_buffer is None or audio_buffer is None or audio_rate is None:
raise RuntimeError("Continuum decoder failed to allocate output buffers")
if frame_cursor != total_retained_frames:
raise RuntimeError(
f"Continuum decode cursor mismatch: {frame_cursor} != {total_retained_frames}"
)
audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate}
reports.append(
f"assembled {len(entries)} chunks into {total_retained_frames} frames with "
"cumulative sample-boundary alignment"
)
return image_buffer.contiguous(), audio, reports
def decode_sequence_with_seam(
*,
entries: list[dict[str, Any]],
video_vae: Any,
audio_vae: Any,
diagnostics_mode: str,
) -> tuple[torch.Tensor, dict[str, Any], list[str]]:
"""Decode with V2.1 Basic correction while preserving legacy output length."""
if not entries:
raise ValueError("cannot decode an empty Continuum sequence")
checked = [validate_chunk_entry(entry) for entry in entries]
plans = [validate_plan(entry["plan"]) for entry in checked]
total_retained_frames = sum(int(plan["net_frames"]) for plan in plans)
if total_retained_frames < 1:
raise ValueError("Continuum sequence contains no retained frames")
image_buffer: torch.Tensor | None = None
audio_buffer: torch.Tensor | None = None
audio_rate: int | None = None
frame_cursor = 0
reports: list[str] = []
for index, (entry, plan) in enumerate(zip(checked, plans), start=1):
latent = entry_to_latent(entry)
raw_images = decode_video(video_vae, latent)
raw_audio = decode_audio(audio_vae, latent)
waveform, sample_rate = validate_audio(raw_audio)
if audio_rate is None:
audio_rate = sample_rate
elif sample_rate != audio_rate:
raise ValueError(
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
)
legacy_frames = int(plan["net_frames"])
legacy_frame_stop = frame_cursor + legacy_frames
frame_start = frame_cursor
trim_frames = int(plan["trim_frames"])
decision: SeamDecision | None = None
video_patch: torch.Tensor | None = None
audio_patch: torch.Tensor | None = None
if index == 1:
images = _trim_images(raw_images, plan)
else:
try:
if image_buffer is None or audio_buffer is None:
raise RuntimeError("seam buffers are unavailable")
limited_raw_images = raw_images[: int(plan["total_frames"])]
images, decision, video_patch = correct_seam(
image_buffer[:frame_cursor],
limited_raw_images,
context_frames=trim_frames,
)
frame_start = frame_cursor - int(decision.cut_rewind_frames)
trim_frames -= int(decision.cut_rewind_frames)
if frame_start < 0 or trim_frames < 0:
raise ValueError("adaptive cut produced a negative boundary")
if frame_start + int(images.shape[0]) != legacy_frame_stop:
raise ValueError("adaptive cut changed the cumulative frame boundary")
sample_start = int(round(frame_start / FPS * sample_rate))
cut_sample = int(round(trim_frames / FPS * sample_rate))
(
audio_patch,
audio_metrics,
crossfade_samples,
level_gain,
dc_bias,
) = correct_audio_seam(
audio_buffer[..., :sample_start],
waveform,
sample_rate=sample_rate,
cut_sample=cut_sample,
)
decision.audio_metrics = audio_metrics
decision.audio_crossfade_samples = crossfade_samples
decision.audio_level_gain = level_gain
decision.audio_dc_bias = dc_bias
except Exception as exc:
images = _trim_images(raw_images, plan)
frame_start = frame_cursor
trim_frames = int(plan["trim_frames"])
video_patch = None
audio_patch = None
decision = SeamDecision(
fallback_reason=f"{type(exc).__name__}: {exc}",
)
segment_frames = int(images.shape[0])
frame_stop = frame_start + segment_frames
if frame_stop != legacy_frame_stop:
raise ValueError(
f"chunk {index} corrected boundary ends at {frame_stop}; "
f"expected {legacy_frame_stop}"
)
segment_waveform = _slice_audio_for_timeline(
raw_audio,
trim_frames=trim_frames,
frame_start=frame_start,
frame_stop=frame_stop,
)
if image_buffer is None:
image_buffer = torch.empty(
(total_retained_frames, *images.shape[1:]),
dtype=images.dtype,
device="cpu",
)
elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]):
raise ValueError(
f"decoded image geometry changed at chunk {index}: "
f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}"
)
if video_patch is not None and video_patch.shape[0] > 0:
patch_start = frame_start - int(video_patch.shape[0])
if patch_start < 0:
raise ValueError("video seam patch starts before the output")
image_buffer[patch_start:frame_start].copy_(video_patch)
image_buffer[frame_start:frame_stop].copy_(images)
if audio_buffer is None:
total_samples = int(round(total_retained_frames / FPS * sample_rate))
audio_buffer = torch.empty(
(*waveform.shape[:-1], total_samples),
dtype=waveform.dtype,
device="cpu",
)
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
raise ValueError("decoded audio batch/channel structure changed between chunks")
sample_start = int(round(frame_start / FPS * sample_rate))
sample_stop = int(round(frame_stop / FPS * sample_rate))
if audio_patch is not None and audio_patch.shape[-1] > 0:
patch_start = sample_start - int(audio_patch.shape[-1])
if patch_start < 0:
raise ValueError("audio seam patch starts before the output")
audio_buffer[..., patch_start:sample_start].copy_(audio_patch)
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
if decision is not None and diagnostics_mode != DIAGNOSTICS_OFF:
reports.append(
format_seam_decision(
index,
decision,
full=diagnostics_mode == DIAGNOSTICS_FULL,
)
)
reports.append(
f"decoded chunk {index}: {segment_frames} retained/corrected frames, "
f"cumulative {frame_stop}/{total_retained_frames}"
)
frame_cursor = frame_stop
del latent, raw_images, raw_audio, images, segment_waveform
if image_buffer is None or audio_buffer is None or audio_rate is None:
raise RuntimeError("Continuum seam decoder failed to allocate output buffers")
if frame_cursor != total_retained_frames:
raise RuntimeError(
f"Continuum seam decode cursor mismatch: {frame_cursor} != {total_retained_frames}"
)
audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate}
reports.append(
f"assembled {len(entries)} chunks into {total_retained_frames} frames with "
"V2.1 Basic seam correction and cumulative sample-boundary alignment"
)
return image_buffer.contiguous(), audio, reports
def enforce_total_frames(
images: torch.Tensor,
audio: dict[str, Any],
*,
target_frames: int,
preserve_final_frame: bool = False,
) -> tuple[torch.Tensor, dict[str, Any], str]:
target_frames = int(target_frames)
if target_frames < 1:
raise ValueError("target_frames must be positive")
current = int(images.shape[0])
adjustment = target_frames - current
anchor_preserved = bool(preserve_final_frame and adjustment < 0 and target_frames >= 2)
if adjustment < 0:
if anchor_preserved:
# H3's valid temporal grid rarely lands on an exact whole-second total.
# When a user supplied a final FL2VA frame, trimming only from the end
# would discard the very anchor they requested. Remove the small excess
# immediately before the final decoded frame instead.
images = torch.cat((images[: target_frames - 1], images[-1:]), dim=0).contiguous()
else:
images = images[:target_frames].contiguous()
elif adjustment > 0:
if current == 0:
raise ValueError("cannot pad an empty image sequence")
pad = images[-1:].expand(adjustment, *images.shape[1:]).clone()
images = torch.cat((images, pad), dim=0).contiguous()
waveform = audio["waveform"]
sample_rate = int(audio["sample_rate"])
original_samples = int(waveform.shape[-1])
target_samples = int(round(target_frames / FPS * sample_rate))
delta_samples = target_samples - original_samples
if delta_samples < 0:
if anchor_preserved and target_samples > 1:
# Preserve the final video-frame interval of audio as well. A short
# 5 ms interpolation at the deletion boundary suppresses a hard click
# without changing the exact output length.
tail_samples = min(
max(1, int(round(sample_rate / FPS))),
target_samples,
original_samples,
)
head_samples = target_samples - tail_samples
head = waveform[..., :head_samples]
tail = waveform[..., -tail_samples:].clone()
fade = min(int(round(sample_rate * 0.005)), head_samples, tail_samples)
if fade > 0:
start = head[..., -1:].to(dtype=tail.dtype)
alpha = torch.linspace(
0.0, 1.0, fade, dtype=tail.dtype, device=tail.device
).reshape(*([1] * (tail.ndim - 1)), fade)
tail[..., :fade] = start * (1.0 - alpha) + tail[..., :fade] * alpha
waveform = torch.cat((head, tail), dim=-1).contiguous()
else:
waveform = waveform[..., :target_samples].contiguous()
elif delta_samples > 0:
# Repeat the final sample instead of introducing a zero click.
if waveform.shape[-1] == 0:
waveform = F.pad(waveform, (0, delta_samples))
else:
tail = waveform[..., -1:].expand(*waveform.shape[:-1], delta_samples).clone()
waveform = torch.cat((waveform, tail), dim=-1).contiguous()
audio = {**audio, "waveform": waveform, "sample_rate": sample_rate}
mode = "final anchor preserved; pre-tail trim" if anchor_preserved else "tail trim/pad"
report = (
f"Exact-duration adjustment: frames {current}->{target_frames} ({adjustment:+d}), "
f"audio samples correction {delta_samples:+d}; {mode}."
)
return images, audio, report
+131
View File
@@ -0,0 +1,131 @@
"""Lightweight join diagnostics without external vision/audio models."""
from __future__ import annotations
import math
from typing import Any
import torch
from ..constants import FPS
from .seam_types import SeamDecision
def _audio_waveform(audio: dict[str, Any]) -> tuple[torch.Tensor, int]:
waveform = audio.get("waveform")
sample_rate = int(audio.get("sample_rate", 0))
if not torch.is_tensor(waveform) or waveform.ndim != 3:
raise ValueError("audio waveform must be [B,C,S]")
if sample_rate <= 0:
raise ValueError("audio sample_rate is invalid")
return waveform.to(dtype=torch.float32), sample_rate
def _normalized_correlation(a: torch.Tensor, b: torch.Tensor) -> float:
length = min(int(a.numel()), int(b.numel()))
if length < 8:
return 0.0
a = a.reshape(-1)[:length].float()
b = b.reshape(-1)[:length].float()
a = a - a.mean()
b = b - b.mean()
denom = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12)
value = float((torch.sum(a * b) / denom).item())
return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0
def overlap_diagnostics(
*,
previous_images: torch.Tensor,
previous_audio: dict[str, Any],
next_raw_images: torch.Tensor,
next_raw_audio: dict[str, Any],
context_frames: int,
) -> dict[str, float]:
context_frames = int(context_frames)
frame_count = min(
context_frames,
int(previous_images.shape[0]),
int(next_raw_images.shape[0]),
)
if frame_count <= 0:
return {
"video_overlap_mae": 0.0,
"video_overlap_psnr": 99.0,
"audio_overlap_correlation": 0.0,
"audio_boundary_jump": 0.0,
}
prev = previous_images[-frame_count:].detach().to(dtype=torch.float32)
nxt = next_raw_images[:frame_count].detach().to(dtype=torch.float32)
mse = float(torch.mean((prev - nxt).square()).item())
mae = float(torch.mean(torch.abs(prev - nxt)).item())
psnr = 99.0 if mse <= 1e-12 else max(0.0, -10.0 * math.log10(mse))
prev_wave, prev_sr = _audio_waveform(previous_audio)
next_wave, next_sr = _audio_waveform(next_raw_audio)
if prev_sr != next_sr:
raise ValueError("diagnostic audio sample rates do not match")
samples = int(round(frame_count / FPS * prev_sr))
samples = min(samples, int(prev_wave.shape[-1]), int(next_wave.shape[-1]))
if samples > 0:
correlation = _normalized_correlation(prev_wave[..., -samples:], next_wave[..., :samples])
else:
correlation = 0.0
if prev_wave.shape[-1] and next_wave.shape[-1]:
jump = float(torch.mean(torch.abs(prev_wave[..., -1] - next_wave[..., 0])).item())
else:
jump = 0.0
return {
"video_overlap_mae": mae,
"video_overlap_psnr": psnr,
"audio_overlap_correlation": correlation,
"audio_boundary_jump": jump,
}
def av_duration_error_samples(images: torch.Tensor, audio: dict[str, Any]) -> int:
waveform, sample_rate = _audio_waveform(audio)
expected = int(round(float(images.shape[0]) / FPS * sample_rate))
return int(waveform.shape[-1]) - expected
def format_join_metrics(index: int, metrics: dict[str, float]) -> str:
return (
f"join {index-1}->{index}: video MAE={metrics['video_overlap_mae']:.6f}, "
f"PSNR={metrics['video_overlap_psnr']:.2f}dB, "
f"audio corr={metrics['audio_overlap_correlation']:.4f}, "
f"boundary jump={metrics['audio_boundary_jump']:.6f}"
)
def format_seam_decision(index: int, decision: SeamDecision, *, full: bool) -> str:
fallback = decision.fallback_reason or "none"
if not full:
return (
f"seam {index-1}->{index}: cut=-{decision.cut_rewind_frames}f, "
f"score {decision.legacy_score:.4f}->{decision.corrected_score:.4f}, "
f"video blend={decision.video_blend_frames}f, "
f"audio offset={decision.audio_metrics.offset_samples:+d} samples, "
f"fallback={fallback}"
)
return (
f"Seam {index-1}->{index}\n"
f" Legacy score : {decision.legacy_score:.6f}\n"
f" Selected cut : -{decision.cut_rewind_frames} frames\n"
f" Cut score : {decision.cut_score:.6f}\n"
f" Corrected score : {decision.corrected_score:.6f}\n"
f" Improvement : {decision.improvement * 100.0:.2f}%\n"
f" Luminance gain : {decision.luma_gain:.4f}\n"
f" Chroma bias : {decision.chroma_bias}\n"
f" Video blend : {decision.video_blend_frames} frames\n"
f" Audio correlation : "
f"{decision.audio_metrics.correlation_before:.4f}"
f"->{decision.audio_metrics.correlation_after:.4f}\n"
f" Audio jump : "
f"{decision.audio_metrics.boundary_jump_before:.6f}"
f"->{decision.audio_metrics.boundary_jump_after:.6f}\n"
f" Audio offset : {decision.audio_metrics.offset_samples:+d} samples\n"
f" Audio crossfade : {decision.audio_crossfade_samples} samples\n"
f" Fallback : {fallback}"
)
+160
View File
@@ -0,0 +1,160 @@
"""Native MiniMax H3 conditioning and latent construction for V2.
All expensive image/VAE work is performed before the sampling phase. Prompt
conditioning is cached per unique prompt and image presentation.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from typing import Any
import torch
from ..continuation import clone_conditioning
from ..temporal import audio_latent_t, video_latent_t
@dataclass(slots=True)
class IdentityAssets:
first_image: torch.Tensor | None
first_latent: torch.Tensor | None
last_image: torch.Tensor | None
last_latent: torch.Tensor | None
identity_hash: str
def _tensor_fingerprint(tensor: torch.Tensor | None) -> str:
if tensor is None:
return "none"
sample = tensor.detach().to("cpu", dtype=torch.float32).contiguous()
# Hash a deterministic downsampled view rather than several gigabytes of bytes.
if sample.ndim == 4:
sample = sample[:1].movedim(-1, 1)
sample = torch.nn.functional.interpolate(sample, size=(64, 64), mode="area")
raw = sample.numpy().tobytes(order="C")
return hashlib.sha256(raw).hexdigest()
def resize_h3_image(image: torch.Tensor, width: int, height: int, crop: str) -> torch.Tensor:
try:
import comfy.utils
except Exception as exc: # pragma: no cover - ComfyUI runtime only
raise RuntimeError(f"ComfyUI resize helper unavailable: {exc}") from exc
if not torch.is_tensor(image) or image.ndim != 4 or image.shape[-1] < 3:
raise ValueError("H3 image input must be [B,H,W,C] with at least three channels")
samples = image[:1, ..., :3].movedim(-1, 1)
resized = comfy.utils.common_upscale(samples, int(width), int(height), "lanczos", crop)
return resized.movedim(1, -1).contiguous()
def prepare_identity_assets(
video_vae: Any,
*,
width: int,
height: int,
first_frame: torch.Tensor | None,
last_frame: torch.Tensor | None,
encode_latents: bool = True,
) -> IdentityAssets:
first_image = first_latent = last_image = last_latent = None
if first_frame is not None:
first_image = resize_h3_image(first_frame, width, height, "disabled")
if encode_latents:
first_latent = video_vae.encode(first_image)
if last_frame is not None:
last_image = resize_h3_image(last_frame, width, height, "center")
if encode_latents:
last_latent = video_vae.encode(last_image)
return IdentityAssets(
first_image=first_image,
first_latent=first_latent,
last_image=last_image,
last_latent=last_latent,
identity_hash=_tensor_fingerprint(first_image),
)
def encode_prompt_conditioning(
clip: Any,
prompt: str,
*,
first_image: torch.Tensor | None,
last_image: torch.Tensor | None,
) -> list:
images = []
if first_image is not None:
images.append(first_image)
if last_image is not None:
images.append(last_image)
tokens = clip.tokenize(str(prompt), images=images)
return clip.encode_from_tokens_scheduled(tokens)
def attach_keyframes(
conditioning: list,
*,
frame_count: int,
first_latent: torch.Tensor | None,
last_latent: torch.Tensor | None,
) -> list:
output = clone_conditioning(conditioning)
keyframes: list[dict[str, Any]] = []
if first_latent is not None:
keyframes.append({"resolved_frame_index": 0, "latent": first_latent})
if last_latent is not None:
keyframes.append(
{"resolved_frame_index": int(frame_count) - 1, "latent": last_latent}
)
for _, metadata in output:
if keyframes:
metadata["minimax_keyframes"] = [dict(item) for item in keyframes]
metadata["minimax_frame_count"] = int(frame_count)
else:
metadata.pop("minimax_keyframes", None)
metadata["minimax_frame_count"] = int(frame_count)
return output
def empty_h3_latent(width: int, height: int, frame_count: int) -> dict[str, Any]:
try:
import comfy.model_management
import comfy.nested_tensor
except Exception as exc: # pragma: no cover - ComfyUI runtime only
raise RuntimeError(f"ComfyUI H3 latent helpers unavailable: {exc}") from exc
width = int(width)
height = int(height)
frame_count = int(frame_count)
if width <= 0 or height <= 0 or width % 32 or height % 32:
raise ValueError("H3 width and height must be positive multiples of 32")
device = comfy.model_management.intermediate_device()
video = torch.zeros(
(1, 24, video_latent_t(frame_count), height // 16, width // 16),
device=device,
dtype=torch.float32,
)
audio = torch.zeros(
(1, 32, 2, audio_latent_t(frame_count)),
device=device,
dtype=torch.float32,
)
return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}
def encode_identity_latents(video_vae: Any, assets: IdentityAssets) -> IdentityAssets:
"""Encode already-resized identity images without changing their fingerprint."""
first_latent = assets.first_latent
last_latent = assets.last_latent
if first_latent is None and assets.first_image is not None:
first_latent = video_vae.encode(assets.first_image)
if last_latent is None and assets.last_image is not None:
last_latent = video_vae.encode(assets.last_image)
return IdentityAssets(
first_image=assets.first_image,
first_latent=first_latent,
last_image=assets.last_image,
last_latent=last_latent,
identity_hash=assets.identity_hash,
)
+53
View File
@@ -0,0 +1,53 @@
"""Conservative latent-motion analysis for automatic context selection."""
from __future__ import annotations
import math
from typing import Any
import torch
from ..constants import CONTINUITY_FRAMES, V2_CONTINUITY_AUTO
from ..state import validate_state
# Conservative defaults. Borderline values intentionally fall back to 22 frames.
_AUTO_STATIC_MAX = 0.035
_AUTO_STRONG_MIN = 0.120
def latent_motion_score(state: dict[str, Any]) -> float:
state = validate_state(state)
video = state["video_tail"].detach().to(dtype=torch.float32)
if video.shape[2] < 2:
return 0.0
temporal_delta = video[:, :, 1:] - video[:, :, :-1]
delta_rms = torch.sqrt(torch.mean(temporal_delta.square()))
signal_rms = torch.sqrt(torch.mean(video.square())).clamp_min(1e-6)
score = float((delta_rms / signal_rms).item())
if not math.isfinite(score):
return _AUTO_STRONG_MIN
return max(0.0, min(score, 1000.0))
def choose_context_frames(mode: str, state: dict[str, Any]) -> tuple[int, float, str]:
state = validate_state(state)
capacity = int(state["capacity_frames"])
score = latent_motion_score(state)
if mode != V2_CONTINUITY_AUTO:
if mode not in CONTINUITY_FRAMES:
raise ValueError(f"unknown continuity mode: {mode!r}")
requested = int(CONTINUITY_FRAMES[mode])
if requested > capacity:
raise ValueError(
f"state retains {capacity} frames but continuity requests {requested}"
)
return requested, score, "fixed profile"
if capacity < 22:
return 5, score, "state capacity below 22 frames"
if score <= _AUTO_STATIC_MAX:
return 5, score, "low latent motion"
if score >= _AUTO_STRONG_MIN and capacity >= 39:
return 39, score, "high latent motion"
return 22, score, "conservative balanced fallback"
+473
View File
@@ -0,0 +1,473 @@
"""ComfyUI node layer for H3 Continuum V2."""
from __future__ import annotations
import logging
from ..constants import (
DIAGNOSTICS_OPTIONS,
PROMPT_FORMAT_OPTIONS,
PROMPT_MODE_OPTIONS,
SEAM_CORRECTION_OFF,
SEAM_CORRECTION_OPTIONS,
V2_CONTINUITY_OPTIONS,
)
from .prompts import (
apply_prompt_overrides,
build_sampler_prompt_plan,
make_prompt_plan,
prompt_plan_report,
)
from .sequence import run_sequence
from .session import entry_to_state, session_summary, validate_session
from .session_io import (
load_session,
save_session,
session_mtime,
session_paths,
)
CATEGORY = "MiniMax H3/Continuum"
LOG = logging.getLogger("h3_continuum_join")
def _repair_v200_example_widget_shift(
*,
audio_continuity,
exact_total_duration,
diagnostics,
reroll_from_chunk,
reroll_nonce,
strict_compatibility,
debug,
):
"""Repair the malformed bundled 2.0.0 example workflow in memory.
``base_seed`` uses ComfyUI's ``control_after_generate`` option, which owns an
additional serialized widget. The original hand-authored V2 example omitted
that widget, shifting every following widget by one slot. A distinctive
signature is therefore ``exact_total_duration == 'Basic'/'Full'/'Off'`` and
numeric ``diagnostics``. Valid workflows never have a diagnostics label in
the BOOLEAN exact-duration input, so this migration is narrow and safe.
The missing exact-duration value cannot be recovered from the malformed JSON;
restore its documented V2 default (True). The remaining values can be shifted
back losslessly.
"""
if isinstance(exact_total_duration, str) and exact_total_duration in DIAGNOSTICS_OPTIONS:
if isinstance(diagnostics, (int, float, bool)) and not isinstance(diagnostics, str):
LOG.warning(
"H3 Continuum V2 detected the malformed 2.0.0 bundled workflow widget "
"layout; repairing shifted settings in memory. exact_total_duration "
"is restored to its default True. Save the workflow again after this run."
)
return (
bool(audio_continuity),
True,
exact_total_duration,
int(diagnostics),
int(reroll_from_chunk),
bool(reroll_nonce),
bool(strict_compatibility),
)
return (
bool(audio_continuity),
bool(exact_total_duration),
diagnostics,
int(reroll_from_chunk),
int(reroll_nonce),
bool(strict_compatibility),
bool(debug),
)
class H3ContinuumSamplerV2:
DESCRIPTION = (
"Integrated N-chunk MiniMax H3 continuation sampler. Precomputes prompt/image "
"conditioning, isolates accelerator runtime state per chunk, captures latent "
"state before decoding, then decodes and assembles the final AV stream."
)
SEARCH_ALIASES = [
"H3 long video sampler",
"MiniMax H3 integrated continuation",
"H3 Continuum V2",
]
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"video_vae": ("VAE",),
"audio_vae": ("VAE",),
"sampler": ("SAMPLER",),
"sigmas": ("SIGMAS",),
"prompt_mode": (
PROMPT_FORMAT_OPTIONS,
{
"default": PROMPT_FORMAT_OPTIONS[0],
"tooltip": (
"Auto detects Fixed text, --- separated List text, or "
"[0-5s] / [Chunk 1] Timeline sections."
),
},
),
"prompt_script": (
"STRING",
{
"default": "Describe one continuous MiniMax H3 shot with audio.",
"multiline": True,
"dynamicPrompts": True,
"tooltip": (
"Fixed: one prompt. List: separate chunks with a line containing ---. "
"Timeline: use [0-5s], [5-10s], ... or [Chunk 1] headers."
),
},
),
"chunks": (
"INT",
{"default": 3, "min": 1, "max": 16, "step": 1},
),
"chunk_seconds": (
"FLOAT",
{"default": 5.0, "min": 4.0, "max": 15.0, "step": 0.1, "tooltip": "Native H3 is trained for roughly 4–15 second outputs."},
),
"width": (
"INT",
{"default": 1344, "min": 32, "max": 16384, "step": 32},
),
"height": (
"INT",
{"default": 768, "min": 32, "max": 16384, "step": 32},
),
"continuity": (
V2_CONTINUITY_OPTIONS,
{
"default": V2_CONTINUITY_OPTIONS[1],
"tooltip": (
"Balanced 22f reproduces the V1 default. Auto uses a conservative "
"latent-motion score and falls back to 22f near thresholds."
),
},
),
"base_seed": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
},
),
"audio_continuity": ("BOOLEAN", {"default": True}),
"exact_total_duration": (
"BOOLEAN",
{
"default": True,
"tooltip": "Trim or minimally pad the assembled result to chunks × seconds × 24fps.",
},
),
"diagnostics": (DIAGNOSTICS_OPTIONS, {"default": DIAGNOSTICS_OPTIONS[0]}),
"reroll_from_chunk": (
"INT",
{
"default": 0,
"min": 0,
"max": 16,
"step": 1,
"tooltip": (
"0 resumes/reuses every matching accepted chunk from the optional session. "
"N preserves chunks before N and regenerates N onward."
),
},
),
"reroll_nonce": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFF,
"step": 1,
"tooltip": "Change this to derive new seeds for regenerated chunks only.",
},
),
"strict_compatibility": ("BOOLEAN", {"default": True}),
"debug": ("BOOLEAN", {"default": False}),
"seam_correction": (
SEAM_CORRECTION_OPTIONS,
{
"default": SEAM_CORRECTION_OFF,
"tooltip": (
"Off preserves the V2.0.2 decode path. Basic applies "
"conservative decoded A/V boundary correction."
),
},
),
},
"optional": {
"sequence_prompt": (
"STRING",
{
"forceInput": True,
"tooltip": (
"Connect one Text (Multiline). Prompt Format parses it for "
"the complete sequence without changing this socket."
),
},
),
"first_frame": ("IMAGE",),
"last_frame": ("IMAGE",),
"session": ("H3_CONTINUUM_SESSION",),
"initial_state": ("H3_CONTINUUM_STATE",),
"prompt_plan": ("H3_CONTINUUM_PROMPT_PLAN",),
"show_preview": (
"BOOLEAN",
{
"default": True,
"tooltip": "Controlled by Settings > H3 Continuum > Show latent preview.",
},
),
**{
f"clip_{index}_prompt": (
"STRING",
{
"forceInput": True,
"tooltip": f"Connect Text (Multiline) for Clip {index}.",
},
)
for index in range(1, 17)
},
},
}
RETURN_TYPES = (
"IMAGE",
"AUDIO",
"H3_CONTINUUM_STATE",
"H3_CONTINUUM_SESSION",
"STRING",
)
RETURN_NAMES = ("images", "audio", "last_state", "session", "report")
FUNCTION = "run"
CATEGORY = CATEGORY
def run(
self,
model,
clip,
video_vae,
audio_vae,
sampler,
sigmas,
prompt_mode,
prompt_script,
chunks,
chunk_seconds,
width,
height,
continuity,
base_seed,
audio_continuity,
exact_total_duration,
diagnostics,
reroll_from_chunk,
reroll_nonce,
strict_compatibility,
debug,
seam_correction=SEAM_CORRECTION_OFF,
first_frame=None,
last_frame=None,
session=None,
initial_state=None,
prompt_plan=None,
sequence_prompt=None,
show_preview=True,
**clip_prompt_inputs,
):
(
audio_continuity,
exact_total_duration,
diagnostics,
reroll_from_chunk,
reroll_nonce,
strict_compatibility,
debug,
) = _repair_v200_example_widget_shift(
audio_continuity=audio_continuity,
exact_total_duration=exact_total_duration,
diagnostics=diagnostics,
reroll_from_chunk=reroll_from_chunk,
reroll_nonce=reroll_nonce,
strict_compatibility=strict_compatibility,
debug=debug,
)
# V2.0.2 workflows stored show_preview in this newly inserted widget slot.
if isinstance(seam_correction, bool):
seam_correction = SEAM_CORRECTION_OFF
prompt_plan = build_sampler_prompt_plan(
prompt_mode=prompt_mode,
prompt_script=prompt_script,
sequence_prompt=sequence_prompt,
prompt_plan=prompt_plan,
chunks=int(chunks),
chunk_seconds=float(chunk_seconds),
)
prompt_plan = apply_prompt_overrides(
prompt_plan,
[
clip_prompt_inputs.get(f"clip_{index}_prompt")
for index in range(1, int(chunks) + 1)
],
)
return run_sequence(
model=model,
clip=clip,
video_vae=video_vae,
audio_vae=audio_vae,
sampler=sampler,
sigmas=sigmas,
first_frame=first_frame,
last_frame=last_frame,
prompt_plan=prompt_plan,
width=int(width),
height=int(height),
continuity=continuity,
base_seed=int(base_seed),
audio_continuity=bool(audio_continuity),
exact_total_duration=bool(exact_total_duration),
diagnostics_mode=diagnostics,
reroll_from_chunk=int(reroll_from_chunk),
reroll_nonce=int(reroll_nonce),
strict_compatibility=bool(strict_compatibility),
debug=bool(debug),
seam_correction=seam_correction,
enable_preview=bool(show_preview),
session=session,
initial_state=initial_state,
)
class H3ContinuumPromptPlanPreview:
DESCRIPTION = "Validate and preview Fixed/List/Timeline prompt parsing without sampling."
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"prompt_mode": (PROMPT_MODE_OPTIONS, {"default": PROMPT_MODE_OPTIONS[0]}),
"prompt_script": ("STRING", {"default": "Prompt", "multiline": True}),
"chunks": ("INT", {"default": 3, "min": 1, "max": 16, "step": 1}),
"chunk_seconds": (
"FLOAT",
{"default": 5.0, "min": 4.0, "max": 15.0, "step": 0.1, "tooltip": "Native H3 is trained for roughly 4–15 second outputs."},
),
}
}
RETURN_TYPES = ("H3_CONTINUUM_PROMPT_PLAN", "STRING")
RETURN_NAMES = ("prompt_plan", "report")
FUNCTION = "build"
CATEGORY = CATEGORY
def build(self, prompt_mode, prompt_script, chunks, chunk_seconds):
plan = make_prompt_plan(
mode=prompt_mode,
script=prompt_script,
chunks=int(chunks),
chunk_seconds=float(chunk_seconds),
)
report = prompt_plan_report(plan) + "\n" + "\n\n".join(
f"Chunk {index + 1}:\n{prompt}" for index, prompt in enumerate(plan["prompts"])
)
return plan, report
class H3ContinuumSaveSession:
DESCRIPTION = "Atomically save every accepted V2 chunk latent plus session metadata."
OUTPUT_NODE = True
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"session": ("H3_CONTINUUM_SESSION",),
"prefix": ("STRING", {"default": "h3_continuum_session"}),
"slot": ("INT", {"default": 1, "min": 1, "max": 9999, "step": 1}),
}
}
RETURN_TYPES = ("H3_CONTINUUM_SESSION", "STRING")
RETURN_NAMES = ("session", "path")
FUNCTION = "save"
CATEGORY = CATEGORY
def save(self, session, prefix, slot):
tensor_path, _json_path = save_session(session, prefix=prefix, slot=int(slot))
return session, str(tensor_path)
class H3ContinuumLoadSession:
DESCRIPTION = "Load an explicitly numbered V2 session for resume or reroll."
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"prefix": ("STRING", {"default": "h3_continuum_session"}),
"slot": ("INT", {"default": 1, "min": 1, "max": 9999, "step": 1}),
}
}
RETURN_TYPES = ("H3_CONTINUUM_SESSION", "STRING")
RETURN_NAMES = ("session", "path")
FUNCTION = "load"
CATEGORY = CATEGORY
@classmethod
def IS_CHANGED(cls, prefix, slot):
return session_mtime(prefix=prefix, slot=int(slot))
def load(self, prefix, slot):
session = load_session(prefix=prefix, slot=int(slot))
tensor_path, _json_path = session_paths(prefix, int(slot))
return session, str(tensor_path)
class H3ContinuumSessionInfo:
DESCRIPTION = "Inspect a V2 session and expose its final V1-compatible continuation state."
@classmethod
def INPUT_TYPES(cls):
return {"required": {"session": ("H3_CONTINUUM_SESSION",)}}
RETURN_TYPES = ("H3_CONTINUUM_SESSION", "H3_CONTINUUM_STATE", "STRING")
RETURN_NAMES = ("session", "last_state", "report")
FUNCTION = "inspect"
CATEGORY = CATEGORY
def inspect(self, session):
session = validate_session(session)
state = entry_to_state(session["chunks"][-1])
report = session_summary(session)
return session, state, report
NODE_CLASS_MAPPINGS = {
"H3ContinuumSamplerV2": H3ContinuumSamplerV2,
"H3ContinuumPromptPlanPreview": H3ContinuumPromptPlanPreview,
"H3ContinuumSaveSession": H3ContinuumSaveSession,
"H3ContinuumLoadSession": H3ContinuumLoadSession,
"H3ContinuumSessionInfo": H3ContinuumSessionInfo,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumSamplerV2": "H3 Continuum Sampler V2",
"H3ContinuumPromptPlanPreview": "H3 Continuum Prompt Plan",
"H3ContinuumSaveSession": "H3 Continuum Save Session",
"H3ContinuumLoadSession": "H3 Continuum Load Session",
"H3ContinuumSessionInfo": "H3 Continuum Session Info",
}
+331
View File
@@ -0,0 +1,331 @@
"""Prompt planning for the integrated H3 Continuum sampler.
The parser is intentionally deterministic and side-effect free so saved workflows
can be reproduced without depending on UI state.
"""
from __future__ import annotations
import hashlib
import json
import re
from typing import Any
from ..constants import (
PROMPT_FORMAT_AUTO,
PROMPT_FORMAT_FIXED,
PROMPT_FORMAT_LIST,
PROMPT_FORMAT_TIMELINE,
PROMPT_MODE_FIXED,
PROMPT_MODE_LIST,
PROMPT_MODE_TIMELINE,
PROMPT_PLAN_MAGIC,
)
from ..version import PROMPT_PLAN_SCHEMA_VERSION
_TIMELINE_HEADER = re.compile(
r"^\s*\[\s*(?P<start>\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*[-–—]\s*"
r"(?P<end>\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*\]\s*$",
re.IGNORECASE,
)
_CHUNK_HEADER = re.compile(
r"^\s*\[\s*(?:chunk|clip)\s*(?P<index>\d+)\s*\]\s*$",
re.IGNORECASE,
)
_LIST_SEPARATOR = re.compile(r"^\s*---+\s*$", re.MULTILINE)
class PromptPlanError(ValueError):
pass
def prompt_hash(prompt: str) -> str:
return hashlib.sha256(prompt.encode("utf-8")).hexdigest()
def _normalize_prompt(value: str, *, label: str) -> str:
text = str(value).strip()
if not text:
raise PromptPlanError(f"{label} is empty")
return text
def _parse_json_list(script: str) -> list[str] | None:
stripped = script.lstrip()
if not stripped.startswith("["):
return None
try:
value = json.loads(script)
except json.JSONDecodeError:
return None
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
raise PromptPlanError("JSON prompt list must be an array of strings")
return [_normalize_prompt(item, label=f"prompt {index + 1}") for index, item in enumerate(value)]
def _parse_list(script: str, chunks: int) -> tuple[list[str], list[str]]:
values = _parse_json_list(script)
if values is None:
values = [
_normalize_prompt(part, label=f"prompt {index + 1}")
for index, part in enumerate(_LIST_SEPARATOR.split(script))
if part.strip()
]
if not values:
raise PromptPlanError("prompt list contains no prompts")
notes: list[str] = []
if len(values) < chunks:
notes.append(f"repeated the last prompt for {chunks - len(values)} chunk(s)")
values.extend([values[-1]] * (chunks - len(values)))
elif len(values) > chunks:
notes.append(f"ignored {len(values) - chunks} extra prompt(s)")
values = values[:chunks]
return values, notes
def _parse_timeline_sections(script: str) -> list[dict[str, Any]]:
sections: list[dict[str, Any]] = []
current: dict[str, Any] | None = None
body: list[str] = []
def finish() -> None:
nonlocal current, body
if current is None:
if any(line.strip() for line in body):
raise PromptPlanError(
"timeline text before the first [0-5s] or [Chunk 1] header is not allowed"
)
body = []
return
current["prompt"] = _normalize_prompt("\n".join(body), label="timeline section")
sections.append(current)
current = None
body = []
for line in str(script).splitlines():
time_match = _TIMELINE_HEADER.match(line)
chunk_match = _CHUNK_HEADER.match(line)
if time_match or chunk_match:
finish()
if time_match:
start = float(time_match.group("start"))
end = float(time_match.group("end"))
if not end > start:
raise PromptPlanError(f"timeline range must increase: {line.strip()}")
current = {"kind": "time", "start": start, "end": end}
else:
index = int(chunk_match.group("index"))
if index < 1:
raise PromptPlanError("chunk numbers are one-based")
current = {"kind": "chunk", "index": index}
continue
body.append(line)
finish()
if not sections:
raise PromptPlanError("timeline contains no sections")
return sections
def _parse_timeline(script: str, chunks: int, chunk_seconds: float) -> tuple[list[str], list[str]]:
sections = _parse_timeline_sections(script)
prompts: list[str] = []
notes: list[str] = []
for chunk_index in range(1, chunks + 1):
chunk_sections = [
item for item in sections if item["kind"] == "chunk" and item["index"] == chunk_index
]
if len(chunk_sections) > 1:
raise PromptPlanError(f"timeline defines Chunk {chunk_index} more than once")
if chunk_sections:
prompts.append(chunk_sections[0]["prompt"])
continue
start = (chunk_index - 1) * chunk_seconds
end = chunk_index * chunk_seconds
scored: list[tuple[float, int, dict[str, Any]]] = []
for order, item in enumerate(sections):
if item["kind"] != "time":
continue
overlap = max(0.0, min(end, item["end"]) - max(start, item["start"]))
if overlap > 0:
scored.append((overlap, -order, item))
if not scored:
raise PromptPlanError(
f"timeline does not cover chunk {chunk_index} ({start:.3f}-{end:.3f}s)"
)
scored.sort(reverse=True, key=lambda row: (row[0], row[1]))
prompts.append(scored[0][2]["prompt"])
if len(scored) > 1 and abs(scored[0][0] - scored[1][0]) < 1e-9:
notes.append(f"chunk {chunk_index} had an equal-overlap timeline tie; used the earlier section")
return prompts, notes
def detect_prompt_mode(script: str) -> str:
"""Detect the stable Prompt Plan mode without changing the input text."""
text = str(script)
for line in text.splitlines():
if _TIMELINE_HEADER.match(line) or _CHUNK_HEADER.match(line):
return PROMPT_MODE_TIMELINE
if _parse_json_list(text) is not None or _LIST_SEPARATOR.search(text):
return PROMPT_MODE_LIST
return PROMPT_MODE_FIXED
def resolve_prompt_mode(mode: str, script: str) -> str:
explicit = {
PROMPT_FORMAT_FIXED: PROMPT_MODE_FIXED,
PROMPT_FORMAT_LIST: PROMPT_MODE_LIST,
PROMPT_FORMAT_TIMELINE: PROMPT_MODE_TIMELINE,
}
if mode == PROMPT_FORMAT_AUTO:
return detect_prompt_mode(script)
if mode in explicit:
return explicit[mode]
if mode in (PROMPT_MODE_FIXED, PROMPT_MODE_LIST, PROMPT_MODE_TIMELINE):
return mode
raise PromptPlanError(f"unknown prompt format: {mode!r}")
def make_prompt_plan(
*, mode: str, script: str, chunks: int, chunk_seconds: float,
) -> dict[str, Any]:
chunks = int(chunks)
chunk_seconds = float(chunk_seconds)
if not (1 <= chunks <= 16):
raise PromptPlanError("chunks must be between 1 and 16")
if not (4.0 <= chunk_seconds <= 15.0):
raise PromptPlanError("chunk_seconds must be between 4.0 and 15.0 for native H3")
resolved_mode = resolve_prompt_mode(mode, script)
notes: list[str] = []
if mode == PROMPT_FORMAT_AUTO:
detected = {
PROMPT_MODE_FIXED: PROMPT_FORMAT_FIXED,
PROMPT_MODE_LIST: PROMPT_FORMAT_LIST,
PROMPT_MODE_TIMELINE: PROMPT_FORMAT_TIMELINE,
}[resolved_mode]
notes.append(f"Auto detected {detected}")
if resolved_mode == PROMPT_MODE_FIXED:
prompt = _normalize_prompt(script, label="fixed prompt")
prompts = [prompt] * chunks
elif resolved_mode == PROMPT_MODE_LIST:
prompts, parse_notes = _parse_list(script, chunks)
notes.extend(parse_notes)
elif resolved_mode == PROMPT_MODE_TIMELINE:
prompts, parse_notes = _parse_timeline(script, chunks, chunk_seconds)
notes.extend(parse_notes)
else:
raise PromptPlanError(f"unknown prompt mode: {resolved_mode!r}")
return {
"magic": PROMPT_PLAN_MAGIC,
"schema_version": PROMPT_PLAN_SCHEMA_VERSION,
"mode": resolved_mode,
"chunks": chunks,
"chunk_seconds": chunk_seconds,
"prompts": prompts,
"hashes": [prompt_hash(prompt) for prompt in prompts],
"notes": notes,
}
def validate_prompt_plan(plan: dict[str, Any]) -> dict[str, Any]:
if not isinstance(plan, dict) or plan.get("magic") != PROMPT_PLAN_MAGIC:
raise PromptPlanError("invalid H3 Continuum prompt plan")
if int(plan.get("schema_version", -1)) != PROMPT_PLAN_SCHEMA_VERSION:
raise PromptPlanError(
f"unsupported prompt-plan schema {plan.get('schema_version')}; "
f"expected {PROMPT_PLAN_SCHEMA_VERSION}"
)
chunks = int(plan.get("chunks", 0))
prompts = plan.get("prompts")
hashes = plan.get("hashes")
if not (1 <= chunks <= 16):
raise PromptPlanError("prompt-plan chunks are invalid")
if not isinstance(prompts, list) or len(prompts) != chunks:
raise PromptPlanError("prompt-plan prompt count does not match chunks")
if not isinstance(hashes, list) or len(hashes) != chunks:
raise PromptPlanError("prompt-plan hash count does not match chunks")
for index, prompt in enumerate(prompts):
_normalize_prompt(prompt, label=f"prompt {index + 1}")
if hashes[index] != prompt_hash(prompt):
raise PromptPlanError(f"prompt-plan hash mismatch at chunk {index + 1}")
return plan
def build_sampler_prompt_plan(
*,
prompt_mode: str,
prompt_script: str,
sequence_prompt: str | None,
prompt_plan: dict[str, Any] | None,
chunks: int,
chunk_seconds: float,
) -> dict[str, Any]:
"""Resolve the fixed Sequence socket while preserving legacy fallbacks."""
chunks = int(chunks)
chunk_seconds = float(chunk_seconds)
if sequence_prompt is not None:
return make_prompt_plan(
mode=prompt_mode,
script=sequence_prompt,
chunks=chunks,
chunk_seconds=chunk_seconds,
)
if prompt_plan is not None:
plan = validate_prompt_plan(prompt_plan)
if int(plan["chunks"]) != chunks:
raise PromptPlanError(
"connected prompt_plan chunks do not match the Sampler chunks widget"
)
if abs(float(plan["chunk_seconds"]) - chunk_seconds) > 1e-6:
raise PromptPlanError(
"connected prompt_plan chunk_seconds do not match the Sampler widget"
)
return plan
return make_prompt_plan(
mode=prompt_mode,
script=prompt_script,
chunks=chunks,
chunk_seconds=chunk_seconds,
)
def apply_prompt_overrides(
plan: dict[str, Any], overrides: list[str | None]
) -> dict[str, Any]:
"""Replace selected chunk prompts without changing the prompt-plan schema."""
plan = validate_prompt_plan(plan)
prompts = list(plan["prompts"])
replaced: list[int] = []
for index, value in enumerate(overrides[: int(plan["chunks"])]):
if value is None:
continue
prompts[index] = _normalize_prompt(value, label=f"Clip {index + 1} Prompt")
replaced.append(index + 1)
if not replaced:
return plan
result = dict(plan)
result["mode"] = PROMPT_MODE_LIST
result["prompts"] = prompts
result["hashes"] = [prompt_hash(prompt) for prompt in prompts]
result["notes"] = list(plan.get("notes") or []) + [
"external Clip Prompt input(s): " + ", ".join(map(str, replaced))
]
return validate_prompt_plan(result)
def prompt_plan_report(plan: dict[str, Any]) -> str:
plan = validate_prompt_plan(plan)
unique = len(set(plan["hashes"]))
note = "; ".join(plan.get("notes") or [])
result = (
f"Prompt plan: {plan['mode']}; {plan['chunks']} chunks; "
f"{unique} unique prompt(s); {plan['chunk_seconds']:.3f}s per chunk."
)
return result + (" " + note + "." if note else "")
+124
View File
@@ -0,0 +1,124 @@
"""ComfyUI-native chunk sampling helpers."""
from __future__ import annotations
from typing import Any
import torch
from ..state import extract_av_streams
class SamplingRuntimeError(RuntimeError):
pass
def _make_basic_guider(model: Any, conditioning: list):
try:
import comfy.samplers
except Exception as exc: # pragma: no cover
raise SamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc
class _BasicGuider(comfy.samplers.CFGGuider):
def set_positive(self, positive):
self.inner_set_conds({"positive": positive})
guider = _BasicGuider(model)
guider.set_positive(conditioning)
return guider
def _prepare_noise(latent: dict[str, Any], seed: int):
try:
import comfy.sample
except Exception as exc: # pragma: no cover
raise SamplingRuntimeError(f"ComfyUI noise API unavailable: {exc}") from exc
batch_inds = latent.get("batch_index")
return comfy.sample.prepare_noise(latent["samples"], int(seed), batch_inds)
def sample_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Run one H3 chunk using the same path as SamplerCustomAdvanced."""
try:
import comfy.model_management
import comfy.sample
import comfy.utils
import latent_preview
except Exception as exc: # pragma: no cover
raise SamplingRuntimeError(f"ComfyUI sampling runtime unavailable: {exc}") from exc
if not isinstance(latent, dict) or "samples" not in latent:
raise SamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
raise SamplingRuntimeError("sigmas must contain at least two values")
working = latent.copy()
latent_image = working["samples"]
latent_image = comfy.sample.fix_empty_latent_channels(
model,
latent_image,
working.get("downscale_ratio_spacial"),
working.get("downscale_ratio_temporal"),
)
working["samples"] = latent_image
noise_mask = working.get("noise_mask")
guider = _make_basic_guider(model, conditioning)
noise = _prepare_noise(working, seed)
x0_output: dict[str, Any] = {}
callback = None
if enable_preview:
callback = latent_preview.prepare_callback(model, int(sigmas.shape[-1]) - 1, x0_output)
disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED
samples = guider.sample(
noise,
latent_image,
sampler,
sigmas,
denoise_mask=noise_mask,
callback=callback,
disable_pbar=disable_pbar,
seed=int(seed),
)
samples = samples.to(comfy.model_management.intermediate_device())
output = working.copy()
output.pop("downscale_ratio_spacial", None)
output.pop("downscale_ratio_temporal", None)
output["samples"] = samples
# Validate the nested AV shape immediately so a bad sampler output cannot be
# committed into a session and fail much later during continuation.
extract_av_streams(output)
return output
def latent_to_cpu(latent: dict[str, Any]) -> tuple[torch.Tensor, torch.Tensor]:
video, audio = extract_av_streams(latent)
video_cpu = video.detach().to("cpu").contiguous().clone()
audio_cpu = audio.detach().to("cpu").contiguous().clone()
if not bool(torch.isfinite(video_cpu.float()).all().item()):
raise SamplingRuntimeError("sampled video latent contains NaN or Inf")
if not bool(torch.isfinite(audio_cpu.float()).all().item()):
raise SamplingRuntimeError("sampled audio latent contains NaN or Inf")
return video_cpu, audio_cpu
def latent_from_cpu(video: torch.Tensor, audio: torch.Tensor) -> dict[str, Any]:
try:
import comfy.nested_tensor
except Exception as exc: # pragma: no cover
raise SamplingRuntimeError(f"ComfyUI NestedTensor API unavailable: {exc}") from exc
return {
"samples": comfy.nested_tensor.NestedTensor(
(video.contiguous(), audio.contiguous())
)
}
+533
View File
@@ -0,0 +1,533 @@
"""Conservative CPU-only decoded seam correction for H3 Continuum V2.1."""
from __future__ import annotations
import math
import torch
import torch.nn.functional as F
from .seam_types import AudioSeamMetrics, SeamDecision, VideoSeamMetrics
COMPARE_SIZE = 96
MAX_CUT_REWIND = 3
MIN_RELATIVE_IMPROVEMENT = 0.10
MIN_ABSOLUTE_IMPROVEMENT = 0.003
SAFE_SCORE_LIMIT = 0.35
WEIGHT_PIXEL = 0.35
WEIGHT_LUMA = 0.20
WEIGHT_CHROMA = 0.10
WEIGHT_EDGE = 0.15
WEIGHT_TEMPORAL = 0.20
def _video_shape(frames: torch.Tensor, label: str) -> None:
if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3:
raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]")
def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor:
_video_shape(frames, "video seam frames")
rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32)
rgb = rgb.permute(0, 3, 1, 2)
if tuple(rgb.shape[-2:]) != (COMPARE_SIZE, COMPARE_SIZE):
rgb = F.interpolate(
rgb,
size=(COMPARE_SIZE, COMPARE_SIZE),
mode="bilinear",
align_corners=False,
)
rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0)
if not bool(torch.isfinite(rgb).all().item()):
raise ValueError("video seam comparison contains NaN or Inf")
return rgb
def _luma(frame: torch.Tensor) -> torch.Tensor:
weights = frame.new_tensor((0.2126, 0.7152, 0.0722))
return torch.sum(frame * weights, dim=-1)
def _boundary_metrics(
previous_images: torch.Tensor,
next_images: torch.Tensor,
) -> VideoSeamMetrics:
if previous_images.shape[0] < 2 or next_images.shape[0] < 2:
raise ValueError("video seam scoring needs two frames on each side")
prev = _downsample_rgb(previous_images[-2:])
nxt = _downsample_rgb(next_images[:2])
prev_before, prev_edge = prev[0], prev[1]
next_edge, next_after = nxt[0], nxt[1]
pixel = torch.mean(torch.abs(prev_edge - next_edge)).clamp(0.0, 1.0)
prev_y = _luma(prev_edge)
next_y = _luma(next_edge)
luma = torch.mean(torch.abs(prev_y - next_y)).clamp(0.0, 1.0)
prev_chroma = prev_edge - prev_y.unsqueeze(-1)
next_chroma = next_edge - next_y.unsqueeze(-1)
chroma = (
torch.mean(torch.abs(prev_chroma - next_chroma)) / 2.0
).clamp(0.0, 1.0)
prev_gx = prev_y[:, 1:] - prev_y[:, :-1]
next_gx = next_y[:, 1:] - next_y[:, :-1]
prev_gy = prev_y[1:, :] - prev_y[:-1, :]
next_gy = next_y[1:, :] - next_y[:-1, :]
edge = (
(
torch.mean(torch.abs(prev_gx - next_gx))
+ torch.mean(torch.abs(prev_gy - next_gy))
)
/ 4.0
).clamp(0.0, 1.0)
prev_delta = prev_edge - prev_before
next_delta = next_after - next_edge
temporal = (
torch.mean(torch.abs(prev_delta - next_delta)) / 2.0
).clamp(0.0, 1.0)
motion = max(
float(torch.mean(torch.abs(prev_delta)).item()),
float(torch.mean(torch.abs(next_delta)).item()),
)
score = (
WEIGHT_PIXEL * float(pixel.item())
+ WEIGHT_LUMA * float(luma.item())
+ WEIGHT_CHROMA * float(chroma.item())
+ WEIGHT_EDGE * float(edge.item())
+ WEIGHT_TEMPORAL * float(temporal.item())
)
if not math.isfinite(score):
raise ValueError("video seam score is not finite")
return VideoSeamMetrics(
pixel_mae_norm=float(pixel.item()),
luma_norm=float(luma.item()),
chroma_norm=float(chroma.item()),
edge_norm=float(edge.item()),
temporal_norm=float(temporal.item()),
motion=motion,
score=max(0.0, min(1.0, score)),
)
def analyze_video_seam(
previous_images: torch.Tensor,
next_raw_images: torch.Tensor,
*,
context_frames: int,
rewind_frames: int = 0,
) -> VideoSeamMetrics:
context_frames = int(context_frames)
rewind_frames = int(rewind_frames)
cut_index = context_frames - rewind_frames
previous_stop = int(previous_images.shape[0]) - rewind_frames
if rewind_frames < 0 or rewind_frames > MAX_CUT_REWIND:
raise ValueError("rewind_frames is outside the V2.1 safe range")
if cut_index < 0 or previous_stop < 2 or cut_index + 2 > next_raw_images.shape[0]:
raise ValueError("insufficient frames for the requested adaptive cut")
return _boundary_metrics(
previous_images[previous_stop - 2 : previous_stop],
next_raw_images[cut_index : cut_index + 2],
)
def choose_adaptive_cut(
previous_images: torch.Tensor,
next_raw_images: torch.Tensor,
*,
context_frames: int,
) -> tuple[int, VideoSeamMetrics, VideoSeamMetrics]:
legacy = analyze_video_seam(
previous_images,
next_raw_images,
context_frames=context_frames,
rewind_frames=0,
)
best_rewind = 0
best = legacy
maximum = min(
MAX_CUT_REWIND,
max(0, int(context_frames)),
max(0, int(previous_images.shape[0]) - 2),
)
for rewind in range(1, maximum + 1):
try:
candidate = analyze_video_seam(
previous_images,
next_raw_images,
context_frames=context_frames,
rewind_frames=rewind,
)
except ValueError:
continue
if candidate.score < best.score:
best_rewind, best = rewind, candidate
absolute = legacy.score - best.score
relative = absolute / max(legacy.score, 1e-12)
accepted = (
best_rewind > 0
and relative >= MIN_RELATIVE_IMPROVEMENT
and absolute >= MIN_ABSOLUTE_IMPROVEMENT
and best.score <= SAFE_SCORE_LIMIT
)
return (best_rewind, legacy, best) if accepted else (0, legacy, legacy)
def apply_color_guard(
previous_images: torch.Tensor,
current_images: torch.Tensor,
) -> tuple[
torch.Tensor,
float,
tuple[float, float, float],
VideoSeamMetrics,
]:
before = _boundary_metrics(previous_images[-2:], current_images[:2])
count = min(4, int(current_images.shape[0]))
if count < 2:
return current_images, 1.0, (0.0, 0.0, 0.0), before
prev_rgb = _downsample_rgb(previous_images[-2:])
current_rgb = _downsample_rgb(current_images[:2])
prev_luma = float(_luma(prev_rgb).mean().item())
current_luma = float(_luma(current_rgb).mean().item())
requested_gain = prev_luma / max(current_luma, 1e-6)
if not 0.85 <= requested_gain <= 1.15:
return current_images, 1.0, (0.0, 0.0, 0.0), before
gain = max(0.94, min(1.06, requested_gain))
prev_y = _luma(prev_rgb).unsqueeze(-1)
current_y = _luma(current_rgb).unsqueeze(-1)
requested_bias = (
(prev_rgb - prev_y).mean(dim=(0, 1, 2))
- (current_rgb - current_y).mean(dim=(0, 1, 2))
)
if float(torch.max(torch.abs(requested_bias)).item()) > 0.08:
return current_images, 1.0, (0.0, 0.0, 0.0), before
bias = requested_bias.clamp(-0.02, 0.02)
result = current_images.clone().contiguous()
original = result[:count].clone()
fade = result.new_tensor((1.0, 0.65, 0.30, 0.0))[:count]
luma_weights = result.new_tensor((0.2126, 0.7152, 0.0722))
for index in range(count):
weight = fade[index]
rgb = result[index, ..., :3]
y = torch.sum(rgb * luma_weights, dim=-1, keepdim=True)
chroma = rgb - y
frame_gain = 1.0 + (gain - 1.0) * weight
frame_bias = bias.to(dtype=rgb.dtype) * weight
result[index, ..., :3] = (
y * frame_gain + chroma + frame_bias
).clamp(0.0, 1.0)
after = _boundary_metrics(previous_images[-2:], result[:2])
if after.score > before.score * 1.01:
result[:count].copy_(original)
return result, 1.0, (0.0, 0.0, 0.0), before
return (
result,
gain,
tuple(float(value) for value in bias.tolist()),
after,
)
def choose_video_blend(metrics: VideoSeamMetrics) -> int:
if metrics.motion <= 0.015:
return 2
if metrics.motion <= 0.040:
return 1
return 0
def apply_video_blend(
previous_frames: torch.Tensor,
aligned_next_frames: torch.Tensor,
*,
blend_frames: int,
) -> torch.Tensor:
blend_frames = min(
int(blend_frames),
int(previous_frames.shape[0]),
int(aligned_next_frames.shape[0]),
2,
)
if blend_frames <= 0:
return previous_frames[:0].clone()
previous = previous_frames[-blend_frames:].clone()
current = aligned_next_frames[-blend_frames:].to(dtype=previous.dtype)
for index in range(blend_frames):
progress = float(index + 1) / float(blend_frames + 1)
alpha = 0.5 - 0.5 * math.cos(math.pi * progress)
previous[index].mul_(1.0 - alpha).add_(current[index], alpha=alpha)
return previous
def correct_seam(
previous_images: torch.Tensor,
next_raw_images: torch.Tensor,
*,
context_frames: int,
) -> tuple[torch.Tensor, SeamDecision, torch.Tensor | None]:
rewind, legacy, selected = choose_adaptive_cut(
previous_images,
next_raw_images,
context_frames=context_frames,
)
cut_index = int(context_frames) - rewind
previous_stop = int(previous_images.shape[0]) - rewind
if previous_stop < 2 or cut_index < 0:
raise ValueError("adaptive cut would exhaust a seam side")
current = next_raw_images[cut_index:].clone().contiguous()
kept_previous = previous_images[:previous_stop]
current, gain, bias, corrected = apply_color_guard(kept_previous, current)
blend_frames = min(
choose_video_blend(selected),
cut_index,
previous_stop,
)
video_patch: torch.Tensor | None = None
if blend_frames > 0:
video_patch = apply_video_blend(
kept_previous[-blend_frames:],
next_raw_images[cut_index - blend_frames : cut_index],
blend_frames=blend_frames,
)
score_previous = kept_previous[-2:].clone()
replaced = min(blend_frames, 2)
score_previous[-replaced:].copy_(video_patch[-replaced:])
corrected = _boundary_metrics(score_previous, current[:2])
improvement = (legacy.score - corrected.score) / max(legacy.score, 1e-12)
if corrected.score > legacy.score * 1.01:
fallback = next_raw_images[int(context_frames) :].clone().contiguous()
return (
fallback,
SeamDecision(
legacy_score=legacy.score,
cut_score=legacy.score,
corrected_score=legacy.score,
fallback_reason="corrected score was worse than legacy",
),
None,
)
decision = SeamDecision(
cut_rewind_frames=rewind,
legacy_score=legacy.score,
cut_score=selected.score,
corrected_score=corrected.score,
improvement=improvement,
video_blend_frames=blend_frames,
luma_gain=gain,
chroma_bias=bias,
)
return current, decision, video_patch
def _audio_mid(waveform: torch.Tensor) -> torch.Tensor:
if not torch.is_tensor(waveform) or waveform.ndim != 3:
raise ValueError("audio seam waveform must be [B,C,S]")
value = waveform.detach().to(device="cpu", dtype=torch.float32)
if not bool(torch.isfinite(value).all().item()):
raise ValueError("audio seam waveform contains NaN or Inf")
return value.mean(dim=(0, 1))
def _correlation(a: torch.Tensor, b: torch.Tensor) -> float:
length = min(int(a.numel()), int(b.numel()))
if length < 8:
return 0.0
a = a[:length] - a[:length].mean()
b = b[:length] - b[:length].mean()
denominator = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12)
value = float((torch.sum(a * b) / denominator).item())
return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0
def analyze_audio_seam(
previous_waveform: torch.Tensor,
next_raw_waveform: torch.Tensor,
*,
sample_rate: int,
cut_sample: int,
) -> AudioSeamMetrics:
sample_rate = int(sample_rate)
cut_sample = int(cut_sample)
previous = _audio_mid(previous_waveform)
current = _audio_mid(next_raw_waveform)
max_lag = max(1, int(round(sample_rate * 0.020)))
window = min(
max(8, int(round(sample_rate * 0.080))),
int(previous.shape[-1]),
max(0, cut_sample - max_lag),
)
if sample_rate <= 0 or window < 8:
raise ValueError("insufficient audio for seam alignment")
previous_tail = previous[-window:]
def correlation_at(lag: int) -> float | None:
stop = cut_sample + int(lag)
start = stop - window
if start < 0 or stop > current.shape[-1]:
return None
return _correlation(previous_tail, current[start:stop])
baseline = correlation_at(0)
if baseline is None:
raise ValueError("audio seam cut is outside the decoded waveform")
best_lag, best_correlation = 0, baseline
for lag in range(-max_lag, max_lag + 1):
correlation = correlation_at(lag)
if correlation is not None and correlation > best_correlation:
best_lag, best_correlation = lag, correlation
if best_correlation < 0.20 or best_correlation - baseline < 0.02:
best_lag, best_correlation = 0, baseline
else:
center = max(0, min(int(current.shape[-1]) - 1, cut_sample + best_lag))
radius = max(1, int(round(sample_rate * 0.001)))
start = max(0, center - radius)
stop = min(int(current.shape[-1]), center + radius + 1)
if stop > start:
zero = int(torch.argmin(torch.abs(current[start:stop])).item()) + start
best_lag = zero - cut_sample
before_jump = float(
torch.mean(
torch.abs(previous_waveform[..., -1] - next_raw_waveform[..., cut_sample])
).item()
)
return AudioSeamMetrics(
correlation_before=baseline,
correlation_after=best_correlation,
boundary_jump_before=before_jump,
boundary_jump_after=before_jump,
offset_samples=int(best_lag),
)
def _audio_crossfade_ms(
previous_waveform: torch.Tensor,
next_raw_waveform: torch.Tensor,
*,
cut_sample: int,
sample_rate: int,
) -> int:
probe = max(8, int(round(sample_rate * 0.020)))
if previous_waveform.shape[-1] < probe or cut_sample < probe:
return 10
previous = _audio_mid(previous_waveform)[-probe:]
current = _audio_mid(next_raw_waveform)[cut_sample - probe : cut_sample]
rms = torch.sqrt(torch.mean(torch.cat((previous, current)).square())).clamp_min(1e-6)
transient = max(
float(torch.mean(torch.abs(torch.diff(previous))).item()),
float(torch.mean(torch.abs(torch.diff(current))).item()),
) / float(rms.item())
if transient > 0.80:
return 10
if transient > 0.35:
return 20
if transient > 0.15:
return 40
return 60
def apply_audio_seam(
previous_waveform: torch.Tensor,
next_raw_waveform: torch.Tensor,
metrics: AudioSeamMetrics,
*,
sample_rate: int,
cut_sample: int,
) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]:
if metrics.correlation_after < 0.20:
return None, metrics, 0, 1.0, 0.0
fade_ms = _audio_crossfade_ms(
previous_waveform,
next_raw_waveform,
cut_sample=cut_sample,
sample_rate=sample_rate,
)
fade_samples = min(
int(round(sample_rate * fade_ms / 1000.0)),
int(previous_waveform.shape[-1]),
int(cut_sample),
)
if fade_samples < 2:
return None, metrics, 0, 1.0, 0.0
progress = torch.linspace(
0.0,
1.0,
fade_samples,
dtype=torch.float32,
device="cpu",
)
base_indices = torch.arange(cut_sample - fade_samples, cut_sample, dtype=torch.long)
offsets = torch.round(
float(metrics.offset_samples) * (1.0 - progress)
).to(dtype=torch.long)
indices = (base_indices + offsets).clamp(0, int(next_raw_waveform.shape[-1]) - 1)
current = next_raw_waveform.detach().to("cpu").index_select(-1, indices)
previous = previous_waveform[..., -fade_samples:].detach().to("cpu")
previous_rms = torch.sqrt(torch.mean(previous.float().square())).clamp_min(1e-6)
current_rms = torch.sqrt(torch.mean(current.float().square())).clamp_min(1e-6)
requested_db = 20.0 * math.log10(float((previous_rms / current_rms).item()))
level_db = max(-1.5, min(1.5, requested_db)) if abs(requested_db) <= 6.0 else 0.0
level_gain = 10.0 ** (level_db / 20.0)
requested_dc = float((previous.float().mean() - current.float().mean()).item())
dc_bias = max(-0.02, min(0.02, requested_dc)) if abs(requested_dc) <= 0.10 else 0.0
current = current * level_gain + dc_bias
shape = [1] * (previous.ndim - 1) + [fade_samples]
fade_in = torch.sin(progress * (math.pi / 2.0)).reshape(shape)
fade_out = torch.cos(progress * (math.pi / 2.0)).reshape(shape)
patch = previous * fade_out + current.to(dtype=previous.dtype) * fade_in
input_peak = max(
float(previous.abs().max().item()),
float(current.abs().max().item()),
1e-6,
)
patch_peak = float(patch.abs().max().item())
if patch_peak > input_peak * 1.05:
patch = patch * ((input_peak * 1.05) / patch_peak)
after_jump = float(
torch.mean(torch.abs(patch[..., -1] - next_raw_waveform[..., cut_sample])).item()
)
updated = AudioSeamMetrics(
correlation_before=metrics.correlation_before,
correlation_after=metrics.correlation_after,
boundary_jump_before=metrics.boundary_jump_before,
boundary_jump_after=after_jump,
offset_samples=metrics.offset_samples,
)
return patch.contiguous(), updated, fade_samples, level_gain, dc_bias
def correct_audio_seam(
previous_waveform: torch.Tensor,
next_raw_waveform: torch.Tensor,
*,
sample_rate: int,
cut_sample: int,
) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]:
metrics = analyze_audio_seam(
previous_waveform,
next_raw_waveform,
sample_rate=sample_rate,
cut_sample=cut_sample,
)
return apply_audio_seam(
previous_waveform,
next_raw_waveform,
metrics,
sample_rate=sample_rate,
cut_sample=cut_sample,
)
+42
View File
@@ -0,0 +1,42 @@
"""Transient types used by V2.1 decoded seam correction."""
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class VideoSeamMetrics:
pixel_mae_norm: float
luma_norm: float
chroma_norm: float
edge_norm: float
temporal_norm: float
motion: float
score: float
@dataclass(frozen=True)
class AudioSeamMetrics:
correlation_before: float = 0.0
correlation_after: float = 0.0
boundary_jump_before: float = 0.0
boundary_jump_after: float = 0.0
offset_samples: int = 0
@dataclass
class SeamDecision:
cut_rewind_frames: int = 0
legacy_score: float = 0.0
cut_score: float = 0.0
corrected_score: float = 0.0
improvement: float = 0.0
video_blend_frames: int = 0
luma_gain: float = 1.0
chroma_bias: tuple[float, float, float] = (0.0, 0.0, 0.0)
audio_metrics: AudioSeamMetrics = AudioSeamMetrics()
audio_crossfade_samples: int = 0
audio_level_gain: float = 1.0
audio_dc_bias: float = 0.0
fallback_reason: str | None = None
+36
View File
@@ -0,0 +1,36 @@
"""Deterministic per-chunk seed derivation."""
from __future__ import annotations
_MASK64 = (1 << 64) - 1
_GOLDEN = 0x9E3779B97F4A7C15
_MIX_A = 0xBF58476D1CE4E5B9
_MIX_B = 0x94D049BB133111EB
def _splitmix64(value: int) -> int:
value = (value + _GOLDEN) & _MASK64
value = ((value ^ (value >> 30)) * _MIX_A) & _MASK64
value = ((value ^ (value >> 27)) * _MIX_B) & _MASK64
return (value ^ (value >> 31)) & _MASK64
def derive_chunk_seed(base_seed: int, chunk_index: int, reroll_nonce: int = 0) -> int:
"""Return a stable 64-bit seed for a zero-based chunk index.
The index and reroll nonce are mixed independently so rerolling chunk N does
not alter accepted chunks before N.
"""
if isinstance(base_seed, bool) or isinstance(chunk_index, bool) or isinstance(reroll_nonce, bool):
raise TypeError("seed inputs must be integers, not booleans")
chunk_index = int(chunk_index)
reroll_nonce = int(reroll_nonce)
if chunk_index < 0:
raise ValueError("chunk_index must be non-negative")
if reroll_nonce < 0:
raise ValueError("reroll_nonce must be non-negative")
value = int(base_seed) & _MASK64
value ^= _splitmix64(chunk_index + 1)
value ^= _splitmix64((reroll_nonce + 1) * 0xD6E8FEB86659FD93)
return _splitmix64(value)
+540
View File
@@ -0,0 +1,540 @@
"""Integrated N-chunk Continuum runtime.
V2 deliberately reuses the proven V1 continuation core. It changes execution
orchestration only: all text/image conditioning is prepared first, H3 sampling
runs for every chunk without intermediate VAE decoding, and decoding/assembly
happens after the sampling phase.
"""
from __future__ import annotations
import copy
import logging
from typing import Any
import torch
from ..compatibility import accelerator_summary, check_comfy_h3_runtime
from ..constants import (
DIAGNOSTICS_FULL,
DIAGNOSTICS_OFF,
DIAGNOSTICS_OPTIONS,
FPS,
CONTINUUM_ACTUAL_PREFIX_STEPS,
SEAM_CORRECTION_OFF,
SEAM_CORRECTION_OPTIONS,
V2_CONTINUITY_AUTO,
)
from ..continuation import POLICY_REPLACE, prepare_conditioning
from ..model_patch import clone_model_for_chunk, patch_model
from ..state import (
assert_context_unchanged,
context_fingerprint,
make_plan,
select_context,
validate_state,
)
from ..temporal import (
align_frame_count_up,
largest_context_capacity,
make_extension_shape,
)
from ..version import PACKAGE_VERSION
from .decoder import decode_sequence, decode_sequence_with_seam, enforce_total_frames
from .h3_builder import (
attach_keyframes,
empty_h3_latent,
encode_identity_latents,
encode_prompt_conditioning,
prepare_identity_assets,
)
from .motion import choose_context_frames
from .prompts import prompt_plan_report, validate_prompt_plan
from .sampling import sample_chunk
from .seeds import derive_chunk_seed
from .session import (
SessionValidationError,
entry_to_state,
make_chunk_entry,
make_session,
model_fingerprint,
session_summary,
validate_chunk_entry,
validate_session,
)
LOG = logging.getLogger("h3_continuum_join")
class SequenceRuntimeError(RuntimeError):
pass
def _check_decode_memory_budget(
*, width: int, height: int, chunks: int, chunk_seconds: float
) -> tuple[float, float | None]:
"""Fail before sampling when the eventual CPU IMAGE output is unsafe.
ComfyUI IMAGE tensors are normally float32. V2 preallocates the final output
once, but one raw VAE chunk and normal process headroom still coexist with it.
The estimate is conservative and intentionally protects long/high-resolution
jobs from failing only after every expensive H3 chunk has completed.
"""
target_frames = max(1, int(round(chunks * chunk_seconds * FPS)))
max_chunk_frames = max(5, int(round(chunk_seconds * FPS)) + 39)
image_bytes = target_frames * width * height * 3 * 4
transient_bytes = max_chunk_frames * width * height * 3 * 4
required_bytes = image_bytes + int(transient_bytes * 1.5) + 1 * 1024**3
estimate_gib = required_bytes / 1024**3
available_gib: float | None = None
try:
import psutil
available = int(psutil.virtual_memory().available)
available_gib = available / 1024**3
if required_bytes > int(available * 0.75):
raise SequenceRuntimeError(
"estimated decoded output requires about "
f"{estimate_gib:.1f} GiB including one-chunk headroom, but only "
f"{available_gib:.1f} GiB RAM is currently available. Reduce chunks, "
"chunk_seconds, or resolution before sampling."
)
except ImportError:
pass
return estimate_gib, available_gib
def _clone_entry_for_reuse(entry: dict[str, Any]) -> dict[str, Any]:
entry = validate_chunk_entry(entry)
result = dict(entry)
result["plan"] = copy.deepcopy(entry["plan"])
result["reused"] = True
return result
def _preserved_prefix(
*,
session: dict[str, Any] | None,
prompt_hashes: list[str],
chunks: int,
reroll_from_chunk: int,
width: int,
height: int,
chunk_seconds: float,
identity_hash: str,
) -> tuple[list[dict[str, Any]], list[str]]:
if session is None:
return [], []
session = validate_session(session)
notes: list[str] = []
if int(session["width"]) != int(width) or int(session["height"]) != int(height):
raise SessionValidationError(
f"session is {session['width']}x{session['height']} but V2 is configured for {width}x{height}"
)
if abs(float(session["chunk_seconds"]) - float(chunk_seconds)) > 1e-6:
raise SessionValidationError(
f"session chunk_seconds={session['chunk_seconds']} does not match {chunk_seconds}"
)
if str(session.get("identity_hash", "none")) != str(identity_hash):
raise SessionValidationError(
"the current first-frame identity image does not match the saved session"
)
if reroll_from_chunk < 0 or reroll_from_chunk > chunks:
raise SessionValidationError("reroll_from_chunk must be 0 or a valid one-based chunk index")
limit = min(len(session["chunks"]), chunks)
if reroll_from_chunk > 0:
limit = min(limit, reroll_from_chunk - 1)
preserved: list[dict[str, Any]] = []
for index in range(limit):
entry = validate_chunk_entry(session["chunks"][index])
if entry["prompt_hash"] != prompt_hashes[index]:
notes.append(
f"session reuse stopped before chunk {index + 1}: prompt changed"
)
break
preserved.append(_clone_entry_for_reuse(entry))
if reroll_from_chunk == 0 and len(preserved) == min(len(session["chunks"]), chunks):
if len(session["chunks"]) < chunks:
notes.append(f"resuming after accepted chunk {len(session['chunks'])}")
else:
notes.append("all requested chunks reused from the session")
elif reroll_from_chunk > 0:
notes.append(
f"preserved chunks 1-{len(preserved)}; regenerated from chunk {reroll_from_chunk}"
)
return preserved, notes
def _conditioning_cache(
*,
clip: Any,
prompts: list[str],
assets: Any,
final_has_last_frame: bool,
) -> dict[tuple[str, bool], list]:
cache: dict[tuple[str, bool], list] = {}
final_index = len(prompts) - 1
for index, prompt in enumerate(prompts):
include_last = bool(final_has_last_frame and index == final_index)
key = (prompt, include_last)
if key in cache:
continue
cache[key] = encode_prompt_conditioning(
clip,
prompt,
first_image=assets.first_image,
last_image=assets.last_image if include_last else None,
)
return cache
def run_sequence(
*,
model: Any,
clip: Any,
video_vae: Any,
audio_vae: Any,
sampler: Any,
sigmas: torch.Tensor,
first_frame: torch.Tensor | None,
last_frame: torch.Tensor | None,
prompt_plan: dict[str, Any],
width: int,
height: int,
continuity: str,
base_seed: int,
audio_continuity: bool,
exact_total_duration: bool,
diagnostics_mode: str,
reroll_from_chunk: int,
reroll_nonce: int,
strict_compatibility: bool,
debug: bool,
seam_correction: str = SEAM_CORRECTION_OFF,
enable_preview: bool = True,
session: dict[str, Any] | None = None,
initial_state: dict[str, Any] | None = None,
) -> tuple[torch.Tensor, dict[str, Any], dict[str, Any], dict[str, Any], str]:
"""Generate or resume a complete Continuum sequence."""
plan = validate_prompt_plan(prompt_plan)
chunks = int(plan["chunks"])
chunk_seconds = float(plan["chunk_seconds"])
prompts = list(plan["prompts"])
prompt_hashes = list(plan["hashes"])
width, height = int(width), int(height)
if width <= 0 or height <= 0 or width % 32 or height % 32:
raise SequenceRuntimeError("width and height must be positive multiples of 32")
if session is not None and initial_state is not None:
raise SequenceRuntimeError("connect either a session or an initial_state, not both")
if diagnostics_mode not in DIAGNOSTICS_OPTIONS:
raise SequenceRuntimeError(f"unknown diagnostics mode: {diagnostics_mode!r}")
if seam_correction not in SEAM_CORRECTION_OPTIONS:
raise SequenceRuntimeError(f"unknown seam correction mode: {seam_correction!r}")
if not (0 <= int(reroll_from_chunk) <= chunks):
raise SequenceRuntimeError(
"reroll_from_chunk must be 0 or a valid one-based chunk index"
)
if initial_state is not None and int(reroll_from_chunk) not in (0, 1):
raise SequenceRuntimeError(
"with initial_state, reroll_from_chunk can only be 0 or 1"
)
decode_estimate_gib, available_ram_gib = _check_decode_memory_budget(
width=width,
height=height,
chunks=chunks,
chunk_seconds=chunk_seconds,
)
issues = check_comfy_h3_runtime()
if issues and strict_compatibility:
raise SequenceRuntimeError("H3 runtime is incompatible: " + "; ".join(issues))
# Phase 1: all CLIP and VAE encoding happens before H3 sampling. This is the
# key residency optimization: no text encoder or VAE is requested between
# H3 chunks.
assets = prepare_identity_assets(
video_vae,
width=width,
height=height,
first_frame=first_frame,
last_frame=last_frame,
encode_latents=False,
)
patched_model = patch_model(
model,
strict=bool(strict_compatibility),
debug=bool(debug),
)
current_model_fingerprint = model_fingerprint(patched_model)
preserved, reuse_notes = _preserved_prefix(
session=session,
prompt_hashes=prompt_hashes,
chunks=chunks,
reroll_from_chunk=int(reroll_from_chunk),
width=width,
height=height,
chunk_seconds=chunk_seconds,
identity_hash=assets.identity_hash,
)
if session is not None:
old_fingerprint = str(session.get("model_fingerprint", ""))
if old_fingerprint and old_fingerprint != current_model_fingerprint:
reuse_notes.append(
"model/accelerator fingerprint differs from the saved session; accepted chunks were kept"
)
cache: dict[tuple[str, bool], list] = {}
if len(preserved) < chunks:
assets = encode_identity_latents(video_vae, assets)
cache = _conditioning_cache(
clip=clip,
prompts=prompts,
assets=assets,
final_has_last_frame=last_frame is not None,
)
entries: list[dict[str, Any]] = preserved[:]
previous_state = None
if entries:
previous_state = entry_to_state(entries[-1])
elif initial_state is not None:
previous_state = validate_state(initial_state)
if int(previous_state["width"]) != width or int(previous_state["height"]) != height:
raise SequenceRuntimeError(
f"initial_state is {previous_state['width']}x{previous_state['height']} "
f"but V2 is configured for {width}x{height}"
)
initial_frame_count = align_frame_count_up(int(round(chunk_seconds * FPS)))
retained_frames = sum(int(entry["plan"]["net_frames"]) for entry in entries)
sampling_reports: list[str] = []
# Phase 2: H3-only loop. Each chunk gets a MODEL clone so accelerator runtime
# history cannot leak across chunks. Sampler/sigmas remain shared, and decoded
# media is deliberately not requested until every chunk is done.
for sequence_index in range(len(entries), chunks):
prompt = prompts[sequence_index]
prompt_hash_value = prompt_hashes[sequence_index]
is_final = sequence_index == chunks - 1
seed = derive_chunk_seed(base_seed, sequence_index, reroll_nonce)
motion_score = 0.0
video_context = None
audio_context = None
context_before = None
if previous_state is None:
total_frames = initial_frame_count
latent = empty_h3_latent(width, height, total_frames)
conditioning = attach_keyframes(
cache[(prompt, bool(last_frame is not None and is_final))],
frame_count=total_frames,
first_latent=assets.first_latent,
last_latent=assets.last_latent if is_final else None,
)
clip_index = 1
context_frames = 0
chunk_plan = make_plan(
continuation=False,
clip_index=clip_index,
total_frames=total_frames,
trim_frames=0,
width=width,
height=height,
context_frames=5,
state_capacity_frames=largest_context_capacity(total_frames),
requested_extend_seconds=chunk_seconds,
debug=debug,
)
reason = "initial clip"
else:
context_frames, motion_score, reason = choose_context_frames(
continuity, previous_state
)
desired_cumulative = int(round((sequence_index + 1) * chunk_seconds * FPS))
requested_new_frames = max(1, desired_cumulative - retained_frames)
shape = make_extension_shape(
context_frames, requested_new_frames / FPS
)
latent = empty_h3_latent(width, height, shape.total_frames)
base_conditioning = attach_keyframes(
cache[(prompt, bool(last_frame is not None and is_final))],
frame_count=shape.total_frames,
first_latent=assets.first_latent,
last_latent=assets.last_latent if is_final else None,
)
video_context, audio_context, grid_offset = select_context(
previous_state,
context_frames,
include_audio=bool(audio_continuity),
)
context_before = context_fingerprint(video_context, audio_context)
conditioning = prepare_conditioning(
base_conditioning,
video_context=video_context,
audio_context=audio_context,
audio_grid_offset=grid_offset,
context_frames=context_frames,
new_frame_count=shape.total_frames,
first_frame_policy=POLICY_REPLACE,
preserve_last_frame=True,
)
clip_index = int(previous_state["clip_index"]) + 1
chunk_plan = make_plan(
continuation=True,
clip_index=clip_index,
total_frames=shape.total_frames,
trim_frames=context_frames,
width=width,
height=height,
context_frames=context_frames,
state_capacity_frames=largest_context_capacity(shape.net_new_frames),
requested_extend_seconds=chunk_seconds,
debug=debug,
)
chunk_model = clone_model_for_chunk(
patched_model,
strict=bool(strict_compatibility),
debug=bool(debug),
chunk_index=clip_index,
context_frames=context_frames if previous_state is not None else None,
)
sampled = sample_chunk(
model=chunk_model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=seed,
enable_preview=bool(enable_preview),
)
if context_before is not None and video_context is not None:
assert_context_unchanged(
video_context,
audio_context,
context_before,
)
# Commit the full chunk to CPU once, then derive the small next-state tail
# from that CPU entry. This avoids a second device-to-host transfer.
entry = make_chunk_entry(
latent=sampled,
plan=chunk_plan,
prompt=prompt,
prompt_hash=prompt_hash_value,
seed=seed,
context_frames=context_frames,
motion_score=motion_score,
reused=False,
)
previous_state = entry_to_state(entry)
entries.append(entry)
retained_frames += int(chunk_plan["net_frames"])
sampling_reports.append(
f"chunk {sequence_index + 1}/{chunks}: seed={seed}, "
f"frames={chunk_plan['total_frames']}, trim={chunk_plan['trim_frames']}, "
f"retained_total={retained_frames}, context={context_frames} "
f"({reason}), motion={motion_score:.6f}, "
+ (
f"interop=emitted actual_prefix={CONTINUUM_ACTUAL_PREFIX_STEPS} "
"consumer=not_observable"
if context_before is not None
else "interop=not_emitted"
)
)
del sampled, latent, conditioning, chunk_model
if len(entries) != chunks:
raise SequenceRuntimeError(
f"internal sequence length mismatch: expected {chunks}, got {len(entries)}"
)
# Phase 3: VAE decode only after all H3 sampling is complete.
if seam_correction == SEAM_CORRECTION_OFF:
images, audio, decode_reports = decode_sequence(
entries=entries,
video_vae=video_vae,
audio_vae=audio_vae,
diagnostics_full=diagnostics_mode == DIAGNOSTICS_FULL,
)
else:
images, audio, decode_reports = decode_sequence_with_seam(
entries=entries,
video_vae=video_vae,
audio_vae=audio_vae,
diagnostics_mode=diagnostics_mode,
)
duration_report = ""
if exact_total_duration:
target_frames = int(round(chunks * chunk_seconds * FPS))
images, audio, duration_report = enforce_total_frames(
images,
audio,
target_frames=target_frames,
preserve_final_frame=last_frame is not None,
)
last_state = entry_to_state(entries[-1])
parent_id = session.get("session_id") if session is not None else None
settings = {
"continuity": continuity,
"audio_continuity": bool(audio_continuity),
"exact_total_duration": bool(exact_total_duration),
"prompt_mode": plan["mode"],
"base_seed": int(base_seed),
"reroll_nonce": int(reroll_nonce),
"diagnostics_mode": diagnostics_mode,
"initial_state_source": initial_state is not None,
}
new_session = make_session(
chunks=entries,
width=width,
height=height,
chunk_seconds=chunk_seconds,
identity_hash=assets.identity_hash,
model_fingerprint_value=current_model_fingerprint,
parent_session_id=parent_id,
reroll_from_chunk=int(reroll_from_chunk),
settings=settings,
)
decoded_gib = (
float(images.shape[0]) * float(width) * float(height) * 3.0 * 4.0 / (1024.0 ** 3)
)
report_lines = [
f"H3 Continuum V2 {PACKAGE_VERSION}",
prompt_plan_report(plan),
f"Seam correction: {seam_correction}.",
accelerator_summary(patched_model),
"Execution: conditioning precomputed; call-local MODEL clone per chunk; decode deferred until sampling completed.",
*reuse_notes,
]
if diagnostics_mode != DIAGNOSTICS_OFF:
report_lines.extend(
[
(
f"Decode RAM estimate: {decode_estimate_gib:.2f} GiB including transient headroom"
+ (
f"; available at start {available_ram_gib:.2f} GiB."
if available_ram_gib is not None
else "."
)
),
*sampling_reports,
*decode_reports,
]
)
if duration_report:
report_lines.append(duration_report)
report_lines.extend(
[
session_summary(new_session),
f"Output: {images.shape[0]} frames ({images.shape[0]/FPS:.3f}s), "
f"audio samples={audio['waveform'].shape[-1]}, decoded tensor≈{decoded_gib:.2f} GiB.",
]
)
return images, audio, last_state, new_session, "\n".join(report_lines)
+261
View File
@@ -0,0 +1,261 @@
"""In-memory session model for accepted chunks, resume, and branch-safe rerolls."""
from __future__ import annotations
import copy
import hashlib
import json
import uuid
from datetime import datetime, timezone
from typing import Any
import torch
from ..constants import SESSION_MAGIC, STATE_MAGIC
from ..state import validate_plan
from ..temporal import (
audio_grid_offset,
audio_latent_t,
context_slots,
is_valid_frame_count,
pixel_frames_for_latent_t,
video_latent_t,
)
from ..version import PACKAGE_VERSION, SESSION_SCHEMA_VERSION, STATE_SCHEMA_VERSION
from .sampling import latent_from_cpu, latent_to_cpu
class SessionValidationError(ValueError):
pass
def _now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def model_fingerprint(model: Any) -> str:
base = getattr(model, "model", None)
inner = getattr(base, "diffusion_model", None)
options = getattr(model, "model_options", {}) or {}
transformer = options.get("transformer_options", {}) or {}
wrappers = getattr(model, "wrappers", {}) or {}
wrapper_keys: list[str] = []
if isinstance(wrappers, dict):
for group in wrappers.values():
if isinstance(group, dict):
wrapper_keys.extend(str(key) for key in group)
descriptor = {
"base": f"{type(base).__module__}.{type(base).__name__}" if base is not None else "missing",
"inner": f"{type(inner).__module__}.{type(inner).__name__}" if inner is not None else "missing",
"dtype": str(getattr(model, "model_dtype", lambda: "unknown")()),
"model_size": int(getattr(model, "model_size", lambda: 0)() or 0),
"wrappers": sorted(wrapper_keys),
"has_attention_override": transformer.get("optimized_attention_override") is not None,
"has_sol_compose": transformer.get("sol_compose") is not None,
}
raw = json.dumps(descriptor, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def make_chunk_entry(
*,
latent: dict[str, Any],
plan: dict[str, Any],
prompt: str,
prompt_hash: str,
seed: int,
context_frames: int,
motion_score: float,
reused: bool,
) -> dict[str, Any]:
plan = validate_plan(plan)
video, audio = latent_to_cpu(latent)
actual_frames = pixel_frames_for_latent_t(int(video.shape[2]))
if actual_frames != int(plan["total_frames"]):
raise SessionValidationError(
f"chunk latent represents {actual_frames} frames but plan declares {plan['total_frames']}"
)
return {
"sequence_index": 0, # populated by make_session
"clip_index": int(plan["clip_index"]),
"prompt": str(prompt),
"prompt_hash": str(prompt_hash),
"seed": int(seed),
"context_frames": int(context_frames),
"motion_score": float(motion_score),
"reused": bool(reused),
"plan": copy.deepcopy(plan),
"video": video,
"audio": audio,
}
def validate_chunk_entry(entry: dict[str, Any]) -> dict[str, Any]:
if not isinstance(entry, dict):
raise SessionValidationError("session chunk must be a dictionary")
video, audio = entry.get("video"), entry.get("audio")
if not torch.is_tensor(video) or video.ndim != 5 or tuple(video.shape[:2]) != (1, 24):
raise SessionValidationError("session chunk video must be [1,24,T,H,W]")
if not torch.is_tensor(audio) or audio.ndim != 4 or tuple(audio.shape[:3]) != (1, 32, 2):
raise SessionValidationError("session chunk audio must be [1,32,2,T]")
if video.device.type != "cpu" or audio.device.type != "cpu":
raise SessionValidationError("session tensors must remain on CPU")
if not bool(torch.isfinite(video.float()).all().item()):
raise SessionValidationError("session video contains NaN or Inf")
if not bool(torch.isfinite(audio.float()).all().item()):
raise SessionValidationError("session audio contains NaN or Inf")
plan = validate_plan(entry.get("plan"))
actual_frames = pixel_frames_for_latent_t(int(video.shape[2]))
if actual_frames != int(plan["total_frames"]):
raise SessionValidationError("session chunk video length does not match its plan")
if not isinstance(entry.get("prompt"), str) or not entry["prompt"].strip():
raise SessionValidationError("session chunk prompt is empty")
if not isinstance(entry.get("prompt_hash"), str) or len(entry["prompt_hash"]) != 64:
raise SessionValidationError("session chunk prompt hash is invalid")
if int(entry.get("seed", -1)) < 0:
raise SessionValidationError("session chunk seed is invalid")
if int(entry.get("context_frames", 0)) not in (0, 5, 22, 39):
raise SessionValidationError("session chunk context_frames is invalid")
return entry
def make_session(
*,
chunks: list[dict[str, Any]],
width: int,
height: int,
chunk_seconds: float,
identity_hash: str,
model_fingerprint_value: str,
parent_session_id: str | None,
reroll_from_chunk: int,
settings: dict[str, Any],
) -> dict[str, Any]:
session_id = uuid.uuid4().hex
normalized: list[dict[str, Any]] = []
for index, entry in enumerate(chunks, start=1):
item = dict(entry)
item["plan"] = copy.deepcopy(entry["plan"])
item["sequence_index"] = index
validate_chunk_entry(item)
normalized.append(item)
return {
"magic": SESSION_MAGIC,
"schema_version": SESSION_SCHEMA_VERSION,
"package_version": PACKAGE_VERSION,
"session_id": session_id,
"parent_session_id": parent_session_id,
"created_utc": _now_iso(),
"reroll_from_chunk": int(reroll_from_chunk),
"width": int(width),
"height": int(height),
"chunk_seconds": float(chunk_seconds),
"identity_hash": str(identity_hash),
"model_fingerprint": str(model_fingerprint_value),
"settings": copy.deepcopy(settings),
"chunks": normalized,
}
def validate_session(session: dict[str, Any]) -> dict[str, Any]:
if not isinstance(session, dict) or session.get("magic") != SESSION_MAGIC:
raise SessionValidationError("invalid H3 Continuum session")
if int(session.get("schema_version", -1)) != SESSION_SCHEMA_VERSION:
raise SessionValidationError(
f"unsupported session schema {session.get('schema_version')}; "
f"expected {SESSION_SCHEMA_VERSION}"
)
if int(session.get("width", 0)) <= 0 or int(session.get("height", 0)) <= 0:
raise SessionValidationError("session dimensions are invalid")
if float(session.get("chunk_seconds", 0.0)) <= 0:
raise SessionValidationError("session chunk_seconds is invalid")
chunks = session.get("chunks")
if not isinstance(chunks, list) or not chunks:
raise SessionValidationError("session contains no chunks")
previous_clip_index = None
for index, entry in enumerate(chunks, start=1):
validate_chunk_entry(entry)
if int(entry.get("sequence_index", index)) != index:
raise SessionValidationError("session chunk sequence indices are not contiguous")
clip_index = int(entry["plan"]["clip_index"])
if previous_clip_index is not None and clip_index != previous_clip_index + 1:
raise SessionValidationError("session clip indices are not contiguous")
previous_clip_index = clip_index
video = entry["video"]
if int(video.shape[-1]) * 16 != int(session["width"]):
raise SessionValidationError("session width does not match chunk latent")
if int(video.shape[-2]) * 16 != int(session["height"]):
raise SessionValidationError("session height does not match chunk latent")
return session
def entry_to_latent(entry: dict[str, Any]) -> dict[str, Any]:
entry = validate_chunk_entry(entry)
return latent_from_cpu(entry["video"], entry["audio"])
def entry_to_state(entry: dict[str, Any], *, capacity_frames: int | None = None) -> dict[str, Any]:
"""Build a V1-compatible continuation state directly from a CPU chunk.
This deliberately avoids reconstructing a ComfyUI ``NestedTensor``. Session
inspection, persistence tests, and branch selection therefore remain usable
outside a live ComfyUI process, while the returned state is byte-for-byte
compatible with the V1 state contract.
"""
entry = validate_chunk_entry(entry)
plan = entry["plan"]
video = entry["video"]
audio = entry["audio"]
source_frames = int(plan["total_frames"])
clip_index = int(plan["clip_index"])
capacity = int(capacity_frames or plan["state_capacity_frames"])
actual_video_t = int(video.shape[2])
actual_frames = pixel_frames_for_latent_t(actual_video_t)
if actual_frames != source_frames:
raise SessionValidationError(
f"chunk plan says {source_frames} frames but latent represents {actual_frames}"
)
if not is_valid_frame_count(actual_frames) or video_latent_t(actual_frames) != actual_video_t:
raise SessionValidationError(
f"chunk video latent T={actual_video_t} is not on the native H3 temporal grid"
)
slots = context_slots(capacity)
audio_steps = audio_latent_t(capacity)
if int(video.shape[2]) < slots or int(audio.shape[-1]) < audio_steps:
raise SessionValidationError(
f"chunk is too short for a {capacity}-frame continuation state"
)
grid_offset = audio_grid_offset(source_frames, int(audio.shape[-1]))
if not (-0.500001 <= grid_offset <= 0.500001):
raise SessionValidationError(
f"unexpected signed audio-grid offset {grid_offset:.6f}"
)
video_tail = video[:, :, -slots:].contiguous().clone()
audio_tail = audio[..., -audio_steps:].contiguous().clone()
return {
"magic": STATE_MAGIC,
"schema_version": STATE_SCHEMA_VERSION,
"clip_index": clip_index,
"source_frame_count": source_frames,
"capacity_frames": capacity,
"width": int(video.shape[-1]) * 16,
"height": int(video.shape[-2]) * 16,
"video_tail": video_tail,
"audio_tail": audio_tail,
"audio_overhang": float(grid_offset),
"source_mode": "latent_direct",
}
def session_summary(session: dict[str, Any]) -> str:
session = validate_session(session)
reused = sum(1 for item in session["chunks"] if item.get("reused"))
frames = sum(int(item["plan"]["net_frames"]) for item in session["chunks"])
return (
f"Session {session['session_id'][:8]}: {len(session['chunks'])} chunks, "
f"{frames} retained frames, {reused} reused, {session['width']}x{session['height']}."
)
+142
View File
@@ -0,0 +1,142 @@
"""Atomic safetensors + JSON persistence for V2 sessions."""
from __future__ import annotations
import json
import logging
import os
import re
import uuid
from pathlib import Path
from typing import Any
from safetensors import safe_open
from safetensors.torch import save_file
from .session import validate_session
LOG = logging.getLogger("h3_continuum_join")
_SAFE_PREFIX = re.compile(r"[^A-Za-z0-9._-]+")
_HEADER_KEY = "h3_continuum_session_json"
def sanitize_prefix(prefix: str) -> str:
value = _SAFE_PREFIX.sub("_", str(prefix).strip()).strip("._")
return value or "h3_continuum_session"
def session_directory() -> Path:
try:
import folder_paths
root = Path(folder_paths.get_output_directory())
except Exception:
root = Path.cwd()
path = root / "h3_continuum_sessions"
path.mkdir(parents=True, exist_ok=True)
return path
def session_paths(prefix: str, slot: int) -> tuple[Path, Path]:
slot = int(slot)
if not (1 <= slot <= 9999):
raise ValueError("session slot must be between 1 and 9999")
name = f"{sanitize_prefix(prefix)}_slot{slot:04d}"
root = session_directory()
return root / f"{name}.safetensors", root / f"{name}.json"
def _fsync_file(path: Path) -> None:
try:
with path.open("rb") as handle:
os.fsync(handle.fileno())
except OSError:
pass
def _fsync_directory(path: Path) -> None:
if os.name == "nt":
return
try:
descriptor = os.open(str(path), os.O_RDONLY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
except OSError:
pass
def _metadata_without_tensors(session: dict[str, Any]) -> dict[str, Any]:
metadata = {key: value for key, value in session.items() if key != "chunks"}
chunks = []
for entry in session["chunks"]:
chunks.append({key: value for key, value in entry.items() if key not in ("video", "audio")})
metadata["chunks"] = chunks
return metadata
def save_session(session: dict[str, Any], *, prefix: str, slot: int) -> tuple[Path, Path]:
session = validate_session(session)
tensor_path, json_path = session_paths(prefix, slot)
token = uuid.uuid4().hex
tensor_tmp = tensor_path.with_name(f".{tensor_path.name}.{token}.tmp")
json_tmp = json_path.with_name(f".{json_path.name}.{token}.tmp")
tensors = {}
for index, entry in enumerate(session["chunks"], start=1):
tensors[f"chunk_{index:04d}_video"] = entry["video"].detach().to("cpu").contiguous()
tensors[f"chunk_{index:04d}_audio"] = entry["audio"].detach().to("cpu").contiguous()
metadata = _metadata_without_tensors(session)
metadata["_session_commit_id"] = token
serialized = json.dumps(metadata, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
try:
save_file(tensors, str(tensor_tmp), metadata={_HEADER_KEY: serialized})
_fsync_file(tensor_tmp)
with json_tmp.open("w", encoding="utf-8", newline="\n") as handle:
json.dump(metadata, handle, ensure_ascii=False, indent=2, sort_keys=True)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(json_tmp, json_path)
os.replace(tensor_tmp, tensor_path)
_fsync_directory(tensor_path.parent)
finally:
for path in (tensor_tmp, json_tmp):
try:
path.unlink(missing_ok=True)
except OSError:
pass
return tensor_path, json_path
def load_session(*, prefix: str, slot: int) -> dict[str, Any]:
tensor_path, json_path = session_paths(prefix, slot)
if not tensor_path.is_file():
raise FileNotFoundError(f"session slot does not exist: {tensor_path.name}")
with safe_open(str(tensor_path), framework="pt", device="cpu") as handle:
serialized = (handle.metadata() or {}).get(_HEADER_KEY)
if serialized is None:
raise ValueError("session safetensors has no embedded Continuum metadata")
session = json.loads(serialized)
for index, entry in enumerate(session.get("chunks") or [], start=1):
entry["video"] = handle.get_tensor(f"chunk_{index:04d}_video")
entry["audio"] = handle.get_tensor(f"chunk_{index:04d}_audio")
if json_path.is_file():
try:
with json_path.open("r", encoding="utf-8") as handle:
mirror = json.load(handle)
if mirror.get("_session_commit_id") != session.get("_session_commit_id"):
LOG.warning("Ignoring stale Continuum session JSON mirror: %s", json_path.name)
except (OSError, ValueError, TypeError) as exc:
LOG.warning("Ignoring unreadable session JSON mirror %s: %s", json_path.name, exc)
return validate_session(session)
def session_mtime(*, prefix: str, slot: int) -> tuple[float, float]:
tensor_path, json_path = session_paths(prefix, slot)
return (
tensor_path.stat().st_mtime if tensor_path.exists() else -1.0,
json_path.stat().st_mtime if json_path.exists() else -1.0,
)