Release V2.1.7 stable facade UI
This commit is contained in:
+102
-443
@@ -1,12 +1,8 @@
|
||||
"""Deferred VAE decoding, overlap trimming, assembly, and exact-duration correction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from ..constants import DIAGNOSTICS_FULL, DIAGNOSTICS_OFF, FPS
|
||||
from ..media import validate_audio
|
||||
from ..state import validate_plan
|
||||
@@ -15,451 +11,114 @@ from .seam_guard import correct_audio_seam, correct_seam
|
||||
from .seam_types import SeamDecision
|
||||
from .session import entry_to_latent, validate_chunk_entry
|
||||
|
||||
|
||||
def decode_video(video_vae: Any, latent: dict[str, Any]) -> torch.Tensor:
|
||||
samples = latent["samples"]
|
||||
if hasattr(samples, "unbind"):
|
||||
video = list(samples.unbind())[0]
|
||||
else:
|
||||
video = samples[0]
|
||||
images = video_vae.decode(video)
|
||||
if images.ndim == 5:
|
||||
images = images.reshape(-1, images.shape[-3], images.shape[-2], images.shape[-1])
|
||||
if images.ndim != 4:
|
||||
raise ValueError(f"video VAE returned unexpected shape {tuple(images.shape)}")
|
||||
# Long sequences must not leave decoded frames resident on the GPU. ComfyUI
|
||||
# IMAGE tensors are conventionally CPU float tensors, so transfer each chunk
|
||||
# immediately and release its raw decode before moving to the next chunk.
|
||||
def decode_video(video_vae:Any,latent:dict[str,Any])->torch.Tensor:
|
||||
samples=latent["samples"]; video=list(samples.unbind())[0] if hasattr(samples,"unbind") else samples[0]; images=video_vae.decode(video)
|
||||
if images.ndim==5: images=images.reshape(-1,images.shape[-3],images.shape[-2],images.shape[-1])
|
||||
if images.ndim!=4: raise ValueError(f"video VAE returned unexpected shape {tuple(images.shape)}")
|
||||
return images.detach().to("cpu").contiguous()
|
||||
|
||||
|
||||
def decode_audio(audio_vae: Any, latent: dict[str, Any]) -> dict[str, Any]:
|
||||
samples = latent["samples"]
|
||||
if hasattr(samples, "unbind"):
|
||||
audio_latent = list(samples.unbind())[-1]
|
||||
else:
|
||||
audio_latent = samples[-1]
|
||||
audio = audio_vae.decode(audio_latent).movedim(-1, 1)
|
||||
std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0
|
||||
std[std < 1.0] = 1.0
|
||||
audio = (audio / std).detach().to("cpu").contiguous()
|
||||
sample_rate = getattr(
|
||||
audio_vae,
|
||||
"audio_sample_rate_output",
|
||||
getattr(audio_vae, "audio_sample_rate", 32000),
|
||||
)
|
||||
return {"waveform": audio, "sample_rate": int(sample_rate)}
|
||||
|
||||
|
||||
def _trim_images(images: torch.Tensor, plan: dict[str, Any]) -> torch.Tensor:
|
||||
plan = validate_plan(plan)
|
||||
total_frames = int(plan["total_frames"])
|
||||
trim_frames = int(plan["trim_frames"])
|
||||
if images.shape[0] < total_frames:
|
||||
raise ValueError(
|
||||
f"decoded batch has {images.shape[0]} frames; expected at least {total_frames}"
|
||||
)
|
||||
def decode_audio(audio_vae:Any,latent:dict[str,Any])->dict[str,Any]:
|
||||
samples=latent["samples"]; audio_latent=list(samples.unbind())[-1] if hasattr(samples,"unbind") else samples[-1]; audio=audio_vae.decode(audio_latent).movedim(-1,1); std=torch.std(audio,dim=[1,2],keepdim=True)*5.0; std[std<1.0]=1.0; audio=(audio/std).detach().to("cpu").contiguous(); sample_rate=getattr(audio_vae,"audio_sample_rate_output",getattr(audio_vae,"audio_sample_rate",32000)); return {"waveform":audio,"sample_rate":int(sample_rate)}
|
||||
def _trim_images(images,plan):
|
||||
plan=validate_plan(plan); total_frames=int(plan["total_frames"]); trim_frames=int(plan["trim_frames"])
|
||||
if images.shape[0]<total_frames: raise ValueError(f"decoded batch has {images.shape[0]} frames; expected at least {total_frames}")
|
||||
return images[:total_frames][trim_frames:].contiguous()
|
||||
|
||||
|
||||
def _slice_audio_for_timeline(
|
||||
audio: dict[str, Any],
|
||||
*,
|
||||
trim_frames: int,
|
||||
frame_start: int,
|
||||
frame_stop: int,
|
||||
) -> torch.Tensor:
|
||||
"""Return one segment at its exact cumulative 24fps sample interval.
|
||||
|
||||
Segment-local rounding drifts by one sample on common 119-frame Continuum
|
||||
chunks. Deriving every boundary from cumulative frame counts guarantees that
|
||||
the assembled audio length is always ``round(total_frames / 24 * rate)``.
|
||||
"""
|
||||
|
||||
waveform, sample_rate = validate_audio(audio)
|
||||
trim_samples = int(round(float(trim_frames) / FPS * sample_rate))
|
||||
target_start = int(round(float(frame_start) / FPS * sample_rate))
|
||||
target_stop = int(round(float(frame_stop) / FPS * sample_rate))
|
||||
wanted = target_stop - target_start
|
||||
if wanted < 0:
|
||||
raise ValueError("audio timeline interval is negative")
|
||||
result = waveform[..., trim_samples : trim_samples + wanted]
|
||||
if result.shape[-1] < wanted:
|
||||
missing = wanted - int(result.shape[-1])
|
||||
if result.shape[-1] > 0:
|
||||
tail = result[..., -1:].expand(*result.shape[:-1], missing).clone()
|
||||
result = torch.cat((result, tail), dim=-1)
|
||||
else:
|
||||
result = F.pad(result, (0, missing))
|
||||
def _slice_audio_for_timeline(audio,*,trim_frames,frame_start,frame_stop):
|
||||
waveform,sample_rate=validate_audio(audio); trim_samples=int(round(float(trim_frames)/FPS*sample_rate)); target_start=int(round(float(frame_start)/FPS*sample_rate)); target_stop=int(round(float(frame_stop)/FPS*sample_rate)); wanted=target_stop-target_start
|
||||
if wanted<0: raise ValueError("audio timeline interval is negative")
|
||||
result=waveform[...,trim_samples:trim_samples+wanted]
|
||||
if result.shape[-1]<wanted:
|
||||
missing=wanted-int(result.shape[-1]); result=torch.cat((result,result[...,-1:].expand(*result.shape[:-1],missing).clone()),dim=-1) if result.shape[-1]>0 else F.pad(result,(0,missing))
|
||||
return result.contiguous()
|
||||
|
||||
|
||||
def decode_sequence(
|
||||
*,
|
||||
entries: list[dict[str, Any]],
|
||||
video_vae: Any,
|
||||
audio_vae: Any,
|
||||
diagnostics_full: bool,
|
||||
) -> tuple[torch.Tensor, dict[str, Any], list[str]]:
|
||||
"""Decode and assemble all chunks with bounded transient memory.
|
||||
|
||||
The final IMAGE tensor is preallocated once. Each raw VAE decode is copied
|
||||
into its destination range and released before the next chunk, avoiding the
|
||||
repeated full-sequence copies produced by progressive ``torch.cat``.
|
||||
"""
|
||||
|
||||
if not entries:
|
||||
raise ValueError("cannot decode an empty Continuum sequence")
|
||||
checked = [validate_chunk_entry(entry) for entry in entries]
|
||||
plans = [validate_plan(entry["plan"]) for entry in checked]
|
||||
total_retained_frames = sum(int(plan["net_frames"]) for plan in plans)
|
||||
if total_retained_frames < 1:
|
||||
raise ValueError("Continuum sequence contains no retained frames")
|
||||
|
||||
image_buffer: torch.Tensor | None = None
|
||||
audio_buffer: torch.Tensor | None = None
|
||||
audio_rate: int | None = None
|
||||
frame_cursor = 0
|
||||
previous_segment_images: torch.Tensor | None = None
|
||||
previous_segment_audio: dict[str, Any] | None = None
|
||||
reports: list[str] = []
|
||||
|
||||
for index, (entry, plan) in enumerate(zip(checked, plans), start=1):
|
||||
latent = entry_to_latent(entry)
|
||||
raw_images = decode_video(video_vae, latent)
|
||||
raw_audio = decode_audio(audio_vae, latent)
|
||||
|
||||
if diagnostics_full and previous_segment_images is not None:
|
||||
metrics = overlap_diagnostics(
|
||||
previous_images=previous_segment_images,
|
||||
previous_audio=previous_segment_audio,
|
||||
next_raw_images=raw_images,
|
||||
next_raw_audio=raw_audio,
|
||||
context_frames=int(plan["trim_frames"]),
|
||||
)
|
||||
reports.append(format_join_metrics(index, metrics))
|
||||
|
||||
images = _trim_images(raw_images, plan)
|
||||
segment_frames = int(images.shape[0])
|
||||
if segment_frames != int(plan["net_frames"]):
|
||||
raise ValueError(
|
||||
f"chunk {index} retained {segment_frames} frames; plan expects {plan['net_frames']}"
|
||||
)
|
||||
frame_stop = frame_cursor + segment_frames
|
||||
|
||||
waveform, sample_rate = validate_audio(raw_audio)
|
||||
if audio_rate is None:
|
||||
audio_rate = sample_rate
|
||||
elif sample_rate != audio_rate:
|
||||
raise ValueError(
|
||||
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
|
||||
)
|
||||
segment_waveform = _slice_audio_for_timeline(
|
||||
raw_audio,
|
||||
trim_frames=int(plan["trim_frames"]),
|
||||
frame_start=frame_cursor,
|
||||
frame_stop=frame_stop,
|
||||
)
|
||||
|
||||
if image_buffer is None:
|
||||
image_buffer = torch.empty(
|
||||
(total_retained_frames, *images.shape[1:]),
|
||||
dtype=images.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]):
|
||||
raise ValueError(
|
||||
f"decoded image geometry changed at chunk {index}: "
|
||||
f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}"
|
||||
)
|
||||
def decode_sequence(*,entries,video_vae,audio_vae,diagnostics_full):
|
||||
if not entries: raise ValueError("cannot decode an empty Continuum sequence")
|
||||
checked=[validate_chunk_entry(e) for e in entries]; plans=[validate_plan(e["plan"]) for e in checked]; total_retained_frames=sum(int(p["net_frames"]) for p in plans)
|
||||
if total_retained_frames<1: raise ValueError("Continuum sequence contains no retained frames")
|
||||
image_buffer=None; audio_buffer=None; audio_rate=None; frame_cursor=0; previous_segment_images=None; previous_segment_audio=None; reports=[]
|
||||
for index,(entry,plan) in enumerate(zip(checked,plans),start=1):
|
||||
latent=entry_to_latent(entry); raw_images=decode_video(video_vae,latent); raw_audio=decode_audio(audio_vae,latent)
|
||||
if diagnostics_full and previous_segment_images is not None: reports.append(format_join_metrics(index,overlap_diagnostics(previous_images=previous_segment_images,previous_audio=previous_segment_audio,next_raw_images=raw_images,next_raw_audio=raw_audio,context_frames=int(plan["trim_frames"]))))
|
||||
images=_trim_images(raw_images,plan); segment_frames=int(images.shape[0])
|
||||
if segment_frames!=int(plan["net_frames"]): raise ValueError(f"chunk {index} retained {segment_frames} frames; plan expects {plan['net_frames']}")
|
||||
frame_stop=frame_cursor+segment_frames; waveform,sample_rate=validate_audio(raw_audio)
|
||||
if audio_rate is None: audio_rate=sample_rate
|
||||
elif sample_rate!=audio_rate: raise ValueError(f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}")
|
||||
segment_waveform=_slice_audio_for_timeline(raw_audio,trim_frames=int(plan["trim_frames"]),frame_start=frame_cursor,frame_stop=frame_stop)
|
||||
if image_buffer is None: image_buffer=torch.empty((total_retained_frames,*images.shape[1:]),dtype=images.dtype,device="cpu")
|
||||
elif tuple(images.shape[1:])!=tuple(image_buffer.shape[1:]): raise ValueError(f"decoded image geometry changed at chunk {index}: {tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}")
|
||||
image_buffer[frame_cursor:frame_stop].copy_(images)
|
||||
|
||||
if audio_buffer is None:
|
||||
total_samples = int(round(total_retained_frames / FPS * sample_rate))
|
||||
audio_buffer = torch.empty(
|
||||
(*waveform.shape[:-1], total_samples),
|
||||
dtype=waveform.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
|
||||
raise ValueError("decoded audio batch/channel structure changed between chunks")
|
||||
sample_start = int(round(frame_cursor / FPS * sample_rate))
|
||||
sample_stop = int(round(frame_stop / FPS * sample_rate))
|
||||
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
|
||||
|
||||
# Retain only what Full diagnostics needs for the next seam. Audio is
|
||||
# small, while video is bounded to the next overlap instead of a full clip.
|
||||
next_context = int(plans[index]["trim_frames"]) if index < len(plans) else 0
|
||||
if next_context > 0:
|
||||
previous_segment_images = images[-next_context:].contiguous()
|
||||
previous_segment_audio = {
|
||||
"waveform": segment_waveform,
|
||||
"sample_rate": sample_rate,
|
||||
}
|
||||
else:
|
||||
previous_segment_images = None
|
||||
previous_segment_audio = None
|
||||
|
||||
reports.append(
|
||||
f"decoded chunk {index}: {segment_frames} retained frames, "
|
||||
f"cumulative {frame_stop}/{total_retained_frames}"
|
||||
)
|
||||
frame_cursor = frame_stop
|
||||
del latent, raw_images, raw_audio, images, segment_waveform
|
||||
|
||||
if image_buffer is None or audio_buffer is None or audio_rate is None:
|
||||
raise RuntimeError("Continuum decoder failed to allocate output buffers")
|
||||
if frame_cursor != total_retained_frames:
|
||||
raise RuntimeError(
|
||||
f"Continuum decode cursor mismatch: {frame_cursor} != {total_retained_frames}"
|
||||
)
|
||||
audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate}
|
||||
reports.append(
|
||||
f"assembled {len(entries)} chunks into {total_retained_frames} frames with "
|
||||
"cumulative sample-boundary alignment"
|
||||
)
|
||||
return image_buffer.contiguous(), audio, reports
|
||||
|
||||
|
||||
def decode_sequence_with_seam(
|
||||
*,
|
||||
entries: list[dict[str, Any]],
|
||||
video_vae: Any,
|
||||
audio_vae: Any,
|
||||
diagnostics_mode: str,
|
||||
automatic: bool = False,
|
||||
) -> tuple[torch.Tensor, dict[str, Any], list[str]]:
|
||||
"""Decode with guarded V2.1 correction while preserving legacy output length."""
|
||||
|
||||
if not entries:
|
||||
raise ValueError("cannot decode an empty Continuum sequence")
|
||||
checked = [validate_chunk_entry(entry) for entry in entries]
|
||||
plans = [validate_plan(entry["plan"]) for entry in checked]
|
||||
total_retained_frames = sum(int(plan["net_frames"]) for plan in plans)
|
||||
if total_retained_frames < 1:
|
||||
raise ValueError("Continuum sequence contains no retained frames")
|
||||
|
||||
image_buffer: torch.Tensor | None = None
|
||||
audio_buffer: torch.Tensor | None = None
|
||||
audio_rate: int | None = None
|
||||
frame_cursor = 0
|
||||
reports: list[str] = []
|
||||
|
||||
for index, (entry, plan) in enumerate(zip(checked, plans), start=1):
|
||||
latent = entry_to_latent(entry)
|
||||
raw_images = decode_video(video_vae, latent)
|
||||
raw_audio = decode_audio(audio_vae, latent)
|
||||
waveform, sample_rate = validate_audio(raw_audio)
|
||||
if audio_rate is None:
|
||||
audio_rate = sample_rate
|
||||
elif sample_rate != audio_rate:
|
||||
raise ValueError(
|
||||
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
|
||||
)
|
||||
|
||||
legacy_frames = int(plan["net_frames"])
|
||||
legacy_frame_stop = frame_cursor + legacy_frames
|
||||
frame_start = frame_cursor
|
||||
trim_frames = int(plan["trim_frames"])
|
||||
decision: SeamDecision | None = None
|
||||
video_patch: torch.Tensor | None = None
|
||||
audio_patch: torch.Tensor | None = None
|
||||
|
||||
if index == 1:
|
||||
images = _trim_images(raw_images, plan)
|
||||
if audio_buffer is None: audio_buffer=torch.empty((*waveform.shape[:-1],int(round(total_retained_frames/FPS*sample_rate))),dtype=waveform.dtype,device="cpu")
|
||||
elif tuple(segment_waveform.shape[:-1])!=tuple(audio_buffer.shape[:-1]): raise ValueError("decoded audio batch/channel structure changed between chunks")
|
||||
sample_start=int(round(frame_cursor/FPS*sample_rate)); sample_stop=int(round(frame_stop/FPS*sample_rate)); audio_buffer[...,sample_start:sample_stop].copy_(segment_waveform)
|
||||
next_context=int(plans[index]["trim_frames"]) if index<len(plans) else 0
|
||||
if next_context>0: previous_segment_images=images[-next_context:].contiguous(); previous_segment_audio={"waveform":segment_waveform,"sample_rate":sample_rate}
|
||||
else: previous_segment_images=None; previous_segment_audio=None
|
||||
reports.append(f"decoded chunk {index}: {segment_frames} retained frames, cumulative {frame_stop}/{total_retained_frames}"); frame_cursor=frame_stop
|
||||
del latent,raw_images,raw_audio,images,segment_waveform
|
||||
if image_buffer is None or audio_buffer is None or audio_rate is None: raise RuntimeError("Continuum decoder failed to allocate output buffers")
|
||||
if frame_cursor!=total_retained_frames: raise RuntimeError(f"Continuum decode cursor mismatch: {frame_cursor} != {total_retained_frames}")
|
||||
reports.append(f"assembled {len(entries)} chunks into {total_retained_frames} frames with cumulative sample-boundary alignment")
|
||||
return image_buffer.contiguous(),{"waveform":audio_buffer.contiguous(),"sample_rate":audio_rate},reports
|
||||
def decode_sequence_with_seam(*,entries,video_vae,audio_vae,diagnostics_mode,automatic=False):
|
||||
if not entries: raise ValueError("cannot decode an empty Continuum sequence")
|
||||
checked=[validate_chunk_entry(e) for e in entries]; plans=[validate_plan(e["plan"]) for e in checked]; total_retained_frames=sum(int(p["net_frames"]) for p in plans)
|
||||
if total_retained_frames<1: raise ValueError("Continuum sequence contains no retained frames")
|
||||
image_buffer=None; audio_buffer=None; audio_rate=None; frame_cursor=0; reports=[]
|
||||
for index,(entry,plan) in enumerate(zip(checked,plans),start=1):
|
||||
latent=entry_to_latent(entry); raw_images=decode_video(video_vae,latent); raw_audio=decode_audio(audio_vae,latent); waveform,sample_rate=validate_audio(raw_audio)
|
||||
if audio_rate is None: audio_rate=sample_rate
|
||||
elif sample_rate!=audio_rate: raise ValueError(f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}")
|
||||
legacy_frames=int(plan["net_frames"]); legacy_frame_stop=frame_cursor+legacy_frames; frame_start=frame_cursor; trim_frames=int(plan["trim_frames"]); decision=None; video_patch=None; audio_patch=None
|
||||
if index==1: images=_trim_images(raw_images,plan)
|
||||
else:
|
||||
try:
|
||||
if image_buffer is None or audio_buffer is None:
|
||||
raise RuntimeError("seam buffers are unavailable")
|
||||
limited_raw_images = raw_images[: int(plan["total_frames"])]
|
||||
images, decision, video_patch = correct_seam(
|
||||
image_buffer[:frame_cursor],
|
||||
limited_raw_images,
|
||||
context_frames=trim_frames,
|
||||
automatic=automatic,
|
||||
)
|
||||
frame_start = frame_cursor - int(decision.cut_rewind_frames)
|
||||
trim_frames -= int(decision.cut_rewind_frames)
|
||||
if frame_start < 0 or trim_frames < 0:
|
||||
raise ValueError("adaptive cut produced a negative boundary")
|
||||
if frame_start + int(images.shape[0]) != legacy_frame_stop:
|
||||
raise ValueError("adaptive cut changed the cumulative frame boundary")
|
||||
|
||||
sample_start = int(round(frame_start / FPS * sample_rate))
|
||||
cut_sample = int(round(trim_frames / FPS * sample_rate))
|
||||
(
|
||||
audio_patch,
|
||||
audio_metrics,
|
||||
crossfade_samples,
|
||||
level_gain,
|
||||
dc_bias,
|
||||
) = correct_audio_seam(
|
||||
audio_buffer[..., :sample_start],
|
||||
waveform,
|
||||
sample_rate=sample_rate,
|
||||
cut_sample=cut_sample,
|
||||
)
|
||||
decision.audio_metrics = audio_metrics
|
||||
decision.audio_crossfade_samples = crossfade_samples
|
||||
decision.audio_level_gain = level_gain
|
||||
decision.audio_dc_bias = dc_bias
|
||||
if image_buffer is None or audio_buffer is None: raise RuntimeError("seam buffers are unavailable")
|
||||
images,decision,video_patch=correct_seam(image_buffer[:frame_cursor],raw_images[:int(plan["total_frames"])],context_frames=trim_frames,automatic=automatic); frame_start=frame_cursor-int(decision.cut_rewind_frames); trim_frames-=int(decision.cut_rewind_frames)
|
||||
if frame_start<0 or trim_frames<0: raise ValueError("adaptive cut produced a negative boundary")
|
||||
if frame_start+int(images.shape[0])!=legacy_frame_stop: raise ValueError("adaptive cut changed the cumulative frame boundary")
|
||||
sample_start=int(round(frame_start/FPS*sample_rate)); cut_sample=int(round(trim_frames/FPS*sample_rate)); audio_patch,audio_metrics,crossfade_samples,level_gain,dc_bias=correct_audio_seam(audio_buffer[...,:sample_start],waveform,sample_rate=sample_rate,cut_sample=cut_sample); decision.audio_metrics=audio_metrics; decision.audio_crossfade_samples=crossfade_samples; decision.audio_level_gain=level_gain; decision.audio_dc_bias=dc_bias
|
||||
except Exception as exc:
|
||||
images = _trim_images(raw_images, plan)
|
||||
frame_start = frame_cursor
|
||||
trim_frames = int(plan["trim_frames"])
|
||||
video_patch = None
|
||||
audio_patch = None
|
||||
decision = SeamDecision(
|
||||
fallback_reason=f"{type(exc).__name__}: {exc}",
|
||||
)
|
||||
|
||||
segment_frames = int(images.shape[0])
|
||||
frame_stop = frame_start + segment_frames
|
||||
if frame_stop != legacy_frame_stop:
|
||||
raise ValueError(
|
||||
f"chunk {index} corrected boundary ends at {frame_stop}; "
|
||||
f"expected {legacy_frame_stop}"
|
||||
)
|
||||
segment_waveform = _slice_audio_for_timeline(
|
||||
raw_audio,
|
||||
trim_frames=trim_frames,
|
||||
frame_start=frame_start,
|
||||
frame_stop=frame_stop,
|
||||
)
|
||||
|
||||
if image_buffer is None:
|
||||
image_buffer = torch.empty(
|
||||
(total_retained_frames, *images.shape[1:]),
|
||||
dtype=images.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]):
|
||||
raise ValueError(
|
||||
f"decoded image geometry changed at chunk {index}: "
|
||||
f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}"
|
||||
)
|
||||
if video_patch is not None and video_patch.shape[0] > 0:
|
||||
patch_start = frame_start - int(video_patch.shape[0])
|
||||
if patch_start < 0:
|
||||
raise ValueError("video seam patch starts before the output")
|
||||
images=_trim_images(raw_images,plan); frame_start=frame_cursor; trim_frames=int(plan["trim_frames"]); video_patch=None; audio_patch=None; decision=SeamDecision(fallback_reason=f"{type(exc).__name__}: {exc}")
|
||||
segment_frames=int(images.shape[0]); frame_stop=frame_start+segment_frames
|
||||
if frame_stop!=legacy_frame_stop: raise ValueError(f"chunk {index} corrected boundary ends at {frame_stop}; expected {legacy_frame_stop}")
|
||||
segment_waveform=_slice_audio_for_timeline(raw_audio,trim_frames=trim_frames,frame_start=frame_start,frame_stop=frame_stop)
|
||||
if image_buffer is None: image_buffer=torch.empty((total_retained_frames,*images.shape[1:]),dtype=images.dtype,device="cpu")
|
||||
elif tuple(images.shape[1:])!=tuple(image_buffer.shape[1:]): raise ValueError(f"decoded image geometry changed at chunk {index}: {tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}")
|
||||
if video_patch is not None and video_patch.shape[0]>0:
|
||||
patch_start=frame_start-int(video_patch.shape[0])
|
||||
if patch_start<0: raise ValueError("video seam patch starts before the output")
|
||||
image_buffer[patch_start:frame_start].copy_(video_patch)
|
||||
image_buffer[frame_start:frame_stop].copy_(images)
|
||||
|
||||
if audio_buffer is None:
|
||||
total_samples = int(round(total_retained_frames / FPS * sample_rate))
|
||||
audio_buffer = torch.empty(
|
||||
(*waveform.shape[:-1], total_samples),
|
||||
dtype=waveform.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
|
||||
raise ValueError("decoded audio batch/channel structure changed between chunks")
|
||||
sample_start = int(round(frame_start / FPS * sample_rate))
|
||||
sample_stop = int(round(frame_stop / FPS * sample_rate))
|
||||
if audio_patch is not None and audio_patch.shape[-1] > 0:
|
||||
patch_start = sample_start - int(audio_patch.shape[-1])
|
||||
if patch_start < 0:
|
||||
raise ValueError("audio seam patch starts before the output")
|
||||
audio_buffer[..., patch_start:sample_start].copy_(audio_patch)
|
||||
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
|
||||
|
||||
if decision is not None and diagnostics_mode != DIAGNOSTICS_OFF:
|
||||
reports.append(
|
||||
format_seam_decision(
|
||||
index,
|
||||
decision,
|
||||
full=diagnostics_mode == DIAGNOSTICS_FULL,
|
||||
)
|
||||
)
|
||||
reports.append(
|
||||
f"decoded chunk {index}: {segment_frames} retained/corrected frames, "
|
||||
f"cumulative {frame_stop}/{total_retained_frames}"
|
||||
)
|
||||
frame_cursor = frame_stop
|
||||
del latent, raw_images, raw_audio, images, segment_waveform
|
||||
|
||||
if image_buffer is None or audio_buffer is None or audio_rate is None:
|
||||
raise RuntimeError("Continuum seam decoder failed to allocate output buffers")
|
||||
if frame_cursor != total_retained_frames:
|
||||
raise RuntimeError(
|
||||
f"Continuum seam decode cursor mismatch: {frame_cursor} != {total_retained_frames}"
|
||||
)
|
||||
audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate}
|
||||
reports.append(
|
||||
f"assembled {len(entries)} chunks into {total_retained_frames} frames with "
|
||||
f"V2.1 {'Auto' if automatic else 'Basic'} seam correction and "
|
||||
"cumulative sample-boundary alignment"
|
||||
)
|
||||
return image_buffer.contiguous(), audio, reports
|
||||
|
||||
|
||||
def enforce_total_frames(
|
||||
images: torch.Tensor,
|
||||
audio: dict[str, Any],
|
||||
*,
|
||||
target_frames: int,
|
||||
preserve_final_frame: bool = False,
|
||||
) -> tuple[torch.Tensor, dict[str, Any], str]:
|
||||
target_frames = int(target_frames)
|
||||
if target_frames < 1:
|
||||
raise ValueError("target_frames must be positive")
|
||||
current = int(images.shape[0])
|
||||
adjustment = target_frames - current
|
||||
anchor_preserved = bool(preserve_final_frame and adjustment < 0 and target_frames >= 2)
|
||||
if adjustment < 0:
|
||||
if anchor_preserved:
|
||||
# H3's valid temporal grid rarely lands on an exact whole-second total.
|
||||
# When a user supplied a final FL2VA frame, trimming only from the end
|
||||
# would discard the very anchor they requested. Remove the small excess
|
||||
# immediately before the final decoded frame instead.
|
||||
images = torch.cat((images[: target_frames - 1], images[-1:]), dim=0).contiguous()
|
||||
else:
|
||||
images = images[:target_frames].contiguous()
|
||||
elif adjustment > 0:
|
||||
if current == 0:
|
||||
raise ValueError("cannot pad an empty image sequence")
|
||||
pad = images[-1:].expand(adjustment, *images.shape[1:]).clone()
|
||||
images = torch.cat((images, pad), dim=0).contiguous()
|
||||
|
||||
waveform = audio["waveform"]
|
||||
sample_rate = int(audio["sample_rate"])
|
||||
original_samples = int(waveform.shape[-1])
|
||||
target_samples = int(round(target_frames / FPS * sample_rate))
|
||||
delta_samples = target_samples - original_samples
|
||||
if delta_samples < 0:
|
||||
if anchor_preserved and target_samples > 1:
|
||||
# Preserve the final video-frame interval of audio as well. A short
|
||||
# 5 ms interpolation at the deletion boundary suppresses a hard click
|
||||
# without changing the exact output length.
|
||||
tail_samples = min(
|
||||
max(1, int(round(sample_rate / FPS))),
|
||||
target_samples,
|
||||
original_samples,
|
||||
)
|
||||
head_samples = target_samples - tail_samples
|
||||
head = waveform[..., :head_samples]
|
||||
tail = waveform[..., -tail_samples:].clone()
|
||||
fade = min(int(round(sample_rate * 0.005)), head_samples, tail_samples)
|
||||
if fade > 0:
|
||||
start = head[..., -1:].to(dtype=tail.dtype)
|
||||
alpha = torch.linspace(
|
||||
0.0, 1.0, fade, dtype=tail.dtype, device=tail.device
|
||||
).reshape(*([1] * (tail.ndim - 1)), fade)
|
||||
tail[..., :fade] = start * (1.0 - alpha) + tail[..., :fade] * alpha
|
||||
waveform = torch.cat((head, tail), dim=-1).contiguous()
|
||||
else:
|
||||
waveform = waveform[..., :target_samples].contiguous()
|
||||
elif delta_samples > 0:
|
||||
# Repeat the final sample instead of introducing a zero click.
|
||||
if waveform.shape[-1] == 0:
|
||||
waveform = F.pad(waveform, (0, delta_samples))
|
||||
else:
|
||||
tail = waveform[..., -1:].expand(*waveform.shape[:-1], delta_samples).clone()
|
||||
waveform = torch.cat((waveform, tail), dim=-1).contiguous()
|
||||
audio = {**audio, "waveform": waveform, "sample_rate": sample_rate}
|
||||
mode = "final anchor preserved; pre-tail trim" if anchor_preserved else "tail trim/pad"
|
||||
report = (
|
||||
f"Exact-duration adjustment: frames {current}->{target_frames} ({adjustment:+d}), "
|
||||
f"audio samples correction {delta_samples:+d}; {mode}."
|
||||
)
|
||||
return images, audio, report
|
||||
if audio_buffer is None: audio_buffer=torch.empty((*waveform.shape[:-1],int(round(total_retained_frames/FPS*sample_rate))),dtype=waveform.dtype,device="cpu")
|
||||
elif tuple(segment_waveform.shape[:-1])!=tuple(audio_buffer.shape[:-1]): raise ValueError("decoded audio batch/channel structure changed between chunks")
|
||||
sample_start=int(round(frame_start/FPS*sample_rate)); sample_stop=int(round(frame_stop/FPS*sample_rate))
|
||||
if audio_patch is not None and audio_patch.shape[-1]>0:
|
||||
patch_start=sample_start-int(audio_patch.shape[-1])
|
||||
if patch_start<0: raise ValueError("audio seam patch starts before the output")
|
||||
audio_buffer[...,patch_start:sample_start].copy_(audio_patch)
|
||||
audio_buffer[...,sample_start:sample_stop].copy_(segment_waveform)
|
||||
if decision is not None and diagnostics_mode!=DIAGNOSTICS_OFF: reports.append(format_seam_decision(index,decision,full=diagnostics_mode==DIAGNOSTICS_FULL))
|
||||
reports.append(f"decoded chunk {index}: {segment_frames} retained/corrected frames, cumulative {frame_stop}/{total_retained_frames}"); frame_cursor=frame_stop
|
||||
del latent,raw_images,raw_audio,images,segment_waveform
|
||||
if image_buffer is None or audio_buffer is None or audio_rate is None: raise RuntimeError("Continuum seam decoder failed to allocate output buffers")
|
||||
if frame_cursor!=total_retained_frames: raise RuntimeError(f"Continuum seam decode cursor mismatch: {frame_cursor} != {total_retained_frames}")
|
||||
reports.append(f"assembled {len(entries)} chunks into {total_retained_frames} frames with V2.1 {'Auto' if automatic else 'Basic'} seam correction and cumulative sample-boundary alignment")
|
||||
return image_buffer.contiguous(),{"waveform":audio_buffer.contiguous(),"sample_rate":audio_rate},reports
|
||||
def enforce_total_frames(images,audio,*,target_frames,preserve_final_frame=False):
|
||||
target_frames=int(target_frames)
|
||||
if target_frames<1: raise ValueError("target_frames must be positive")
|
||||
current=int(images.shape[0]); adjustment=target_frames-current; anchor_preserved=bool(preserve_final_frame and adjustment<0 and target_frames>=2)
|
||||
if adjustment<0: images=torch.cat((images[:target_frames-1],images[-1:]),dim=0).contiguous() if anchor_preserved else images[:target_frames].contiguous()
|
||||
elif adjustment>0:
|
||||
if current==0: raise ValueError("cannot pad an empty image sequence")
|
||||
images=torch.cat((images,images[-1:].expand(adjustment,*images.shape[1:]).clone()),dim=0).contiguous()
|
||||
waveform=audio["waveform"]; sample_rate=int(audio["sample_rate"]); original_samples=int(waveform.shape[-1]); target_samples=int(round(target_frames/FPS*sample_rate)); delta_samples=target_samples-original_samples
|
||||
if delta_samples<0:
|
||||
if anchor_preserved and target_samples>1:
|
||||
tail_samples=min(max(1,int(round(sample_rate/FPS))),target_samples,original_samples); head_samples=target_samples-tail_samples; head=waveform[...,:head_samples]; tail=waveform[...,-tail_samples:].clone(); fade=min(int(round(sample_rate*0.005)),head_samples,tail_samples)
|
||||
if fade>0:
|
||||
start=head[...,-1:].to(dtype=tail.dtype); alpha=torch.linspace(0.0,1.0,fade,dtype=tail.dtype,device=tail.device).reshape(*([1]*(tail.ndim-1)),fade); tail[...,:fade]=start*(1.0-alpha)+tail[...,:fade]*alpha
|
||||
waveform=torch.cat((head,tail),dim=-1).contiguous()
|
||||
else: waveform=waveform[...,:target_samples].contiguous()
|
||||
elif delta_samples>0:
|
||||
waveform=F.pad(waveform,(0,delta_samples)) if waveform.shape[-1]==0 else torch.cat((waveform,waveform[...,-1:].expand(*waveform.shape[:-1],delta_samples).clone()),dim=-1).contiguous()
|
||||
audio={**audio,"waveform":waveform,"sample_rate":sample_rate}; mode="final anchor preserved; pre-tail trim" if anchor_preserved else "tail trim/pad"; return images,audio,f"Exact-duration adjustment: frames {current}->{target_frames} ({adjustment:+d}), audio samples correction {delta_samples:+d}; {mode}."
|
||||
|
||||
+24
-124
@@ -1,131 +1,31 @@
|
||||
"""Lightweight join diagnostics without external vision/audio models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..constants import FPS
|
||||
from .seam_types import SeamDecision
|
||||
|
||||
|
||||
def _audio_waveform(audio: dict[str, Any]) -> tuple[torch.Tensor, int]:
|
||||
waveform = audio.get("waveform")
|
||||
sample_rate = int(audio.get("sample_rate", 0))
|
||||
if not torch.is_tensor(waveform) or waveform.ndim != 3:
|
||||
raise ValueError("audio waveform must be [B,C,S]")
|
||||
if sample_rate <= 0:
|
||||
raise ValueError("audio sample_rate is invalid")
|
||||
return waveform.to(dtype=torch.float32), sample_rate
|
||||
|
||||
|
||||
def _normalized_correlation(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
length = min(int(a.numel()), int(b.numel()))
|
||||
if length < 8:
|
||||
return 0.0
|
||||
a = a.reshape(-1)[:length].float()
|
||||
b = b.reshape(-1)[:length].float()
|
||||
a = a - a.mean()
|
||||
b = b - b.mean()
|
||||
denom = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12)
|
||||
value = float((torch.sum(a * b) / denom).item())
|
||||
return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0
|
||||
|
||||
|
||||
def overlap_diagnostics(
|
||||
*,
|
||||
previous_images: torch.Tensor,
|
||||
previous_audio: dict[str, Any],
|
||||
next_raw_images: torch.Tensor,
|
||||
next_raw_audio: dict[str, Any],
|
||||
context_frames: int,
|
||||
) -> dict[str, float]:
|
||||
context_frames = int(context_frames)
|
||||
frame_count = min(
|
||||
context_frames,
|
||||
int(previous_images.shape[0]),
|
||||
int(next_raw_images.shape[0]),
|
||||
)
|
||||
if frame_count <= 0:
|
||||
return {
|
||||
"video_overlap_mae": 0.0,
|
||||
"video_overlap_psnr": 99.0,
|
||||
"audio_overlap_correlation": 0.0,
|
||||
"audio_boundary_jump": 0.0,
|
||||
}
|
||||
|
||||
prev = previous_images[-frame_count:].detach().to(dtype=torch.float32)
|
||||
nxt = next_raw_images[:frame_count].detach().to(dtype=torch.float32)
|
||||
mse = float(torch.mean((prev - nxt).square()).item())
|
||||
mae = float(torch.mean(torch.abs(prev - nxt)).item())
|
||||
psnr = 99.0 if mse <= 1e-12 else max(0.0, -10.0 * math.log10(mse))
|
||||
|
||||
prev_wave, prev_sr = _audio_waveform(previous_audio)
|
||||
next_wave, next_sr = _audio_waveform(next_raw_audio)
|
||||
if prev_sr != next_sr:
|
||||
raise ValueError("diagnostic audio sample rates do not match")
|
||||
samples = int(round(frame_count / FPS * prev_sr))
|
||||
samples = min(samples, int(prev_wave.shape[-1]), int(next_wave.shape[-1]))
|
||||
if samples > 0:
|
||||
correlation = _normalized_correlation(prev_wave[..., -samples:], next_wave[..., :samples])
|
||||
else:
|
||||
correlation = 0.0
|
||||
if prev_wave.shape[-1] and next_wave.shape[-1]:
|
||||
jump = float(torch.mean(torch.abs(prev_wave[..., -1] - next_wave[..., 0])).item())
|
||||
else:
|
||||
jump = 0.0
|
||||
return {
|
||||
"video_overlap_mae": mae,
|
||||
"video_overlap_psnr": psnr,
|
||||
"audio_overlap_correlation": correlation,
|
||||
"audio_boundary_jump": jump,
|
||||
}
|
||||
|
||||
|
||||
def av_duration_error_samples(images: torch.Tensor, audio: dict[str, Any]) -> int:
|
||||
waveform, sample_rate = _audio_waveform(audio)
|
||||
expected = int(round(float(images.shape[0]) / FPS * sample_rate))
|
||||
return int(waveform.shape[-1]) - expected
|
||||
|
||||
|
||||
def format_join_metrics(index: int, metrics: dict[str, float]) -> str:
|
||||
return (
|
||||
f"join {index-1}->{index}: video MAE={metrics['video_overlap_mae']:.6f}, "
|
||||
f"PSNR={metrics['video_overlap_psnr']:.2f}dB, "
|
||||
f"audio corr={metrics['audio_overlap_correlation']:.4f}, "
|
||||
f"boundary jump={metrics['audio_boundary_jump']:.6f}"
|
||||
)
|
||||
|
||||
|
||||
def format_seam_decision(index: int, decision: SeamDecision, *, full: bool) -> str:
|
||||
fallback = decision.fallback_reason or "none"
|
||||
if not full:
|
||||
return (
|
||||
f"seam {index-1}->{index}: cut=-{decision.cut_rewind_frames}f, "
|
||||
f"score {decision.legacy_score:.4f}->{decision.corrected_score:.4f}, "
|
||||
f"video blend={decision.video_blend_frames}f, "
|
||||
f"audio offset={decision.audio_metrics.offset_samples:+d} samples, "
|
||||
f"fallback={fallback}"
|
||||
)
|
||||
return (
|
||||
f"Seam {index-1}->{index}\n"
|
||||
f" Legacy score : {decision.legacy_score:.6f}\n"
|
||||
f" Selected cut : -{decision.cut_rewind_frames} frames\n"
|
||||
f" Cut score : {decision.cut_score:.6f}\n"
|
||||
f" Corrected score : {decision.corrected_score:.6f}\n"
|
||||
f" Improvement : {decision.improvement * 100.0:.2f}%\n"
|
||||
f" Luminance gain : {decision.luma_gain:.4f}\n"
|
||||
f" Chroma bias : {decision.chroma_bias}\n"
|
||||
f" Video blend : {decision.video_blend_frames} frames\n"
|
||||
f" Audio correlation : "
|
||||
f"{decision.audio_metrics.correlation_before:.4f}"
|
||||
f"->{decision.audio_metrics.correlation_after:.4f}\n"
|
||||
f" Audio jump : "
|
||||
f"{decision.audio_metrics.boundary_jump_before:.6f}"
|
||||
f"->{decision.audio_metrics.boundary_jump_after:.6f}\n"
|
||||
f" Audio offset : {decision.audio_metrics.offset_samples:+d} samples\n"
|
||||
f" Audio crossfade : {decision.audio_crossfade_samples} samples\n"
|
||||
f" Fallback : {fallback}"
|
||||
)
|
||||
def _audio_waveform(audio:dict[str,Any]):
|
||||
waveform=audio.get("waveform"); sample_rate=int(audio.get("sample_rate",0))
|
||||
if not torch.is_tensor(waveform) or waveform.ndim!=3: raise ValueError("audio waveform must be [B,C,S]")
|
||||
if sample_rate<=0: raise ValueError("audio sample_rate is invalid")
|
||||
return waveform.to(dtype=torch.float32),sample_rate
|
||||
def _normalized_correlation(a,b):
|
||||
length=min(int(a.numel()),int(b.numel()))
|
||||
if length<8:return 0.0
|
||||
a=a.reshape(-1)[:length].float(); b=b.reshape(-1)[:length].float(); a=a-a.mean(); b=b-b.mean(); denom=torch.sqrt(torch.sum(a.square())*torch.sum(b.square())).clamp_min(1e-12); value=float((torch.sum(a*b)/denom).item()); return max(-1.0,min(1.0,value)) if math.isfinite(value) else 0.0
|
||||
def overlap_diagnostics(*,previous_images,previous_audio,next_raw_images,next_raw_audio,context_frames):
|
||||
frame_count=min(int(context_frames),int(previous_images.shape[0]),int(next_raw_images.shape[0]))
|
||||
if frame_count<=0:return {"video_overlap_mae":0.0,"video_overlap_psnr":99.0,"audio_overlap_correlation":0.0,"audio_boundary_jump":0.0}
|
||||
prev=previous_images[-frame_count:].detach().to(dtype=torch.float32); nxt=next_raw_images[:frame_count].detach().to(dtype=torch.float32); mse=float(torch.mean((prev-nxt).square()).item()); mae=float(torch.mean(torch.abs(prev-nxt)).item()); psnr=99.0 if mse<=1e-12 else max(0.0,-10.0*math.log10(mse))
|
||||
prev_wave,prev_sr=_audio_waveform(previous_audio); next_wave,next_sr=_audio_waveform(next_raw_audio)
|
||||
if prev_sr!=next_sr: raise ValueError("diagnostic audio sample rates do not match")
|
||||
samples=min(int(round(frame_count/FPS*prev_sr)),int(prev_wave.shape[-1]),int(next_wave.shape[-1])); correlation=_normalized_correlation(prev_wave[...,-samples:],next_wave[...,:samples]) if samples>0 else 0.0; jump=float(torch.mean(torch.abs(prev_wave[...,-1]-next_wave[...,0])).item()) if prev_wave.shape[-1] and next_wave.shape[-1] else 0.0
|
||||
return {"video_overlap_mae":mae,"video_overlap_psnr":psnr,"audio_overlap_correlation":correlation,"audio_boundary_jump":jump}
|
||||
def av_duration_error_samples(images,audio):
|
||||
waveform,sample_rate=_audio_waveform(audio); return int(waveform.shape[-1])-int(round(float(images.shape[0])/FPS*sample_rate))
|
||||
def format_join_metrics(index,metrics): return f"join {index-1}->{index}: video MAE={metrics['video_overlap_mae']:.6f}, PSNR={metrics['video_overlap_psnr']:.2f}dB, audio corr={metrics['audio_overlap_correlation']:.4f}, boundary jump={metrics['audio_boundary_jump']:.6f}"
|
||||
def format_seam_decision(index,decision:SeamDecision,*,full:bool):
|
||||
fallback=decision.fallback_reason or "none"
|
||||
if not full:return f"seam {index-1}->{index}: cut=-{decision.cut_rewind_frames}f, score {decision.legacy_score:.4f}->{decision.corrected_score:.4f}, video blend={decision.video_blend_frames}f, audio offset={decision.audio_metrics.offset_samples:+d} samples, fallback={fallback}"
|
||||
return (f"Seam {index-1}->{index}\n Legacy score : {decision.legacy_score:.6f}\n Selected cut : -{decision.cut_rewind_frames} frames\n Cut score : {decision.cut_score:.6f}\n Corrected score : {decision.corrected_score:.6f}\n Improvement : {decision.improvement*100.0:.2f}%\n Luminance gain : {decision.luma_gain:.4f}\n Chroma bias : {decision.chroma_bias}\n Video blend : {decision.video_blend_frames} frames\n Audio correlation : {decision.audio_metrics.correlation_before:.4f}->{decision.audio_metrics.correlation_after:.4f}\n Audio jump : {decision.audio_metrics.boundary_jump_before:.6f}->{decision.audio_metrics.boundary_jump_after:.6f}\n Audio offset : {decision.audio_metrics.offset_samples:+d} samples\n Audio crossfade : {decision.audio_crossfade_samples} samples\n Fallback : {fallback}")
|
||||
|
||||
+322
-5
@@ -86,8 +86,8 @@ def _repair_v200_example_widget_shift(
|
||||
class H3ContinuumSamplerV2:
|
||||
DESCRIPTION = (
|
||||
"Integrated N-chunk MiniMax H3 continuation sampler. Precomputes prompt/image "
|
||||
"conditioning, isolates accelerator runtime state per chunk, captures latent "
|
||||
"state before decoding, then decodes and assembles the final AV stream."
|
||||
"conditioning, isolates accelerator runtime state per chunk, captures "
|
||||
"latent state before decoding, then decodes and assembles the final AV stream."
|
||||
)
|
||||
SEARCH_ALIASES = [
|
||||
"H3 long video sampler",
|
||||
@@ -228,7 +228,7 @@ class H3ContinuumSamplerV2:
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": "Controlled by Settings > H3 Continuum > Show latent preview.",
|
||||
"tooltip": "Show the in-progress latent thumbnail for this sampler node.",
|
||||
},
|
||||
),
|
||||
**{
|
||||
@@ -236,7 +236,7 @@ class H3ContinuumSamplerV2:
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": f"Connect Text (Multiline) for Clip {index}.",
|
||||
"tooltip": f"Optional prompt override for Clip {index}.",
|
||||
},
|
||||
)
|
||||
for index in range(1, 17)
|
||||
@@ -360,7 +360,16 @@ class H3ContinuumPromptPlanPreview:
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prompt_mode": (PROMPT_MODE_OPTIONS, {"default": PROMPT_MODE_OPTIONS[0]}),
|
||||
"prompt_mode": (
|
||||
PROMPT_FORMAT_OPTIONS,
|
||||
{
|
||||
"default": PROMPT_FORMAT_OPTIONS[0],
|
||||
"tooltip": (
|
||||
"Auto detects Fixed text, --- separated List text, or "
|
||||
"[0-5s] / [Chunk 1] Timeline sections."
|
||||
),
|
||||
},
|
||||
),
|
||||
"prompt_script": ("STRING", {"default": "Prompt", "multiline": True}),
|
||||
"chunks": ("INT", {"default": 3, "min": 1, "max": 16, "step": 1}),
|
||||
"chunk_seconds": (
|
||||
@@ -458,7 +467,311 @@ class H3ContinuumSessionInfo:
|
||||
return session, state, report
|
||||
|
||||
|
||||
class H3ContinuumClipOverrides:
|
||||
"""Build a static prompt-override pack without per-clip sockets."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prompt_mode": (
|
||||
PROMPT_FORMAT_OPTIONS,
|
||||
{
|
||||
"default": PROMPT_FORMAT_OPTIONS[0],
|
||||
"display_name": "Prompt Format",
|
||||
},
|
||||
),
|
||||
"override_script": (
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"display_name": "Override Script",
|
||||
"tooltip": (
|
||||
"Connect one Text (Multiline). Use ---, [Chunk N], or "
|
||||
"timeline sections according to Prompt Format."
|
||||
),
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("H3_CONTINUUM_CLIP_OVERRIDES",)
|
||||
RETURN_NAMES = ("prompt_overrides",)
|
||||
FUNCTION = "build"
|
||||
CATEGORY = CATEGORY
|
||||
|
||||
def build(self, prompt_mode, override_script):
|
||||
return (
|
||||
{
|
||||
"prompt_mode": str(prompt_mode),
|
||||
"prompt_script": str(override_script),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class H3ContinuumAdvanced:
|
||||
"""Package optional continuation inputs and infrequent sampler settings."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"audio_continuity": ("BOOLEAN", {"default": True}),
|
||||
"exact_total_duration": ("BOOLEAN", {"default": True}),
|
||||
"diagnostics": (DIAGNOSTICS_OPTIONS, {"default": DIAGNOSTICS_OPTIONS[0]}),
|
||||
"reroll_from_chunk": (
|
||||
"INT",
|
||||
{"default": 0, "min": 0, "max": 16, "step": 1},
|
||||
),
|
||||
"reroll_nonce": (
|
||||
"INT",
|
||||
{"default": 0, "min": 0, "max": 0xFFFFFFFF, "step": 1},
|
||||
),
|
||||
"strict_compatibility": ("BOOLEAN", {"default": True}),
|
||||
"debug": ("BOOLEAN", {"default": False}),
|
||||
"show_preview": ("BOOLEAN", {"default": True}),
|
||||
},
|
||||
"optional": {
|
||||
"last_frame": ("IMAGE",),
|
||||
"session": ("H3_CONTINUUM_SESSION",),
|
||||
"initial_state": ("H3_CONTINUUM_STATE",),
|
||||
"prompt_plan": ("H3_CONTINUUM_PROMPT_PLAN",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("H3_CONTINUUM_ADVANCED",)
|
||||
RETURN_NAMES = ("advanced",)
|
||||
FUNCTION = "build"
|
||||
CATEGORY = CATEGORY
|
||||
|
||||
def build(
|
||||
self,
|
||||
audio_continuity,
|
||||
exact_total_duration,
|
||||
diagnostics,
|
||||
reroll_from_chunk,
|
||||
reroll_nonce,
|
||||
strict_compatibility,
|
||||
debug,
|
||||
show_preview,
|
||||
last_frame=None,
|
||||
session=None,
|
||||
initial_state=None,
|
||||
prompt_plan=None,
|
||||
):
|
||||
return (
|
||||
{
|
||||
"audio_continuity": bool(audio_continuity),
|
||||
"exact_total_duration": bool(exact_total_duration),
|
||||
"diagnostics": diagnostics,
|
||||
"reroll_from_chunk": int(reroll_from_chunk),
|
||||
"reroll_nonce": int(reroll_nonce),
|
||||
"strict_compatibility": bool(strict_compatibility),
|
||||
"debug": bool(debug),
|
||||
"show_preview": bool(show_preview),
|
||||
"last_frame": last_frame,
|
||||
"session": session,
|
||||
"initial_state": initial_state,
|
||||
"prompt_plan": prompt_plan,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class H3ContinuumSampler:
|
||||
"""Stable, compact facade over the fully compatible V2 sampler core."""
|
||||
|
||||
DESCRIPTION = (
|
||||
"Compact H3 Continuum sampler facade. Uses the legacy V2 execution core "
|
||||
"without dynamic sockets or frontend display patches."
|
||||
)
|
||||
SEARCH_ALIASES = ["H3 Continuum", "MiniMax H3 long video"]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"clip": ("CLIP",),
|
||||
"video_vae": ("VAE",),
|
||||
"audio_vae": ("VAE",),
|
||||
"sampler": ("SAMPLER",),
|
||||
"sigmas": ("SIGMAS",),
|
||||
"sequence_prompt": (
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"display_name": "Sequence Prompt",
|
||||
"tooltip": "Connect one Text (Multiline) for the complete sequence.",
|
||||
},
|
||||
),
|
||||
"prompt_mode": (
|
||||
PROMPT_FORMAT_OPTIONS,
|
||||
{
|
||||
"default": PROMPT_FORMAT_OPTIONS[0],
|
||||
"display_name": "Prompt Format",
|
||||
},
|
||||
),
|
||||
"chunks": ("INT", {"default": 3, "min": 1, "max": 16, "step": 1}),
|
||||
"chunk_seconds": (
|
||||
"FLOAT",
|
||||
{"default": 5.0, "min": 4.0, "max": 15.0, "step": 0.1},
|
||||
),
|
||||
"width": ("INT", {"default": 1344, "min": 32, "max": 16384, "step": 32}),
|
||||
"height": ("INT", {"default": 768, "min": 32, "max": 16384, "step": 32}),
|
||||
"continuity": (
|
||||
V2_CONTINUITY_OPTIONS,
|
||||
{"default": V2_CONTINUITY_OPTIONS[1]},
|
||||
),
|
||||
"base_seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": True,
|
||||
},
|
||||
),
|
||||
"seam_correction": (
|
||||
SEAM_CORRECTION_OPTIONS,
|
||||
{"default": SEAM_CORRECTION_AUTO},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"first_frame": ("IMAGE",),
|
||||
"prompt_overrides": ("H3_CONTINUUM_CLIP_OVERRIDES",),
|
||||
"advanced": ("H3_CONTINUUM_ADVANCED",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_RESULT")
|
||||
RETURN_NAMES = ("images", "audio", "result")
|
||||
FUNCTION = "run"
|
||||
CATEGORY = CATEGORY
|
||||
|
||||
def run(
|
||||
self,
|
||||
model,
|
||||
clip,
|
||||
video_vae,
|
||||
audio_vae,
|
||||
sampler,
|
||||
sigmas,
|
||||
sequence_prompt,
|
||||
prompt_mode,
|
||||
chunks,
|
||||
chunk_seconds,
|
||||
width,
|
||||
height,
|
||||
continuity,
|
||||
base_seed,
|
||||
seam_correction,
|
||||
first_frame=None,
|
||||
prompt_overrides=None,
|
||||
advanced=None,
|
||||
):
|
||||
if prompt_overrides is not None and not isinstance(prompt_overrides, dict):
|
||||
raise TypeError("prompt_overrides must be an H3_CONTINUUM_CLIP_OVERRIDES pack")
|
||||
if advanced is not None and not isinstance(advanced, dict):
|
||||
raise TypeError("advanced must be an H3_CONTINUUM_ADVANCED pack")
|
||||
|
||||
advanced_values = {
|
||||
"audio_continuity": True,
|
||||
"exact_total_duration": True,
|
||||
"diagnostics": DIAGNOSTICS_OPTIONS[0],
|
||||
"reroll_from_chunk": 0,
|
||||
"reroll_nonce": 0,
|
||||
"strict_compatibility": True,
|
||||
"debug": False,
|
||||
"show_preview": True,
|
||||
"last_frame": None,
|
||||
"session": None,
|
||||
"initial_state": None,
|
||||
"prompt_plan": None,
|
||||
}
|
||||
if advanced:
|
||||
advanced_values.update(advanced)
|
||||
|
||||
clip_prompt_inputs = {}
|
||||
if prompt_overrides:
|
||||
override_plan = make_prompt_plan(
|
||||
mode=prompt_overrides.get("prompt_mode", PROMPT_FORMAT_OPTIONS[0]),
|
||||
script=prompt_overrides.get("prompt_script", ""),
|
||||
chunks=int(chunks),
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
)
|
||||
for index, prompt in enumerate(override_plan.get("prompts", []), start=1):
|
||||
if isinstance(prompt, str) and prompt.strip():
|
||||
clip_prompt_inputs[f"clip_{index}_prompt"] = prompt
|
||||
|
||||
images, audio, last_state, session, report = H3ContinuumSamplerV2().run(
|
||||
model=model,
|
||||
clip=clip,
|
||||
video_vae=video_vae,
|
||||
audio_vae=audio_vae,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
prompt_mode=prompt_mode,
|
||||
prompt_script=sequence_prompt,
|
||||
chunks=chunks,
|
||||
chunk_seconds=chunk_seconds,
|
||||
width=width,
|
||||
height=height,
|
||||
continuity=continuity,
|
||||
base_seed=base_seed,
|
||||
audio_continuity=advanced_values["audio_continuity"],
|
||||
exact_total_duration=advanced_values["exact_total_duration"],
|
||||
diagnostics=advanced_values["diagnostics"],
|
||||
reroll_from_chunk=advanced_values["reroll_from_chunk"],
|
||||
reroll_nonce=advanced_values["reroll_nonce"],
|
||||
strict_compatibility=advanced_values["strict_compatibility"],
|
||||
debug=advanced_values["debug"],
|
||||
seam_correction=seam_correction,
|
||||
first_frame=first_frame,
|
||||
last_frame=advanced_values["last_frame"],
|
||||
session=advanced_values["session"],
|
||||
initial_state=advanced_values["initial_state"],
|
||||
prompt_plan=advanced_values["prompt_plan"],
|
||||
sequence_prompt=sequence_prompt,
|
||||
show_preview=advanced_values["show_preview"],
|
||||
**clip_prompt_inputs,
|
||||
)
|
||||
return (
|
||||
images,
|
||||
audio,
|
||||
{
|
||||
"last_state": last_state,
|
||||
"session": session,
|
||||
"report": report,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class H3ContinuumResult:
|
||||
"""Expand the compact facade result pack only when advanced outputs are needed."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {"required": {"result": ("H3_CONTINUUM_RESULT",)}}
|
||||
|
||||
RETURN_TYPES = ("H3_CONTINUUM_STATE", "H3_CONTINUUM_SESSION", "STRING")
|
||||
RETURN_NAMES = ("last_state", "session", "report")
|
||||
FUNCTION = "unpack"
|
||||
CATEGORY = CATEGORY
|
||||
|
||||
def unpack(self, result):
|
||||
if not isinstance(result, dict):
|
||||
raise TypeError("result must be an H3_CONTINUUM_RESULT pack")
|
||||
missing = {"last_state", "session", "report"} - set(result)
|
||||
if missing:
|
||||
raise ValueError("H3_CONTINUUM_RESULT is missing: " + ", ".join(sorted(missing)))
|
||||
return result["last_state"], result["session"], str(result["report"])
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumSampler": H3ContinuumSampler,
|
||||
"H3ContinuumClipOverrides": H3ContinuumClipOverrides,
|
||||
"H3ContinuumAdvanced": H3ContinuumAdvanced,
|
||||
"H3ContinuumResult": H3ContinuumResult,
|
||||
"H3ContinuumSamplerV2": H3ContinuumSamplerV2,
|
||||
"H3ContinuumPromptPlanPreview": H3ContinuumPromptPlanPreview,
|
||||
"H3ContinuumSaveSession": H3ContinuumSaveSession,
|
||||
@@ -467,6 +780,10 @@ NODE_CLASS_MAPPINGS = {
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumSampler": "H3 Continuum Sampler",
|
||||
"H3ContinuumClipOverrides": "H3 Continuum Clip Overrides",
|
||||
"H3ContinuumAdvanced": "H3 Continuum Advanced",
|
||||
"H3ContinuumResult": "H3 Continuum Result",
|
||||
"H3ContinuumSamplerV2": "H3 Continuum Sampler V2",
|
||||
"H3ContinuumPromptPlanPreview": "H3 Continuum Prompt Plan",
|
||||
"H3ContinuumSaveSession": "H3 Continuum Save Session",
|
||||
|
||||
+105
-303
@@ -3,329 +3,131 @@
|
||||
The parser is intentionally deterministic and side-effect free so saved workflows
|
||||
can be reproduced without depending on UI state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import hashlib, json, re
|
||||
from typing import Any
|
||||
|
||||
from ..constants import (
|
||||
PROMPT_FORMAT_AUTO,
|
||||
PROMPT_FORMAT_FIXED,
|
||||
PROMPT_FORMAT_LIST,
|
||||
PROMPT_FORMAT_TIMELINE,
|
||||
PROMPT_MODE_FIXED,
|
||||
PROMPT_MODE_LIST,
|
||||
PROMPT_MODE_TIMELINE,
|
||||
PROMPT_PLAN_MAGIC,
|
||||
PROMPT_FORMAT_AUTO,PROMPT_FORMAT_FIXED,PROMPT_FORMAT_LIST,PROMPT_FORMAT_TIMELINE,
|
||||
PROMPT_MODE_FIXED,PROMPT_MODE_LIST,PROMPT_MODE_TIMELINE,PROMPT_PLAN_MAGIC,
|
||||
)
|
||||
from ..version import PROMPT_PLAN_SCHEMA_VERSION
|
||||
|
||||
_TIMELINE_HEADER = re.compile(
|
||||
r"^\s*\[\s*(?P<start>\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*[-–—]\s*"
|
||||
r"(?P<end>\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*\]\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CHUNK_HEADER = re.compile(
|
||||
r"^\s*\[\s*(?:chunk|clip)\s*(?P<index>\d+)\s*\]\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_LIST_SEPARATOR = re.compile(r"^\s*---+\s*$", re.MULTILINE)
|
||||
|
||||
|
||||
class PromptPlanError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def prompt_hash(prompt: str) -> str:
|
||||
return hashlib.sha256(prompt.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _normalize_prompt(value: str, *, label: str) -> str:
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
raise PromptPlanError(f"{label} is empty")
|
||||
_TIMELINE_HEADER=re.compile(r"^\s*\[\s*(?P<start>\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*[-–—]\s*(?P<end>\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*\]\s*$",re.IGNORECASE)
|
||||
_CHUNK_HEADER=re.compile(r"^\s*\[\s*(?:chunk|clip)\s*(?P<index>\d+)\s*\]\s*$",re.IGNORECASE)
|
||||
_LIST_SEPARATOR=re.compile(r"^\s*---+\s*$",re.MULTILINE)
|
||||
class PromptPlanError(ValueError): pass
|
||||
def prompt_hash(prompt:str)->str: return hashlib.sha256(prompt.encode("utf-8")).hexdigest()
|
||||
def _normalize_prompt(value:str,*,label:str)->str:
|
||||
text=str(value).strip()
|
||||
if not text: raise PromptPlanError(f"{label} is empty")
|
||||
return text
|
||||
|
||||
|
||||
def _parse_json_list(script: str) -> list[str] | None:
|
||||
stripped = script.lstrip()
|
||||
if not stripped.startswith("["):
|
||||
return None
|
||||
try:
|
||||
value = json.loads(script)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
||||
raise PromptPlanError("JSON prompt list must be an array of strings")
|
||||
return [_normalize_prompt(item, label=f"prompt {index + 1}") for index, item in enumerate(value)]
|
||||
|
||||
|
||||
def _parse_list(script: str, chunks: int) -> tuple[list[str], list[str]]:
|
||||
values = _parse_json_list(script)
|
||||
if values is None:
|
||||
values = [
|
||||
_normalize_prompt(part, label=f"prompt {index + 1}")
|
||||
for index, part in enumerate(_LIST_SEPARATOR.split(script))
|
||||
if part.strip()
|
||||
]
|
||||
if not values:
|
||||
raise PromptPlanError("prompt list contains no prompts")
|
||||
notes: list[str] = []
|
||||
if len(values) < chunks:
|
||||
notes.append(f"repeated the last prompt for {chunks - len(values)} chunk(s)")
|
||||
values.extend([values[-1]] * (chunks - len(values)))
|
||||
elif len(values) > chunks:
|
||||
notes.append(f"ignored {len(values) - chunks} extra prompt(s)")
|
||||
values = values[:chunks]
|
||||
return values, notes
|
||||
|
||||
|
||||
def _parse_timeline_sections(script: str) -> list[dict[str, Any]]:
|
||||
sections: list[dict[str, Any]] = []
|
||||
current: dict[str, Any] | None = None
|
||||
body: list[str] = []
|
||||
|
||||
def finish() -> None:
|
||||
nonlocal current, body
|
||||
def _parse_json_list(script:str):
|
||||
stripped=script.lstrip()
|
||||
if not stripped.startswith("["): return None
|
||||
try: value=json.loads(script)
|
||||
except json.JSONDecodeError: return None
|
||||
if not isinstance(value,list) or not all(isinstance(item,str) for item in value): raise PromptPlanError("JSON prompt list must be an array of strings")
|
||||
return [_normalize_prompt(item,label=f"prompt {index+1}") for index,item in enumerate(value)]
|
||||
def _parse_list(script,chunks):
|
||||
values=_parse_json_list(script)
|
||||
if values is None: values=[_normalize_prompt(part,label=f"prompt {index+1}") for index,part in enumerate(_LIST_SEPARATOR.split(script)) if part.strip()]
|
||||
if not values: raise PromptPlanError("prompt list contains no prompts")
|
||||
notes=[]
|
||||
if len(values)<chunks: notes.append(f"repeated the last prompt for {chunks-len(values)} chunk(s)"); values.extend([values[-1]]*(chunks-len(values)))
|
||||
elif len(values)>chunks: notes.append(f"ignored {len(values)-chunks} extra prompt(s)"); values=values[:chunks]
|
||||
return values,notes
|
||||
def _parse_timeline_sections(script):
|
||||
sections=[]; current=None; body=[]
|
||||
def finish():
|
||||
nonlocal current,body
|
||||
if current is None:
|
||||
if any(line.strip() for line in body):
|
||||
raise PromptPlanError(
|
||||
"timeline text before the first [0-5s] or [Chunk 1] header is not allowed"
|
||||
)
|
||||
body = []
|
||||
return
|
||||
current["prompt"] = _normalize_prompt("\n".join(body), label="timeline section")
|
||||
sections.append(current)
|
||||
current = None
|
||||
body = []
|
||||
|
||||
if any(line.strip() for line in body): raise PromptPlanError("timeline text before the first [0-5s] or [Chunk 1] header is not allowed")
|
||||
body=[]; return
|
||||
current["prompt"]=_normalize_prompt("\n".join(body),label="timeline section"); sections.append(current); current=None; body=[]
|
||||
for line in str(script).splitlines():
|
||||
time_match = _TIMELINE_HEADER.match(line)
|
||||
chunk_match = _CHUNK_HEADER.match(line)
|
||||
time_match=_TIMELINE_HEADER.match(line); chunk_match=_CHUNK_HEADER.match(line)
|
||||
if time_match or chunk_match:
|
||||
finish()
|
||||
if time_match:
|
||||
start = float(time_match.group("start"))
|
||||
end = float(time_match.group("end"))
|
||||
if not end > start:
|
||||
raise PromptPlanError(f"timeline range must increase: {line.strip()}")
|
||||
current = {"kind": "time", "start": start, "end": end}
|
||||
start=float(time_match.group("start")); end=float(time_match.group("end"))
|
||||
if not end>start: raise PromptPlanError(f"timeline range must increase: {line.strip()}")
|
||||
current={"kind":"time","start":start,"end":end}
|
||||
else:
|
||||
index = int(chunk_match.group("index"))
|
||||
if index < 1:
|
||||
raise PromptPlanError("chunk numbers are one-based")
|
||||
current = {"kind": "chunk", "index": index}
|
||||
index=int(chunk_match.group("index"))
|
||||
if index<1: raise PromptPlanError("chunk numbers are one-based")
|
||||
current={"kind":"chunk","index":index}
|
||||
continue
|
||||
body.append(line)
|
||||
finish()
|
||||
if not sections:
|
||||
raise PromptPlanError("timeline contains no sections")
|
||||
if not sections: raise PromptPlanError("timeline contains no sections")
|
||||
return sections
|
||||
|
||||
|
||||
def _parse_timeline(script: str, chunks: int, chunk_seconds: float) -> tuple[list[str], list[str]]:
|
||||
sections = _parse_timeline_sections(script)
|
||||
prompts: list[str] = []
|
||||
notes: list[str] = []
|
||||
for chunk_index in range(1, chunks + 1):
|
||||
chunk_sections = [
|
||||
item for item in sections if item["kind"] == "chunk" and item["index"] == chunk_index
|
||||
]
|
||||
if len(chunk_sections) > 1:
|
||||
raise PromptPlanError(f"timeline defines Chunk {chunk_index} more than once")
|
||||
if chunk_sections:
|
||||
prompts.append(chunk_sections[0]["prompt"])
|
||||
continue
|
||||
|
||||
start = (chunk_index - 1) * chunk_seconds
|
||||
end = chunk_index * chunk_seconds
|
||||
scored: list[tuple[float, int, dict[str, Any]]] = []
|
||||
for order, item in enumerate(sections):
|
||||
if item["kind"] != "time":
|
||||
continue
|
||||
overlap = max(0.0, min(end, item["end"]) - max(start, item["start"]))
|
||||
if overlap > 0:
|
||||
scored.append((overlap, -order, item))
|
||||
if not scored:
|
||||
raise PromptPlanError(
|
||||
f"timeline does not cover chunk {chunk_index} ({start:.3f}-{end:.3f}s)"
|
||||
)
|
||||
scored.sort(reverse=True, key=lambda row: (row[0], row[1]))
|
||||
prompts.append(scored[0][2]["prompt"])
|
||||
if len(scored) > 1 and abs(scored[0][0] - scored[1][0]) < 1e-9:
|
||||
notes.append(f"chunk {chunk_index} had an equal-overlap timeline tie; used the earlier section")
|
||||
return prompts, notes
|
||||
|
||||
|
||||
def detect_prompt_mode(script: str) -> str:
|
||||
"""Detect the stable Prompt Plan mode without changing the input text."""
|
||||
|
||||
text = str(script)
|
||||
def _parse_timeline(script,chunks,chunk_seconds):
|
||||
sections=_parse_timeline_sections(script); prompts=[]; notes=[]
|
||||
for chunk_index in range(1,chunks+1):
|
||||
chunk_sections=[item for item in sections if item["kind"]=="chunk" and item["index"]==chunk_index]
|
||||
if len(chunk_sections)>1: raise PromptPlanError(f"timeline defines Chunk {chunk_index} more than once")
|
||||
if chunk_sections: prompts.append(chunk_sections[0]["prompt"]); continue
|
||||
start=(chunk_index-1)*chunk_seconds; end=chunk_index*chunk_seconds; scored=[]
|
||||
for order,item in enumerate(sections):
|
||||
if item["kind"]!="time": continue
|
||||
overlap=max(0.0,min(end,item["end"])-max(start,item["start"]))
|
||||
if overlap>0: scored.append((overlap,-order,item))
|
||||
if not scored: raise PromptPlanError(f"timeline does not cover chunk {chunk_index} ({start:.3f}-{end:.3f}s)")
|
||||
scored.sort(reverse=True,key=lambda row:(row[0],row[1])); prompts.append(scored[0][2]["prompt"])
|
||||
if len(scored)>1 and abs(scored[0][0]-scored[1][0])<1e-9: notes.append(f"chunk {chunk_index} had an equal-overlap timeline tie; used the earlier section")
|
||||
return prompts,notes
|
||||
def detect_prompt_mode(script):
|
||||
text=str(script)
|
||||
for line in text.splitlines():
|
||||
if _TIMELINE_HEADER.match(line) or _CHUNK_HEADER.match(line):
|
||||
return PROMPT_MODE_TIMELINE
|
||||
if _parse_json_list(text) is not None or _LIST_SEPARATOR.search(text):
|
||||
return PROMPT_MODE_LIST
|
||||
if _TIMELINE_HEADER.match(line) or _CHUNK_HEADER.match(line): return PROMPT_MODE_TIMELINE
|
||||
if _parse_json_list(text) is not None or _LIST_SEPARATOR.search(text): return PROMPT_MODE_LIST
|
||||
return PROMPT_MODE_FIXED
|
||||
|
||||
|
||||
def resolve_prompt_mode(mode: str, script: str) -> str:
|
||||
explicit = {
|
||||
PROMPT_FORMAT_FIXED: PROMPT_MODE_FIXED,
|
||||
PROMPT_FORMAT_LIST: PROMPT_MODE_LIST,
|
||||
PROMPT_FORMAT_TIMELINE: PROMPT_MODE_TIMELINE,
|
||||
}
|
||||
if mode == PROMPT_FORMAT_AUTO:
|
||||
return detect_prompt_mode(script)
|
||||
if mode in explicit:
|
||||
return explicit[mode]
|
||||
if mode in (PROMPT_MODE_FIXED, PROMPT_MODE_LIST, PROMPT_MODE_TIMELINE):
|
||||
return mode
|
||||
def resolve_prompt_mode(mode,script):
|
||||
explicit={PROMPT_FORMAT_FIXED:PROMPT_MODE_FIXED,PROMPT_FORMAT_LIST:PROMPT_MODE_LIST,PROMPT_FORMAT_TIMELINE:PROMPT_MODE_TIMELINE}
|
||||
if mode==PROMPT_FORMAT_AUTO: return detect_prompt_mode(script)
|
||||
if mode in explicit: return explicit[mode]
|
||||
if mode in (PROMPT_MODE_FIXED,PROMPT_MODE_LIST,PROMPT_MODE_TIMELINE): return mode
|
||||
raise PromptPlanError(f"unknown prompt format: {mode!r}")
|
||||
|
||||
|
||||
def make_prompt_plan(
|
||||
*, mode: str, script: str, chunks: int, chunk_seconds: float,
|
||||
) -> dict[str, Any]:
|
||||
chunks = int(chunks)
|
||||
chunk_seconds = float(chunk_seconds)
|
||||
if not (1 <= chunks <= 16):
|
||||
raise PromptPlanError("chunks must be between 1 and 16")
|
||||
if not (4.0 <= chunk_seconds <= 15.0):
|
||||
raise PromptPlanError("chunk_seconds must be between 4.0 and 15.0 for native H3")
|
||||
|
||||
resolved_mode = resolve_prompt_mode(mode, script)
|
||||
notes: list[str] = []
|
||||
if mode == PROMPT_FORMAT_AUTO:
|
||||
detected = {
|
||||
PROMPT_MODE_FIXED: PROMPT_FORMAT_FIXED,
|
||||
PROMPT_MODE_LIST: PROMPT_FORMAT_LIST,
|
||||
PROMPT_MODE_TIMELINE: PROMPT_FORMAT_TIMELINE,
|
||||
}[resolved_mode]
|
||||
notes.append(f"Auto detected {detected}")
|
||||
|
||||
if resolved_mode == PROMPT_MODE_FIXED:
|
||||
prompt = _normalize_prompt(script, label="fixed prompt")
|
||||
prompts = [prompt] * chunks
|
||||
elif resolved_mode == PROMPT_MODE_LIST:
|
||||
prompts, parse_notes = _parse_list(script, chunks)
|
||||
notes.extend(parse_notes)
|
||||
elif resolved_mode == PROMPT_MODE_TIMELINE:
|
||||
prompts, parse_notes = _parse_timeline(script, chunks, chunk_seconds)
|
||||
notes.extend(parse_notes)
|
||||
else:
|
||||
raise PromptPlanError(f"unknown prompt mode: {resolved_mode!r}")
|
||||
|
||||
return {
|
||||
"magic": PROMPT_PLAN_MAGIC,
|
||||
"schema_version": PROMPT_PLAN_SCHEMA_VERSION,
|
||||
"mode": resolved_mode,
|
||||
"chunks": chunks,
|
||||
"chunk_seconds": chunk_seconds,
|
||||
"prompts": prompts,
|
||||
"hashes": [prompt_hash(prompt) for prompt in prompts],
|
||||
"notes": notes,
|
||||
}
|
||||
|
||||
|
||||
def validate_prompt_plan(plan: dict[str, Any]) -> dict[str, Any]:
|
||||
if not isinstance(plan, dict) or plan.get("magic") != PROMPT_PLAN_MAGIC:
|
||||
raise PromptPlanError("invalid H3 Continuum prompt plan")
|
||||
if int(plan.get("schema_version", -1)) != PROMPT_PLAN_SCHEMA_VERSION:
|
||||
raise PromptPlanError(
|
||||
f"unsupported prompt-plan schema {plan.get('schema_version')}; "
|
||||
f"expected {PROMPT_PLAN_SCHEMA_VERSION}"
|
||||
)
|
||||
chunks = int(plan.get("chunks", 0))
|
||||
prompts = plan.get("prompts")
|
||||
hashes = plan.get("hashes")
|
||||
if not (1 <= chunks <= 16):
|
||||
raise PromptPlanError("prompt-plan chunks are invalid")
|
||||
if not isinstance(prompts, list) or len(prompts) != chunks:
|
||||
raise PromptPlanError("prompt-plan prompt count does not match chunks")
|
||||
if not isinstance(hashes, list) or len(hashes) != chunks:
|
||||
raise PromptPlanError("prompt-plan hash count does not match chunks")
|
||||
for index, prompt in enumerate(prompts):
|
||||
_normalize_prompt(prompt, label=f"prompt {index + 1}")
|
||||
if hashes[index] != prompt_hash(prompt):
|
||||
raise PromptPlanError(f"prompt-plan hash mismatch at chunk {index + 1}")
|
||||
def make_prompt_plan(*,mode,script,chunks,chunk_seconds):
|
||||
chunks=int(chunks); chunk_seconds=float(chunk_seconds)
|
||||
if not 1<=chunks<=16: raise PromptPlanError("chunks must be between 1 and 16")
|
||||
if not 4.0<=chunk_seconds<=15.0: raise PromptPlanError("chunk_seconds must be between 4.0 and 15.0 for native H3")
|
||||
resolved_mode=resolve_prompt_mode(mode,script); notes=[]
|
||||
if mode==PROMPT_FORMAT_AUTO:
|
||||
detected={PROMPT_MODE_FIXED:PROMPT_FORMAT_FIXED,PROMPT_MODE_LIST:PROMPT_FORMAT_LIST,PROMPT_MODE_TIMELINE:PROMPT_FORMAT_TIMELINE}[resolved_mode]; notes.append(f"Auto detected {detected}")
|
||||
if resolved_mode==PROMPT_MODE_FIXED: prompt=_normalize_prompt(script,label="fixed prompt"); prompts=[prompt]*chunks
|
||||
elif resolved_mode==PROMPT_MODE_LIST: prompts,parse_notes=_parse_list(script,chunks); notes.extend(parse_notes)
|
||||
elif resolved_mode==PROMPT_MODE_TIMELINE: prompts,parse_notes=_parse_timeline(script,chunks,chunk_seconds); notes.extend(parse_notes)
|
||||
else: raise PromptPlanError(f"unknown prompt mode: {resolved_mode!r}")
|
||||
return {"magic":PROMPT_PLAN_MAGIC,"schema_version":PROMPT_PLAN_SCHEMA_VERSION,"mode":resolved_mode,"chunks":chunks,"chunk_seconds":chunk_seconds,"prompts":prompts,"hashes":[prompt_hash(p) for p in prompts],"notes":notes}
|
||||
def validate_prompt_plan(plan):
|
||||
if not isinstance(plan,dict) or plan.get("magic")!=PROMPT_PLAN_MAGIC: raise PromptPlanError("invalid H3 Continuum prompt plan")
|
||||
if int(plan.get("schema_version",-1))!=PROMPT_PLAN_SCHEMA_VERSION: raise PromptPlanError(f"unsupported prompt-plan schema {plan.get('schema_version')}; expected {PROMPT_PLAN_SCHEMA_VERSION}")
|
||||
chunks=int(plan.get("chunks",0)); prompts=plan.get("prompts"); hashes=plan.get("hashes")
|
||||
if not 1<=chunks<=16: raise PromptPlanError("prompt-plan chunks are invalid")
|
||||
if not isinstance(prompts,list) or len(prompts)!=chunks: raise PromptPlanError("prompt-plan prompt count does not match chunks")
|
||||
if not isinstance(hashes,list) or len(hashes)!=chunks: raise PromptPlanError("prompt-plan hash count does not match chunks")
|
||||
for index,prompt in enumerate(prompts):
|
||||
_normalize_prompt(prompt,label=f"prompt {index+1}")
|
||||
if hashes[index]!=prompt_hash(prompt): raise PromptPlanError(f"prompt-plan hash mismatch at chunk {index+1}")
|
||||
return plan
|
||||
|
||||
|
||||
def build_sampler_prompt_plan(
|
||||
*,
|
||||
prompt_mode: str,
|
||||
prompt_script: str,
|
||||
sequence_prompt: str | None,
|
||||
prompt_plan: dict[str, Any] | None,
|
||||
chunks: int,
|
||||
chunk_seconds: float,
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve the fixed Sequence socket while preserving legacy fallbacks."""
|
||||
|
||||
chunks = int(chunks)
|
||||
chunk_seconds = float(chunk_seconds)
|
||||
if sequence_prompt is not None:
|
||||
return make_prompt_plan(
|
||||
mode=prompt_mode,
|
||||
script=sequence_prompt,
|
||||
chunks=chunks,
|
||||
chunk_seconds=chunk_seconds,
|
||||
)
|
||||
def build_sampler_prompt_plan(*,prompt_mode,prompt_script,sequence_prompt,prompt_plan,chunks,chunk_seconds):
|
||||
chunks=int(chunks); chunk_seconds=float(chunk_seconds)
|
||||
if sequence_prompt is not None: return make_prompt_plan(mode=prompt_mode,script=sequence_prompt,chunks=chunks,chunk_seconds=chunk_seconds)
|
||||
if prompt_plan is not None:
|
||||
plan = validate_prompt_plan(prompt_plan)
|
||||
if int(plan["chunks"]) != chunks:
|
||||
raise PromptPlanError(
|
||||
"connected prompt_plan chunks do not match the Sampler chunks widget"
|
||||
)
|
||||
if abs(float(plan["chunk_seconds"]) - chunk_seconds) > 1e-6:
|
||||
raise PromptPlanError(
|
||||
"connected prompt_plan chunk_seconds do not match the Sampler widget"
|
||||
)
|
||||
plan=validate_prompt_plan(prompt_plan)
|
||||
if int(plan["chunks"])!=chunks: raise PromptPlanError("connected prompt_plan chunks do not match the Sampler chunks widget")
|
||||
if abs(float(plan["chunk_seconds"])-chunk_seconds)>1e-6: raise PromptPlanError("connected prompt_plan chunk_seconds do not match the Sampler widget")
|
||||
return plan
|
||||
return make_prompt_plan(
|
||||
mode=prompt_mode,
|
||||
script=prompt_script,
|
||||
chunks=chunks,
|
||||
chunk_seconds=chunk_seconds,
|
||||
)
|
||||
|
||||
|
||||
def apply_prompt_overrides(
|
||||
plan: dict[str, Any], overrides: list[str | None]
|
||||
) -> dict[str, Any]:
|
||||
"""Replace selected chunk prompts without changing the prompt-plan schema."""
|
||||
|
||||
plan = validate_prompt_plan(plan)
|
||||
prompts = list(plan["prompts"])
|
||||
replaced: list[int] = []
|
||||
for index, value in enumerate(overrides[: int(plan["chunks"])]):
|
||||
if value is None:
|
||||
continue
|
||||
prompts[index] = _normalize_prompt(value, label=f"Clip {index + 1} Prompt")
|
||||
replaced.append(index + 1)
|
||||
if not replaced:
|
||||
return plan
|
||||
|
||||
result = dict(plan)
|
||||
result["mode"] = PROMPT_MODE_LIST
|
||||
result["prompts"] = prompts
|
||||
result["hashes"] = [prompt_hash(prompt) for prompt in prompts]
|
||||
result["notes"] = list(plan.get("notes") or []) + [
|
||||
"external Clip Prompt input(s): " + ", ".join(map(str, replaced))
|
||||
]
|
||||
return make_prompt_plan(mode=prompt_mode,script=prompt_script,chunks=chunks,chunk_seconds=chunk_seconds)
|
||||
def apply_prompt_overrides(plan,overrides):
|
||||
plan=validate_prompt_plan(plan); prompts=list(plan["prompts"]); replaced=[]
|
||||
for index,value in enumerate(overrides[:int(plan["chunks"])]):
|
||||
if value is None: continue
|
||||
prompts[index]=_normalize_prompt(value,label=f"Clip {index+1} Prompt"); replaced.append(index+1)
|
||||
if not replaced: return plan
|
||||
result=dict(plan); result["mode"]=PROMPT_MODE_LIST; result["prompts"]=prompts; result["hashes"]=[prompt_hash(p) for p in prompts]; result["notes"]=list(plan.get("notes") or [])+["external Clip Prompt input(s): "+", ".join(map(str,replaced))]
|
||||
return validate_prompt_plan(result)
|
||||
|
||||
|
||||
def prompt_plan_report(plan: dict[str, Any]) -> str:
|
||||
plan = validate_prompt_plan(plan)
|
||||
unique = len(set(plan["hashes"]))
|
||||
note = "; ".join(plan.get("notes") or [])
|
||||
result = (
|
||||
f"Prompt plan: {plan['mode']}; {plan['chunks']} chunks; "
|
||||
f"{unique} unique prompt(s); {plan['chunk_seconds']:.3f}s per chunk."
|
||||
)
|
||||
return result + (" " + note + "." if note else "")
|
||||
def prompt_plan_report(plan):
|
||||
plan=validate_prompt_plan(plan); unique=len(set(plan["hashes"])); note="; ".join(plan.get("notes") or [])
|
||||
result=f"Prompt plan: {plan['mode']}; {plan['chunks']} chunks; {unique} unique prompt(s); {plan['chunk_seconds']:.3f}s per chunk."
|
||||
return result+(" "+note+"." if note else "")
|
||||
|
||||
+117
-536
@@ -1,551 +1,132 @@
|
||||
"""Conservative CPU-only decoded seam correction for H3 Continuum V2.1."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .seam_types import AudioSeamMetrics, SeamDecision, VideoSeamMetrics
|
||||
COMPARE_SIZE=96; MAX_CUT_REWIND=3; MIN_RELATIVE_IMPROVEMENT=0.10; MIN_ABSOLUTE_IMPROVEMENT=0.003; SAFE_SCORE_LIMIT=0.35
|
||||
WEIGHT_PIXEL=0.35; WEIGHT_LUMA=0.20; WEIGHT_CHROMA=0.10; WEIGHT_EDGE=0.15; WEIGHT_TEMPORAL=0.20
|
||||
|
||||
COMPARE_SIZE = 96
|
||||
MAX_CUT_REWIND = 3
|
||||
MIN_RELATIVE_IMPROVEMENT = 0.10
|
||||
MIN_ABSOLUTE_IMPROVEMENT = 0.003
|
||||
SAFE_SCORE_LIMIT = 0.35
|
||||
|
||||
WEIGHT_PIXEL = 0.35
|
||||
WEIGHT_LUMA = 0.20
|
||||
WEIGHT_CHROMA = 0.10
|
||||
WEIGHT_EDGE = 0.15
|
||||
WEIGHT_TEMPORAL = 0.20
|
||||
|
||||
|
||||
def _video_shape(frames: torch.Tensor, label: str) -> None:
|
||||
if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3:
|
||||
raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]")
|
||||
|
||||
|
||||
def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor:
|
||||
_video_shape(frames, "video seam frames")
|
||||
rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32)
|
||||
rgb = rgb.permute(0, 3, 1, 2)
|
||||
if tuple(rgb.shape[-2:]) != (COMPARE_SIZE, COMPARE_SIZE):
|
||||
rgb = F.interpolate(
|
||||
rgb,
|
||||
size=(COMPARE_SIZE, COMPARE_SIZE),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0)
|
||||
if not bool(torch.isfinite(rgb).all().item()):
|
||||
raise ValueError("video seam comparison contains NaN or Inf")
|
||||
def _video_shape(frames,label):
|
||||
if not torch.is_tensor(frames) or frames.ndim!=4 or frames.shape[-1]<3: raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]")
|
||||
def _downsample_rgb(frames):
|
||||
_video_shape(frames,"video seam frames"); rgb=frames[...,:3].detach().to(device="cpu",dtype=torch.float32).permute(0,3,1,2)
|
||||
if tuple(rgb.shape[-2:])!=(COMPARE_SIZE,COMPARE_SIZE): rgb=F.interpolate(rgb,size=(COMPARE_SIZE,COMPARE_SIZE),mode="bilinear",align_corners=False)
|
||||
rgb=rgb.permute(0,2,3,1).clamp(0.0,1.0)
|
||||
if not bool(torch.isfinite(rgb).all().item()): raise ValueError("video seam comparison contains NaN or Inf")
|
||||
return rgb
|
||||
|
||||
|
||||
def _luma(frame: torch.Tensor) -> torch.Tensor:
|
||||
weights = frame.new_tensor((0.2126, 0.7152, 0.0722))
|
||||
return torch.sum(frame * weights, dim=-1)
|
||||
|
||||
|
||||
def _boundary_metrics(
|
||||
previous_images: torch.Tensor,
|
||||
next_images: torch.Tensor,
|
||||
) -> VideoSeamMetrics:
|
||||
if previous_images.shape[0] < 2 or next_images.shape[0] < 2:
|
||||
raise ValueError("video seam scoring needs two frames on each side")
|
||||
prev = _downsample_rgb(previous_images[-2:])
|
||||
nxt = _downsample_rgb(next_images[:2])
|
||||
prev_before, prev_edge = prev[0], prev[1]
|
||||
next_edge, next_after = nxt[0], nxt[1]
|
||||
|
||||
pixel = torch.mean(torch.abs(prev_edge - next_edge)).clamp(0.0, 1.0)
|
||||
prev_y = _luma(prev_edge)
|
||||
next_y = _luma(next_edge)
|
||||
luma = torch.mean(torch.abs(prev_y - next_y)).clamp(0.0, 1.0)
|
||||
prev_chroma = prev_edge - prev_y.unsqueeze(-1)
|
||||
next_chroma = next_edge - next_y.unsqueeze(-1)
|
||||
chroma = (
|
||||
torch.mean(torch.abs(prev_chroma - next_chroma)) / 2.0
|
||||
).clamp(0.0, 1.0)
|
||||
|
||||
prev_gx = prev_y[:, 1:] - prev_y[:, :-1]
|
||||
next_gx = next_y[:, 1:] - next_y[:, :-1]
|
||||
prev_gy = prev_y[1:, :] - prev_y[:-1, :]
|
||||
next_gy = next_y[1:, :] - next_y[:-1, :]
|
||||
edge = (
|
||||
(
|
||||
torch.mean(torch.abs(prev_gx - next_gx))
|
||||
+ torch.mean(torch.abs(prev_gy - next_gy))
|
||||
)
|
||||
/ 4.0
|
||||
).clamp(0.0, 1.0)
|
||||
|
||||
prev_delta = prev_edge - prev_before
|
||||
next_delta = next_after - next_edge
|
||||
temporal = (
|
||||
torch.mean(torch.abs(prev_delta - next_delta)) / 2.0
|
||||
).clamp(0.0, 1.0)
|
||||
motion = max(
|
||||
float(torch.mean(torch.abs(prev_delta)).item()),
|
||||
float(torch.mean(torch.abs(next_delta)).item()),
|
||||
)
|
||||
score = (
|
||||
WEIGHT_PIXEL * float(pixel.item())
|
||||
+ WEIGHT_LUMA * float(luma.item())
|
||||
+ WEIGHT_CHROMA * float(chroma.item())
|
||||
+ WEIGHT_EDGE * float(edge.item())
|
||||
+ WEIGHT_TEMPORAL * float(temporal.item())
|
||||
)
|
||||
if not math.isfinite(score):
|
||||
raise ValueError("video seam score is not finite")
|
||||
return VideoSeamMetrics(
|
||||
pixel_mae_norm=float(pixel.item()),
|
||||
luma_norm=float(luma.item()),
|
||||
chroma_norm=float(chroma.item()),
|
||||
edge_norm=float(edge.item()),
|
||||
temporal_norm=float(temporal.item()),
|
||||
motion=motion,
|
||||
score=max(0.0, min(1.0, score)),
|
||||
)
|
||||
|
||||
|
||||
def analyze_video_seam(
|
||||
previous_images: torch.Tensor,
|
||||
next_raw_images: torch.Tensor,
|
||||
*,
|
||||
context_frames: int,
|
||||
rewind_frames: int = 0,
|
||||
) -> VideoSeamMetrics:
|
||||
context_frames = int(context_frames)
|
||||
rewind_frames = int(rewind_frames)
|
||||
cut_index = context_frames - rewind_frames
|
||||
previous_stop = int(previous_images.shape[0]) - rewind_frames
|
||||
if rewind_frames < 0 or rewind_frames > MAX_CUT_REWIND:
|
||||
raise ValueError("rewind_frames is outside the V2.1 safe range")
|
||||
if cut_index < 0 or previous_stop < 2 or cut_index + 2 > next_raw_images.shape[0]:
|
||||
raise ValueError("insufficient frames for the requested adaptive cut")
|
||||
return _boundary_metrics(
|
||||
previous_images[previous_stop - 2 : previous_stop],
|
||||
next_raw_images[cut_index : cut_index + 2],
|
||||
)
|
||||
|
||||
|
||||
def choose_adaptive_cut(
|
||||
previous_images: torch.Tensor,
|
||||
next_raw_images: torch.Tensor,
|
||||
*,
|
||||
context_frames: int,
|
||||
) -> tuple[int, VideoSeamMetrics, VideoSeamMetrics]:
|
||||
legacy = analyze_video_seam(
|
||||
previous_images,
|
||||
next_raw_images,
|
||||
context_frames=context_frames,
|
||||
rewind_frames=0,
|
||||
)
|
||||
best_rewind = 0
|
||||
best = legacy
|
||||
maximum = min(
|
||||
MAX_CUT_REWIND,
|
||||
max(0, int(context_frames)),
|
||||
max(0, int(previous_images.shape[0]) - 2),
|
||||
)
|
||||
for rewind in range(1, maximum + 1):
|
||||
try:
|
||||
candidate = analyze_video_seam(
|
||||
previous_images,
|
||||
next_raw_images,
|
||||
context_frames=context_frames,
|
||||
rewind_frames=rewind,
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
if candidate.score < best.score:
|
||||
best_rewind, best = rewind, candidate
|
||||
|
||||
absolute = legacy.score - best.score
|
||||
relative = absolute / max(legacy.score, 1e-12)
|
||||
accepted = (
|
||||
best_rewind > 0
|
||||
and relative >= MIN_RELATIVE_IMPROVEMENT
|
||||
and absolute >= MIN_ABSOLUTE_IMPROVEMENT
|
||||
and best.score <= SAFE_SCORE_LIMIT
|
||||
)
|
||||
return (best_rewind, legacy, best) if accepted else (0, legacy, legacy)
|
||||
|
||||
|
||||
def apply_color_guard(
|
||||
previous_images: torch.Tensor,
|
||||
current_images: torch.Tensor,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
float,
|
||||
tuple[float, float, float],
|
||||
VideoSeamMetrics,
|
||||
]:
|
||||
before = _boundary_metrics(previous_images[-2:], current_images[:2])
|
||||
count = min(4, int(current_images.shape[0]))
|
||||
if count < 2:
|
||||
return current_images, 1.0, (0.0, 0.0, 0.0), before
|
||||
|
||||
prev_rgb = _downsample_rgb(previous_images[-2:])
|
||||
current_rgb = _downsample_rgb(current_images[:2])
|
||||
prev_luma = float(_luma(prev_rgb).mean().item())
|
||||
current_luma = float(_luma(current_rgb).mean().item())
|
||||
requested_gain = prev_luma / max(current_luma, 1e-6)
|
||||
if not 0.85 <= requested_gain <= 1.15:
|
||||
return current_images, 1.0, (0.0, 0.0, 0.0), before
|
||||
gain = max(0.94, min(1.06, requested_gain))
|
||||
|
||||
prev_y = _luma(prev_rgb).unsqueeze(-1)
|
||||
current_y = _luma(current_rgb).unsqueeze(-1)
|
||||
requested_bias = (
|
||||
(prev_rgb - prev_y).mean(dim=(0, 1, 2))
|
||||
- (current_rgb - current_y).mean(dim=(0, 1, 2))
|
||||
)
|
||||
if float(torch.max(torch.abs(requested_bias)).item()) > 0.08:
|
||||
return current_images, 1.0, (0.0, 0.0, 0.0), before
|
||||
bias = requested_bias.clamp(-0.02, 0.02)
|
||||
|
||||
result = current_images.clone().contiguous()
|
||||
original = result[:count].clone()
|
||||
fade = result.new_tensor((1.0, 0.65, 0.30, 0.0))[:count]
|
||||
luma_weights = result.new_tensor((0.2126, 0.7152, 0.0722))
|
||||
def _luma(frame): return torch.sum(frame*frame.new_tensor((0.2126,0.7152,0.0722)),dim=-1)
|
||||
def _boundary_metrics(previous_images,next_images):
|
||||
if previous_images.shape[0]<2 or next_images.shape[0]<2: raise ValueError("video seam scoring needs two frames on each side")
|
||||
prev=_downsample_rgb(previous_images[-2:]); nxt=_downsample_rgb(next_images[:2]); prev_before,prev_edge=prev[0],prev[1]; next_edge,next_after=nxt[0],nxt[1]
|
||||
pixel=torch.mean(torch.abs(prev_edge-next_edge)).clamp(0,1); prev_y=_luma(prev_edge); next_y=_luma(next_edge); luma=torch.mean(torch.abs(prev_y-next_y)).clamp(0,1)
|
||||
prev_chroma=prev_edge-prev_y.unsqueeze(-1); next_chroma=next_edge-next_y.unsqueeze(-1); chroma=(torch.mean(torch.abs(prev_chroma-next_chroma))/2.0).clamp(0,1)
|
||||
prev_gx=prev_y[:,1:]-prev_y[:,:-1]; next_gx=next_y[:,1:]-next_y[:,:-1]; prev_gy=prev_y[1:,:]-prev_y[:-1,:]; next_gy=next_y[1:,:]-next_y[:-1,:]
|
||||
edge=((torch.mean(torch.abs(prev_gx-next_gx))+torch.mean(torch.abs(prev_gy-next_gy)))/4.0).clamp(0,1)
|
||||
prev_delta=prev_edge-prev_before; next_delta=next_after-next_edge; temporal=(torch.mean(torch.abs(prev_delta-next_delta))/2.0).clamp(0,1)
|
||||
motion=max(float(torch.mean(torch.abs(prev_delta)).item()),float(torch.mean(torch.abs(next_delta)).item()))
|
||||
score=WEIGHT_PIXEL*float(pixel.item())+WEIGHT_LUMA*float(luma.item())+WEIGHT_CHROMA*float(chroma.item())+WEIGHT_EDGE*float(edge.item())+WEIGHT_TEMPORAL*float(temporal.item())
|
||||
if not math.isfinite(score): raise ValueError("video seam score is not finite")
|
||||
return VideoSeamMetrics(float(pixel.item()),float(luma.item()),float(chroma.item()),float(edge.item()),float(temporal.item()),motion,max(0.0,min(1.0,score)))
|
||||
def analyze_video_seam(previous_images,next_raw_images,*,context_frames,rewind_frames=0):
|
||||
context_frames=int(context_frames); rewind_frames=int(rewind_frames); cut_index=context_frames-rewind_frames; previous_stop=int(previous_images.shape[0])-rewind_frames
|
||||
if rewind_frames<0 or rewind_frames>MAX_CUT_REWIND: raise ValueError("rewind_frames is outside the V2.1 safe range")
|
||||
if cut_index<0 or previous_stop<2 or cut_index+2>next_raw_images.shape[0]: raise ValueError("insufficient frames for the requested adaptive cut")
|
||||
return _boundary_metrics(previous_images[previous_stop-2:previous_stop],next_raw_images[cut_index:cut_index+2])
|
||||
def choose_adaptive_cut(previous_images,next_raw_images,*,context_frames):
|
||||
legacy=analyze_video_seam(previous_images,next_raw_images,context_frames=context_frames,rewind_frames=0); best_rewind=0; best=legacy
|
||||
maximum=min(MAX_CUT_REWIND,max(0,int(context_frames)),max(0,int(previous_images.shape[0])-2))
|
||||
for rewind in range(1,maximum+1):
|
||||
try: candidate=analyze_video_seam(previous_images,next_raw_images,context_frames=context_frames,rewind_frames=rewind)
|
||||
except ValueError: continue
|
||||
if candidate.score<best.score: best_rewind,best=rewind,candidate
|
||||
absolute=legacy.score-best.score; relative=absolute/max(legacy.score,1e-12)
|
||||
accepted=best_rewind>0 and relative>=MIN_RELATIVE_IMPROVEMENT and absolute>=MIN_ABSOLUTE_IMPROVEMENT and best.score<=SAFE_SCORE_LIMIT
|
||||
return (best_rewind,legacy,best) if accepted else (0,legacy,legacy)
|
||||
def apply_color_guard(previous_images,current_images):
|
||||
before=_boundary_metrics(previous_images[-2:],current_images[:2]); count=min(4,int(current_images.shape[0]))
|
||||
if count<2: return current_images,1.0,(0.0,0.0,0.0),before
|
||||
prev_rgb=_downsample_rgb(previous_images[-2:]); current_rgb=_downsample_rgb(current_images[:2]); prev_luma=float(_luma(prev_rgb).mean().item()); current_luma=float(_luma(current_rgb).mean().item()); requested_gain=prev_luma/max(current_luma,1e-6)
|
||||
if not 0.85<=requested_gain<=1.15: return current_images,1.0,(0.0,0.0,0.0),before
|
||||
gain=max(0.94,min(1.06,requested_gain)); prev_y=_luma(prev_rgb).unsqueeze(-1); current_y=_luma(current_rgb).unsqueeze(-1); requested_bias=(prev_rgb-prev_y).mean(dim=(0,1,2))-(current_rgb-current_y).mean(dim=(0,1,2))
|
||||
if float(torch.max(torch.abs(requested_bias)).item())>0.08: return current_images,1.0,(0.0,0.0,0.0),before
|
||||
bias=requested_bias.clamp(-0.02,0.02); result=current_images.clone().contiguous(); original=result[:count].clone(); fade=result.new_tensor((1.0,0.65,0.30,0.0))[:count]; luma_weights=result.new_tensor((0.2126,0.7152,0.0722))
|
||||
for index in range(count):
|
||||
weight = fade[index]
|
||||
rgb = result[index, ..., :3]
|
||||
y = torch.sum(rgb * luma_weights, dim=-1, keepdim=True)
|
||||
chroma = rgb - y
|
||||
frame_gain = 1.0 + (gain - 1.0) * weight
|
||||
frame_bias = bias.to(dtype=rgb.dtype) * weight
|
||||
result[index, ..., :3] = (
|
||||
y * frame_gain + chroma + frame_bias
|
||||
).clamp(0.0, 1.0)
|
||||
|
||||
after = _boundary_metrics(previous_images[-2:], result[:2])
|
||||
if after.score > before.score * 1.01:
|
||||
result[:count].copy_(original)
|
||||
return result, 1.0, (0.0, 0.0, 0.0), before
|
||||
return (
|
||||
result,
|
||||
gain,
|
||||
tuple(float(value) for value in bias.tolist()),
|
||||
after,
|
||||
)
|
||||
|
||||
|
||||
def choose_video_blend(metrics: VideoSeamMetrics) -> int:
|
||||
if metrics.motion <= 0.015:
|
||||
return 2
|
||||
if metrics.motion <= 0.040:
|
||||
return 1
|
||||
weight=fade[index]; rgb=result[index,...,:3]; y=torch.sum(rgb*luma_weights,dim=-1,keepdim=True); chroma=rgb-y; frame_gain=1.0+(gain-1.0)*weight; frame_bias=bias.to(dtype=rgb.dtype)*weight; result[index,...,:3]=(y*frame_gain+chroma+frame_bias).clamp(0.0,1.0)
|
||||
after=_boundary_metrics(previous_images[-2:],result[:2])
|
||||
if after.score>before.score*1.01: result[:count].copy_(original); return result,1.0,(0.0,0.0,0.0),before
|
||||
return result,gain,tuple(float(v) for v in bias.tolist()),after
|
||||
def choose_video_blend(metrics):
|
||||
if metrics.motion<=0.015: return 2
|
||||
if metrics.motion<=0.040: return 1
|
||||
return 0
|
||||
|
||||
|
||||
def apply_video_blend(
|
||||
previous_frames: torch.Tensor,
|
||||
aligned_next_frames: torch.Tensor,
|
||||
*,
|
||||
blend_frames: int,
|
||||
) -> torch.Tensor:
|
||||
blend_frames = min(
|
||||
int(blend_frames),
|
||||
int(previous_frames.shape[0]),
|
||||
int(aligned_next_frames.shape[0]),
|
||||
2,
|
||||
)
|
||||
if blend_frames <= 0:
|
||||
return previous_frames[:0].clone()
|
||||
previous = previous_frames[-blend_frames:].clone()
|
||||
current = aligned_next_frames[-blend_frames:].to(dtype=previous.dtype)
|
||||
def apply_video_blend(previous_frames,aligned_next_frames,*,blend_frames):
|
||||
blend_frames=min(int(blend_frames),int(previous_frames.shape[0]),int(aligned_next_frames.shape[0]),2)
|
||||
if blend_frames<=0: return previous_frames[:0].clone()
|
||||
previous=previous_frames[-blend_frames:].clone(); current=aligned_next_frames[-blend_frames:].to(dtype=previous.dtype)
|
||||
for index in range(blend_frames):
|
||||
progress = float(index + 1) / float(blend_frames + 1)
|
||||
alpha = 0.5 - 0.5 * math.cos(math.pi * progress)
|
||||
previous[index].mul_(1.0 - alpha).add_(current[index], alpha=alpha)
|
||||
progress=float(index+1)/float(blend_frames+1); alpha=0.5-0.5*math.cos(math.pi*progress); previous[index].mul_(1.0-alpha).add_(current[index],alpha=alpha)
|
||||
return previous
|
||||
def correct_seam(previous_images,next_raw_images,*,context_frames,automatic=False):
|
||||
rewind,legacy,selected=choose_adaptive_cut(previous_images,next_raw_images,context_frames=context_frames); cut_index=int(context_frames)-rewind; previous_stop=int(previous_images.shape[0])-rewind
|
||||
if previous_stop<2 or cut_index<0: raise ValueError("adaptive cut would exhaust a seam side")
|
||||
current=next_raw_images[cut_index:].clone().contiguous(); kept_previous=previous_images[:previous_stop]; current,gain,bias,corrected=apply_color_guard(kept_previous,current)
|
||||
blend_frames=min(choose_video_blend(selected),cut_index,previous_stop); video_patch=None
|
||||
if blend_frames>0:
|
||||
video_patch=apply_video_blend(kept_previous[-blend_frames:],next_raw_images[cut_index-blend_frames:cut_index],blend_frames=blend_frames); score_previous=kept_previous[-2:].clone(); replaced=min(blend_frames,2); score_previous[-replaced:].copy_(video_patch[-replaced:]); corrected=_boundary_metrics(score_previous,current[:2])
|
||||
improvement=(legacy.score-corrected.score)/max(legacy.score,1e-12)
|
||||
if corrected.score>legacy.score*1.01:
|
||||
return next_raw_images[int(context_frames):].clone().contiguous(),SeamDecision(legacy_score=legacy.score,cut_score=legacy.score,corrected_score=legacy.score,fallback_reason="corrected score was worse than legacy"),None
|
||||
absolute_improvement=legacy.score-corrected.score
|
||||
if automatic and (improvement<MIN_RELATIVE_IMPROVEMENT or absolute_improvement<MIN_ABSOLUTE_IMPROVEMENT):
|
||||
return next_raw_images[int(context_frames):].clone().contiguous(),SeamDecision(legacy_score=legacy.score,cut_score=selected.score,corrected_score=legacy.score,fallback_reason="Auto kept the legacy video seam; improvement was below threshold"),None
|
||||
return current,SeamDecision(cut_rewind_frames=rewind,legacy_score=legacy.score,cut_score=selected.score,corrected_score=corrected.score,improvement=improvement,video_blend_frames=blend_frames,luma_gain=gain,chroma_bias=bias),video_patch
|
||||
|
||||
|
||||
def correct_seam(
|
||||
previous_images: torch.Tensor,
|
||||
next_raw_images: torch.Tensor,
|
||||
*,
|
||||
context_frames: int,
|
||||
automatic: bool = False,
|
||||
) -> tuple[torch.Tensor, SeamDecision, torch.Tensor | None]:
|
||||
rewind, legacy, selected = choose_adaptive_cut(
|
||||
previous_images,
|
||||
next_raw_images,
|
||||
context_frames=context_frames,
|
||||
)
|
||||
cut_index = int(context_frames) - rewind
|
||||
previous_stop = int(previous_images.shape[0]) - rewind
|
||||
if previous_stop < 2 or cut_index < 0:
|
||||
raise ValueError("adaptive cut would exhaust a seam side")
|
||||
|
||||
current = next_raw_images[cut_index:].clone().contiguous()
|
||||
kept_previous = previous_images[:previous_stop]
|
||||
current, gain, bias, corrected = apply_color_guard(kept_previous, current)
|
||||
|
||||
blend_frames = min(
|
||||
choose_video_blend(selected),
|
||||
cut_index,
|
||||
previous_stop,
|
||||
)
|
||||
video_patch: torch.Tensor | None = None
|
||||
if blend_frames > 0:
|
||||
video_patch = apply_video_blend(
|
||||
kept_previous[-blend_frames:],
|
||||
next_raw_images[cut_index - blend_frames : cut_index],
|
||||
blend_frames=blend_frames,
|
||||
)
|
||||
score_previous = kept_previous[-2:].clone()
|
||||
replaced = min(blend_frames, 2)
|
||||
score_previous[-replaced:].copy_(video_patch[-replaced:])
|
||||
corrected = _boundary_metrics(score_previous, current[:2])
|
||||
|
||||
improvement = (legacy.score - corrected.score) / max(legacy.score, 1e-12)
|
||||
if corrected.score > legacy.score * 1.01:
|
||||
fallback = next_raw_images[int(context_frames) :].clone().contiguous()
|
||||
return (
|
||||
fallback,
|
||||
SeamDecision(
|
||||
legacy_score=legacy.score,
|
||||
cut_score=legacy.score,
|
||||
corrected_score=legacy.score,
|
||||
fallback_reason="corrected score was worse than legacy",
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
absolute_improvement = legacy.score - corrected.score
|
||||
if automatic and (
|
||||
improvement < MIN_RELATIVE_IMPROVEMENT
|
||||
or absolute_improvement < MIN_ABSOLUTE_IMPROVEMENT
|
||||
):
|
||||
fallback = next_raw_images[int(context_frames) :].clone().contiguous()
|
||||
return (
|
||||
fallback,
|
||||
SeamDecision(
|
||||
legacy_score=legacy.score,
|
||||
cut_score=selected.score,
|
||||
corrected_score=legacy.score,
|
||||
fallback_reason="Auto kept the legacy video seam; improvement was below threshold",
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
decision = SeamDecision(
|
||||
cut_rewind_frames=rewind,
|
||||
legacy_score=legacy.score,
|
||||
cut_score=selected.score,
|
||||
corrected_score=corrected.score,
|
||||
improvement=improvement,
|
||||
video_blend_frames=blend_frames,
|
||||
luma_gain=gain,
|
||||
chroma_bias=bias,
|
||||
)
|
||||
return current, decision, video_patch
|
||||
|
||||
|
||||
def _audio_mid(waveform: torch.Tensor) -> torch.Tensor:
|
||||
if not torch.is_tensor(waveform) or waveform.ndim != 3:
|
||||
raise ValueError("audio seam waveform must be [B,C,S]")
|
||||
value = waveform.detach().to(device="cpu", dtype=torch.float32)
|
||||
if not bool(torch.isfinite(value).all().item()):
|
||||
raise ValueError("audio seam waveform contains NaN or Inf")
|
||||
return value.mean(dim=(0, 1))
|
||||
|
||||
|
||||
def _correlation(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
length = min(int(a.numel()), int(b.numel()))
|
||||
if length < 8:
|
||||
return 0.0
|
||||
a = a[:length] - a[:length].mean()
|
||||
b = b[:length] - b[:length].mean()
|
||||
denominator = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12)
|
||||
value = float((torch.sum(a * b) / denominator).item())
|
||||
return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0
|
||||
|
||||
|
||||
def analyze_audio_seam(
|
||||
previous_waveform: torch.Tensor,
|
||||
next_raw_waveform: torch.Tensor,
|
||||
*,
|
||||
sample_rate: int,
|
||||
cut_sample: int,
|
||||
) -> AudioSeamMetrics:
|
||||
sample_rate = int(sample_rate)
|
||||
cut_sample = int(cut_sample)
|
||||
previous = _audio_mid(previous_waveform)
|
||||
current = _audio_mid(next_raw_waveform)
|
||||
max_lag = max(1, int(round(sample_rate * 0.020)))
|
||||
window = min(
|
||||
max(8, int(round(sample_rate * 0.080))),
|
||||
int(previous.shape[-1]),
|
||||
max(0, cut_sample - max_lag),
|
||||
)
|
||||
if sample_rate <= 0 or window < 8:
|
||||
raise ValueError("insufficient audio for seam alignment")
|
||||
previous_tail = previous[-window:]
|
||||
|
||||
def correlation_at(lag: int) -> float | None:
|
||||
stop = cut_sample + int(lag)
|
||||
start = stop - window
|
||||
if start < 0 or stop > current.shape[-1]:
|
||||
return None
|
||||
return _correlation(previous_tail, current[start:stop])
|
||||
|
||||
baseline = correlation_at(0)
|
||||
if baseline is None:
|
||||
raise ValueError("audio seam cut is outside the decoded waveform")
|
||||
best_lag, best_correlation = 0, baseline
|
||||
for lag in range(-max_lag, max_lag + 1):
|
||||
correlation = correlation_at(lag)
|
||||
if correlation is not None and correlation > best_correlation:
|
||||
best_lag, best_correlation = lag, correlation
|
||||
|
||||
if best_correlation < 0.20 or best_correlation - baseline < 0.02:
|
||||
best_lag, best_correlation = 0, baseline
|
||||
def _audio_mid(waveform):
|
||||
if not torch.is_tensor(waveform) or waveform.ndim!=3: raise ValueError("audio seam waveform must be [B,C,S]")
|
||||
value=waveform.detach().to(device="cpu",dtype=torch.float32)
|
||||
if not bool(torch.isfinite(value).all().item()): raise ValueError("audio seam waveform contains NaN or Inf")
|
||||
return value.mean(dim=(0,1))
|
||||
def _correlation(a,b):
|
||||
length=min(int(a.numel()),int(b.numel()))
|
||||
if length<8: return 0.0
|
||||
a=a[:length]-a[:length].mean(); b=b[:length]-b[:length].mean(); denominator=torch.sqrt(torch.sum(a.square())*torch.sum(b.square())).clamp_min(1e-12); value=float((torch.sum(a*b)/denominator).item()); return max(-1.0,min(1.0,value)) if math.isfinite(value) else 0.0
|
||||
def analyze_audio_seam(previous_waveform,next_raw_waveform,*,sample_rate,cut_sample):
|
||||
sample_rate=int(sample_rate); cut_sample=int(cut_sample); previous=_audio_mid(previous_waveform); current=_audio_mid(next_raw_waveform); max_lag=max(1,int(round(sample_rate*0.020))); window=min(max(8,int(round(sample_rate*0.080))),int(previous.shape[-1]),max(0,cut_sample-max_lag))
|
||||
if sample_rate<=0 or window<8: raise ValueError("insufficient audio for seam alignment")
|
||||
previous_tail=previous[-window:]
|
||||
def correlation_at(lag):
|
||||
stop=cut_sample+int(lag); start=stop-window
|
||||
if start<0 or stop>current.shape[-1]: return None
|
||||
return _correlation(previous_tail,current[start:stop])
|
||||
baseline=correlation_at(0)
|
||||
if baseline is None: raise ValueError("audio seam cut is outside the decoded waveform")
|
||||
best_lag,best_correlation=0,baseline
|
||||
for lag in range(-max_lag,max_lag+1):
|
||||
corr=correlation_at(lag)
|
||||
if corr is not None and corr>best_correlation: best_lag,best_correlation=lag,corr
|
||||
if best_correlation<0.20 or best_correlation-baseline<0.02: best_lag,best_correlation=0,baseline
|
||||
else:
|
||||
center = max(0, min(int(current.shape[-1]) - 1, cut_sample + best_lag))
|
||||
radius = max(1, int(round(sample_rate * 0.001)))
|
||||
start = max(0, center - radius)
|
||||
stop = min(int(current.shape[-1]), center + radius + 1)
|
||||
if stop > start:
|
||||
zero = int(torch.argmin(torch.abs(current[start:stop])).item()) + start
|
||||
best_lag = zero - cut_sample
|
||||
|
||||
before_jump = float(
|
||||
torch.mean(
|
||||
torch.abs(previous_waveform[..., -1] - next_raw_waveform[..., cut_sample])
|
||||
).item()
|
||||
)
|
||||
return AudioSeamMetrics(
|
||||
correlation_before=baseline,
|
||||
correlation_after=best_correlation,
|
||||
boundary_jump_before=before_jump,
|
||||
boundary_jump_after=before_jump,
|
||||
offset_samples=int(best_lag),
|
||||
)
|
||||
|
||||
|
||||
def _audio_crossfade_ms(
|
||||
previous_waveform: torch.Tensor,
|
||||
next_raw_waveform: torch.Tensor,
|
||||
*,
|
||||
cut_sample: int,
|
||||
sample_rate: int,
|
||||
) -> int:
|
||||
probe = max(8, int(round(sample_rate * 0.020)))
|
||||
if previous_waveform.shape[-1] < probe or cut_sample < probe:
|
||||
return 10
|
||||
previous = _audio_mid(previous_waveform)[-probe:]
|
||||
current = _audio_mid(next_raw_waveform)[cut_sample - probe : cut_sample]
|
||||
rms = torch.sqrt(torch.mean(torch.cat((previous, current)).square())).clamp_min(1e-6)
|
||||
transient = max(
|
||||
float(torch.mean(torch.abs(torch.diff(previous))).item()),
|
||||
float(torch.mean(torch.abs(torch.diff(current))).item()),
|
||||
) / float(rms.item())
|
||||
if transient > 0.80:
|
||||
return 10
|
||||
if transient > 0.35:
|
||||
return 20
|
||||
if transient > 0.15:
|
||||
return 40
|
||||
center=max(0,min(int(current.shape[-1])-1,cut_sample+best_lag)); radius=max(1,int(round(sample_rate*0.001))); start=max(0,center-radius); stop=min(int(current.shape[-1]),center+radius+1)
|
||||
if stop>start: best_lag=int(torch.argmin(torch.abs(current[start:stop])).item())+start-cut_sample
|
||||
before_jump=float(torch.mean(torch.abs(previous_waveform[...,-1]-next_raw_waveform[...,cut_sample])).item())
|
||||
return AudioSeamMetrics(baseline,best_correlation,before_jump,before_jump,int(best_lag))
|
||||
def _audio_crossfade_ms(previous_waveform,next_raw_waveform,*,cut_sample,sample_rate):
|
||||
probe=max(8,int(round(sample_rate*0.020)))
|
||||
if previous_waveform.shape[-1]<probe or cut_sample<probe: return 10
|
||||
previous=_audio_mid(previous_waveform)[-probe:]; current=_audio_mid(next_raw_waveform)[cut_sample-probe:cut_sample]; rms=torch.sqrt(torch.mean(torch.cat((previous,current)).square())).clamp_min(1e-6); transient=max(float(torch.mean(torch.abs(torch.diff(previous))).item()),float(torch.mean(torch.abs(torch.diff(current))).item()))/float(rms.item())
|
||||
if transient>0.80:return 10
|
||||
if transient>0.35:return 20
|
||||
if transient>0.15:return 40
|
||||
return 60
|
||||
|
||||
|
||||
def apply_audio_seam(
|
||||
previous_waveform: torch.Tensor,
|
||||
next_raw_waveform: torch.Tensor,
|
||||
metrics: AudioSeamMetrics,
|
||||
*,
|
||||
sample_rate: int,
|
||||
cut_sample: int,
|
||||
) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]:
|
||||
if metrics.correlation_after < 0.20:
|
||||
return None, metrics, 0, 1.0, 0.0
|
||||
fade_ms = _audio_crossfade_ms(
|
||||
previous_waveform,
|
||||
next_raw_waveform,
|
||||
cut_sample=cut_sample,
|
||||
sample_rate=sample_rate,
|
||||
)
|
||||
fade_samples = min(
|
||||
int(round(sample_rate * fade_ms / 1000.0)),
|
||||
int(previous_waveform.shape[-1]),
|
||||
int(cut_sample),
|
||||
)
|
||||
if fade_samples < 2:
|
||||
return None, metrics, 0, 1.0, 0.0
|
||||
|
||||
progress = torch.linspace(
|
||||
0.0,
|
||||
1.0,
|
||||
fade_samples,
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
base_indices = torch.arange(cut_sample - fade_samples, cut_sample, dtype=torch.long)
|
||||
offsets = torch.round(
|
||||
float(metrics.offset_samples) * (1.0 - progress)
|
||||
).to(dtype=torch.long)
|
||||
indices = (base_indices + offsets).clamp(0, int(next_raw_waveform.shape[-1]) - 1)
|
||||
current = next_raw_waveform.detach().to("cpu").index_select(-1, indices)
|
||||
previous = previous_waveform[..., -fade_samples:].detach().to("cpu")
|
||||
|
||||
previous_rms = torch.sqrt(torch.mean(previous.float().square())).clamp_min(1e-6)
|
||||
current_rms = torch.sqrt(torch.mean(current.float().square())).clamp_min(1e-6)
|
||||
requested_db = 20.0 * math.log10(float((previous_rms / current_rms).item()))
|
||||
level_db = max(-1.5, min(1.5, requested_db)) if abs(requested_db) <= 6.0 else 0.0
|
||||
level_gain = 10.0 ** (level_db / 20.0)
|
||||
requested_dc = float((previous.float().mean() - current.float().mean()).item())
|
||||
dc_bias = max(-0.02, min(0.02, requested_dc)) if abs(requested_dc) <= 0.10 else 0.0
|
||||
current = current * level_gain + dc_bias
|
||||
|
||||
shape = [1] * (previous.ndim - 1) + [fade_samples]
|
||||
fade_in = torch.sin(progress * (math.pi / 2.0)).reshape(shape)
|
||||
fade_out = torch.cos(progress * (math.pi / 2.0)).reshape(shape)
|
||||
patch = previous * fade_out + current.to(dtype=previous.dtype) * fade_in
|
||||
input_peak = max(
|
||||
float(previous.abs().max().item()),
|
||||
float(current.abs().max().item()),
|
||||
1e-6,
|
||||
)
|
||||
patch_peak = float(patch.abs().max().item())
|
||||
if patch_peak > input_peak * 1.05:
|
||||
patch = patch * ((input_peak * 1.05) / patch_peak)
|
||||
after_jump = float(
|
||||
torch.mean(torch.abs(patch[..., -1] - next_raw_waveform[..., cut_sample])).item()
|
||||
)
|
||||
updated = AudioSeamMetrics(
|
||||
correlation_before=metrics.correlation_before,
|
||||
correlation_after=metrics.correlation_after,
|
||||
boundary_jump_before=metrics.boundary_jump_before,
|
||||
boundary_jump_after=after_jump,
|
||||
offset_samples=metrics.offset_samples,
|
||||
)
|
||||
return patch.contiguous(), updated, fade_samples, level_gain, dc_bias
|
||||
|
||||
|
||||
def correct_audio_seam(
|
||||
previous_waveform: torch.Tensor,
|
||||
next_raw_waveform: torch.Tensor,
|
||||
*,
|
||||
sample_rate: int,
|
||||
cut_sample: int,
|
||||
) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]:
|
||||
metrics = analyze_audio_seam(
|
||||
previous_waveform,
|
||||
next_raw_waveform,
|
||||
sample_rate=sample_rate,
|
||||
cut_sample=cut_sample,
|
||||
)
|
||||
return apply_audio_seam(
|
||||
previous_waveform,
|
||||
next_raw_waveform,
|
||||
metrics,
|
||||
sample_rate=sample_rate,
|
||||
cut_sample=cut_sample,
|
||||
)
|
||||
def apply_audio_seam(previous_waveform,next_raw_waveform,metrics,*,sample_rate,cut_sample):
|
||||
if metrics.correlation_after<0.20:return None,metrics,0,1.0,0.0
|
||||
fade_ms=_audio_crossfade_ms(previous_waveform,next_raw_waveform,cut_sample=cut_sample,sample_rate=sample_rate); fade_samples=min(int(round(sample_rate*fade_ms/1000.0)),int(previous_waveform.shape[-1]),int(cut_sample))
|
||||
if fade_samples<2:return None,metrics,0,1.0,0.0
|
||||
progress=torch.linspace(0.0,1.0,fade_samples,dtype=torch.float32,device="cpu"); base_indices=torch.arange(cut_sample-fade_samples,cut_sample,dtype=torch.long); offsets=torch.round(float(metrics.offset_samples)*(1.0-progress)).to(dtype=torch.long); indices=(base_indices+offsets).clamp(0,int(next_raw_waveform.shape[-1])-1)
|
||||
current=next_raw_waveform.detach().to("cpu").index_select(-1,indices); previous=previous_waveform[...,-fade_samples:].detach().to("cpu"); previous_rms=torch.sqrt(torch.mean(previous.float().square())).clamp_min(1e-6); current_rms=torch.sqrt(torch.mean(current.float().square())).clamp_min(1e-6); requested_db=20.0*math.log10(float((previous_rms/current_rms).item())); level_db=max(-1.5,min(1.5,requested_db)) if abs(requested_db)<=6.0 else 0.0; level_gain=10.0**(level_db/20.0); requested_dc=float((previous.float().mean()-current.float().mean()).item()); dc_bias=max(-0.02,min(0.02,requested_dc)) if abs(requested_dc)<=0.10 else 0.0; current=current*level_gain+dc_bias
|
||||
shape=[1]*(previous.ndim-1)+[fade_samples]; fade_in=torch.sin(progress*(math.pi/2.0)).reshape(shape); fade_out=torch.cos(progress*(math.pi/2.0)).reshape(shape); patch=previous*fade_out+current.to(dtype=previous.dtype)*fade_in; input_peak=max(float(previous.abs().max().item()),float(current.abs().max().item()),1e-6); patch_peak=float(patch.abs().max().item())
|
||||
if patch_peak>input_peak*1.05: patch=patch*((input_peak*1.05)/patch_peak)
|
||||
after_jump=float(torch.mean(torch.abs(patch[...,-1]-next_raw_waveform[...,cut_sample])).item()); updated=AudioSeamMetrics(metrics.correlation_before,metrics.correlation_after,metrics.boundary_jump_before,after_jump,metrics.offset_samples); return patch.contiguous(),updated,fade_samples,level_gain,dc_bias
|
||||
def correct_audio_seam(previous_waveform,next_raw_waveform,*,sample_rate,cut_sample):
|
||||
metrics=analyze_audio_seam(previous_waveform,next_raw_waveform,sample_rate=sample_rate,cut_sample=cut_sample); return apply_audio_seam(previous_waveform,next_raw_waveform,metrics,sample_rate=sample_rate,cut_sample=cut_sample)
|
||||
|
||||
+1
-9
@@ -1,10 +1,6 @@
|
||||
"""Transient types used by V2.1 decoded seam correction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VideoSeamMetrics:
|
||||
pixel_mae_norm: float
|
||||
@@ -14,8 +10,6 @@ class VideoSeamMetrics:
|
||||
temporal_norm: float
|
||||
motion: float
|
||||
score: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AudioSeamMetrics:
|
||||
correlation_before: float = 0.0
|
||||
@@ -23,8 +17,6 @@ class AudioSeamMetrics:
|
||||
boundary_jump_before: float = 0.0
|
||||
boundary_jump_after: float = 0.0
|
||||
offset_samples: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SeamDecision:
|
||||
cut_rewind_frames: int = 0
|
||||
@@ -34,7 +26,7 @@ class SeamDecision:
|
||||
improvement: float = 0.0
|
||||
video_blend_frames: int = 0
|
||||
luma_gain: float = 1.0
|
||||
chroma_bias: tuple[float, float, float] = (0.0, 0.0, 0.0)
|
||||
chroma_bias: tuple[float,float,float] = (0.0,0.0,0.0)
|
||||
audio_metrics: AudioSeamMetrics = AudioSeamMetrics()
|
||||
audio_crossfade_samples: int = 0
|
||||
audio_level_gain: float = 1.0
|
||||
|
||||
+97
-504
@@ -5,538 +5,131 @@ orchestration only: all text/image conditioning is prepared first, H3 sampling
|
||||
runs for every chunk without intermediate VAE decoding, and decoding/assembly
|
||||
happens after the sampling phase.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import copy, logging
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..compatibility import accelerator_summary, check_comfy_h3_runtime
|
||||
from ..constants import (
|
||||
DIAGNOSTICS_FULL,
|
||||
DIAGNOSTICS_OFF,
|
||||
DIAGNOSTICS_OPTIONS,
|
||||
FPS,
|
||||
CONTINUUM_ACTUAL_PREFIX_STEPS,
|
||||
SEAM_CORRECTION_AUTO,
|
||||
SEAM_CORRECTION_OFF,
|
||||
SEAM_CORRECTION_OPTIONS,
|
||||
V2_CONTINUITY_AUTO,
|
||||
DIAGNOSTICS_FULL, DIAGNOSTICS_OFF, DIAGNOSTICS_OPTIONS, FPS,
|
||||
CONTINUUM_ACTUAL_PREFIX_STEPS, SEAM_CORRECTION_AUTO, SEAM_CORRECTION_OFF,
|
||||
SEAM_CORRECTION_OPTIONS, V2_CONTINUITY_AUTO,
|
||||
)
|
||||
from ..continuation import POLICY_REPLACE, prepare_conditioning
|
||||
from ..model_patch import clone_model_for_chunk, patch_model
|
||||
from ..state import (
|
||||
assert_context_unchanged,
|
||||
context_fingerprint,
|
||||
make_plan,
|
||||
select_context,
|
||||
validate_state,
|
||||
)
|
||||
from ..temporal import (
|
||||
align_frame_count_up,
|
||||
largest_context_capacity,
|
||||
make_extension_shape,
|
||||
)
|
||||
from ..state import assert_context_unchanged, context_fingerprint, make_plan, select_context, validate_state
|
||||
from ..temporal import align_frame_count_up, largest_context_capacity, make_extension_shape
|
||||
from ..version import PACKAGE_VERSION
|
||||
from .decoder import decode_sequence, decode_sequence_with_seam, enforce_total_frames
|
||||
from .h3_builder import (
|
||||
attach_keyframes,
|
||||
empty_h3_latent,
|
||||
encode_identity_latents,
|
||||
encode_prompt_conditioning,
|
||||
prepare_identity_assets,
|
||||
)
|
||||
from .h3_builder import attach_keyframes, empty_h3_latent, encode_identity_latents, encode_prompt_conditioning, prepare_identity_assets
|
||||
from .motion import choose_context_frames
|
||||
from .prompts import prompt_plan_report, validate_prompt_plan
|
||||
from .sampling import sample_chunk
|
||||
from .seeds import derive_chunk_seed
|
||||
from .session import (
|
||||
SessionValidationError,
|
||||
entry_to_state,
|
||||
make_chunk_entry,
|
||||
make_session,
|
||||
model_fingerprint,
|
||||
session_summary,
|
||||
validate_chunk_entry,
|
||||
validate_session,
|
||||
SessionValidationError, entry_to_state, make_chunk_entry, make_session, model_fingerprint,
|
||||
session_summary, validate_chunk_entry, validate_session,
|
||||
)
|
||||
LOG=logging.getLogger("h3_continuum_join")
|
||||
class SequenceRuntimeError(RuntimeError): pass
|
||||
|
||||
LOG = logging.getLogger("h3_continuum_join")
|
||||
|
||||
|
||||
class SequenceRuntimeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _check_decode_memory_budget(
|
||||
*, width: int, height: int, chunks: int, chunk_seconds: float
|
||||
) -> tuple[float, float | None]:
|
||||
"""Fail before sampling when the eventual CPU IMAGE output is unsafe.
|
||||
|
||||
ComfyUI IMAGE tensors are normally float32. V2 preallocates the final output
|
||||
once, but one raw VAE chunk and normal process headroom still coexist with it.
|
||||
The estimate is conservative and intentionally protects long/high-resolution
|
||||
jobs from failing only after every expensive H3 chunk has completed.
|
||||
"""
|
||||
|
||||
target_frames = max(1, int(round(chunks * chunk_seconds * FPS)))
|
||||
max_chunk_frames = max(5, int(round(chunk_seconds * FPS)) + 39)
|
||||
image_bytes = target_frames * width * height * 3 * 4
|
||||
transient_bytes = max_chunk_frames * width * height * 3 * 4
|
||||
required_bytes = image_bytes + int(transient_bytes * 1.5) + 1 * 1024**3
|
||||
estimate_gib = required_bytes / 1024**3
|
||||
available_gib: float | None = None
|
||||
def _check_decode_memory_budget(*,width,height,chunks,chunk_seconds):
|
||||
target_frames=max(1,int(round(chunks*chunk_seconds*FPS))); max_chunk_frames=max(5,int(round(chunk_seconds*FPS))+39)
|
||||
image_bytes=target_frames*width*height*3*4; transient_bytes=max_chunk_frames*width*height*3*4
|
||||
required_bytes=image_bytes+int(transient_bytes*1.5)+1*1024**3; estimate_gib=required_bytes/1024**3; available_gib=None
|
||||
try:
|
||||
import psutil
|
||||
available=int(psutil.virtual_memory().available); available_gib=available/1024**3
|
||||
if required_bytes>int(available*0.75):
|
||||
raise SequenceRuntimeError(f"estimated decoded output requires about {estimate_gib:.1f} GiB including one-chunk headroom, but only {available_gib:.1f} GiB RAM is currently available. Reduce chunks, chunk_seconds, or resolution before sampling.")
|
||||
except ImportError: pass
|
||||
return estimate_gib,available_gib
|
||||
|
||||
available = int(psutil.virtual_memory().available)
|
||||
available_gib = available / 1024**3
|
||||
if required_bytes > int(available * 0.75):
|
||||
raise SequenceRuntimeError(
|
||||
"estimated decoded output requires about "
|
||||
f"{estimate_gib:.1f} GiB including one-chunk headroom, but only "
|
||||
f"{available_gib:.1f} GiB RAM is currently available. Reduce chunks, "
|
||||
"chunk_seconds, or resolution before sampling."
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
return estimate_gib, available_gib
|
||||
def _clone_entry_for_reuse(entry):
|
||||
entry=validate_chunk_entry(entry); result=dict(entry); result["plan"]=copy.deepcopy(entry["plan"]); result["reused"]=True; return result
|
||||
|
||||
|
||||
def _clone_entry_for_reuse(entry: dict[str, Any]) -> dict[str, Any]:
|
||||
entry = validate_chunk_entry(entry)
|
||||
result = dict(entry)
|
||||
result["plan"] = copy.deepcopy(entry["plan"])
|
||||
result["reused"] = True
|
||||
return result
|
||||
|
||||
|
||||
def _preserved_prefix(
|
||||
*,
|
||||
session: dict[str, Any] | None,
|
||||
prompt_hashes: list[str],
|
||||
chunks: int,
|
||||
reroll_from_chunk: int,
|
||||
width: int,
|
||||
height: int,
|
||||
chunk_seconds: float,
|
||||
identity_hash: str,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
if session is None:
|
||||
return [], []
|
||||
session = validate_session(session)
|
||||
notes: list[str] = []
|
||||
if int(session["width"]) != int(width) or int(session["height"]) != int(height):
|
||||
raise SessionValidationError(
|
||||
f"session is {session['width']}x{session['height']} but V2 is configured for {width}x{height}"
|
||||
)
|
||||
if abs(float(session["chunk_seconds"]) - float(chunk_seconds)) > 1e-6:
|
||||
raise SessionValidationError(
|
||||
f"session chunk_seconds={session['chunk_seconds']} does not match {chunk_seconds}"
|
||||
)
|
||||
if str(session.get("identity_hash", "none")) != str(identity_hash):
|
||||
raise SessionValidationError(
|
||||
"the current first-frame identity image does not match the saved session"
|
||||
)
|
||||
|
||||
if reroll_from_chunk < 0 or reroll_from_chunk > chunks:
|
||||
raise SessionValidationError("reroll_from_chunk must be 0 or a valid one-based chunk index")
|
||||
limit = min(len(session["chunks"]), chunks)
|
||||
if reroll_from_chunk > 0:
|
||||
limit = min(limit, reroll_from_chunk - 1)
|
||||
|
||||
preserved: list[dict[str, Any]] = []
|
||||
def _preserved_prefix(*,session,prompt_hashes,chunks,reroll_from_chunk,width,height,chunk_seconds,identity_hash):
|
||||
if session is None: return [],[]
|
||||
session=validate_session(session); notes=[]
|
||||
if int(session["width"])!=int(width) or int(session["height"])!=int(height): raise SessionValidationError(f"session is {session['width']}x{session['height']} but V2 is configured for {width}x{height}")
|
||||
if abs(float(session["chunk_seconds"])-float(chunk_seconds))>1e-6: raise SessionValidationError(f"session chunk_seconds={session['chunk_seconds']} does not match {chunk_seconds}")
|
||||
if str(session.get("identity_hash","none"))!=str(identity_hash): raise SessionValidationError("the current first-frame identity image does not match the saved session")
|
||||
if reroll_from_chunk<0 or reroll_from_chunk>chunks: raise SessionValidationError("reroll_from_chunk must be 0 or a valid one-based chunk index")
|
||||
limit=min(len(session["chunks"]),chunks)
|
||||
if reroll_from_chunk>0: limit=min(limit,reroll_from_chunk-1)
|
||||
preserved=[]
|
||||
for index in range(limit):
|
||||
entry = validate_chunk_entry(session["chunks"][index])
|
||||
if entry["prompt_hash"] != prompt_hashes[index]:
|
||||
notes.append(
|
||||
f"session reuse stopped before chunk {index + 1}: prompt changed"
|
||||
)
|
||||
break
|
||||
entry=validate_chunk_entry(session["chunks"][index])
|
||||
if entry["prompt_hash"]!=prompt_hashes[index]: notes.append(f"session reuse stopped before chunk {index+1}: prompt changed"); break
|
||||
preserved.append(_clone_entry_for_reuse(entry))
|
||||
if reroll_from_chunk == 0 and len(preserved) == min(len(session["chunks"]), chunks):
|
||||
if len(session["chunks"]) < chunks:
|
||||
notes.append(f"resuming after accepted chunk {len(session['chunks'])}")
|
||||
else:
|
||||
notes.append("all requested chunks reused from the session")
|
||||
elif reroll_from_chunk > 0:
|
||||
notes.append(
|
||||
f"preserved chunks 1-{len(preserved)}; regenerated from chunk {reroll_from_chunk}"
|
||||
)
|
||||
return preserved, notes
|
||||
if reroll_from_chunk==0 and len(preserved)==min(len(session["chunks"]),chunks):
|
||||
notes.append(f"resuming after accepted chunk {len(session['chunks'])}" if len(session["chunks"])<chunks else "all requested chunks reused from the session")
|
||||
elif reroll_from_chunk>0: notes.append(f"preserved chunks 1-{len(preserved)}; regenerated from chunk {reroll_from_chunk}")
|
||||
return preserved,notes
|
||||
|
||||
|
||||
def _conditioning_cache(
|
||||
*,
|
||||
clip: Any,
|
||||
prompts: list[str],
|
||||
assets: Any,
|
||||
final_has_last_frame: bool,
|
||||
) -> dict[tuple[str, bool], list]:
|
||||
cache: dict[tuple[str, bool], list] = {}
|
||||
final_index = len(prompts) - 1
|
||||
for index, prompt in enumerate(prompts):
|
||||
include_last = bool(final_has_last_frame and index == final_index)
|
||||
key = (prompt, include_last)
|
||||
if key in cache:
|
||||
continue
|
||||
cache[key] = encode_prompt_conditioning(
|
||||
clip,
|
||||
prompt,
|
||||
first_image=assets.first_image,
|
||||
last_image=assets.last_image if include_last else None,
|
||||
)
|
||||
def _conditioning_cache(*,clip,prompts,assets,final_has_last_frame):
|
||||
cache={}; final_index=len(prompts)-1
|
||||
for index,prompt in enumerate(prompts):
|
||||
include_last=bool(final_has_last_frame and index==final_index); key=(prompt,include_last)
|
||||
if key in cache: continue
|
||||
cache[key]=encode_prompt_conditioning(clip,prompt,first_image=assets.first_image,last_image=assets.last_image if include_last else None)
|
||||
return cache
|
||||
|
||||
|
||||
def run_sequence(
|
||||
*,
|
||||
model: Any,
|
||||
clip: Any,
|
||||
video_vae: Any,
|
||||
audio_vae: Any,
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
first_frame: torch.Tensor | None,
|
||||
last_frame: torch.Tensor | None,
|
||||
prompt_plan: dict[str, Any],
|
||||
width: int,
|
||||
height: int,
|
||||
continuity: str,
|
||||
base_seed: int,
|
||||
audio_continuity: bool,
|
||||
exact_total_duration: bool,
|
||||
diagnostics_mode: str,
|
||||
reroll_from_chunk: int,
|
||||
reroll_nonce: int,
|
||||
strict_compatibility: bool,
|
||||
debug: bool,
|
||||
seam_correction: str = SEAM_CORRECTION_OFF,
|
||||
enable_preview: bool = True,
|
||||
session: dict[str, Any] | None = None,
|
||||
initial_state: dict[str, Any] | None = None,
|
||||
) -> tuple[torch.Tensor, dict[str, Any], dict[str, Any], dict[str, Any], str]:
|
||||
"""Generate or resume a complete Continuum sequence."""
|
||||
|
||||
plan = validate_prompt_plan(prompt_plan)
|
||||
chunks = int(plan["chunks"])
|
||||
chunk_seconds = float(plan["chunk_seconds"])
|
||||
prompts = list(plan["prompts"])
|
||||
prompt_hashes = list(plan["hashes"])
|
||||
width, height = int(width), int(height)
|
||||
if width <= 0 or height <= 0 or width % 32 or height % 32:
|
||||
raise SequenceRuntimeError("width and height must be positive multiples of 32")
|
||||
if session is not None and initial_state is not None:
|
||||
raise SequenceRuntimeError("connect either a session or an initial_state, not both")
|
||||
if diagnostics_mode not in DIAGNOSTICS_OPTIONS:
|
||||
raise SequenceRuntimeError(f"unknown diagnostics mode: {diagnostics_mode!r}")
|
||||
if seam_correction not in SEAM_CORRECTION_OPTIONS:
|
||||
raise SequenceRuntimeError(f"unknown seam correction mode: {seam_correction!r}")
|
||||
if not (0 <= int(reroll_from_chunk) <= chunks):
|
||||
raise SequenceRuntimeError(
|
||||
"reroll_from_chunk must be 0 or a valid one-based chunk index"
|
||||
)
|
||||
if initial_state is not None and int(reroll_from_chunk) not in (0, 1):
|
||||
raise SequenceRuntimeError(
|
||||
"with initial_state, reroll_from_chunk can only be 0 or 1"
|
||||
)
|
||||
decode_estimate_gib, available_ram_gib = _check_decode_memory_budget(
|
||||
width=width,
|
||||
height=height,
|
||||
chunks=chunks,
|
||||
chunk_seconds=chunk_seconds,
|
||||
)
|
||||
|
||||
issues = check_comfy_h3_runtime()
|
||||
if issues and strict_compatibility:
|
||||
raise SequenceRuntimeError("H3 runtime is incompatible: " + "; ".join(issues))
|
||||
|
||||
# Phase 1: all CLIP and VAE encoding happens before H3 sampling. This is the
|
||||
# key residency optimization: no text encoder or VAE is requested between
|
||||
# H3 chunks.
|
||||
assets = prepare_identity_assets(
|
||||
video_vae,
|
||||
width=width,
|
||||
height=height,
|
||||
first_frame=first_frame,
|
||||
last_frame=last_frame,
|
||||
encode_latents=False,
|
||||
)
|
||||
|
||||
patched_model = patch_model(
|
||||
model,
|
||||
strict=bool(strict_compatibility),
|
||||
debug=bool(debug),
|
||||
)
|
||||
current_model_fingerprint = model_fingerprint(patched_model)
|
||||
preserved, reuse_notes = _preserved_prefix(
|
||||
session=session,
|
||||
prompt_hashes=prompt_hashes,
|
||||
chunks=chunks,
|
||||
reroll_from_chunk=int(reroll_from_chunk),
|
||||
width=width,
|
||||
height=height,
|
||||
chunk_seconds=chunk_seconds,
|
||||
identity_hash=assets.identity_hash,
|
||||
)
|
||||
def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,sigmas:torch.Tensor,first_frame:torch.Tensor|None,last_frame:torch.Tensor|None,prompt_plan:dict[str,Any],width:int,height:int,continuity:str,base_seed:int,audio_continuity:bool,exact_total_duration:bool,diagnostics_mode:str,reroll_from_chunk:int,reroll_nonce:int,strict_compatibility:bool,debug:bool,seam_correction:str=SEAM_CORRECTION_OFF,enable_preview:bool=True,session:dict[str,Any]|None=None,initial_state:dict[str,Any]|None=None):
|
||||
plan=validate_prompt_plan(prompt_plan); chunks=int(plan["chunks"]); chunk_seconds=float(plan["chunk_seconds"]); prompts=list(plan["prompts"]); prompt_hashes=list(plan["hashes"]); width,height=int(width),int(height)
|
||||
if width<=0 or height<=0 or width%32 or height%32: raise SequenceRuntimeError("width and height must be positive multiples of 32")
|
||||
if session is not None and initial_state is not None: raise SequenceRuntimeError("connect either a session or an initial_state, not both")
|
||||
if diagnostics_mode not in DIAGNOSTICS_OPTIONS: raise SequenceRuntimeError(f"unknown diagnostics mode: {diagnostics_mode!r}")
|
||||
if seam_correction not in SEAM_CORRECTION_OPTIONS: raise SequenceRuntimeError(f"unknown seam correction mode: {seam_correction!r}")
|
||||
if not 0<=int(reroll_from_chunk)<=chunks: raise SequenceRuntimeError("reroll_from_chunk must be 0 or a valid one-based chunk index")
|
||||
if initial_state is not None and int(reroll_from_chunk) not in (0,1): raise SequenceRuntimeError("with initial_state, reroll_from_chunk can only be 0 or 1")
|
||||
decode_estimate_gib,available_ram_gib=_check_decode_memory_budget(width=width,height=height,chunks=chunks,chunk_seconds=chunk_seconds)
|
||||
issues=check_comfy_h3_runtime()
|
||||
if issues and strict_compatibility: raise SequenceRuntimeError("H3 runtime is incompatible: "+"; ".join(issues))
|
||||
assets=prepare_identity_assets(video_vae,width=width,height=height,first_frame=first_frame,last_frame=last_frame,encode_latents=False)
|
||||
patched_model=patch_model(model,strict=bool(strict_compatibility),debug=bool(debug)); current_model_fingerprint=model_fingerprint(patched_model)
|
||||
preserved,reuse_notes=_preserved_prefix(session=session,prompt_hashes=prompt_hashes,chunks=chunks,reroll_from_chunk=int(reroll_from_chunk),width=width,height=height,chunk_seconds=chunk_seconds,identity_hash=assets.identity_hash)
|
||||
if session is not None:
|
||||
old_fingerprint = str(session.get("model_fingerprint", ""))
|
||||
if old_fingerprint and old_fingerprint != current_model_fingerprint:
|
||||
reuse_notes.append(
|
||||
"model/accelerator fingerprint differs from the saved session; accepted chunks were kept"
|
||||
)
|
||||
|
||||
cache: dict[tuple[str, bool], list] = {}
|
||||
if len(preserved) < chunks:
|
||||
assets = encode_identity_latents(video_vae, assets)
|
||||
cache = _conditioning_cache(
|
||||
clip=clip,
|
||||
prompts=prompts,
|
||||
assets=assets,
|
||||
final_has_last_frame=last_frame is not None,
|
||||
)
|
||||
|
||||
entries: list[dict[str, Any]] = preserved[:]
|
||||
previous_state = None
|
||||
if entries:
|
||||
previous_state = entry_to_state(entries[-1])
|
||||
old_fingerprint=str(session.get("model_fingerprint",""))
|
||||
if old_fingerprint and old_fingerprint!=current_model_fingerprint: reuse_notes.append("model/accelerator fingerprint differs from the saved session; accepted chunks were kept")
|
||||
cache={}
|
||||
if len(preserved)<chunks:
|
||||
assets=encode_identity_latents(video_vae,assets); cache=_conditioning_cache(clip=clip,prompts=prompts,assets=assets,final_has_last_frame=last_frame is not None)
|
||||
entries=preserved[:]; previous_state=None
|
||||
if entries: previous_state=entry_to_state(entries[-1])
|
||||
elif initial_state is not None:
|
||||
previous_state = validate_state(initial_state)
|
||||
if int(previous_state["width"]) != width or int(previous_state["height"]) != height:
|
||||
raise SequenceRuntimeError(
|
||||
f"initial_state is {previous_state['width']}x{previous_state['height']} "
|
||||
f"but V2 is configured for {width}x{height}"
|
||||
)
|
||||
|
||||
initial_frame_count = align_frame_count_up(int(round(chunk_seconds * FPS)))
|
||||
retained_frames = sum(int(entry["plan"]["net_frames"]) for entry in entries)
|
||||
sampling_reports: list[str] = []
|
||||
|
||||
# Phase 2: H3-only loop. Each chunk gets a MODEL clone so accelerator runtime
|
||||
# history cannot leak across chunks. Sampler/sigmas remain shared, and decoded
|
||||
# media is deliberately not requested until every chunk is done.
|
||||
for sequence_index in range(len(entries), chunks):
|
||||
prompt = prompts[sequence_index]
|
||||
prompt_hash_value = prompt_hashes[sequence_index]
|
||||
is_final = sequence_index == chunks - 1
|
||||
seed = derive_chunk_seed(base_seed, sequence_index, reroll_nonce)
|
||||
motion_score = 0.0
|
||||
video_context = None
|
||||
audio_context = None
|
||||
context_before = None
|
||||
|
||||
previous_state=validate_state(initial_state)
|
||||
if int(previous_state["width"])!=width or int(previous_state["height"])!=height: raise SequenceRuntimeError(f"initial_state is {previous_state['width']}x{previous_state['height']} but V2 is configured for {width}x{height}")
|
||||
initial_frame_count=align_frame_count_up(int(round(chunk_seconds*FPS))); retained_frames=sum(int(entry["plan"]["net_frames"]) for entry in entries); sampling_reports=[]
|
||||
for sequence_index in range(len(entries),chunks):
|
||||
prompt=prompts[sequence_index]; prompt_hash_value=prompt_hashes[sequence_index]; is_final=sequence_index==chunks-1; seed=derive_chunk_seed(base_seed,sequence_index,reroll_nonce); motion_score=0.0; video_context=None; audio_context=None; context_before=None
|
||||
if previous_state is None:
|
||||
total_frames = initial_frame_count
|
||||
latent = empty_h3_latent(width, height, total_frames)
|
||||
conditioning = attach_keyframes(
|
||||
cache[(prompt, bool(last_frame is not None and is_final))],
|
||||
frame_count=total_frames,
|
||||
first_latent=assets.first_latent,
|
||||
last_latent=assets.last_latent if is_final else None,
|
||||
)
|
||||
clip_index = 1
|
||||
context_frames = 0
|
||||
chunk_plan = make_plan(
|
||||
continuation=False,
|
||||
clip_index=clip_index,
|
||||
total_frames=total_frames,
|
||||
trim_frames=0,
|
||||
width=width,
|
||||
height=height,
|
||||
context_frames=5,
|
||||
state_capacity_frames=largest_context_capacity(total_frames),
|
||||
requested_extend_seconds=chunk_seconds,
|
||||
debug=debug,
|
||||
)
|
||||
reason = "initial clip"
|
||||
total_frames=initial_frame_count; latent=empty_h3_latent(width,height,total_frames); conditioning=attach_keyframes(cache[(prompt,bool(last_frame is not None and is_final))],frame_count=total_frames,first_latent=assets.first_latent,last_latent=assets.last_latent if is_final else None); clip_index=1; context_frames=0
|
||||
chunk_plan=make_plan(continuation=False,clip_index=clip_index,total_frames=total_frames,trim_frames=0,width=width,height=height,context_frames=5,state_capacity_frames=largest_context_capacity(total_frames),requested_extend_seconds=chunk_seconds,debug=debug); reason="initial clip"
|
||||
else:
|
||||
context_frames, motion_score, reason = choose_context_frames(
|
||||
continuity, previous_state
|
||||
)
|
||||
desired_cumulative = int(round((sequence_index + 1) * chunk_seconds * FPS))
|
||||
requested_new_frames = max(1, desired_cumulative - retained_frames)
|
||||
shape = make_extension_shape(
|
||||
context_frames, requested_new_frames / FPS
|
||||
)
|
||||
latent = empty_h3_latent(width, height, shape.total_frames)
|
||||
base_conditioning = attach_keyframes(
|
||||
cache[(prompt, bool(last_frame is not None and is_final))],
|
||||
frame_count=shape.total_frames,
|
||||
first_latent=assets.first_latent,
|
||||
last_latent=assets.last_latent if is_final else None,
|
||||
)
|
||||
video_context, audio_context, grid_offset = select_context(
|
||||
previous_state,
|
||||
context_frames,
|
||||
include_audio=bool(audio_continuity),
|
||||
)
|
||||
context_before = context_fingerprint(video_context, audio_context)
|
||||
conditioning = prepare_conditioning(
|
||||
base_conditioning,
|
||||
video_context=video_context,
|
||||
audio_context=audio_context,
|
||||
audio_grid_offset=grid_offset,
|
||||
context_frames=context_frames,
|
||||
new_frame_count=shape.total_frames,
|
||||
first_frame_policy=POLICY_REPLACE,
|
||||
preserve_last_frame=True,
|
||||
)
|
||||
clip_index = int(previous_state["clip_index"]) + 1
|
||||
chunk_plan = make_plan(
|
||||
continuation=True,
|
||||
clip_index=clip_index,
|
||||
total_frames=shape.total_frames,
|
||||
trim_frames=context_frames,
|
||||
width=width,
|
||||
height=height,
|
||||
context_frames=context_frames,
|
||||
state_capacity_frames=largest_context_capacity(shape.net_new_frames),
|
||||
requested_extend_seconds=chunk_seconds,
|
||||
debug=debug,
|
||||
)
|
||||
|
||||
chunk_model = clone_model_for_chunk(
|
||||
patched_model,
|
||||
strict=bool(strict_compatibility),
|
||||
debug=bool(debug),
|
||||
chunk_index=clip_index,
|
||||
context_frames=context_frames if previous_state is not None else None,
|
||||
)
|
||||
sampled = sample_chunk(
|
||||
model=chunk_model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=seed,
|
||||
enable_preview=bool(enable_preview),
|
||||
)
|
||||
if context_before is not None and video_context is not None:
|
||||
assert_context_unchanged(
|
||||
video_context,
|
||||
audio_context,
|
||||
context_before,
|
||||
)
|
||||
# Commit the full chunk to CPU once, then derive the small next-state tail
|
||||
# from that CPU entry. This avoids a second device-to-host transfer.
|
||||
entry = make_chunk_entry(
|
||||
latent=sampled,
|
||||
plan=chunk_plan,
|
||||
prompt=prompt,
|
||||
prompt_hash=prompt_hash_value,
|
||||
seed=seed,
|
||||
context_frames=context_frames,
|
||||
motion_score=motion_score,
|
||||
reused=False,
|
||||
)
|
||||
previous_state = entry_to_state(entry)
|
||||
entries.append(entry)
|
||||
retained_frames += int(chunk_plan["net_frames"])
|
||||
sampling_reports.append(
|
||||
f"chunk {sequence_index + 1}/{chunks}: seed={seed}, "
|
||||
f"frames={chunk_plan['total_frames']}, trim={chunk_plan['trim_frames']}, "
|
||||
f"retained_total={retained_frames}, context={context_frames} "
|
||||
f"({reason}), motion={motion_score:.6f}, "
|
||||
+ (
|
||||
f"interop=emitted actual_prefix={CONTINUUM_ACTUAL_PREFIX_STEPS} "
|
||||
"consumer=not_observable"
|
||||
if context_before is not None
|
||||
else "interop=not_emitted"
|
||||
)
|
||||
)
|
||||
del sampled, latent, conditioning, chunk_model
|
||||
|
||||
if len(entries) != chunks:
|
||||
raise SequenceRuntimeError(
|
||||
f"internal sequence length mismatch: expected {chunks}, got {len(entries)}"
|
||||
)
|
||||
|
||||
# Phase 3: VAE decode only after all H3 sampling is complete.
|
||||
if seam_correction == SEAM_CORRECTION_OFF:
|
||||
images, audio, decode_reports = decode_sequence(
|
||||
entries=entries,
|
||||
video_vae=video_vae,
|
||||
audio_vae=audio_vae,
|
||||
diagnostics_full=diagnostics_mode == DIAGNOSTICS_FULL,
|
||||
)
|
||||
else:
|
||||
images, audio, decode_reports = decode_sequence_with_seam(
|
||||
entries=entries,
|
||||
video_vae=video_vae,
|
||||
audio_vae=audio_vae,
|
||||
diagnostics_mode=diagnostics_mode,
|
||||
automatic=seam_correction == SEAM_CORRECTION_AUTO,
|
||||
)
|
||||
duration_report = ""
|
||||
context_frames,motion_score,reason=choose_context_frames(continuity,previous_state); desired_cumulative=int(round((sequence_index+1)*chunk_seconds*FPS)); requested_new_frames=max(1,desired_cumulative-retained_frames); shape=make_extension_shape(context_frames,requested_new_frames/FPS); latent=empty_h3_latent(width,height,shape.total_frames)
|
||||
base_conditioning=attach_keyframes(cache[(prompt,bool(last_frame is not None and is_final))],frame_count=shape.total_frames,first_latent=assets.first_latent,last_latent=assets.last_latent if is_final else None)
|
||||
video_context,audio_context,grid_offset=select_context(previous_state,context_frames,include_audio=bool(audio_continuity)); context_before=context_fingerprint(video_context,audio_context)
|
||||
conditioning=prepare_conditioning(base_conditioning,video_context=video_context,audio_context=audio_context,audio_grid_offset=grid_offset,context_frames=context_frames,new_frame_count=shape.total_frames,first_frame_policy=POLICY_REPLACE,preserve_last_frame=True)
|
||||
clip_index=int(previous_state["clip_index"])+1; chunk_plan=make_plan(continuation=True,clip_index=clip_index,total_frames=shape.total_frames,trim_frames=context_frames,width=width,height=height,context_frames=context_frames,state_capacity_frames=largest_context_capacity(shape.net_new_frames),requested_extend_seconds=chunk_seconds,debug=debug)
|
||||
chunk_model=clone_model_for_chunk(patched_model,strict=bool(strict_compatibility),debug=bool(debug),chunk_index=clip_index,context_frames=context_frames if previous_state is not None else None)
|
||||
sampled=sample_chunk(model=chunk_model,conditioning=conditioning,latent=latent,sampler=sampler,sigmas=sigmas,seed=seed,enable_preview=bool(enable_preview))
|
||||
if context_before is not None and video_context is not None: assert_context_unchanged(video_context,audio_context,context_before)
|
||||
entry=make_chunk_entry(latent=sampled,plan=chunk_plan,prompt=prompt,prompt_hash=prompt_hash_value,seed=seed,context_frames=context_frames,motion_score=motion_score,reused=False); previous_state=entry_to_state(entry); entries.append(entry); retained_frames+=int(chunk_plan["net_frames"])
|
||||
sampling_reports.append(f"chunk {sequence_index+1}/{chunks}: seed={seed}, frames={chunk_plan['total_frames']}, trim={chunk_plan['trim_frames']}, retained_total={retained_frames}, context={context_frames} ({reason}), motion={motion_score:.6f}, "+(f"interop=emitted actual_prefix={CONTINUUM_ACTUAL_PREFIX_STEPS} consumer=not_observable" if context_before is not None else "interop=not_emitted"))
|
||||
del sampled,latent,conditioning,chunk_model
|
||||
if len(entries)!=chunks: raise SequenceRuntimeError(f"internal sequence length mismatch: expected {chunks}, got {len(entries)}")
|
||||
if seam_correction==SEAM_CORRECTION_OFF: images,audio,decode_reports=decode_sequence(entries=entries,video_vae=video_vae,audio_vae=audio_vae,diagnostics_full=diagnostics_mode==DIAGNOSTICS_FULL)
|
||||
else: images,audio,decode_reports=decode_sequence_with_seam(entries=entries,video_vae=video_vae,audio_vae=audio_vae,diagnostics_mode=diagnostics_mode,automatic=seam_correction==SEAM_CORRECTION_AUTO)
|
||||
duration_report=""
|
||||
if exact_total_duration:
|
||||
target_frames = int(round(chunks * chunk_seconds * FPS))
|
||||
images, audio, duration_report = enforce_total_frames(
|
||||
images,
|
||||
audio,
|
||||
target_frames=target_frames,
|
||||
preserve_final_frame=last_frame is not None,
|
||||
)
|
||||
|
||||
last_state = entry_to_state(entries[-1])
|
||||
parent_id = session.get("session_id") if session is not None else None
|
||||
settings = {
|
||||
"continuity": continuity,
|
||||
"audio_continuity": bool(audio_continuity),
|
||||
"exact_total_duration": bool(exact_total_duration),
|
||||
"prompt_mode": plan["mode"],
|
||||
"base_seed": int(base_seed),
|
||||
"reroll_nonce": int(reroll_nonce),
|
||||
"diagnostics_mode": diagnostics_mode,
|
||||
"initial_state_source": initial_state is not None,
|
||||
}
|
||||
new_session = make_session(
|
||||
chunks=entries,
|
||||
width=width,
|
||||
height=height,
|
||||
chunk_seconds=chunk_seconds,
|
||||
identity_hash=assets.identity_hash,
|
||||
model_fingerprint_value=current_model_fingerprint,
|
||||
parent_session_id=parent_id,
|
||||
reroll_from_chunk=int(reroll_from_chunk),
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
decoded_gib = (
|
||||
float(images.shape[0]) * float(width) * float(height) * 3.0 * 4.0 / (1024.0 ** 3)
|
||||
)
|
||||
report_lines = [
|
||||
f"H3 Continuum V2 {PACKAGE_VERSION}",
|
||||
prompt_plan_report(plan),
|
||||
f"Seam correction: {seam_correction}.",
|
||||
accelerator_summary(patched_model),
|
||||
"Execution: conditioning precomputed; call-local MODEL clone per chunk; decode deferred until sampling completed.",
|
||||
*reuse_notes,
|
||||
]
|
||||
if diagnostics_mode != DIAGNOSTICS_OFF:
|
||||
report_lines.extend(
|
||||
[
|
||||
(
|
||||
f"Decode RAM estimate: {decode_estimate_gib:.2f} GiB including transient headroom"
|
||||
+ (
|
||||
f"; available at start {available_ram_gib:.2f} GiB."
|
||||
if available_ram_gib is not None
|
||||
else "."
|
||||
)
|
||||
),
|
||||
*sampling_reports,
|
||||
*decode_reports,
|
||||
]
|
||||
)
|
||||
if duration_report:
|
||||
report_lines.append(duration_report)
|
||||
report_lines.extend(
|
||||
[
|
||||
session_summary(new_session),
|
||||
f"Output: {images.shape[0]} frames ({images.shape[0]/FPS:.3f}s), "
|
||||
f"audio samples={audio['waveform'].shape[-1]}, decoded tensor≈{decoded_gib:.2f} GiB.",
|
||||
]
|
||||
)
|
||||
return images, audio, last_state, new_session, "\n".join(report_lines)
|
||||
target_frames=int(round(chunks*chunk_seconds*FPS)); images,audio,duration_report=enforce_total_frames(images,audio,target_frames=target_frames,preserve_final_frame=last_frame is not None)
|
||||
last_state=entry_to_state(entries[-1]); parent_id=session.get("session_id") if session is not None else None
|
||||
settings={"continuity":continuity,"audio_continuity":bool(audio_continuity),"exact_total_duration":bool(exact_total_duration),"prompt_mode":plan["mode"],"base_seed":int(base_seed),"reroll_nonce":int(reroll_nonce),"diagnostics_mode":diagnostics_mode,"initial_state_source":initial_state is not None}
|
||||
new_session=make_session(chunks=entries,width=width,height=height,chunk_seconds=chunk_seconds,identity_hash=assets.identity_hash,model_fingerprint_value=current_model_fingerprint,parent_session_id=parent_id,reroll_from_chunk=int(reroll_from_chunk),settings=settings)
|
||||
decoded_gib=float(images.shape[0])*float(width)*float(height)*3.0*4.0/(1024.0**3)
|
||||
report_lines=[f"H3 Continuum V2 {PACKAGE_VERSION}",prompt_plan_report(plan),f"Seam correction: {seam_correction}.",accelerator_summary(patched_model),"Execution: conditioning precomputed; call-local MODEL clone per chunk; decode deferred until sampling completed.",*reuse_notes]
|
||||
if diagnostics_mode!=DIAGNOSTICS_OFF: report_lines.extend([f"Decode RAM estimate: {decode_estimate_gib:.2f} GiB including transient headroom"+(f"; available at start {available_ram_gib:.2f} GiB." if available_ram_gib is not None else "."),*sampling_reports,*decode_reports])
|
||||
if duration_report: report_lines.append(duration_report)
|
||||
report_lines.extend([session_summary(new_session),f"Output: {images.shape[0]} frames ({images.shape[0]/FPS:.3f}s), audio samples={audio['waveform'].shape[-1]}, decoded tensor≈{decoded_gib:.2f} GiB."])
|
||||
return images,audio,last_state,new_session,"\n".join(report_lines)
|
||||
|
||||
+57
-127
@@ -1,142 +1,72 @@
|
||||
"""Atomic safetensors + JSON persistence for V2 sessions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
import json, logging, os, re, uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from .session import validate_session
|
||||
|
||||
LOG = logging.getLogger("h3_continuum_join")
|
||||
_SAFE_PREFIX = re.compile(r"[^A-Za-z0-9._-]+")
|
||||
_HEADER_KEY = "h3_continuum_session_json"
|
||||
|
||||
|
||||
def sanitize_prefix(prefix: str) -> str:
|
||||
value = _SAFE_PREFIX.sub("_", str(prefix).strip()).strip("._")
|
||||
return value or "h3_continuum_session"
|
||||
|
||||
|
||||
def session_directory() -> Path:
|
||||
LOG=logging.getLogger("h3_continuum_join")
|
||||
_SAFE_PREFIX=re.compile(r"[^A-Za-z0-9._-]+")
|
||||
_HEADER_KEY="h3_continuum_session_json"
|
||||
def sanitize_prefix(prefix:str)->str:
|
||||
value=_SAFE_PREFIX.sub("_",str(prefix).strip()).strip("._"); return value or "h3_continuum_session"
|
||||
def session_directory()->Path:
|
||||
try:
|
||||
import folder_paths
|
||||
root = Path(folder_paths.get_output_directory())
|
||||
except Exception:
|
||||
root = Path.cwd()
|
||||
path = root / "h3_continuum_sessions"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def session_paths(prefix: str, slot: int) -> tuple[Path, Path]:
|
||||
slot = int(slot)
|
||||
if not (1 <= slot <= 9999):
|
||||
raise ValueError("session slot must be between 1 and 9999")
|
||||
name = f"{sanitize_prefix(prefix)}_slot{slot:04d}"
|
||||
root = session_directory()
|
||||
return root / f"{name}.safetensors", root / f"{name}.json"
|
||||
|
||||
|
||||
def _fsync_file(path: Path) -> None:
|
||||
import folder_paths; root=Path(folder_paths.get_output_directory())
|
||||
except Exception: root=Path.cwd()
|
||||
path=root/"h3_continuum_sessions"; path.mkdir(parents=True,exist_ok=True); return path
|
||||
def session_paths(prefix:str,slot:int)->tuple[Path,Path]:
|
||||
slot=int(slot)
|
||||
if not 1<=slot<=9999: raise ValueError("session slot must be between 1 and 9999")
|
||||
name=f"{sanitize_prefix(prefix)}_slot{slot:04d}"; root=session_directory(); return root/f"{name}.safetensors",root/f"{name}.json"
|
||||
def _fsync_file(path):
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
if os.name == "nt":
|
||||
return
|
||||
with path.open("rb") as handle: os.fsync(handle.fileno())
|
||||
except OSError: pass
|
||||
def _fsync_directory(path):
|
||||
if os.name=="nt": return
|
||||
try:
|
||||
descriptor = os.open(str(path), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _metadata_without_tensors(session: dict[str, Any]) -> dict[str, Any]:
|
||||
metadata = {key: value for key, value in session.items() if key != "chunks"}
|
||||
chunks = []
|
||||
for entry in session["chunks"]:
|
||||
chunks.append({key: value for key, value in entry.items() if key not in ("video", "audio")})
|
||||
metadata["chunks"] = chunks
|
||||
return metadata
|
||||
|
||||
|
||||
def save_session(session: dict[str, Any], *, prefix: str, slot: int) -> tuple[Path, Path]:
|
||||
session = validate_session(session)
|
||||
tensor_path, json_path = session_paths(prefix, slot)
|
||||
token = uuid.uuid4().hex
|
||||
tensor_tmp = tensor_path.with_name(f".{tensor_path.name}.{token}.tmp")
|
||||
json_tmp = json_path.with_name(f".{json_path.name}.{token}.tmp")
|
||||
|
||||
tensors = {}
|
||||
for index, entry in enumerate(session["chunks"], start=1):
|
||||
tensors[f"chunk_{index:04d}_video"] = entry["video"].detach().to("cpu").contiguous()
|
||||
tensors[f"chunk_{index:04d}_audio"] = entry["audio"].detach().to("cpu").contiguous()
|
||||
metadata = _metadata_without_tensors(session)
|
||||
metadata["_session_commit_id"] = token
|
||||
serialized = json.dumps(metadata, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
descriptor=os.open(str(path),os.O_RDONLY)
|
||||
try: os.fsync(descriptor)
|
||||
finally: os.close(descriptor)
|
||||
except OSError: pass
|
||||
def _metadata_without_tensors(session):
|
||||
metadata={k:v for k,v in session.items() if k!="chunks"}; chunks=[]
|
||||
for entry in session["chunks"]: chunks.append({k:v for k,v in entry.items() if k not in ("video","audio")})
|
||||
metadata["chunks"]=chunks; return metadata
|
||||
def save_session(session,*,prefix,slot):
|
||||
session=validate_session(session); tensor_path,json_path=session_paths(prefix,slot); token=uuid.uuid4().hex
|
||||
tensor_tmp=tensor_path.with_name(f".{tensor_path.name}.{token}.tmp"); json_tmp=json_path.with_name(f".{json_path.name}.{token}.tmp")
|
||||
tensors={}
|
||||
for index,entry in enumerate(session["chunks"],start=1):
|
||||
tensors[f"chunk_{index:04d}_video"]=entry["video"].detach().to("cpu").contiguous(); tensors[f"chunk_{index:04d}_audio"]=entry["audio"].detach().to("cpu").contiguous()
|
||||
metadata=_metadata_without_tensors(session); metadata["_session_commit_id"]=token
|
||||
serialized=json.dumps(metadata,ensure_ascii=False,sort_keys=True,separators=(",",":"))
|
||||
try:
|
||||
save_file(tensors, str(tensor_tmp), metadata={_HEADER_KEY: serialized})
|
||||
_fsync_file(tensor_tmp)
|
||||
with json_tmp.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(metadata, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(json_tmp, json_path)
|
||||
os.replace(tensor_tmp, tensor_path)
|
||||
_fsync_directory(tensor_path.parent)
|
||||
save_file(tensors,str(tensor_tmp),metadata={_HEADER_KEY:serialized}); _fsync_file(tensor_tmp)
|
||||
with json_tmp.open("w",encoding="utf-8",newline="\n") as handle:
|
||||
json.dump(metadata,handle,ensure_ascii=False,indent=2,sort_keys=True); handle.write("\n"); handle.flush(); os.fsync(handle.fileno())
|
||||
os.replace(json_tmp,json_path); os.replace(tensor_tmp,tensor_path); _fsync_directory(tensor_path.parent)
|
||||
finally:
|
||||
for path in (tensor_tmp, json_tmp):
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return tensor_path, json_path
|
||||
|
||||
|
||||
def load_session(*, prefix: str, slot: int) -> dict[str, Any]:
|
||||
tensor_path, json_path = session_paths(prefix, slot)
|
||||
if not tensor_path.is_file():
|
||||
raise FileNotFoundError(f"session slot does not exist: {tensor_path.name}")
|
||||
with safe_open(str(tensor_path), framework="pt", device="cpu") as handle:
|
||||
serialized = (handle.metadata() or {}).get(_HEADER_KEY)
|
||||
if serialized is None:
|
||||
raise ValueError("session safetensors has no embedded Continuum metadata")
|
||||
session = json.loads(serialized)
|
||||
for index, entry in enumerate(session.get("chunks") or [], start=1):
|
||||
entry["video"] = handle.get_tensor(f"chunk_{index:04d}_video")
|
||||
entry["audio"] = handle.get_tensor(f"chunk_{index:04d}_audio")
|
||||
|
||||
for path in (tensor_tmp,json_tmp):
|
||||
try: path.unlink(missing_ok=True)
|
||||
except OSError: pass
|
||||
return tensor_path,json_path
|
||||
def load_session(*,prefix,slot):
|
||||
tensor_path,json_path=session_paths(prefix,slot)
|
||||
if not tensor_path.is_file(): raise FileNotFoundError(f"session slot does not exist: {tensor_path.name}")
|
||||
with safe_open(str(tensor_path),framework="pt",device="cpu") as handle:
|
||||
serialized=(handle.metadata() or {}).get(_HEADER_KEY)
|
||||
if serialized is None: raise ValueError("session safetensors has no embedded Continuum metadata")
|
||||
session=json.loads(serialized)
|
||||
for index,entry in enumerate(session.get("chunks") or [],start=1):
|
||||
entry["video"]=handle.get_tensor(f"chunk_{index:04d}_video"); entry["audio"]=handle.get_tensor(f"chunk_{index:04d}_audio")
|
||||
if json_path.is_file():
|
||||
try:
|
||||
with json_path.open("r", encoding="utf-8") as handle:
|
||||
mirror = json.load(handle)
|
||||
if mirror.get("_session_commit_id") != session.get("_session_commit_id"):
|
||||
LOG.warning("Ignoring stale Continuum session JSON mirror: %s", json_path.name)
|
||||
except (OSError, ValueError, TypeError) as exc:
|
||||
LOG.warning("Ignoring unreadable session JSON mirror %s: %s", json_path.name, exc)
|
||||
with json_path.open("r",encoding="utf-8") as handle: mirror=json.load(handle)
|
||||
if mirror.get("_session_commit_id")!=session.get("_session_commit_id"): LOG.warning("Ignoring stale Continuum session JSON mirror: %s",json_path.name)
|
||||
except (OSError,ValueError,TypeError) as exc: LOG.warning("Ignoring unreadable session JSON mirror %s: %s",json_path.name,exc)
|
||||
return validate_session(session)
|
||||
|
||||
|
||||
def session_mtime(*, prefix: str, slot: int) -> tuple[float, float]:
|
||||
tensor_path, json_path = session_paths(prefix, slot)
|
||||
return (
|
||||
tensor_path.stat().st_mtime if tensor_path.exists() else -1.0,
|
||||
json_path.stat().st_mtime if json_path.exists() else -1.0,
|
||||
)
|
||||
def session_mtime(*,prefix,slot):
|
||||
tensor_path,json_path=session_paths(prefix,slot); return (tensor_path.stat().st_mtime if tensor_path.exists() else -1.0,json_path.stat().st_mtime if json_path.exists() else -1.0)
|
||||
|
||||
Reference in New Issue
Block a user