Files
DumasNodes/video_reference_modes.py
T
chris.dumas bf8f76505f Squashed 'vendor/dumas_h3_continuum/' content from commit d07ed31
git-subtree-dir: vendor/dumas_h3_continuum
git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
2026-10-20 13:43:38 +00:00

300 lines
11 KiB
Python

"""Experimental Follow Timeline over the existing 24-fps IMAGE input.
Repeat Reference deliberately continues through reference_video.py unchanged.
A Follow source has a different identity, but its identity does not depend on
Chunks. A physical window is selected from the existing retained-frame plan,
not a Queue-local counter. No sampling, continuation or audio code lives here.
"""
from __future__ import annotations
from dataclasses import dataclass
import math
from typing import Any, Callable
import torch
from .reference_video import (
REFERENCE_VIDEO_PREPROCESS_VERSION,
REFERENCE_VIDEO_SIZE_EFFICIENT,
REFERENCE_VIDEO_SIZE_OPTIONS,
ReferenceVideoAssets,
ReferenceVideoError,
ReferenceVideoSource,
_canonical_hash,
_prepare_reference_frames,
encode_reference_video,
encode_reference_video_cached,
resolve_reference_video_size,
)
from .temporal import align_frame_count_up
REPEAT_REFERENCE = "Repeat Reference"
FOLLOW_TIMELINE = "Follow Timeline"
VIDEO_REFERENCE_MODES = (FOLLOW_TIMELINE, REPEAT_REFERENCE)
FOLLOW_VERSION = 1
FPS = 24
FOLLOW_PLAN_KEY = "_h3_continuum_video_reference_v1"
WINDOW_POLICY = "natural_visible_frames_excluding_context_v1"
END_POLICY = "remaining_interval_then_no_reference_v1"
def mode_input_definition() -> tuple:
"""Append this widget; never insert it into the old serialized prefix."""
return (
VIDEO_REFERENCE_MODES,
{
"default": REPEAT_REFERENCE,
"display_name": "Video Reference Mode",
"tooltip": (
"Experimental. Follow Timeline selects the source interval for each "
"physical generation group, then continues without video reference "
"after the source ends. Repeat Reference preserves the previous "
"Video Guide behavior: the same bounded prefix for every group. "
"This is independent of Prompt Format. Input frames must be 24 fps."
),
},
)
@dataclass(frozen=True)
class FollowVideoSource:
"""Run-scoped, normalized source. Do not mutate frames after preparation."""
frames: torch.Tensor
source_shape: tuple[int, ...]
source_dtype: str
source_sha256: str
size_mode: str
target_width: int
target_height: int
chunk_seconds: float
combined_hash: str
@property
def frame_count(self) -> int:
return int(self.source_shape[0])
@property
def contract(self) -> dict[str, Any]:
# No configured Chunks or total duration: adding future groups must not
# invalidate an otherwise unchanged saved prefix. Mode is nevertheless
# part of the reference identity, so Follow and Repeat never collide.
return {
"follow_video_contract_version": FOLLOW_VERSION,
"mode": FOLLOW_TIMELINE,
"source_shape": list(self.source_shape),
"source_dtype": self.source_dtype,
"source_sha256": self.source_sha256,
"source_fps": FPS,
"size_mode": self.size_mode,
"target_width": self.target_width,
"target_height": self.target_height,
"chunk_seconds": self.chunk_seconds,
"preprocess_version": REFERENCE_VIDEO_PREPROCESS_VERSION,
"window_policy": WINDOW_POLICY,
"end_policy": END_POLICY,
"combined_hash": self.combined_hash,
}
@dataclass(frozen=True)
class FollowVideoWindow:
"""Half-open source frame range, before H3's short alignment padding."""
requested_start: int
requested_stop: int
source_start: int
source_stop: int
encoded_frames: int
source_frames: int
identity: str
@property
def available_frames(self) -> int:
return self.source_stop - self.source_start
@property
def active(self) -> bool:
return self.available_frames >= 5
@property
def status(self) -> str:
if self.available_frames == 0:
return "source_exhausted"
if not self.active:
return "short_remainder_skipped"
if self.source_stop < self.requested_stop:
return "partial_source"
return "active"
@property
def contract(self) -> dict[str, Any]:
return {
"version": FOLLOW_VERSION,
"mode": FOLLOW_TIMELINE,
"requested_start_frame": self.requested_start,
"requested_stop_frame": self.requested_stop,
"source_start_frame": self.source_start,
"source_stop_frame": self.source_stop,
"source_frames": self.source_frames,
"encoded_frames": self.encoded_frames,
"padding_frames": self.encoded_frames - self.available_frames if self.active else 0,
"fps": FPS,
"status": self.status,
"slice_sha256": self.identity,
}
def report(self) -> str:
if not self.active:
return (
f"Timeline Video: output start={self.requested_start / FPS:.3f}s; "
f"{self.status}; no video reference for this physical group."
)
return (
"Timeline Video: source "
f"[{self.source_start / FPS:.3f}, {self.source_stop / FPS:.3f})s "
f"({self.available_frames} frames, H3={self.encoded_frames}); {self.status}."
)
def prepare_follow_video_source(
frames: torch.Tensor,
*,
chunk_seconds: float,
output_width: int,
output_height: int,
size_mode: str = REFERENCE_VIDEO_SIZE_EFFICIENT,
) -> FollowVideoSource:
"""Keep the full normalized source for future windows, without a GPU copy.
This IMAGE-input implementation is not a lazy media-file reader. Its CPU
source copy is intentional: hashing and encoding must see the same pixels.
It makes no claim of reducing long-input RAM use.
"""
if not torch.is_tensor(frames) or frames.ndim != 4 or int(frames.shape[-1]) < 3:
raise ReferenceVideoError("Timeline Video Frames requires IMAGE [T,H,W,C] with RGB")
if int(frames.shape[1]) < 1 or int(frames.shape[2]) < 1:
raise ReferenceVideoError("Timeline Video Frames has empty spatial dimensions")
seconds = float(chunk_seconds)
if not math.isfinite(seconds) or seconds <= 0:
raise ValueError("Invalid internal chunk duration for Timeline Video")
mode = str(size_mode) if str(size_mode) in REFERENCE_VIDEO_SIZE_OPTIONS else REFERENCE_VIDEO_SIZE_EFFICIENT
normalized, shape, dtype, fingerprint = _prepare_reference_frames(
frames, frame_count=int(frames.shape[0])
)
width, height = resolve_reference_video_size(
int(shape[2]), int(shape[1]),
output_width=int(output_width), output_height=int(output_height), size_mode=mode,
)
descriptor = {
"follow_video_contract_version": FOLLOW_VERSION,
"mode": FOLLOW_TIMELINE,
"source_shape": list(shape),
"source_dtype": dtype,
"source_sha256": fingerprint,
"source_fps": FPS,
"size_mode": mode,
"target_width": width,
"target_height": height,
"chunk_seconds": seconds,
"preprocess_version": REFERENCE_VIDEO_PREPROCESS_VERSION,
"window_policy": WINDOW_POLICY,
"end_policy": END_POLICY,
}
return FollowVideoSource(
normalized, shape, dtype, fingerprint, mode, width, height, seconds,
_canonical_hash(descriptor),
)
def select_follow_window(
source: FollowVideoSource, *, start_frame: int, visible_frames: int,
) -> FollowVideoWindow:
"""Select the existing physical plan's visible range, never its context.
Use the natural (pre-final-crop) output range. Clamping this to the current
requested total would change an old final group's conditioning on extension.
Only the available source and H3's <=16-frame grid padding limit the window.
"""
if isinstance(start_frame, bool) or isinstance(visible_frames, bool):
raise ValueError("Physical video window indices must be integers")
if int(start_frame) != start_frame or int(visible_frames) != visible_frames:
raise ValueError("Physical video window indices must be integers")
start, count = int(start_frame), int(visible_frames)
if start < 0 or count < 1:
raise ValueError("Invalid internal visible-frame window")
stop = start + count
first = min(start, source.frame_count)
last = min(stop, source.frame_count)
available = last - first
encoded = align_frame_count_up(available) if available >= 5 else 0
identity = _canonical_hash({
"source": source.combined_hash,
"requested_start": start,
"requested_stop": stop,
"source_start": first,
"source_stop": last,
"encoded_frames": encoded,
"window_policy": WINDOW_POLICY,
})
return FollowVideoWindow(start, stop, first, last, encoded, source.frame_count, identity)
def encode_follow_video_group(
video_vae: Any,
source: FollowVideoSource,
*,
start_frame: int,
visible_frames: int,
cache_enabled: bool = False,
cache_event: Callable | None = None,
) -> tuple[ReferenceVideoAssets | None, FollowVideoWindow]:
window = select_follow_window(source, start_frame=start_frame, visible_frames=visible_frames)
if not window.active:
return None, window
selected = source.frames[window.source_start:window.source_stop]
# Both the Qwen item and native Video VAE latent are made by the SAME
# existing encoder from this one selected frame batch. Slice identity is
# included in the old cache's source.combined_hash slot.
selected_source = ReferenceVideoSource(
frames=selected,
source_shape=tuple(int(v) for v in selected.shape),
source_dtype=source.source_dtype,
source_sha256=window.identity,
size_mode=source.size_mode,
target_width=source.target_width,
target_height=source.target_height,
frame_count=window.encoded_frames,
combined_hash=window.identity,
)
if cache_enabled:
assets = encode_reference_video_cached(video_vae, selected_source, cache_event=cache_event)
else:
assets = encode_reference_video(video_vae, selected_source)
return assets, window
def build_follow_group_conditioning(
source: FollowVideoSource,
video_vae: Any,
*,
start_frame: int,
visible_frames: int,
conditioning_builder: Callable,
conditioning_kwargs: dict[str, Any],
cache_enabled: bool = False,
cache_event: Callable | None = None,
) -> tuple[dict, FollowVideoWindow]:
"""Use a fresh prompt-cache dictionary per physical reference interval.
Never reuse a prior group's text conditioning merely because prompt text
matches. The VAE cache remains reusable because its key includes the slice.
"""
assets, window = encode_follow_video_group(
video_vae, source, start_frame=start_frame, visible_frames=visible_frames,
cache_enabled=cache_enabled, cache_event=cache_event,
)
kwargs = dict(conditioning_kwargs)
kwargs["timeline_video_assets"] = assets
return conditioning_builder(**kwargs), window