git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
497 lines
16 KiB
Python
497 lines
16 KiB
Python
"""Target-aware refinement sampling for frozen Video and Audio policies."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Callable
|
|
|
|
import torch
|
|
|
|
from ..state import extract_av_streams
|
|
from .refine_sampling import (
|
|
RefineSamplingRuntimeError,
|
|
_load_runtime_modules,
|
|
_make_basic_guider,
|
|
sample_refine_chunk,
|
|
)
|
|
from .refine_target import (
|
|
MODE_AUDIO_ONLY,
|
|
MODE_VIDEO_AUDIO,
|
|
MODE_VIDEO_ONLY,
|
|
RefineTarget,
|
|
RefineTargetError,
|
|
resolve_refine_target,
|
|
)
|
|
from .refine_window import GroupTemporalWindow
|
|
|
|
|
|
class TargetedRefineExecutionUnavailable(RefineTargetError):
|
|
"""Raised before Sampling when a future target mode is not implemented."""
|
|
|
|
|
|
def _validate_ranges(
|
|
ranges: tuple[tuple[int, int], ...],
|
|
length: int,
|
|
*,
|
|
name: str,
|
|
) -> tuple[tuple[int, int], ...]:
|
|
if not ranges:
|
|
raise RefineSamplingRuntimeError(f"{name} temporal window is empty")
|
|
previous_stop = 0
|
|
for start, stop in ranges:
|
|
if start < previous_stop or stop <= start or stop > int(length):
|
|
raise RefineSamplingRuntimeError(
|
|
f"{name} temporal window is outside its latent length"
|
|
)
|
|
previous_stop = stop
|
|
return ranges
|
|
|
|
|
|
def _window_masks(
|
|
video: torch.Tensor,
|
|
audio: torch.Tensor,
|
|
temporal_window: GroupTemporalWindow,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
if not isinstance(temporal_window, GroupTemporalWindow):
|
|
raise RefineSamplingRuntimeError(
|
|
"temporal window must be a resolved GroupTemporalWindow"
|
|
)
|
|
video_ranges = _validate_ranges(
|
|
temporal_window.video_slot_ranges,
|
|
int(video.shape[2]),
|
|
name="Video",
|
|
)
|
|
audio_ranges = _validate_ranges(
|
|
temporal_window.audio_tick_ranges,
|
|
int(audio.shape[-1]),
|
|
name="Audio",
|
|
)
|
|
video_mask = torch.zeros(
|
|
(
|
|
int(video.shape[0]),
|
|
1,
|
|
int(video.shape[2]),
|
|
int(video.shape[3]),
|
|
int(video.shape[4]),
|
|
),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
audio_mask = torch.zeros(
|
|
(
|
|
int(audio.shape[0]),
|
|
1,
|
|
int(audio.shape[2]),
|
|
int(audio.shape[3]),
|
|
),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
for start, stop in video_ranges:
|
|
video_mask[:, :, start:stop] = 1.0
|
|
for start, stop in audio_ranges:
|
|
audio_mask[..., start:stop] = 1.0
|
|
return video_mask, audio_mask
|
|
|
|
|
|
def _prepare_video_only_noise_and_mask(
|
|
video: torch.Tensor,
|
|
audio: torch.Tensor,
|
|
*,
|
|
seed: int,
|
|
batch_inds: Any = None,
|
|
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
|
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
|
temporal_window: GroupTemporalWindow,
|
|
) -> tuple[Any, Any]:
|
|
"""Build windowed Video noise/mask while locking Audio completely."""
|
|
|
|
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
|
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
|
raise RefineSamplingRuntimeError(
|
|
"ComfyUI returned invalid Video Only noise geometry"
|
|
)
|
|
video_mask, _audio_window_mask = _window_masks(video, audio, temporal_window)
|
|
audio_noise = torch.zeros_like(audio, device="cpu")
|
|
audio_mask = torch.zeros(
|
|
(int(audio.shape[0]), 1, 1, 1),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
return (
|
|
nested_builder((video_noise, audio_noise)),
|
|
nested_builder((video_mask, audio_mask)),
|
|
)
|
|
|
|
|
|
def require_video_only_execution(
|
|
refine_target: RefineTarget | str | None,
|
|
) -> RefineTarget:
|
|
target = resolve_refine_target(refine_target)
|
|
if target.mode != MODE_VIDEO_ONLY:
|
|
raise TargetedRefineExecutionUnavailable(
|
|
f"refine target {target.mode!r} is contract-only in Phase A1"
|
|
)
|
|
return target
|
|
|
|
|
|
def _prepare_audio_only_noise_and_mask(
|
|
video: torch.Tensor,
|
|
audio: torch.Tensor,
|
|
*,
|
|
seed: int,
|
|
batch_inds: Any = None,
|
|
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
|
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
|
temporal_window: GroupTemporalWindow | None = None,
|
|
) -> tuple[Any, Any]:
|
|
"""Build zero/locked Video and seeded-random/sampled Audio streams."""
|
|
|
|
if video.ndim != 5 or audio.ndim != 4:
|
|
raise RefineSamplingRuntimeError(
|
|
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
|
)
|
|
if int(video.shape[0]) != int(audio.shape[0]):
|
|
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
|
|
|
|
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
|
|
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
|
|
raise RefineSamplingRuntimeError(
|
|
"ComfyUI returned invalid Audio Only noise geometry"
|
|
)
|
|
video_noise = torch.zeros_like(video, device="cpu")
|
|
if temporal_window is None:
|
|
video_mask = torch.zeros(
|
|
(int(video.shape[0]), 1, 1, 1, 1),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
audio_mask = torch.ones(
|
|
(int(audio.shape[0]), 1, 1, 1),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
else:
|
|
_video_window_mask, audio_mask = _window_masks(
|
|
video,
|
|
audio,
|
|
temporal_window,
|
|
)
|
|
video_mask = torch.zeros(
|
|
(int(video.shape[0]), 1, 1, 1, 1),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
noise = nested_builder((video_noise, audio_noise))
|
|
noise_mask = nested_builder((video_mask, audio_mask))
|
|
return noise, noise_mask
|
|
|
|
|
|
def _prepare_video_audio_noise_and_mask(
|
|
video: torch.Tensor,
|
|
audio: torch.Tensor,
|
|
*,
|
|
seed: int,
|
|
batch_inds: Any = None,
|
|
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
|
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
|
temporal_window: GroupTemporalWindow | None = None,
|
|
) -> tuple[Any, Any]:
|
|
"""Build seeded-random noise and sampled masks for both AV streams."""
|
|
|
|
if video.ndim != 5 or audio.ndim != 4:
|
|
raise RefineSamplingRuntimeError(
|
|
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
|
)
|
|
if int(video.shape[0]) != int(audio.shape[0]):
|
|
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
|
|
|
|
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
|
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
|
|
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
|
raise RefineSamplingRuntimeError(
|
|
"ComfyUI returned invalid Video + Audio video noise geometry"
|
|
)
|
|
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
|
|
raise RefineSamplingRuntimeError(
|
|
"ComfyUI returned invalid Video + Audio audio noise geometry"
|
|
)
|
|
if temporal_window is None:
|
|
video_mask = torch.ones(
|
|
(int(video.shape[0]), 1, 1, 1, 1),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
audio_mask = torch.ones(
|
|
(int(audio.shape[0]), 1, 1, 1),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
else:
|
|
video_mask, audio_mask = _window_masks(video, audio, temporal_window)
|
|
noise = nested_builder((video_noise, audio_noise))
|
|
noise_mask = nested_builder((video_mask, audio_mask))
|
|
return noise, noise_mask
|
|
|
|
|
|
def _sample_with_target_noise_and_mask(
|
|
*,
|
|
model: Any,
|
|
conditioning: list,
|
|
latent: dict[str, Any],
|
|
sampler: Any,
|
|
sigmas: torch.Tensor,
|
|
seed: int,
|
|
enable_preview: bool,
|
|
noise_mask_builder: Callable[..., tuple[Any, Any]],
|
|
temporal_window: GroupTemporalWindow | None = None,
|
|
restore_video_outside_window: bool = False,
|
|
restore_audio_outside_window: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Run Core once after constructing one explicit AV target policy."""
|
|
|
|
if not isinstance(latent, dict) or "samples" not in latent:
|
|
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
|
|
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
|
|
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
|
|
|
|
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
|
|
_load_runtime_modules()
|
|
)
|
|
working = latent.copy()
|
|
latent_image = comfy_sample.fix_empty_latent_channels(
|
|
model,
|
|
working["samples"],
|
|
working.get("downscale_ratio_spacial"),
|
|
working.get("downscale_ratio_temporal"),
|
|
)
|
|
working["samples"] = latent_image
|
|
video, audio = extract_av_streams(working)
|
|
noise, noise_mask = noise_mask_builder(
|
|
video,
|
|
audio,
|
|
seed=int(seed),
|
|
batch_inds=working.get("batch_index"),
|
|
prepare_noise_fn=comfy_sample.prepare_noise,
|
|
nested_builder=nested_tensor.NestedTensor,
|
|
temporal_window=temporal_window,
|
|
)
|
|
guider = _make_basic_guider(model, conditioning)
|
|
|
|
x0_output: dict[str, Any] = {}
|
|
callback = None
|
|
if enable_preview:
|
|
callback = preview.prepare_callback(
|
|
model,
|
|
int(sigmas.shape[-1]) - 1,
|
|
x0_output,
|
|
)
|
|
samples = guider.sample(
|
|
noise,
|
|
latent_image,
|
|
sampler,
|
|
sigmas,
|
|
denoise_mask=noise_mask,
|
|
callback=callback,
|
|
disable_pbar=not comfy_utils.PROGRESS_BAR_ENABLED,
|
|
seed=int(seed),
|
|
)
|
|
samples = samples.to(model_management.intermediate_device())
|
|
if temporal_window is not None:
|
|
sampled_output = {"samples": samples}
|
|
sampled_video, sampled_audio = extract_av_streams(sampled_output)
|
|
if restore_video_outside_window:
|
|
sampled_video = _restore_outside_temporal_ranges(
|
|
sampled_video,
|
|
video,
|
|
temporal_window.video_slot_ranges,
|
|
time_dim=2,
|
|
name="Video",
|
|
)
|
|
if restore_audio_outside_window:
|
|
sampled_audio = _restore_outside_temporal_ranges(
|
|
sampled_audio,
|
|
audio,
|
|
temporal_window.audio_tick_ranges,
|
|
time_dim=-1,
|
|
name="Audio",
|
|
)
|
|
samples = nested_tensor.NestedTensor((sampled_video, sampled_audio))
|
|
|
|
output = working.copy()
|
|
output.pop("downscale_ratio_spacial", None)
|
|
output.pop("downscale_ratio_temporal", None)
|
|
output["samples"] = samples
|
|
extract_av_streams(output)
|
|
return output
|
|
|
|
|
|
def _restore_outside_temporal_ranges(
|
|
sampled: torch.Tensor,
|
|
source: torch.Tensor,
|
|
ranges: tuple[tuple[int, int], ...],
|
|
*,
|
|
time_dim: int,
|
|
name: str,
|
|
) -> torch.Tensor:
|
|
"""Restore every inactive temporal position from the input bit-exactly."""
|
|
|
|
if not torch.is_tensor(sampled) or not torch.is_tensor(source):
|
|
raise RefineSamplingRuntimeError(f"{name} temporal restore requires Tensors")
|
|
if tuple(sampled.shape) != tuple(source.shape):
|
|
raise RefineSamplingRuntimeError(
|
|
f"{name} temporal restore input/output shapes differ"
|
|
)
|
|
if sampled.dtype != source.dtype:
|
|
raise RefineSamplingRuntimeError(
|
|
f"{name} temporal restore input/output dtypes differ"
|
|
)
|
|
dim = int(time_dim)
|
|
if dim < 0:
|
|
dim += sampled.ndim
|
|
if dim < 0 or dim >= sampled.ndim:
|
|
raise RefineSamplingRuntimeError(f"{name} temporal restore dimension is invalid")
|
|
validated = _validate_ranges(ranges, int(sampled.shape[dim]), name=name)
|
|
restored = sampled.clone()
|
|
original = source.to(device=restored.device)
|
|
cursor = 0
|
|
for start, stop in validated:
|
|
if start > cursor:
|
|
restored.narrow(dim, cursor, start - cursor).copy_(
|
|
original.narrow(dim, cursor, start - cursor)
|
|
)
|
|
cursor = stop
|
|
if cursor < int(restored.shape[dim]):
|
|
restored.narrow(dim, cursor, int(restored.shape[dim]) - cursor).copy_(
|
|
original.narrow(dim, cursor, int(restored.shape[dim]) - cursor)
|
|
)
|
|
return restored
|
|
|
|
|
|
def sample_video_only_window_refine_chunk(
|
|
*,
|
|
model: Any,
|
|
conditioning: list,
|
|
latent: dict[str, Any],
|
|
sampler: Any,
|
|
sigmas: torch.Tensor,
|
|
seed: int,
|
|
temporal_window: GroupTemporalWindow,
|
|
enable_preview: bool = True,
|
|
) -> dict[str, Any]:
|
|
"""Refine only selected Video slots and restore all other AV positions."""
|
|
|
|
return _sample_with_target_noise_and_mask(
|
|
model=model,
|
|
conditioning=conditioning,
|
|
latent=latent,
|
|
sampler=sampler,
|
|
sigmas=sigmas,
|
|
seed=int(seed),
|
|
enable_preview=bool(enable_preview),
|
|
noise_mask_builder=_prepare_video_only_noise_and_mask,
|
|
temporal_window=temporal_window,
|
|
restore_video_outside_window=True,
|
|
restore_audio_outside_window=True,
|
|
)
|
|
|
|
|
|
def sample_audio_only_refine_chunk(
|
|
*,
|
|
model: Any,
|
|
conditioning: list,
|
|
latent: dict[str, Any],
|
|
sampler: Any,
|
|
sigmas: torch.Tensor,
|
|
seed: int,
|
|
temporal_window: GroupTemporalWindow | None = None,
|
|
enable_preview: bool = True,
|
|
) -> dict[str, Any]:
|
|
"""Sample Audio while Core carries Video with zero noise and mask zero."""
|
|
|
|
return _sample_with_target_noise_and_mask(
|
|
model=model,
|
|
conditioning=conditioning,
|
|
latent=latent,
|
|
sampler=sampler,
|
|
sigmas=sigmas,
|
|
seed=int(seed),
|
|
enable_preview=bool(enable_preview),
|
|
noise_mask_builder=_prepare_audio_only_noise_and_mask,
|
|
temporal_window=temporal_window,
|
|
restore_video_outside_window=temporal_window is not None,
|
|
restore_audio_outside_window=temporal_window is not None,
|
|
)
|
|
|
|
|
|
def sample_video_audio_refine_chunk(
|
|
*,
|
|
model: Any,
|
|
conditioning: list,
|
|
latent: dict[str, Any],
|
|
sampler: Any,
|
|
sigmas: torch.Tensor,
|
|
seed: int,
|
|
temporal_window: GroupTemporalWindow | None = None,
|
|
enable_preview: bool = True,
|
|
) -> dict[str, Any]:
|
|
"""Sample and adopt both Video and Audio using mask one for each stream."""
|
|
|
|
return _sample_with_target_noise_and_mask(
|
|
model=model,
|
|
conditioning=conditioning,
|
|
latent=latent,
|
|
sampler=sampler,
|
|
sigmas=sigmas,
|
|
seed=int(seed),
|
|
enable_preview=bool(enable_preview),
|
|
noise_mask_builder=_prepare_video_audio_noise_and_mask,
|
|
temporal_window=temporal_window,
|
|
restore_video_outside_window=temporal_window is not None,
|
|
restore_audio_outside_window=temporal_window is not None,
|
|
)
|
|
|
|
|
|
def sample_targeted_refine_chunk(
|
|
*,
|
|
model: Any,
|
|
conditioning: list,
|
|
latent: dict[str, Any],
|
|
sampler: Any,
|
|
sigmas: torch.Tensor,
|
|
seed: int,
|
|
refine_target: RefineTarget | str | None = MODE_VIDEO_ONLY,
|
|
temporal_window: GroupTemporalWindow | None = None,
|
|
enable_preview: bool = True,
|
|
legacy_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
|
video_only_window_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
|
audio_only_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
|
video_audio_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Dispatch an implemented target without changing its stream policy."""
|
|
|
|
target = resolve_refine_target(refine_target)
|
|
if target.mode == MODE_VIDEO_ONLY and temporal_window is not None:
|
|
sample_fn = video_only_window_sample_fn or sample_video_only_window_refine_chunk
|
|
elif target.mode == MODE_VIDEO_ONLY:
|
|
sample_fn = legacy_sample_fn or sample_refine_chunk
|
|
elif target.mode == MODE_AUDIO_ONLY:
|
|
sample_fn = audio_only_sample_fn or sample_audio_only_refine_chunk
|
|
elif target.mode == MODE_VIDEO_AUDIO:
|
|
sample_fn = video_audio_sample_fn or sample_video_audio_refine_chunk
|
|
else: # pragma: no cover - resolve_refine_target validates the mode.
|
|
raise TargetedRefineExecutionUnavailable(
|
|
f"refine target {target.mode!r} is unavailable"
|
|
)
|
|
arguments = dict(
|
|
model=model,
|
|
conditioning=conditioning,
|
|
latent=latent,
|
|
sampler=sampler,
|
|
sigmas=sigmas,
|
|
seed=int(seed),
|
|
enable_preview=bool(enable_preview),
|
|
)
|
|
if temporal_window is not None:
|
|
arguments["temporal_window"] = temporal_window
|
|
return sample_fn(**arguments)
|