Files
ComfyUI-H3-Continuum-Tiled/v3/targeted_refine_sampling.py
T

497 lines
16 KiB
Python

"""Target-aware refinement sampling for frozen Video and Audio policies."""
from __future__ import annotations
from typing import Any, Callable
import torch
from ..state import extract_av_streams
from .refine_sampling import (
RefineSamplingRuntimeError,
_load_runtime_modules,
_make_basic_guider,
sample_refine_chunk,
)
from .refine_target import (
MODE_AUDIO_ONLY,
MODE_VIDEO_AUDIO,
MODE_VIDEO_ONLY,
RefineTarget,
RefineTargetError,
resolve_refine_target,
)
from .refine_window import GroupTemporalWindow
class TargetedRefineExecutionUnavailable(RefineTargetError):
"""Raised before Sampling when a future target mode is not implemented."""
def _validate_ranges(
ranges: tuple[tuple[int, int], ...],
length: int,
*,
name: str,
) -> tuple[tuple[int, int], ...]:
if not ranges:
raise RefineSamplingRuntimeError(f"{name} temporal window is empty")
previous_stop = 0
for start, stop in ranges:
if start < previous_stop or stop <= start or stop > int(length):
raise RefineSamplingRuntimeError(
f"{name} temporal window is outside its latent length"
)
previous_stop = stop
return ranges
def _window_masks(
video: torch.Tensor,
audio: torch.Tensor,
temporal_window: GroupTemporalWindow,
) -> tuple[torch.Tensor, torch.Tensor]:
if not isinstance(temporal_window, GroupTemporalWindow):
raise RefineSamplingRuntimeError(
"temporal window must be a resolved GroupTemporalWindow"
)
video_ranges = _validate_ranges(
temporal_window.video_slot_ranges,
int(video.shape[2]),
name="Video",
)
audio_ranges = _validate_ranges(
temporal_window.audio_tick_ranges,
int(audio.shape[-1]),
name="Audio",
)
video_mask = torch.zeros(
(
int(video.shape[0]),
1,
int(video.shape[2]),
int(video.shape[3]),
int(video.shape[4]),
),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.zeros(
(
int(audio.shape[0]),
1,
int(audio.shape[2]),
int(audio.shape[3]),
),
dtype=torch.float32,
device="cpu",
)
for start, stop in video_ranges:
video_mask[:, :, start:stop] = 1.0
for start, stop in audio_ranges:
audio_mask[..., start:stop] = 1.0
return video_mask, audio_mask
def _prepare_video_only_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
temporal_window: GroupTemporalWindow,
) -> tuple[Any, Any]:
"""Build windowed Video noise/mask while locking Audio completely."""
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
raise RefineSamplingRuntimeError(
"ComfyUI returned invalid Video Only noise geometry"
)
video_mask, _audio_window_mask = _window_masks(video, audio, temporal_window)
audio_noise = torch.zeros_like(audio, device="cpu")
audio_mask = torch.zeros(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
return (
nested_builder((video_noise, audio_noise)),
nested_builder((video_mask, audio_mask)),
)
def require_video_only_execution(
refine_target: RefineTarget | str | None,
) -> RefineTarget:
target = resolve_refine_target(refine_target)
if target.mode != MODE_VIDEO_ONLY:
raise TargetedRefineExecutionUnavailable(
f"refine target {target.mode!r} is contract-only in Phase A1"
)
return target
def _prepare_audio_only_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
temporal_window: GroupTemporalWindow | None = None,
) -> tuple[Any, Any]:
"""Build zero/locked Video and seeded-random/sampled Audio streams."""
if video.ndim != 5 or audio.ndim != 4:
raise RefineSamplingRuntimeError(
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
)
if int(video.shape[0]) != int(audio.shape[0]):
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
raise RefineSamplingRuntimeError(
"ComfyUI returned invalid Audio Only noise geometry"
)
video_noise = torch.zeros_like(video, device="cpu")
if temporal_window is None:
video_mask = torch.zeros(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.ones(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
else:
_video_window_mask, audio_mask = _window_masks(
video,
audio,
temporal_window,
)
video_mask = torch.zeros(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
noise = nested_builder((video_noise, audio_noise))
noise_mask = nested_builder((video_mask, audio_mask))
return noise, noise_mask
def _prepare_video_audio_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
temporal_window: GroupTemporalWindow | None = None,
) -> tuple[Any, Any]:
"""Build seeded-random noise and sampled masks for both AV streams."""
if video.ndim != 5 or audio.ndim != 4:
raise RefineSamplingRuntimeError(
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
)
if int(video.shape[0]) != int(audio.shape[0]):
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
raise RefineSamplingRuntimeError(
"ComfyUI returned invalid Video + Audio video noise geometry"
)
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
raise RefineSamplingRuntimeError(
"ComfyUI returned invalid Video + Audio audio noise geometry"
)
if temporal_window is None:
video_mask = torch.ones(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.ones(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
else:
video_mask, audio_mask = _window_masks(video, audio, temporal_window)
noise = nested_builder((video_noise, audio_noise))
noise_mask = nested_builder((video_mask, audio_mask))
return noise, noise_mask
def _sample_with_target_noise_and_mask(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
enable_preview: bool,
noise_mask_builder: Callable[..., tuple[Any, Any]],
temporal_window: GroupTemporalWindow | None = None,
restore_video_outside_window: bool = False,
restore_audio_outside_window: bool = False,
) -> dict[str, Any]:
"""Run Core once after constructing one explicit AV target policy."""
if not isinstance(latent, dict) or "samples" not in latent:
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
_load_runtime_modules()
)
working = latent.copy()
latent_image = comfy_sample.fix_empty_latent_channels(
model,
working["samples"],
working.get("downscale_ratio_spacial"),
working.get("downscale_ratio_temporal"),
)
working["samples"] = latent_image
video, audio = extract_av_streams(working)
noise, noise_mask = noise_mask_builder(
video,
audio,
seed=int(seed),
batch_inds=working.get("batch_index"),
prepare_noise_fn=comfy_sample.prepare_noise,
nested_builder=nested_tensor.NestedTensor,
temporal_window=temporal_window,
)
guider = _make_basic_guider(model, conditioning)
x0_output: dict[str, Any] = {}
callback = None
if enable_preview:
callback = preview.prepare_callback(
model,
int(sigmas.shape[-1]) - 1,
x0_output,
)
samples = guider.sample(
noise,
latent_image,
sampler,
sigmas,
denoise_mask=noise_mask,
callback=callback,
disable_pbar=not comfy_utils.PROGRESS_BAR_ENABLED,
seed=int(seed),
)
samples = samples.to(model_management.intermediate_device())
if temporal_window is not None:
sampled_output = {"samples": samples}
sampled_video, sampled_audio = extract_av_streams(sampled_output)
if restore_video_outside_window:
sampled_video = _restore_outside_temporal_ranges(
sampled_video,
video,
temporal_window.video_slot_ranges,
time_dim=2,
name="Video",
)
if restore_audio_outside_window:
sampled_audio = _restore_outside_temporal_ranges(
sampled_audio,
audio,
temporal_window.audio_tick_ranges,
time_dim=-1,
name="Audio",
)
samples = nested_tensor.NestedTensor((sampled_video, sampled_audio))
output = working.copy()
output.pop("downscale_ratio_spacial", None)
output.pop("downscale_ratio_temporal", None)
output["samples"] = samples
extract_av_streams(output)
return output
def _restore_outside_temporal_ranges(
sampled: torch.Tensor,
source: torch.Tensor,
ranges: tuple[tuple[int, int], ...],
*,
time_dim: int,
name: str,
) -> torch.Tensor:
"""Restore every inactive temporal position from the input bit-exactly."""
if not torch.is_tensor(sampled) or not torch.is_tensor(source):
raise RefineSamplingRuntimeError(f"{name} temporal restore requires Tensors")
if tuple(sampled.shape) != tuple(source.shape):
raise RefineSamplingRuntimeError(
f"{name} temporal restore input/output shapes differ"
)
if sampled.dtype != source.dtype:
raise RefineSamplingRuntimeError(
f"{name} temporal restore input/output dtypes differ"
)
dim = int(time_dim)
if dim < 0:
dim += sampled.ndim
if dim < 0 or dim >= sampled.ndim:
raise RefineSamplingRuntimeError(f"{name} temporal restore dimension is invalid")
validated = _validate_ranges(ranges, int(sampled.shape[dim]), name=name)
restored = sampled.clone()
original = source.to(device=restored.device)
cursor = 0
for start, stop in validated:
if start > cursor:
restored.narrow(dim, cursor, start - cursor).copy_(
original.narrow(dim, cursor, start - cursor)
)
cursor = stop
if cursor < int(restored.shape[dim]):
restored.narrow(dim, cursor, int(restored.shape[dim]) - cursor).copy_(
original.narrow(dim, cursor, int(restored.shape[dim]) - cursor)
)
return restored
def sample_video_only_window_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
temporal_window: GroupTemporalWindow,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Refine only selected Video slots and restore all other AV positions."""
return _sample_with_target_noise_and_mask(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
noise_mask_builder=_prepare_video_only_noise_and_mask,
temporal_window=temporal_window,
restore_video_outside_window=True,
restore_audio_outside_window=True,
)
def sample_audio_only_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
temporal_window: GroupTemporalWindow | None = None,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Sample Audio while Core carries Video with zero noise and mask zero."""
return _sample_with_target_noise_and_mask(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
noise_mask_builder=_prepare_audio_only_noise_and_mask,
temporal_window=temporal_window,
restore_video_outside_window=temporal_window is not None,
restore_audio_outside_window=temporal_window is not None,
)
def sample_video_audio_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
temporal_window: GroupTemporalWindow | None = None,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Sample and adopt both Video and Audio using mask one for each stream."""
return _sample_with_target_noise_and_mask(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
noise_mask_builder=_prepare_video_audio_noise_and_mask,
temporal_window=temporal_window,
restore_video_outside_window=temporal_window is not None,
restore_audio_outside_window=temporal_window is not None,
)
def sample_targeted_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
refine_target: RefineTarget | str | None = MODE_VIDEO_ONLY,
temporal_window: GroupTemporalWindow | None = None,
enable_preview: bool = True,
legacy_sample_fn: Callable[..., dict[str, Any]] | None = None,
video_only_window_sample_fn: Callable[..., dict[str, Any]] | None = None,
audio_only_sample_fn: Callable[..., dict[str, Any]] | None = None,
video_audio_sample_fn: Callable[..., dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Dispatch an implemented target without changing its stream policy."""
target = resolve_refine_target(refine_target)
if target.mode == MODE_VIDEO_ONLY and temporal_window is not None:
sample_fn = video_only_window_sample_fn or sample_video_only_window_refine_chunk
elif target.mode == MODE_VIDEO_ONLY:
sample_fn = legacy_sample_fn or sample_refine_chunk
elif target.mode == MODE_AUDIO_ONLY:
sample_fn = audio_only_sample_fn or sample_audio_only_refine_chunk
elif target.mode == MODE_VIDEO_AUDIO:
sample_fn = video_audio_sample_fn or sample_video_audio_refine_chunk
else: # pragma: no cover - resolve_refine_target validates the mode.
raise TargetedRefineExecutionUnavailable(
f"refine target {target.mode!r} is unavailable"
)
arguments = dict(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
)
if temporal_window is not None:
arguments["temporal_window"] = temporal_window
return sample_fn(**arguments)