Release V3.8 source, illustrated manual and Spectrum workflow
This commit is contained in:
@@ -0,0 +1,496 @@
|
||||
"""Target-aware refinement sampling for frozen Video and Audio policies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import extract_av_streams
|
||||
from .refine_sampling import (
|
||||
RefineSamplingRuntimeError,
|
||||
_load_runtime_modules,
|
||||
_make_basic_guider,
|
||||
sample_refine_chunk,
|
||||
)
|
||||
from .refine_target import (
|
||||
MODE_AUDIO_ONLY,
|
||||
MODE_VIDEO_AUDIO,
|
||||
MODE_VIDEO_ONLY,
|
||||
RefineTarget,
|
||||
RefineTargetError,
|
||||
resolve_refine_target,
|
||||
)
|
||||
from .refine_window import GroupTemporalWindow
|
||||
|
||||
|
||||
class TargetedRefineExecutionUnavailable(RefineTargetError):
|
||||
"""Raised before Sampling when a future target mode is not implemented."""
|
||||
|
||||
|
||||
def _validate_ranges(
|
||||
ranges: tuple[tuple[int, int], ...],
|
||||
length: int,
|
||||
*,
|
||||
name: str,
|
||||
) -> tuple[tuple[int, int], ...]:
|
||||
if not ranges:
|
||||
raise RefineSamplingRuntimeError(f"{name} temporal window is empty")
|
||||
previous_stop = 0
|
||||
for start, stop in ranges:
|
||||
if start < previous_stop or stop <= start or stop > int(length):
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"{name} temporal window is outside its latent length"
|
||||
)
|
||||
previous_stop = stop
|
||||
return ranges
|
||||
|
||||
|
||||
def _window_masks(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
temporal_window: GroupTemporalWindow,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not isinstance(temporal_window, GroupTemporalWindow):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"temporal window must be a resolved GroupTemporalWindow"
|
||||
)
|
||||
video_ranges = _validate_ranges(
|
||||
temporal_window.video_slot_ranges,
|
||||
int(video.shape[2]),
|
||||
name="Video",
|
||||
)
|
||||
audio_ranges = _validate_ranges(
|
||||
temporal_window.audio_tick_ranges,
|
||||
int(audio.shape[-1]),
|
||||
name="Audio",
|
||||
)
|
||||
video_mask = torch.zeros(
|
||||
(
|
||||
int(video.shape[0]),
|
||||
1,
|
||||
int(video.shape[2]),
|
||||
int(video.shape[3]),
|
||||
int(video.shape[4]),
|
||||
),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.zeros(
|
||||
(
|
||||
int(audio.shape[0]),
|
||||
1,
|
||||
int(audio.shape[2]),
|
||||
int(audio.shape[3]),
|
||||
),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
for start, stop in video_ranges:
|
||||
video_mask[:, :, start:stop] = 1.0
|
||||
for start, stop in audio_ranges:
|
||||
audio_mask[..., start:stop] = 1.0
|
||||
return video_mask, audio_mask
|
||||
|
||||
|
||||
def _prepare_video_only_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
temporal_window: GroupTemporalWindow,
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build windowed Video noise/mask while locking Audio completely."""
|
||||
|
||||
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
||||
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"ComfyUI returned invalid Video Only noise geometry"
|
||||
)
|
||||
video_mask, _audio_window_mask = _window_masks(video, audio, temporal_window)
|
||||
audio_noise = torch.zeros_like(audio, device="cpu")
|
||||
audio_mask = torch.zeros(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
return (
|
||||
nested_builder((video_noise, audio_noise)),
|
||||
nested_builder((video_mask, audio_mask)),
|
||||
)
|
||||
|
||||
|
||||
def require_video_only_execution(
|
||||
refine_target: RefineTarget | str | None,
|
||||
) -> RefineTarget:
|
||||
target = resolve_refine_target(refine_target)
|
||||
if target.mode != MODE_VIDEO_ONLY:
|
||||
raise TargetedRefineExecutionUnavailable(
|
||||
f"refine target {target.mode!r} is contract-only in Phase A1"
|
||||
)
|
||||
return target
|
||||
|
||||
|
||||
def _prepare_audio_only_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build zero/locked Video and seeded-random/sampled Audio streams."""
|
||||
|
||||
if video.ndim != 5 or audio.ndim != 4:
|
||||
raise RefineSamplingRuntimeError(
|
||||
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
||||
)
|
||||
if int(video.shape[0]) != int(audio.shape[0]):
|
||||
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
|
||||
|
||||
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
|
||||
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"ComfyUI returned invalid Audio Only noise geometry"
|
||||
)
|
||||
video_noise = torch.zeros_like(video, device="cpu")
|
||||
if temporal_window is None:
|
||||
video_mask = torch.zeros(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.ones(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
else:
|
||||
_video_window_mask, audio_mask = _window_masks(
|
||||
video,
|
||||
audio,
|
||||
temporal_window,
|
||||
)
|
||||
video_mask = torch.zeros(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
noise = nested_builder((video_noise, audio_noise))
|
||||
noise_mask = nested_builder((video_mask, audio_mask))
|
||||
return noise, noise_mask
|
||||
|
||||
|
||||
def _prepare_video_audio_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build seeded-random noise and sampled masks for both AV streams."""
|
||||
|
||||
if video.ndim != 5 or audio.ndim != 4:
|
||||
raise RefineSamplingRuntimeError(
|
||||
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
||||
)
|
||||
if int(video.shape[0]) != int(audio.shape[0]):
|
||||
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
|
||||
|
||||
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
||||
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
|
||||
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"ComfyUI returned invalid Video + Audio video noise geometry"
|
||||
)
|
||||
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"ComfyUI returned invalid Video + Audio audio noise geometry"
|
||||
)
|
||||
if temporal_window is None:
|
||||
video_mask = torch.ones(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.ones(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
else:
|
||||
video_mask, audio_mask = _window_masks(video, audio, temporal_window)
|
||||
noise = nested_builder((video_noise, audio_noise))
|
||||
noise_mask = nested_builder((video_mask, audio_mask))
|
||||
return noise, noise_mask
|
||||
|
||||
|
||||
def _sample_with_target_noise_and_mask(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
enable_preview: bool,
|
||||
noise_mask_builder: Callable[..., tuple[Any, Any]],
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
restore_video_outside_window: bool = False,
|
||||
restore_audio_outside_window: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Run Core once after constructing one explicit AV target policy."""
|
||||
|
||||
if not isinstance(latent, dict) or "samples" not in latent:
|
||||
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
|
||||
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
|
||||
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
|
||||
|
||||
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
|
||||
_load_runtime_modules()
|
||||
)
|
||||
working = latent.copy()
|
||||
latent_image = comfy_sample.fix_empty_latent_channels(
|
||||
model,
|
||||
working["samples"],
|
||||
working.get("downscale_ratio_spacial"),
|
||||
working.get("downscale_ratio_temporal"),
|
||||
)
|
||||
working["samples"] = latent_image
|
||||
video, audio = extract_av_streams(working)
|
||||
noise, noise_mask = noise_mask_builder(
|
||||
video,
|
||||
audio,
|
||||
seed=int(seed),
|
||||
batch_inds=working.get("batch_index"),
|
||||
prepare_noise_fn=comfy_sample.prepare_noise,
|
||||
nested_builder=nested_tensor.NestedTensor,
|
||||
temporal_window=temporal_window,
|
||||
)
|
||||
guider = _make_basic_guider(model, conditioning)
|
||||
|
||||
x0_output: dict[str, Any] = {}
|
||||
callback = None
|
||||
if enable_preview:
|
||||
callback = preview.prepare_callback(
|
||||
model,
|
||||
int(sigmas.shape[-1]) - 1,
|
||||
x0_output,
|
||||
)
|
||||
samples = guider.sample(
|
||||
noise,
|
||||
latent_image,
|
||||
sampler,
|
||||
sigmas,
|
||||
denoise_mask=noise_mask,
|
||||
callback=callback,
|
||||
disable_pbar=not comfy_utils.PROGRESS_BAR_ENABLED,
|
||||
seed=int(seed),
|
||||
)
|
||||
samples = samples.to(model_management.intermediate_device())
|
||||
if temporal_window is not None:
|
||||
sampled_output = {"samples": samples}
|
||||
sampled_video, sampled_audio = extract_av_streams(sampled_output)
|
||||
if restore_video_outside_window:
|
||||
sampled_video = _restore_outside_temporal_ranges(
|
||||
sampled_video,
|
||||
video,
|
||||
temporal_window.video_slot_ranges,
|
||||
time_dim=2,
|
||||
name="Video",
|
||||
)
|
||||
if restore_audio_outside_window:
|
||||
sampled_audio = _restore_outside_temporal_ranges(
|
||||
sampled_audio,
|
||||
audio,
|
||||
temporal_window.audio_tick_ranges,
|
||||
time_dim=-1,
|
||||
name="Audio",
|
||||
)
|
||||
samples = nested_tensor.NestedTensor((sampled_video, sampled_audio))
|
||||
|
||||
output = working.copy()
|
||||
output.pop("downscale_ratio_spacial", None)
|
||||
output.pop("downscale_ratio_temporal", None)
|
||||
output["samples"] = samples
|
||||
extract_av_streams(output)
|
||||
return output
|
||||
|
||||
|
||||
def _restore_outside_temporal_ranges(
|
||||
sampled: torch.Tensor,
|
||||
source: torch.Tensor,
|
||||
ranges: tuple[tuple[int, int], ...],
|
||||
*,
|
||||
time_dim: int,
|
||||
name: str,
|
||||
) -> torch.Tensor:
|
||||
"""Restore every inactive temporal position from the input bit-exactly."""
|
||||
|
||||
if not torch.is_tensor(sampled) or not torch.is_tensor(source):
|
||||
raise RefineSamplingRuntimeError(f"{name} temporal restore requires Tensors")
|
||||
if tuple(sampled.shape) != tuple(source.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"{name} temporal restore input/output shapes differ"
|
||||
)
|
||||
if sampled.dtype != source.dtype:
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"{name} temporal restore input/output dtypes differ"
|
||||
)
|
||||
dim = int(time_dim)
|
||||
if dim < 0:
|
||||
dim += sampled.ndim
|
||||
if dim < 0 or dim >= sampled.ndim:
|
||||
raise RefineSamplingRuntimeError(f"{name} temporal restore dimension is invalid")
|
||||
validated = _validate_ranges(ranges, int(sampled.shape[dim]), name=name)
|
||||
restored = sampled.clone()
|
||||
original = source.to(device=restored.device)
|
||||
cursor = 0
|
||||
for start, stop in validated:
|
||||
if start > cursor:
|
||||
restored.narrow(dim, cursor, start - cursor).copy_(
|
||||
original.narrow(dim, cursor, start - cursor)
|
||||
)
|
||||
cursor = stop
|
||||
if cursor < int(restored.shape[dim]):
|
||||
restored.narrow(dim, cursor, int(restored.shape[dim]) - cursor).copy_(
|
||||
original.narrow(dim, cursor, int(restored.shape[dim]) - cursor)
|
||||
)
|
||||
return restored
|
||||
|
||||
|
||||
def sample_video_only_window_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
temporal_window: GroupTemporalWindow,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Refine only selected Video slots and restore all other AV positions."""
|
||||
|
||||
return _sample_with_target_noise_and_mask(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
noise_mask_builder=_prepare_video_only_noise_and_mask,
|
||||
temporal_window=temporal_window,
|
||||
restore_video_outside_window=True,
|
||||
restore_audio_outside_window=True,
|
||||
)
|
||||
|
||||
|
||||
def sample_audio_only_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Sample Audio while Core carries Video with zero noise and mask zero."""
|
||||
|
||||
return _sample_with_target_noise_and_mask(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
noise_mask_builder=_prepare_audio_only_noise_and_mask,
|
||||
temporal_window=temporal_window,
|
||||
restore_video_outside_window=temporal_window is not None,
|
||||
restore_audio_outside_window=temporal_window is not None,
|
||||
)
|
||||
|
||||
|
||||
def sample_video_audio_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Sample and adopt both Video and Audio using mask one for each stream."""
|
||||
|
||||
return _sample_with_target_noise_and_mask(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
noise_mask_builder=_prepare_video_audio_noise_and_mask,
|
||||
temporal_window=temporal_window,
|
||||
restore_video_outside_window=temporal_window is not None,
|
||||
restore_audio_outside_window=temporal_window is not None,
|
||||
)
|
||||
|
||||
|
||||
def sample_targeted_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
refine_target: RefineTarget | str | None = MODE_VIDEO_ONLY,
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
enable_preview: bool = True,
|
||||
legacy_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
||||
video_only_window_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
||||
audio_only_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
||||
video_audio_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Dispatch an implemented target without changing its stream policy."""
|
||||
|
||||
target = resolve_refine_target(refine_target)
|
||||
if target.mode == MODE_VIDEO_ONLY and temporal_window is not None:
|
||||
sample_fn = video_only_window_sample_fn or sample_video_only_window_refine_chunk
|
||||
elif target.mode == MODE_VIDEO_ONLY:
|
||||
sample_fn = legacy_sample_fn or sample_refine_chunk
|
||||
elif target.mode == MODE_AUDIO_ONLY:
|
||||
sample_fn = audio_only_sample_fn or sample_audio_only_refine_chunk
|
||||
elif target.mode == MODE_VIDEO_AUDIO:
|
||||
sample_fn = video_audio_sample_fn or sample_video_audio_refine_chunk
|
||||
else: # pragma: no cover - resolve_refine_target validates the mode.
|
||||
raise TargetedRefineExecutionUnavailable(
|
||||
f"refine target {target.mode!r} is unavailable"
|
||||
)
|
||||
arguments = dict(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
)
|
||||
if temporal_window is not None:
|
||||
arguments["temporal_window"] = temporal_window
|
||||
return sample_fn(**arguments)
|
||||
Reference in New Issue
Block a user