git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
159 lines
5.1 KiB
Python
159 lines
5.1 KiB
Python
"""V3.5 Second Pass sampling with the first-pass audio stream locked."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Callable
|
|
|
|
import torch
|
|
|
|
from ..state import extract_av_streams
|
|
|
|
|
|
class RefineSamplingRuntimeError(RuntimeError):
|
|
"""Raised when the ComfyUI sampling runtime cannot satisfy the refine contract."""
|
|
|
|
|
|
def _make_basic_guider(model: Any, conditioning: list):
|
|
try:
|
|
import comfy.samplers
|
|
except Exception as exc: # pragma: no cover
|
|
raise RefineSamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc
|
|
|
|
class _BasicGuider(comfy.samplers.CFGGuider):
|
|
def set_positive(self, positive):
|
|
self.inner_set_conds({"positive": positive})
|
|
|
|
guider = _BasicGuider(model)
|
|
guider.set_positive(conditioning)
|
|
return guider
|
|
|
|
|
|
def _load_runtime_modules():
|
|
try:
|
|
import comfy.model_management
|
|
import comfy.nested_tensor
|
|
import comfy.sample
|
|
import comfy.utils
|
|
import latent_preview
|
|
except Exception as exc: # pragma: no cover
|
|
raise RefineSamplingRuntimeError(
|
|
f"ComfyUI refine sampling runtime unavailable: {exc}"
|
|
) from exc
|
|
return (
|
|
comfy.model_management,
|
|
comfy.nested_tensor,
|
|
comfy.sample,
|
|
comfy.utils,
|
|
latent_preview,
|
|
)
|
|
|
|
|
|
def _prepare_refine_noise_and_mask(
|
|
video: torch.Tensor,
|
|
audio: torch.Tensor,
|
|
*,
|
|
seed: int,
|
|
batch_inds: Any = None,
|
|
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
|
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
|
) -> tuple[Any, Any]:
|
|
"""Build random video noise and a Core-expandable audio-lock mask."""
|
|
|
|
if video.ndim != 5 or audio.ndim != 4:
|
|
raise RefineSamplingRuntimeError(
|
|
"refine latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
|
)
|
|
if int(video.shape[0]) != int(audio.shape[0]):
|
|
raise RefineSamplingRuntimeError("refine video/audio batch sizes differ")
|
|
|
|
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
|
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
|
raise RefineSamplingRuntimeError("ComfyUI returned invalid video noise geometry")
|
|
|
|
# Core prepare_noise creates CPU noise. Keep both streams on that same handoff
|
|
# device so NestedTensor packing remains valid before Core moves it to the model.
|
|
audio_noise = torch.zeros_like(audio, device="cpu")
|
|
|
|
# These masks deliberately use the smallest shapes accepted by Core reshape_mask.
|
|
# Core expands them to the full stream geometry on the sampling device.
|
|
video_mask = torch.ones(
|
|
(int(video.shape[0]), 1, 1, 1, 1),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
audio_mask = torch.zeros(
|
|
(int(audio.shape[0]), 1, 1, 1),
|
|
dtype=torch.float32,
|
|
device="cpu",
|
|
)
|
|
noise = nested_builder((video_noise, audio_noise))
|
|
noise_mask = nested_builder((video_mask, audio_mask))
|
|
return noise, noise_mask
|
|
|
|
|
|
def sample_refine_chunk(
|
|
*,
|
|
model: Any,
|
|
conditioning: list,
|
|
latent: dict[str, Any],
|
|
sampler: Any,
|
|
sigmas: torch.Tensor,
|
|
seed: int,
|
|
enable_preview: bool = True,
|
|
) -> dict[str, Any]:
|
|
"""Refine video while Core carries the first-pass audio as a locked AV stream."""
|
|
|
|
if not isinstance(latent, dict) or "samples" not in latent:
|
|
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
|
|
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
|
|
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
|
|
|
|
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
|
|
_load_runtime_modules()
|
|
)
|
|
working = latent.copy()
|
|
latent_image = comfy_sample.fix_empty_latent_channels(
|
|
model,
|
|
working["samples"],
|
|
working.get("downscale_ratio_spacial"),
|
|
working.get("downscale_ratio_temporal"),
|
|
)
|
|
working["samples"] = latent_image
|
|
video, audio = extract_av_streams(working)
|
|
noise, noise_mask = _prepare_refine_noise_and_mask(
|
|
video,
|
|
audio,
|
|
seed=int(seed),
|
|
batch_inds=working.get("batch_index"),
|
|
prepare_noise_fn=comfy_sample.prepare_noise,
|
|
nested_builder=nested_tensor.NestedTensor,
|
|
)
|
|
guider = _make_basic_guider(model, conditioning)
|
|
|
|
x0_output: dict[str, Any] = {}
|
|
callback = None
|
|
if enable_preview:
|
|
callback = preview.prepare_callback(
|
|
model,
|
|
int(sigmas.shape[-1]) - 1,
|
|
x0_output,
|
|
)
|
|
disable_pbar = not comfy_utils.PROGRESS_BAR_ENABLED
|
|
samples = guider.sample(
|
|
noise,
|
|
latent_image,
|
|
sampler,
|
|
sigmas,
|
|
denoise_mask=noise_mask,
|
|
callback=callback,
|
|
disable_pbar=disable_pbar,
|
|
seed=int(seed),
|
|
)
|
|
samples = samples.to(model_management.intermediate_device())
|
|
output = working.copy()
|
|
output.pop("downscale_ratio_spacial", None)
|
|
output.pop("downscale_ratio_temporal", None)
|
|
output["samples"] = samples
|
|
extract_av_streams(output)
|
|
return output
|