Files
ComfyUI-H3-Continuum-Tiled/v3/refine_sampling.py
T
2026-08-24 14:24:25 +09:00

159 lines
5.1 KiB
Python

"""V3.5 Second Pass sampling with the first-pass audio stream locked."""
from __future__ import annotations
from typing import Any, Callable
import torch
from ..state import extract_av_streams
class RefineSamplingRuntimeError(RuntimeError):
"""Raised when the ComfyUI sampling runtime cannot satisfy the refine contract."""
def _make_basic_guider(model: Any, conditioning: list):
try:
import comfy.samplers
except Exception as exc: # pragma: no cover
raise RefineSamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc
class _BasicGuider(comfy.samplers.CFGGuider):
def set_positive(self, positive):
self.inner_set_conds({"positive": positive})
guider = _BasicGuider(model)
guider.set_positive(conditioning)
return guider
def _load_runtime_modules():
try:
import comfy.model_management
import comfy.nested_tensor
import comfy.sample
import comfy.utils
import latent_preview
except Exception as exc: # pragma: no cover
raise RefineSamplingRuntimeError(
f"ComfyUI refine sampling runtime unavailable: {exc}"
) from exc
return (
comfy.model_management,
comfy.nested_tensor,
comfy.sample,
comfy.utils,
latent_preview,
)
def _prepare_refine_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
) -> tuple[Any, Any]:
"""Build random video noise and a Core-expandable audio-lock mask."""
if video.ndim != 5 or audio.ndim != 4:
raise RefineSamplingRuntimeError(
"refine latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
)
if int(video.shape[0]) != int(audio.shape[0]):
raise RefineSamplingRuntimeError("refine video/audio batch sizes differ")
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
raise RefineSamplingRuntimeError("ComfyUI returned invalid video noise geometry")
# Core prepare_noise creates CPU noise. Keep both streams on that same handoff
# device so NestedTensor packing remains valid before Core moves it to the model.
audio_noise = torch.zeros_like(audio, device="cpu")
# These masks deliberately use the smallest shapes accepted by Core reshape_mask.
# Core expands them to the full stream geometry on the sampling device.
video_mask = torch.ones(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.zeros(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
noise = nested_builder((video_noise, audio_noise))
noise_mask = nested_builder((video_mask, audio_mask))
return noise, noise_mask
def sample_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Refine video while Core carries the first-pass audio as a locked AV stream."""
if not isinstance(latent, dict) or "samples" not in latent:
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
_load_runtime_modules()
)
working = latent.copy()
latent_image = comfy_sample.fix_empty_latent_channels(
model,
working["samples"],
working.get("downscale_ratio_spacial"),
working.get("downscale_ratio_temporal"),
)
working["samples"] = latent_image
video, audio = extract_av_streams(working)
noise, noise_mask = _prepare_refine_noise_and_mask(
video,
audio,
seed=int(seed),
batch_inds=working.get("batch_index"),
prepare_noise_fn=comfy_sample.prepare_noise,
nested_builder=nested_tensor.NestedTensor,
)
guider = _make_basic_guider(model, conditioning)
x0_output: dict[str, Any] = {}
callback = None
if enable_preview:
callback = preview.prepare_callback(
model,
int(sigmas.shape[-1]) - 1,
x0_output,
)
disable_pbar = not comfy_utils.PROGRESS_BAR_ENABLED
samples = guider.sample(
noise,
latent_image,
sampler,
sigmas,
denoise_mask=noise_mask,
callback=callback,
disable_pbar=disable_pbar,
seed=int(seed),
)
samples = samples.to(model_management.intermediate_device())
output = working.copy()
output.pop("downscale_ratio_spacial", None)
output.pop("downscale_ratio_temporal", None)
output["samples"] = samples
extract_av_streams(output)
return output