Merge commit 'bf8f76505fdd25ea99770f262eb9371509bf226d' as 'vendor/dumas_h3_continuum'
This commit is contained in:
+158
@@ -0,0 +1,158 @@
|
||||
"""V3.5 Second Pass sampling with the first-pass audio stream locked."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import extract_av_streams
|
||||
|
||||
|
||||
class RefineSamplingRuntimeError(RuntimeError):
|
||||
"""Raised when the ComfyUI sampling runtime cannot satisfy the refine contract."""
|
||||
|
||||
|
||||
def _make_basic_guider(model: Any, conditioning: list):
|
||||
try:
|
||||
import comfy.samplers
|
||||
except Exception as exc: # pragma: no cover
|
||||
raise RefineSamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc
|
||||
|
||||
class _BasicGuider(comfy.samplers.CFGGuider):
|
||||
def set_positive(self, positive):
|
||||
self.inner_set_conds({"positive": positive})
|
||||
|
||||
guider = _BasicGuider(model)
|
||||
guider.set_positive(conditioning)
|
||||
return guider
|
||||
|
||||
|
||||
def _load_runtime_modules():
|
||||
try:
|
||||
import comfy.model_management
|
||||
import comfy.nested_tensor
|
||||
import comfy.sample
|
||||
import comfy.utils
|
||||
import latent_preview
|
||||
except Exception as exc: # pragma: no cover
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"ComfyUI refine sampling runtime unavailable: {exc}"
|
||||
) from exc
|
||||
return (
|
||||
comfy.model_management,
|
||||
comfy.nested_tensor,
|
||||
comfy.sample,
|
||||
comfy.utils,
|
||||
latent_preview,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_refine_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build random video noise and a Core-expandable audio-lock mask."""
|
||||
|
||||
if video.ndim != 5 or audio.ndim != 4:
|
||||
raise RefineSamplingRuntimeError(
|
||||
"refine latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
||||
)
|
||||
if int(video.shape[0]) != int(audio.shape[0]):
|
||||
raise RefineSamplingRuntimeError("refine video/audio batch sizes differ")
|
||||
|
||||
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
||||
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
||||
raise RefineSamplingRuntimeError("ComfyUI returned invalid video noise geometry")
|
||||
|
||||
# Core prepare_noise creates CPU noise. Keep both streams on that same handoff
|
||||
# device so NestedTensor packing remains valid before Core moves it to the model.
|
||||
audio_noise = torch.zeros_like(audio, device="cpu")
|
||||
|
||||
# These masks deliberately use the smallest shapes accepted by Core reshape_mask.
|
||||
# Core expands them to the full stream geometry on the sampling device.
|
||||
video_mask = torch.ones(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.zeros(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
noise = nested_builder((video_noise, audio_noise))
|
||||
noise_mask = nested_builder((video_mask, audio_mask))
|
||||
return noise, noise_mask
|
||||
|
||||
|
||||
def sample_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Refine video while Core carries the first-pass audio as a locked AV stream."""
|
||||
|
||||
if not isinstance(latent, dict) or "samples" not in latent:
|
||||
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
|
||||
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
|
||||
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
|
||||
|
||||
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
|
||||
_load_runtime_modules()
|
||||
)
|
||||
working = latent.copy()
|
||||
latent_image = comfy_sample.fix_empty_latent_channels(
|
||||
model,
|
||||
working["samples"],
|
||||
working.get("downscale_ratio_spacial"),
|
||||
working.get("downscale_ratio_temporal"),
|
||||
)
|
||||
working["samples"] = latent_image
|
||||
video, audio = extract_av_streams(working)
|
||||
noise, noise_mask = _prepare_refine_noise_and_mask(
|
||||
video,
|
||||
audio,
|
||||
seed=int(seed),
|
||||
batch_inds=working.get("batch_index"),
|
||||
prepare_noise_fn=comfy_sample.prepare_noise,
|
||||
nested_builder=nested_tensor.NestedTensor,
|
||||
)
|
||||
guider = _make_basic_guider(model, conditioning)
|
||||
|
||||
x0_output: dict[str, Any] = {}
|
||||
callback = None
|
||||
if enable_preview:
|
||||
callback = preview.prepare_callback(
|
||||
model,
|
||||
int(sigmas.shape[-1]) - 1,
|
||||
x0_output,
|
||||
)
|
||||
disable_pbar = not comfy_utils.PROGRESS_BAR_ENABLED
|
||||
samples = guider.sample(
|
||||
noise,
|
||||
latent_image,
|
||||
sampler,
|
||||
sigmas,
|
||||
denoise_mask=noise_mask,
|
||||
callback=callback,
|
||||
disable_pbar=disable_pbar,
|
||||
seed=int(seed),
|
||||
)
|
||||
samples = samples.to(model_management.intermediate_device())
|
||||
output = working.copy()
|
||||
output.pop("downscale_ratio_spacial", None)
|
||||
output.pop("downscale_ratio_temporal", None)
|
||||
output["samples"] = samples
|
||||
extract_av_streams(output)
|
||||
return output
|
||||
Reference in New Issue
Block a user