"""Sequential pixel/VAE resize for Continuum physical video groups.""" from __future__ import annotations from collections.abc import Callable, Mapping, Sequence from typing import Any import torch from .plan import SECOND_PASS_CONTRACT_VERSION PIXEL_UPSCALE_METHODS = ( "nearest-exact", "bilinear", "area", "bicubic", "bislerp", "lanczos", ) _PIXEL_RESIZE_FRAME_BATCH = 8 class PixelVaeResizeError(ValueError): """Raised when a pixel/VAE resize would violate the physical-group contract.""" def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]: if not isinstance(assembly_plan, Mapping): raise PixelVaeResizeError("assembly_plan must be a mapping") contract = assembly_plan.get("second_pass_contract") if not isinstance(contract, Mapping): raise PixelVaeResizeError("assembly_plan has no second_pass_contract") if contract.get("version") != SECOND_PASS_CONTRACT_VERSION: raise PixelVaeResizeError("unsupported second_pass_contract version") groups = contract.get("physical_groups") if ( not isinstance(groups, list) or not groups or not all(isinstance(group, Mapping) for group in groups) ): raise PixelVaeResizeError( "second_pass_contract.physical_groups must be non-empty" ) return groups def resolve_pixel_latent_scale( assembly_plan: Mapping[str, Any], ) -> tuple[int, int]: """Return plan-derived pixel-per-latent ratios as ``(width, height)``.""" groups = _contract_groups(assembly_plan) scale: tuple[int, int] | None = None for position, group in enumerate(groups): if int(group.get("group_id", -1)) != position: raise PixelVaeResizeError("physical group order or group_id changed") source_width = int(group.get("source_width", -1)) source_height = int(group.get("source_height", -1)) source_latent_width = int(group.get("source_latent_w", -1)) source_latent_height = int(group.get("source_latent_h", -1)) if min( source_width, source_height, source_latent_width, source_latent_height, ) < 1: raise PixelVaeResizeError("source pixel/latent geometry is invalid") if ( source_width % source_latent_width or source_height % source_latent_height ): raise PixelVaeResizeError( "source pixel/latent geometry is not integral" ) group_scale = ( source_width // source_latent_width, source_height // source_latent_height, ) if scale is None: scale = group_scale elif scale != group_scale: raise PixelVaeResizeError( "source pixel/latent scale differs between physical groups" ) assert scale is not None return scale def _latent_samples( latent: Any, *, position: int, group: Mapping[str, Any], ) -> torch.Tensor: if not isinstance(latent, Mapping) or "samples" not in latent: raise PixelVaeResizeError( f"video_latents[{position}] must be a LATENT mapping with samples" ) samples = latent["samples"] if not isinstance(samples, torch.Tensor) or samples.ndim != 5: raise PixelVaeResizeError("video latent must have shape [B,C,T,H,W]") batch, channels, temporal, height, width = ( int(value) for value in samples.shape ) expected = ( int(group.get("source_batch", -1)), int(group.get("latent_channels", -1)), int(group.get("source_latent_t", -1)), int(group.get("source_latent_h", -1)), int(group.get("source_latent_w", -1)), ) if (batch, channels, temporal, height, width) != expected: raise PixelVaeResizeError( "video latent shape or physical group order changed" ) if channels != 24: raise PixelVaeResizeError("MiniMax H3 video latent must have 24 channels") if not bool(torch.isfinite(samples).all()): raise PixelVaeResizeError("input video latent contains NaN or Inf") return samples def _resize_one_group( samples: torch.Tensor, *, video_vae: Any, source_width: int, source_height: int, target_width: int, target_height: int, target_latent_width: int, target_latent_height: int, upscale_method: str, upscale_fn: Callable[..., Any], ) -> torch.Tensor: """Decode, resize, and encode one group so its pixel buffer dies per call.""" decoded_pixels = video_vae.decode(samples) if not isinstance(decoded_pixels, torch.Tensor) or decoded_pixels.ndim != 5: raise PixelVaeResizeError( "Video VAE decode must return pixels with shape [B,F,H,W,C]" ) batch, frames, pixel_height, pixel_width, channels = ( int(value) for value in decoded_pixels.shape ) if batch != int(samples.shape[0]): raise PixelVaeResizeError("Video VAE decode changed the batch size") if frames < 1 or channels != 3: raise PixelVaeResizeError( "Video VAE decode returned invalid frame or channel geometry" ) if (pixel_width, pixel_height) != (source_width, source_height): raise PixelVaeResizeError( "Video VAE decode geometry differs from the assembly plan" ) for frame_start in range(0, frames, _PIXEL_RESIZE_FRAME_BATCH): if not bool( torch.isfinite( decoded_pixels[ :, frame_start : frame_start + _PIXEL_RESIZE_FRAME_BATCH, ] ).all() ): raise PixelVaeResizeError("Video VAE decode contains NaN or Inf") # Pixel resizing is intentionally CPU-owned. Core's Lanczos implementation # is CPU/PIL-based already, and this also avoids retaining a full video-sized # resize workspace in VRAM under --gpu-only configurations. pixels = decoded_pixels.detach().to("cpu").contiguous() del decoded_pixels # Core common_upscale is a 4D image boundary. Make B*F explicit so video # frame order is unambiguous. Resize bounded frame batches into one CPU # [B,F,H,W,C] buffer required by the temporal H3 VAE encoder. source_flat = pixels.reshape( batch * frames, pixel_height, pixel_width, channels ) resized_pixels = torch.empty( (batch, frames, target_height, target_width, channels), dtype=pixels.dtype, device="cpu", ) target_flat = resized_pixels.reshape( batch * frames, target_height, target_width, channels ) for frame_start in range( 0, batch * frames, _PIXEL_RESIZE_FRAME_BATCH, ): frame_stop = min( frame_start + _PIXEL_RESIZE_FRAME_BATCH, batch * frames, ) pixel_nchw = ( source_flat[frame_start:frame_stop] .movedim(-1, 1) .contiguous() ) resized_nchw = upscale_fn( pixel_nchw, int(target_width), int(target_height), upscale_method, "disabled", ) expected_resize_shape = ( frame_stop - frame_start, channels, target_height, target_width, ) if not isinstance(resized_nchw, torch.Tensor) or tuple( resized_nchw.shape ) != expected_resize_shape: raise PixelVaeResizeError("Core pixel resize returned invalid geometry") resized_nchw = resized_nchw.detach().to("cpu") if not bool(torch.isfinite(resized_nchw).all()): raise PixelVaeResizeError("Core pixel resize contains NaN or Inf") target_flat[frame_start:frame_stop].copy_( resized_nchw.movedim(1, -1) ) del pixel_nchw, resized_nchw del pixels, source_flat, target_flat # Match Core VAEEncode exactly: H3's VAE wrapper expects a 4D IMAGE batch # and promotes its frame axis to one temporal video batch. Passing the 5D # [B,F,H,W,C] tensor directly makes Core crop F as if it were spatial. # Encode each Continuum batch independently, then restore B explicitly. encoded_batches: list[torch.Tensor] = [] expected_batch_shape = ( 1, int(samples.shape[1]), int(samples.shape[2]), int(target_latent_height), int(target_latent_width), ) for batch_index in range(batch): encoded_batch = video_vae.encode(resized_pixels[batch_index]) if not isinstance(encoded_batch, torch.Tensor) or encoded_batch.ndim != 5: raise PixelVaeResizeError( "Video VAE encode must return a latent with shape [B,C,T,H,W]" ) if int(encoded_batch.shape[0]) != 1: raise PixelVaeResizeError( "Core Video VAE Encode must return one batch per IMAGE sequence" ) encoded_batch_shape = tuple(int(value) for value in encoded_batch.shape) if encoded_batch_shape != expected_batch_shape: raise PixelVaeResizeError( "Video VAE encode changed B/C/T or returned the wrong target H/W " f"for batch {batch_index}: expected {expected_batch_shape}, " f"got {encoded_batch_shape}" ) encoded_batches.append(encoded_batch) encoded = ( encoded_batches[0] if len(encoded_batches) == 1 else torch.cat(encoded_batches, dim=0) ) if not isinstance(encoded, torch.Tensor) or encoded.ndim != 5: raise PixelVaeResizeError( "Video VAE encode must return a latent with shape [B,C,T,H,W]" ) expected_shape = ( int(samples.shape[0]), int(samples.shape[1]), int(samples.shape[2]), int(target_latent_height), int(target_latent_width), ) if tuple(int(value) for value in encoded.shape) != expected_shape: raise PixelVaeResizeError( "Video VAE encode changed B/C/T or returned the wrong target H/W: " f"expected {expected_shape}, got {tuple(int(value) for value in encoded.shape)}" ) if not bool(torch.isfinite(encoded).all()): raise PixelVaeResizeError("Video VAE encode contains NaN or Inf") # Continuum stores accepted physical latents on CPU; do not retain one GPU # allocation per physical group while later groups are being converted. output = encoded.detach().to("cpu").contiguous() del resized_pixels, encoded_batches, encoded return output @torch.inference_mode() def resize_video_latents_via_vae( video_latents: Sequence[Mapping[str, Any]], assembly_plan: Mapping[str, Any], *, video_vae: Any, target_width: int, target_height: int, upscale_method: str = "lanczos", upscale_fn: Callable[..., Any] | None = None, ) -> list[dict[str, Any]]: """Resize physical groups sequentially through the connected H3 Video VAE. ``target_width`` and ``target_height`` are pixel dimensions. The expected encoded latent geometry is derived exclusively from assembly-plan metadata. VAE failures are intentionally not caught: Core owns its OOM/tiled fallback, and this path must never fall back to direct latent interpolation. """ groups = _contract_groups(assembly_plan) if ( not isinstance(video_latents, Sequence) or isinstance(video_latents, (str, bytes)) or len(video_latents) != len(groups) ): raise PixelVaeResizeError("video latent group count changed") if ( video_vae is None or not callable(getattr(video_vae, "decode", None)) or not callable(getattr(video_vae, "encode", None)) ): raise PixelVaeResizeError("a usable H3 Video VAE is required") if upscale_method not in PIXEL_UPSCALE_METHODS: raise PixelVaeResizeError( f"unsupported pixel upscale method {upscale_method!r}" ) target_width = int(target_width) target_height = int(target_height) if target_width < 1 or target_height < 1: raise PixelVaeResizeError("target pixel width and height must be positive") scale_width, scale_height = resolve_pixel_latent_scale(assembly_plan) if target_width % scale_width or target_height % scale_height: raise PixelVaeResizeError( "target pixel geometry is not aligned to the plan-derived latent scale" ) target_latent_width = target_width // scale_width target_latent_height = target_height // scale_height if upscale_fn is None: from comfy.utils import common_upscale upscale_fn = common_upscale outputs: list[dict[str, Any]] = [] for position, (latent, group) in enumerate( zip(video_latents, groups, strict=True) ): samples = _latent_samples(latent, position=position, group=group) source_width = int(group["source_width"]) source_height = int(group["source_height"]) resized_samples = _resize_one_group( samples, video_vae=video_vae, source_width=source_width, source_height=source_height, target_width=target_width, target_height=target_height, target_latent_width=target_latent_width, target_latent_height=target_latent_height, upscale_method=upscale_method, upscale_fn=upscale_fn, ) output = dict(latent) output["samples"] = resized_samples outputs.append(output) return outputs