"""Native MiniMax H3 conditioning and latent construction for V2. All expensive image/VAE work is performed before the sampling phase. Prompt conditioning is cached per unique prompt and image presentation. """ from __future__ import annotations from collections import OrderedDict import hashlib from dataclasses import dataclass from typing import Any, Callable import torch from ..continuation import clone_conditioning from ..temporal import audio_latent_t, video_latent_t _PROMPT_CACHE_ATTR = "_h3_continuum_prompt_conditioning_cache_v1" _PROMPT_CACHE_VERSION = 1 _PROMPT_CACHE_MAX_ENTRIES = 16 @dataclass(slots=True) class IdentityAssets: first_image: torch.Tensor | None first_latent: torch.Tensor | None last_image: torch.Tensor | None last_latent: torch.Tensor | None identity_hash: str @property def first_frame_hash(self) -> str: """Version-neutral alias retained alongside the legacy identity hash.""" return self.identity_hash @property def last_frame_hash(self) -> str: """Fingerprint the resized final keyframe independently from identity.""" return _tensor_fingerprint(self.last_image) def _tensor_fingerprint(tensor: torch.Tensor | None) -> str: if tensor is None: return "none" sample = tensor.detach().to("cpu").contiguous() digest = hashlib.sha256() digest.update(str(tuple(int(value) for value in sample.shape)).encode("ascii")) digest.update(str(sample.dtype).encode("ascii")) digest.update(sample.view(torch.uint8).numpy().tobytes(order="C")) return digest.hexdigest() def _cache_event(callback: Callable[[str], None] | None, event: str) -> None: if callback is None: return try: callback(str(event)) except Exception: pass def _clone_metadata_containers(value: Any) -> Any: """Clone mutable metadata containers while sharing immutable tensor payloads.""" if isinstance(value, dict): return { key: _clone_metadata_containers(item) for key, item in value.items() } if isinstance(value, list): return [_clone_metadata_containers(item) for item in value] if isinstance(value, tuple): return tuple(_clone_metadata_containers(item) for item in value) return value def _clone_cached_conditioning(conditioning: list) -> list: output = [] for index, item in enumerate(conditioning): if not isinstance(item, (tuple, list)) or len(item) != 2: raise ValueError(f"conditioning entry {index} is not [tensor, metadata]") if not isinstance(item[1], dict): raise ValueError(f"conditioning entry {index} metadata is not a dictionary") output.append([item[0], _clone_metadata_containers(item[1])]) if not output: raise ValueError("conditioning is empty") return output def _contains_cuda_tensor(value: Any) -> bool: if torch.is_tensor(value): return value.device.type == "cuda" if isinstance(value, dict): return any(_contains_cuda_tensor(item) for item in value.values()) if isinstance(value, (list, tuple)): return any(_contains_cuda_tensor(item) for item in value) return False def _prompt_cache_clip_state(clip: Any) -> tuple[str, Any] | None: """Return a conservative state key, or None for CLIP modes we do not cache.""" patcher = getattr(clip, "patcher", None) patches_uuid = getattr(patcher, "patches_uuid", None) if patcher is None or patches_uuid is None: return None if bool(getattr(clip, "use_clip_schedule", False)): return None if getattr(clip, "apply_hooks_to_conds", None) is not None: return None if getattr(patcher, "forced_hooks", None) is not None: return None if getattr(patcher, "current_hooks", None) is not None: return None if bool(getattr(patcher, "hook_patches", {}) or {}): return None if bool(getattr(clip, "tokenizer_options", {}) or {}): return None return str(patches_uuid), getattr(clip, "layer_idx", None) def encode_prompt_conditioning_cached( clip: Any, prompt: str, *, first_image: torch.Tensor | None, last_image: torch.Tensor | None, first_image_fingerprint: str, last_image_fingerprint: str, cache_enabled: bool, cache_event: Callable[[str], None] | None = None, ) -> list: """V3.5-only bounded prompt cache; every unsafe state falls back to encode.""" if not bool(cache_enabled): return encode_prompt_conditioning( clip, prompt, first_image=first_image, last_image=last_image, ) state = _prompt_cache_clip_state(clip) if state is None: result = encode_prompt_conditioning( clip, prompt, first_image=first_image, last_image=last_image, ) _cache_event(cache_event, "bypass_special_clip") return result key = ( _PROMPT_CACHE_VERSION, str(prompt), str(first_image_fingerprint), str(last_image_fingerprint), state, ) try: cache = getattr(clip, _PROMPT_CACHE_ATTR, None) if cache is None: cache = OrderedDict() setattr(clip, _PROMPT_CACHE_ATTR, cache) if not isinstance(cache, OrderedDict): raise TypeError("unexpected prompt cache container") if key in cache: cached = cache.pop(key) cache[key] = cached result = _clone_cached_conditioning(cached) _cache_event(cache_event, "hit") return result except Exception: result = encode_prompt_conditioning( clip, prompt, first_image=first_image, last_image=last_image, ) _cache_event(cache_event, "bypass_cache_error") return result result = encode_prompt_conditioning( clip, prompt, first_image=first_image, last_image=last_image, ) if _contains_cuda_tensor(result): _cache_event(cache_event, "bypass_cuda_output") return result try: cache[key] = _clone_cached_conditioning(result) cache.move_to_end(key) while len(cache) > _PROMPT_CACHE_MAX_ENTRIES: cache.popitem(last=False) _cache_event(cache_event, "miss") except Exception: _cache_event(cache_event, "bypass_cache_error") return result def resize_h3_image(image: torch.Tensor, width: int, height: int, crop: str) -> torch.Tensor: try: import comfy.utils except Exception as exc: # pragma: no cover - ComfyUI runtime only raise RuntimeError(f"ComfyUI resize helper unavailable: {exc}") from exc if not torch.is_tensor(image) or image.ndim != 4 or image.shape[-1] < 3: raise ValueError("H3 image input must be [B,H,W,C] with at least three channels") samples = image[:1, ..., :3].movedim(-1, 1) resized = comfy.utils.common_upscale(samples, int(width), int(height), "lanczos", crop) return resized.movedim(1, -1).contiguous() def prepare_identity_assets( video_vae: Any, *, width: int, height: int, first_frame: torch.Tensor | None, last_frame: torch.Tensor | None, encode_latents: bool = True, ) -> IdentityAssets: first_image = first_latent = last_image = last_latent = None if first_frame is not None: first_image = resize_h3_image(first_frame, width, height, "disabled") if encode_latents: first_latent = video_vae.encode(first_image) if last_frame is not None: last_image = resize_h3_image(last_frame, width, height, "center") if encode_latents: last_latent = video_vae.encode(last_image) return IdentityAssets( first_image=first_image, first_latent=first_latent, last_image=last_image, last_latent=last_latent, identity_hash=_tensor_fingerprint(first_image), ) def encode_prompt_conditioning( clip: Any, prompt: str, *, first_image: torch.Tensor | None, last_image: torch.Tensor | None, reference_audio_assets=None, timeline_video_assets=None, ) -> list: images = [] if first_image is not None: images.append(first_image) if last_image is not None: images.append(last_image) tokenize_options = {"images": images} if reference_audio_assets is not None or timeline_video_assets is not None: from ..reference_audio import ( reference_audio_asset_items, reference_audio_item, ) ref_items = [ *({"type": "image", "data": image} for image in images), ] if timeline_video_assets is not None: ref_items.append(dict(timeline_video_assets.item)) if reference_audio_assets is not None: ref_items.extend( reference_audio_item() for _ in reference_audio_asset_items(reference_audio_assets) ) tokenize_options = { "minimax_ref_items": ref_items } tokens = clip.tokenize(str(prompt), **tokenize_options) conditioning = clip.encode_from_tokens_scheduled(tokens) if reference_audio_assets is None and timeline_video_assets is None: return conditioning refs = [] if timeline_video_assets is not None: refs.append(dict(timeline_video_assets.block)) if reference_audio_assets is not None: from ..reference_audio import reference_audio_asset_items, reference_audio_block refs.extend( dict(reference_audio_block(item)) for item in reference_audio_asset_items(reference_audio_assets) ) return [ [tensor, {**dict(metadata), "minimax_refs": list(refs)}] for tensor, metadata in conditioning ] def attach_keyframes( conditioning: list, *, frame_count: int, first_latent: torch.Tensor | None, last_latent: torch.Tensor | None, ) -> list: output = clone_conditioning(conditioning) keyframes: list[dict[str, Any]] = [] if first_latent is not None: keyframes.append({"resolved_frame_index": 0, "latent": first_latent}) if last_latent is not None: keyframes.append( {"resolved_frame_index": int(frame_count) - 1, "latent": last_latent} ) for _, metadata in output: if keyframes: metadata["minimax_keyframes"] = [dict(item) for item in keyframes] metadata["minimax_frame_count"] = int(frame_count) else: metadata.pop("minimax_keyframes", None) metadata["minimax_frame_count"] = int(frame_count) return output def empty_h3_latent(width: int, height: int, frame_count: int) -> dict[str, Any]: try: import comfy.model_management import comfy.nested_tensor except Exception as exc: # pragma: no cover - ComfyUI runtime only raise RuntimeError(f"ComfyUI H3 latent helpers unavailable: {exc}") from exc width = int(width) height = int(height) frame_count = int(frame_count) if width <= 0 or height <= 0 or width % 32 or height % 32: raise ValueError("H3 width and height must be positive multiples of 32") device = comfy.model_management.intermediate_device() video = torch.zeros( (1, 24, video_latent_t(frame_count), height // 16, width // 16), device=device, dtype=torch.float32, ) audio = torch.zeros( (1, 32, 2, audio_latent_t(frame_count)), device=device, dtype=torch.float32, ) return {"samples": comfy.nested_tensor.NestedTensor((video, audio))} def encode_identity_latents(video_vae: Any, assets: IdentityAssets) -> IdentityAssets: """Encode already-resized identity images without changing their fingerprint.""" first_latent = assets.first_latent last_latent = assets.last_latent if first_latent is None and assets.first_image is not None: first_latent = video_vae.encode(assets.first_image) if last_latent is None and assets.last_image is not None: last_latent = video_vae.encode(assets.last_image) return IdentityAssets( first_image=assets.first_image, first_latent=first_latent, last_image=assets.last_image, last_latent=last_latent, identity_hash=assets.identity_hash, )