Files
ComfyUI-H3-Continuum-Tiled/v2/h3_builder.py
T

363 lines
12 KiB
Python

"""Native MiniMax H3 conditioning and latent construction for V2.
All expensive image/VAE work is performed before the sampling phase. Prompt
conditioning is cached per unique prompt and image presentation.
"""
from __future__ import annotations
from collections import OrderedDict
import hashlib
from dataclasses import dataclass
from typing import Any, Callable
import torch
from ..continuation import clone_conditioning
from ..temporal import audio_latent_t, video_latent_t
_PROMPT_CACHE_ATTR = "_h3_continuum_prompt_conditioning_cache_v1"
_PROMPT_CACHE_VERSION = 1
_PROMPT_CACHE_MAX_ENTRIES = 16
@dataclass(slots=True)
class IdentityAssets:
first_image: torch.Tensor | None
first_latent: torch.Tensor | None
last_image: torch.Tensor | None
last_latent: torch.Tensor | None
identity_hash: str
@property
def first_frame_hash(self) -> str:
"""Version-neutral alias retained alongside the legacy identity hash."""
return self.identity_hash
@property
def last_frame_hash(self) -> str:
"""Fingerprint the resized final keyframe independently from identity."""
return _tensor_fingerprint(self.last_image)
def _tensor_fingerprint(tensor: torch.Tensor | None) -> str:
if tensor is None:
return "none"
sample = tensor.detach().to("cpu").contiguous()
digest = hashlib.sha256()
digest.update(str(tuple(int(value) for value in sample.shape)).encode("ascii"))
digest.update(str(sample.dtype).encode("ascii"))
digest.update(sample.view(torch.uint8).numpy().tobytes(order="C"))
return digest.hexdigest()
def _cache_event(callback: Callable[[str], None] | None, event: str) -> None:
if callback is None:
return
try:
callback(str(event))
except Exception:
pass
def _clone_metadata_containers(value: Any) -> Any:
"""Clone mutable metadata containers while sharing immutable tensor payloads."""
if isinstance(value, dict):
return {
key: _clone_metadata_containers(item)
for key, item in value.items()
}
if isinstance(value, list):
return [_clone_metadata_containers(item) for item in value]
if isinstance(value, tuple):
return tuple(_clone_metadata_containers(item) for item in value)
return value
def _clone_cached_conditioning(conditioning: list) -> list:
output = []
for index, item in enumerate(conditioning):
if not isinstance(item, (tuple, list)) or len(item) != 2:
raise ValueError(f"conditioning entry {index} is not [tensor, metadata]")
if not isinstance(item[1], dict):
raise ValueError(f"conditioning entry {index} metadata is not a dictionary")
output.append([item[0], _clone_metadata_containers(item[1])])
if not output:
raise ValueError("conditioning is empty")
return output
def _contains_cuda_tensor(value: Any) -> bool:
if torch.is_tensor(value):
return value.device.type == "cuda"
if isinstance(value, dict):
return any(_contains_cuda_tensor(item) for item in value.values())
if isinstance(value, (list, tuple)):
return any(_contains_cuda_tensor(item) for item in value)
return False
def _prompt_cache_clip_state(clip: Any) -> tuple[str, Any] | None:
"""Return a conservative state key, or None for CLIP modes we do not cache."""
patcher = getattr(clip, "patcher", None)
patches_uuid = getattr(patcher, "patches_uuid", None)
if patcher is None or patches_uuid is None:
return None
if bool(getattr(clip, "use_clip_schedule", False)):
return None
if getattr(clip, "apply_hooks_to_conds", None) is not None:
return None
if getattr(patcher, "forced_hooks", None) is not None:
return None
if getattr(patcher, "current_hooks", None) is not None:
return None
if bool(getattr(patcher, "hook_patches", {}) or {}):
return None
if bool(getattr(clip, "tokenizer_options", {}) or {}):
return None
return str(patches_uuid), getattr(clip, "layer_idx", None)
def encode_prompt_conditioning_cached(
clip: Any,
prompt: str,
*,
first_image: torch.Tensor | None,
last_image: torch.Tensor | None,
first_image_fingerprint: str,
last_image_fingerprint: str,
cache_enabled: bool,
cache_event: Callable[[str], None] | None = None,
) -> list:
"""V3.5-only bounded prompt cache; every unsafe state falls back to encode."""
if not bool(cache_enabled):
return encode_prompt_conditioning(
clip,
prompt,
first_image=first_image,
last_image=last_image,
)
state = _prompt_cache_clip_state(clip)
if state is None:
result = encode_prompt_conditioning(
clip,
prompt,
first_image=first_image,
last_image=last_image,
)
_cache_event(cache_event, "bypass_special_clip")
return result
key = (
_PROMPT_CACHE_VERSION,
str(prompt),
str(first_image_fingerprint),
str(last_image_fingerprint),
state,
)
try:
cache = getattr(clip, _PROMPT_CACHE_ATTR, None)
if cache is None:
cache = OrderedDict()
setattr(clip, _PROMPT_CACHE_ATTR, cache)
if not isinstance(cache, OrderedDict):
raise TypeError("unexpected prompt cache container")
if key in cache:
cached = cache.pop(key)
cache[key] = cached
result = _clone_cached_conditioning(cached)
_cache_event(cache_event, "hit")
return result
except Exception:
result = encode_prompt_conditioning(
clip,
prompt,
first_image=first_image,
last_image=last_image,
)
_cache_event(cache_event, "bypass_cache_error")
return result
result = encode_prompt_conditioning(
clip,
prompt,
first_image=first_image,
last_image=last_image,
)
if _contains_cuda_tensor(result):
_cache_event(cache_event, "bypass_cuda_output")
return result
try:
cache[key] = _clone_cached_conditioning(result)
cache.move_to_end(key)
while len(cache) > _PROMPT_CACHE_MAX_ENTRIES:
cache.popitem(last=False)
_cache_event(cache_event, "miss")
except Exception:
_cache_event(cache_event, "bypass_cache_error")
return result
def resize_h3_image(image: torch.Tensor, width: int, height: int, crop: str) -> torch.Tensor:
try:
import comfy.utils
except Exception as exc: # pragma: no cover - ComfyUI runtime only
raise RuntimeError(f"ComfyUI resize helper unavailable: {exc}") from exc
if not torch.is_tensor(image) or image.ndim != 4 or image.shape[-1] < 3:
raise ValueError("H3 image input must be [B,H,W,C] with at least three channels")
samples = image[:1, ..., :3].movedim(-1, 1)
resized = comfy.utils.common_upscale(samples, int(width), int(height), "lanczos", crop)
return resized.movedim(1, -1).contiguous()
def prepare_identity_assets(
video_vae: Any,
*,
width: int,
height: int,
first_frame: torch.Tensor | None,
last_frame: torch.Tensor | None,
encode_latents: bool = True,
) -> IdentityAssets:
first_image = first_latent = last_image = last_latent = None
if first_frame is not None:
first_image = resize_h3_image(first_frame, width, height, "disabled")
if encode_latents:
first_latent = video_vae.encode(first_image)
if last_frame is not None:
last_image = resize_h3_image(last_frame, width, height, "center")
if encode_latents:
last_latent = video_vae.encode(last_image)
return IdentityAssets(
first_image=first_image,
first_latent=first_latent,
last_image=last_image,
last_latent=last_latent,
identity_hash=_tensor_fingerprint(first_image),
)
def encode_prompt_conditioning(
clip: Any,
prompt: str,
*,
first_image: torch.Tensor | None,
last_image: torch.Tensor | None,
reference_audio_assets=None,
timeline_video_assets=None,
) -> list:
images = []
if first_image is not None:
images.append(first_image)
if last_image is not None:
images.append(last_image)
tokenize_options = {"images": images}
if reference_audio_assets is not None or timeline_video_assets is not None:
from ..reference_audio import (
reference_audio_asset_items,
reference_audio_item,
)
ref_items = [
*({"type": "image", "data": image} for image in images),
]
if timeline_video_assets is not None:
ref_items.append(dict(timeline_video_assets.item))
if reference_audio_assets is not None:
ref_items.extend(
reference_audio_item()
for _ in reference_audio_asset_items(reference_audio_assets)
)
tokenize_options = {
"minimax_ref_items": ref_items
}
tokens = clip.tokenize(str(prompt), **tokenize_options)
conditioning = clip.encode_from_tokens_scheduled(tokens)
if reference_audio_assets is None and timeline_video_assets is None:
return conditioning
refs = []
if timeline_video_assets is not None:
refs.append(dict(timeline_video_assets.block))
if reference_audio_assets is not None:
from ..reference_audio import reference_audio_asset_items, reference_audio_block
refs.extend(
dict(reference_audio_block(item))
for item in reference_audio_asset_items(reference_audio_assets)
)
return [
[tensor, {**dict(metadata), "minimax_refs": list(refs)}]
for tensor, metadata in conditioning
]
def attach_keyframes(
conditioning: list,
*,
frame_count: int,
first_latent: torch.Tensor | None,
last_latent: torch.Tensor | None,
) -> list:
output = clone_conditioning(conditioning)
keyframes: list[dict[str, Any]] = []
if first_latent is not None:
keyframes.append({"resolved_frame_index": 0, "latent": first_latent})
if last_latent is not None:
keyframes.append(
{"resolved_frame_index": int(frame_count) - 1, "latent": last_latent}
)
for _, metadata in output:
if keyframes:
metadata["minimax_keyframes"] = [dict(item) for item in keyframes]
metadata["minimax_frame_count"] = int(frame_count)
else:
metadata.pop("minimax_keyframes", None)
metadata["minimax_frame_count"] = int(frame_count)
return output
def empty_h3_latent(width: int, height: int, frame_count: int) -> dict[str, Any]:
try:
import comfy.model_management
import comfy.nested_tensor
except Exception as exc: # pragma: no cover - ComfyUI runtime only
raise RuntimeError(f"ComfyUI H3 latent helpers unavailable: {exc}") from exc
width = int(width)
height = int(height)
frame_count = int(frame_count)
if width <= 0 or height <= 0 or width % 32 or height % 32:
raise ValueError("H3 width and height must be positive multiples of 32")
device = comfy.model_management.intermediate_device()
video = torch.zeros(
(1, 24, video_latent_t(frame_count), height // 16, width // 16),
device=device,
dtype=torch.float32,
)
audio = torch.zeros(
(1, 32, 2, audio_latent_t(frame_count)),
device=device,
dtype=torch.float32,
)
return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}
def encode_identity_latents(video_vae: Any, assets: IdentityAssets) -> IdentityAssets:
"""Encode already-resized identity images without changing their fingerprint."""
first_latent = assets.first_latent
last_latent = assets.last_latent
if first_latent is None and assets.first_image is not None:
first_latent = video_vae.encode(assets.first_image)
if last_latent is None and assets.last_image is not None:
last_latent = video_vae.encode(assets.last_image)
return IdentityAssets(
first_image=assets.first_image,
first_latent=first_latent,
last_image=assets.last_image,
last_latent=last_latent,
identity_hash=assets.identity_hash,
)