git-subtree-dir: vendor/dumas_minimax_h3_refmod git-subtree-split: 517e8c419a8584fbf69fceb487bc39d83e6a6ec8
103 lines
5.3 KiB
Python
103 lines
5.3 KiB
Python
"""Present saved RefMod references to H3's native text/vision encoder."""
|
|
|
|
import math
|
|
import inspect
|
|
|
|
from comfy.text_encoders.minimax import MiniMaxH3Tokenizer
|
|
from comfy.ldm.minimax.vae import MiniMaxH3VideoVAE
|
|
|
|
|
|
def reference_map(mods):
|
|
counters = {"image": 0, "video": 0, "audio": 0}
|
|
labels = {"image": "Picture", "video": "Video", "audio": "Audio"}
|
|
mapping = []
|
|
for mod, strength in mods:
|
|
if not math.isfinite(strength) or not 0 <= strength <= 1:
|
|
raise ValueError("RefMod strength must be between 0 and 1.")
|
|
if strength == 0:
|
|
continue
|
|
counters[mod.kind] += 1
|
|
mapping.append(f"<{labels[mod.kind]} {counters[mod.kind]}> = {mod.name}")
|
|
return "\n".join(mapping) or "No active RefMods."
|
|
|
|
|
|
class MiniMaxH3RefModTextEncode:
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {"required": {
|
|
"clip": ("CLIP",),
|
|
"mods": ("H3_REF_MODS",),
|
|
"prompt": ("STRING", {"multiline": True, "dynamicPrompts": True}),
|
|
"reference_fps": ("FLOAT", {"default": 24.0, "min": 1.0, "max": 120.0,
|
|
"tooltip": "Playback FPS of the reconstructed video, used for Qwen timestamps. Saved compressed or stacked refs do not preserve original timing."}),
|
|
"max_total_tokens": ("INT", {"default": 0, "min": 0, "max": 2147483647}),
|
|
}, "optional": {
|
|
"vae": ("VAE", {"tooltip": "H3 video VAE: reconstructs visual refs for Qwen. Not needed for audio-only bundles."}),
|
|
}}
|
|
|
|
RETURN_TYPES = ("CONDITIONING", "STRING")
|
|
RETURN_NAMES = ("conditioning", "reference_map")
|
|
FUNCTION = "encode"
|
|
CATEGORY = "MiniMax-H3/mod"
|
|
DESCRIPTION = "Encode the prompt together with numbered RefMod references. Already attaches refs; connect directly to the sampler, without applying the same mods again."
|
|
|
|
def encode(self, clip, mods, prompt, reference_fps=24.0, max_total_tokens=0, vae=None):
|
|
from .nodes import _check_token_budget
|
|
|
|
native = isinstance(clip.tokenizer, MiniMaxH3Tokenizer)
|
|
# Projected encoders retain their smaller model's tokenizer, but expose
|
|
# H3 reference presentation on the CLIP wrapper itself (e.g. ClipProj).
|
|
if not native and "minimax_ref_items" not in inspect.signature(clip.tokenize).parameters:
|
|
raise ValueError("Connect an H3 CLIP or a projected CLIP supporting minimax_ref_items (such as current ClipProj).")
|
|
if not math.isfinite(reference_fps) or not 1 <= reference_fps <= 120:
|
|
raise ValueError("reference_fps must be between 1 and 120.")
|
|
# Keep zero-strength slots out of both the presentation and DiT payload.
|
|
mapping = reference_map(mods)
|
|
active = [(mod, strength) for mod, strength in mods if strength > 0]
|
|
_check_token_budget(active, max_total_tokens)
|
|
if any(mod.kind != "audio" for mod, _ in active) and vae is None:
|
|
raise ValueError("Connect the H3 video VAE to present visual RefMods to CLIP.")
|
|
if any(mod.kind != "audio" for mod, _ in active) and not isinstance(vae.first_stage_model, MiniMaxH3VideoVAE):
|
|
raise ValueError("Visual RefMods require the MiniMax H3 video VAE.")
|
|
|
|
items, blocks = [], []
|
|
for mod, strength in active:
|
|
block = mod.ref_block(strength)
|
|
block["refmod"] = True
|
|
kind = block["kind"]
|
|
item = {"type": kind}
|
|
if kind != "audio":
|
|
# Decode the same weakened latent that the DiT receives. ComfyUI
|
|
# owns device placement and its decode OOM/tiled fallback.
|
|
pixels = vae.decode(block["latent"])
|
|
# ComfyUI video VAEs return BTHWC; older wrappers may already
|
|
# expose THWC. Each RefMod is one video, never a batch of videos.
|
|
if pixels.ndim == 5 and pixels.shape[0] == 1:
|
|
pixels = pixels[0]
|
|
if pixels.ndim != 4 or pixels.shape[-1] != 3 or pixels.shape[0] < 1:
|
|
raise ValueError(f"H3 video VAE must decode to [1, frames, height, width, 3] or [frames, height, width, 3]; got {tuple(pixels.shape)}.")
|
|
if kind == "image":
|
|
item["data"] = pixels[:1].cpu().clone()
|
|
else:
|
|
# Native H3 presents video at 2 fps. Index by timestamps so
|
|
# non-integer frame rates do not accumulate rounding drift.
|
|
times = [i / 2 for i in range(math.ceil(pixels.shape[0] * 2 / reference_fps))]
|
|
indices = [min(round(t * reference_fps), pixels.shape[0] - 1) for t in times]
|
|
item["data"] = pixels[indices].cpu()
|
|
item["timestamps"] = times
|
|
del pixels
|
|
items.append(item)
|
|
blocks.append(block)
|
|
|
|
tokens = clip.tokenize(prompt, minimax_ref_items=items)
|
|
conditioning = clip.encode_from_tokens_scheduled(tokens)
|
|
out = []
|
|
for embedding, metadata in conditioning:
|
|
if "minimax_token_tags" not in metadata:
|
|
raise ValueError("The CLIP encoder did not return H3 minimax_token_tags. Use an H3-compatible encoder/projection.")
|
|
metadata = dict(metadata)
|
|
if blocks:
|
|
metadata["minimax_refs"] = list(blocks)
|
|
out.append([embedding, metadata])
|
|
return out, mapping
|