Files
DumasNodes/prompt.py
T
chris.dumas 70d0672ee3 Squashed 'vendor/dumas_minimax_h3_refmod/' content from commit 517e8c4
git-subtree-dir: vendor/dumas_minimax_h3_refmod
git-subtree-split: 517e8c419a8584fbf69fceb487bc39d83e6a6ec8
2026-10-20 13:43:44 +00:00

103 lines
5.3 KiB
Python

"""Present saved RefMod references to H3's native text/vision encoder."""
import math
import inspect
from comfy.text_encoders.minimax import MiniMaxH3Tokenizer
from comfy.ldm.minimax.vae import MiniMaxH3VideoVAE
def reference_map(mods):
counters = {"image": 0, "video": 0, "audio": 0}
labels = {"image": "Picture", "video": "Video", "audio": "Audio"}
mapping = []
for mod, strength in mods:
if not math.isfinite(strength) or not 0 <= strength <= 1:
raise ValueError("RefMod strength must be between 0 and 1.")
if strength == 0:
continue
counters[mod.kind] += 1
mapping.append(f"<{labels[mod.kind]} {counters[mod.kind]}> = {mod.name}")
return "\n".join(mapping) or "No active RefMods."
class MiniMaxH3RefModTextEncode:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"clip": ("CLIP",),
"mods": ("H3_REF_MODS",),
"prompt": ("STRING", {"multiline": True, "dynamicPrompts": True}),
"reference_fps": ("FLOAT", {"default": 24.0, "min": 1.0, "max": 120.0,
"tooltip": "Playback FPS of the reconstructed video, used for Qwen timestamps. Saved compressed or stacked refs do not preserve original timing."}),
"max_total_tokens": ("INT", {"default": 0, "min": 0, "max": 2147483647}),
}, "optional": {
"vae": ("VAE", {"tooltip": "H3 video VAE: reconstructs visual refs for Qwen. Not needed for audio-only bundles."}),
}}
RETURN_TYPES = ("CONDITIONING", "STRING")
RETURN_NAMES = ("conditioning", "reference_map")
FUNCTION = "encode"
CATEGORY = "MiniMax-H3/mod"
DESCRIPTION = "Encode the prompt together with numbered RefMod references. Already attaches refs; connect directly to the sampler, without applying the same mods again."
def encode(self, clip, mods, prompt, reference_fps=24.0, max_total_tokens=0, vae=None):
from .nodes import _check_token_budget
native = isinstance(clip.tokenizer, MiniMaxH3Tokenizer)
# Projected encoders retain their smaller model's tokenizer, but expose
# H3 reference presentation on the CLIP wrapper itself (e.g. ClipProj).
if not native and "minimax_ref_items" not in inspect.signature(clip.tokenize).parameters:
raise ValueError("Connect an H3 CLIP or a projected CLIP supporting minimax_ref_items (such as current ClipProj).")
if not math.isfinite(reference_fps) or not 1 <= reference_fps <= 120:
raise ValueError("reference_fps must be between 1 and 120.")
# Keep zero-strength slots out of both the presentation and DiT payload.
mapping = reference_map(mods)
active = [(mod, strength) for mod, strength in mods if strength > 0]
_check_token_budget(active, max_total_tokens)
if any(mod.kind != "audio" for mod, _ in active) and vae is None:
raise ValueError("Connect the H3 video VAE to present visual RefMods to CLIP.")
if any(mod.kind != "audio" for mod, _ in active) and not isinstance(vae.first_stage_model, MiniMaxH3VideoVAE):
raise ValueError("Visual RefMods require the MiniMax H3 video VAE.")
items, blocks = [], []
for mod, strength in active:
block = mod.ref_block(strength)
block["refmod"] = True
kind = block["kind"]
item = {"type": kind}
if kind != "audio":
# Decode the same weakened latent that the DiT receives. ComfyUI
# owns device placement and its decode OOM/tiled fallback.
pixels = vae.decode(block["latent"])
# ComfyUI video VAEs return BTHWC; older wrappers may already
# expose THWC. Each RefMod is one video, never a batch of videos.
if pixels.ndim == 5 and pixels.shape[0] == 1:
pixels = pixels[0]
if pixels.ndim != 4 or pixels.shape[-1] != 3 or pixels.shape[0] < 1:
raise ValueError(f"H3 video VAE must decode to [1, frames, height, width, 3] or [frames, height, width, 3]; got {tuple(pixels.shape)}.")
if kind == "image":
item["data"] = pixels[:1].cpu().clone()
else:
# Native H3 presents video at 2 fps. Index by timestamps so
# non-integer frame rates do not accumulate rounding drift.
times = [i / 2 for i in range(math.ceil(pixels.shape[0] * 2 / reference_fps))]
indices = [min(round(t * reference_fps), pixels.shape[0] - 1) for t in times]
item["data"] = pixels[indices].cpu()
item["timestamps"] = times
del pixels
items.append(item)
blocks.append(block)
tokens = clip.tokenize(prompt, minimax_ref_items=items)
conditioning = clip.encode_from_tokens_scheduled(tokens)
out = []
for embedding, metadata in conditioning:
if "minimax_token_tags" not in metadata:
raise ValueError("The CLIP encoder did not return H3 minimax_token_tags. Use an H3-compatible encoder/projection.")
metadata = dict(metadata)
if blocks:
metadata["minimax_refs"] = list(blocks)
out.append([embedding, metadata])
return out, mapping