Merge commit 'c9eb575e2fd25f74b15f1d7336af4932c7fe6338' as 'vendor/dumas_h3_refmod_picker'

This commit is contained in:
2026-10-20 13:43:48 +00:00
30 changed files with 5783 additions and 0 deletions
+183
View File
@@ -0,0 +1,183 @@
"""
H3RefModApplySimple — identity-oriented RefMod apply node with one strength dial
"""
from __future__ import annotations
import random
from dataclasses import replace
from typing import Dict, List, Optional
from comfy_api.latest import io
# reference retention presets (master strength multiplier on Apply)
RETENTION = {
"fully_preserved": 1.0,
"partially_preserved": 0.7,
"attribute_transfer": 0.4,
"weak_reference": 0.15,
}
def _filter_legacy_ref_block(block: Optional[Dict],
use_video: bool,
use_audio: bool) -> Optional[Dict]:
"""Apply media toggles to ref blocks produced by older RefMod APIs."""
if block is None or (not use_video and not use_audio):
return None
kind = str(block.get("kind") or "").strip().lower()
has_audio = block.get("audio_latent") is not None and int(block.get("ref_audio_t") or 0) > 0
if kind == "audio":
return block if use_audio else None
if not use_video:
if use_audio and has_audio:
return {
"kind": "audio",
"ref_audio_t": int(block.get("ref_audio_t") or 0),
"audio_latent": block.get("audio_latent"),
}
return None
if not use_audio and has_audio:
filtered = dict(block)
filtered["kind"] = "video"
filtered["ref_audio_t"] = 0
filtered["audio_latent"] = None
return filtered
return block
def _build_ref_block(mod, eff: float, curve, use_video: bool, use_audio: bool) -> Optional[Dict]:
try:
return mod.ref_block(eff, curve=curve, use_video=use_video, use_audio=use_audio)
except TypeError as exc:
message = str(exc)
if "unexpected keyword argument 'use_video'" not in message and "unexpected keyword argument 'use_audio'" not in message:
raise
legacy_block = mod.ref_block(eff, curve=curve)
return _filter_legacy_ref_block(legacy_block, use_video=use_video, use_audio=use_audio)
def _ref_blocks(mods, retention, curve=None, seed=-1,
use_video: bool = True, use_audio: bool = True) -> List[Dict]:
"""Ref blocks for a loader bundle, scaled by row strength x retention.
``retention`` is a master strength multiplier: a float 0-1 (1.0 =
fully_preserved, 0.7 = partially_preserved, 0.4 = attribute_transfer,
0.15 = weak_reference), or one of those preset names for legacy
workflows saved with the old combo widget.
``curve`` (optional) is a per-frame strength spec — a ``(direction,
shape, value)`` tuple, a legacy preset name, per-frame values, control
points (see ``core.curve_strengths``) — applied on top of the row
strength. A flat/no curve keeps today's behavior.
``seed`` (default -1 = off) enables ref scrambling: with 2+ refs in the
bundle, the order is shuffled and a random subset kept, so a different
ref leads each run instead of the same one always "popping". Same seed
-> same scramble; connect/randomize the seed for per-run variation.
"""
if isinstance(retention, str):
factor = RETENTION.get(retention, 1.0)
else:
factor = float(retention)
items = [] if mods is None else list(mods)
if int(seed) >= 0 and len(items) > 1:
rng = random.Random(int(seed))
rng.shuffle(items)
keep = rng.randint(max(1, len(items) // 2), len(items))
items = items[:keep]
print(f"[H3RefModApply] scramble seed={int(seed)}: "
f"{len(mods)} refs -> kept {len(items)} (order shuffled)")
blocks = []
debug_rows = []
for mod, strength in items:
eff = min(1.0, max(0.0, strength * factor))
block = _build_ref_block(mod, eff, curve=curve, use_video=use_video, use_audio=use_audio)
if block is not None:
blocks.append(block)
debug_rows.append(f"{mod.name}@{eff:.2f}")
else:
debug_rows.append(f"{mod.name}@{eff:.2f} (skipped)")
if debug_rows:
print("[H3RefModApply] effective refs: " + ", ".join(debug_rows))
return blocks
class H3RefModApplySimple(io.ComfyNode):
"""Identity-oriented RefMod apply node with one strength dial."""
@classmethod
def define_schema(cls):
template = io.MatchType.Template(
"cond",
allowed_types=[io.Custom("MINIMAX_H3_COND"), io.Conditioning])
return io.Schema(
node_id="H3RefModApplySimple",
display_name="Apply H3 RefMod Simple",
description=(
"Apply one or more RefMods to a MiniMax H3 conditioning with a single "
"Strength control. This identity-oriented version uses a flat full-length "
"reference curve under the hood."
),
category="H3RefModPicker",
inputs=[
io.MatchType.Input("conditioning", template=template,
tooltip="MINIMAX_H3_COND (ComfyUI-MiniMaxH3 pack) or CONDITIONING "
"(core MiniMaxH3ReferenceToVideo)."),
io.Custom("H3_REF_MODS").Input("mods",
optional=True,
tooltip="Bundle of RefMods to inject. Leave unconnected to bypass unchanged."),
io.Boolean.Input("use_video", default=True,
tooltip="Apply the visual latent from each RefMod. Turn this off to use only embedded audio."),
io.Boolean.Input("use_audio", default=True,
tooltip="Apply the embedded audio latent from each RefMod when present. Turn this off for visual-only application."),
io.Float.Input("strength", default=1.0, min=0.0, max=1.0, step=0.01,
tooltip="Master reference strength. For identity work this is the main "
"dial: higher = tighter identity lock, lower = more freedom but "
"more drift."),
],
outputs=[
io.MatchType.Output(template=template, display_name="conditioning",
tooltip="The conditioning with the ref blocks injected, same type as the input."),
],
)
@classmethod
def execute(cls, conditioning, mods, use_video=True, use_audio=True, strength=1.0):
if mods is None:
print("[H3RefModApplySimple] no mods connected — bypassing conditioning unchanged")
return io.NodeOutput(conditioning)
if not use_video and not use_audio:
print("[H3RefModApplySimple] use_video=False and use_audio=False — bypassing conditioning unchanged")
return io.NodeOutput(conditioning)
blocks = _ref_blocks(mods, strength, ("constant", "linear", 1.0), seed=-1,
use_video=bool(use_video), use_audio=bool(use_audio))
if isinstance(conditioning, list):
out = []
for t in conditioning:
d = dict(t[1])
d["minimax_refs"] = list(d.get("minimax_refs", [])) + blocks
out.append([t[0], d])
print(f"[H3RefModApplySimple] strength={strength} "
f"({len(blocks)} ref block(s) injected)")
else:
out = replace(conditioning, refs=list(conditioning.refs) + blocks)
print(f"[H3RefModApplySimple] strength={strength} "
f"({len(blocks)} ref block(s) injected, {len(out.refs)} total)")
return io.NodeOutput(out)
NODE_CLASS_MAPPINGS = {
"H3RefModApplySimple": H3RefModApplySimple,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3RefModApplySimple": "Apply H3 RefMod Simple",
}
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
+186
View File
@@ -0,0 +1,186 @@
"""
H3RefModApplyAxis — choose between two picker bundles with separate video/audio signed values
Think of it as creating a Slider from two RefMods
"""
from __future__ import annotations
import os
from typing import List, Optional, Tuple
from ..py.refmod_common import (
_prompt_hint,
)
from ..py.refmod_core import (
H3RefMod,
)
_PACK_DIR = os.path.dirname(os.path.abspath(__file__))
LEGACY_MODS_DIR = os.path.join(_PACK_DIR, "mods") # pre-models/refmods storage, still read
# reference retention presets (master strength multiplier on Apply)
RETENTION = {
"fully_preserved": 1.0,
"partially_preserved": 0.7,
"attribute_transfer": 0.4,
"weak_reference": 0.15,
}
def _info_lines(mod):
opt = mod.optimize_steps
if mod.mode == "encode":
opt = f"n/a ({mod.optimize_steps} — encode mode stores the actual encode)"
return [
"=" * 52,
f" MiniMax H3 RefMod: {mod.name}",
f" {'concept_type':<18} {mod.concept_type}",
f" {'mode':<18} {mod.mode}",
f" {'kind':<18} {mod.kind}",
f" {'latent':<18} {tuple(mod.latent.shape)}",
f" {'tokens injected':<18} {mod.token_count}",
f" {'source':<18} {mod.source} ({mod.source_shape})",
f" {'pool':<18} {mod.pool}",
f" {'identity':<18} {opt}",
f" {'tags':<18} {', '.join(mod.tags) if mod.tags else '-'}",
f" {'description':<18} {mod.description or '-'}",
"=" * 52,
]
def _mod_group_key(mod: H3RefMod) -> str:
name = str(getattr(mod, "name", "") or "")
lower_name = name.lower()
for suffix in ("_Video", "_Audio"):
if lower_name.endswith(suffix.lower()):
return name[:-len(suffix)]
return name
def _split_trailing_selection(rows, label: str) -> Tuple[List[Tuple[H3RefMod, float]], Optional[Tuple[H3RefMod, float]], Optional[Tuple[H3RefMod, float]]]:
"""Return the last logical RefMod selection from a bundle."""
items = [] if rows is None else list(rows)
if not items:
return [], None, None
last_key = _mod_group_key(items[-1][0])
start = len(items) - 1
while start > 0 and _mod_group_key(items[start - 1][0]) == last_key:
start -= 1
group = items[start:]
if start > 0:
print(f"[H3RefModApplyAxis] warning: {label} input contains {start} earlier row(s); using only the last RefMod selection '{last_key}'.")
video_row = None
audio_row = None
for row in group:
mod, _strength = row
if mod.kind == "audio":
audio_row = row
else:
video_row = row
return group, video_row, audio_row
class H3RefModApplyAxis:
"""Choose between two picker bundles with separate video/audio signed values."""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"mods_a": ("H3_REF_MODS", {
"tooltip": "RefMod bundle from one Visual RefMod Picker. If older chained selections are present too, only the last logical RefMod selection is used.",
}),
"mods_b": ("H3_REF_MODS", {
"tooltip": "RefMod bundle from another Visual RefMod Picker. If older chained selections are present too, only the last logical RefMod selection is used.",
}),
"video_value": ("FLOAT", {
"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01,
"display": "number",
"tooltip": "Signed video selector: negative uses A video, positive uses B video. Magnitude becomes the output video strength. 0 skips video.",
}),
"audio_value": ("FLOAT", {
"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01,
"display": "number",
"tooltip": "Signed audio selector: negative uses A audio, positive uses B audio. If only one side has audio, that audio is used whenever this is non-zero.",
}),
"show_info": ("BOOLEAN", {
"default": False,
"tooltip": "Print the selected output RefMods and strengths to the console.",
}),
},
}
RETURN_TYPES = ("H3_REF_MODS", "STRING")
RETURN_NAMES = ("mods", "prompt_hint")
FUNCTION = "select"
CATEGORY = "H3RefModPicker"
def select(self, mods_a, mods_b, video_value=0.0, audio_value=0.0, show_info=False):
_group_a, video_a, audio_a = _split_trailing_selection(mods_a, "A")
_group_b, video_b, audio_b = _split_trailing_selection(mods_b, "B")
loads: List[Tuple[H3RefMod, float]] = []
def pick_signed(value: float, row_a, row_b, kind: str):
mag = min(1.0, abs(float(value)))
if mag < 1e-6:
return None
chosen = row_b if value > 0 else row_a
chosen_side = "B" if value > 0 else "A"
fallback = row_a if value > 0 else row_b
fallback_side = "A" if value > 0 else "B"
if chosen is None:
if fallback is not None:
print(f"[H3RefModApplyAxis] warning: {kind} {chosen_side} is missing; using {fallback_side} {kind} at {mag:.2f}.")
return fallback[0], mag
print(f"[H3RefModApplyAxis] warning: no {kind} RefMod available on either side.")
return None
return chosen[0], mag
video_pick = pick_signed(video_value, video_a, video_b, "video")
if video_pick is not None:
loads.append(video_pick)
if audio_a is not None and audio_b is not None:
audio_pick = pick_signed(audio_value, audio_a, audio_b, "audio")
elif audio_a is not None or audio_b is not None:
mag = min(1.0, abs(float(audio_value)))
if mag < 1e-6:
audio_pick = None
else:
only = audio_a or audio_b
side = "A" if audio_a is not None else "B"
print(f"[H3RefModApplyAxis] audio exists only on {side}; using that audio at {mag:.2f}.")
audio_pick = (only[0], mag)
else:
audio_pick = None
if audio_pick is not None:
loads.append(audio_pick)
if loads:
print("[H3RefModApplyAxis] " + ", ".join(
f"{mod.name}@{strength:.2f}" for mod, strength in loads
))
else:
print("[H3RefModApplyAxis] no output refs selected")
if show_info:
for mod, strength in loads:
print("\n".join(_info_lines(mod)))
print(f" {'strength':<18} {strength:.2f}")
hint = _prompt_hint(loads)
if hint:
print(f"[H3RefModApplyAxis] prompt_hint: {hint}")
return (loads, hint)
# ═══════════════════════════════════════════════════════════════════════════
# Registration
# ═══════════════════════════════════════════════════════════════════════════
NODE_CLASS_MAPPINGS = {
"H3RefModApplyAxis": H3RefModApplyAxis,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3RefModApplyAxis": "Apply H3 RefMod Axis",
}
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
File diff suppressed because it is too large Load Diff
+116
View File
@@ -0,0 +1,116 @@
"""
decode_refmod.py — H3RefModDecode node.
Decode a saved RefMod's latent back into images, so you can see what the mod
actually captured. Purely a debugging / inspection tool — it plays no part in
generation.
A RefMod stores a video-VAE latent [1, 24, T, h, w] (the reference). This node
runs it back through the H3 video VAE decoder and returns the frames as an IMAGE
batch [T, H, W, 3], so you can preview a mod, sanity-check an extraction, or
eyeball what a pooled (training-mode) mod actually kept vs an encode-mode one.
It decodes the full latent timeline exactly as stored in the RefMod; it does
not resample or guess a frame subset.
Note: a training-mode (pooled) mod decodes to a tiny, blurry grid — that's the
point of the mode, it only stores a concept thumbnail. An encode-mode mod
decodes to recognizable full-res frames. Audio (if the mod has it) is not
decoded here — this is the visual reference only.
"""
from __future__ import annotations
import torch
import torch.nn.functional as F
import comfy.utils
from comfy_api.latest import io
from .refmod_loader import _list_mod_names, _load_mod
def _reshape_decoded_frames(frames: torch.Tensor) -> torch.Tensor:
if frames.dim() == 5: # [B, T, H, W, C] -> [B*T, H, W, C]
frames = frames.reshape(-1, frames.shape[-3], frames.shape[-2], frames.shape[-1])
return frames
def _upscale_for_inspection(frames: torch.Tensor, min_edge: int, mode: str) -> torch.Tensor:
if min_edge <= 0 or frames.dim() != 4 or frames.shape[0] <= 0:
return frames
height, width = int(frames.shape[1]), int(frames.shape[2])
shortest = min(height, width)
if shortest <= 0 or shortest >= min_edge:
return frames
scale = float(min_edge) / float(shortest)
target_h = max(1, int(round(height * scale)))
target_w = max(1, int(round(width * scale)))
nchw = frames.movedim(-1, 1)
if mode == "training":
upscaled = F.interpolate(nchw, size=(target_h, target_w), mode="nearest")
else:
upscaled = comfy.utils.common_upscale(nchw, target_w, target_h, "lanczos", "disabled")
return upscaled.movedim(1, -1)
class H3RefModDecode(io.ComfyNode):
"""Decode a RefMod's latent back to images for inspection."""
@classmethod
def define_schema(cls):
return io.Schema(
node_id="H3RefModDecode",
display_name="Decode H3 RefMod",
category="H3RefMod",
description="Decode a saved RefMod's latent back into images so you can see "
"what it captured. Inspection only — not part of generation. "
"The full stored latent timeline is decoded as-is; no frame "
"sampling is applied. "
"An encode-mode mod decodes to recognizable frames; a "
"training-mode (pooled) mod decodes to a tiny blurry grid "
"(that's all it stores). Audio is not decoded.",
inputs=[
io.Combo.Input("mod", options=_list_mod_names(),
tooltip="The RefMod to decode (from models/refmods/)."),
io.Vae.Input("vae",
tooltip="The MiniMax H3 video VAE."),
io.Int.Input("inspect_min_edge", default=320, min=0, max=2048, step=32,
tooltip="Upscale small decoded previews so pooled/training mods are easier "
"to inspect. 0 disables inspection upscaling. Training-mode uses "
"nearest-neighbor to keep the stored cell structure visible."),
],
outputs=[
io.Image.Output("images", display_name="images",
tooltip="The decoded reference frames [T, H, W, 3]."),
],
)
@classmethod
def execute(cls, mod, vae, inspect_min_edge=320) -> io.NodeOutput:
m = _load_mod(mod)
if m.kind == "audio":
raise ValueError("H3RefModDecode only decodes visual RefMods; audio-only mods are not supported.")
z = m.latent # [1, 24, T, h, w]
total_t = z.shape[2]
with torch.no_grad():
frames = vae.decode(z)
# video latent decodes channel-last; frame count rides the batch dim, so a
# 5-dim result is reshaped like the core VAEDecode node does
frames = _reshape_decoded_frames(frames)
frames = frames.clamp(0.0, 1.0).float().cpu()
frames = _upscale_for_inspection(frames, int(inspect_min_edge), m.mode)
print(f"[H3RefModDecode] '{m.name}' ({m.mode}, {m.kind}, "
f"latent_t={total_t}, inspect_min_edge={inspect_min_edge}) -> {tuple(frames.shape)}")
return io.NodeOutput(frames)
NODE_CLASS_MAPPINGS = {
"H3RefModDecode": H3RefModDecode,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3RefModDecode": "Decode H3 RefMod",
}
+283
View File
@@ -0,0 +1,283 @@
"""
H3RefModLoader — load one RefMod and optionally append it to another
"""
from __future__ import annotations
from typing import Any, Dict, List, Tuple
import os
import folder_paths
import server
from ..py.refmod_core import H3RefMod, load_refmods_from_file, read_refmod_meta, refmod_capabilities
from ..py.refmod_common import (
_prompt_hint,
)
from ..py.refmod_common import (
refmods_dir,
refmods_dirs,
)
_MOD_CACHE: Dict[str, Tuple[H3RefMod, ...]] = {}
_MOD_CACHE_MAX = 24 # cap: never pin more mods in RAM than this (FIFO eviction)
_MOD_LIST_CACHE_KEY = None # (dirs, mtimes, sizes) signature of the last _list_mod_names() scan
_MOD_LIST_CACHE_VAL = None
_MOD_PATH_CACHE_KEY = None
_MOD_PATH_CACHE_VAL: Dict[str, str] = {}
_MOD_SKIP_DIRS = {"graph_presets", ".git", "__pycache__"}
_MAX_WEIGHT = 10.0
# Mod storage lives in ComfyUI's models/ tree (created on first run) and is
# registered as a first-class folder type so it shows up next to loras/unet.
try:
folder_paths.add_model_folder_path("refmods", refmods_dir())
except Exception:
pass
# ═══════════════════════════════════════════════════════════════════════════
# Mods folder helpers
# ═══════════════════════════════════════════════════════════════════════════
def _iter_mod_paths(base_dir: str):
"""Yield ``(relative_stem, absolute_stem)`` for RefMods under ``base_dir``."""
for root, dirnames, filenames in os.walk(base_dir):
dirnames[:] = sorted(d for d in dirnames if d not in _MOD_SKIP_DIRS)
for fn in sorted(filenames):
if not fn.endswith(".safetensors"):
continue
abs_stem = os.path.join(root, fn[:-len(".safetensors")])
rel_stem = os.path.relpath(abs_stem, base_dir).replace(os.sep, "/")
yield rel_stem, abs_stem
def _mod_scan_signature() -> tuple[str, List[str]]:
mods_dirs = refmods_dirs()
sig = []
for mods_dir in mods_dirs:
for rel_stem, abs_stem in _iter_mod_paths(mods_dir):
try:
st = os.stat(abs_stem + ".safetensors")
sig.append(f"{mods_dir}:{rel_stem}:{st.st_size}:{int(st.st_mtime)}")
except OSError:
pass
return "\n".join(sig), mods_dirs
def _root_display_labels(roots: List[str]) -> List[str]:
labels: List[str] = []
counts: Dict[str, int] = {}
for index, root in enumerate(roots, start=1):
base = os.path.basename(os.path.normpath(root)) or f"root{index}"
count = counts.get(base, 0) + 1
counts[base] = count
labels.append(base if count == 1 else f"{base}#{count}")
return labels
def _refresh_mod_lookup() -> Dict[str, str]:
global _MOD_LIST_CACHE_KEY, _MOD_LIST_CACHE_VAL, _MOD_PATH_CACHE_KEY, _MOD_PATH_CACHE_VAL
key, mods_dirs = _mod_scan_signature()
if key == _MOD_LIST_CACHE_KEY and key == _MOD_PATH_CACHE_KEY and _MOD_LIST_CACHE_VAL is not None:
return _MOD_PATH_CACHE_VAL
names_by_rel: Dict[str, List[Tuple[int, str]]] = {}
for index, mods_dir in enumerate(mods_dirs):
for rel_stem, abs_stem in _iter_mod_paths(mods_dir):
meta = read_refmod_meta(abs_stem)
if meta is None or meta.get("kind") not in ("image", "video", "audio", "bundle"):
continue
names_by_rel.setdefault(rel_stem, []).append((index, abs_stem))
labels = _root_display_labels(mods_dirs)
lookup: Dict[str, str] = {}
names: List[str] = []
for rel_stem in sorted(names_by_rel, key=str.lower):
matches = names_by_rel[rel_stem]
first_index, first_path = matches[0]
lookup[rel_stem] = first_path
names.append(rel_stem)
for dup_index, dup_path in matches[1:]:
display_name = f"[{labels[dup_index]}] {rel_stem}"
suffix = 2
while display_name in lookup:
display_name = f"[{labels[dup_index]} #{suffix}] {rel_stem}"
suffix += 1
lookup[display_name] = dup_path
names.append(display_name)
_MOD_LIST_CACHE_KEY = key
_MOD_LIST_CACHE_VAL = names
_MOD_PATH_CACHE_KEY = key
_MOD_PATH_CACHE_VAL = lookup
return lookup
def _list_mod_names() -> List[str]:
"""Available RefMod names across the search dirs (for the loader dropdown).
Only entries with valid RefMod metadata (embedded in the safetensors header
or a legacy sidecar .json) are listed, so other mod formats in
models/mods/ (e.g. LTXMod files) don't show up.
Called by INPUT_TYPES/VALIDATE_INPUTS on every prompt validation, so the
result is cached until any mod file appears/disappears/changes (checked
via cheap os.stat, not by re-reading every safetensors header).
"""
_refresh_mod_lookup()
return _MOD_LIST_CACHE_VAL
def _find_mod_path(name: str) -> str:
lookup = _refresh_mod_lookup()
if name in lookup:
return lookup[name]
mods_dirs = refmods_dirs()
raise FileNotFoundError(
f"RefMod '{name}' not found. Searched:\n" +
"\n".join(f" - {d}/{name}.safetensors" for d in mods_dirs))
def _load_mods(name: str) -> List[H3RefMod]:
if name in _MOD_CACHE:
return list(_MOD_CACHE[name])
path = _find_mod_path(name)
mods = tuple(load_refmods_from_file(path, device="cpu"))
_MOD_CACHE[name] = mods
if len(_MOD_CACHE) > _MOD_CACHE_MAX:
_MOD_CACHE.pop(next(iter(_MOD_CACHE)))
return list(mods)
def _load_mod(name: str) -> H3RefMod:
mods = _load_mods(name)
for mod in mods:
if mod.kind != "audio":
return mod
return mods[0]
def _mod_capabilities(name: str) -> Dict[str, bool]:
path = _find_mod_path(name)
meta = read_refmod_meta(path)
if meta is None:
return {"has_visual": True, "has_audio": False}
caps = refmod_capabilities(meta)
return {
"has_visual": bool(caps.get("has_visual", False)),
"has_audio": bool(caps.get("has_audio", False)),
}
def _normalize_weight(weight: float) -> float:
return min(_MAX_WEIGHT, max(0.0, float(weight)))
def _expand_weight(weight: float) -> List[float]:
clipped = _normalize_weight(weight)
whole = int(clipped)
remainder = clipped - whole
strengths = [1.0] * whole
if remainder > 1e-6:
strengths.append(remainder)
return strengths
def _append_weighted_mod(rows: List[tuple[H3RefMod, float]], mod: H3RefMod, weight: float) -> float:
clipped = _normalize_weight(weight)
rows.extend((mod, strength) for strength in _expand_weight(clipped))
return clipped
def _weight_display(weight: float) -> str:
clipped = _normalize_weight(weight)
whole = int(clipped)
remainder = clipped - whole
if whole <= 0:
return f"{clipped:.2f}"
if remainder <= 1e-6:
return f"x{whole}"
return f"x{whole} + {remainder:.2f}"
class H3RefModLoader:
"""Load one RefMod and append it to an existing bundle."""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"mod": (_list_mod_names(), {"tooltip": "The RefMod to load."}),
"weight": ("FLOAT", {"default": 1.0, "min": 0.0, "max": _MAX_WEIGHT, "step": 0.01,
"display": "number",
"tooltip": "Legacy unified weight control kept for workflow compatibility. New graphs should use video_weight and audio_weight."}),
"video_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "max": _MAX_WEIGHT, "step": 0.01,
"display": "number",
"tooltip": "Unified video weight control. 0 skips video. 0..1 behaves like the old strength control. Values above 1 repeat the same visual RefMod as extra copies."}),
"audio_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "max": _MAX_WEIGHT, "step": 0.01,
"display": "number",
"tooltip": "Unified audio weight control. 0 skips audio. 0..1 behaves like the old strength control. Values above 1 repeat the same audio RefMod as extra copies."}),
},
"optional": {
"mods": ("H3_REF_MODS",),
},
}
RETURN_TYPES = ("H3_REF_MODS", "STRING")
RETURN_NAMES = ("mods", "prompt_hint")
FUNCTION = "load"
CATEGORY = "H3RefModPicker"
@classmethod
def VALIDATE_INPUTS(cls, mod, **kwargs):
if mod not in set(_list_mod_names()):
return f"RefMod '{mod}' not found in any configured refmods folder. Run Create H3 RefMod first."
return True
def load(self, mod, weight=1.0, video_weight=1.0, audio_weight=1.0, mods=None, strength=None, audio_strength=None):
rows = list(mods) if mods is not None else []
if strength is not None:
weight = strength
if audio_strength is not None:
audio_weight = audio_strength
if strength is not None or (weight != 1.0 and video_weight == 1.0 and audio_weight == 1.0):
video_weight = weight
audio_weight = weight
loaded = _load_mods(mod)
summaries = []
hint_rows = list(mods) if mods is not None else []
for item in loaded:
current_weight = audio_weight if item.kind == "audio" else video_weight
clipped = _append_weighted_mod(rows, item, current_weight)
summaries.append(f"{item.name}({_weight_display(clipped)})")
if clipped > 0.0:
hint_rows.append((item, min(1.0, clipped)))
print(f"[H3RefModLoader] {mod}: " + ", ".join(summaries))
hint = _prompt_hint(hint_rows)
if hint:
print(f"[H3RefModLoader] prompt_hint: {hint}")
return (rows, hint)
NODE_CLASS_MAPPINGS = {
"H3RefModLoader": H3RefModLoader
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3RefModLoader": "Load H3 RefMod Simple"
}
@server.PromptServer.instance.routes.get("/h3refmods/refmod-loader/capabilities")
async def refmod_loader_capabilities(request):
try:
name = str(request.query.get("mod", "") or "").strip()
if not name:
return server.web.json_response({"ok": True, "has_visual": True, "has_audio": True})
caps = _mod_capabilities(name)
return server.web.json_response({"ok": True, **caps})
except Exception as exc:
return server.web.json_response({"ok": False, "error": str(exc)}, status=500)
@@ -0,0 +1,165 @@
from __future__ import annotations
import os
from typing import Dict, List, Tuple
import server
from ..py.refmod_browser import (
PREVIEW_EXTS,
browser_root,
list_browser_dir,
paired_mod_path,
safe_file_path,
)
from ..py.refmod_core import H3RefMod, load_refmods_from_file
from ..py.refmod_common import _prompt_hint
from .refmod_loader import _MAX_WEIGHT, _append_weighted_mod, _weight_display
_VISUAL_MOD_CACHE: Dict[str, Tuple[H3RefMod, ...]] = {}
_VISUAL_MOD_CACHE_MAX = 24
def _load_mods_from_path(mod_path: str) -> List[H3RefMod]:
path = safe_file_path(mod_path)
if not path.lower().endswith(".safetensors"):
raise ValueError("Selected file is not a RefMod safetensors file.")
if path in _VISUAL_MOD_CACHE:
return list(_VISUAL_MOD_CACHE[path])
mods = list(load_refmods_from_file(path[:-len(".safetensors")], device="cpu"))
if not any(mod.kind == "audio" for mod in mods):
paired_audio = paired_mod_path(path[:-len(".safetensors")], "audio")
if paired_audio is not None:
mods.extend(load_refmods_from_file(paired_audio[:-len(".safetensors")], device="cpu"))
_VISUAL_MOD_CACHE[path] = tuple(mods)
if len(_VISUAL_MOD_CACHE) > _VISUAL_MOD_CACHE_MAX:
_VISUAL_MOD_CACHE.pop(next(iter(_VISUAL_MOD_CACHE)))
return mods
class H3RefModVisualPicker:
"""Pick a RefMod by browsing models/refmods thumbnails and append it to a bundle."""
NAME = "Visual RefMod Picker"
CATEGORY = "H3RefModPicker"
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"mod_path": (
"STRING",
{
"default": "",
"tooltip": "Selected RefMod safetensors path from the thumbnail browser.",
},
),
"video_weight": (
"FLOAT",
{
"default": 1.0,
"min": 0.0,
"max": _MAX_WEIGHT,
"step": 0.01,
"tooltip": "Unified video weight control. 0 skips video. 0..1 behaves like the old strength control. Values above 1 repeat the same video RefMod as extra copies.",
},
),
"audio_weight": (
"FLOAT",
{
"default": 1.0,
"min": 0.0,
"max": _MAX_WEIGHT,
"step": 0.01,
"tooltip": "Unified audio weight control. 0 skips audio. 0..1 behaves like the old audio strength control. Values above 1 repeat the same audio RefMod as extra copies.",
},
),
},
"optional": {
"mods": ("H3_REF_MODS",),
},
}
RETURN_TYPES = ("H3_REF_MODS", "STRING")
RETURN_NAMES = ("mods", "prompt_hint")
FUNCTION = "pick"
DESCRIPTION = (
"Pick a RefMod by browsing models/refmods and its subfolders. "
"A matching preview image with the same base name is shown when present; "
"otherwise a placeholder is used. When a matching *_Audio file exists "
"beside a *_Video or *_Visual file, both are loaded together and appended to the bundle, "
"with separate video/audio weights. A weight in 0..1 behaves like the old strength control; "
"a weight above 1 repeats the same RefMod as extra copies. Legacy combined RefMods still use one shared video weight."
)
@classmethod
def VALIDATE_INPUTS(cls, **kwargs):
return True
def pick(self, mod_path: str, video_weight: float = 1.0, audio_weight: float = 1.0,
mods=None, strength=None, audio_strength=None):
rows: List[Tuple[H3RefMod, float]] = []
if mods is not None and isinstance(mods, (list, tuple)):
rows.extend(list(mods))
selected = str(mod_path or "").strip()
if not selected or selected.lower() == "(none)":
hint = _prompt_hint(rows)
return (rows, hint)
if strength is not None:
video_weight = strength
if audio_strength is not None:
audio_weight = audio_strength
selected_mods = _load_mods_from_path(selected)
hint_rows = list(mods) if mods is not None and isinstance(mods, (list, tuple)) else []
summaries = []
for mod in selected_mods:
weight = audio_weight if mod.kind == "audio" else video_weight
clipped = _append_weighted_mod(rows, mod, weight)
if clipped > 0.0:
hint_rows.append((mod, min(1.0, clipped)))
summaries.append(f"{mod.name}({_weight_display(clipped)})")
print("[H3RefModVisualPicker] " + ", ".join(summaries))
hint = _prompt_hint(hint_rows)
if hint:
print(f"[H3RefModVisualPicker] prompt_hint: {hint}")
return (rows, hint)
NODE_CLASS_MAPPINGS = {"H3RefModVisualPicker": H3RefModVisualPicker}
NODE_DISPLAY_NAME_MAPPINGS = {"H3RefModVisualPicker": "Visual H3 RefMod Picker"}
@server.PromptServer.instance.routes.get("/h3refmods/refmod-browser/root")
async def refmod_browser_root(request):
try:
return server.web.json_response({"ok": True, "root": browser_root()})
except Exception as exc:
return server.web.json_response({"ok": False, "error": str(exc)}, status=500)
@server.PromptServer.instance.routes.get("/h3refmods/refmod-browser/list")
async def refmod_browser_list(request):
try:
data = list_browser_dir(request.query.get("path", "").strip())
return server.web.json_response({"ok": True, **data})
except ValueError as exc:
return server.web.json_response({"ok": False, "error": str(exc)}, status=404)
except Exception as exc:
return server.web.json_response({"ok": False, "error": str(exc)}, status=500)
@server.PromptServer.instance.routes.get("/h3refmods/refmod-browser/file")
async def refmod_browser_file(request):
try:
path = safe_file_path(request.query.get("path", "").strip())
if os.path.splitext(path)[1].lower() not in PREVIEW_EXTS:
return server.web.json_response(
{"ok": False, "error": "Unsupported file type"}, status=403
)
return server.web.FileResponse(path, headers={"Cache-Control": "no-cache"})
except ValueError as exc:
return server.web.json_response({"ok": False, "error": str(exc)}, status=404)
except Exception as exc:
return server.web.json_response({"ok": False, "error": str(exc)}, status=500)
+293
View File
@@ -0,0 +1,293 @@
"""Native H3 reference conditioning with grounded RefMod stacks."""
from __future__ import annotations
import io as bytes_io
import math
import torch
import torch.nn.functional as functional
from comfy_api.latest import io
REFERENCE_TYPE = "H3_REFMOD_REFERENCES"
def _mod_rows(mods):
if mods is None:
return []
if hasattr(mods, "ref_block"):
return [(mods, 1.0)]
if isinstance(mods, (tuple, list)) and mods and hasattr(mods[0], "ref_block"):
mods = [mods]
rows = []
for row in mods:
if not isinstance(row, (tuple, list)) or len(row) not in (2, 3):
raise ValueError("mods must contain (RefMod, strength) rows.")
mod, strength = row[:2]
if mod is None:
continue
if not callable(getattr(mod, "ref_block", None)):
raise ValueError("mods contains an unsupported RefMod object.")
strength = float(strength)
if not math.isfinite(strength):
raise ValueError("RefMod strengths must be finite.")
if strength > 0:
rows.append((mod, min(1.0, strength)))
return rows
def _decoded_frames(video_vae, latent):
with torch.no_grad():
frames = video_vae.decode(latent)
if frames.ndim == 5 and frames.shape[0] == 1:
frames = frames[0]
if frames.ndim != 4 or frames.shape[-1] != 3 or frames.shape[0] < 1:
raise ValueError(f"H3 video VAE returned invalid images: {tuple(frames.shape)}")
return frames.detach().float().clamp(0, 1).cpu()
def _encoder_frames(mod, video_vae, reference_fps):
times = getattr(mod, "enc_times", None)
path = getattr(mod, "path", "")
packed_frames = getattr(mod, "enc_frames", None)
is_stack = getattr(mod, "source", "") == "stack"
if times and (packed_frames or path) and (is_stack or abs(getattr(mod, "enc_fps", 0) - reference_fps) < 1e-6):
import numpy as np
from PIL import Image
def unpack(get_tensor):
frames = []
for index in range(len(times)):
packed = get_tensor(f"enc_{index}").numpy().tobytes()
with Image.open(bytes_io.BytesIO(packed)) as image:
frames.append(torch.from_numpy(np.array(image.convert("RGB"))).float() / 255)
return torch.stack(frames), list(times), "stored pictures"
if packed_frames:
return unpack(packed_frames.__getitem__)
from safetensors import safe_open
filename = path if path.endswith(".safetensors") else path + ".safetensors"
with safe_open(filename, framework="pt", device="cpu") as handle:
return unpack(handle.get_tensor)
if video_vae is None:
raise ValueError(f"Connect video_vae to reconstruct visual RefMod '{mod.name}'.")
if is_stack:
frames = [_decoded_frames(video_vae, mod.latent[:, :, index:index + 1])[:1]
for index in range(mod.latent.shape[2])]
return torch.cat(frames), [float(index) for index in range(len(frames))], "decoded stack"
frames = _decoded_frames(video_vae, mod.latent)
if mod.kind == "image":
return frames[:1], [0.0], "decoded image"
times = [index / 2 for index in range(math.ceil(frames.shape[0] * 2 / reference_fps))]
indices = [min(round(time * reference_fps), frames.shape[0] - 1) for time in times]
return frames[indices], times, "decoded video"
def _visual_item(mod, frames, times, strength, stack_pictures, visual_kind=None):
if strength < 1:
samples = frames.movedim(-1, 1)
blurred = functional.adaptive_avg_pool2d(samples, (max(1, mod.latent_h // 8),
max(1, mod.latent_w // 8)))
blurred = functional.interpolate(blurred, size=samples.shape[-2:], mode="bilinear", align_corners=False)
frames = (strength * samples + (1 - strength) * blurred).movedim(1, -1)
if (visual_kind or mod.kind) == "image":
return {"type": "image", "data": frames[:1]}, 1
if getattr(mod, "source", "") == "stack":
count = frames.shape[0]
if stack_pictures == "every 4th":
indices = list(range(0, count, 4))
frames = frames[indices]
times = [index / 2 for index in range(len(indices))]
else:
indices = ([round(index * (count - 1) / 7) for index in range(8)]
if stack_pictures == "up to 8" and count > 8 else list(range(count)))
frames = frames[indices].repeat_interleave(2, dim=0)
times = [index + offset for index in range(len(indices)) for offset in (0.0, 0.5)]
shown = len(indices)
else:
shown = frames.shape[0]
return {"type": "video", "data": frames, "timestamps": times}, shown
class H3RefModReferences(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="H3RefModReferences",
display_name="Collect H3 References",
category="H3RefModPicker",
inputs=[
io.Custom(REFERENCE_TYPE).Input("references", optional=True,
tooltip="Optional references from another collector."),
io.Autogrow.Input("ref_images", optional=True, template=io.Autogrow.TemplatePrefix(
input=io.Image.Input("ref_image"), prefix="ref_image_", min=0, max=9)),
io.Autogrow.Input("ref_videos", optional=True, template=io.Autogrow.TemplatePrefix(
input=io.Image.Input("ref_video", tooltip="Video frame batch at 24 fps."),
prefix="ref_video_", min=0, max=3)),
io.Autogrow.Input("ref_video_audios", optional=True, template=io.Autogrow.TemplatePrefix(
input=io.Audio.Input("ref_video_audio", tooltip="Soundtrack for the same-numbered video input."),
prefix="ref_video_audio_", min=0, max=3)),
io.Autogrow.Input("ref_audios", optional=True, template=io.Autogrow.TemplatePrefix(
input=io.Audio.Input("ref_audio"), prefix="ref_audio_", min=0, max=3)),
],
outputs=[io.Custom(REFERENCE_TYPE).Output("references")],
)
@classmethod
def execute(cls, references=None, ref_images=None, ref_videos=None,
ref_video_audios=None, ref_audios=None):
entries = list(references or [])
for name, image in (ref_images or {}).items():
if image is not None:
entries.append({"kind": "image", "data": image, "name": name})
videos = ref_videos or {}
soundtracks = ref_video_audios or {}
for name, soundtrack in soundtracks.items():
if soundtrack is not None and videos.get("ref_video_" + name.rsplit("_", 1)[-1]) is None:
raise ValueError(f"{name} needs its same-numbered ref_video input; use ref_audio for standalone audio.")
for name, frames in videos.items():
if frames is not None:
entries.append({"kind": "video", "data": frames, "name": name,
"audio": soundtracks.get("ref_video_audio_" + name.rsplit("_", 1)[-1])})
for name, audio in (ref_audios or {}).items():
if audio is not None:
entries.append({"kind": "audio", "data": audio, "name": name})
return io.NodeOutput(entries)
class _ReferencePresentation:
def __init__(self):
self.items = []
def tokenize(self, prompt, minimax_ref_items):
self.items = minimax_ref_items
return None
def encode_from_tokens_scheduled(self, tokens):
return [[None, {}]]
def _native_references(references, video_vae, audio_vae, width, height, length, ref_image_size):
from comfy_extras.nodes_minimax_h3 import MiniMaxH3ReferenceToVideo
images, videos, soundtracks, audios = {}, {}, {}, {}
names = []
for kind in ("image", "video", "audio"):
for index, entry in enumerate(references or []):
if entry.get("kind") != kind:
continue
name = entry["name"]
if kind == "image":
images[f"ref_image_{index}"] = entry["data"]
elif kind == "video":
videos[f"ref_video_{index}"] = entry["data"]
if entry.get("audio") is not None:
soundtracks[f"ref_video_audio_{index}"] = entry["audio"]
names.append(name + " soundtrack")
else:
audios[f"ref_audio_{index}"] = entry["data"]
names.append(name)
presentation = _ReferencePresentation()
output = MiniMaxH3ReferenceToVideo.execute(
presentation, "", width, height, length, ref_image_size=ref_image_size,
vae=video_vae, audio_vae=audio_vae, ref_images=images, ref_videos=videos,
ref_video_audios=soundtracks, ref_audios=audios)
conditioning, latent = output.result
return presentation.items, list(conditioning[0][1].get("minimax_refs", [])), names, latent
class H3RefModsToVideo(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="H3RefModsToVideo",
display_name="MiniMax H3 RefMods to Video",
category="model/conditioning/minimax",
description="Reference-to-Video with grounded RefMods and optional collected media references.",
inputs=[
io.Clip.Input("clip"),
io.Vae.Input("video_vae", optional=True,
tooltip="Encodes raw pictures/videos and reconstructs RefMods without stored encoder pictures."),
io.Vae.Input("audio_vae", optional=True,
tooltip="Encodes raw reference audio. Audio RefMods are already encoded."),
io.Custom(REFERENCE_TYPE).Input("references", optional=True),
io.MultiType.Input("mods", types=[io.Custom("H3_REFMOD"), io.Custom("H3_REF_MODS")], optional=True),
io.String.Input("prompt", multiline=True, dynamic_prompts=True),
io.Int.Input("width", default=1344, min=32, max=8192, step=32),
io.Int.Input("height", default=768, min=32, max=8192, step=32),
io.Int.Input("length", default=124, min=5, max=3600, step=17),
io.Combo.Input("ref_image_size", options=["match", "max"], default="match"),
io.Combo.Input("stack_pictures", options=["every 4th", "up to 8", "all"], default="every 4th",
tooltip="Pictures shown to the text encoder for a stacked RefMod. All latent frames remain applied."),
],
outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output(),
io.String.Output("reference_map")],
)
@classmethod
def execute(cls, clip, prompt, width, height, length, video_vae=None, audio_vae=None,
references=None, mods=None, ref_image_size="match", stack_pictures="every 4th"):
if stack_pictures not in ("every 4th", "up to 8", "all"):
raise ValueError("Unknown stack_pictures mode.")
items, blocks, names, latent = _native_references(
references, video_vae, audio_vae, width, height, length, ref_image_size)
counters = {"image": 0, "video": 0, "audio": 0}
labels = {"image": "Picture", "video": "Video", "audio": "Audio"}
mapping = []
def append_label(item, name):
kind = item["type"]
counters[kind] += 1
mapping.append(f"<{labels[kind]} {counters[kind]}> = {name}")
for item, name in zip(items, names):
append_label(item, name)
decoded = {}
for mod, strength in _mod_rows(mods):
block = mod.ref_block(strength)
if block is None:
continue
block = dict(block)
kind = block["kind"]
name = mod.name
if kind not in ("image", "video", "video_audio", "audio"):
raise ValueError(f"Unsupported RefMod kind: {kind}")
if kind == "audio" or (block.get("audio_latent") is not None and block.get("ref_audio_t", 0) > 0):
item = {"type": "audio"}
items.append(item)
append_label(item, name + (" soundtrack" if kind != "audio" else ""))
if kind != "audio":
if id(mod) not in decoded:
decoded[id(mod)] = _encoder_frames(mod, video_vae, 24.0)
frames, times, _source = decoded[id(mod)]
visual_kind = "video" if kind == "video_audio" else kind
item, _shown = _visual_item(mod, frames, times, strength, stack_pictures, visual_kind)
items.append(item)
append_label(item, name)
block["refmod"] = True
blocks.append(block)
if video_vae is not None or audio_vae is not None:
import comfy.model_management
comfy.model_management.soft_empty_cache()
tokens = clip.tokenize(prompt, minimax_ref_items=items)
conditioning = clip.encode_from_tokens_scheduled(tokens)
output = []
for embedding, metadata in conditioning:
if items and "minimax_token_tags" not in metadata:
raise ValueError("Connect a MiniMax H3 CLIP that supports reference token tags.")
metadata = dict(metadata)
if blocks:
metadata["minimax_refs"] = list(metadata.get("minimax_refs", [])) + blocks
output.append([embedding, metadata])
reference_map = "\n".join(mapping) or "No references."
return io.NodeOutput(output, latent, reference_map)
NODE_CLASS_MAPPINGS = {
"H3RefModsToVideo": H3RefModsToVideo,
"H3RefModReferences": H3RefModReferences,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3RefModsToVideo": "MiniMax H3 RefMods to Video",
"H3RefModReferences": "Collect H3 References",
}