Merge commit 'c9eb575e2fd25f74b15f1d7336af4932c7fe6338' as 'vendor/dumas_h3_refmod_picker'
This commit is contained in:
+183
@@ -0,0 +1,183 @@
|
||||
"""
|
||||
H3RefModApplySimple — identity-oriented RefMod apply node with one strength dial
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from dataclasses import replace
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from comfy_api.latest import io
|
||||
|
||||
# reference retention presets (master strength multiplier on Apply)
|
||||
RETENTION = {
|
||||
"fully_preserved": 1.0,
|
||||
"partially_preserved": 0.7,
|
||||
"attribute_transfer": 0.4,
|
||||
"weak_reference": 0.15,
|
||||
}
|
||||
|
||||
|
||||
def _filter_legacy_ref_block(block: Optional[Dict],
|
||||
use_video: bool,
|
||||
use_audio: bool) -> Optional[Dict]:
|
||||
"""Apply media toggles to ref blocks produced by older RefMod APIs."""
|
||||
if block is None or (not use_video and not use_audio):
|
||||
return None
|
||||
|
||||
kind = str(block.get("kind") or "").strip().lower()
|
||||
has_audio = block.get("audio_latent") is not None and int(block.get("ref_audio_t") or 0) > 0
|
||||
|
||||
if kind == "audio":
|
||||
return block if use_audio else None
|
||||
|
||||
if not use_video:
|
||||
if use_audio and has_audio:
|
||||
return {
|
||||
"kind": "audio",
|
||||
"ref_audio_t": int(block.get("ref_audio_t") or 0),
|
||||
"audio_latent": block.get("audio_latent"),
|
||||
}
|
||||
return None
|
||||
|
||||
if not use_audio and has_audio:
|
||||
filtered = dict(block)
|
||||
filtered["kind"] = "video"
|
||||
filtered["ref_audio_t"] = 0
|
||||
filtered["audio_latent"] = None
|
||||
return filtered
|
||||
|
||||
return block
|
||||
|
||||
|
||||
def _build_ref_block(mod, eff: float, curve, use_video: bool, use_audio: bool) -> Optional[Dict]:
|
||||
try:
|
||||
return mod.ref_block(eff, curve=curve, use_video=use_video, use_audio=use_audio)
|
||||
except TypeError as exc:
|
||||
message = str(exc)
|
||||
if "unexpected keyword argument 'use_video'" not in message and "unexpected keyword argument 'use_audio'" not in message:
|
||||
raise
|
||||
legacy_block = mod.ref_block(eff, curve=curve)
|
||||
return _filter_legacy_ref_block(legacy_block, use_video=use_video, use_audio=use_audio)
|
||||
|
||||
|
||||
def _ref_blocks(mods, retention, curve=None, seed=-1,
|
||||
use_video: bool = True, use_audio: bool = True) -> List[Dict]:
|
||||
"""Ref blocks for a loader bundle, scaled by row strength x retention.
|
||||
|
||||
``retention`` is a master strength multiplier: a float 0-1 (1.0 =
|
||||
fully_preserved, 0.7 = partially_preserved, 0.4 = attribute_transfer,
|
||||
0.15 = weak_reference), or one of those preset names for legacy
|
||||
workflows saved with the old combo widget.
|
||||
|
||||
``curve`` (optional) is a per-frame strength spec — a ``(direction,
|
||||
shape, value)`` tuple, a legacy preset name, per-frame values, control
|
||||
points (see ``core.curve_strengths``) — applied on top of the row
|
||||
strength. A flat/no curve keeps today's behavior.
|
||||
|
||||
``seed`` (default -1 = off) enables ref scrambling: with 2+ refs in the
|
||||
bundle, the order is shuffled and a random subset kept, so a different
|
||||
ref leads each run instead of the same one always "popping". Same seed
|
||||
-> same scramble; connect/randomize the seed for per-run variation.
|
||||
"""
|
||||
if isinstance(retention, str):
|
||||
factor = RETENTION.get(retention, 1.0)
|
||||
else:
|
||||
factor = float(retention)
|
||||
items = [] if mods is None else list(mods)
|
||||
if int(seed) >= 0 and len(items) > 1:
|
||||
rng = random.Random(int(seed))
|
||||
rng.shuffle(items)
|
||||
keep = rng.randint(max(1, len(items) // 2), len(items))
|
||||
items = items[:keep]
|
||||
print(f"[H3RefModApply] scramble seed={int(seed)}: "
|
||||
f"{len(mods)} refs -> kept {len(items)} (order shuffled)")
|
||||
blocks = []
|
||||
debug_rows = []
|
||||
for mod, strength in items:
|
||||
eff = min(1.0, max(0.0, strength * factor))
|
||||
block = _build_ref_block(mod, eff, curve=curve, use_video=use_video, use_audio=use_audio)
|
||||
if block is not None:
|
||||
blocks.append(block)
|
||||
debug_rows.append(f"{mod.name}@{eff:.2f}")
|
||||
else:
|
||||
debug_rows.append(f"{mod.name}@{eff:.2f} (skipped)")
|
||||
if debug_rows:
|
||||
print("[H3RefModApply] effective refs: " + ", ".join(debug_rows))
|
||||
return blocks
|
||||
|
||||
|
||||
class H3RefModApplySimple(io.ComfyNode):
|
||||
"""Identity-oriented RefMod apply node with one strength dial."""
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
template = io.MatchType.Template(
|
||||
"cond",
|
||||
allowed_types=[io.Custom("MINIMAX_H3_COND"), io.Conditioning])
|
||||
return io.Schema(
|
||||
node_id="H3RefModApplySimple",
|
||||
display_name="Apply H3 RefMod Simple",
|
||||
description=(
|
||||
"Apply one or more RefMods to a MiniMax H3 conditioning with a single "
|
||||
"Strength control. This identity-oriented version uses a flat full-length "
|
||||
"reference curve under the hood."
|
||||
),
|
||||
category="H3RefModPicker",
|
||||
inputs=[
|
||||
io.MatchType.Input("conditioning", template=template,
|
||||
tooltip="MINIMAX_H3_COND (ComfyUI-MiniMaxH3 pack) or CONDITIONING "
|
||||
"(core MiniMaxH3ReferenceToVideo)."),
|
||||
io.Custom("H3_REF_MODS").Input("mods",
|
||||
optional=True,
|
||||
tooltip="Bundle of RefMods to inject. Leave unconnected to bypass unchanged."),
|
||||
io.Boolean.Input("use_video", default=True,
|
||||
tooltip="Apply the visual latent from each RefMod. Turn this off to use only embedded audio."),
|
||||
io.Boolean.Input("use_audio", default=True,
|
||||
tooltip="Apply the embedded audio latent from each RefMod when present. Turn this off for visual-only application."),
|
||||
io.Float.Input("strength", default=1.0, min=0.0, max=1.0, step=0.01,
|
||||
tooltip="Master reference strength. For identity work this is the main "
|
||||
"dial: higher = tighter identity lock, lower = more freedom but "
|
||||
"more drift."),
|
||||
],
|
||||
outputs=[
|
||||
io.MatchType.Output(template=template, display_name="conditioning",
|
||||
tooltip="The conditioning with the ref blocks injected, same type as the input."),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, conditioning, mods, use_video=True, use_audio=True, strength=1.0):
|
||||
if mods is None:
|
||||
print("[H3RefModApplySimple] no mods connected — bypassing conditioning unchanged")
|
||||
return io.NodeOutput(conditioning)
|
||||
if not use_video and not use_audio:
|
||||
print("[H3RefModApplySimple] use_video=False and use_audio=False — bypassing conditioning unchanged")
|
||||
return io.NodeOutput(conditioning)
|
||||
blocks = _ref_blocks(mods, strength, ("constant", "linear", 1.0), seed=-1,
|
||||
use_video=bool(use_video), use_audio=bool(use_audio))
|
||||
if isinstance(conditioning, list):
|
||||
out = []
|
||||
for t in conditioning:
|
||||
d = dict(t[1])
|
||||
d["minimax_refs"] = list(d.get("minimax_refs", [])) + blocks
|
||||
out.append([t[0], d])
|
||||
print(f"[H3RefModApplySimple] strength={strength} "
|
||||
f"({len(blocks)} ref block(s) injected)")
|
||||
else:
|
||||
out = replace(conditioning, refs=list(conditioning.refs) + blocks)
|
||||
print(f"[H3RefModApplySimple] strength={strength} "
|
||||
f"({len(blocks)} ref block(s) injected, {len(out.refs)} total)")
|
||||
return io.NodeOutput(out)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3RefModApplySimple": H3RefModApplySimple,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3RefModApplySimple": "Apply H3 RefMod Simple",
|
||||
}
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
"""
|
||||
H3RefModApplyAxis — choose between two picker bundles with separate video/audio signed values
|
||||
Think of it as creating a Slider from two RefMods
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from ..py.refmod_common import (
|
||||
_prompt_hint,
|
||||
)
|
||||
|
||||
from ..py.refmod_core import (
|
||||
H3RefMod,
|
||||
)
|
||||
_PACK_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
LEGACY_MODS_DIR = os.path.join(_PACK_DIR, "mods") # pre-models/refmods storage, still read
|
||||
|
||||
|
||||
# reference retention presets (master strength multiplier on Apply)
|
||||
RETENTION = {
|
||||
"fully_preserved": 1.0,
|
||||
"partially_preserved": 0.7,
|
||||
"attribute_transfer": 0.4,
|
||||
"weak_reference": 0.15,
|
||||
}
|
||||
|
||||
def _info_lines(mod):
|
||||
opt = mod.optimize_steps
|
||||
if mod.mode == "encode":
|
||||
opt = f"n/a ({mod.optimize_steps} — encode mode stores the actual encode)"
|
||||
return [
|
||||
"=" * 52,
|
||||
f" MiniMax H3 RefMod: {mod.name}",
|
||||
f" {'concept_type':<18} {mod.concept_type}",
|
||||
f" {'mode':<18} {mod.mode}",
|
||||
f" {'kind':<18} {mod.kind}",
|
||||
f" {'latent':<18} {tuple(mod.latent.shape)}",
|
||||
f" {'tokens injected':<18} {mod.token_count}",
|
||||
f" {'source':<18} {mod.source} ({mod.source_shape})",
|
||||
f" {'pool':<18} {mod.pool}",
|
||||
f" {'identity':<18} {opt}",
|
||||
f" {'tags':<18} {', '.join(mod.tags) if mod.tags else '-'}",
|
||||
f" {'description':<18} {mod.description or '-'}",
|
||||
"=" * 52,
|
||||
]
|
||||
|
||||
def _mod_group_key(mod: H3RefMod) -> str:
|
||||
name = str(getattr(mod, "name", "") or "")
|
||||
lower_name = name.lower()
|
||||
for suffix in ("_Video", "_Audio"):
|
||||
if lower_name.endswith(suffix.lower()):
|
||||
return name[:-len(suffix)]
|
||||
return name
|
||||
|
||||
|
||||
def _split_trailing_selection(rows, label: str) -> Tuple[List[Tuple[H3RefMod, float]], Optional[Tuple[H3RefMod, float]], Optional[Tuple[H3RefMod, float]]]:
|
||||
"""Return the last logical RefMod selection from a bundle."""
|
||||
items = [] if rows is None else list(rows)
|
||||
if not items:
|
||||
return [], None, None
|
||||
last_key = _mod_group_key(items[-1][0])
|
||||
start = len(items) - 1
|
||||
while start > 0 and _mod_group_key(items[start - 1][0]) == last_key:
|
||||
start -= 1
|
||||
group = items[start:]
|
||||
if start > 0:
|
||||
print(f"[H3RefModApplyAxis] warning: {label} input contains {start} earlier row(s); using only the last RefMod selection '{last_key}'.")
|
||||
video_row = None
|
||||
audio_row = None
|
||||
for row in group:
|
||||
mod, _strength = row
|
||||
if mod.kind == "audio":
|
||||
audio_row = row
|
||||
else:
|
||||
video_row = row
|
||||
return group, video_row, audio_row
|
||||
|
||||
|
||||
class H3RefModApplyAxis:
|
||||
"""Choose between two picker bundles with separate video/audio signed values."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"mods_a": ("H3_REF_MODS", {
|
||||
"tooltip": "RefMod bundle from one Visual RefMod Picker. If older chained selections are present too, only the last logical RefMod selection is used.",
|
||||
}),
|
||||
"mods_b": ("H3_REF_MODS", {
|
||||
"tooltip": "RefMod bundle from another Visual RefMod Picker. If older chained selections are present too, only the last logical RefMod selection is used.",
|
||||
}),
|
||||
"video_value": ("FLOAT", {
|
||||
"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01,
|
||||
"display": "number",
|
||||
"tooltip": "Signed video selector: negative uses A video, positive uses B video. Magnitude becomes the output video strength. 0 skips video.",
|
||||
}),
|
||||
"audio_value": ("FLOAT", {
|
||||
"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01,
|
||||
"display": "number",
|
||||
"tooltip": "Signed audio selector: negative uses A audio, positive uses B audio. If only one side has audio, that audio is used whenever this is non-zero.",
|
||||
}),
|
||||
"show_info": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Print the selected output RefMods and strengths to the console.",
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("H3_REF_MODS", "STRING")
|
||||
RETURN_NAMES = ("mods", "prompt_hint")
|
||||
FUNCTION = "select"
|
||||
CATEGORY = "H3RefModPicker"
|
||||
|
||||
def select(self, mods_a, mods_b, video_value=0.0, audio_value=0.0, show_info=False):
|
||||
_group_a, video_a, audio_a = _split_trailing_selection(mods_a, "A")
|
||||
_group_b, video_b, audio_b = _split_trailing_selection(mods_b, "B")
|
||||
|
||||
loads: List[Tuple[H3RefMod, float]] = []
|
||||
|
||||
def pick_signed(value: float, row_a, row_b, kind: str):
|
||||
mag = min(1.0, abs(float(value)))
|
||||
if mag < 1e-6:
|
||||
return None
|
||||
chosen = row_b if value > 0 else row_a
|
||||
chosen_side = "B" if value > 0 else "A"
|
||||
fallback = row_a if value > 0 else row_b
|
||||
fallback_side = "A" if value > 0 else "B"
|
||||
if chosen is None:
|
||||
if fallback is not None:
|
||||
print(f"[H3RefModApplyAxis] warning: {kind} {chosen_side} is missing; using {fallback_side} {kind} at {mag:.2f}.")
|
||||
return fallback[0], mag
|
||||
print(f"[H3RefModApplyAxis] warning: no {kind} RefMod available on either side.")
|
||||
return None
|
||||
return chosen[0], mag
|
||||
|
||||
video_pick = pick_signed(video_value, video_a, video_b, "video")
|
||||
if video_pick is not None:
|
||||
loads.append(video_pick)
|
||||
|
||||
if audio_a is not None and audio_b is not None:
|
||||
audio_pick = pick_signed(audio_value, audio_a, audio_b, "audio")
|
||||
elif audio_a is not None or audio_b is not None:
|
||||
mag = min(1.0, abs(float(audio_value)))
|
||||
if mag < 1e-6:
|
||||
audio_pick = None
|
||||
else:
|
||||
only = audio_a or audio_b
|
||||
side = "A" if audio_a is not None else "B"
|
||||
print(f"[H3RefModApplyAxis] audio exists only on {side}; using that audio at {mag:.2f}.")
|
||||
audio_pick = (only[0], mag)
|
||||
else:
|
||||
audio_pick = None
|
||||
if audio_pick is not None:
|
||||
loads.append(audio_pick)
|
||||
|
||||
if loads:
|
||||
print("[H3RefModApplyAxis] " + ", ".join(
|
||||
f"{mod.name}@{strength:.2f}" for mod, strength in loads
|
||||
))
|
||||
else:
|
||||
print("[H3RefModApplyAxis] no output refs selected")
|
||||
if show_info:
|
||||
for mod, strength in loads:
|
||||
print("\n".join(_info_lines(mod)))
|
||||
print(f" {'strength':<18} {strength:.2f}")
|
||||
hint = _prompt_hint(loads)
|
||||
if hint:
|
||||
print(f"[H3RefModApplyAxis] prompt_hint: {hint}")
|
||||
return (loads, hint)
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Registration
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3RefModApplyAxis": H3RefModApplyAxis,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3RefModApplyAxis": "Apply H3 RefMod Axis",
|
||||
}
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
+1143
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,116 @@
|
||||
"""
|
||||
decode_refmod.py — H3RefModDecode node.
|
||||
|
||||
Decode a saved RefMod's latent back into images, so you can see what the mod
|
||||
actually captured. Purely a debugging / inspection tool — it plays no part in
|
||||
generation.
|
||||
|
||||
A RefMod stores a video-VAE latent [1, 24, T, h, w] (the reference). This node
|
||||
runs it back through the H3 video VAE decoder and returns the frames as an IMAGE
|
||||
batch [T, H, W, 3], so you can preview a mod, sanity-check an extraction, or
|
||||
eyeball what a pooled (training-mode) mod actually kept vs an encode-mode one.
|
||||
|
||||
It decodes the full latent timeline exactly as stored in the RefMod; it does
|
||||
not resample or guess a frame subset.
|
||||
|
||||
Note: a training-mode (pooled) mod decodes to a tiny, blurry grid — that's the
|
||||
point of the mode, it only stores a concept thumbnail. An encode-mode mod
|
||||
decodes to recognizable full-res frames. Audio (if the mod has it) is not
|
||||
decoded here — this is the visual reference only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import comfy.utils
|
||||
from comfy_api.latest import io
|
||||
|
||||
from .refmod_loader import _list_mod_names, _load_mod
|
||||
|
||||
|
||||
def _reshape_decoded_frames(frames: torch.Tensor) -> torch.Tensor:
|
||||
if frames.dim() == 5: # [B, T, H, W, C] -> [B*T, H, W, C]
|
||||
frames = frames.reshape(-1, frames.shape[-3], frames.shape[-2], frames.shape[-1])
|
||||
return frames
|
||||
|
||||
|
||||
def _upscale_for_inspection(frames: torch.Tensor, min_edge: int, mode: str) -> torch.Tensor:
|
||||
if min_edge <= 0 or frames.dim() != 4 or frames.shape[0] <= 0:
|
||||
return frames
|
||||
height, width = int(frames.shape[1]), int(frames.shape[2])
|
||||
shortest = min(height, width)
|
||||
if shortest <= 0 or shortest >= min_edge:
|
||||
return frames
|
||||
scale = float(min_edge) / float(shortest)
|
||||
target_h = max(1, int(round(height * scale)))
|
||||
target_w = max(1, int(round(width * scale)))
|
||||
nchw = frames.movedim(-1, 1)
|
||||
if mode == "training":
|
||||
upscaled = F.interpolate(nchw, size=(target_h, target_w), mode="nearest")
|
||||
else:
|
||||
upscaled = comfy.utils.common_upscale(nchw, target_w, target_h, "lanczos", "disabled")
|
||||
return upscaled.movedim(1, -1)
|
||||
|
||||
|
||||
class H3RefModDecode(io.ComfyNode):
|
||||
"""Decode a RefMod's latent back to images for inspection."""
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="H3RefModDecode",
|
||||
display_name="Decode H3 RefMod",
|
||||
category="H3RefMod",
|
||||
description="Decode a saved RefMod's latent back into images so you can see "
|
||||
"what it captured. Inspection only — not part of generation. "
|
||||
"The full stored latent timeline is decoded as-is; no frame "
|
||||
"sampling is applied. "
|
||||
"An encode-mode mod decodes to recognizable frames; a "
|
||||
"training-mode (pooled) mod decodes to a tiny blurry grid "
|
||||
"(that's all it stores). Audio is not decoded.",
|
||||
inputs=[
|
||||
io.Combo.Input("mod", options=_list_mod_names(),
|
||||
tooltip="The RefMod to decode (from models/refmods/)."),
|
||||
io.Vae.Input("vae",
|
||||
tooltip="The MiniMax H3 video VAE."),
|
||||
io.Int.Input("inspect_min_edge", default=320, min=0, max=2048, step=32,
|
||||
tooltip="Upscale small decoded previews so pooled/training mods are easier "
|
||||
"to inspect. 0 disables inspection upscaling. Training-mode uses "
|
||||
"nearest-neighbor to keep the stored cell structure visible."),
|
||||
],
|
||||
outputs=[
|
||||
io.Image.Output("images", display_name="images",
|
||||
tooltip="The decoded reference frames [T, H, W, 3]."),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, mod, vae, inspect_min_edge=320) -> io.NodeOutput:
|
||||
m = _load_mod(mod)
|
||||
if m.kind == "audio":
|
||||
raise ValueError("H3RefModDecode only decodes visual RefMods; audio-only mods are not supported.")
|
||||
z = m.latent # [1, 24, T, h, w]
|
||||
total_t = z.shape[2]
|
||||
|
||||
with torch.no_grad():
|
||||
frames = vae.decode(z)
|
||||
# video latent decodes channel-last; frame count rides the batch dim, so a
|
||||
# 5-dim result is reshaped like the core VAEDecode node does
|
||||
frames = _reshape_decoded_frames(frames)
|
||||
frames = frames.clamp(0.0, 1.0).float().cpu()
|
||||
frames = _upscale_for_inspection(frames, int(inspect_min_edge), m.mode)
|
||||
|
||||
print(f"[H3RefModDecode] '{m.name}' ({m.mode}, {m.kind}, "
|
||||
f"latent_t={total_t}, inspect_min_edge={inspect_min_edge}) -> {tuple(frames.shape)}")
|
||||
return io.NodeOutput(frames)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3RefModDecode": H3RefModDecode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3RefModDecode": "Decode H3 RefMod",
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
"""
|
||||
H3RefModLoader — load one RefMod and optionally append it to another
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Tuple
|
||||
import os
|
||||
import folder_paths
|
||||
import server
|
||||
|
||||
from ..py.refmod_core import H3RefMod, load_refmods_from_file, read_refmod_meta, refmod_capabilities
|
||||
|
||||
from ..py.refmod_common import (
|
||||
_prompt_hint,
|
||||
)
|
||||
|
||||
|
||||
from ..py.refmod_common import (
|
||||
refmods_dir,
|
||||
refmods_dirs,
|
||||
)
|
||||
|
||||
_MOD_CACHE: Dict[str, Tuple[H3RefMod, ...]] = {}
|
||||
_MOD_CACHE_MAX = 24 # cap: never pin more mods in RAM than this (FIFO eviction)
|
||||
_MOD_LIST_CACHE_KEY = None # (dirs, mtimes, sizes) signature of the last _list_mod_names() scan
|
||||
_MOD_LIST_CACHE_VAL = None
|
||||
_MOD_PATH_CACHE_KEY = None
|
||||
_MOD_PATH_CACHE_VAL: Dict[str, str] = {}
|
||||
_MOD_SKIP_DIRS = {"graph_presets", ".git", "__pycache__"}
|
||||
_MAX_WEIGHT = 10.0
|
||||
|
||||
# Mod storage lives in ComfyUI's models/ tree (created on first run) and is
|
||||
# registered as a first-class folder type so it shows up next to loras/unet.
|
||||
try:
|
||||
folder_paths.add_model_folder_path("refmods", refmods_dir())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Mods folder helpers
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def _iter_mod_paths(base_dir: str):
|
||||
"""Yield ``(relative_stem, absolute_stem)`` for RefMods under ``base_dir``."""
|
||||
for root, dirnames, filenames in os.walk(base_dir):
|
||||
dirnames[:] = sorted(d for d in dirnames if d not in _MOD_SKIP_DIRS)
|
||||
for fn in sorted(filenames):
|
||||
if not fn.endswith(".safetensors"):
|
||||
continue
|
||||
abs_stem = os.path.join(root, fn[:-len(".safetensors")])
|
||||
rel_stem = os.path.relpath(abs_stem, base_dir).replace(os.sep, "/")
|
||||
yield rel_stem, abs_stem
|
||||
|
||||
|
||||
def _mod_scan_signature() -> tuple[str, List[str]]:
|
||||
mods_dirs = refmods_dirs()
|
||||
sig = []
|
||||
for mods_dir in mods_dirs:
|
||||
for rel_stem, abs_stem in _iter_mod_paths(mods_dir):
|
||||
try:
|
||||
st = os.stat(abs_stem + ".safetensors")
|
||||
sig.append(f"{mods_dir}:{rel_stem}:{st.st_size}:{int(st.st_mtime)}")
|
||||
except OSError:
|
||||
pass
|
||||
return "\n".join(sig), mods_dirs
|
||||
|
||||
|
||||
def _root_display_labels(roots: List[str]) -> List[str]:
|
||||
labels: List[str] = []
|
||||
counts: Dict[str, int] = {}
|
||||
for index, root in enumerate(roots, start=1):
|
||||
base = os.path.basename(os.path.normpath(root)) or f"root{index}"
|
||||
count = counts.get(base, 0) + 1
|
||||
counts[base] = count
|
||||
labels.append(base if count == 1 else f"{base}#{count}")
|
||||
return labels
|
||||
|
||||
|
||||
def _refresh_mod_lookup() -> Dict[str, str]:
|
||||
global _MOD_LIST_CACHE_KEY, _MOD_LIST_CACHE_VAL, _MOD_PATH_CACHE_KEY, _MOD_PATH_CACHE_VAL
|
||||
|
||||
key, mods_dirs = _mod_scan_signature()
|
||||
if key == _MOD_LIST_CACHE_KEY and key == _MOD_PATH_CACHE_KEY and _MOD_LIST_CACHE_VAL is not None:
|
||||
return _MOD_PATH_CACHE_VAL
|
||||
|
||||
names_by_rel: Dict[str, List[Tuple[int, str]]] = {}
|
||||
for index, mods_dir in enumerate(mods_dirs):
|
||||
for rel_stem, abs_stem in _iter_mod_paths(mods_dir):
|
||||
meta = read_refmod_meta(abs_stem)
|
||||
if meta is None or meta.get("kind") not in ("image", "video", "audio", "bundle"):
|
||||
continue
|
||||
names_by_rel.setdefault(rel_stem, []).append((index, abs_stem))
|
||||
|
||||
labels = _root_display_labels(mods_dirs)
|
||||
lookup: Dict[str, str] = {}
|
||||
names: List[str] = []
|
||||
for rel_stem in sorted(names_by_rel, key=str.lower):
|
||||
matches = names_by_rel[rel_stem]
|
||||
first_index, first_path = matches[0]
|
||||
lookup[rel_stem] = first_path
|
||||
names.append(rel_stem)
|
||||
for dup_index, dup_path in matches[1:]:
|
||||
display_name = f"[{labels[dup_index]}] {rel_stem}"
|
||||
suffix = 2
|
||||
while display_name in lookup:
|
||||
display_name = f"[{labels[dup_index]} #{suffix}] {rel_stem}"
|
||||
suffix += 1
|
||||
lookup[display_name] = dup_path
|
||||
names.append(display_name)
|
||||
|
||||
_MOD_LIST_CACHE_KEY = key
|
||||
_MOD_LIST_CACHE_VAL = names
|
||||
_MOD_PATH_CACHE_KEY = key
|
||||
_MOD_PATH_CACHE_VAL = lookup
|
||||
return lookup
|
||||
|
||||
|
||||
def _list_mod_names() -> List[str]:
|
||||
"""Available RefMod names across the search dirs (for the loader dropdown).
|
||||
|
||||
Only entries with valid RefMod metadata (embedded in the safetensors header
|
||||
or a legacy sidecar .json) are listed, so other mod formats in
|
||||
models/mods/ (e.g. LTXMod files) don't show up.
|
||||
|
||||
Called by INPUT_TYPES/VALIDATE_INPUTS on every prompt validation, so the
|
||||
result is cached until any mod file appears/disappears/changes (checked
|
||||
via cheap os.stat, not by re-reading every safetensors header).
|
||||
"""
|
||||
_refresh_mod_lookup()
|
||||
return _MOD_LIST_CACHE_VAL
|
||||
|
||||
|
||||
def _find_mod_path(name: str) -> str:
|
||||
lookup = _refresh_mod_lookup()
|
||||
if name in lookup:
|
||||
return lookup[name]
|
||||
mods_dirs = refmods_dirs()
|
||||
raise FileNotFoundError(
|
||||
f"RefMod '{name}' not found. Searched:\n" +
|
||||
"\n".join(f" - {d}/{name}.safetensors" for d in mods_dirs))
|
||||
|
||||
|
||||
def _load_mods(name: str) -> List[H3RefMod]:
|
||||
if name in _MOD_CACHE:
|
||||
return list(_MOD_CACHE[name])
|
||||
path = _find_mod_path(name)
|
||||
mods = tuple(load_refmods_from_file(path, device="cpu"))
|
||||
_MOD_CACHE[name] = mods
|
||||
if len(_MOD_CACHE) > _MOD_CACHE_MAX:
|
||||
_MOD_CACHE.pop(next(iter(_MOD_CACHE)))
|
||||
return list(mods)
|
||||
|
||||
|
||||
def _load_mod(name: str) -> H3RefMod:
|
||||
mods = _load_mods(name)
|
||||
for mod in mods:
|
||||
if mod.kind != "audio":
|
||||
return mod
|
||||
return mods[0]
|
||||
|
||||
|
||||
def _mod_capabilities(name: str) -> Dict[str, bool]:
|
||||
path = _find_mod_path(name)
|
||||
meta = read_refmod_meta(path)
|
||||
if meta is None:
|
||||
return {"has_visual": True, "has_audio": False}
|
||||
caps = refmod_capabilities(meta)
|
||||
return {
|
||||
"has_visual": bool(caps.get("has_visual", False)),
|
||||
"has_audio": bool(caps.get("has_audio", False)),
|
||||
}
|
||||
|
||||
|
||||
def _normalize_weight(weight: float) -> float:
|
||||
return min(_MAX_WEIGHT, max(0.0, float(weight)))
|
||||
|
||||
|
||||
def _expand_weight(weight: float) -> List[float]:
|
||||
clipped = _normalize_weight(weight)
|
||||
whole = int(clipped)
|
||||
remainder = clipped - whole
|
||||
strengths = [1.0] * whole
|
||||
if remainder > 1e-6:
|
||||
strengths.append(remainder)
|
||||
return strengths
|
||||
|
||||
|
||||
def _append_weighted_mod(rows: List[tuple[H3RefMod, float]], mod: H3RefMod, weight: float) -> float:
|
||||
clipped = _normalize_weight(weight)
|
||||
rows.extend((mod, strength) for strength in _expand_weight(clipped))
|
||||
return clipped
|
||||
|
||||
|
||||
def _weight_display(weight: float) -> str:
|
||||
clipped = _normalize_weight(weight)
|
||||
whole = int(clipped)
|
||||
remainder = clipped - whole
|
||||
if whole <= 0:
|
||||
return f"{clipped:.2f}"
|
||||
if remainder <= 1e-6:
|
||||
return f"x{whole}"
|
||||
return f"x{whole} + {remainder:.2f}"
|
||||
|
||||
class H3RefModLoader:
|
||||
"""Load one RefMod and append it to an existing bundle."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"mod": (_list_mod_names(), {"tooltip": "The RefMod to load."}),
|
||||
"weight": ("FLOAT", {"default": 1.0, "min": 0.0, "max": _MAX_WEIGHT, "step": 0.01,
|
||||
"display": "number",
|
||||
"tooltip": "Legacy unified weight control kept for workflow compatibility. New graphs should use video_weight and audio_weight."}),
|
||||
"video_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "max": _MAX_WEIGHT, "step": 0.01,
|
||||
"display": "number",
|
||||
"tooltip": "Unified video weight control. 0 skips video. 0..1 behaves like the old strength control. Values above 1 repeat the same visual RefMod as extra copies."}),
|
||||
"audio_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "max": _MAX_WEIGHT, "step": 0.01,
|
||||
"display": "number",
|
||||
"tooltip": "Unified audio weight control. 0 skips audio. 0..1 behaves like the old strength control. Values above 1 repeat the same audio RefMod as extra copies."}),
|
||||
},
|
||||
"optional": {
|
||||
"mods": ("H3_REF_MODS",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("H3_REF_MODS", "STRING")
|
||||
RETURN_NAMES = ("mods", "prompt_hint")
|
||||
FUNCTION = "load"
|
||||
CATEGORY = "H3RefModPicker"
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, mod, **kwargs):
|
||||
if mod not in set(_list_mod_names()):
|
||||
return f"RefMod '{mod}' not found in any configured refmods folder. Run Create H3 RefMod first."
|
||||
return True
|
||||
|
||||
def load(self, mod, weight=1.0, video_weight=1.0, audio_weight=1.0, mods=None, strength=None, audio_strength=None):
|
||||
rows = list(mods) if mods is not None else []
|
||||
if strength is not None:
|
||||
weight = strength
|
||||
if audio_strength is not None:
|
||||
audio_weight = audio_strength
|
||||
if strength is not None or (weight != 1.0 and video_weight == 1.0 and audio_weight == 1.0):
|
||||
video_weight = weight
|
||||
audio_weight = weight
|
||||
loaded = _load_mods(mod)
|
||||
summaries = []
|
||||
hint_rows = list(mods) if mods is not None else []
|
||||
for item in loaded:
|
||||
current_weight = audio_weight if item.kind == "audio" else video_weight
|
||||
clipped = _append_weighted_mod(rows, item, current_weight)
|
||||
summaries.append(f"{item.name}({_weight_display(clipped)})")
|
||||
if clipped > 0.0:
|
||||
hint_rows.append((item, min(1.0, clipped)))
|
||||
print(f"[H3RefModLoader] {mod}: " + ", ".join(summaries))
|
||||
hint = _prompt_hint(hint_rows)
|
||||
if hint:
|
||||
print(f"[H3RefModLoader] prompt_hint: {hint}")
|
||||
return (rows, hint)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3RefModLoader": H3RefModLoader
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3RefModLoader": "Load H3 RefMod Simple"
|
||||
}
|
||||
|
||||
|
||||
@server.PromptServer.instance.routes.get("/h3refmods/refmod-loader/capabilities")
|
||||
async def refmod_loader_capabilities(request):
|
||||
try:
|
||||
name = str(request.query.get("mod", "") or "").strip()
|
||||
if not name:
|
||||
return server.web.json_response({"ok": True, "has_visual": True, "has_audio": True})
|
||||
caps = _mod_capabilities(name)
|
||||
return server.web.json_response({"ok": True, **caps})
|
||||
except Exception as exc:
|
||||
return server.web.json_response({"ok": False, "error": str(exc)}, status=500)
|
||||
@@ -0,0 +1,165 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
import server
|
||||
|
||||
from ..py.refmod_browser import (
|
||||
PREVIEW_EXTS,
|
||||
browser_root,
|
||||
list_browser_dir,
|
||||
paired_mod_path,
|
||||
safe_file_path,
|
||||
)
|
||||
from ..py.refmod_core import H3RefMod, load_refmods_from_file
|
||||
from ..py.refmod_common import _prompt_hint
|
||||
from .refmod_loader import _MAX_WEIGHT, _append_weighted_mod, _weight_display
|
||||
|
||||
_VISUAL_MOD_CACHE: Dict[str, Tuple[H3RefMod, ...]] = {}
|
||||
_VISUAL_MOD_CACHE_MAX = 24
|
||||
|
||||
|
||||
def _load_mods_from_path(mod_path: str) -> List[H3RefMod]:
|
||||
path = safe_file_path(mod_path)
|
||||
if not path.lower().endswith(".safetensors"):
|
||||
raise ValueError("Selected file is not a RefMod safetensors file.")
|
||||
if path in _VISUAL_MOD_CACHE:
|
||||
return list(_VISUAL_MOD_CACHE[path])
|
||||
mods = list(load_refmods_from_file(path[:-len(".safetensors")], device="cpu"))
|
||||
if not any(mod.kind == "audio" for mod in mods):
|
||||
paired_audio = paired_mod_path(path[:-len(".safetensors")], "audio")
|
||||
if paired_audio is not None:
|
||||
mods.extend(load_refmods_from_file(paired_audio[:-len(".safetensors")], device="cpu"))
|
||||
_VISUAL_MOD_CACHE[path] = tuple(mods)
|
||||
if len(_VISUAL_MOD_CACHE) > _VISUAL_MOD_CACHE_MAX:
|
||||
_VISUAL_MOD_CACHE.pop(next(iter(_VISUAL_MOD_CACHE)))
|
||||
return mods
|
||||
|
||||
|
||||
class H3RefModVisualPicker:
|
||||
"""Pick a RefMod by browsing models/refmods thumbnails and append it to a bundle."""
|
||||
|
||||
NAME = "Visual RefMod Picker"
|
||||
CATEGORY = "H3RefModPicker"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"mod_path": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"tooltip": "Selected RefMod safetensors path from the thumbnail browser.",
|
||||
},
|
||||
),
|
||||
"video_weight": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"min": 0.0,
|
||||
"max": _MAX_WEIGHT,
|
||||
"step": 0.01,
|
||||
"tooltip": "Unified video weight control. 0 skips video. 0..1 behaves like the old strength control. Values above 1 repeat the same video RefMod as extra copies.",
|
||||
},
|
||||
),
|
||||
"audio_weight": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"min": 0.0,
|
||||
"max": _MAX_WEIGHT,
|
||||
"step": 0.01,
|
||||
"tooltip": "Unified audio weight control. 0 skips audio. 0..1 behaves like the old audio strength control. Values above 1 repeat the same audio RefMod as extra copies.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"mods": ("H3_REF_MODS",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("H3_REF_MODS", "STRING")
|
||||
RETURN_NAMES = ("mods", "prompt_hint")
|
||||
FUNCTION = "pick"
|
||||
DESCRIPTION = (
|
||||
"Pick a RefMod by browsing models/refmods and its subfolders. "
|
||||
"A matching preview image with the same base name is shown when present; "
|
||||
"otherwise a placeholder is used. When a matching *_Audio file exists "
|
||||
"beside a *_Video or *_Visual file, both are loaded together and appended to the bundle, "
|
||||
"with separate video/audio weights. A weight in 0..1 behaves like the old strength control; "
|
||||
"a weight above 1 repeats the same RefMod as extra copies. Legacy combined RefMods still use one shared video weight."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, **kwargs):
|
||||
return True
|
||||
|
||||
def pick(self, mod_path: str, video_weight: float = 1.0, audio_weight: float = 1.0,
|
||||
mods=None, strength=None, audio_strength=None):
|
||||
rows: List[Tuple[H3RefMod, float]] = []
|
||||
if mods is not None and isinstance(mods, (list, tuple)):
|
||||
rows.extend(list(mods))
|
||||
|
||||
selected = str(mod_path or "").strip()
|
||||
if not selected or selected.lower() == "(none)":
|
||||
hint = _prompt_hint(rows)
|
||||
return (rows, hint)
|
||||
|
||||
if strength is not None:
|
||||
video_weight = strength
|
||||
if audio_strength is not None:
|
||||
audio_weight = audio_strength
|
||||
selected_mods = _load_mods_from_path(selected)
|
||||
hint_rows = list(mods) if mods is not None and isinstance(mods, (list, tuple)) else []
|
||||
summaries = []
|
||||
for mod in selected_mods:
|
||||
weight = audio_weight if mod.kind == "audio" else video_weight
|
||||
clipped = _append_weighted_mod(rows, mod, weight)
|
||||
if clipped > 0.0:
|
||||
hint_rows.append((mod, min(1.0, clipped)))
|
||||
summaries.append(f"{mod.name}({_weight_display(clipped)})")
|
||||
print("[H3RefModVisualPicker] " + ", ".join(summaries))
|
||||
hint = _prompt_hint(hint_rows)
|
||||
if hint:
|
||||
print(f"[H3RefModVisualPicker] prompt_hint: {hint}")
|
||||
return (rows, hint)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"H3RefModVisualPicker": H3RefModVisualPicker}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"H3RefModVisualPicker": "Visual H3 RefMod Picker"}
|
||||
|
||||
|
||||
@server.PromptServer.instance.routes.get("/h3refmods/refmod-browser/root")
|
||||
async def refmod_browser_root(request):
|
||||
try:
|
||||
return server.web.json_response({"ok": True, "root": browser_root()})
|
||||
except Exception as exc:
|
||||
return server.web.json_response({"ok": False, "error": str(exc)}, status=500)
|
||||
|
||||
|
||||
@server.PromptServer.instance.routes.get("/h3refmods/refmod-browser/list")
|
||||
async def refmod_browser_list(request):
|
||||
try:
|
||||
data = list_browser_dir(request.query.get("path", "").strip())
|
||||
return server.web.json_response({"ok": True, **data})
|
||||
except ValueError as exc:
|
||||
return server.web.json_response({"ok": False, "error": str(exc)}, status=404)
|
||||
except Exception as exc:
|
||||
return server.web.json_response({"ok": False, "error": str(exc)}, status=500)
|
||||
|
||||
|
||||
@server.PromptServer.instance.routes.get("/h3refmods/refmod-browser/file")
|
||||
async def refmod_browser_file(request):
|
||||
try:
|
||||
path = safe_file_path(request.query.get("path", "").strip())
|
||||
if os.path.splitext(path)[1].lower() not in PREVIEW_EXTS:
|
||||
return server.web.json_response(
|
||||
{"ok": False, "error": "Unsupported file type"}, status=403
|
||||
)
|
||||
return server.web.FileResponse(path, headers={"Cache-Control": "no-cache"})
|
||||
except ValueError as exc:
|
||||
return server.web.json_response({"ok": False, "error": str(exc)}, status=404)
|
||||
except Exception as exc:
|
||||
return server.web.json_response({"ok": False, "error": str(exc)}, status=500)
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Native H3 reference conditioning with grounded RefMod stacks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io as bytes_io
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as functional
|
||||
from comfy_api.latest import io
|
||||
|
||||
REFERENCE_TYPE = "H3_REFMOD_REFERENCES"
|
||||
|
||||
|
||||
def _mod_rows(mods):
|
||||
if mods is None:
|
||||
return []
|
||||
if hasattr(mods, "ref_block"):
|
||||
return [(mods, 1.0)]
|
||||
if isinstance(mods, (tuple, list)) and mods and hasattr(mods[0], "ref_block"):
|
||||
mods = [mods]
|
||||
rows = []
|
||||
for row in mods:
|
||||
if not isinstance(row, (tuple, list)) or len(row) not in (2, 3):
|
||||
raise ValueError("mods must contain (RefMod, strength) rows.")
|
||||
mod, strength = row[:2]
|
||||
if mod is None:
|
||||
continue
|
||||
if not callable(getattr(mod, "ref_block", None)):
|
||||
raise ValueError("mods contains an unsupported RefMod object.")
|
||||
strength = float(strength)
|
||||
if not math.isfinite(strength):
|
||||
raise ValueError("RefMod strengths must be finite.")
|
||||
if strength > 0:
|
||||
rows.append((mod, min(1.0, strength)))
|
||||
return rows
|
||||
|
||||
|
||||
def _decoded_frames(video_vae, latent):
|
||||
with torch.no_grad():
|
||||
frames = video_vae.decode(latent)
|
||||
if frames.ndim == 5 and frames.shape[0] == 1:
|
||||
frames = frames[0]
|
||||
if frames.ndim != 4 or frames.shape[-1] != 3 or frames.shape[0] < 1:
|
||||
raise ValueError(f"H3 video VAE returned invalid images: {tuple(frames.shape)}")
|
||||
return frames.detach().float().clamp(0, 1).cpu()
|
||||
|
||||
|
||||
def _encoder_frames(mod, video_vae, reference_fps):
|
||||
times = getattr(mod, "enc_times", None)
|
||||
path = getattr(mod, "path", "")
|
||||
packed_frames = getattr(mod, "enc_frames", None)
|
||||
is_stack = getattr(mod, "source", "") == "stack"
|
||||
if times and (packed_frames or path) and (is_stack or abs(getattr(mod, "enc_fps", 0) - reference_fps) < 1e-6):
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
def unpack(get_tensor):
|
||||
frames = []
|
||||
for index in range(len(times)):
|
||||
packed = get_tensor(f"enc_{index}").numpy().tobytes()
|
||||
with Image.open(bytes_io.BytesIO(packed)) as image:
|
||||
frames.append(torch.from_numpy(np.array(image.convert("RGB"))).float() / 255)
|
||||
return torch.stack(frames), list(times), "stored pictures"
|
||||
|
||||
if packed_frames:
|
||||
return unpack(packed_frames.__getitem__)
|
||||
from safetensors import safe_open
|
||||
filename = path if path.endswith(".safetensors") else path + ".safetensors"
|
||||
with safe_open(filename, framework="pt", device="cpu") as handle:
|
||||
return unpack(handle.get_tensor)
|
||||
if video_vae is None:
|
||||
raise ValueError(f"Connect video_vae to reconstruct visual RefMod '{mod.name}'.")
|
||||
if is_stack:
|
||||
frames = [_decoded_frames(video_vae, mod.latent[:, :, index:index + 1])[:1]
|
||||
for index in range(mod.latent.shape[2])]
|
||||
return torch.cat(frames), [float(index) for index in range(len(frames))], "decoded stack"
|
||||
frames = _decoded_frames(video_vae, mod.latent)
|
||||
if mod.kind == "image":
|
||||
return frames[:1], [0.0], "decoded image"
|
||||
times = [index / 2 for index in range(math.ceil(frames.shape[0] * 2 / reference_fps))]
|
||||
indices = [min(round(time * reference_fps), frames.shape[0] - 1) for time in times]
|
||||
return frames[indices], times, "decoded video"
|
||||
|
||||
|
||||
def _visual_item(mod, frames, times, strength, stack_pictures, visual_kind=None):
|
||||
if strength < 1:
|
||||
samples = frames.movedim(-1, 1)
|
||||
blurred = functional.adaptive_avg_pool2d(samples, (max(1, mod.latent_h // 8),
|
||||
max(1, mod.latent_w // 8)))
|
||||
blurred = functional.interpolate(blurred, size=samples.shape[-2:], mode="bilinear", align_corners=False)
|
||||
frames = (strength * samples + (1 - strength) * blurred).movedim(1, -1)
|
||||
if (visual_kind or mod.kind) == "image":
|
||||
return {"type": "image", "data": frames[:1]}, 1
|
||||
if getattr(mod, "source", "") == "stack":
|
||||
count = frames.shape[0]
|
||||
if stack_pictures == "every 4th":
|
||||
indices = list(range(0, count, 4))
|
||||
frames = frames[indices]
|
||||
times = [index / 2 for index in range(len(indices))]
|
||||
else:
|
||||
indices = ([round(index * (count - 1) / 7) for index in range(8)]
|
||||
if stack_pictures == "up to 8" and count > 8 else list(range(count)))
|
||||
frames = frames[indices].repeat_interleave(2, dim=0)
|
||||
times = [index + offset for index in range(len(indices)) for offset in (0.0, 0.5)]
|
||||
shown = len(indices)
|
||||
else:
|
||||
shown = frames.shape[0]
|
||||
return {"type": "video", "data": frames, "timestamps": times}, shown
|
||||
|
||||
|
||||
class H3RefModReferences(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="H3RefModReferences",
|
||||
display_name="Collect H3 References",
|
||||
category="H3RefModPicker",
|
||||
inputs=[
|
||||
io.Custom(REFERENCE_TYPE).Input("references", optional=True,
|
||||
tooltip="Optional references from another collector."),
|
||||
io.Autogrow.Input("ref_images", optional=True, template=io.Autogrow.TemplatePrefix(
|
||||
input=io.Image.Input("ref_image"), prefix="ref_image_", min=0, max=9)),
|
||||
io.Autogrow.Input("ref_videos", optional=True, template=io.Autogrow.TemplatePrefix(
|
||||
input=io.Image.Input("ref_video", tooltip="Video frame batch at 24 fps."),
|
||||
prefix="ref_video_", min=0, max=3)),
|
||||
io.Autogrow.Input("ref_video_audios", optional=True, template=io.Autogrow.TemplatePrefix(
|
||||
input=io.Audio.Input("ref_video_audio", tooltip="Soundtrack for the same-numbered video input."),
|
||||
prefix="ref_video_audio_", min=0, max=3)),
|
||||
io.Autogrow.Input("ref_audios", optional=True, template=io.Autogrow.TemplatePrefix(
|
||||
input=io.Audio.Input("ref_audio"), prefix="ref_audio_", min=0, max=3)),
|
||||
],
|
||||
outputs=[io.Custom(REFERENCE_TYPE).Output("references")],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, references=None, ref_images=None, ref_videos=None,
|
||||
ref_video_audios=None, ref_audios=None):
|
||||
entries = list(references or [])
|
||||
for name, image in (ref_images or {}).items():
|
||||
if image is not None:
|
||||
entries.append({"kind": "image", "data": image, "name": name})
|
||||
videos = ref_videos or {}
|
||||
soundtracks = ref_video_audios or {}
|
||||
for name, soundtrack in soundtracks.items():
|
||||
if soundtrack is not None and videos.get("ref_video_" + name.rsplit("_", 1)[-1]) is None:
|
||||
raise ValueError(f"{name} needs its same-numbered ref_video input; use ref_audio for standalone audio.")
|
||||
for name, frames in videos.items():
|
||||
if frames is not None:
|
||||
entries.append({"kind": "video", "data": frames, "name": name,
|
||||
"audio": soundtracks.get("ref_video_audio_" + name.rsplit("_", 1)[-1])})
|
||||
for name, audio in (ref_audios or {}).items():
|
||||
if audio is not None:
|
||||
entries.append({"kind": "audio", "data": audio, "name": name})
|
||||
return io.NodeOutput(entries)
|
||||
|
||||
|
||||
class _ReferencePresentation:
|
||||
def __init__(self):
|
||||
self.items = []
|
||||
|
||||
def tokenize(self, prompt, minimax_ref_items):
|
||||
self.items = minimax_ref_items
|
||||
return None
|
||||
|
||||
def encode_from_tokens_scheduled(self, tokens):
|
||||
return [[None, {}]]
|
||||
|
||||
|
||||
def _native_references(references, video_vae, audio_vae, width, height, length, ref_image_size):
|
||||
from comfy_extras.nodes_minimax_h3 import MiniMaxH3ReferenceToVideo
|
||||
|
||||
images, videos, soundtracks, audios = {}, {}, {}, {}
|
||||
names = []
|
||||
for kind in ("image", "video", "audio"):
|
||||
for index, entry in enumerate(references or []):
|
||||
if entry.get("kind") != kind:
|
||||
continue
|
||||
name = entry["name"]
|
||||
if kind == "image":
|
||||
images[f"ref_image_{index}"] = entry["data"]
|
||||
elif kind == "video":
|
||||
videos[f"ref_video_{index}"] = entry["data"]
|
||||
if entry.get("audio") is not None:
|
||||
soundtracks[f"ref_video_audio_{index}"] = entry["audio"]
|
||||
names.append(name + " soundtrack")
|
||||
else:
|
||||
audios[f"ref_audio_{index}"] = entry["data"]
|
||||
names.append(name)
|
||||
presentation = _ReferencePresentation()
|
||||
output = MiniMaxH3ReferenceToVideo.execute(
|
||||
presentation, "", width, height, length, ref_image_size=ref_image_size,
|
||||
vae=video_vae, audio_vae=audio_vae, ref_images=images, ref_videos=videos,
|
||||
ref_video_audios=soundtracks, ref_audios=audios)
|
||||
conditioning, latent = output.result
|
||||
return presentation.items, list(conditioning[0][1].get("minimax_refs", [])), names, latent
|
||||
|
||||
|
||||
class H3RefModsToVideo(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="H3RefModsToVideo",
|
||||
display_name="MiniMax H3 RefMods to Video",
|
||||
category="model/conditioning/minimax",
|
||||
description="Reference-to-Video with grounded RefMods and optional collected media references.",
|
||||
inputs=[
|
||||
io.Clip.Input("clip"),
|
||||
io.Vae.Input("video_vae", optional=True,
|
||||
tooltip="Encodes raw pictures/videos and reconstructs RefMods without stored encoder pictures."),
|
||||
io.Vae.Input("audio_vae", optional=True,
|
||||
tooltip="Encodes raw reference audio. Audio RefMods are already encoded."),
|
||||
io.Custom(REFERENCE_TYPE).Input("references", optional=True),
|
||||
io.MultiType.Input("mods", types=[io.Custom("H3_REFMOD"), io.Custom("H3_REF_MODS")], optional=True),
|
||||
io.String.Input("prompt", multiline=True, dynamic_prompts=True),
|
||||
io.Int.Input("width", default=1344, min=32, max=8192, step=32),
|
||||
io.Int.Input("height", default=768, min=32, max=8192, step=32),
|
||||
io.Int.Input("length", default=124, min=5, max=3600, step=17),
|
||||
io.Combo.Input("ref_image_size", options=["match", "max"], default="match"),
|
||||
io.Combo.Input("stack_pictures", options=["every 4th", "up to 8", "all"], default="every 4th",
|
||||
tooltip="Pictures shown to the text encoder for a stacked RefMod. All latent frames remain applied."),
|
||||
],
|
||||
outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output(),
|
||||
io.String.Output("reference_map")],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, clip, prompt, width, height, length, video_vae=None, audio_vae=None,
|
||||
references=None, mods=None, ref_image_size="match", stack_pictures="every 4th"):
|
||||
if stack_pictures not in ("every 4th", "up to 8", "all"):
|
||||
raise ValueError("Unknown stack_pictures mode.")
|
||||
items, blocks, names, latent = _native_references(
|
||||
references, video_vae, audio_vae, width, height, length, ref_image_size)
|
||||
counters = {"image": 0, "video": 0, "audio": 0}
|
||||
labels = {"image": "Picture", "video": "Video", "audio": "Audio"}
|
||||
mapping = []
|
||||
|
||||
def append_label(item, name):
|
||||
kind = item["type"]
|
||||
counters[kind] += 1
|
||||
mapping.append(f"<{labels[kind]} {counters[kind]}> = {name}")
|
||||
|
||||
for item, name in zip(items, names):
|
||||
append_label(item, name)
|
||||
decoded = {}
|
||||
for mod, strength in _mod_rows(mods):
|
||||
block = mod.ref_block(strength)
|
||||
if block is None:
|
||||
continue
|
||||
block = dict(block)
|
||||
kind = block["kind"]
|
||||
name = mod.name
|
||||
if kind not in ("image", "video", "video_audio", "audio"):
|
||||
raise ValueError(f"Unsupported RefMod kind: {kind}")
|
||||
if kind == "audio" or (block.get("audio_latent") is not None and block.get("ref_audio_t", 0) > 0):
|
||||
item = {"type": "audio"}
|
||||
items.append(item)
|
||||
append_label(item, name + (" soundtrack" if kind != "audio" else ""))
|
||||
if kind != "audio":
|
||||
if id(mod) not in decoded:
|
||||
decoded[id(mod)] = _encoder_frames(mod, video_vae, 24.0)
|
||||
frames, times, _source = decoded[id(mod)]
|
||||
visual_kind = "video" if kind == "video_audio" else kind
|
||||
item, _shown = _visual_item(mod, frames, times, strength, stack_pictures, visual_kind)
|
||||
items.append(item)
|
||||
append_label(item, name)
|
||||
block["refmod"] = True
|
||||
blocks.append(block)
|
||||
if video_vae is not None or audio_vae is not None:
|
||||
import comfy.model_management
|
||||
comfy.model_management.soft_empty_cache()
|
||||
tokens = clip.tokenize(prompt, minimax_ref_items=items)
|
||||
conditioning = clip.encode_from_tokens_scheduled(tokens)
|
||||
output = []
|
||||
for embedding, metadata in conditioning:
|
||||
if items and "minimax_token_tags" not in metadata:
|
||||
raise ValueError("Connect a MiniMax H3 CLIP that supports reference token tags.")
|
||||
metadata = dict(metadata)
|
||||
if blocks:
|
||||
metadata["minimax_refs"] = list(metadata.get("minimax_refs", [])) + blocks
|
||||
output.append([embedding, metadata])
|
||||
reference_map = "\n".join(mapping) or "No references."
|
||||
return io.NodeOutput(output, latent, reference_map)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3RefModsToVideo": H3RefModsToVideo,
|
||||
"H3RefModReferences": H3RefModReferences,
|
||||
}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3RefModsToVideo": "MiniMax H3 RefMods to Video",
|
||||
"H3RefModReferences": "Collect H3 References",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user