Files
DumasNodes/py/refmod_core.py
T
chris.dumas c9eb575e2f Squashed 'vendor/dumas_h3_refmod_picker/' content from commit c732f8b
git-subtree-dir: vendor/dumas_h3_refmod_picker
git-subtree-split: c732f8b8893bf4f00c6974c9b9941c763496fe48
2026-10-20 13:43:48 +00:00

959 lines
43 KiB
Python

"""
core.py — "RefMod" adapter for MiniMax H3 (no training required)
The reference-video path of MiniMax H3 works by injecting *reference tokens*
into the packed sequence: the ref is VAE-encoded, patchified, projected and
placed on the 3D RoPE grid, then every DiT block attends to it. A video ref
is expensive because it contributes thousands of tokens.
Mode naming: ``encode`` (was ``full``) = straight full-res VAE encode;
``training`` (was ``pooled``) = compressed grid refined by gradient steps
(the only mode that "trains" — the refinement loop). The old names are
accepted everywhere as legacy aliases so saved workflows and mods keep
working.
A RefMod is the same reference, compressed to a handful of tokens:
* the ref is VAE-encoded to its full latent [1, 24, T, H, W],
* the latent is average-pooled to a tiny grid (default 4x4) and a few
latent frames, so the patchified token count drops to ~4-16,
* optionally the small latent is refined with a few gradient steps that
reconstruct the full latent (still no model weights involved).
At generation time the pooled latent is handed back to the model through the
native ``refs`` payload, so it flows through the exact same per-block
attention machinery as a full reference — the only thing that changes is the
token budget.
This is the MiniMax H3 analog of the LTX "Mod" concept tokens + per-block
injection: instead of a trained hypernetwork predicting AdaLN deltas, the
model's own cross-attention over the compressed ref tokens does the work.
"""
from __future__ import annotations
import json
import math
import os
import tempfile
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors import safe_open
from safetensors.torch import load_file, save_file
# Metadata key embedded in the safetensors header (keeps a mod in one file,
# so it can be shared/uploaded as a single artifact).
META_KEY = "refmod_meta"
# What a mod is standing in for. This is documentation + a routing hint, not
# a hard constraint on the tensors — but it drives two things: (1) the
# "identity in training mode" warning in nodes.py, and (2) the loader's
# prompt_hint output, which merges each loaded mod's concept_type +
# description into a string you can concat onto your CLIP Text Encode prompt.
# The model only reads these labels through the text prompt, so prompt_hint is
# the only place they are surfaced.
CONCEPT_TYPES = (
"generic", # unspecified / mixed
"identity", # a specific person/character — use mode="encode" for this
"pose_motion", # a pose, dance, gesture, camera move
"clothing", # an outfit / garment, decoupled from who's wearing it
"background", # environment / set / location plate
"voice",
"singing",
"music_style",
"sound_fx",
"ambience",
"style", # look/grade/animation style, not a concrete subject
)
AUDIO_CONCEPT_TYPES = ("voice", "singing", "music_style", "sound_fx", "ambience")
def _blur_latent(z: torch.Tensor, factor: int = 8) -> torch.Tensor:
"""Heavy spatial low-pass (downsample then upsample) used as the
weakening target for ``ref_block``'s ``strength`` below ``1.0``.
Originally this mixed toward ``torch.randn()`` noise, mirroring what the
docstring below describes as the model's own ``visual_cond_noise_aug``
mechanism. In practice, at the retention range people actually use this
dial for (0.15-0.7, i.e. large noise fractions), it produces a visible
woven/static texture instead of a gracefully weaker reference: a VAE
latent's channels are correlated, so raw iid noise isn't read as "less
reference," it's read as real-but-garbled content and gets rendered as
an actual (wrong) texture. ``visual_cond_noise_aug`` in the model's own
training is likely a small-magnitude robustness augmentation, not
something swept this far — so copying its math doesn't reproduce its
behavior at these strengths. A blurred copy stays on the latent
manifold (smooth, plausible) while still discarding detail as strength
drops, which is what "weaker reference" should actually look like.
"""
if z.dim() == 4:
b, c, stereo, t = z.shape
if t <= 1:
return z
flat = z.reshape(b * c * stereo, 1, t).float()
down = F.adaptive_avg_pool1d(flat, max(1, t // factor))
up = F.interpolate(down, size=t, mode="linear", align_corners=False)
return up.reshape_as(z).to(z.dtype)
if z.dim() != 5:
raise ValueError(f"Unsupported RefMod latent shape: {tuple(z.shape)}")
t, h, w = z.shape[2], z.shape[3], z.shape[4]
sh, sw = max(1, h // factor), max(1, w // factor)
down = F.adaptive_avg_pool3d(z.float(), (t, sh, sw))
up = F.interpolate(down, size=(t, h, w), mode="trilinear", align_corners=False)
return up.to(z.dtype)
def read_refmod_meta(path_no_ext: str) -> Optional[Dict]:
"""
Read the metadata block from a mod/preset file without loading tensors.
Prefers the metadata embedded in the safetensors header; falls back to
the legacy sidecar ``{path}.json`` written by older versions.
"""
try:
with safe_open(path_no_ext + ".safetensors", framework="pt") as f:
meta = f.metadata()
if meta:
for key in (META_KEY, "audio_refmod_meta"):
if key in meta:
return json.loads(meta[key])
except Exception:
pass
jpath = path_no_ext + ".json"
if os.path.isfile(jpath):
try:
with open(jpath) as f:
return json.load(f)
except Exception:
return None
return None
def refmod_bundle_members(meta: Dict) -> List[Dict]:
if not isinstance(meta, dict) or meta.get("_format_version") != 5 or meta.get("kind") != "bundle":
raise ValueError("Unsupported RefMod bundle format.")
refs = meta.get("members")
if not isinstance(refs, list) or not refs or len(refs) > 256:
raise ValueError("A RefMod bundle must contain 1-256 members.")
for ref in refs:
if not isinstance(ref, dict) or ref.get("kind") not in ("image", "video", "audio"):
raise ValueError("Invalid RefMod bundle member.")
return refs
def refmod_capabilities(meta: Optional[Dict]) -> Dict[str, object]:
caps: Dict[str, object] = {
"kind": "unknown",
"has_visual": False,
"has_audio": False,
"member_count": 0,
}
if not isinstance(meta, dict):
return caps
kind = str(meta.get("kind", "") or "unknown")
caps["kind"] = kind
if kind == "bundle":
refs = refmod_bundle_members(meta)
caps["member_count"] = len(refs)
caps["has_visual"] = any(ref.get("kind") in ("image", "video") for ref in refs)
caps["has_audio"] = any(ref.get("kind") == "audio" for ref in refs)
elif kind in ("image", "video"):
caps["has_visual"] = True
caps["has_audio"] = int(meta.get("ref_audio_t", 0) or 0) > 0
caps["member_count"] = 1 + (1 if caps["has_audio"] else 0)
elif kind == "audio":
caps["has_audio"] = True
caps["member_count"] = 1
return caps
def _standalone_audio_member_meta(meta: Dict) -> Dict:
name = str(meta.get("name", "refmod") or "refmod")
if not name.lower().endswith("_audio"):
name = f"{name}_Audio"
return {
**meta,
"name": name,
"kind": "audio",
"latent_h": 0,
"latent_w": 0,
"latent_t": int(meta.get("ref_audio_t", 0) or 0),
"concept_type": str(meta.get("audio_concept_type", meta.get("concept_type", "voice")) or "voice"),
"audio_concept_type": str(meta.get("audio_concept_type", "") or ""),
"ref_audio_t": int(meta.get("ref_audio_t", 0) or 0),
"sample_rate": int(meta.get("sample_rate", 32000) or 32000),
}
def load_refmods_from_file(path_no_ext: str, device: str = "cpu") -> List["H3RefMod"]:
meta = read_refmod_meta(path_no_ext)
if not isinstance(meta, dict):
raise ValueError(
f"{path_no_ext}.safetensors has no RefMod metadata "
f"(header key '{META_KEY}' or sidecar .json missing)."
)
if meta.get("kind") == "bundle":
loaded: List[H3RefMod] = []
refs = refmod_bundle_members(meta)
with safe_open(path_no_ext + ".safetensors", framework="pt", device=device) as f:
for idx, member_meta in enumerate(refs):
latent = f.get_tensor(f"ref_{idx}").clone()
enc_frames = _read_encoder_images(member_meta, f.get_tensor)
loaded.append(H3RefMod.from_metadata(member_meta, latent, enc_frames=enc_frames))
return loaded
tensors = load_file(path_no_ext + ".safetensors", device=device)
if "latent" not in tensors:
raise ValueError(f"{path_no_ext}.safetensors is missing its latent tensor.")
latent = tensors["latent"].clone()
audio_latent = tensors["audio_latent"].clone() if "audio_latent" in tensors else None
if meta.get("kind") == "audio":
return [H3RefMod.from_metadata(meta, latent)]
visual_meta = dict(meta)
visual_meta["ref_audio_t"] = 0
visual_meta["audio_concept_type"] = ""
enc_frames = _read_encoder_images(meta, tensors.__getitem__)
loaded = [H3RefMod.from_metadata(visual_meta, latent, enc_frames=enc_frames)]
if audio_latent is not None and int(meta.get("ref_audio_t", 0) or 0) > 0:
loaded.append(H3RefMod.from_metadata(_standalone_audio_member_meta(meta), audio_latent))
return loaded
def save_refmod_bundle(path_no_ext: str, name: str, mods: List["H3RefMod"]) -> str:
unique = list({id(mod): mod for mod in mods}.values())
if not unique:
raise ValueError("RefMod bundle is empty.")
if sum(bool(mod.enc_frames) for mod in unique) > 1:
raise ValueError("Save RefMods with encoder pictures in separate files when more than one member has pictures.")
metadata = {
"_format_version": 5,
"kind": "bundle",
"name": str(name or "refmod"),
"members": [mod.metadata() for mod in unique],
}
tensors = {
f"ref_{idx}": mod.latent.detach().cpu().contiguous().clone()
for idx, mod in enumerate(unique)
}
for mod in unique:
tensors.update({key: value.contiguous() for key, value in mod.enc_frames.items()})
destination = path_no_ext + ".safetensors"
directory = os.path.dirname(destination) or "."
os.makedirs(directory, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix=".refmod-", suffix=".tmp", dir=directory)
os.close(fd)
try:
save_file(tensors, temporary, metadata={META_KEY: json.dumps(metadata)})
os.replace(temporary, destination)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
return destination
def _read_encoder_images(meta, get_tensor):
record = meta.get("encoder_images") or {}
times = record.get("times", meta.get("enc_times", []))
prefix = record.get("prefix", "enc_")
return {f"enc_{index}": get_tensor(f"{prefix}{index}").detach().cpu().clone()
for index in range(len(times))}
def pack_encoder_images(mod, frames, times, fps=24.0):
import io as bytes_io
from PIL import Image
if frames.shape[0] != len(times):
raise ValueError("Encoder picture count does not match its timestamps.")
packed = {}
for index, frame in enumerate(frames):
buffer = bytes_io.BytesIO()
pixels = (frame.detach().cpu().float().clamp(0, 1) * 255).round().to(torch.uint8).numpy()
Image.fromarray(pixels).save(buffer, "JPEG", quality=95)
packed[f"enc_{index}"] = torch.frombuffer(bytearray(buffer.getvalue()), dtype=torch.uint8)
mod.enc_frames = packed
mod.enc_times = list(times)
mod.enc_fps = float(fps)
# ═══════════════════════════════════════════════════════════════════════════
# Latent compression
# ═══════════════════════════════════════════════════════════════════════════
MODE_ALIASES = {
"full": "encode",
"pooled": "training",
"Full Reference": "encode",
"Compressed Reference": "training",
}
def normalize_mode(mode: str) -> str:
"""Map legacy mode names (``full``/``pooled``) to the current ones."""
return MODE_ALIASES.get(mode, mode)
def aspect_grid(pool_h: int, pool_w: int, aspect: float):
"""Even pool grid dims that match ``aspect`` (h/w) within the dial caps.
The DiT patches each 2x2 latent cell into one token and the rope grid is
area-normalized per axis, so a pooled grid that ignores the source aspect
squishes the subject: a portrait latent (tall person) forced into a square
16x16 grid is compressed 5x in height but only 3x in width, and comes out
"fat". The official ref2video node never pools — it feeds the model
aspect-correct latent dims, which is what the DiT was trained on.
The long edge follows the user's dial (``max(pool_h, pool_w)``) and the
short edge is derived from the source aspect, both rounded to even for the
2x2 patch. Square sources keep the exact dial value (16x16 stays 16x16).
"""
long_edge = max(pool_h, pool_w)
if aspect >= 1.0:
h, w = long_edge, long_edge / aspect
else:
w, h = long_edge, long_edge * aspect
h = max(2, round(h / 2) * 2)
w = max(2, round(w / 2) * 2)
return h, w
def pool_latent(
z: torch.Tensor,
latent_t: int,
latent_h: int,
latent_w: int,
) -> torch.Tensor:
"""
Average-pool a VAE latent ``[1, 24, T, H, W]`` down to a tiny grid.
``latent_h/latent_w`` are the *latent* grid dims (the DiT patches each 2x2
latent cell into one token), so a 4x4 latent = 2x2 = 4 tokens per frame.
Dims must be even (the DiT's 2x2 patch). Pooling runs in fp32; the result
keeps the input dtype.
"""
if z.shape[2] == latent_t and z.shape[3] == latent_h and z.shape[4] == latent_w:
return z
if latent_h % 2 != 0 or latent_w % 2 != 0:
raise ValueError(f"pool_h/pool_w must be even (got {latent_h}x{latent_w})")
pooled = F.adaptive_avg_pool3d(
z.float(), (latent_t, latent_h, latent_w)
)
return pooled.to(z.dtype)
def optimize_latent(
z_small: torch.Tensor,
z_full: torch.Tensor,
steps: int = 150,
lr: float = 00.02,
device: Optional[torch.device] = None,
progress_every: int = 0,
) -> torch.Tensor:
"""
Model-free refinement of the compressed latent.
Optimizes the small latent so its trilinearly upsampled reconstruction
matches the full reference latent. This pulls the pooled representation
closer to what the DiT would see from the full ref, with nothing but the
tiny latent trainable (~1-2K params) and no diffusion model loaded.
``progress_every`` > 0 prints a ``[RefMod] identity <step>/<steps>`` line
every N steps so long pooled-mode refinement isn't silent (the default
``identity=500`` takes a while per ref on CPU/VRAM-bound setups).
Returns the refined latent detached, same shape/dtype as ``z_small``.
"""
if steps <= 0:
return z_small
device = device or z_full.device
# Node execution runs under ComfyUI's global inference mode, which would
# disable the autograd this refine loop needs. Re-enable it for this
# scope and clone the inputs to drop the inference flag.
with torch.inference_mode(False), torch.set_grad_enabled(True):
target = z_full.clone().float().to(device)
param = nn.Parameter(z_small.clone().float().to(device))
opt = torch.optim.Adam([param], lr=lr)
size = tuple(target.shape[2:])
for i in range(steps):
opt.zero_grad()
up = F.interpolate(param, size=size, mode="trilinear", align_corners=False)
loss = F.mse_loss(up, target)
loss.backward()
opt.step()
if progress_every and (i + 1) % progress_every == 0:
print(f"[RefMod] identity {i + 1}/{steps}")
# materialize inside the scope so the result is a normal tensor, not
# an inference-mode tensor (it gets stored in the mod and reused)
refined = param.detach().to(z_small.dtype)
return refined
def optimize_latent_multi(z_init, targets, steps=150, lr=0.02, device=None,
progress_every=0, strategy="grouped"):
"""Mean reconstruction objective with bounded activation memory."""
if not targets or steps <= 0:
return z_init
if strategy not in ("grouped", "stream", "resident"):
raise ValueError("Unknown multi-reference optimization strategy.")
device = device or z_init.device
with torch.inference_mode(False), torch.set_grad_enabled(True):
count = len(targets)
if strategy == "grouped":
groups = {}
for target in targets:
key = tuple(target.shape)
value = target.detach().to(device="cpu", dtype=torch.float32).clone()
if key in groups:
groups[key][0].add_(value)
groups[key][1] += 1
else:
groups[key] = [value, 1]
grouped = []
for total, group_count in groups.values():
grouped.append((total.div_(group_count), group_count))
elif strategy == "resident":
resident = [target.detach().float().to(device) for target in targets]
param = nn.Parameter(z_init.clone().float().to(device))
opt = torch.optim.Adam([param], lr=lr)
for step_idx in range(steps):
opt.zero_grad()
loss = None
if strategy == "grouped":
total_weight = 0
for target_mean, group_count in grouped:
target = target_mean.to(device)
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
align_corners=False)
contribution = F.mse_loss(up, target)
loss = contribution * group_count if loss is None else loss + contribution * group_count
total_weight += group_count
loss = loss / total_weight
elif strategy == "stream":
for target_cpu in targets:
target = target_cpu.detach().float().to(device)
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
align_corners=False)
contribution = F.mse_loss(up, target)
loss = contribution if loss is None else loss + contribution
loss = loss / count
else:
for target in resident:
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
align_corners=False)
contribution = F.mse_loss(up, target)
loss = contribution if loss is None else loss + contribution
loss = loss / count
loss.backward()
opt.step()
if progress_every and (step_idx + 1) % progress_every == 0:
print(f"[RefMod] identity {step_idx + 1}/{steps}")
refined = param.detach().to(z_init.dtype)
return refined
# ═══════════════════════════════════════════════════════════════════════════
# Per-frame strength curve
# ═══════════════════════════════════════════════════════════════════════════
# Direction names describe where the *concept* shows up in the output, which
# is the mirror of the strength envelope over the ref's own latent timeline:
# concept_at_end locks the ref's literal footage at the START and releases it
# toward the end (was "decrease"), concept_at_start opens free and locks onto
# the ref near the END (was "increase"). shape is how the envelope travels
# between its endpoints; value is the non-zero endpoint ("user input").
CURVE_DIRECTIONS = ("constant", "concept_at_start", "concept_at_middle",
"concept_at_end", "concept_at_ends")
CURVE_SHAPES = ("linear", "ease", "sigmoid", "tanh", "quadratic", "cubic",
"exponential", "stair", "elastic", "bump", "dip")
# old single-combo preset names -> (direction, shape, value) so workflows
# saved before the curve was split still resolve
_LEGACY_CURVES = {
"flat": ("constant", "linear", 1.0),
"fade_in": ("concept_at_start", "linear", 1.0),
"fade_out": ("concept_at_end", "linear", 1.0),
"bump": ("concept_at_start", "bump", 1.0),
"dip": ("constant", "dip", 1.0),
}
def _ease(shape: str, x: float) -> float:
"""Shape a progress ``x`` in [0, 1] per the easing name (may overshoot)."""
if shape == "linear":
return x
if shape == "ease": # smoothstep
return x * x * (3.0 - 2.0 * x)
if shape == "sigmoid": # logistic S-curve
return 1.0 / (1.0 + math.exp(-12.0 * (x - 0.5)))
if shape == "tanh": # sharper S-curve (steeper knee than sigmoid)
return 0.5 * (math.tanh(8.0 * (x - 0.5)) + 1.0)
if shape == "quadratic":
return x * x
if shape == "cubic":
return x * x * x
if shape == "exponential":
return 2.0 ** x - 1.0
if shape == "stair":
return min(1.0, math.floor(x * 4) / 3.0)
if shape == "elastic":
if x <= 0.0:
return 0.0
if x >= 1.0:
return 1.0
return 2.0 ** (-10.0 * x) * math.sin((x * 10.0 - 0.75) * (2.0 * math.pi / 3.0)) + 1.0
if shape == "bump":
return 1.0 - abs(2.0 * x - 1.0)
if shape == "dip":
return abs(2.0 * x - 1.0)
return x
def curve_strengths(spec, t: int) -> Optional[List[float]]:
"""Resolve a curve spec to ``t`` per-frame strength multipliers in [0, 1].
Kept as plain Python so the Apply node exposes the curve as normal
combo/float widgets instead of depending on ComfyUI's new Curve widget.
Accepts:
* a ``(direction, shape, value)`` tuple — direction is ``constant``
(one strength everywhere), ``concept_at_start`` (0 -> value, a
crescent — the concept shows early; legacy ``increase``),
``concept_at_end`` (value -> 0, a decrescent — the concept pops at
the end; legacy ``decrease``), ``concept_at_middle`` ([0..1..0] —
concept peaks mid-timeline) or ``concept_at_ends`` ([1..0..1] —
concept at both ends); shape is how the envelope travels between
its endpoints (see ``_ease``); value is the non-zero endpoint
(1.0 = full strength there);
* a legacy preset name (``flat``/``fade_in``/``fade_out``/``bump``/
``dip``) from before the split;
* a list of per-frame floats (len == t), used as-is;
* a list of ``(x, y)`` control points (x in [0, 1]), linearly
interpolated across the frames;
* any object with ``interp(x) -> float`` (legacy CurveInput saved in an
old workflow).
Returns None for flat/no curve so the caller keeps its single-strength
path unchanged (a ``constant`` curve at value 1.0 included).
"""
if t <= 1 or spec is None or spec == "":
return None
if isinstance(spec, str):
legacy = _LEGACY_CURVES.get(spec)
return curve_strengths(legacy, t) if legacy is not None else None
if (isinstance(spec, tuple) and len(spec) == 3
and isinstance(spec[0], str) and isinstance(spec[1], str)):
direction, shape, value = spec
value = float(value)
if direction == "constant":
# linear = flat at ``value`` (1.0 == no curve); the other shapes
# modulate around the constant, so constant + dip is a trough,
# constant + bump is a peak (both endpoints equal)
if shape == "linear":
return None if value >= 1.0 else [value] * t
p = [_ease(shape, i / (t - 1)) for i in range(t)]
return [max(0.0, min(1.0, value * y)) for y in p]
p = [_ease(shape, i / (t - 1)) for i in range(t)]
# legacy "increase"/"decrease" (pre-rename workflows) map onto the same
# envelopes as the concept-placement names
if direction in ("increase", "concept_at_start"):
return [max(0.0, min(1.0, value * y)) for y in p]
if direction in ("decrease", "concept_at_end"):
return [max(0.0, min(1.0, value * (1.0 - y))) for y in p]
if direction == "concept_at_middle": # [0..1..0]: concept peaks mid-timeline
return [max(0.0, min(1.0, value * (1.0 - abs(2.0 * y - 1.0)))) for y in p]
if direction == "concept_at_ends": # [1..0..1]: concept at both ends
return [max(0.0, min(1.0, value * abs(2.0 * y - 1.0))) for y in p]
return None # unknown direction: treat as flat
if isinstance(spec, (list, tuple)):
if len(spec) == t and all(isinstance(v, (int, float)) for v in spec):
return [float(v) for v in spec]
pts = [(float(x), float(y)) for x, y in spec
if isinstance(x, (int, float)) and isinstance(y, (int, float))]
if pts:
pts = sorted(pts, key=lambda p: p[0])
out = []
for i in range(t):
x = i * (1.0 / (t - 1))
if x <= pts[0][0]:
out.append(pts[0][1])
elif x >= pts[-1][0]:
out.append(pts[-1][1])
else:
for (x0, y0), (x1, y1) in zip(pts, pts[1:]):
if x0 <= x <= x1:
out.append(y0 + (y1 - y0) * (x - x0) / (x1 - x0))
break
return out
interp = getattr(spec, "interp", None)
if callable(interp):
return [float(interp(i / (t - 1))) for i in range(t)]
return None
def curve_value_at(spec, x: float) -> float:
"""Per-progress strength of a curve spec at one point ``x`` in [0, 1].
The continuous analog of ``curve_strengths`` — same direction/shape/value
math, but evaluated at a single progress value instead of discretized over
frames. The step-curve node uses it to re-mix the ref latents once per
denoising step, where ``x`` is the denoise progress (0 = first step,
1 = last step). Returns 1.0 (no weakening) for anything unrecognized.
"""
if spec is None or spec == "":
return 1.0
if isinstance(spec, str):
legacy = _LEGACY_CURVES.get(spec)
return curve_value_at(legacy, x) if legacy is not None else 1.0
if (isinstance(spec, tuple) and len(spec) == 3
and isinstance(spec[0], str) and isinstance(spec[1], str)):
direction, shape, value = spec
value = float(value)
if direction == "constant":
if shape == "linear":
return value
return max(0.0, min(1.0, value * _ease(shape, x)))
y = _ease(shape, x)
if direction in ("increase", "concept_at_start"):
return max(0.0, min(1.0, value * y))
if direction in ("decrease", "concept_at_end"):
return max(0.0, min(1.0, value * (1.0 - y)))
if direction == "concept_at_middle": # [0..1..0]: concept peaks mid-timeline
return max(0.0, min(1.0, value * (1.0 - abs(2.0 * y - 1.0))))
if direction == "concept_at_ends": # [1..0..1]: concept at both ends
return max(0.0, min(1.0, value * abs(2.0 * y - 1.0)))
return 1.0
def dedup_frame_indices(z: torch.Tensor, threshold: float = 0.02) -> List[int]:
"""Indices of latent frames kept by greedy temporal dedup.
Each frame is compared to the last *kept* frame: if the mean-abs
difference between them, normalized by the frames' own magnitude, is
below ``threshold`` the frame is dropped as a (near-)duplicate. Video
refs (a dance loop, a static shot, a talking head) contain long runs of
frames that differ only by codec noise — each one still costs a token
per spatial patch in every DiT block, so cutting them is the cheapest
way to honor a token cap. Stacked image refs survive: different
angles/expressions land well above the threshold.
"""
t = z.shape[2]
if t <= 1:
return list(range(t))
flat = z[0].float() # [24, T, H, W]
kept = [0]
prev = flat[:, 0]
for i in range(1, t):
cur = flat[:, i]
denom = (cur.abs().mean() + prev.abs().mean()) / 2 + 1e-6
diff = (cur - prev).abs().mean() / denom
if diff >= threshold:
kept.append(i)
prev = cur
return kept
def fit_token_budget(latent: torch.Tensor, budget: int, label: str) -> torch.Tensor:
"""Bring a stacked latent's injected token count under ``budget``.
Tokens = (H/2)*(W/2) per frame (the DiT's 2x2 patch). When the stack
exceeds the budget it is cut in two cheap, loss-ordered passes:
1. temporal dedup — drop latent frames that are (near-)identical to
the last kept frame (free: they carry no new reference info);
2. budget-fit resample — uniformly subsample the remaining frames down
to the largest count that fits the budget.
The spatial dims are never touched: resizing a latent's H/W changes the
rope grid and degrades the reference, so time is the only honest lever.
``label`` is the mod name for the console notes.
"""
h, w = latent.shape[3], latent.shape[4]
per_frame = (h // 2) * (w // 2)
t = latent.shape[2]
if per_frame * t <= budget:
return latent
kept = dedup_frame_indices(latent)
if len(kept) < t:
print(f"[RefMod] {label}: over {budget}-token cap, "
f"dropped {t - len(kept)} near-duplicate frame(s) "
f"({t} -> {len(kept)})")
latent = latent[:, :, kept]
t = latent.shape[2]
if per_frame * t > budget:
fit_t = max(1, budget // per_frame)
if fit_t < t:
idx = torch.linspace(0, t - 1, fit_t).round().long()
latent = latent[:, :, idx]
print(f"[RefMod] {label}: still over {budget}-token cap, "
f"resampled {t} -> {fit_t} frame(s) to fit")
if per_frame * latent.shape[2] > budget:
print(f"[RefMod] {label}: one frame alone is {per_frame} tokens > "
f"{budget} — lower ref_resolution or pool_h/pool_w to meet "
f"the cap.")
return latent
# ═══════════════════════════════════════════════════════════════════════════
# H3RefMod — the saved artifact
# ═══════════════════════════════════════════════════════════════════════════
@dataclass
class H3RefMod:
"""
A compressed reference for MiniMax H3.
``latent`` is the VAE latent ``[1, 24, latent_t, latent_h, latent_w]`` — a
full-resolution encode (``mode="encode"``) or a pooled thumbnail
(``mode="training"``). ``kind`` is ``"image"`` (single latent frame) or
``"video"`` (a few frames), matching the native ref block kinds the
model's ``PackedLayout`` understands.
``mode="encode"`` stores the encode at the resolution the official ref2video
path uses, so the injected ref carries real identity detail;
``mode="training"`` stores a tiny average-pooled grid refined by gradient
steps (concept/motion, or identity at high pool sizes).
"""
name: str
kind: str
latent: torch.Tensor
latent_h: int = 4
latent_w: int = 4
latent_t: int = 1
mode: str = "training"
source: str = "" # "image" | "video" | "manual"
source_shape: str = "" # original latent dims as "TxHxW"
pool: str = "4x4x1" # pool_t x pool_h x pool_w
optimize_steps: int = 0
tags: List[str] = field(default_factory=list)
description: str = "" # optional text describing the concept (emitted by the loaders)
concept_type: str = "generic" # what this mod represents; see CONCEPT_TYPES above
audio_concept_type: str = "" # upstream-style audio role label for embedded audio, if present
# Optional audio identity: the audio-VAE latent [1, 32, 2, ref_audio_t] and
# its frame count. When present the mod is emitted as a "video_audio" (or
# standalone "audio") ref block so the DiT attends to the soundtrack too.
audio_latent: Optional[torch.Tensor] = None
ref_audio_t: int = 0
sample_rate: int = 32000
enc_frames: Dict[str, torch.Tensor] = field(default_factory=dict)
enc_times: List[float] = field(default_factory=list)
enc_fps: float = 0.0
def __post_init__(self):
if self.kind == "audio":
if self.latent.ndim != 4 or tuple(self.latent.shape[:3]) != (1, 32, 2) or self.latent.shape[-1] < 1:
raise ValueError("H3 audio RefMod must contain [1,32,2,T] with T >= 1.")
self.latent_t = int(self.latent.shape[-1])
self.latent_h = 0
self.latent_w = 0
self.audio_latent = None
self.ref_audio_t = 0
return
if self.kind not in ("image", "video"):
raise ValueError(f"kind must be 'image', 'video' or 'audio' (got {self.kind!r})")
if self.kind == "image":
self.latent_t = 1
if self.audio_latent is None or self.ref_audio_t <= 0:
self.audio_latent = None
self.audio_concept_type = ""
self.ref_audio_t = 0
# ── token budget ──────────────────────────────────────────────────
@property
def token_count(self) -> int:
"""Number of patchified tokens the mod injects into the packed sequence."""
if self.kind == "audio":
return 2 * self.latent_t
per_frame = (self.latent_h // 2) * (self.latent_w // 2)
return self.latent_t * per_frame
# ── native ref block ──────────────────────────────────────────────
def ref_block(self, strength: float = 1.0,
curve=None, use_video: bool = True,
use_audio: bool = True) -> Optional[Dict]:
"""
Build the ref block dict the model's ``PackedLayout`` / payload consumes.
The shape mirrors what the native ref2va nodes emit, so the pooled
latent rides the exact same path: ``cond_video_latents`` -> patchify ->
``video_patch_proj`` -> packed sequence with a 3D RoPE grid.
``strength`` weakens the ref by mixing the latent toward a heavily
blurred copy of itself (see ``_blur_latent``) rather than toward
zero or random noise: scaling toward zero pushes values out of the
normalized latent distribution (reads as grey output), and mixing
toward random noise reads as real-but-garbled content and decodes
as a wrong texture rather than "weaker" — a blurred copy stays
on-manifold while still discarding the specific detail that makes a
reference strong. ``strength <= 0`` drops the block entirely (no
tokens injected).
``curve`` (optional) is a per-frame strength spec for the ref's
latent timeline — a ``(direction, shape, value)`` tuple (e.g.
``("concept_at_end", "ease", 1.0)``), a legacy preset name, a list of
per-frame floats, ``(x, y)`` control points, or any object with
``interp(x)`` (see ``curve_strengths``). Each frame is mixed with
its own strength ``retention * curve(x)`` instead of one flat value,
so the ref can fade in/out across the video. A flat curve is
identical to passing ``strength`` alone.
"""
if strength <= 0.0 or (not use_video and not use_audio):
return None
if self.kind == "audio":
if not use_audio:
return None
latent = self.latent
strengths = curve_strengths(curve, self.latent_t) if curve is not None else None
if strengths is not None:
weights = [max(0.0, min(1.0, strength * value)) for value in strengths]
if any(value < 1.0 for value in weights):
st = latent.new_tensor(weights).view(1, 1, 1, self.latent_t)
latent = st * latent + (1.0 - st) * _blur_latent(latent)
elif strength < 1.0:
latent = strength * latent + (1.0 - strength) * _blur_latent(latent)
return {
"kind": "audio",
"ref_audio_t": self.latent_t,
"audio_latent": latent,
}
has_audio = use_audio and self.audio_latent is not None and self.ref_audio_t > 0
if not use_video:
if has_audio:
return {
"kind": "audio",
"ref_audio_t": self.ref_audio_t,
"audio_latent": self.audio_latent,
}
return None
latent = self.latent
if curve is not None and self.latent_t > 1:
strengths = curve_strengths(curve, self.latent_t)
if strengths is not None:
t = self.latent_t
st = torch.tensor(
[max(0.0, min(1.0, strength * s)) for s in strengths],
dtype=latent.dtype, device=latent.device)
st = st.view(1, 1, t, 1, 1)
latent = st * latent + (1.0 - st) * _blur_latent(latent)
elif strength < 1.0:
latent = strength * latent + (1.0 - strength) * _blur_latent(latent)
block: Dict = {
"kind": self.kind,
"latent_h": self.latent_h,
"latent_w": self.latent_w,
"latent": latent,
}
if self.kind == "video" or has_audio:
# The DiT only reads audio off video / video_audio / audio blocks —
# its "image" branch ignores audio_latent. So a mod carrying audio
# is always emitted as video_audio (an image mod is just a 1-frame
# video), which packs the audio rows immediately before the video
# rows on the shared cursor.
block["kind"] = "video_audio" if has_audio else "video"
block["latent_t"] = self.latent_t
block["ref_audio_t"] = self.ref_audio_t if has_audio else 0
block["audio_latent"] = self.audio_latent if has_audio else None
return block
# ── serialization ─────────────────────────────────────────────────
def metadata(self) -> Dict:
meta = {
"name": self.name,
"kind": self.kind,
"latent_h": self.latent_h,
"latent_w": self.latent_w,
"latent_t": self.latent_t,
"mode": self.mode,
"source": self.source,
"source_shape": self.source_shape,
"pool": self.pool,
"optimize_steps": self.optimize_steps,
"tags": self.tags,
"description": self.description,
"concept_type": self.concept_type,
"_format_version": 4,
}
if self.kind == "audio" or (self.audio_latent is not None and self.ref_audio_t > 0):
meta["audio_concept_type"] = self.audio_concept_type
meta["ref_audio_t"] = self.ref_audio_t
meta["sample_rate"] = self.sample_rate
if self.enc_frames:
meta["enc_times"] = self.enc_times
meta["enc_fps"] = self.enc_fps
return meta
def save(self, path_no_ext: str) -> str:
"""Save as a single ``{path}.safetensors`` with metadata in the header."""
os.makedirs(os.path.dirname(path_no_ext) or ".", exist_ok=True)
meta = self.metadata()
tensors = {"latent": self.latent.contiguous()}
tensors.update({key: value.contiguous() for key, value in self.enc_frames.items()})
if self.audio_latent is not None and self.ref_audio_t > 0:
tensors["audio_latent"] = self.audio_latent.contiguous()
save_file(tensors, path_no_ext + ".safetensors",
metadata={META_KEY: json.dumps(meta)})
return path_no_ext + ".safetensors"
@classmethod
def from_metadata(
cls,
meta: Dict,
latent: torch.Tensor,
*,
audio_latent: Optional[torch.Tensor] = None,
enc_frames: Optional[Dict[str, torch.Tensor]] = None,
) -> "H3RefMod":
if not isinstance(meta, dict):
raise ValueError("RefMod metadata must be a dict.")
record = meta.get("encoder_images") or {}
return cls(
name=meta.get("name", "refmod"),
kind=meta.get("kind", "image"),
latent=latent,
latent_h=int(meta.get("latent_h", latent.shape[3] if latent.ndim == 5 else 0)),
latent_w=int(meta.get("latent_w", latent.shape[4] if latent.ndim == 5 else 0)),
latent_t=int(meta.get("latent_t", latent.shape[2] if latent.ndim == 5 else latent.shape[-1])),
mode=normalize_mode(meta.get("mode", "training")),
source=meta.get("source", ""),
source_shape=meta.get("source_shape", ""),
pool=meta.get("pool", ""),
optimize_steps=int(meta.get("optimize_steps", 0)),
tags=list(meta.get("tags", [])),
description=str(meta.get("description", "") or ""),
concept_type=str(meta.get("concept_type", "generic") or "generic"),
audio_concept_type=str(meta.get("audio_concept_type", "") or ""),
audio_latent=audio_latent,
ref_audio_t=int(meta.get("ref_audio_t", 0) or 0) if audio_latent is not None else 0,
sample_rate=int(meta.get("sample_rate", 32000) or 32000),
enc_frames=enc_frames or {},
enc_times=list(record.get("times", meta.get("enc_times", []))),
enc_fps=float(record.get("fps", meta.get("enc_fps", 0.0))),
)
@classmethod
def load(cls, path_no_ext: str, device: str = "cpu") -> "H3RefMod":
"""Load from ``{path}.safetensors`` (metadata in header, or legacy .json)."""
meta = read_refmod_meta(path_no_ext)
if meta is None:
raise ValueError(
f"{path_no_ext}.safetensors has no RefMod metadata "
f"(header key '{META_KEY}' or sidecar .json missing).")
if meta.get("kind") == "bundle":
raise ValueError("This is a RefMod bundle. Load its members instead of treating it as one latent.")
# clone drops the file mmap, so the file isn't locked on Windows and
# can be re-saved over the same name
all_tensors = load_file(path_no_ext + ".safetensors", device=device)
latent = all_tensors["latent"].clone()
audio_latent = all_tensors["audio_latent"].clone() if "audio_latent" in all_tensors else None
enc_frames = _read_encoder_images(meta, all_tensors.__getitem__)
return cls.from_metadata(meta, latent, audio_latent=audio_latent, enc_frames=enc_frames)