Squashed 'vendor/dumas_h3_refmod_picker/' content from commit c732f8b
git-subtree-dir: vendor/dumas_h3_refmod_picker git-subtree-split: c732f8b8893bf4f00c6974c9b9941c763496fe48
This commit is contained in:
@@ -0,0 +1,958 @@
|
||||
"""
|
||||
core.py — "RefMod" adapter for MiniMax H3 (no training required)
|
||||
|
||||
The reference-video path of MiniMax H3 works by injecting *reference tokens*
|
||||
into the packed sequence: the ref is VAE-encoded, patchified, projected and
|
||||
placed on the 3D RoPE grid, then every DiT block attends to it. A video ref
|
||||
is expensive because it contributes thousands of tokens.
|
||||
|
||||
Mode naming: ``encode`` (was ``full``) = straight full-res VAE encode;
|
||||
``training`` (was ``pooled``) = compressed grid refined by gradient steps
|
||||
(the only mode that "trains" — the refinement loop). The old names are
|
||||
accepted everywhere as legacy aliases so saved workflows and mods keep
|
||||
working.
|
||||
|
||||
A RefMod is the same reference, compressed to a handful of tokens:
|
||||
|
||||
* the ref is VAE-encoded to its full latent [1, 24, T, H, W],
|
||||
* the latent is average-pooled to a tiny grid (default 4x4) and a few
|
||||
latent frames, so the patchified token count drops to ~4-16,
|
||||
* optionally the small latent is refined with a few gradient steps that
|
||||
reconstruct the full latent (still no model weights involved).
|
||||
|
||||
At generation time the pooled latent is handed back to the model through the
|
||||
native ``refs`` payload, so it flows through the exact same per-block
|
||||
attention machinery as a full reference — the only thing that changes is the
|
||||
token budget.
|
||||
|
||||
This is the MiniMax H3 analog of the LTX "Mod" concept tokens + per-block
|
||||
injection: instead of a trained hypernetwork predicting AdaLN deltas, the
|
||||
model's own cross-attention over the compressed ref tokens does the work.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
# Metadata key embedded in the safetensors header (keeps a mod in one file,
|
||||
# so it can be shared/uploaded as a single artifact).
|
||||
META_KEY = "refmod_meta"
|
||||
|
||||
# What a mod is standing in for. This is documentation + a routing hint, not
|
||||
# a hard constraint on the tensors — but it drives two things: (1) the
|
||||
# "identity in training mode" warning in nodes.py, and (2) the loader's
|
||||
# prompt_hint output, which merges each loaded mod's concept_type +
|
||||
# description into a string you can concat onto your CLIP Text Encode prompt.
|
||||
# The model only reads these labels through the text prompt, so prompt_hint is
|
||||
# the only place they are surfaced.
|
||||
CONCEPT_TYPES = (
|
||||
"generic", # unspecified / mixed
|
||||
"identity", # a specific person/character — use mode="encode" for this
|
||||
"pose_motion", # a pose, dance, gesture, camera move
|
||||
"clothing", # an outfit / garment, decoupled from who's wearing it
|
||||
"background", # environment / set / location plate
|
||||
"voice",
|
||||
"singing",
|
||||
"music_style",
|
||||
"sound_fx",
|
||||
"ambience",
|
||||
"style", # look/grade/animation style, not a concrete subject
|
||||
)
|
||||
|
||||
AUDIO_CONCEPT_TYPES = ("voice", "singing", "music_style", "sound_fx", "ambience")
|
||||
|
||||
|
||||
def _blur_latent(z: torch.Tensor, factor: int = 8) -> torch.Tensor:
|
||||
"""Heavy spatial low-pass (downsample then upsample) used as the
|
||||
weakening target for ``ref_block``'s ``strength`` below ``1.0``.
|
||||
|
||||
Originally this mixed toward ``torch.randn()`` noise, mirroring what the
|
||||
docstring below describes as the model's own ``visual_cond_noise_aug``
|
||||
mechanism. In practice, at the retention range people actually use this
|
||||
dial for (0.15-0.7, i.e. large noise fractions), it produces a visible
|
||||
woven/static texture instead of a gracefully weaker reference: a VAE
|
||||
latent's channels are correlated, so raw iid noise isn't read as "less
|
||||
reference," it's read as real-but-garbled content and gets rendered as
|
||||
an actual (wrong) texture. ``visual_cond_noise_aug`` in the model's own
|
||||
training is likely a small-magnitude robustness augmentation, not
|
||||
something swept this far — so copying its math doesn't reproduce its
|
||||
behavior at these strengths. A blurred copy stays on the latent
|
||||
manifold (smooth, plausible) while still discarding detail as strength
|
||||
drops, which is what "weaker reference" should actually look like.
|
||||
"""
|
||||
if z.dim() == 4:
|
||||
b, c, stereo, t = z.shape
|
||||
if t <= 1:
|
||||
return z
|
||||
flat = z.reshape(b * c * stereo, 1, t).float()
|
||||
down = F.adaptive_avg_pool1d(flat, max(1, t // factor))
|
||||
up = F.interpolate(down, size=t, mode="linear", align_corners=False)
|
||||
return up.reshape_as(z).to(z.dtype)
|
||||
if z.dim() != 5:
|
||||
raise ValueError(f"Unsupported RefMod latent shape: {tuple(z.shape)}")
|
||||
t, h, w = z.shape[2], z.shape[3], z.shape[4]
|
||||
sh, sw = max(1, h // factor), max(1, w // factor)
|
||||
down = F.adaptive_avg_pool3d(z.float(), (t, sh, sw))
|
||||
up = F.interpolate(down, size=(t, h, w), mode="trilinear", align_corners=False)
|
||||
return up.to(z.dtype)
|
||||
|
||||
|
||||
def read_refmod_meta(path_no_ext: str) -> Optional[Dict]:
|
||||
"""
|
||||
Read the metadata block from a mod/preset file without loading tensors.
|
||||
|
||||
Prefers the metadata embedded in the safetensors header; falls back to
|
||||
the legacy sidecar ``{path}.json`` written by older versions.
|
||||
"""
|
||||
try:
|
||||
with safe_open(path_no_ext + ".safetensors", framework="pt") as f:
|
||||
meta = f.metadata()
|
||||
if meta:
|
||||
for key in (META_KEY, "audio_refmod_meta"):
|
||||
if key in meta:
|
||||
return json.loads(meta[key])
|
||||
except Exception:
|
||||
pass
|
||||
jpath = path_no_ext + ".json"
|
||||
if os.path.isfile(jpath):
|
||||
try:
|
||||
with open(jpath) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def refmod_bundle_members(meta: Dict) -> List[Dict]:
|
||||
if not isinstance(meta, dict) or meta.get("_format_version") != 5 or meta.get("kind") != "bundle":
|
||||
raise ValueError("Unsupported RefMod bundle format.")
|
||||
refs = meta.get("members")
|
||||
if not isinstance(refs, list) or not refs or len(refs) > 256:
|
||||
raise ValueError("A RefMod bundle must contain 1-256 members.")
|
||||
for ref in refs:
|
||||
if not isinstance(ref, dict) or ref.get("kind") not in ("image", "video", "audio"):
|
||||
raise ValueError("Invalid RefMod bundle member.")
|
||||
return refs
|
||||
|
||||
|
||||
def refmod_capabilities(meta: Optional[Dict]) -> Dict[str, object]:
|
||||
caps: Dict[str, object] = {
|
||||
"kind": "unknown",
|
||||
"has_visual": False,
|
||||
"has_audio": False,
|
||||
"member_count": 0,
|
||||
}
|
||||
if not isinstance(meta, dict):
|
||||
return caps
|
||||
kind = str(meta.get("kind", "") or "unknown")
|
||||
caps["kind"] = kind
|
||||
if kind == "bundle":
|
||||
refs = refmod_bundle_members(meta)
|
||||
caps["member_count"] = len(refs)
|
||||
caps["has_visual"] = any(ref.get("kind") in ("image", "video") for ref in refs)
|
||||
caps["has_audio"] = any(ref.get("kind") == "audio" for ref in refs)
|
||||
elif kind in ("image", "video"):
|
||||
caps["has_visual"] = True
|
||||
caps["has_audio"] = int(meta.get("ref_audio_t", 0) or 0) > 0
|
||||
caps["member_count"] = 1 + (1 if caps["has_audio"] else 0)
|
||||
elif kind == "audio":
|
||||
caps["has_audio"] = True
|
||||
caps["member_count"] = 1
|
||||
return caps
|
||||
|
||||
|
||||
def _standalone_audio_member_meta(meta: Dict) -> Dict:
|
||||
name = str(meta.get("name", "refmod") or "refmod")
|
||||
if not name.lower().endswith("_audio"):
|
||||
name = f"{name}_Audio"
|
||||
return {
|
||||
**meta,
|
||||
"name": name,
|
||||
"kind": "audio",
|
||||
"latent_h": 0,
|
||||
"latent_w": 0,
|
||||
"latent_t": int(meta.get("ref_audio_t", 0) or 0),
|
||||
"concept_type": str(meta.get("audio_concept_type", meta.get("concept_type", "voice")) or "voice"),
|
||||
"audio_concept_type": str(meta.get("audio_concept_type", "") or ""),
|
||||
"ref_audio_t": int(meta.get("ref_audio_t", 0) or 0),
|
||||
"sample_rate": int(meta.get("sample_rate", 32000) or 32000),
|
||||
}
|
||||
|
||||
|
||||
def load_refmods_from_file(path_no_ext: str, device: str = "cpu") -> List["H3RefMod"]:
|
||||
meta = read_refmod_meta(path_no_ext)
|
||||
if not isinstance(meta, dict):
|
||||
raise ValueError(
|
||||
f"{path_no_ext}.safetensors has no RefMod metadata "
|
||||
f"(header key '{META_KEY}' or sidecar .json missing)."
|
||||
)
|
||||
if meta.get("kind") == "bundle":
|
||||
loaded: List[H3RefMod] = []
|
||||
refs = refmod_bundle_members(meta)
|
||||
with safe_open(path_no_ext + ".safetensors", framework="pt", device=device) as f:
|
||||
for idx, member_meta in enumerate(refs):
|
||||
latent = f.get_tensor(f"ref_{idx}").clone()
|
||||
enc_frames = _read_encoder_images(member_meta, f.get_tensor)
|
||||
loaded.append(H3RefMod.from_metadata(member_meta, latent, enc_frames=enc_frames))
|
||||
return loaded
|
||||
|
||||
tensors = load_file(path_no_ext + ".safetensors", device=device)
|
||||
if "latent" not in tensors:
|
||||
raise ValueError(f"{path_no_ext}.safetensors is missing its latent tensor.")
|
||||
latent = tensors["latent"].clone()
|
||||
audio_latent = tensors["audio_latent"].clone() if "audio_latent" in tensors else None
|
||||
if meta.get("kind") == "audio":
|
||||
return [H3RefMod.from_metadata(meta, latent)]
|
||||
|
||||
visual_meta = dict(meta)
|
||||
visual_meta["ref_audio_t"] = 0
|
||||
visual_meta["audio_concept_type"] = ""
|
||||
enc_frames = _read_encoder_images(meta, tensors.__getitem__)
|
||||
loaded = [H3RefMod.from_metadata(visual_meta, latent, enc_frames=enc_frames)]
|
||||
if audio_latent is not None and int(meta.get("ref_audio_t", 0) or 0) > 0:
|
||||
loaded.append(H3RefMod.from_metadata(_standalone_audio_member_meta(meta), audio_latent))
|
||||
return loaded
|
||||
|
||||
|
||||
def save_refmod_bundle(path_no_ext: str, name: str, mods: List["H3RefMod"]) -> str:
|
||||
unique = list({id(mod): mod for mod in mods}.values())
|
||||
if not unique:
|
||||
raise ValueError("RefMod bundle is empty.")
|
||||
if sum(bool(mod.enc_frames) for mod in unique) > 1:
|
||||
raise ValueError("Save RefMods with encoder pictures in separate files when more than one member has pictures.")
|
||||
metadata = {
|
||||
"_format_version": 5,
|
||||
"kind": "bundle",
|
||||
"name": str(name or "refmod"),
|
||||
"members": [mod.metadata() for mod in unique],
|
||||
}
|
||||
tensors = {
|
||||
f"ref_{idx}": mod.latent.detach().cpu().contiguous().clone()
|
||||
for idx, mod in enumerate(unique)
|
||||
}
|
||||
for mod in unique:
|
||||
tensors.update({key: value.contiguous() for key, value in mod.enc_frames.items()})
|
||||
destination = path_no_ext + ".safetensors"
|
||||
directory = os.path.dirname(destination) or "."
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix=".refmod-", suffix=".tmp", dir=directory)
|
||||
os.close(fd)
|
||||
try:
|
||||
save_file(tensors, temporary, metadata={META_KEY: json.dumps(metadata)})
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
if os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
return destination
|
||||
|
||||
|
||||
def _read_encoder_images(meta, get_tensor):
|
||||
record = meta.get("encoder_images") or {}
|
||||
times = record.get("times", meta.get("enc_times", []))
|
||||
prefix = record.get("prefix", "enc_")
|
||||
return {f"enc_{index}": get_tensor(f"{prefix}{index}").detach().cpu().clone()
|
||||
for index in range(len(times))}
|
||||
|
||||
|
||||
def pack_encoder_images(mod, frames, times, fps=24.0):
|
||||
import io as bytes_io
|
||||
from PIL import Image
|
||||
|
||||
if frames.shape[0] != len(times):
|
||||
raise ValueError("Encoder picture count does not match its timestamps.")
|
||||
packed = {}
|
||||
for index, frame in enumerate(frames):
|
||||
buffer = bytes_io.BytesIO()
|
||||
pixels = (frame.detach().cpu().float().clamp(0, 1) * 255).round().to(torch.uint8).numpy()
|
||||
Image.fromarray(pixels).save(buffer, "JPEG", quality=95)
|
||||
packed[f"enc_{index}"] = torch.frombuffer(bytearray(buffer.getvalue()), dtype=torch.uint8)
|
||||
mod.enc_frames = packed
|
||||
mod.enc_times = list(times)
|
||||
mod.enc_fps = float(fps)
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Latent compression
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
MODE_ALIASES = {
|
||||
"full": "encode",
|
||||
"pooled": "training",
|
||||
"Full Reference": "encode",
|
||||
"Compressed Reference": "training",
|
||||
}
|
||||
|
||||
|
||||
def normalize_mode(mode: str) -> str:
|
||||
"""Map legacy mode names (``full``/``pooled``) to the current ones."""
|
||||
return MODE_ALIASES.get(mode, mode)
|
||||
|
||||
|
||||
def aspect_grid(pool_h: int, pool_w: int, aspect: float):
|
||||
"""Even pool grid dims that match ``aspect`` (h/w) within the dial caps.
|
||||
|
||||
The DiT patches each 2x2 latent cell into one token and the rope grid is
|
||||
area-normalized per axis, so a pooled grid that ignores the source aspect
|
||||
squishes the subject: a portrait latent (tall person) forced into a square
|
||||
16x16 grid is compressed 5x in height but only 3x in width, and comes out
|
||||
"fat". The official ref2video node never pools — it feeds the model
|
||||
aspect-correct latent dims, which is what the DiT was trained on.
|
||||
|
||||
The long edge follows the user's dial (``max(pool_h, pool_w)``) and the
|
||||
short edge is derived from the source aspect, both rounded to even for the
|
||||
2x2 patch. Square sources keep the exact dial value (16x16 stays 16x16).
|
||||
"""
|
||||
long_edge = max(pool_h, pool_w)
|
||||
if aspect >= 1.0:
|
||||
h, w = long_edge, long_edge / aspect
|
||||
else:
|
||||
w, h = long_edge, long_edge * aspect
|
||||
h = max(2, round(h / 2) * 2)
|
||||
w = max(2, round(w / 2) * 2)
|
||||
return h, w
|
||||
|
||||
|
||||
def pool_latent(
|
||||
z: torch.Tensor,
|
||||
latent_t: int,
|
||||
latent_h: int,
|
||||
latent_w: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Average-pool a VAE latent ``[1, 24, T, H, W]`` down to a tiny grid.
|
||||
|
||||
``latent_h/latent_w`` are the *latent* grid dims (the DiT patches each 2x2
|
||||
latent cell into one token), so a 4x4 latent = 2x2 = 4 tokens per frame.
|
||||
Dims must be even (the DiT's 2x2 patch). Pooling runs in fp32; the result
|
||||
keeps the input dtype.
|
||||
"""
|
||||
if z.shape[2] == latent_t and z.shape[3] == latent_h and z.shape[4] == latent_w:
|
||||
return z
|
||||
if latent_h % 2 != 0 or latent_w % 2 != 0:
|
||||
raise ValueError(f"pool_h/pool_w must be even (got {latent_h}x{latent_w})")
|
||||
pooled = F.adaptive_avg_pool3d(
|
||||
z.float(), (latent_t, latent_h, latent_w)
|
||||
)
|
||||
return pooled.to(z.dtype)
|
||||
|
||||
|
||||
def optimize_latent(
|
||||
z_small: torch.Tensor,
|
||||
z_full: torch.Tensor,
|
||||
steps: int = 150,
|
||||
lr: float = 00.02,
|
||||
device: Optional[torch.device] = None,
|
||||
progress_every: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Model-free refinement of the compressed latent.
|
||||
|
||||
Optimizes the small latent so its trilinearly upsampled reconstruction
|
||||
matches the full reference latent. This pulls the pooled representation
|
||||
closer to what the DiT would see from the full ref, with nothing but the
|
||||
tiny latent trainable (~1-2K params) and no diffusion model loaded.
|
||||
|
||||
``progress_every`` > 0 prints a ``[RefMod] identity <step>/<steps>`` line
|
||||
every N steps so long pooled-mode refinement isn't silent (the default
|
||||
``identity=500`` takes a while per ref on CPU/VRAM-bound setups).
|
||||
|
||||
Returns the refined latent detached, same shape/dtype as ``z_small``.
|
||||
"""
|
||||
if steps <= 0:
|
||||
return z_small
|
||||
device = device or z_full.device
|
||||
# Node execution runs under ComfyUI's global inference mode, which would
|
||||
# disable the autograd this refine loop needs. Re-enable it for this
|
||||
# scope and clone the inputs to drop the inference flag.
|
||||
with torch.inference_mode(False), torch.set_grad_enabled(True):
|
||||
target = z_full.clone().float().to(device)
|
||||
param = nn.Parameter(z_small.clone().float().to(device))
|
||||
opt = torch.optim.Adam([param], lr=lr)
|
||||
size = tuple(target.shape[2:])
|
||||
for i in range(steps):
|
||||
opt.zero_grad()
|
||||
up = F.interpolate(param, size=size, mode="trilinear", align_corners=False)
|
||||
loss = F.mse_loss(up, target)
|
||||
loss.backward()
|
||||
opt.step()
|
||||
if progress_every and (i + 1) % progress_every == 0:
|
||||
print(f"[RefMod] identity {i + 1}/{steps}")
|
||||
# materialize inside the scope so the result is a normal tensor, not
|
||||
# an inference-mode tensor (it gets stored in the mod and reused)
|
||||
refined = param.detach().to(z_small.dtype)
|
||||
return refined
|
||||
|
||||
|
||||
def optimize_latent_multi(z_init, targets, steps=150, lr=0.02, device=None,
|
||||
progress_every=0, strategy="grouped"):
|
||||
"""Mean reconstruction objective with bounded activation memory."""
|
||||
if not targets or steps <= 0:
|
||||
return z_init
|
||||
if strategy not in ("grouped", "stream", "resident"):
|
||||
raise ValueError("Unknown multi-reference optimization strategy.")
|
||||
device = device or z_init.device
|
||||
with torch.inference_mode(False), torch.set_grad_enabled(True):
|
||||
count = len(targets)
|
||||
if strategy == "grouped":
|
||||
groups = {}
|
||||
for target in targets:
|
||||
key = tuple(target.shape)
|
||||
value = target.detach().to(device="cpu", dtype=torch.float32).clone()
|
||||
if key in groups:
|
||||
groups[key][0].add_(value)
|
||||
groups[key][1] += 1
|
||||
else:
|
||||
groups[key] = [value, 1]
|
||||
grouped = []
|
||||
for total, group_count in groups.values():
|
||||
grouped.append((total.div_(group_count), group_count))
|
||||
elif strategy == "resident":
|
||||
resident = [target.detach().float().to(device) for target in targets]
|
||||
param = nn.Parameter(z_init.clone().float().to(device))
|
||||
opt = torch.optim.Adam([param], lr=lr)
|
||||
for step_idx in range(steps):
|
||||
opt.zero_grad()
|
||||
loss = None
|
||||
if strategy == "grouped":
|
||||
total_weight = 0
|
||||
for target_mean, group_count in grouped:
|
||||
target = target_mean.to(device)
|
||||
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
|
||||
align_corners=False)
|
||||
contribution = F.mse_loss(up, target)
|
||||
loss = contribution * group_count if loss is None else loss + contribution * group_count
|
||||
total_weight += group_count
|
||||
loss = loss / total_weight
|
||||
elif strategy == "stream":
|
||||
for target_cpu in targets:
|
||||
target = target_cpu.detach().float().to(device)
|
||||
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
|
||||
align_corners=False)
|
||||
contribution = F.mse_loss(up, target)
|
||||
loss = contribution if loss is None else loss + contribution
|
||||
loss = loss / count
|
||||
else:
|
||||
for target in resident:
|
||||
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
|
||||
align_corners=False)
|
||||
contribution = F.mse_loss(up, target)
|
||||
loss = contribution if loss is None else loss + contribution
|
||||
loss = loss / count
|
||||
loss.backward()
|
||||
opt.step()
|
||||
if progress_every and (step_idx + 1) % progress_every == 0:
|
||||
print(f"[RefMod] identity {step_idx + 1}/{steps}")
|
||||
refined = param.detach().to(z_init.dtype)
|
||||
return refined
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Per-frame strength curve
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
# Direction names describe where the *concept* shows up in the output, which
|
||||
# is the mirror of the strength envelope over the ref's own latent timeline:
|
||||
# concept_at_end locks the ref's literal footage at the START and releases it
|
||||
# toward the end (was "decrease"), concept_at_start opens free and locks onto
|
||||
# the ref near the END (was "increase"). shape is how the envelope travels
|
||||
# between its endpoints; value is the non-zero endpoint ("user input").
|
||||
CURVE_DIRECTIONS = ("constant", "concept_at_start", "concept_at_middle",
|
||||
"concept_at_end", "concept_at_ends")
|
||||
CURVE_SHAPES = ("linear", "ease", "sigmoid", "tanh", "quadratic", "cubic",
|
||||
"exponential", "stair", "elastic", "bump", "dip")
|
||||
|
||||
# old single-combo preset names -> (direction, shape, value) so workflows
|
||||
# saved before the curve was split still resolve
|
||||
_LEGACY_CURVES = {
|
||||
"flat": ("constant", "linear", 1.0),
|
||||
"fade_in": ("concept_at_start", "linear", 1.0),
|
||||
"fade_out": ("concept_at_end", "linear", 1.0),
|
||||
"bump": ("concept_at_start", "bump", 1.0),
|
||||
"dip": ("constant", "dip", 1.0),
|
||||
}
|
||||
|
||||
|
||||
def _ease(shape: str, x: float) -> float:
|
||||
"""Shape a progress ``x`` in [0, 1] per the easing name (may overshoot)."""
|
||||
if shape == "linear":
|
||||
return x
|
||||
if shape == "ease": # smoothstep
|
||||
return x * x * (3.0 - 2.0 * x)
|
||||
if shape == "sigmoid": # logistic S-curve
|
||||
return 1.0 / (1.0 + math.exp(-12.0 * (x - 0.5)))
|
||||
if shape == "tanh": # sharper S-curve (steeper knee than sigmoid)
|
||||
return 0.5 * (math.tanh(8.0 * (x - 0.5)) + 1.0)
|
||||
if shape == "quadratic":
|
||||
return x * x
|
||||
if shape == "cubic":
|
||||
return x * x * x
|
||||
if shape == "exponential":
|
||||
return 2.0 ** x - 1.0
|
||||
if shape == "stair":
|
||||
return min(1.0, math.floor(x * 4) / 3.0)
|
||||
if shape == "elastic":
|
||||
if x <= 0.0:
|
||||
return 0.0
|
||||
if x >= 1.0:
|
||||
return 1.0
|
||||
return 2.0 ** (-10.0 * x) * math.sin((x * 10.0 - 0.75) * (2.0 * math.pi / 3.0)) + 1.0
|
||||
if shape == "bump":
|
||||
return 1.0 - abs(2.0 * x - 1.0)
|
||||
if shape == "dip":
|
||||
return abs(2.0 * x - 1.0)
|
||||
return x
|
||||
|
||||
|
||||
def curve_strengths(spec, t: int) -> Optional[List[float]]:
|
||||
"""Resolve a curve spec to ``t`` per-frame strength multipliers in [0, 1].
|
||||
|
||||
Kept as plain Python so the Apply node exposes the curve as normal
|
||||
combo/float widgets instead of depending on ComfyUI's new Curve widget.
|
||||
Accepts:
|
||||
|
||||
* a ``(direction, shape, value)`` tuple — direction is ``constant``
|
||||
(one strength everywhere), ``concept_at_start`` (0 -> value, a
|
||||
crescent — the concept shows early; legacy ``increase``),
|
||||
``concept_at_end`` (value -> 0, a decrescent — the concept pops at
|
||||
the end; legacy ``decrease``), ``concept_at_middle`` ([0..1..0] —
|
||||
concept peaks mid-timeline) or ``concept_at_ends`` ([1..0..1] —
|
||||
concept at both ends); shape is how the envelope travels between
|
||||
its endpoints (see ``_ease``); value is the non-zero endpoint
|
||||
(1.0 = full strength there);
|
||||
* a legacy preset name (``flat``/``fade_in``/``fade_out``/``bump``/
|
||||
``dip``) from before the split;
|
||||
* a list of per-frame floats (len == t), used as-is;
|
||||
* a list of ``(x, y)`` control points (x in [0, 1]), linearly
|
||||
interpolated across the frames;
|
||||
* any object with ``interp(x) -> float`` (legacy CurveInput saved in an
|
||||
old workflow).
|
||||
|
||||
Returns None for flat/no curve so the caller keeps its single-strength
|
||||
path unchanged (a ``constant`` curve at value 1.0 included).
|
||||
"""
|
||||
if t <= 1 or spec is None or spec == "":
|
||||
return None
|
||||
if isinstance(spec, str):
|
||||
legacy = _LEGACY_CURVES.get(spec)
|
||||
return curve_strengths(legacy, t) if legacy is not None else None
|
||||
if (isinstance(spec, tuple) and len(spec) == 3
|
||||
and isinstance(spec[0], str) and isinstance(spec[1], str)):
|
||||
direction, shape, value = spec
|
||||
value = float(value)
|
||||
if direction == "constant":
|
||||
# linear = flat at ``value`` (1.0 == no curve); the other shapes
|
||||
# modulate around the constant, so constant + dip is a trough,
|
||||
# constant + bump is a peak (both endpoints equal)
|
||||
if shape == "linear":
|
||||
return None if value >= 1.0 else [value] * t
|
||||
p = [_ease(shape, i / (t - 1)) for i in range(t)]
|
||||
return [max(0.0, min(1.0, value * y)) for y in p]
|
||||
p = [_ease(shape, i / (t - 1)) for i in range(t)]
|
||||
# legacy "increase"/"decrease" (pre-rename workflows) map onto the same
|
||||
# envelopes as the concept-placement names
|
||||
if direction in ("increase", "concept_at_start"):
|
||||
return [max(0.0, min(1.0, value * y)) for y in p]
|
||||
if direction in ("decrease", "concept_at_end"):
|
||||
return [max(0.0, min(1.0, value * (1.0 - y))) for y in p]
|
||||
if direction == "concept_at_middle": # [0..1..0]: concept peaks mid-timeline
|
||||
return [max(0.0, min(1.0, value * (1.0 - abs(2.0 * y - 1.0)))) for y in p]
|
||||
if direction == "concept_at_ends": # [1..0..1]: concept at both ends
|
||||
return [max(0.0, min(1.0, value * abs(2.0 * y - 1.0))) for y in p]
|
||||
return None # unknown direction: treat as flat
|
||||
if isinstance(spec, (list, tuple)):
|
||||
if len(spec) == t and all(isinstance(v, (int, float)) for v in spec):
|
||||
return [float(v) for v in spec]
|
||||
pts = [(float(x), float(y)) for x, y in spec
|
||||
if isinstance(x, (int, float)) and isinstance(y, (int, float))]
|
||||
if pts:
|
||||
pts = sorted(pts, key=lambda p: p[0])
|
||||
out = []
|
||||
for i in range(t):
|
||||
x = i * (1.0 / (t - 1))
|
||||
if x <= pts[0][0]:
|
||||
out.append(pts[0][1])
|
||||
elif x >= pts[-1][0]:
|
||||
out.append(pts[-1][1])
|
||||
else:
|
||||
for (x0, y0), (x1, y1) in zip(pts, pts[1:]):
|
||||
if x0 <= x <= x1:
|
||||
out.append(y0 + (y1 - y0) * (x - x0) / (x1 - x0))
|
||||
break
|
||||
return out
|
||||
interp = getattr(spec, "interp", None)
|
||||
if callable(interp):
|
||||
return [float(interp(i / (t - 1))) for i in range(t)]
|
||||
return None
|
||||
|
||||
|
||||
def curve_value_at(spec, x: float) -> float:
|
||||
"""Per-progress strength of a curve spec at one point ``x`` in [0, 1].
|
||||
|
||||
The continuous analog of ``curve_strengths`` — same direction/shape/value
|
||||
math, but evaluated at a single progress value instead of discretized over
|
||||
frames. The step-curve node uses it to re-mix the ref latents once per
|
||||
denoising step, where ``x`` is the denoise progress (0 = first step,
|
||||
1 = last step). Returns 1.0 (no weakening) for anything unrecognized.
|
||||
"""
|
||||
if spec is None or spec == "":
|
||||
return 1.0
|
||||
if isinstance(spec, str):
|
||||
legacy = _LEGACY_CURVES.get(spec)
|
||||
return curve_value_at(legacy, x) if legacy is not None else 1.0
|
||||
if (isinstance(spec, tuple) and len(spec) == 3
|
||||
and isinstance(spec[0], str) and isinstance(spec[1], str)):
|
||||
direction, shape, value = spec
|
||||
value = float(value)
|
||||
if direction == "constant":
|
||||
if shape == "linear":
|
||||
return value
|
||||
return max(0.0, min(1.0, value * _ease(shape, x)))
|
||||
y = _ease(shape, x)
|
||||
if direction in ("increase", "concept_at_start"):
|
||||
return max(0.0, min(1.0, value * y))
|
||||
if direction in ("decrease", "concept_at_end"):
|
||||
return max(0.0, min(1.0, value * (1.0 - y)))
|
||||
if direction == "concept_at_middle": # [0..1..0]: concept peaks mid-timeline
|
||||
return max(0.0, min(1.0, value * (1.0 - abs(2.0 * y - 1.0))))
|
||||
if direction == "concept_at_ends": # [1..0..1]: concept at both ends
|
||||
return max(0.0, min(1.0, value * abs(2.0 * y - 1.0)))
|
||||
return 1.0
|
||||
|
||||
|
||||
def dedup_frame_indices(z: torch.Tensor, threshold: float = 0.02) -> List[int]:
|
||||
"""Indices of latent frames kept by greedy temporal dedup.
|
||||
|
||||
Each frame is compared to the last *kept* frame: if the mean-abs
|
||||
difference between them, normalized by the frames' own magnitude, is
|
||||
below ``threshold`` the frame is dropped as a (near-)duplicate. Video
|
||||
refs (a dance loop, a static shot, a talking head) contain long runs of
|
||||
frames that differ only by codec noise — each one still costs a token
|
||||
per spatial patch in every DiT block, so cutting them is the cheapest
|
||||
way to honor a token cap. Stacked image refs survive: different
|
||||
angles/expressions land well above the threshold.
|
||||
"""
|
||||
t = z.shape[2]
|
||||
if t <= 1:
|
||||
return list(range(t))
|
||||
flat = z[0].float() # [24, T, H, W]
|
||||
kept = [0]
|
||||
prev = flat[:, 0]
|
||||
for i in range(1, t):
|
||||
cur = flat[:, i]
|
||||
denom = (cur.abs().mean() + prev.abs().mean()) / 2 + 1e-6
|
||||
diff = (cur - prev).abs().mean() / denom
|
||||
if diff >= threshold:
|
||||
kept.append(i)
|
||||
prev = cur
|
||||
return kept
|
||||
|
||||
|
||||
def fit_token_budget(latent: torch.Tensor, budget: int, label: str) -> torch.Tensor:
|
||||
"""Bring a stacked latent's injected token count under ``budget``.
|
||||
|
||||
Tokens = (H/2)*(W/2) per frame (the DiT's 2x2 patch). When the stack
|
||||
exceeds the budget it is cut in two cheap, loss-ordered passes:
|
||||
|
||||
1. temporal dedup — drop latent frames that are (near-)identical to
|
||||
the last kept frame (free: they carry no new reference info);
|
||||
2. budget-fit resample — uniformly subsample the remaining frames down
|
||||
to the largest count that fits the budget.
|
||||
|
||||
The spatial dims are never touched: resizing a latent's H/W changes the
|
||||
rope grid and degrades the reference, so time is the only honest lever.
|
||||
``label`` is the mod name for the console notes.
|
||||
"""
|
||||
h, w = latent.shape[3], latent.shape[4]
|
||||
per_frame = (h // 2) * (w // 2)
|
||||
t = latent.shape[2]
|
||||
if per_frame * t <= budget:
|
||||
return latent
|
||||
kept = dedup_frame_indices(latent)
|
||||
if len(kept) < t:
|
||||
print(f"[RefMod] {label}: over {budget}-token cap, "
|
||||
f"dropped {t - len(kept)} near-duplicate frame(s) "
|
||||
f"({t} -> {len(kept)})")
|
||||
latent = latent[:, :, kept]
|
||||
t = latent.shape[2]
|
||||
if per_frame * t > budget:
|
||||
fit_t = max(1, budget // per_frame)
|
||||
if fit_t < t:
|
||||
idx = torch.linspace(0, t - 1, fit_t).round().long()
|
||||
latent = latent[:, :, idx]
|
||||
print(f"[RefMod] {label}: still over {budget}-token cap, "
|
||||
f"resampled {t} -> {fit_t} frame(s) to fit")
|
||||
if per_frame * latent.shape[2] > budget:
|
||||
print(f"[RefMod] {label}: one frame alone is {per_frame} tokens > "
|
||||
f"{budget} — lower ref_resolution or pool_h/pool_w to meet "
|
||||
f"the cap.")
|
||||
return latent
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# H3RefMod — the saved artifact
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
@dataclass
|
||||
class H3RefMod:
|
||||
"""
|
||||
A compressed reference for MiniMax H3.
|
||||
|
||||
``latent`` is the VAE latent ``[1, 24, latent_t, latent_h, latent_w]`` — a
|
||||
full-resolution encode (``mode="encode"``) or a pooled thumbnail
|
||||
(``mode="training"``). ``kind`` is ``"image"`` (single latent frame) or
|
||||
``"video"`` (a few frames), matching the native ref block kinds the
|
||||
model's ``PackedLayout`` understands.
|
||||
|
||||
``mode="encode"`` stores the encode at the resolution the official ref2video
|
||||
path uses, so the injected ref carries real identity detail;
|
||||
``mode="training"`` stores a tiny average-pooled grid refined by gradient
|
||||
steps (concept/motion, or identity at high pool sizes).
|
||||
"""
|
||||
|
||||
name: str
|
||||
kind: str
|
||||
latent: torch.Tensor
|
||||
latent_h: int = 4
|
||||
latent_w: int = 4
|
||||
latent_t: int = 1
|
||||
mode: str = "training"
|
||||
source: str = "" # "image" | "video" | "manual"
|
||||
source_shape: str = "" # original latent dims as "TxHxW"
|
||||
pool: str = "4x4x1" # pool_t x pool_h x pool_w
|
||||
optimize_steps: int = 0
|
||||
tags: List[str] = field(default_factory=list)
|
||||
description: str = "" # optional text describing the concept (emitted by the loaders)
|
||||
concept_type: str = "generic" # what this mod represents; see CONCEPT_TYPES above
|
||||
audio_concept_type: str = "" # upstream-style audio role label for embedded audio, if present
|
||||
# Optional audio identity: the audio-VAE latent [1, 32, 2, ref_audio_t] and
|
||||
# its frame count. When present the mod is emitted as a "video_audio" (or
|
||||
# standalone "audio") ref block so the DiT attends to the soundtrack too.
|
||||
audio_latent: Optional[torch.Tensor] = None
|
||||
ref_audio_t: int = 0
|
||||
sample_rate: int = 32000
|
||||
enc_frames: Dict[str, torch.Tensor] = field(default_factory=dict)
|
||||
enc_times: List[float] = field(default_factory=list)
|
||||
enc_fps: float = 0.0
|
||||
|
||||
def __post_init__(self):
|
||||
if self.kind == "audio":
|
||||
if self.latent.ndim != 4 or tuple(self.latent.shape[:3]) != (1, 32, 2) or self.latent.shape[-1] < 1:
|
||||
raise ValueError("H3 audio RefMod must contain [1,32,2,T] with T >= 1.")
|
||||
self.latent_t = int(self.latent.shape[-1])
|
||||
self.latent_h = 0
|
||||
self.latent_w = 0
|
||||
self.audio_latent = None
|
||||
self.ref_audio_t = 0
|
||||
return
|
||||
if self.kind not in ("image", "video"):
|
||||
raise ValueError(f"kind must be 'image', 'video' or 'audio' (got {self.kind!r})")
|
||||
if self.kind == "image":
|
||||
self.latent_t = 1
|
||||
if self.audio_latent is None or self.ref_audio_t <= 0:
|
||||
self.audio_latent = None
|
||||
self.audio_concept_type = ""
|
||||
self.ref_audio_t = 0
|
||||
|
||||
# ── token budget ──────────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def token_count(self) -> int:
|
||||
"""Number of patchified tokens the mod injects into the packed sequence."""
|
||||
if self.kind == "audio":
|
||||
return 2 * self.latent_t
|
||||
per_frame = (self.latent_h // 2) * (self.latent_w // 2)
|
||||
return self.latent_t * per_frame
|
||||
|
||||
# ── native ref block ──────────────────────────────────────────────
|
||||
|
||||
def ref_block(self, strength: float = 1.0,
|
||||
curve=None, use_video: bool = True,
|
||||
use_audio: bool = True) -> Optional[Dict]:
|
||||
"""
|
||||
Build the ref block dict the model's ``PackedLayout`` / payload consumes.
|
||||
|
||||
The shape mirrors what the native ref2va nodes emit, so the pooled
|
||||
latent rides the exact same path: ``cond_video_latents`` -> patchify ->
|
||||
``video_patch_proj`` -> packed sequence with a 3D RoPE grid.
|
||||
|
||||
``strength`` weakens the ref by mixing the latent toward a heavily
|
||||
blurred copy of itself (see ``_blur_latent``) rather than toward
|
||||
zero or random noise: scaling toward zero pushes values out of the
|
||||
normalized latent distribution (reads as grey output), and mixing
|
||||
toward random noise reads as real-but-garbled content and decodes
|
||||
as a wrong texture rather than "weaker" — a blurred copy stays
|
||||
on-manifold while still discarding the specific detail that makes a
|
||||
reference strong. ``strength <= 0`` drops the block entirely (no
|
||||
tokens injected).
|
||||
|
||||
``curve`` (optional) is a per-frame strength spec for the ref's
|
||||
latent timeline — a ``(direction, shape, value)`` tuple (e.g.
|
||||
``("concept_at_end", "ease", 1.0)``), a legacy preset name, a list of
|
||||
per-frame floats, ``(x, y)`` control points, or any object with
|
||||
``interp(x)`` (see ``curve_strengths``). Each frame is mixed with
|
||||
its own strength ``retention * curve(x)`` instead of one flat value,
|
||||
so the ref can fade in/out across the video. A flat curve is
|
||||
identical to passing ``strength`` alone.
|
||||
"""
|
||||
if strength <= 0.0 or (not use_video and not use_audio):
|
||||
return None
|
||||
if self.kind == "audio":
|
||||
if not use_audio:
|
||||
return None
|
||||
latent = self.latent
|
||||
strengths = curve_strengths(curve, self.latent_t) if curve is not None else None
|
||||
if strengths is not None:
|
||||
weights = [max(0.0, min(1.0, strength * value)) for value in strengths]
|
||||
if any(value < 1.0 for value in weights):
|
||||
st = latent.new_tensor(weights).view(1, 1, 1, self.latent_t)
|
||||
latent = st * latent + (1.0 - st) * _blur_latent(latent)
|
||||
elif strength < 1.0:
|
||||
latent = strength * latent + (1.0 - strength) * _blur_latent(latent)
|
||||
return {
|
||||
"kind": "audio",
|
||||
"ref_audio_t": self.latent_t,
|
||||
"audio_latent": latent,
|
||||
}
|
||||
has_audio = use_audio and self.audio_latent is not None and self.ref_audio_t > 0
|
||||
if not use_video:
|
||||
if has_audio:
|
||||
return {
|
||||
"kind": "audio",
|
||||
"ref_audio_t": self.ref_audio_t,
|
||||
"audio_latent": self.audio_latent,
|
||||
}
|
||||
return None
|
||||
latent = self.latent
|
||||
if curve is not None and self.latent_t > 1:
|
||||
strengths = curve_strengths(curve, self.latent_t)
|
||||
if strengths is not None:
|
||||
t = self.latent_t
|
||||
st = torch.tensor(
|
||||
[max(0.0, min(1.0, strength * s)) for s in strengths],
|
||||
dtype=latent.dtype, device=latent.device)
|
||||
st = st.view(1, 1, t, 1, 1)
|
||||
latent = st * latent + (1.0 - st) * _blur_latent(latent)
|
||||
elif strength < 1.0:
|
||||
latent = strength * latent + (1.0 - strength) * _blur_latent(latent)
|
||||
block: Dict = {
|
||||
"kind": self.kind,
|
||||
"latent_h": self.latent_h,
|
||||
"latent_w": self.latent_w,
|
||||
"latent": latent,
|
||||
}
|
||||
if self.kind == "video" or has_audio:
|
||||
# The DiT only reads audio off video / video_audio / audio blocks —
|
||||
# its "image" branch ignores audio_latent. So a mod carrying audio
|
||||
# is always emitted as video_audio (an image mod is just a 1-frame
|
||||
# video), which packs the audio rows immediately before the video
|
||||
# rows on the shared cursor.
|
||||
block["kind"] = "video_audio" if has_audio else "video"
|
||||
block["latent_t"] = self.latent_t
|
||||
block["ref_audio_t"] = self.ref_audio_t if has_audio else 0
|
||||
block["audio_latent"] = self.audio_latent if has_audio else None
|
||||
return block
|
||||
|
||||
# ── serialization ─────────────────────────────────────────────────
|
||||
|
||||
def metadata(self) -> Dict:
|
||||
meta = {
|
||||
"name": self.name,
|
||||
"kind": self.kind,
|
||||
"latent_h": self.latent_h,
|
||||
"latent_w": self.latent_w,
|
||||
"latent_t": self.latent_t,
|
||||
"mode": self.mode,
|
||||
"source": self.source,
|
||||
"source_shape": self.source_shape,
|
||||
"pool": self.pool,
|
||||
"optimize_steps": self.optimize_steps,
|
||||
"tags": self.tags,
|
||||
"description": self.description,
|
||||
"concept_type": self.concept_type,
|
||||
"_format_version": 4,
|
||||
}
|
||||
if self.kind == "audio" or (self.audio_latent is not None and self.ref_audio_t > 0):
|
||||
meta["audio_concept_type"] = self.audio_concept_type
|
||||
meta["ref_audio_t"] = self.ref_audio_t
|
||||
meta["sample_rate"] = self.sample_rate
|
||||
if self.enc_frames:
|
||||
meta["enc_times"] = self.enc_times
|
||||
meta["enc_fps"] = self.enc_fps
|
||||
return meta
|
||||
|
||||
def save(self, path_no_ext: str) -> str:
|
||||
"""Save as a single ``{path}.safetensors`` with metadata in the header."""
|
||||
os.makedirs(os.path.dirname(path_no_ext) or ".", exist_ok=True)
|
||||
meta = self.metadata()
|
||||
tensors = {"latent": self.latent.contiguous()}
|
||||
tensors.update({key: value.contiguous() for key, value in self.enc_frames.items()})
|
||||
if self.audio_latent is not None and self.ref_audio_t > 0:
|
||||
tensors["audio_latent"] = self.audio_latent.contiguous()
|
||||
save_file(tensors, path_no_ext + ".safetensors",
|
||||
metadata={META_KEY: json.dumps(meta)})
|
||||
return path_no_ext + ".safetensors"
|
||||
|
||||
@classmethod
|
||||
def from_metadata(
|
||||
cls,
|
||||
meta: Dict,
|
||||
latent: torch.Tensor,
|
||||
*,
|
||||
audio_latent: Optional[torch.Tensor] = None,
|
||||
enc_frames: Optional[Dict[str, torch.Tensor]] = None,
|
||||
) -> "H3RefMod":
|
||||
if not isinstance(meta, dict):
|
||||
raise ValueError("RefMod metadata must be a dict.")
|
||||
record = meta.get("encoder_images") or {}
|
||||
return cls(
|
||||
name=meta.get("name", "refmod"),
|
||||
kind=meta.get("kind", "image"),
|
||||
latent=latent,
|
||||
latent_h=int(meta.get("latent_h", latent.shape[3] if latent.ndim == 5 else 0)),
|
||||
latent_w=int(meta.get("latent_w", latent.shape[4] if latent.ndim == 5 else 0)),
|
||||
latent_t=int(meta.get("latent_t", latent.shape[2] if latent.ndim == 5 else latent.shape[-1])),
|
||||
mode=normalize_mode(meta.get("mode", "training")),
|
||||
source=meta.get("source", ""),
|
||||
source_shape=meta.get("source_shape", ""),
|
||||
pool=meta.get("pool", ""),
|
||||
optimize_steps=int(meta.get("optimize_steps", 0)),
|
||||
tags=list(meta.get("tags", [])),
|
||||
description=str(meta.get("description", "") or ""),
|
||||
concept_type=str(meta.get("concept_type", "generic") or "generic"),
|
||||
audio_concept_type=str(meta.get("audio_concept_type", "") or ""),
|
||||
audio_latent=audio_latent,
|
||||
ref_audio_t=int(meta.get("ref_audio_t", 0) or 0) if audio_latent is not None else 0,
|
||||
sample_rate=int(meta.get("sample_rate", 32000) or 32000),
|
||||
enc_frames=enc_frames or {},
|
||||
enc_times=list(record.get("times", meta.get("enc_times", []))),
|
||||
enc_fps=float(record.get("fps", meta.get("enc_fps", 0.0))),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path_no_ext: str, device: str = "cpu") -> "H3RefMod":
|
||||
"""Load from ``{path}.safetensors`` (metadata in header, or legacy .json)."""
|
||||
meta = read_refmod_meta(path_no_ext)
|
||||
if meta is None:
|
||||
raise ValueError(
|
||||
f"{path_no_ext}.safetensors has no RefMod metadata "
|
||||
f"(header key '{META_KEY}' or sidecar .json missing).")
|
||||
if meta.get("kind") == "bundle":
|
||||
raise ValueError("This is a RefMod bundle. Load its members instead of treating it as one latent.")
|
||||
# clone drops the file mmap, so the file isn't locked on Windows and
|
||||
# can be re-saved over the same name
|
||||
all_tensors = load_file(path_no_ext + ".safetensors", device=device)
|
||||
latent = all_tensors["latent"].clone()
|
||||
audio_latent = all_tensors["audio_latent"].clone() if "audio_latent" in all_tensors else None
|
||||
enc_frames = _read_encoder_images(meta, all_tensors.__getitem__)
|
||||
return cls.from_metadata(meta, latent, audio_latent=audio_latent, enc_frames=enc_frames)
|
||||
|
||||
Reference in New Issue
Block a user