Merge commit 'c9eb575e2fd25f74b15f1d7336af4932c7fe6338' as 'vendor/dumas_h3_refmod_picker'
This commit is contained in:
+253
@@ -0,0 +1,253 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .refmod_common import refmods_dir, refmods_dirs
|
||||
from .refmod_core import read_refmod_meta, refmod_capabilities
|
||||
|
||||
|
||||
PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".webp")
|
||||
HIDDEN_BROWSER_DIRS = {"graph_presets"}
|
||||
VISUAL_SUFFIXES = ("_visual", "_video")
|
||||
AUDIO_SUFFIXES = ("_audio",)
|
||||
VISUAL_FILE_SUFFIXES = ("_visual", "_Visual", "_video", "_Video")
|
||||
AUDIO_FILE_SUFFIXES = ("_audio", "_Audio")
|
||||
|
||||
|
||||
def browser_root() -> str:
|
||||
root = os.path.abspath(refmods_dir())
|
||||
os.makedirs(root, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def browser_roots() -> List[str]:
|
||||
"""Every declared RefMod folder; the first one is browser_root()."""
|
||||
return refmods_dirs()
|
||||
|
||||
|
||||
def _browser_root_labels() -> Dict[str, str]:
|
||||
labels: Dict[str, str] = {}
|
||||
counts: Dict[str, int] = {}
|
||||
for index, root in enumerate(browser_roots(), start=1):
|
||||
base = os.path.basename(os.path.normpath(root)) or f"root{index}"
|
||||
count = counts.get(base, 0) + 1
|
||||
counts[base] = count
|
||||
labels[os.path.realpath(root)] = base if count == 1 else f"{base}#{count}"
|
||||
return labels
|
||||
|
||||
|
||||
def _is_root(path: str) -> bool:
|
||||
real = os.path.realpath(_safe_abspath(path))
|
||||
return any(real == os.path.realpath(root) for root in browser_roots())
|
||||
|
||||
|
||||
def _root_label_for_path(path: str) -> str:
|
||||
current = os.path.realpath(_safe_abspath(path))
|
||||
for root_real, label in _browser_root_labels().items():
|
||||
if current == root_real or current.startswith(root_real + os.sep):
|
||||
return label
|
||||
return "refmods"
|
||||
|
||||
|
||||
def _safe_abspath(path: str) -> str:
|
||||
return os.path.abspath(os.path.expanduser(path))
|
||||
|
||||
|
||||
def _resolve_browser_path(path: str) -> str:
|
||||
raw = str(path or "").strip()
|
||||
if not raw:
|
||||
return browser_root()
|
||||
expanded = os.path.expanduser(raw)
|
||||
if os.path.isabs(expanded):
|
||||
return os.path.abspath(expanded)
|
||||
normalized = raw.replace("\\", os.sep).replace("/", os.sep)
|
||||
for root in browser_roots():
|
||||
candidate = os.path.abspath(os.path.join(root, normalized))
|
||||
if os.path.exists(candidate):
|
||||
return candidate
|
||||
return os.path.abspath(os.path.join(browser_root(), normalized))
|
||||
|
||||
|
||||
def _is_under_root(path: str) -> bool:
|
||||
current = os.path.realpath(_safe_abspath(path))
|
||||
for root_dir in browser_roots():
|
||||
root = os.path.realpath(root_dir)
|
||||
if current == root or current.startswith(root + os.sep):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def safe_dir_path(path: str = "") -> str:
|
||||
if not path:
|
||||
return browser_root()
|
||||
current = _resolve_browser_path(path)
|
||||
if not _is_under_root(current):
|
||||
raise ValueError("Path is outside models/refmods")
|
||||
if not os.path.isdir(current):
|
||||
raise ValueError("Folder not found")
|
||||
return current
|
||||
|
||||
|
||||
def safe_file_path(path: str) -> str:
|
||||
if not path:
|
||||
raise ValueError("Missing path")
|
||||
current = _resolve_browser_path(path)
|
||||
if not _is_under_root(current):
|
||||
raise ValueError("Path is outside models/refmods")
|
||||
if not os.path.isfile(current):
|
||||
raise ValueError("File not found")
|
||||
return current
|
||||
|
||||
|
||||
def _strip_known_suffix(path_no_ext: str) -> tuple[str, str | None, str | None]:
|
||||
name = os.path.basename(path_no_ext)
|
||||
lower_name = name.lower()
|
||||
for suffix in VISUAL_SUFFIXES:
|
||||
if lower_name.endswith(suffix):
|
||||
return (path_no_ext[:-len(suffix)], "visual", suffix)
|
||||
for suffix in AUDIO_SUFFIXES:
|
||||
if lower_name.endswith(suffix):
|
||||
return (path_no_ext[:-len(suffix)], "audio", suffix)
|
||||
return (path_no_ext, None, None)
|
||||
|
||||
|
||||
def paired_mod_path(path_no_ext: str, target_kind: str) -> Optional[str]:
|
||||
base, current_kind, _suffix = _strip_known_suffix(path_no_ext)
|
||||
if current_kind is None:
|
||||
return None
|
||||
current_path = path_no_ext + ".safetensors"
|
||||
suffixes = VISUAL_FILE_SUFFIXES if target_kind == "visual" else AUDIO_FILE_SUFFIXES
|
||||
for suffix in suffixes:
|
||||
candidate = base + suffix + ".safetensors"
|
||||
if candidate != current_path and os.path.isfile(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _find_preview(path_no_ext: str) -> Optional[str]:
|
||||
candidates = [path_no_ext]
|
||||
base, kind, _suffix = _strip_known_suffix(path_no_ext)
|
||||
if kind is not None:
|
||||
candidates.insert(0, base)
|
||||
for stem in candidates:
|
||||
for ext in PREVIEW_EXTS:
|
||||
candidate = stem + ext
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _display_mod_name(path_no_ext: str) -> str:
|
||||
base, kind, _suffix = _strip_known_suffix(path_no_ext)
|
||||
if kind is not None:
|
||||
name = os.path.basename(base)
|
||||
return name or os.path.basename(path_no_ext)
|
||||
return os.path.basename(path_no_ext)
|
||||
|
||||
|
||||
def _mod_entry(path: str) -> Optional[Dict]:
|
||||
if not path.lower().endswith(".safetensors"):
|
||||
return None
|
||||
path_no_ext = path[:-len(".safetensors")]
|
||||
meta = read_refmod_meta(path_no_ext)
|
||||
if meta is None:
|
||||
return None
|
||||
caps = refmod_capabilities(meta)
|
||||
meta_kind = str(caps.get("kind", "") or "")
|
||||
if meta_kind not in ("image", "video", "audio", "bundle"):
|
||||
return None
|
||||
_base, paired_kind, _suffix = _strip_known_suffix(path_no_ext)
|
||||
paired_visual = paired_mod_path(path_no_ext, "visual")
|
||||
paired_audio = paired_mod_path(path_no_ext, "audio")
|
||||
paired_split = bool(paired_visual or paired_audio)
|
||||
if paired_kind == "audio" and paired_visual:
|
||||
return None
|
||||
has_visual = bool(caps.get("has_visual", False))
|
||||
has_audio = bool(caps.get("has_audio", False))
|
||||
if paired_split:
|
||||
has_visual = True
|
||||
has_audio = True
|
||||
preview = _find_preview(path_no_ext) if has_visual else None
|
||||
return {
|
||||
"name": _display_mod_name(path_no_ext) + ".safetensors",
|
||||
"path": path,
|
||||
"preview_path": preview,
|
||||
"kind": meta_kind or "unknown",
|
||||
"has_visual": has_visual,
|
||||
"has_audio": has_audio,
|
||||
"paired_split": paired_split,
|
||||
"concept_type": str(meta.get("concept_type", "generic") or "generic"),
|
||||
"description": str(meta.get("description", "") or ""),
|
||||
}
|
||||
|
||||
|
||||
def _scan_dir(current: str) -> tuple[List[Dict], List[Dict]]:
|
||||
dirs: List[Dict] = []
|
||||
mods: List[Dict] = []
|
||||
|
||||
try:
|
||||
entries = list(os.scandir(current))
|
||||
except PermissionError as exc:
|
||||
raise ValueError("Access denied") from exc
|
||||
|
||||
for entry in entries:
|
||||
name = entry.name
|
||||
if name.startswith(".") or name in HIDDEN_BROWSER_DIRS:
|
||||
continue
|
||||
try:
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
dirs.append({"name": name, "path": os.path.abspath(entry.path)})
|
||||
elif entry.is_file(follow_symlinks=False) and name.lower().endswith(".safetensors"):
|
||||
item = _mod_entry(os.path.abspath(entry.path))
|
||||
if item is not None:
|
||||
mods.append(item)
|
||||
except OSError:
|
||||
continue
|
||||
return dirs, mods
|
||||
|
||||
|
||||
def _annotate_merged_duplicates(items: List[Dict]) -> None:
|
||||
counts: Dict[str, int] = {}
|
||||
for item in items:
|
||||
key = str(item.get("name", "")).lower()
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
for item in items:
|
||||
key = str(item.get("name", "")).lower()
|
||||
if counts.get(key, 0) > 1:
|
||||
item["name"] = f"{item['name']} [{_root_label_for_path(str(item.get('path', '')))}]"
|
||||
|
||||
|
||||
def list_browser_dir(path: str = "") -> Dict:
|
||||
current = safe_dir_path(path)
|
||||
|
||||
if _is_root(current):
|
||||
dirs: List[Dict] = []
|
||||
mods: List[Dict] = []
|
||||
for root in browser_roots():
|
||||
root_dirs, root_mods = _scan_dir(root)
|
||||
dirs.extend(root_dirs)
|
||||
mods.extend(root_mods)
|
||||
_annotate_merged_duplicates(dirs)
|
||||
_annotate_merged_duplicates(mods)
|
||||
current = browser_root()
|
||||
parent_path = None
|
||||
else:
|
||||
dirs, mods = _scan_dir(current)
|
||||
parent = os.path.dirname(current.rstrip("\\/"))
|
||||
if parent and _is_root(parent):
|
||||
parent_path = browser_root()
|
||||
elif parent and _is_under_root(parent) and parent != current:
|
||||
parent_path = parent
|
||||
else:
|
||||
parent_path = None
|
||||
|
||||
dirs.sort(key=lambda item: item["name"].lower())
|
||||
mods.sort(key=lambda item: item["name"].lower())
|
||||
return {
|
||||
"root": browser_root(),
|
||||
"current_path": current,
|
||||
"parent_path": parent_path,
|
||||
"dirs": dirs,
|
||||
"mods": mods,
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Shared helpers for the RefMod pack: media loading and the refmods folder."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif"}
|
||||
|
||||
|
||||
def refmods_dir() -> str:
|
||||
"""ComfyUI models/refmods — the mod storage folder, created on first use.
|
||||
|
||||
Sits next to loras/, unet/ etc. instead of inside the pack folder, so mods
|
||||
live in the standard model tree.
|
||||
"""
|
||||
import folder_paths
|
||||
d = os.path.join(folder_paths.models_dir, "refmods")
|
||||
os.makedirs(d, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def refmods_dirs() -> List[str]:
|
||||
"""All RefMod folders ComfyUI knows about, write folder first.
|
||||
|
||||
Includes folders declared under ``refmods:`` in extra_model_paths.yaml
|
||||
(or --extra-model-paths-config). Duplicates are removed by real path,
|
||||
since a root may be a symlink to another one.
|
||||
"""
|
||||
import folder_paths
|
||||
|
||||
candidates = [refmods_dir()]
|
||||
try:
|
||||
candidates += list(folder_paths.get_folder_paths("refmods"))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
roots: List[str] = []
|
||||
seen = set()
|
||||
for directory in candidates:
|
||||
if not os.path.isdir(directory):
|
||||
continue
|
||||
real = os.path.realpath(directory)
|
||||
if real in seen:
|
||||
continue
|
||||
seen.add(real)
|
||||
roots.append(os.path.abspath(directory))
|
||||
return roots
|
||||
|
||||
|
||||
def list_image_files(folder: str) -> List[str]:
|
||||
"""Images directly under ``folder`` (top level only), sorted by name."""
|
||||
images = []
|
||||
if os.path.isdir(folder):
|
||||
for fn in sorted(os.listdir(folder)):
|
||||
ext = os.path.splitext(fn)[1].lower()
|
||||
p = os.path.join(folder, fn)
|
||||
if os.path.isfile(p):
|
||||
if ext in IMAGE_EXTS:
|
||||
images.append(p)
|
||||
return images
|
||||
|
||||
|
||||
def load_image_file(path: str, max_edge: Optional[int] = None) -> torch.Tensor:
|
||||
"""Load one image file -> [1, H, W, 3] float32 in [0, 1]."""
|
||||
import numpy as np
|
||||
from PIL import Image, ImageOps
|
||||
with Image.open(path) as img:
|
||||
img = ImageOps.exif_transpose(img)
|
||||
img = img.convert("RGB")
|
||||
w, h = img.size
|
||||
if max_edge is not None:
|
||||
scale = min(1.0, max_edge / max(w, h))
|
||||
if scale < 1.0:
|
||||
img = img.resize((max(1, round(w * scale)), max(1, round(h * scale))),
|
||||
Image.LANCZOS)
|
||||
arr = torch.from_numpy(np.asarray(img).copy()).float() / 255.0
|
||||
return arr.unsqueeze(0) # [1, H, W, 3]
|
||||
|
||||
|
||||
def _prompt_hint(loads) -> str:
|
||||
"""Merge loaded mods' concept_type + description into one prompt-ready string.
|
||||
|
||||
e.g. "identity: ginger woman, tattooed neck, black lipstick; pose_motion:
|
||||
slow twirl into camera, hair whipping". Concat this onto your positive
|
||||
prompt (a string-concat node ahead of CLIP Text Encode) instead of
|
||||
retyping each mod's description by hand. Mods with no description are
|
||||
skipped — a bare concept_type with nothing to say isn't a useful clue.
|
||||
"""
|
||||
parts = []
|
||||
for mod, _strength in loads:
|
||||
if mod.description:
|
||||
parts.append(f"{mod.concept_type}: {mod.description}")
|
||||
return "; ".join(parts)
|
||||
+958
@@ -0,0 +1,958 @@
|
||||
"""
|
||||
core.py — "RefMod" adapter for MiniMax H3 (no training required)
|
||||
|
||||
The reference-video path of MiniMax H3 works by injecting *reference tokens*
|
||||
into the packed sequence: the ref is VAE-encoded, patchified, projected and
|
||||
placed on the 3D RoPE grid, then every DiT block attends to it. A video ref
|
||||
is expensive because it contributes thousands of tokens.
|
||||
|
||||
Mode naming: ``encode`` (was ``full``) = straight full-res VAE encode;
|
||||
``training`` (was ``pooled``) = compressed grid refined by gradient steps
|
||||
(the only mode that "trains" — the refinement loop). The old names are
|
||||
accepted everywhere as legacy aliases so saved workflows and mods keep
|
||||
working.
|
||||
|
||||
A RefMod is the same reference, compressed to a handful of tokens:
|
||||
|
||||
* the ref is VAE-encoded to its full latent [1, 24, T, H, W],
|
||||
* the latent is average-pooled to a tiny grid (default 4x4) and a few
|
||||
latent frames, so the patchified token count drops to ~4-16,
|
||||
* optionally the small latent is refined with a few gradient steps that
|
||||
reconstruct the full latent (still no model weights involved).
|
||||
|
||||
At generation time the pooled latent is handed back to the model through the
|
||||
native ``refs`` payload, so it flows through the exact same per-block
|
||||
attention machinery as a full reference — the only thing that changes is the
|
||||
token budget.
|
||||
|
||||
This is the MiniMax H3 analog of the LTX "Mod" concept tokens + per-block
|
||||
injection: instead of a trained hypernetwork predicting AdaLN deltas, the
|
||||
model's own cross-attention over the compressed ref tokens does the work.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
# Metadata key embedded in the safetensors header (keeps a mod in one file,
|
||||
# so it can be shared/uploaded as a single artifact).
|
||||
META_KEY = "refmod_meta"
|
||||
|
||||
# What a mod is standing in for. This is documentation + a routing hint, not
|
||||
# a hard constraint on the tensors — but it drives two things: (1) the
|
||||
# "identity in training mode" warning in nodes.py, and (2) the loader's
|
||||
# prompt_hint output, which merges each loaded mod's concept_type +
|
||||
# description into a string you can concat onto your CLIP Text Encode prompt.
|
||||
# The model only reads these labels through the text prompt, so prompt_hint is
|
||||
# the only place they are surfaced.
|
||||
CONCEPT_TYPES = (
|
||||
"generic", # unspecified / mixed
|
||||
"identity", # a specific person/character — use mode="encode" for this
|
||||
"pose_motion", # a pose, dance, gesture, camera move
|
||||
"clothing", # an outfit / garment, decoupled from who's wearing it
|
||||
"background", # environment / set / location plate
|
||||
"voice",
|
||||
"singing",
|
||||
"music_style",
|
||||
"sound_fx",
|
||||
"ambience",
|
||||
"style", # look/grade/animation style, not a concrete subject
|
||||
)
|
||||
|
||||
AUDIO_CONCEPT_TYPES = ("voice", "singing", "music_style", "sound_fx", "ambience")
|
||||
|
||||
|
||||
def _blur_latent(z: torch.Tensor, factor: int = 8) -> torch.Tensor:
|
||||
"""Heavy spatial low-pass (downsample then upsample) used as the
|
||||
weakening target for ``ref_block``'s ``strength`` below ``1.0``.
|
||||
|
||||
Originally this mixed toward ``torch.randn()`` noise, mirroring what the
|
||||
docstring below describes as the model's own ``visual_cond_noise_aug``
|
||||
mechanism. In practice, at the retention range people actually use this
|
||||
dial for (0.15-0.7, i.e. large noise fractions), it produces a visible
|
||||
woven/static texture instead of a gracefully weaker reference: a VAE
|
||||
latent's channels are correlated, so raw iid noise isn't read as "less
|
||||
reference," it's read as real-but-garbled content and gets rendered as
|
||||
an actual (wrong) texture. ``visual_cond_noise_aug`` in the model's own
|
||||
training is likely a small-magnitude robustness augmentation, not
|
||||
something swept this far — so copying its math doesn't reproduce its
|
||||
behavior at these strengths. A blurred copy stays on the latent
|
||||
manifold (smooth, plausible) while still discarding detail as strength
|
||||
drops, which is what "weaker reference" should actually look like.
|
||||
"""
|
||||
if z.dim() == 4:
|
||||
b, c, stereo, t = z.shape
|
||||
if t <= 1:
|
||||
return z
|
||||
flat = z.reshape(b * c * stereo, 1, t).float()
|
||||
down = F.adaptive_avg_pool1d(flat, max(1, t // factor))
|
||||
up = F.interpolate(down, size=t, mode="linear", align_corners=False)
|
||||
return up.reshape_as(z).to(z.dtype)
|
||||
if z.dim() != 5:
|
||||
raise ValueError(f"Unsupported RefMod latent shape: {tuple(z.shape)}")
|
||||
t, h, w = z.shape[2], z.shape[3], z.shape[4]
|
||||
sh, sw = max(1, h // factor), max(1, w // factor)
|
||||
down = F.adaptive_avg_pool3d(z.float(), (t, sh, sw))
|
||||
up = F.interpolate(down, size=(t, h, w), mode="trilinear", align_corners=False)
|
||||
return up.to(z.dtype)
|
||||
|
||||
|
||||
def read_refmod_meta(path_no_ext: str) -> Optional[Dict]:
|
||||
"""
|
||||
Read the metadata block from a mod/preset file without loading tensors.
|
||||
|
||||
Prefers the metadata embedded in the safetensors header; falls back to
|
||||
the legacy sidecar ``{path}.json`` written by older versions.
|
||||
"""
|
||||
try:
|
||||
with safe_open(path_no_ext + ".safetensors", framework="pt") as f:
|
||||
meta = f.metadata()
|
||||
if meta:
|
||||
for key in (META_KEY, "audio_refmod_meta"):
|
||||
if key in meta:
|
||||
return json.loads(meta[key])
|
||||
except Exception:
|
||||
pass
|
||||
jpath = path_no_ext + ".json"
|
||||
if os.path.isfile(jpath):
|
||||
try:
|
||||
with open(jpath) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def refmod_bundle_members(meta: Dict) -> List[Dict]:
|
||||
if not isinstance(meta, dict) or meta.get("_format_version") != 5 or meta.get("kind") != "bundle":
|
||||
raise ValueError("Unsupported RefMod bundle format.")
|
||||
refs = meta.get("members")
|
||||
if not isinstance(refs, list) or not refs or len(refs) > 256:
|
||||
raise ValueError("A RefMod bundle must contain 1-256 members.")
|
||||
for ref in refs:
|
||||
if not isinstance(ref, dict) or ref.get("kind") not in ("image", "video", "audio"):
|
||||
raise ValueError("Invalid RefMod bundle member.")
|
||||
return refs
|
||||
|
||||
|
||||
def refmod_capabilities(meta: Optional[Dict]) -> Dict[str, object]:
|
||||
caps: Dict[str, object] = {
|
||||
"kind": "unknown",
|
||||
"has_visual": False,
|
||||
"has_audio": False,
|
||||
"member_count": 0,
|
||||
}
|
||||
if not isinstance(meta, dict):
|
||||
return caps
|
||||
kind = str(meta.get("kind", "") or "unknown")
|
||||
caps["kind"] = kind
|
||||
if kind == "bundle":
|
||||
refs = refmod_bundle_members(meta)
|
||||
caps["member_count"] = len(refs)
|
||||
caps["has_visual"] = any(ref.get("kind") in ("image", "video") for ref in refs)
|
||||
caps["has_audio"] = any(ref.get("kind") == "audio" for ref in refs)
|
||||
elif kind in ("image", "video"):
|
||||
caps["has_visual"] = True
|
||||
caps["has_audio"] = int(meta.get("ref_audio_t", 0) or 0) > 0
|
||||
caps["member_count"] = 1 + (1 if caps["has_audio"] else 0)
|
||||
elif kind == "audio":
|
||||
caps["has_audio"] = True
|
||||
caps["member_count"] = 1
|
||||
return caps
|
||||
|
||||
|
||||
def _standalone_audio_member_meta(meta: Dict) -> Dict:
|
||||
name = str(meta.get("name", "refmod") or "refmod")
|
||||
if not name.lower().endswith("_audio"):
|
||||
name = f"{name}_Audio"
|
||||
return {
|
||||
**meta,
|
||||
"name": name,
|
||||
"kind": "audio",
|
||||
"latent_h": 0,
|
||||
"latent_w": 0,
|
||||
"latent_t": int(meta.get("ref_audio_t", 0) or 0),
|
||||
"concept_type": str(meta.get("audio_concept_type", meta.get("concept_type", "voice")) or "voice"),
|
||||
"audio_concept_type": str(meta.get("audio_concept_type", "") or ""),
|
||||
"ref_audio_t": int(meta.get("ref_audio_t", 0) or 0),
|
||||
"sample_rate": int(meta.get("sample_rate", 32000) or 32000),
|
||||
}
|
||||
|
||||
|
||||
def load_refmods_from_file(path_no_ext: str, device: str = "cpu") -> List["H3RefMod"]:
|
||||
meta = read_refmod_meta(path_no_ext)
|
||||
if not isinstance(meta, dict):
|
||||
raise ValueError(
|
||||
f"{path_no_ext}.safetensors has no RefMod metadata "
|
||||
f"(header key '{META_KEY}' or sidecar .json missing)."
|
||||
)
|
||||
if meta.get("kind") == "bundle":
|
||||
loaded: List[H3RefMod] = []
|
||||
refs = refmod_bundle_members(meta)
|
||||
with safe_open(path_no_ext + ".safetensors", framework="pt", device=device) as f:
|
||||
for idx, member_meta in enumerate(refs):
|
||||
latent = f.get_tensor(f"ref_{idx}").clone()
|
||||
enc_frames = _read_encoder_images(member_meta, f.get_tensor)
|
||||
loaded.append(H3RefMod.from_metadata(member_meta, latent, enc_frames=enc_frames))
|
||||
return loaded
|
||||
|
||||
tensors = load_file(path_no_ext + ".safetensors", device=device)
|
||||
if "latent" not in tensors:
|
||||
raise ValueError(f"{path_no_ext}.safetensors is missing its latent tensor.")
|
||||
latent = tensors["latent"].clone()
|
||||
audio_latent = tensors["audio_latent"].clone() if "audio_latent" in tensors else None
|
||||
if meta.get("kind") == "audio":
|
||||
return [H3RefMod.from_metadata(meta, latent)]
|
||||
|
||||
visual_meta = dict(meta)
|
||||
visual_meta["ref_audio_t"] = 0
|
||||
visual_meta["audio_concept_type"] = ""
|
||||
enc_frames = _read_encoder_images(meta, tensors.__getitem__)
|
||||
loaded = [H3RefMod.from_metadata(visual_meta, latent, enc_frames=enc_frames)]
|
||||
if audio_latent is not None and int(meta.get("ref_audio_t", 0) or 0) > 0:
|
||||
loaded.append(H3RefMod.from_metadata(_standalone_audio_member_meta(meta), audio_latent))
|
||||
return loaded
|
||||
|
||||
|
||||
def save_refmod_bundle(path_no_ext: str, name: str, mods: List["H3RefMod"]) -> str:
|
||||
unique = list({id(mod): mod for mod in mods}.values())
|
||||
if not unique:
|
||||
raise ValueError("RefMod bundle is empty.")
|
||||
if sum(bool(mod.enc_frames) for mod in unique) > 1:
|
||||
raise ValueError("Save RefMods with encoder pictures in separate files when more than one member has pictures.")
|
||||
metadata = {
|
||||
"_format_version": 5,
|
||||
"kind": "bundle",
|
||||
"name": str(name or "refmod"),
|
||||
"members": [mod.metadata() for mod in unique],
|
||||
}
|
||||
tensors = {
|
||||
f"ref_{idx}": mod.latent.detach().cpu().contiguous().clone()
|
||||
for idx, mod in enumerate(unique)
|
||||
}
|
||||
for mod in unique:
|
||||
tensors.update({key: value.contiguous() for key, value in mod.enc_frames.items()})
|
||||
destination = path_no_ext + ".safetensors"
|
||||
directory = os.path.dirname(destination) or "."
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix=".refmod-", suffix=".tmp", dir=directory)
|
||||
os.close(fd)
|
||||
try:
|
||||
save_file(tensors, temporary, metadata={META_KEY: json.dumps(metadata)})
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
if os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
return destination
|
||||
|
||||
|
||||
def _read_encoder_images(meta, get_tensor):
|
||||
record = meta.get("encoder_images") or {}
|
||||
times = record.get("times", meta.get("enc_times", []))
|
||||
prefix = record.get("prefix", "enc_")
|
||||
return {f"enc_{index}": get_tensor(f"{prefix}{index}").detach().cpu().clone()
|
||||
for index in range(len(times))}
|
||||
|
||||
|
||||
def pack_encoder_images(mod, frames, times, fps=24.0):
|
||||
import io as bytes_io
|
||||
from PIL import Image
|
||||
|
||||
if frames.shape[0] != len(times):
|
||||
raise ValueError("Encoder picture count does not match its timestamps.")
|
||||
packed = {}
|
||||
for index, frame in enumerate(frames):
|
||||
buffer = bytes_io.BytesIO()
|
||||
pixels = (frame.detach().cpu().float().clamp(0, 1) * 255).round().to(torch.uint8).numpy()
|
||||
Image.fromarray(pixels).save(buffer, "JPEG", quality=95)
|
||||
packed[f"enc_{index}"] = torch.frombuffer(bytearray(buffer.getvalue()), dtype=torch.uint8)
|
||||
mod.enc_frames = packed
|
||||
mod.enc_times = list(times)
|
||||
mod.enc_fps = float(fps)
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Latent compression
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
MODE_ALIASES = {
|
||||
"full": "encode",
|
||||
"pooled": "training",
|
||||
"Full Reference": "encode",
|
||||
"Compressed Reference": "training",
|
||||
}
|
||||
|
||||
|
||||
def normalize_mode(mode: str) -> str:
|
||||
"""Map legacy mode names (``full``/``pooled``) to the current ones."""
|
||||
return MODE_ALIASES.get(mode, mode)
|
||||
|
||||
|
||||
def aspect_grid(pool_h: int, pool_w: int, aspect: float):
|
||||
"""Even pool grid dims that match ``aspect`` (h/w) within the dial caps.
|
||||
|
||||
The DiT patches each 2x2 latent cell into one token and the rope grid is
|
||||
area-normalized per axis, so a pooled grid that ignores the source aspect
|
||||
squishes the subject: a portrait latent (tall person) forced into a square
|
||||
16x16 grid is compressed 5x in height but only 3x in width, and comes out
|
||||
"fat". The official ref2video node never pools — it feeds the model
|
||||
aspect-correct latent dims, which is what the DiT was trained on.
|
||||
|
||||
The long edge follows the user's dial (``max(pool_h, pool_w)``) and the
|
||||
short edge is derived from the source aspect, both rounded to even for the
|
||||
2x2 patch. Square sources keep the exact dial value (16x16 stays 16x16).
|
||||
"""
|
||||
long_edge = max(pool_h, pool_w)
|
||||
if aspect >= 1.0:
|
||||
h, w = long_edge, long_edge / aspect
|
||||
else:
|
||||
w, h = long_edge, long_edge * aspect
|
||||
h = max(2, round(h / 2) * 2)
|
||||
w = max(2, round(w / 2) * 2)
|
||||
return h, w
|
||||
|
||||
|
||||
def pool_latent(
|
||||
z: torch.Tensor,
|
||||
latent_t: int,
|
||||
latent_h: int,
|
||||
latent_w: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Average-pool a VAE latent ``[1, 24, T, H, W]`` down to a tiny grid.
|
||||
|
||||
``latent_h/latent_w`` are the *latent* grid dims (the DiT patches each 2x2
|
||||
latent cell into one token), so a 4x4 latent = 2x2 = 4 tokens per frame.
|
||||
Dims must be even (the DiT's 2x2 patch). Pooling runs in fp32; the result
|
||||
keeps the input dtype.
|
||||
"""
|
||||
if z.shape[2] == latent_t and z.shape[3] == latent_h and z.shape[4] == latent_w:
|
||||
return z
|
||||
if latent_h % 2 != 0 or latent_w % 2 != 0:
|
||||
raise ValueError(f"pool_h/pool_w must be even (got {latent_h}x{latent_w})")
|
||||
pooled = F.adaptive_avg_pool3d(
|
||||
z.float(), (latent_t, latent_h, latent_w)
|
||||
)
|
||||
return pooled.to(z.dtype)
|
||||
|
||||
|
||||
def optimize_latent(
|
||||
z_small: torch.Tensor,
|
||||
z_full: torch.Tensor,
|
||||
steps: int = 150,
|
||||
lr: float = 00.02,
|
||||
device: Optional[torch.device] = None,
|
||||
progress_every: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Model-free refinement of the compressed latent.
|
||||
|
||||
Optimizes the small latent so its trilinearly upsampled reconstruction
|
||||
matches the full reference latent. This pulls the pooled representation
|
||||
closer to what the DiT would see from the full ref, with nothing but the
|
||||
tiny latent trainable (~1-2K params) and no diffusion model loaded.
|
||||
|
||||
``progress_every`` > 0 prints a ``[RefMod] identity <step>/<steps>`` line
|
||||
every N steps so long pooled-mode refinement isn't silent (the default
|
||||
``identity=500`` takes a while per ref on CPU/VRAM-bound setups).
|
||||
|
||||
Returns the refined latent detached, same shape/dtype as ``z_small``.
|
||||
"""
|
||||
if steps <= 0:
|
||||
return z_small
|
||||
device = device or z_full.device
|
||||
# Node execution runs under ComfyUI's global inference mode, which would
|
||||
# disable the autograd this refine loop needs. Re-enable it for this
|
||||
# scope and clone the inputs to drop the inference flag.
|
||||
with torch.inference_mode(False), torch.set_grad_enabled(True):
|
||||
target = z_full.clone().float().to(device)
|
||||
param = nn.Parameter(z_small.clone().float().to(device))
|
||||
opt = torch.optim.Adam([param], lr=lr)
|
||||
size = tuple(target.shape[2:])
|
||||
for i in range(steps):
|
||||
opt.zero_grad()
|
||||
up = F.interpolate(param, size=size, mode="trilinear", align_corners=False)
|
||||
loss = F.mse_loss(up, target)
|
||||
loss.backward()
|
||||
opt.step()
|
||||
if progress_every and (i + 1) % progress_every == 0:
|
||||
print(f"[RefMod] identity {i + 1}/{steps}")
|
||||
# materialize inside the scope so the result is a normal tensor, not
|
||||
# an inference-mode tensor (it gets stored in the mod and reused)
|
||||
refined = param.detach().to(z_small.dtype)
|
||||
return refined
|
||||
|
||||
|
||||
def optimize_latent_multi(z_init, targets, steps=150, lr=0.02, device=None,
|
||||
progress_every=0, strategy="grouped"):
|
||||
"""Mean reconstruction objective with bounded activation memory."""
|
||||
if not targets or steps <= 0:
|
||||
return z_init
|
||||
if strategy not in ("grouped", "stream", "resident"):
|
||||
raise ValueError("Unknown multi-reference optimization strategy.")
|
||||
device = device or z_init.device
|
||||
with torch.inference_mode(False), torch.set_grad_enabled(True):
|
||||
count = len(targets)
|
||||
if strategy == "grouped":
|
||||
groups = {}
|
||||
for target in targets:
|
||||
key = tuple(target.shape)
|
||||
value = target.detach().to(device="cpu", dtype=torch.float32).clone()
|
||||
if key in groups:
|
||||
groups[key][0].add_(value)
|
||||
groups[key][1] += 1
|
||||
else:
|
||||
groups[key] = [value, 1]
|
||||
grouped = []
|
||||
for total, group_count in groups.values():
|
||||
grouped.append((total.div_(group_count), group_count))
|
||||
elif strategy == "resident":
|
||||
resident = [target.detach().float().to(device) for target in targets]
|
||||
param = nn.Parameter(z_init.clone().float().to(device))
|
||||
opt = torch.optim.Adam([param], lr=lr)
|
||||
for step_idx in range(steps):
|
||||
opt.zero_grad()
|
||||
loss = None
|
||||
if strategy == "grouped":
|
||||
total_weight = 0
|
||||
for target_mean, group_count in grouped:
|
||||
target = target_mean.to(device)
|
||||
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
|
||||
align_corners=False)
|
||||
contribution = F.mse_loss(up, target)
|
||||
loss = contribution * group_count if loss is None else loss + contribution * group_count
|
||||
total_weight += group_count
|
||||
loss = loss / total_weight
|
||||
elif strategy == "stream":
|
||||
for target_cpu in targets:
|
||||
target = target_cpu.detach().float().to(device)
|
||||
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
|
||||
align_corners=False)
|
||||
contribution = F.mse_loss(up, target)
|
||||
loss = contribution if loss is None else loss + contribution
|
||||
loss = loss / count
|
||||
else:
|
||||
for target in resident:
|
||||
up = F.interpolate(param, size=tuple(target.shape[2:]), mode="trilinear",
|
||||
align_corners=False)
|
||||
contribution = F.mse_loss(up, target)
|
||||
loss = contribution if loss is None else loss + contribution
|
||||
loss = loss / count
|
||||
loss.backward()
|
||||
opt.step()
|
||||
if progress_every and (step_idx + 1) % progress_every == 0:
|
||||
print(f"[RefMod] identity {step_idx + 1}/{steps}")
|
||||
refined = param.detach().to(z_init.dtype)
|
||||
return refined
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Per-frame strength curve
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
# Direction names describe where the *concept* shows up in the output, which
|
||||
# is the mirror of the strength envelope over the ref's own latent timeline:
|
||||
# concept_at_end locks the ref's literal footage at the START and releases it
|
||||
# toward the end (was "decrease"), concept_at_start opens free and locks onto
|
||||
# the ref near the END (was "increase"). shape is how the envelope travels
|
||||
# between its endpoints; value is the non-zero endpoint ("user input").
|
||||
CURVE_DIRECTIONS = ("constant", "concept_at_start", "concept_at_middle",
|
||||
"concept_at_end", "concept_at_ends")
|
||||
CURVE_SHAPES = ("linear", "ease", "sigmoid", "tanh", "quadratic", "cubic",
|
||||
"exponential", "stair", "elastic", "bump", "dip")
|
||||
|
||||
# old single-combo preset names -> (direction, shape, value) so workflows
|
||||
# saved before the curve was split still resolve
|
||||
_LEGACY_CURVES = {
|
||||
"flat": ("constant", "linear", 1.0),
|
||||
"fade_in": ("concept_at_start", "linear", 1.0),
|
||||
"fade_out": ("concept_at_end", "linear", 1.0),
|
||||
"bump": ("concept_at_start", "bump", 1.0),
|
||||
"dip": ("constant", "dip", 1.0),
|
||||
}
|
||||
|
||||
|
||||
def _ease(shape: str, x: float) -> float:
|
||||
"""Shape a progress ``x`` in [0, 1] per the easing name (may overshoot)."""
|
||||
if shape == "linear":
|
||||
return x
|
||||
if shape == "ease": # smoothstep
|
||||
return x * x * (3.0 - 2.0 * x)
|
||||
if shape == "sigmoid": # logistic S-curve
|
||||
return 1.0 / (1.0 + math.exp(-12.0 * (x - 0.5)))
|
||||
if shape == "tanh": # sharper S-curve (steeper knee than sigmoid)
|
||||
return 0.5 * (math.tanh(8.0 * (x - 0.5)) + 1.0)
|
||||
if shape == "quadratic":
|
||||
return x * x
|
||||
if shape == "cubic":
|
||||
return x * x * x
|
||||
if shape == "exponential":
|
||||
return 2.0 ** x - 1.0
|
||||
if shape == "stair":
|
||||
return min(1.0, math.floor(x * 4) / 3.0)
|
||||
if shape == "elastic":
|
||||
if x <= 0.0:
|
||||
return 0.0
|
||||
if x >= 1.0:
|
||||
return 1.0
|
||||
return 2.0 ** (-10.0 * x) * math.sin((x * 10.0 - 0.75) * (2.0 * math.pi / 3.0)) + 1.0
|
||||
if shape == "bump":
|
||||
return 1.0 - abs(2.0 * x - 1.0)
|
||||
if shape == "dip":
|
||||
return abs(2.0 * x - 1.0)
|
||||
return x
|
||||
|
||||
|
||||
def curve_strengths(spec, t: int) -> Optional[List[float]]:
|
||||
"""Resolve a curve spec to ``t`` per-frame strength multipliers in [0, 1].
|
||||
|
||||
Kept as plain Python so the Apply node exposes the curve as normal
|
||||
combo/float widgets instead of depending on ComfyUI's new Curve widget.
|
||||
Accepts:
|
||||
|
||||
* a ``(direction, shape, value)`` tuple — direction is ``constant``
|
||||
(one strength everywhere), ``concept_at_start`` (0 -> value, a
|
||||
crescent — the concept shows early; legacy ``increase``),
|
||||
``concept_at_end`` (value -> 0, a decrescent — the concept pops at
|
||||
the end; legacy ``decrease``), ``concept_at_middle`` ([0..1..0] —
|
||||
concept peaks mid-timeline) or ``concept_at_ends`` ([1..0..1] —
|
||||
concept at both ends); shape is how the envelope travels between
|
||||
its endpoints (see ``_ease``); value is the non-zero endpoint
|
||||
(1.0 = full strength there);
|
||||
* a legacy preset name (``flat``/``fade_in``/``fade_out``/``bump``/
|
||||
``dip``) from before the split;
|
||||
* a list of per-frame floats (len == t), used as-is;
|
||||
* a list of ``(x, y)`` control points (x in [0, 1]), linearly
|
||||
interpolated across the frames;
|
||||
* any object with ``interp(x) -> float`` (legacy CurveInput saved in an
|
||||
old workflow).
|
||||
|
||||
Returns None for flat/no curve so the caller keeps its single-strength
|
||||
path unchanged (a ``constant`` curve at value 1.0 included).
|
||||
"""
|
||||
if t <= 1 or spec is None or spec == "":
|
||||
return None
|
||||
if isinstance(spec, str):
|
||||
legacy = _LEGACY_CURVES.get(spec)
|
||||
return curve_strengths(legacy, t) if legacy is not None else None
|
||||
if (isinstance(spec, tuple) and len(spec) == 3
|
||||
and isinstance(spec[0], str) and isinstance(spec[1], str)):
|
||||
direction, shape, value = spec
|
||||
value = float(value)
|
||||
if direction == "constant":
|
||||
# linear = flat at ``value`` (1.0 == no curve); the other shapes
|
||||
# modulate around the constant, so constant + dip is a trough,
|
||||
# constant + bump is a peak (both endpoints equal)
|
||||
if shape == "linear":
|
||||
return None if value >= 1.0 else [value] * t
|
||||
p = [_ease(shape, i / (t - 1)) for i in range(t)]
|
||||
return [max(0.0, min(1.0, value * y)) for y in p]
|
||||
p = [_ease(shape, i / (t - 1)) for i in range(t)]
|
||||
# legacy "increase"/"decrease" (pre-rename workflows) map onto the same
|
||||
# envelopes as the concept-placement names
|
||||
if direction in ("increase", "concept_at_start"):
|
||||
return [max(0.0, min(1.0, value * y)) for y in p]
|
||||
if direction in ("decrease", "concept_at_end"):
|
||||
return [max(0.0, min(1.0, value * (1.0 - y))) for y in p]
|
||||
if direction == "concept_at_middle": # [0..1..0]: concept peaks mid-timeline
|
||||
return [max(0.0, min(1.0, value * (1.0 - abs(2.0 * y - 1.0)))) for y in p]
|
||||
if direction == "concept_at_ends": # [1..0..1]: concept at both ends
|
||||
return [max(0.0, min(1.0, value * abs(2.0 * y - 1.0))) for y in p]
|
||||
return None # unknown direction: treat as flat
|
||||
if isinstance(spec, (list, tuple)):
|
||||
if len(spec) == t and all(isinstance(v, (int, float)) for v in spec):
|
||||
return [float(v) for v in spec]
|
||||
pts = [(float(x), float(y)) for x, y in spec
|
||||
if isinstance(x, (int, float)) and isinstance(y, (int, float))]
|
||||
if pts:
|
||||
pts = sorted(pts, key=lambda p: p[0])
|
||||
out = []
|
||||
for i in range(t):
|
||||
x = i * (1.0 / (t - 1))
|
||||
if x <= pts[0][0]:
|
||||
out.append(pts[0][1])
|
||||
elif x >= pts[-1][0]:
|
||||
out.append(pts[-1][1])
|
||||
else:
|
||||
for (x0, y0), (x1, y1) in zip(pts, pts[1:]):
|
||||
if x0 <= x <= x1:
|
||||
out.append(y0 + (y1 - y0) * (x - x0) / (x1 - x0))
|
||||
break
|
||||
return out
|
||||
interp = getattr(spec, "interp", None)
|
||||
if callable(interp):
|
||||
return [float(interp(i / (t - 1))) for i in range(t)]
|
||||
return None
|
||||
|
||||
|
||||
def curve_value_at(spec, x: float) -> float:
|
||||
"""Per-progress strength of a curve spec at one point ``x`` in [0, 1].
|
||||
|
||||
The continuous analog of ``curve_strengths`` — same direction/shape/value
|
||||
math, but evaluated at a single progress value instead of discretized over
|
||||
frames. The step-curve node uses it to re-mix the ref latents once per
|
||||
denoising step, where ``x`` is the denoise progress (0 = first step,
|
||||
1 = last step). Returns 1.0 (no weakening) for anything unrecognized.
|
||||
"""
|
||||
if spec is None or spec == "":
|
||||
return 1.0
|
||||
if isinstance(spec, str):
|
||||
legacy = _LEGACY_CURVES.get(spec)
|
||||
return curve_value_at(legacy, x) if legacy is not None else 1.0
|
||||
if (isinstance(spec, tuple) and len(spec) == 3
|
||||
and isinstance(spec[0], str) and isinstance(spec[1], str)):
|
||||
direction, shape, value = spec
|
||||
value = float(value)
|
||||
if direction == "constant":
|
||||
if shape == "linear":
|
||||
return value
|
||||
return max(0.0, min(1.0, value * _ease(shape, x)))
|
||||
y = _ease(shape, x)
|
||||
if direction in ("increase", "concept_at_start"):
|
||||
return max(0.0, min(1.0, value * y))
|
||||
if direction in ("decrease", "concept_at_end"):
|
||||
return max(0.0, min(1.0, value * (1.0 - y)))
|
||||
if direction == "concept_at_middle": # [0..1..0]: concept peaks mid-timeline
|
||||
return max(0.0, min(1.0, value * (1.0 - abs(2.0 * y - 1.0))))
|
||||
if direction == "concept_at_ends": # [1..0..1]: concept at both ends
|
||||
return max(0.0, min(1.0, value * abs(2.0 * y - 1.0)))
|
||||
return 1.0
|
||||
|
||||
|
||||
def dedup_frame_indices(z: torch.Tensor, threshold: float = 0.02) -> List[int]:
|
||||
"""Indices of latent frames kept by greedy temporal dedup.
|
||||
|
||||
Each frame is compared to the last *kept* frame: if the mean-abs
|
||||
difference between them, normalized by the frames' own magnitude, is
|
||||
below ``threshold`` the frame is dropped as a (near-)duplicate. Video
|
||||
refs (a dance loop, a static shot, a talking head) contain long runs of
|
||||
frames that differ only by codec noise — each one still costs a token
|
||||
per spatial patch in every DiT block, so cutting them is the cheapest
|
||||
way to honor a token cap. Stacked image refs survive: different
|
||||
angles/expressions land well above the threshold.
|
||||
"""
|
||||
t = z.shape[2]
|
||||
if t <= 1:
|
||||
return list(range(t))
|
||||
flat = z[0].float() # [24, T, H, W]
|
||||
kept = [0]
|
||||
prev = flat[:, 0]
|
||||
for i in range(1, t):
|
||||
cur = flat[:, i]
|
||||
denom = (cur.abs().mean() + prev.abs().mean()) / 2 + 1e-6
|
||||
diff = (cur - prev).abs().mean() / denom
|
||||
if diff >= threshold:
|
||||
kept.append(i)
|
||||
prev = cur
|
||||
return kept
|
||||
|
||||
|
||||
def fit_token_budget(latent: torch.Tensor, budget: int, label: str) -> torch.Tensor:
|
||||
"""Bring a stacked latent's injected token count under ``budget``.
|
||||
|
||||
Tokens = (H/2)*(W/2) per frame (the DiT's 2x2 patch). When the stack
|
||||
exceeds the budget it is cut in two cheap, loss-ordered passes:
|
||||
|
||||
1. temporal dedup — drop latent frames that are (near-)identical to
|
||||
the last kept frame (free: they carry no new reference info);
|
||||
2. budget-fit resample — uniformly subsample the remaining frames down
|
||||
to the largest count that fits the budget.
|
||||
|
||||
The spatial dims are never touched: resizing a latent's H/W changes the
|
||||
rope grid and degrades the reference, so time is the only honest lever.
|
||||
``label`` is the mod name for the console notes.
|
||||
"""
|
||||
h, w = latent.shape[3], latent.shape[4]
|
||||
per_frame = (h // 2) * (w // 2)
|
||||
t = latent.shape[2]
|
||||
if per_frame * t <= budget:
|
||||
return latent
|
||||
kept = dedup_frame_indices(latent)
|
||||
if len(kept) < t:
|
||||
print(f"[RefMod] {label}: over {budget}-token cap, "
|
||||
f"dropped {t - len(kept)} near-duplicate frame(s) "
|
||||
f"({t} -> {len(kept)})")
|
||||
latent = latent[:, :, kept]
|
||||
t = latent.shape[2]
|
||||
if per_frame * t > budget:
|
||||
fit_t = max(1, budget // per_frame)
|
||||
if fit_t < t:
|
||||
idx = torch.linspace(0, t - 1, fit_t).round().long()
|
||||
latent = latent[:, :, idx]
|
||||
print(f"[RefMod] {label}: still over {budget}-token cap, "
|
||||
f"resampled {t} -> {fit_t} frame(s) to fit")
|
||||
if per_frame * latent.shape[2] > budget:
|
||||
print(f"[RefMod] {label}: one frame alone is {per_frame} tokens > "
|
||||
f"{budget} — lower ref_resolution or pool_h/pool_w to meet "
|
||||
f"the cap.")
|
||||
return latent
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# H3RefMod — the saved artifact
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
@dataclass
|
||||
class H3RefMod:
|
||||
"""
|
||||
A compressed reference for MiniMax H3.
|
||||
|
||||
``latent`` is the VAE latent ``[1, 24, latent_t, latent_h, latent_w]`` — a
|
||||
full-resolution encode (``mode="encode"``) or a pooled thumbnail
|
||||
(``mode="training"``). ``kind`` is ``"image"`` (single latent frame) or
|
||||
``"video"`` (a few frames), matching the native ref block kinds the
|
||||
model's ``PackedLayout`` understands.
|
||||
|
||||
``mode="encode"`` stores the encode at the resolution the official ref2video
|
||||
path uses, so the injected ref carries real identity detail;
|
||||
``mode="training"`` stores a tiny average-pooled grid refined by gradient
|
||||
steps (concept/motion, or identity at high pool sizes).
|
||||
"""
|
||||
|
||||
name: str
|
||||
kind: str
|
||||
latent: torch.Tensor
|
||||
latent_h: int = 4
|
||||
latent_w: int = 4
|
||||
latent_t: int = 1
|
||||
mode: str = "training"
|
||||
source: str = "" # "image" | "video" | "manual"
|
||||
source_shape: str = "" # original latent dims as "TxHxW"
|
||||
pool: str = "4x4x1" # pool_t x pool_h x pool_w
|
||||
optimize_steps: int = 0
|
||||
tags: List[str] = field(default_factory=list)
|
||||
description: str = "" # optional text describing the concept (emitted by the loaders)
|
||||
concept_type: str = "generic" # what this mod represents; see CONCEPT_TYPES above
|
||||
audio_concept_type: str = "" # upstream-style audio role label for embedded audio, if present
|
||||
# Optional audio identity: the audio-VAE latent [1, 32, 2, ref_audio_t] and
|
||||
# its frame count. When present the mod is emitted as a "video_audio" (or
|
||||
# standalone "audio") ref block so the DiT attends to the soundtrack too.
|
||||
audio_latent: Optional[torch.Tensor] = None
|
||||
ref_audio_t: int = 0
|
||||
sample_rate: int = 32000
|
||||
enc_frames: Dict[str, torch.Tensor] = field(default_factory=dict)
|
||||
enc_times: List[float] = field(default_factory=list)
|
||||
enc_fps: float = 0.0
|
||||
|
||||
def __post_init__(self):
|
||||
if self.kind == "audio":
|
||||
if self.latent.ndim != 4 or tuple(self.latent.shape[:3]) != (1, 32, 2) or self.latent.shape[-1] < 1:
|
||||
raise ValueError("H3 audio RefMod must contain [1,32,2,T] with T >= 1.")
|
||||
self.latent_t = int(self.latent.shape[-1])
|
||||
self.latent_h = 0
|
||||
self.latent_w = 0
|
||||
self.audio_latent = None
|
||||
self.ref_audio_t = 0
|
||||
return
|
||||
if self.kind not in ("image", "video"):
|
||||
raise ValueError(f"kind must be 'image', 'video' or 'audio' (got {self.kind!r})")
|
||||
if self.kind == "image":
|
||||
self.latent_t = 1
|
||||
if self.audio_latent is None or self.ref_audio_t <= 0:
|
||||
self.audio_latent = None
|
||||
self.audio_concept_type = ""
|
||||
self.ref_audio_t = 0
|
||||
|
||||
# ── token budget ──────────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def token_count(self) -> int:
|
||||
"""Number of patchified tokens the mod injects into the packed sequence."""
|
||||
if self.kind == "audio":
|
||||
return 2 * self.latent_t
|
||||
per_frame = (self.latent_h // 2) * (self.latent_w // 2)
|
||||
return self.latent_t * per_frame
|
||||
|
||||
# ── native ref block ──────────────────────────────────────────────
|
||||
|
||||
def ref_block(self, strength: float = 1.0,
|
||||
curve=None, use_video: bool = True,
|
||||
use_audio: bool = True) -> Optional[Dict]:
|
||||
"""
|
||||
Build the ref block dict the model's ``PackedLayout`` / payload consumes.
|
||||
|
||||
The shape mirrors what the native ref2va nodes emit, so the pooled
|
||||
latent rides the exact same path: ``cond_video_latents`` -> patchify ->
|
||||
``video_patch_proj`` -> packed sequence with a 3D RoPE grid.
|
||||
|
||||
``strength`` weakens the ref by mixing the latent toward a heavily
|
||||
blurred copy of itself (see ``_blur_latent``) rather than toward
|
||||
zero or random noise: scaling toward zero pushes values out of the
|
||||
normalized latent distribution (reads as grey output), and mixing
|
||||
toward random noise reads as real-but-garbled content and decodes
|
||||
as a wrong texture rather than "weaker" — a blurred copy stays
|
||||
on-manifold while still discarding the specific detail that makes a
|
||||
reference strong. ``strength <= 0`` drops the block entirely (no
|
||||
tokens injected).
|
||||
|
||||
``curve`` (optional) is a per-frame strength spec for the ref's
|
||||
latent timeline — a ``(direction, shape, value)`` tuple (e.g.
|
||||
``("concept_at_end", "ease", 1.0)``), a legacy preset name, a list of
|
||||
per-frame floats, ``(x, y)`` control points, or any object with
|
||||
``interp(x)`` (see ``curve_strengths``). Each frame is mixed with
|
||||
its own strength ``retention * curve(x)`` instead of one flat value,
|
||||
so the ref can fade in/out across the video. A flat curve is
|
||||
identical to passing ``strength`` alone.
|
||||
"""
|
||||
if strength <= 0.0 or (not use_video and not use_audio):
|
||||
return None
|
||||
if self.kind == "audio":
|
||||
if not use_audio:
|
||||
return None
|
||||
latent = self.latent
|
||||
strengths = curve_strengths(curve, self.latent_t) if curve is not None else None
|
||||
if strengths is not None:
|
||||
weights = [max(0.0, min(1.0, strength * value)) for value in strengths]
|
||||
if any(value < 1.0 for value in weights):
|
||||
st = latent.new_tensor(weights).view(1, 1, 1, self.latent_t)
|
||||
latent = st * latent + (1.0 - st) * _blur_latent(latent)
|
||||
elif strength < 1.0:
|
||||
latent = strength * latent + (1.0 - strength) * _blur_latent(latent)
|
||||
return {
|
||||
"kind": "audio",
|
||||
"ref_audio_t": self.latent_t,
|
||||
"audio_latent": latent,
|
||||
}
|
||||
has_audio = use_audio and self.audio_latent is not None and self.ref_audio_t > 0
|
||||
if not use_video:
|
||||
if has_audio:
|
||||
return {
|
||||
"kind": "audio",
|
||||
"ref_audio_t": self.ref_audio_t,
|
||||
"audio_latent": self.audio_latent,
|
||||
}
|
||||
return None
|
||||
latent = self.latent
|
||||
if curve is not None and self.latent_t > 1:
|
||||
strengths = curve_strengths(curve, self.latent_t)
|
||||
if strengths is not None:
|
||||
t = self.latent_t
|
||||
st = torch.tensor(
|
||||
[max(0.0, min(1.0, strength * s)) for s in strengths],
|
||||
dtype=latent.dtype, device=latent.device)
|
||||
st = st.view(1, 1, t, 1, 1)
|
||||
latent = st * latent + (1.0 - st) * _blur_latent(latent)
|
||||
elif strength < 1.0:
|
||||
latent = strength * latent + (1.0 - strength) * _blur_latent(latent)
|
||||
block: Dict = {
|
||||
"kind": self.kind,
|
||||
"latent_h": self.latent_h,
|
||||
"latent_w": self.latent_w,
|
||||
"latent": latent,
|
||||
}
|
||||
if self.kind == "video" or has_audio:
|
||||
# The DiT only reads audio off video / video_audio / audio blocks —
|
||||
# its "image" branch ignores audio_latent. So a mod carrying audio
|
||||
# is always emitted as video_audio (an image mod is just a 1-frame
|
||||
# video), which packs the audio rows immediately before the video
|
||||
# rows on the shared cursor.
|
||||
block["kind"] = "video_audio" if has_audio else "video"
|
||||
block["latent_t"] = self.latent_t
|
||||
block["ref_audio_t"] = self.ref_audio_t if has_audio else 0
|
||||
block["audio_latent"] = self.audio_latent if has_audio else None
|
||||
return block
|
||||
|
||||
# ── serialization ─────────────────────────────────────────────────
|
||||
|
||||
def metadata(self) -> Dict:
|
||||
meta = {
|
||||
"name": self.name,
|
||||
"kind": self.kind,
|
||||
"latent_h": self.latent_h,
|
||||
"latent_w": self.latent_w,
|
||||
"latent_t": self.latent_t,
|
||||
"mode": self.mode,
|
||||
"source": self.source,
|
||||
"source_shape": self.source_shape,
|
||||
"pool": self.pool,
|
||||
"optimize_steps": self.optimize_steps,
|
||||
"tags": self.tags,
|
||||
"description": self.description,
|
||||
"concept_type": self.concept_type,
|
||||
"_format_version": 4,
|
||||
}
|
||||
if self.kind == "audio" or (self.audio_latent is not None and self.ref_audio_t > 0):
|
||||
meta["audio_concept_type"] = self.audio_concept_type
|
||||
meta["ref_audio_t"] = self.ref_audio_t
|
||||
meta["sample_rate"] = self.sample_rate
|
||||
if self.enc_frames:
|
||||
meta["enc_times"] = self.enc_times
|
||||
meta["enc_fps"] = self.enc_fps
|
||||
return meta
|
||||
|
||||
def save(self, path_no_ext: str) -> str:
|
||||
"""Save as a single ``{path}.safetensors`` with metadata in the header."""
|
||||
os.makedirs(os.path.dirname(path_no_ext) or ".", exist_ok=True)
|
||||
meta = self.metadata()
|
||||
tensors = {"latent": self.latent.contiguous()}
|
||||
tensors.update({key: value.contiguous() for key, value in self.enc_frames.items()})
|
||||
if self.audio_latent is not None and self.ref_audio_t > 0:
|
||||
tensors["audio_latent"] = self.audio_latent.contiguous()
|
||||
save_file(tensors, path_no_ext + ".safetensors",
|
||||
metadata={META_KEY: json.dumps(meta)})
|
||||
return path_no_ext + ".safetensors"
|
||||
|
||||
@classmethod
|
||||
def from_metadata(
|
||||
cls,
|
||||
meta: Dict,
|
||||
latent: torch.Tensor,
|
||||
*,
|
||||
audio_latent: Optional[torch.Tensor] = None,
|
||||
enc_frames: Optional[Dict[str, torch.Tensor]] = None,
|
||||
) -> "H3RefMod":
|
||||
if not isinstance(meta, dict):
|
||||
raise ValueError("RefMod metadata must be a dict.")
|
||||
record = meta.get("encoder_images") or {}
|
||||
return cls(
|
||||
name=meta.get("name", "refmod"),
|
||||
kind=meta.get("kind", "image"),
|
||||
latent=latent,
|
||||
latent_h=int(meta.get("latent_h", latent.shape[3] if latent.ndim == 5 else 0)),
|
||||
latent_w=int(meta.get("latent_w", latent.shape[4] if latent.ndim == 5 else 0)),
|
||||
latent_t=int(meta.get("latent_t", latent.shape[2] if latent.ndim == 5 else latent.shape[-1])),
|
||||
mode=normalize_mode(meta.get("mode", "training")),
|
||||
source=meta.get("source", ""),
|
||||
source_shape=meta.get("source_shape", ""),
|
||||
pool=meta.get("pool", ""),
|
||||
optimize_steps=int(meta.get("optimize_steps", 0)),
|
||||
tags=list(meta.get("tags", [])),
|
||||
description=str(meta.get("description", "") or ""),
|
||||
concept_type=str(meta.get("concept_type", "generic") or "generic"),
|
||||
audio_concept_type=str(meta.get("audio_concept_type", "") or ""),
|
||||
audio_latent=audio_latent,
|
||||
ref_audio_t=int(meta.get("ref_audio_t", 0) or 0) if audio_latent is not None else 0,
|
||||
sample_rate=int(meta.get("sample_rate", 32000) or 32000),
|
||||
enc_frames=enc_frames or {},
|
||||
enc_times=list(record.get("times", meta.get("enc_times", []))),
|
||||
enc_fps=float(record.get("fps", meta.get("enc_fps", 0.0))),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path_no_ext: str, device: str = "cpu") -> "H3RefMod":
|
||||
"""Load from ``{path}.safetensors`` (metadata in header, or legacy .json)."""
|
||||
meta = read_refmod_meta(path_no_ext)
|
||||
if meta is None:
|
||||
raise ValueError(
|
||||
f"{path_no_ext}.safetensors has no RefMod metadata "
|
||||
f"(header key '{META_KEY}' or sidecar .json missing).")
|
||||
if meta.get("kind") == "bundle":
|
||||
raise ValueError("This is a RefMod bundle. Load its members instead of treating it as one latent.")
|
||||
# clone drops the file mmap, so the file isn't locked on Windows and
|
||||
# can be re-saved over the same name
|
||||
all_tensors = load_file(path_no_ext + ".safetensors", device=device)
|
||||
latent = all_tensors["latent"].clone()
|
||||
audio_latent = all_tensors["audio_latent"].clone() if "audio_latent" in all_tensors else None
|
||||
enc_frames = _read_encoder_images(meta, all_tensors.__getitem__)
|
||||
return cls.from_metadata(meta, latent, audio_latent=audio_latent, enc_frames=enc_frames)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
|
||||
_H3_VAE_CACHE: Dict[Tuple[str, str, str, bool], Tuple[Any, Optional[Any]]] = {}
|
||||
|
||||
|
||||
def _cache_key(video_path: str, audio_path: str, device, load_audio: bool) -> Tuple[str, str, str, bool]:
|
||||
return (
|
||||
os.path.abspath(video_path or ""),
|
||||
os.path.abspath(audio_path or ""),
|
||||
str(device),
|
||||
bool(load_audio),
|
||||
)
|
||||
|
||||
|
||||
def load_h3_vae_pair(video_path: str, audio_path: str = "", device=None,
|
||||
load_audio: bool = True) -> Tuple[Any, Optional[Any]]:
|
||||
"""Load and cache the H3 video/audio VAE wrappers from safetensors paths."""
|
||||
import comfy.model_management
|
||||
import comfy.sd
|
||||
import comfy.utils
|
||||
|
||||
if not video_path:
|
||||
raise ValueError("load_h3_vae_pair: video_path is required")
|
||||
device = device or comfy.model_management.get_torch_device()
|
||||
key = _cache_key(video_path, audio_path, device, load_audio)
|
||||
cached = _H3_VAE_CACHE.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
sd, metadata = comfy.utils.load_torch_file(video_path, return_metadata=True)
|
||||
video_vae = comfy.sd.VAE(sd=sd, metadata=metadata, device=device)
|
||||
video_vae.throw_exception_if_invalid()
|
||||
|
||||
audio_vae = None
|
||||
if load_audio and audio_path:
|
||||
asd, ameta = comfy.utils.load_torch_file(audio_path, return_metadata=True)
|
||||
audio_vae = comfy.sd.VAE(sd=asd, metadata=ameta, device=device)
|
||||
audio_vae.throw_exception_if_invalid()
|
||||
|
||||
pair = (video_vae, audio_vae)
|
||||
_H3_VAE_CACHE[key] = pair
|
||||
return pair
|
||||
|
||||
|
||||
def load_h3_vaes_from_av_encoder(av_encoder, device=None,
|
||||
load_audio: bool = True) -> Tuple[Any, Optional[Any]]:
|
||||
"""Resolve a MiniMax H3 VAE-loader payload into local video/audio VAE wrappers."""
|
||||
video_path = getattr(av_encoder, "video_path", None)
|
||||
audio_path = getattr(av_encoder, "audio_path", None) or ""
|
||||
if not video_path:
|
||||
raise ValueError(
|
||||
"H3RefModExtract: av_encoder is missing video_path. Connect a valid "
|
||||
"MiniMax H3 VAE loader output."
|
||||
)
|
||||
return load_h3_vae_pair(video_path, audio_path, device=device,
|
||||
load_audio=load_audio)
|
||||
Reference in New Issue
Block a user