Files
DumasNodes/extract_mod.py
T
chris.dumas 70d0672ee3 Squashed 'vendor/dumas_minimax_h3_refmod/' content from commit 517e8c4
git-subtree-dir: vendor/dumas_minimax_h3_refmod
git-subtree-split: 517e8c419a8584fbf69fceb487bc39d83e6a6ec8
2026-10-20 13:43:44 +00:00

272 lines
14 KiB
Python

"""
extract_mod.py — standalone MiniMax H3 RefMod extractor (no training)
Turns reference images/videos into a ``.safetensors`` mod that the ComfyUI
Load H3 RefMods / Apply nodes can inject into MiniMax H3 generation. Only
the H3 video VAE is needed; the 29B DiT is never loaded.
Two modes:
* ``encode`` (default) — refs are resized to ``--resolution`` short edge (down
only) and encoded at that resolution, exactly like the official ref2video
node. This is what carries identity (a face, an outfit); files are
~0.2-1 MB per frame. (Old name: ``full``.)
* ``training`` — refs are resized to ``--resolution`` short edge too (the
latent is pooled to a tiny grid anyway, so encoding at native resolution
is wasted compute), then average-pooled to a small grid (4x4 by default
= 4 tokens per latent frame) and optionally refined with a few gradient
steps (still model-free; that refinement loop is the only "training" in
the whole pack). Nearly free to inject but only carries concept / motion,
not fine identity. (Old name: ``pooled``.)
Usage
-----
# full-res identity mod (recommended for characters)
python custom_nodes/ComfyUI-MiniMaxH3Mod/extract_mod.py \
--image char.png --vae path/to/h3_video_vae.safetensors \
--name my_character --mode encode --resolution 1024
# tiny concept/motion mod
python custom_nodes/ComfyUI-MiniMaxH3Mod/extract_mod.py \
--video dance.mp4 --vae path/to/h3_video_vae.safetensors \
--name dance --mode training --pool 4 --latent-frames 2
Multi-reference concept (each ref becomes its own latent frame):
python custom_nodes/ComfyUI-MiniMaxH3Mod/extract_mod.py \
--image face_a.png --image face_b.png --image full.png \
--video dance.mp4 --vae path/to/h3_video_vae.safetensors \
--name disney_char --mode encode --resolution 1024
Output goes to ``ComfyUI/models/refmods/`` by default (created on first run,
next to loras/ and unet/).
Run it with the same Python/venv that runs ComfyUI (it imports ``comfy``
from the install the script lives in).
"""
from __future__ import annotations
import argparse
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
if HERE not in sys.path:
sys.path.insert(0, HERE)
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
if ROOT not in sys.path:
sys.path.insert(0, ROOT)
import torch
from common import (load_image_file, load_video_file, refmods_dir, mod_output_path,
resize_ref as _resize_ref, ensure_min_size, sample_video_for_vae)
from core import (CONCEPT_TYPES, H3RefMod, aspect_grid, fit_token_budget,
normalize_mode, optimize_latent, pool_latent)
MAX_VIDEO_FRAMES = 60 # uniform sample cap; temporal pooling averages anyway
# ═══════════════════════════════════════════════════════════════════════════
# Media loading (delegates to common.py: image/video -> [B, H, W, 3] float32)
# ═══════════════════════════════════════════════════════════════════════════
def _load_image(path: str, max_edge: int) -> torch.Tensor:
return load_image_file(path, max_edge=max_edge)
def _load_video(path: str, max_edge: int, max_frames: int = MAX_VIDEO_FRAMES) -> torch.Tensor:
return load_video_file(path, max_frames=max_frames, max_edge=max_edge)
# ═══════════════════════════════════════════════════════════════════════════
# Main
# ═══════════════════════════════════════════════════════════════════════════
def main():
ap = argparse.ArgumentParser(
description="Extract a MiniMax H3 RefMod from one or more images/videos "
"of the same concept (they are stacked into a video-kind mod)")
ap.add_argument("--image", action="append", default=[], metavar="PATH",
help="reference image path (repeatable for multi-view concepts)")
ap.add_argument("--video", action="append", default=[], metavar="PATH",
help="reference video path (repeatable)")
ap.add_argument("--vae", required=True, help="MiniMax H3 video VAE .safetensors")
ap.add_argument("--name", default=None, help="mod name (default: first source file stem)")
ap.add_argument("--subfolder", default="", help="optional subfolder inside models/refmods")
ap.add_argument("--output", default=None,
help="output dir (default: ComfyUI/models/refmods)")
ap.add_argument("--mode", choices=["training", "encode", "full", "pooled"],
default="training",
help="training = compressed grid refined by --identity (default, good balance); encode = straight full-res VAE encode (~1K tokens/img); full/pooled = old names, still accepted")
ap.add_argument("--resolution", type=int, default=1024,
help="both modes: target short edge in px, downscale only (default 1024; 2048 = max fidelity). training mode encodes at this size too — the main speed dial for it")
ap.add_argument("--pool", type=int, default=16, help="training mode: spatial latent grid (even, default 16)")
ap.add_argument("--pool-w", type=int, default=None, help="pool width (default: == --pool)")
ap.add_argument("--latent-frames", type=int, default=16,
help="frames kept per video ref (default 16; images use 1): training mode pools latent frames; encode mode samples source frames on H3's 17k+5 grid (16 selects 5, below 5 selects the first image)")
ap.add_argument("--identity", type=int, default=500,
help="training mode: how tightly the mod clings to the reference (gradient refinement steps; default 500, 0 = pure pooling)")
ap.add_argument("--multiplier", type=int, default=1,
help="data multiplier: repeat the extracted ref N times along time so a short video/GIF isn't drowned out by the main video's tokens (default 1 = no repeat)")
ap.add_argument("--max-tokens", type=int, default=5120,
help="hard cap on the total injected tokens (0 = off; default 5120): drops near-duplicate latent frames first, then resamples the rest to fit")
ap.add_argument("--max-edge", type=int, default=1536,
help="resize source so the longest edge is <= this before encoding (default 1536)")
ap.add_argument("--max-frames", type=int, default=60,
help="max video frames sampled (uniform) before encoding (default 60; lower for CPU)")
ap.add_argument("--description", default="",
help="optional text describing the concept (e.g. 'a ginger woman with messy hair', "
"'an animation style'). Stored in the mod and emitted by the loaders so it "
"can be merged into the prompt without retyping it.")
ap.add_argument("--concept-type", choices=list(CONCEPT_TYPES), default="generic",
help="what this mod represents (default: generic). 'identity' in --mode training "
"prints a warning: pooling is lossy in exactly the way that destroys faces — "
"use --mode encode for people.")
ap.add_argument("--device", default="auto", help="auto / cuda / mps / cpu (VAE device)")
args = ap.parse_args()
args.mode = normalize_mode(args.mode) # accept legacy 'full'/'pooled'
if not args.image and not args.video:
ap.error("provide at least one --image or --video")
if args.concept_type == "identity" and args.mode == "training" and args.pool < 16:
print(
f"[extract] warning: --concept-type identity with --mode training at "
f"--pool {args.pool} — pooling averages away exactly the detail that "
f"carries a face. Use --mode encode for people, or at least --pool 32."
)
import comfy.model_management
import comfy.sd
import comfy.utils
device = comfy.model_management.get_torch_device() if args.device == "auto" \
else torch.device(args.device)
# ── load VAE ─────────────────────────────────────────────────────
print(f"[extract] loading VAE {args.vae} (device={args.device})")
sd, metadata = comfy.utils.load_torch_file(args.vae, return_metadata=True)
vae = comfy.sd.VAE(sd=sd, metadata=metadata, device=device)
vae.throw_exception_if_invalid()
# ── encode each source (full-res or pooled), then stack along time ──
pool_w = args.pool_w or args.pool
# in encode mode --resolution is the target short edge, so keep the
# load-time longest-edge cap from interfering with it
# both modes resize to --resolution before encode now, so the load-time
# longest-edge cap just needs to stay ahead of that target
load_max_edge = max(args.resolution * 2, args.max_edge)
# full-res refs must share one spatial canvas so the stacked latent has a
# single H/W: anchor on the first source, cover-crop the rest to it
canvas = None
if args.mode == "encode" and len(args.image) + len(args.video) > 1:
if args.image:
first = _load_image(args.image[0], load_max_edge)
else:
first = _load_video(args.video[0], load_max_edge, args.max_frames)
h, w = first.shape[1], first.shape[2]
scale = min(1.0, args.resolution / min(h, w))
canvas = (max(32, round(w * scale / 32) * 32),
max(32, round(h * scale / 32) * 32))
# training mode: anchor the pool grid to the first source's aspect so a
# portrait person isn't squished into a square grid ("fat" distortion).
pool_grid = None
if args.mode == "training":
if args.image:
first = _load_image(args.image[0], load_max_edge)
else:
first = _load_video(args.video[0], load_max_edge, args.max_frames)
h0, w0 = first.shape[1], first.shape[2]
pool_grid = aspect_grid(args.pool, pool_w, h0 / w0)
if pool_grid != (args.pool, pool_w):
print(f"[extract] pooled grid {args.pool}x{pool_w} -> "
f"{pool_grid[0]}x{pool_grid[1]} to match source aspect "
f"{w0}x{h0} (avoids squishing the subject wide)")
frames = []
shapes = []
n_img = n_vid = 0
for path in args.image:
src = _load_image(path, load_max_edge)
src = ensure_min_size(_resize_ref(src, args.resolution, canvas))
print(f"[extract] image {path}: {tuple(src.shape)} (mode={args.mode})")
with torch.no_grad():
z = vae.encode(src.to(device)).float().cpu()
if args.mode == "encode":
pooled = z.to(torch.float16)
else:
gh, gw = pool_grid if pool_grid is not None else (args.pool, pool_w)
pooled = pool_latent(z, 1, gh, gw).to(torch.float16)
if args.identity > 0:
pooled = optimize_latent(pooled, z, steps=args.identity, device=device,
progress_every=100)
frames.append(pooled)
shapes.append(f"{z.shape[2]}x{z.shape[3]}x{z.shape[4]}")
n_img += 1
for path in args.video:
src = _load_video(path, load_max_edge, args.max_frames)
src = sample_video_for_vae(src, args.latent_frames if args.mode == "encode" else None)
src = ensure_min_size(_resize_ref(src, args.resolution, canvas))
print(f"[extract] video {path}: {tuple(src.shape)} (mode={args.mode})")
with torch.no_grad():
z = vae.encode(src.to(device)).float().cpu()
if args.mode == "encode":
pooled = z.to(torch.float16)
else:
pool_t = min(args.latent_frames, z.shape[2])
gh, gw = pool_grid if pool_grid is not None else (args.pool, pool_w)
pooled = pool_latent(z, pool_t, gh, gw).to(torch.float16)
if args.identity > 0:
pooled = optimize_latent(pooled, z, steps=args.identity, device=device,
progress_every=100)
frames.append(pooled)
shapes.append(f"{z.shape[2]}x{z.shape[3]}x{z.shape[4]}")
n_vid += 1
if args.mode == "encode" and args.identity > 0:
print(f"[extract] warning: --identity only applies to training mode — "
f"encode mode stores the actual encode, so --identity {args.identity} "
f"was ignored.")
latent = torch.cat(frames, dim=2)
if args.multiplier > 1:
latent = latent.repeat(1, 1, args.multiplier, 1, 1)
name = args.name or (os.path.splitext(os.path.basename(args.image[0]))[0]
if args.image else os.path.splitext(os.path.basename(args.video[0]))[0])
if args.max_tokens > 0:
latent = fit_token_budget(latent, args.max_tokens, name)
total_t = latent.shape[2]
kind = "video" if total_t > 1 else "image"
out_dir = args.output or refmods_dir()
px_w, px_h = latent.shape[4] * 16, latent.shape[3] * 16
mod = H3RefMod(
name=name,
kind=kind,
latent=latent,
latent_h=latent.shape[3],
latent_w=latent.shape[4],
latent_t=total_t,
mode=args.mode,
source="stack" if len(frames) > 1 else ("video" if n_vid else "image"),
source_shape=" +".join(shapes),
pool=f"full-res {px_w}x{px_h}px (short-edge cap {args.resolution}px)" if args.mode == "encode" else f"{total_t}x{gh}x{gw}",
optimize_steps=args.identity if args.mode == "training" else 0,
tags=[f"{n_img} img, {n_vid} vid"] + ([f"x{args.multiplier} repeat"] if args.multiplier > 1 else []),
description=args.description.strip(),
concept_type=args.concept_type,
)
mod.path = os.path.join(out_dir, name) if args.output else mod_output_path(name, args.subfolder) # so Fix H3 RefMod Config can re-save in place
path = mod.save(mod.path)
mb = latent.numel() * latent.element_size() / 1024 / 1024
print(f"[extract] saved {kind} mod '{name}' "
f"({mod.token_count} tokens, {mb:.2f} MB) -> {path}")
print(f"[extract] load it in ComfyUI with the Load H3 RefMods node "
f"(dropdown '{name}', strength 1.0 = full ref), then chain "
f"Apply H3 RefMod (Cond) into your sampling.")
if __name__ == "__main__":
main()