git-subtree-dir: vendor/dumas_h3_refmod_picker git-subtree-split: c732f8b8893bf4f00c6974c9b9941c763496fe48
1144 lines
55 KiB
Python
1144 lines
55 KiB
Python
"""
|
|
H3RefModCreateFromFolder — folder-driven extraction of reference media into a saved RefMod.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
import os
|
|
import shutil
|
|
from typing import List, Optional, Tuple
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
import comfy.model_management
|
|
import comfy.utils
|
|
import folder_paths
|
|
from comfy_api.latest import io
|
|
|
|
from ..py.refmod_common import (
|
|
list_image_files,
|
|
refmods_dir,
|
|
)
|
|
from ..py.refmod_core import (
|
|
AUDIO_CONCEPT_TYPES,
|
|
CONCEPT_TYPES,
|
|
H3RefMod,
|
|
_blur_latent,
|
|
aspect_grid,
|
|
fit_token_budget,
|
|
normalize_mode,
|
|
optimize_latent,
|
|
optimize_latent_multi,
|
|
pack_encoder_images,
|
|
pool_latent,
|
|
save_refmod_bundle,
|
|
)
|
|
from ..py.refmod_vae_loader import load_h3_vaes_from_av_encoder
|
|
from .refmods_to_video import _encoder_frames
|
|
# reuse the encode helpers shared with the loader/mods-listing side of the pack
|
|
from . import refmod_loader as _nodes_mod # loader-side cache/list refresh
|
|
from .refmod_loader import (
|
|
_MOD_CACHE,
|
|
_MOD_CACHE_MAX,
|
|
)
|
|
|
|
AUDIO_EXTS = {".wav", ".mp3", ".flac", ".aac", ".m4a", ".ogg", ".opus"}
|
|
THUMB_TARGET_ASPECT = 3.0 / 4.0
|
|
THUMB_ASPECT_TOLERANCE = 0.05
|
|
VISUAL_SUFFIX = "_visual"
|
|
AUDIO_SUFFIX = "_audio"
|
|
|
|
|
|
def _resize_ref(image, short_edge: int, canvas=None):
|
|
"""Aspect-preserving downscale (never upscale) to ``short_edge`` px; dims to /32."""
|
|
h, w = image.shape[1], image.shape[2]
|
|
if h <= 0 or w <= 0:
|
|
raise ValueError(
|
|
f"_resize_ref: source has an empty frame ({h}x{w}) before any "
|
|
f"resize — the reference itself is invalid.")
|
|
scale = min(1.0, short_edge / min(h, w))
|
|
tw = max(32, round(w * scale / 32) * 32)
|
|
th = max(32, round(h * scale / 32) * 32)
|
|
crop = "disabled"
|
|
if canvas is not None:
|
|
tw, th = canvas
|
|
crop = "center"
|
|
if tw <= 0 or th <= 0:
|
|
raise ValueError(
|
|
f"_resize_ref: computed a zero-size resize target ({tw}x{th}) "
|
|
f"for a {h}x{w} source (short_edge={short_edge}, canvas={canvas}). "
|
|
f"This should be impossible — please report this shape.")
|
|
samples = image[..., :3].movedim(-1, 1)
|
|
samples = comfy.utils.common_upscale(samples, tw, th, "lanczos", crop)
|
|
if samples.shape[2] <= 0 or samples.shape[3] <= 0:
|
|
raise ValueError(
|
|
f"_resize_ref: common_upscale produced an empty result "
|
|
f"{tuple(samples.shape)} from a {h}x{w} source targeting "
|
|
f"{tw}x{th} (crop={crop}, canvas={canvas}). This points to a bug "
|
|
f"in comfy.utils.common_upscale for this input, not in RefMod's "
|
|
f"own math.")
|
|
return samples.movedim(1, -1)
|
|
|
|
|
|
def _ensure_min_size(image, floor: int = 320):
|
|
"""Upscale (never downscale) so both spatial dims are >= ``floor`` px."""
|
|
h, w = image.shape[1], image.shape[2]
|
|
if h >= floor and w >= floor:
|
|
return image
|
|
scale = floor / min(h, w)
|
|
tw = max(floor, round(w * scale / 32) * 32)
|
|
th = max(floor, round(h * scale / 32) * 32)
|
|
samples = image[..., :3].movedim(-1, 1)
|
|
samples = comfy.utils.common_upscale(samples, tw, th, "lanczos", "disabled")
|
|
return samples.movedim(1, -1)
|
|
|
|
|
|
def _normalize_mask_batch(mask, label: str = "mask") -> torch.Tensor:
|
|
"""Canonicalize a MASK input to ``[N, H, W]`` float32 in [0, 1]."""
|
|
if mask is None:
|
|
return None
|
|
if not isinstance(mask, torch.Tensor):
|
|
raise ValueError(f"H3RefModExtract: {label} must be a MASK tensor, "
|
|
f"got {type(mask)}")
|
|
if mask.dim() == 2:
|
|
mask = mask.unsqueeze(0)
|
|
if mask.dim() != 3:
|
|
raise ValueError(f"H3RefModExtract: {label} has unexpected shape "
|
|
f"{tuple(mask.shape)} (expected [H,W] or [N,H,W])")
|
|
return mask.float().clamp(0.0, 1.0)
|
|
|
|
|
|
def _resize_mask(mask: torch.Tensor, target_h: int, target_w: int, crop="disabled") -> torch.Tensor:
|
|
"""Resize a ``[T, H, W]`` mask to ``target_h x target_w`` (bilinear)."""
|
|
samples = mask.unsqueeze(1)
|
|
samples = comfy.utils.common_upscale(samples, target_w, target_h, "bilinear", crop)
|
|
return samples.squeeze(1).clamp(0.0, 1.0)
|
|
|
|
|
|
def _mask_latent(z: torch.Tensor, mask_px: torch.Tensor, background_retention: float,
|
|
seed_key: str) -> torch.Tensor:
|
|
"""Suppress the latent outside ``mask_px`` toward a blurred copy of itself, per cell."""
|
|
del seed_key
|
|
t, h, w = z.shape[2], z.shape[3], z.shape[4]
|
|
mp = mask_px.unsqueeze(1)
|
|
if mp.shape[0] == 1 and t > 1:
|
|
mp = mp.expand(t, -1, -1, -1)
|
|
elif mp.shape[0] != t:
|
|
idx = torch.linspace(0, mp.shape[0] - 1, t).round().long()
|
|
mp = mp[idx]
|
|
mp = F.adaptive_avg_pool2d(mp.float(), (h, w))
|
|
mp = mp.permute(1, 0, 2, 3).unsqueeze(0).clamp(0.0, 1.0)
|
|
weight = background_retention + (1.0 - background_retention) * mp
|
|
blurred = _blur_latent(z)
|
|
return (weight * z.float() + (1.0 - weight) * blurred).to(z.dtype)
|
|
|
|
|
|
def _normalize_ref(src, label: str = "reference") -> torch.Tensor:
|
|
"""Canonicalize any ref source to ``[T, H, W, C]`` (T=1 for stills)."""
|
|
if not isinstance(src, torch.Tensor) or src.dim() not in (3, 4, 5):
|
|
raise ValueError(
|
|
f"H3RefModExtract: {label} must be a 3-5D tensor, "
|
|
f"got {getattr(src, 'shape', src)}")
|
|
if src.dim() == 5:
|
|
if src.shape[0] == 0:
|
|
raise ValueError(
|
|
f"H3RefModExtract: {label} has no frames "
|
|
f"(T=0) — check the source image.")
|
|
src = src[0] if src.shape[0] == 1 else src.reshape(-1, *src.shape[2:])
|
|
if src.dim() == 3:
|
|
src = src.unsqueeze(0)
|
|
if src.shape[-1] != 3 and src.shape[1] == 3:
|
|
src = src.movedim(1, -1)
|
|
if src.dim() != 4 or src.shape[-1] != 3:
|
|
raise ValueError(
|
|
f"H3RefModExtract: {label} has an unexpected layout "
|
|
f"{tuple(src.shape)} (expected [T, H, W, 3])")
|
|
if src.shape[0] <= 0:
|
|
raise ValueError(
|
|
f"H3RefModExtract: {label} has no frames (T={src.shape[0]}) "
|
|
f"— check the source image.")
|
|
if src.shape[1] <= 0 or src.shape[2] <= 0:
|
|
raise ValueError(
|
|
f"H3RefModExtract: {label} has an empty frame "
|
|
f"({src.shape[1]}x{src.shape[2]}) — check the source image.")
|
|
return src
|
|
|
|
|
|
def _sanitize_name(name: str) -> str:
|
|
name = name.strip().replace("/", "_").replace("\\", "_")
|
|
if not name:
|
|
raise ValueError("mod name must not be empty")
|
|
return name
|
|
|
|
|
|
def _resolve_folder(folder: str) -> str:
|
|
"""Resolve a folder input: absolute path, a name inside input/, or input/ itself."""
|
|
folder = (folder or "").strip().strip('"')
|
|
if not folder:
|
|
return folder_paths.get_input_directory()
|
|
if os.path.isabs(folder):
|
|
resolved = os.path.normpath(folder)
|
|
else:
|
|
resolved = os.path.join(folder_paths.get_input_directory(), folder)
|
|
if not os.path.isdir(resolved):
|
|
raise ValueError(
|
|
f"folder not found: {folder!r} (looked at '{resolved}'; use an "
|
|
"absolute path or a folder name inside input/).")
|
|
return resolved
|
|
|
|
|
|
def _summarize(mod: H3RefMod) -> str:
|
|
if mod.kind == "audio":
|
|
return f"{mod.name}: audio, {mod.latent_t / 40:.2f}s, {mod.token_count} tokens"
|
|
mb = mod.latent.numel() * mod.latent.element_size() / 1024 / 1024
|
|
return (f"'{mod.name}' {mod.mode} {mod.kind} {tuple(mod.latent.shape)} "
|
|
f"({mod.token_count} tokens, {mb:.2f} MB)")
|
|
|
|
|
|
def _auto_folder_description(folder: str, concept_type: str) -> str:
|
|
"""Default metadata description for folder-generated RefMods."""
|
|
folder_label = os.path.basename(os.path.normpath(folder)) or "refmod"
|
|
if concept_type == "identity":
|
|
return f"{folder_label} persona reference mod"
|
|
if concept_type and concept_type != "generic":
|
|
return f"{folder_label} {concept_type} reference mod"
|
|
return f"{folder_label} reference mod"
|
|
|
|
|
|
def _folder_refmod_name(folder: str) -> str:
|
|
"""RefMod base name derived from a folder, matching subfolder mode."""
|
|
folder_name = _sanitize_name(os.path.basename(os.path.normpath(folder)))
|
|
if "refmod" not in folder_name.lower():
|
|
folder_name = f"{folder_name}_refMod"
|
|
return folder_name
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Folder scanning (images / audio)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
def _scan_folder(folder: str) -> "tuple[List[str], List[str]]":
|
|
"""(images, audios) directly under ``folder``, sorted by name."""
|
|
images = list_image_files(folder)
|
|
audios = []
|
|
if os.path.isdir(folder):
|
|
for fn in sorted(os.listdir(folder)):
|
|
if os.path.splitext(fn)[1].lower() in AUDIO_EXTS:
|
|
p = os.path.join(folder, fn)
|
|
if os.path.isfile(p):
|
|
audios.append(p)
|
|
if not images and not audios:
|
|
print(
|
|
f"[H3RefModCreateFromFolder] scan: '{folder}' contains no supported image or audio files "
|
|
"at the top level."
|
|
)
|
|
return images, audios
|
|
|
|
|
|
def _image_size(path: str) -> "tuple[int, int]":
|
|
from PIL import Image, ImageOps
|
|
|
|
with Image.open(path) as img:
|
|
img = ImageOps.exif_transpose(img)
|
|
return img.size
|
|
|
|
|
|
def _load_image_and_alpha_mask_file(path: str, max_edge: Optional[int] = None) -> "tuple[torch.Tensor, Optional[torch.Tensor]]":
|
|
"""Load one image file and an optional embedded alpha mask.
|
|
|
|
Returns ``(image, mask)`` where image is ``[1, H, W, 3]`` RGB float32 in
|
|
``[0, 1]`` and mask is ``[1, H, W]`` float32 in ``[0, 1]`` when the source
|
|
contains a non-trivial alpha channel. Fully opaque images return ``None``
|
|
for the mask.
|
|
"""
|
|
import numpy as np
|
|
from PIL import Image, ImageOps
|
|
|
|
with Image.open(path) as img:
|
|
img = ImageOps.exif_transpose(img)
|
|
rgba = img.convert("RGBA")
|
|
rgb = rgba.convert("RGB")
|
|
alpha = rgba.getchannel("A")
|
|
w, h = rgb.size
|
|
if max_edge is not None:
|
|
scale = min(1.0, max_edge / max(w, h))
|
|
if scale < 1.0:
|
|
size = (max(1, round(w * scale)), max(1, round(h * scale)))
|
|
rgb = rgb.resize(size, Image.LANCZOS)
|
|
alpha = alpha.resize(size, Image.BILINEAR)
|
|
image = torch.from_numpy(np.asarray(rgb).copy()).float() / 255.0
|
|
alpha_arr = np.asarray(alpha).copy()
|
|
mask = None
|
|
if alpha_arr.min() < 255:
|
|
mask = torch.from_numpy(alpha_arr).float() / 255.0
|
|
mask = mask.unsqueeze(0)
|
|
return image.unsqueeze(0), mask
|
|
|
|
|
|
def _is_preferred_thumb_aspect(width: int, height: int) -> bool:
|
|
if width <= 0 or height <= 0:
|
|
return False
|
|
return abs((width / float(height)) - THUMB_TARGET_ASPECT) <= THUMB_ASPECT_TOLERANCE
|
|
|
|
|
|
def _choose_thumb_source(images: List[str]) -> Optional[str]:
|
|
"""Prefer a 3:4 image; otherwise use the largest image. Stable on ties."""
|
|
ranked = []
|
|
for idx, path in enumerate(images):
|
|
try:
|
|
width, height = _image_size(path)
|
|
except Exception:
|
|
continue
|
|
ranked.append((
|
|
0 if _is_preferred_thumb_aspect(width, height) else 1,
|
|
-(width * height),
|
|
idx,
|
|
path,
|
|
))
|
|
if not ranked:
|
|
return None
|
|
ranked.sort()
|
|
return ranked[0][3]
|
|
|
|
|
|
def _copy_refmod_thumbnail(images: List[str], path_no_ext: str) -> None:
|
|
"""Copy a representative source image beside the saved RefMod."""
|
|
thumb_src = _choose_thumb_source(images)
|
|
if not thumb_src:
|
|
return
|
|
ext = os.path.splitext(thumb_src)[1].lower()
|
|
thumb_base = path_no_ext
|
|
if thumb_base.lower().endswith(VISUAL_SUFFIX.lower()):
|
|
thumb_base = thumb_base[:-len(VISUAL_SUFFIX)]
|
|
thumb_dst = thumb_base + ext
|
|
shutil.copy2(thumb_src, thumb_dst)
|
|
print(f"[H3RefModCreateFromFolder] thumbnail {os.path.basename(thumb_src)} -> {thumb_dst}")
|
|
|
|
|
|
def _latent_token_count(latent: torch.Tensor) -> int:
|
|
return int(latent.shape[2]) * (int(latent.shape[3]) // 2) * (int(latent.shape[4]) // 2)
|
|
|
|
|
|
def _resolve_output_dir(subfolder: str = "") -> str:
|
|
base = refmods_dir()
|
|
subfolder = str(subfolder or "").strip().strip('"').strip("/\\")
|
|
return os.path.join(base, subfolder) if subfolder else base
|
|
|
|
|
|
def _base_refmod_name(name: str) -> str:
|
|
base = _sanitize_name(name)
|
|
lower_base = base.lower()
|
|
for suffix in (VISUAL_SUFFIX, AUDIO_SUFFIX):
|
|
if lower_base.endswith(suffix):
|
|
base = base[:-len(suffix)]
|
|
lower_base = base.lower()
|
|
return base or _sanitize_name(name)
|
|
|
|
|
|
def _paired_refmod_names(base_name: str) -> Tuple[str, str]:
|
|
base = _base_refmod_name(base_name)
|
|
return f"{base}{VISUAL_SUFFIX}", f"{base}{AUDIO_SUFFIX}"
|
|
|
|
|
|
def _unique_split_mod_paths(out_dir: str, base_name: str, include_audio: bool) -> Tuple[str, str, str, str, str]:
|
|
root = _base_refmod_name(base_name)
|
|
candidate = root
|
|
attempt = 1
|
|
while True:
|
|
visual_name, audio_name = _paired_refmod_names(candidate)
|
|
visual_path = os.path.join(out_dir, visual_name)
|
|
audio_path = os.path.join(out_dir, audio_name)
|
|
visual_exists = os.path.isfile(visual_path + ".safetensors")
|
|
audio_exists = os.path.isfile(audio_path + ".safetensors")
|
|
if not visual_exists and not audio_exists:
|
|
return candidate, visual_name, visual_path, audio_name, audio_path
|
|
attempt += 1
|
|
candidate = f"{root}_{attempt}"
|
|
|
|
|
|
def _unique_bundle_mod_path(out_dir: str, base_name: str) -> Tuple[str, str]:
|
|
root = _base_refmod_name(base_name)
|
|
candidate = root
|
|
attempt = 1
|
|
while True:
|
|
bundle_path = os.path.join(out_dir, candidate)
|
|
if not os.path.isfile(bundle_path + ".safetensors"):
|
|
return candidate, bundle_path
|
|
attempt += 1
|
|
candidate = f"{root}_{attempt}"
|
|
|
|
|
|
def _make_audio_refmod(name: str, audio_latent: torch.Tensor, audio_concept_type: str,
|
|
description: str, sample_rate: int = 32000) -> H3RefMod:
|
|
return H3RefMod(
|
|
name=name,
|
|
kind="audio",
|
|
latent=audio_latent,
|
|
mode="encode",
|
|
source="audio",
|
|
source_shape=f"audio:{int(audio_latent.shape[-1])}",
|
|
pool=f"{int(audio_latent.shape[-1])} audio",
|
|
tags=[f"{int(audio_latent.shape[-1])} audio", f"audio:{audio_concept_type}"],
|
|
description=description,
|
|
concept_type=audio_concept_type,
|
|
audio_concept_type=audio_concept_type,
|
|
sample_rate=sample_rate,
|
|
)
|
|
|
|
|
|
def _apply_extraction_preset(mode, ref_resolution, pool_h, pool_w, identity,
|
|
merge, motion_only, concept_type,
|
|
audio_concept_type, extraction_preset, log_prefix):
|
|
preset_replaced = ""
|
|
if extraction_preset == "identity_encode":
|
|
concept_type, audio_concept_type = "identity", "voice"
|
|
mode, ref_resolution, identity, merge, motion_only = "encode", 1024, 0, False, False
|
|
preset_replaced = "concept_type=identity, audio_concept_type=voice, mode=Full Reference, ref_resolution=1024, Refinement Steps=0, merge=False, motion_only=False"
|
|
elif extraction_preset == "style_experimental":
|
|
concept_type, audio_concept_type = "style", "voice"
|
|
mode, pool_h, pool_w, identity, merge, motion_only = "training", 8, 8, 150, False, False
|
|
preset_replaced = "concept_type=style, audio_concept_type=voice, mode=Compressed Reference, pool_h=8, pool_w=8, Refinement Steps=150, merge=False, motion_only=False"
|
|
elif extraction_preset == "motion_sequence":
|
|
concept_type, audio_concept_type = "pose_motion", "voice"
|
|
mode, pool_h, pool_w, merge, motion_only = "training", 16, 16, False, False
|
|
preset_replaced = "concept_type=pose_motion, audio_concept_type=voice, mode=Compressed Reference, pool_h=16, pool_w=16, merge=False (Refinement Steps preserved)"
|
|
elif extraction_preset != "manual":
|
|
raise ValueError("Unknown extraction preset.")
|
|
mode = normalize_mode(mode)
|
|
if preset_replaced:
|
|
print(f"[{log_prefix}] preset={extraction_preset} replaces {preset_replaced}")
|
|
return mode, ref_resolution, pool_h, pool_w, identity, merge, motion_only, concept_type, audio_concept_type
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Audio loading + encoding
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
def _load_audio_waveform(path: str) -> "tuple[torch.Tensor, int]":
|
|
"""Load an audio file -> (waveform [C, L] float32, sample_rate)."""
|
|
errors = []
|
|
try:
|
|
import torchaudio
|
|
waveform, sr = torchaudio.load(path)
|
|
if waveform.dim() == 1:
|
|
waveform = waveform.unsqueeze(0)
|
|
return waveform.float(), sr
|
|
except Exception as exc:
|
|
errors.append(f"torchaudio: {exc}")
|
|
try:
|
|
import soundfile as sf
|
|
data, sr = sf.read(path, always_2d=True) # [L, C]
|
|
return torch.from_numpy(data.T).float(), sr
|
|
except Exception as exc:
|
|
errors.append(f"soundfile: {exc}")
|
|
try:
|
|
from comfy_extras.nodes_audio import load
|
|
return load(path)
|
|
except Exception as exc:
|
|
errors.append(f"ComfyUI audio decoder: {exc}")
|
|
if os.path.splitext(path)[1].lower() == ".wav":
|
|
try:
|
|
from scipy.io import wavfile
|
|
sr, data = wavfile.read(path)
|
|
waveform = torch.from_numpy(data).float()
|
|
if waveform.dim() == 1:
|
|
waveform = waveform.unsqueeze(0)
|
|
else:
|
|
waveform = waveform.T
|
|
if data.dtype.kind in ("i", "u"):
|
|
waveform = waveform / float(2 ** (8 * data.dtype.itemsize - 1))
|
|
return waveform, sr
|
|
except Exception as exc:
|
|
errors.append(f"scipy WAV reader: {exc}")
|
|
raise RuntimeError(
|
|
f"Could not decode audio file {path!r}. Try converting it to PCM WAV. "
|
|
+ " | ".join(errors)
|
|
)
|
|
|
|
|
|
def _resample(waveform: torch.Tensor, sr: int, target_sr: int) -> torch.Tensor:
|
|
"""Resample [C, L] -> [C, L'] via torchaudio, falling back to scipy."""
|
|
try:
|
|
import torchaudio
|
|
return torchaudio.functional.resample(waveform, sr, target_sr)
|
|
except Exception:
|
|
import numpy as np
|
|
from math import gcd
|
|
from scipy.signal import resample_poly
|
|
g = gcd(sr, target_sr)
|
|
out = resample_poly(waveform.numpy(), target_sr // g, sr // g, axis=-1)
|
|
return torch.from_numpy(np.ascontiguousarray(out)).float()
|
|
|
|
|
|
def _encode_ref_audio(audio_vae, waveform: torch.Tensor, sr: int, device) -> torch.Tensor:
|
|
"""Encode a waveform to an H3 audio latent [1, 32, 2, T]."""
|
|
import comfy.model_management
|
|
vae_sr = getattr(audio_vae, "audio_sample_rate", 32000)
|
|
if sr != vae_sr:
|
|
waveform = _resample(waveform, sr, vae_sr)
|
|
if waveform.shape[0] == 1:
|
|
waveform = waveform.repeat(2, 1) # mono -> stereo
|
|
elif waveform.shape[0] > 2:
|
|
waveform = waveform[:2]
|
|
batch = waveform.unsqueeze(0) # [1, 2, L]
|
|
model = audio_vae.first_stage_model
|
|
last_err = None
|
|
for dev in (device, torch.device("cpu")):
|
|
try:
|
|
if dev.type == "cuda":
|
|
comfy.model_management.load_models_gpu([audio_vae.patcher])
|
|
else:
|
|
model.to(dev)
|
|
with torch.no_grad():
|
|
z = model.encode(batch.to(dev)).float().cpu()
|
|
if dev.type == "cuda":
|
|
try:
|
|
audio_vae.patcher.unpatch_model()
|
|
except Exception:
|
|
pass
|
|
return z
|
|
except (torch.OutOfMemoryError, RuntimeError) as e:
|
|
last_err = e
|
|
if "out of memory" not in str(e).lower() and "OOM" not in str(e):
|
|
raise
|
|
continue
|
|
if last_err is not None:
|
|
raise last_err
|
|
raise RuntimeError("Failed to encode folder audio with the H3 audio VAE.")
|
|
|
|
|
|
def _encode_input_audio(audio_vae, audio, max_seconds: float = 30.0,
|
|
chunk_seconds: float = 10.0):
|
|
waveform = audio["waveform"]
|
|
sample_rate = int(audio["sample_rate"])
|
|
if waveform.ndim != 3 or waveform.shape[0] != 1 or waveform.shape[1] not in (1, 2):
|
|
raise ValueError("Audio must have one batch of mono or stereo samples [1,C,L].")
|
|
if sample_rate <= 0 or waveform.shape[-1] < 1:
|
|
raise ValueError("Audio is empty or has an invalid sample rate.")
|
|
if (not math.isfinite(max_seconds) or not math.isfinite(chunk_seconds)
|
|
or max_seconds <= 0 or chunk_seconds <= 0):
|
|
raise ValueError("Audio duration and chunk length must be positive.")
|
|
|
|
vae_sr = int(getattr(audio_vae, "audio_sample_rate", 32000))
|
|
waveform = waveform[..., :max(1, round(max_seconds * sample_rate))]
|
|
if waveform.shape[1] == 1:
|
|
waveform = waveform.repeat(1, 2, 1)
|
|
if sample_rate != vae_sr:
|
|
waveform = _resample(waveform[0], sample_rate, vae_sr).unsqueeze(0)
|
|
|
|
model = audio_vae.first_stage_model
|
|
chunk = max(800, round(chunk_seconds * 40) * 800)
|
|
last_err = None
|
|
for dev in (comfy.model_management.get_torch_device(), torch.device("cpu")):
|
|
try:
|
|
if dev.type == "cuda":
|
|
comfy.model_management.load_models_gpu([audio_vae.patcher])
|
|
else:
|
|
model.to(dev)
|
|
latents = []
|
|
with torch.no_grad():
|
|
for start in range(0, waveform.shape[-1], chunk):
|
|
piece = waveform[..., start:start + chunk].to(dev)
|
|
z = model.encode(piece).float().cpu()
|
|
if z.ndim != 4 or tuple(z.shape[:3]) != (1, 32, 2):
|
|
raise ValueError(f"H3 audio VAE returned an invalid latent: {tuple(z.shape)}")
|
|
latents.append(z)
|
|
if dev.type == "cuda":
|
|
try:
|
|
audio_vae.patcher.unpatch_model()
|
|
except Exception:
|
|
pass
|
|
return torch.cat(latents, dim=-1)
|
|
except (torch.OutOfMemoryError, RuntimeError) as e:
|
|
last_err = e
|
|
if "out of memory" not in str(e).lower() and "oom" not in str(e).lower():
|
|
raise
|
|
continue
|
|
if last_err is not None:
|
|
raise last_err
|
|
raise RuntimeError("Failed to encode input audio with the H3 audio VAE.")
|
|
|
|
|
|
def _make_audio_refmod_from_input(name: str, audio_vae, audio, max_seconds: float = 30.0,
|
|
max_tokens: int = 5120, budget_policy: str = "error",
|
|
description: str = "", concept_type: str = "voice") -> H3RefMod:
|
|
if not isinstance(max_tokens, int) or isinstance(max_tokens, bool) or max_tokens < 0:
|
|
raise ValueError("Audio token budget must be a non-negative integer.")
|
|
if budget_policy not in ("error", "truncate"):
|
|
raise ValueError("Audio budget policy must be error or truncate.")
|
|
latent = _encode_input_audio(audio_vae, audio, max_seconds=max_seconds)
|
|
tokens = latent.shape[-1] * 2
|
|
if max_tokens and tokens > max_tokens:
|
|
if max_tokens < 2 or budget_policy == "error":
|
|
raise ValueError(
|
|
f"Audio requires {tokens} tokens; budget is {max_tokens}. "
|
|
"Lower audio_max_seconds or choose truncate."
|
|
)
|
|
latent = latent[..., :max_tokens // 2].clone()
|
|
return _make_audio_refmod(
|
|
name, latent, concept_type, description,
|
|
sample_rate=int(getattr(audio_vae, "audio_sample_rate", 32000)),
|
|
)
|
|
|
|
|
|
def _crop_audio_latent(latent: torch.Tensor, max_tokens: int, budget_policy: str, label: str) -> torch.Tensor:
|
|
if not isinstance(max_tokens, int) or isinstance(max_tokens, bool) or max_tokens < 0:
|
|
raise ValueError("Audio token budget must be a non-negative integer.")
|
|
if budget_policy not in ("error", "truncate"):
|
|
raise ValueError("Audio budget policy must be error or truncate.")
|
|
tokens = int(latent.shape[-1]) * 2
|
|
if max_tokens and tokens > max_tokens:
|
|
if max_tokens < 2 or budget_policy == "error":
|
|
raise ValueError(
|
|
f"{label} requires {tokens} audio tokens; budget is {max_tokens}. "
|
|
"Lower audio_max_seconds or choose truncate."
|
|
)
|
|
latent = latent[..., :max_tokens // 2].clone()
|
|
return latent
|
|
|
|
|
|
def _check_total_token_budget(rows: List[Tuple[H3RefMod, float]], max_total_tokens: int = 0) -> int:
|
|
if not isinstance(max_total_tokens, int) or isinstance(max_total_tokens, bool) or max_total_tokens < 0:
|
|
raise ValueError("Combined token budget must be a non-negative integer.")
|
|
total = sum(mod.token_count for mod, strength in rows if float(strength) > 0)
|
|
if max_total_tokens and total > max_total_tokens:
|
|
raise ValueError(
|
|
f"Combined visual+audio references require {total} tokens; budget is {max_total_tokens}. "
|
|
"Lower the visual/audio budgets or set max_total_tokens to 0 to disable the extra cap."
|
|
)
|
|
return total
|
|
|
|
|
|
def _create_mod_from_folder(
|
|
folder: str,
|
|
name: str,
|
|
mode: str,
|
|
concept_type: str,
|
|
audio_concept_type: str,
|
|
vae,
|
|
audio_vae=None,
|
|
ref_resolution: int = 1024,
|
|
pool_h: int = 16,
|
|
pool_w: int = 16,
|
|
latent_frames: int = 16,
|
|
max_tokens: int = 8192,
|
|
identity: int = 500,
|
|
multiplier: int = 1,
|
|
max_frames: int = 240,
|
|
background_retention: float = 0.0,
|
|
audio_max_seconds: float = 30.0,
|
|
audio_max_tokens: int = 5120,
|
|
audio_budget_policy: str = "error",
|
|
max_total_tokens: int = 0,
|
|
description: str = "",
|
|
subfolder: str = "",
|
|
save: bool = True,
|
|
save_layout: str = "bundle",
|
|
budget_policy: str = "truncate",
|
|
merge: bool = False,
|
|
motion_only: bool = False,
|
|
extraction_preset: str = "manual",
|
|
include_images: bool = False,
|
|
) -> List[Tuple[H3RefMod, float]]:
|
|
"""Create refs from one folder and optionally save them as split files or one bundle."""
|
|
if budget_policy not in ("truncate", "error"):
|
|
raise ValueError("Unknown visual token budget policy.")
|
|
if save_layout not in ("bundle", "separate_files"):
|
|
raise ValueError("Unknown save layout.")
|
|
if audio_budget_policy not in ("error", "truncate"):
|
|
raise ValueError("Unknown audio token budget policy.")
|
|
if not math.isfinite(audio_max_seconds) or audio_max_seconds <= 0:
|
|
raise ValueError("audio_max_seconds must be a positive finite number.")
|
|
name = _sanitize_name(name)
|
|
mode, ref_resolution, pool_h, pool_w, identity, merge, motion_only, concept_type, audio_concept_type = _apply_extraction_preset(
|
|
mode, ref_resolution, pool_h, pool_w, identity, merge, motion_only,
|
|
concept_type, audio_concept_type, extraction_preset,
|
|
"H3RefModCreateFromFolder"
|
|
)
|
|
description = (description or "").strip() or _auto_folder_description(folder, concept_type)
|
|
|
|
images, audios = _scan_folder(folder)
|
|
has_visual = bool(images)
|
|
has_audio_files = bool(audios)
|
|
if not has_visual and not has_audio_files:
|
|
raise ValueError(
|
|
f"H3RefModCreateFromFolder: no images or audio found in '{folder}'. "
|
|
"Video files are not supported. Use around six reference images instead.")
|
|
print(f"[H3RefModCreateFromFolder] {folder}: {len(images)} image(s), "
|
|
f"{len(audios)} audio file(s)")
|
|
|
|
ignored = []
|
|
if has_visual and mode == "encode":
|
|
ignored.append("pool_h, pool_w, Refinement Steps, merge (Full Reference)")
|
|
if motion_only:
|
|
ignored.append("motion_only (video-file support removed)")
|
|
if has_visual and max_tokens == 0:
|
|
ignored.append("budget_policy (max_tokens=0)")
|
|
if ignored:
|
|
print("[H3RefModCreateFromFolder] ignored: " + "; ".join(ignored))
|
|
if has_visual and concept_type == "identity" and mode == "training" and max(pool_h, pool_w) < 16:
|
|
print(
|
|
f"[H3RefModCreateFromFolder] warning: concept_type='identity' with "
|
|
f"mode='training' at a {pool_h}x{pool_w} grid — pooling averages away "
|
|
f"exactly the detail that carries a face. For a person, either switch "
|
|
f"mode='encode' or raise pool_h/pool_w toward 32x32+."
|
|
)
|
|
|
|
device = comfy.model_management.get_torch_device()
|
|
|
|
# ── audio identity (optional) ────────────────────────────────────
|
|
audio_latent = None
|
|
ref_audio_t = 0
|
|
if audios:
|
|
if audio_vae is not None:
|
|
aframes = []
|
|
remaining_seconds = float(audio_max_seconds)
|
|
for ap in audios:
|
|
if remaining_seconds <= 1e-9:
|
|
print("[H3RefModCreateFromFolder] audio_max_seconds reached; skipping remaining audio files.")
|
|
break
|
|
waveform, sr = _load_audio_waveform(ap)
|
|
max_samples = max(1, int(round(remaining_seconds * sr)))
|
|
if waveform.shape[-1] > max_samples:
|
|
waveform = waveform[..., :max_samples].contiguous()
|
|
z = _encode_ref_audio(audio_vae, waveform, sr, device)
|
|
print(f"[H3RefModCreateFromFolder] audio {os.path.basename(ap)}: "
|
|
f"latent {tuple(z.shape)}")
|
|
aframes.append(z.to(torch.float16))
|
|
remaining_seconds -= waveform.shape[-1] / float(sr)
|
|
audio_latent = torch.cat(aframes, dim=-1) if len(aframes) > 1 else aframes[0]
|
|
audio_latent = _crop_audio_latent(
|
|
audio_latent, audio_max_tokens, audio_budget_policy,
|
|
"H3RefModCreateFromFolder"
|
|
) if audio_latent is not None else None
|
|
ref_audio_t = 0 if audio_latent is None else audio_latent.shape[-1]
|
|
else:
|
|
print(f"[H3RefModCreateFromFolder] {len(audios)} audio file(s) found but no "
|
|
"audio_vae connected — audio NOT embedded.")
|
|
|
|
include_audio = ref_audio_t > 0
|
|
if not has_visual and not include_audio:
|
|
if has_audio_files and audio_vae is None:
|
|
raise ValueError(
|
|
f"H3RefModCreateFromFolder: '{folder}' contains only audio files, but no audio_vae is connected. "
|
|
"Connect the MiniMax H3 audio VAE or add reference images."
|
|
)
|
|
raise ValueError(
|
|
f"H3RefModCreateFromFolder: '{folder}' did not produce any usable visual or audio references."
|
|
)
|
|
|
|
# ── load visual refs as tensors ──────────────────────────────────
|
|
sources = []
|
|
for p in images:
|
|
image, mask = _load_image_and_alpha_mask_file(p, max_edge=ref_resolution * 2)
|
|
sources.append((image, mask))
|
|
if merge and mode != "training":
|
|
print("[H3RefModCreateFromFolder] warning: 'merge' only applies to "
|
|
"training mode — stacking the refs as usual for mode='encode'.")
|
|
|
|
# shared spatial canvas (encode mode) / pool grid (training mode)
|
|
canvas = None
|
|
if mode == "encode" and len(sources) > 1:
|
|
h, w = sources[0][0].shape[1], sources[0][0].shape[2]
|
|
scale = min(1.0, ref_resolution / min(h, w))
|
|
canvas = (max(32, round(w * scale / 32) * 32),
|
|
max(32, round(h * scale / 32) * 32))
|
|
pool_grid = None
|
|
if mode == "training" and sources:
|
|
h0, w0 = sources[0][0].shape[1], sources[0][0].shape[2]
|
|
pool_grid = aspect_grid(pool_h, pool_w, h0 / w0)
|
|
if pool_grid != (pool_h, pool_w):
|
|
print(f"[H3RefModCreateFromFolder] pooled grid {pool_h}x{pool_w} -> "
|
|
f"{pool_grid[0]}x{pool_grid[1]} to match source aspect "
|
|
f"{w0}x{h0} (avoids squishing the subject wide)")
|
|
gh, gw = pool_grid if pool_grid is not None else (pool_h, pool_w)
|
|
|
|
# ── encode each source ───────────────────────────────────────────
|
|
frames = []
|
|
source_shapes = []
|
|
n_img = 0
|
|
n_refs = len(sources)
|
|
mask_applied = False
|
|
merge_refs = [] if (merge and mode == "training" and n_refs > 1) else None
|
|
pbar = comfy.utils.ProgressBar(n_refs)
|
|
for idx, (src, source_mask) in enumerate(sources):
|
|
label = f"ref {idx + 1}/{n_refs} (image)"
|
|
src = src[:1]
|
|
if mode == "encode":
|
|
src = _resize_ref(src, ref_resolution, canvas)
|
|
else:
|
|
src = _resize_ref(src, ref_resolution, None)
|
|
src = _ensure_min_size(src)
|
|
mask_px = None
|
|
if source_mask is not None:
|
|
mask_px = _resize_mask(source_mask, src.shape[1], src.shape[2])
|
|
with torch.no_grad():
|
|
z = vae.encode(src)
|
|
if z.dim() != 5 or z.shape[1] != 24:
|
|
raise ValueError(f"Expected a MiniMax H3 video VAE latent [1,24,T,H,W], "
|
|
f"got {tuple(z.shape)}. The connected VAE is not the H3 VAE.")
|
|
source_shapes.append(f"{z.shape[2]}x{z.shape[3]}x{z.shape[4]}")
|
|
if mask_px is not None:
|
|
z = _mask_latent(z, mask_px, background_retention, seed_key=f"{name}:{idx}")
|
|
mask_applied = True
|
|
print(f"[H3RefModCreateFromFolder] {label}: applied embedded alpha mask "
|
|
f"(background_retention={background_retention})")
|
|
if mode == "encode":
|
|
pooled = z.to(torch.float16)
|
|
else:
|
|
pooled = pool_latent(z, 1, gh, gw).to(torch.float16)
|
|
if merge_refs is not None:
|
|
merge_refs.append((pooled.cpu(), z.float().cpu()))
|
|
elif identity > 0:
|
|
pooled = optimize_latent(pooled, z.float(), steps=int(identity),
|
|
progress_every=100)
|
|
if merge_refs is None:
|
|
frames.append(pooled)
|
|
n_img += 1
|
|
print(f"[H3RefModCreateFromFolder] {label}: encoded {tuple(pooled.shape)}")
|
|
pbar.update_absolute(idx + 1)
|
|
|
|
merged_n = 0
|
|
latent = None
|
|
if merge_refs is not None:
|
|
common_t = max(p.shape[2] for p, _ in merge_refs)
|
|
init = torch.stack([pool_latent(p, common_t, gh, gw) for p, _ in merge_refs]).mean(0)
|
|
print(f"[H3RefModCreateFromFolder] merging {len(merge_refs)} references into one shared {common_t}x{gh}x{gw} latent"
|
|
+ (f", {int(identity)} joint gradient steps" if identity > 0 else " (pure pooling mean — identity=0)"))
|
|
if identity > 0:
|
|
init = optimize_latent_multi(init, [f for _, f in merge_refs], steps=int(identity), progress_every=100)
|
|
print("[H3RefModCreateFromFolder] merge refinement done")
|
|
latent = init.to(torch.float16)
|
|
merged_n = len(merge_refs)
|
|
elif frames:
|
|
latent = torch.cat(frames, dim=2)
|
|
if latent is not None and multiplier > 1:
|
|
latent = latent.repeat(1, 1, multiplier, 1, 1)
|
|
if latent is not None and max_tokens > 0 and budget_policy == "error":
|
|
tokens = _latent_token_count(latent)
|
|
if tokens > max_tokens:
|
|
raise ValueError(
|
|
f"H3RefModCreateFromFolder: extracted {tokens} visual tokens, which exceeds "
|
|
f"max_tokens={max_tokens}. Increase the budget, reduce frames/resolution, "
|
|
"or switch budget_policy to 'truncate'."
|
|
)
|
|
if latent is not None and max_tokens > 0:
|
|
latent = fit_token_budget(latent, max_tokens, name)
|
|
if latent is not None:
|
|
total_t = latent.shape[2]
|
|
kind = "video" if total_t > 1 else "image"
|
|
px_w, px_h = latent.shape[4] * 16, latent.shape[3] * 16
|
|
if merged_n:
|
|
source = f"merge {merged_n} refs"
|
|
elif len(frames) > 1:
|
|
source = "stack"
|
|
else:
|
|
source = "image"
|
|
|
|
out_dir = _resolve_output_dir(subfolder)
|
|
requested_base = _base_refmod_name(name)
|
|
bundle_name = resolved_base = requested_base
|
|
bundle_path_no_ext = os.path.join(out_dir, requested_base)
|
|
if save and save_layout == "separate_files":
|
|
resolved_base, visual_name, visual_path_no_ext, audio_name, audio_path_no_ext = _unique_split_mod_paths(
|
|
out_dir, requested_base, include_audio)
|
|
if resolved_base != requested_base:
|
|
print(f"[H3RefModCreateFromFolder] '{requested_base}' already exists in {out_dir} — "
|
|
f"saving as '{resolved_base}{VISUAL_SUFFIX}'"
|
|
+ (f" and '{resolved_base}{AUDIO_SUFFIX}'" if include_audio else "")
|
|
+ " instead (existing mods are never overwritten).")
|
|
elif save:
|
|
bundle_name, bundle_path_no_ext = _unique_bundle_mod_path(out_dir, requested_base)
|
|
resolved_base = bundle_name
|
|
if bundle_name != requested_base:
|
|
print(f"[H3RefModCreateFromFolder] '{requested_base}' already exists in {out_dir} — "
|
|
f"saving as '{bundle_name}' instead (existing mods are never overwritten).")
|
|
visual_name, audio_name = _paired_refmod_names(bundle_name)
|
|
visual_path_no_ext = os.path.join(out_dir, visual_name)
|
|
audio_path_no_ext = os.path.join(out_dir, audio_name)
|
|
else:
|
|
resolved_base = requested_base
|
|
visual_name, audio_name = _paired_refmod_names(resolved_base)
|
|
visual_path_no_ext = os.path.join(out_dir, visual_name)
|
|
audio_path_no_ext = os.path.join(out_dir, audio_name)
|
|
|
|
rows: List[Tuple[H3RefMod, float]] = []
|
|
visual_mod = None
|
|
if latent is not None:
|
|
visual_mod = H3RefMod(
|
|
name=visual_name,
|
|
kind=kind,
|
|
latent=latent,
|
|
latent_h=latent.shape[3],
|
|
latent_w=latent.shape[4],
|
|
latent_t=total_t,
|
|
mode=mode,
|
|
source=source,
|
|
source_shape=" +".join(source_shapes),
|
|
pool=(f"full-res {px_w}x{px_h}px (short-edge cap {ref_resolution}px)"
|
|
if mode == "encode" else f"{total_t}x{gh}x{gw}"),
|
|
optimize_steps=int(identity) if mode == "training" else 0,
|
|
tags=[f"{n_img} img"]
|
|
+ ([f"merged {merged_n} refs"] if merged_n else [])
|
|
+ ([f"masked (bg_retention={background_retention})"] if mask_applied else [])
|
|
+ ([f"x{multiplier} repeat"] if multiplier > 1 else [])
|
|
+ ([f"paired {audio_name}"] if include_audio else []),
|
|
description=description,
|
|
concept_type=concept_type,
|
|
audio_concept_type="",
|
|
)
|
|
rows.append((visual_mod, 1.0))
|
|
if include_audio:
|
|
audio_mod = _make_audio_refmod(
|
|
audio_name, audio_latent, audio_concept_type, description,
|
|
sample_rate=getattr(audio_vae, "audio_sample_rate", 32000),
|
|
)
|
|
rows.append((audio_mod, 1.0))
|
|
total_tokens = _check_total_token_budget(rows, max_total_tokens)
|
|
if include_images and visual_mod is not None:
|
|
encoder_frames, encoder_times, _source = _encoder_frames(visual_mod, vae, 24.0)
|
|
pack_encoder_images(visual_mod, encoder_frames, encoder_times)
|
|
if save:
|
|
if save_layout == "bundle":
|
|
destination = save_refmod_bundle(bundle_path_no_ext, bundle_name, [mod for mod, _ in rows])
|
|
_copy_refmod_thumbnail(images, bundle_path_no_ext)
|
|
_MOD_CACHE[bundle_name] = tuple(mod for mod, _ in rows)
|
|
print(f"[CreateH3RefMod] saved bundle '{bundle_name}' with {len(rows)} member(s) -> {destination}")
|
|
else:
|
|
if visual_mod is not None:
|
|
visual_path = visual_mod.save(visual_path_no_ext)
|
|
_copy_refmod_thumbnail(images, visual_path_no_ext)
|
|
_MOD_CACHE[visual_mod.name] = (visual_mod,)
|
|
print(f"[CreateH3RefMod] saved {_summarize(visual_mod)} -> {visual_path}")
|
|
if include_audio:
|
|
audio_mod = rows[-1][0]
|
|
audio_path = audio_mod.save(audio_path_no_ext)
|
|
_MOD_CACHE[audio_mod.name] = (audio_mod,)
|
|
print(f"[CreateH3RefMod] saved {_summarize(audio_mod)} -> {audio_path}")
|
|
if len(_MOD_CACHE) > _MOD_CACHE_MAX:
|
|
_MOD_CACHE.pop(next(iter(_MOD_CACHE)))
|
|
_nodes_mod._MOD_LIST_CACHE_KEY = None # refresh the loader dropdown
|
|
else:
|
|
if visual_mod is not None:
|
|
print(f"[CreateH3RefMod] {_summarize(visual_mod)} (not saved)")
|
|
if include_audio:
|
|
print(f"[CreateH3RefMod] {_summarize(rows[-1][0])} (not saved)")
|
|
print(f"[CreateH3RefMod] complete: {len(rows)} reference(s), {total_tokens} tokens")
|
|
return rows
|
|
|
|
|
|
class H3RefModCreateFromFolder(io.ComfyNode):
|
|
"""Create a RefMod from reference images and optional audio in a folder.
|
|
|
|
Scans a dataset folder, encodes the media with the connected H3 VAEs,
|
|
and saves a visual RefMod.
|
|
When audio is present and an audio VAE is connected, it also saves a
|
|
paired audio RefMod. Defaults to an ``identity`` concept in ``encode``
|
|
mode — the right choice for a person/character.
|
|
|
|
Flagged as an output node (``is_output_node=True``) so it actually runs
|
|
even when nothing is wired to its ``mods`` output — it still saves to
|
|
disk either way. ``mods`` carries the freshly created mod at strength
|
|
1.0 (same ``H3_REF_MODS`` shape ``H3RefModExtract`` outputs), so you can
|
|
chain it straight into Apply H3 RefMod / MiniMax H3 RefMods to Video
|
|
without a separate reload step — or just ignore it and reload later with
|
|
Load H3 RefMods.
|
|
"""
|
|
|
|
@classmethod
|
|
def define_schema(cls):
|
|
return io.Schema(
|
|
node_id="H3RefModCreateFromFolder",
|
|
display_name="Create H3 RefMod From Folder",
|
|
category="H3RefModPicker",
|
|
description="Create RefMods from reference images and optional audio. Around six images usually suffice. "
|
|
"Saves to models/refmods/ by default.",
|
|
inputs=[
|
|
io.String.Input("folder", default="",
|
|
tooltip="Dataset folder with reference images and optional audio. REQUIRED — "
|
|
"an absolute path, or a folder name inside ComfyUI's input/ "
|
|
"directory. The node will NOT run if left empty. "
|
|
"Use around six images with the same aspect ratio whenever possible. Video files are ignored."),
|
|
io.Boolean.Input("use_subfolders", default=False,
|
|
label_on="subfolders", label_off="single folder",
|
|
tooltip="When ON, the folder input is treated as a parent directory: "
|
|
"every immediate subfolder is scanned and turned into its own "
|
|
"RefMod, named after the subfolder. The 'name' input is ignored "
|
|
"in this mode, and the description is auto-generated from each "
|
|
"subfolder name. Great for batch-processing a dataset of concepts."),
|
|
io.Boolean.Input("use_folder_as_name", default=True,
|
|
label_on="folder name", label_off="manual name",
|
|
tooltip="When ON, single-folder mode names the RefMod from the folder, using "
|
|
"the same naming scheme as subfolder mode. The 'name' input is ignored. "
|
|
"Turn this OFF to type a manual name instead."),
|
|
io.String.Input("name", default="name_refMod",
|
|
tooltip="Saved mod name when 'use folder as name' is OFF. Appears in the "
|
|
"Load H3 RefMods dropdown after a reload."),
|
|
io.Combo.Input("extraction_preset", options=["manual", "identity_encode", "style_experimental", "motion_sequence"], default="identity_encode",
|
|
tooltip="manual preserves controls. identity_encode: identity + voice, Full Reference, resolution=1024, steps=0. "
|
|
"style_experimental: style + voice, Compressed Reference, 8x8 pool, 150 steps. "
|
|
"motion_sequence: pose references from images + voice, Compressed Reference, 16x16 pool; preserves Refinement Steps."),
|
|
io.Boolean.Input("advanced", default=False,
|
|
label_on="advanced", label_off="simple",
|
|
tooltip="Preset display mode. simple hides most preset-managed controls; advanced shows the fuller control set while preserving the same preset values."),
|
|
io.Combo.Input("concept_type", options=list(CONCEPT_TYPES), default="identity",
|
|
tooltip="What this mod represents. 'identity' (default) = a specific "
|
|
"person/character; also pose_motion, clothing, background, style, "
|
|
"generic. Audio-style labels are also accepted for compatibility with "
|
|
"upstream RefMods, but visual concepts should usually stay on identity/"
|
|
"pose_motion/clothing/background/style."),
|
|
io.Combo.Input("audio_concept_type", options=list(AUDIO_CONCEPT_TYPES), default="voice",
|
|
tooltip="How to label embedded audio in this RefMod. Matches upstream audio "
|
|
"concept labels: voice (default), singing, music_style, sound_fx, "
|
|
"ambience."),
|
|
io.Combo.Input("mode", options=["Full Reference", "Compressed Reference"], default="Full Reference",
|
|
tooltip="'Full Reference' (default) = full-res VAE encode, max identity (~1K "
|
|
"tokens/img) — recommended for people. 'Compressed Reference' = pooled grid "
|
|
"refined by 'Refinement Steps' — cheaper, concept/motion only. Legacy "
|
|
"'encode'/'training' values remain accepted."),
|
|
io.Vae.Input("vae",
|
|
tooltip="The MiniMax H3 video VAE."),
|
|
io.Vae.Input("audio_vae", optional=True,
|
|
tooltip="The MiniMax H3 audio VAE — needed to embed audio files found in "
|
|
"the folder. If not connected, audio is skipped with a note."),
|
|
io.Int.Input("ref_resolution", default=1024, min=256, max=2048, step=64,
|
|
tooltip="Target short edge in px (downscale only). 1024 default; 2048 = "
|
|
"official max fidelity, 4x the tokens."),
|
|
io.Int.Input("pool_h", default=16, min=2, max=64, step=2,
|
|
tooltip="Compressed Reference mode: pooled latent height. The grid is auto-fit "
|
|
"to the first source's aspect ratio so a portrait subject isn't squished."),
|
|
io.Int.Input("pool_w", default=16, min=2, max=64, step=2,
|
|
tooltip="Compressed Reference mode: pooled latent width (long edge if the source is wider than tall)."),
|
|
io.Int.Input("max_tokens", default=8192, min=0, max=65536, step=512,
|
|
tooltip="Hard cap on total injected tokens (0 = no cap). Near-duplicate "
|
|
"frames dropped first, then resampled to fit."),
|
|
io.Int.Input("identity", display_name="Refinement Steps", default=500, min=0, max=2000, step=50,
|
|
tooltip="Compressed Reference only: gradient refinement steps (0 = pure pooling)."),
|
|
io.Boolean.Input("merge", default=False,
|
|
label_on="merge", label_off="stack",
|
|
tooltip="Compressed Reference only: optimize one shared consensus latent against "
|
|
"all refs instead of stacking each ref separately."),
|
|
io.Int.Input("multiplier", default=1, min=1, max=10, step=1,
|
|
tooltip="Repeat the extracted latent along time so a short reference is not drowned out by "
|
|
"the main video's token budget. 1 = no repeat."),
|
|
io.Float.Input("background_retention", default=0.0, min=0.0, max=1.0, step=0.05,
|
|
tooltip="Only used for still images in the folder that carry an embedded alpha mask. "
|
|
"Outside the masked subject, 0 collapses the latent toward a blurred copy of "
|
|
"itself, 1 keeps the full background. Opaque images ignore this."),
|
|
io.Float.Input("audio_max_seconds", default=30.0, min=0.025, max=600.0,
|
|
tooltip="Combined audio reference length cap across folder audio files before encoding. "
|
|
"Longer audio creates more tokens; 30 seconds matches the upstream master default."),
|
|
io.Int.Input("audio_max_tokens", default=5120, min=0, max=2147483647, step=512,
|
|
tooltip="Hard cap on the paired audio RefMod token count (0 = no cap)."),
|
|
io.Combo.Input("audio_budget_policy", options=["error", "truncate"], default="error",
|
|
tooltip="On audio_max_tokens overflow: error stops without saving, truncate crops "
|
|
"the combined folder audio latent to fit the token budget."),
|
|
io.Int.Input("max_total_tokens", default=0, min=0, max=1048576, step=512,
|
|
tooltip="Extra combined cap across visual and audio tokens. 0 disables this shared limit."),
|
|
io.Combo.Input("budget_policy", options=["truncate", "error"], default="truncate",
|
|
tooltip="On max_tokens overflow: truncate uses the existing frame reduction; "
|
|
"error stops without saving. 0 max_tokens disables the cap."),
|
|
io.String.Input("description", default="", multiline=True,
|
|
tooltip="Optional text describing the concept, stored in the mod and "
|
|
"emitted by the loaders' prompt_hint output. If left empty, "
|
|
"Create From Folder auto-generates one from the folder name. "
|
|
"In subfolder mode this field is ignored and each subfolder gets "
|
|
"its own auto-generated description. A thumbnail is also copied "
|
|
"from the source images: 3:4 images first, otherwise the largest one."),
|
|
io.String.Input("subfolder", default="", optional=True,
|
|
tooltip="Optional folder inside the RefMods save directory, for example celebs or voices."),
|
|
io.Boolean.Input("save", default=True, label_on="save", label_off="don't save",
|
|
tooltip="Save the mod to disk so Load H3 RefMods can pick it up later."),
|
|
io.Combo.Input("save_layout", options=["bundle", "separate_files"], default="bundle",
|
|
tooltip="bundle (default) saves one single-file RefMod that can contain visual and audio members together. separate_files keeps the older *_Video and *_Audio files."),
|
|
io.Boolean.Input("include_images", default=False,
|
|
tooltip="Save reconstructed pictures so RefMods to Video can visually ground prompt labels "
|
|
"without decoding the latent again. Increases file size."),
|
|
],
|
|
outputs=[
|
|
io.Custom("H3_REF_MODS").Output("mods",
|
|
tooltip="Bundle with the freshly created mod(s) at strength 1.0 — feed "
|
|
"straight into Apply H3 RefMod / MiniMax H3 RefMods to Video, no "
|
|
"reload needed. In subfolder mode this "
|
|
"contains one mod per subfolder."),
|
|
],
|
|
is_output_node=True,
|
|
)
|
|
|
|
@classmethod
|
|
def execute(cls, folder, use_folder_as_name, name, mode, concept_type, audio_concept_type, vae, audio_vae=None,
|
|
ref_resolution=1024, pool_h=16, pool_w=16, latent_frames=16,
|
|
max_tokens=8192, identity=500, merge=False, motion_only=False,
|
|
multiplier=1, max_frames=240, background_retention=0.0, extraction_preset="identity_encode",
|
|
audio_max_seconds=30.0, audio_max_tokens=5120,
|
|
audio_budget_policy="error", max_total_tokens=0,
|
|
description="", subfolder="", budget_policy="truncate",
|
|
save=True, save_layout="bundle", advanced=False, use_subfolders=False, save_dir="",
|
|
include_images=False) -> io.NodeOutput:
|
|
del advanced
|
|
del save_dir
|
|
raw_folder = (folder or "").strip().strip('"')
|
|
if not (folder or "").strip().strip('"'):
|
|
print("[H3RefModCreateFromFolder] folder input is empty; refusing to scan.")
|
|
raise ValueError(
|
|
"Create H3 RefMod: 'folder' is empty. Point it at a dataset folder "
|
|
"(absolute path, or a name inside input/) — the node refuses to run "
|
|
"without one so it can't accidentally scan your whole input/ directory.")
|
|
folder = _resolve_folder(folder)
|
|
mode = normalize_mode(mode)
|
|
print(
|
|
f"[H3RefModCreateFromFolder] start: folder='{raw_folder}' -> '{folder}', "
|
|
f"use_subfolders={bool(use_subfolders)}, preset='{extraction_preset}', mode='{mode}'"
|
|
)
|
|
|
|
if use_subfolders:
|
|
subfolders = [
|
|
os.path.join(folder, d)
|
|
for d in sorted(os.listdir(folder))
|
|
if os.path.isdir(os.path.join(folder, d))
|
|
]
|
|
if not subfolders:
|
|
raise ValueError(
|
|
f"H3RefModCreateFromFolder: use_subfolders=True but no subfolders "
|
|
f"found in '{folder}'. In subfolder mode, files placed directly in "
|
|
"that folder are ignored.")
|
|
print(f"[H3RefModCreateFromFolder] subfolder mode: {len(subfolders)} folder(s)")
|
|
mods = []
|
|
for sub in subfolders:
|
|
sub_name = _folder_refmod_name(sub)
|
|
created = _create_mod_from_folder(
|
|
sub, sub_name, mode, concept_type, audio_concept_type, vae, audio_vae,
|
|
ref_resolution, pool_h, pool_w, latent_frames, max_tokens,
|
|
identity, multiplier, max_frames, background_retention,
|
|
audio_max_seconds, audio_max_tokens,
|
|
audio_budget_policy, max_total_tokens,
|
|
_auto_folder_description(sub, concept_type), subfolder,
|
|
save, save_layout, budget_policy, merge, motion_only, extraction_preset,
|
|
include_images=include_images,
|
|
)
|
|
mods.extend(created)
|
|
return io.NodeOutput(mods)
|
|
|
|
name = _folder_refmod_name(folder) if use_folder_as_name else _sanitize_name(name)
|
|
rows = _create_mod_from_folder(
|
|
folder, name, mode, concept_type, audio_concept_type, vae, audio_vae,
|
|
ref_resolution, pool_h, pool_w, latent_frames, max_tokens,
|
|
identity, multiplier, max_frames, background_retention,
|
|
audio_max_seconds, audio_max_tokens,
|
|
audio_budget_policy, max_total_tokens, description, subfolder,
|
|
save, save_layout, budget_policy, merge, motion_only, extraction_preset,
|
|
include_images=include_images,
|
|
)
|
|
return io.NodeOutput(rows)
|
|
|
|
|
|
NODE_CLASS_MAPPINGS = {
|
|
"H3RefModCreateFromFolder": H3RefModCreateFromFolder,
|
|
}
|
|
|
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
|
"H3RefModCreateFromFolder": "Create H3 RefMod From Folder",
|
|
}
|