Files
DumasNodes/common.py
T
chris.dumas 70d0672ee3 Squashed 'vendor/dumas_minimax_h3_refmod/' content from commit 517e8c4
git-subtree-dir: vendor/dumas_minimax_h3_refmod
git-subtree-split: 517e8c419a8584fbf69fceb487bc39d83e6a6ec8
2026-10-20 13:43:44 +00:00

238 lines
9.6 KiB
Python

"""Shared helpers for the RefMod pack: media loading and the refmods folder."""
from __future__ import annotations
import os
import ntpath
import math
import random
from typing import List, Optional, Tuple
import torch
import comfy.utils
IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif"}
VIDEO_EXTS = {".mp4", ".webm", ".mov", ".mkv", ".avi", ".m4v"}
def refmods_dir() -> str:
"""First registered RefMod root, or the default; selection does not write."""
import folder_paths
try:
roots = folder_paths.get_folder_paths("refmods")
except KeyError:
roots = []
return roots[0] if roots else os.path.join(folder_paths.models_dir, "refmods")
def list_media_files(folder: str) -> Tuple[List[str], List[str]]:
"""(images, videos) directly under ``folder`` (top level only), sorted by name."""
images, videos = [], []
if os.path.isdir(folder):
for fn in sorted(os.listdir(folder)):
ext = os.path.splitext(fn)[1].lower()
p = os.path.join(folder, fn)
if os.path.isfile(p):
if ext in IMAGE_EXTS:
images.append(p)
elif ext in VIDEO_EXTS:
videos.append(p)
return images, videos
def load_image_file(path: str, max_edge: Optional[int] = None) -> torch.Tensor:
"""Load one image file -> [1, H, W, 3] float32 in [0, 1]."""
import numpy as np
from PIL import Image
with Image.open(path) as img:
img = img.convert("RGB")
w, h = img.size
if max_edge is not None:
scale = min(1.0, max_edge / max(w, h))
if scale < 1.0:
img = img.resize((max(1, round(w * scale)), max(1, round(h * scale))),
Image.LANCZOS)
arr = torch.from_numpy(np.asarray(img).copy()).float() / 255.0
return arr.unsqueeze(0) # [1, H, W, 3]
def _sample_video_frames(frames, total, max_frames, max_edge):
"""Uniform known-length sampling; bounded deterministic reservoir otherwise."""
import numpy as np
from PIL import Image
known = math.isfinite(total) and total > 0
total = int(total) if known else 0
targets = None
if total > max_frames:
targets = {round(i * (total - 1) / max(1, max_frames - 1)) for i in range(max_frames)}
kept = []
rng = random.Random(0)
for index, frame in enumerate(frames):
if targets is not None and index not in targets:
continue
# Treat underestimated counts as unknown after filling the reservoir.
slot = len(kept) if len(kept) < max_frames else rng.randrange(index + 1)
if slot >= max_frames:
continue
frame = np.asarray(frame)
if max_edge is not None:
h, w = frame.shape[:2]
scale = min(1.0, max_edge / max(h, w))
if scale < 1.0:
frame = np.asarray(Image.fromarray(frame).resize(
(max(1, round(w * scale)), max(1, round(h * scale))), Image.Resampling.BILINEAR))
item = (index, frame.copy())
if slot == len(kept):
kept.append(item)
else:
kept[slot] = item
if targets is not None and len(kept) == len(targets):
break
if not kept:
raise ValueError("Video contains no decodable frames.")
return torch.from_numpy(np.stack([frame for _, frame in sorted(kept)])).float() / 255.0
def load_video_file(path: str, max_frames: int = 240,
max_edge: Optional[int] = None) -> torch.Tensor:
"""Load bounded, ordered RGB frames; release either decoder on every exit."""
if max_frames < 1:
raise ValueError("max_frames must be at least 1.")
errors = []
try:
import cv2
cap = cv2.VideoCapture(path)
try:
total = cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0
def frames():
while True:
ok, frame = cap.read()
if not ok:
break
yield cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
return _sample_video_frames(frames(), total, max_frames, max_edge)
finally:
cap.release()
except (ImportError, OSError, ValueError, RuntimeError) as exc:
errors.append(str(exc))
try:
import imageio.v2 as imageio
reader = imageio.get_reader(path)
try:
total = reader.get_meta_data().get("nframes", 0) or 0
return _sample_video_frames(iter(reader), total, max_frames, max_edge)
finally:
reader.close()
except (ImportError, OSError, ValueError, RuntimeError) as exc:
errors.append(str(exc))
raise RuntimeError(f"Cannot decode video '{path}': " + "; ".join(errors))
def mod_output_path(name, subfolder=""):
relative = "/".join(part for part in (subfolder.replace("\\", "/"), name) if part)
if (ntpath.splitdrive(relative)[0] or relative.startswith("/") or
any(p in ("..", ".", "graph_presets", ".git", "__pycache__") for p in relative.split("/"))):
raise ValueError("Use a relative RefMod subfolder without '..' or reserved directories.")
root = os.path.realpath(refmods_dir())
path = os.path.join(root, relative)
if os.path.commonpath((root, os.path.realpath(path + ".safetensors"))) != root:
raise ValueError("RefMod save path leaves the model folder.")
return path
def resize_ref(image, short_edge: int, canvas=None):
"""Aspect-preserving downscale (never upscale) to ``short_edge`` px; dims to /32.
When several refs are stacked into one mod they must share a single spatial
canvas, so ``canvas`` (tw, th) cover-crops each ref to it (like the official
node's follower keyframes). Mirrors the official ref2video node: refs are
resized before VAE encode, so the stored latent rides the same
full-resolution path the model was trained with (the pooled path below is
the cheap "thumbnail" alternative).
"""
h, w = image.shape[1], image.shape[2]
if h <= 0 or w <= 0:
raise ValueError(
f"_resize_ref: source has an empty frame ({h}x{w}) before any "
f"resize — the reference itself is invalid.")
scale = min(1.0, short_edge / min(h, w))
tw = max(32, round(w * scale / 32) * 32)
th = max(32, round(h * scale / 32) * 32)
crop = "disabled"
if canvas is not None:
tw, th = canvas
crop = "center"
if tw <= 0 or th <= 0:
raise ValueError(
f"_resize_ref: computed a zero-size resize target ({tw}x{th}) "
f"for a {h}x{w} source (short_edge={short_edge}, canvas={canvas}). "
f"This should be impossible — please report this shape.")
samples = image[..., :3].movedim(-1, 1)
samples = comfy.utils.common_upscale(samples, tw, th, "lanczos", crop)
if samples.shape[2] <= 0 or samples.shape[3] <= 0:
raise ValueError(
f"_resize_ref: common_upscale produced an empty result "
f"{tuple(samples.shape)} from a {h}x{w} source targeting "
f"{tw}x{th} (crop={crop}, canvas={canvas}). This points to a bug "
f"in comfy.utils.common_upscale for this input, not in RefMod's "
f"own math.")
return samples.movedim(1, -1)
def snap_to_causal_grid(n_frames: int) -> int:
"""Round down to H3's reference-video grid: 5, 22, 39, ... (17k+5).
The VAE encodes 17-frame chunks, pads the final chunk and drops three
trailing latent tokens. Its effective video grid is not the encoder's
standalone 4x temporal stride. RefMod keeps its single-image fallback
when fewer than five frames are selected.
"""
if n_frames < 1:
raise ValueError("A reference must contain at least one frame.")
if n_frames < 5:
return 1
return ((n_frames - 5) // 17) * 17 + 5
def sample_video_for_vae(frames: torch.Tensor, max_frames: Optional[int] = None) -> torch.Tensor:
"""Sample the full clip onto H3's grid without exceeding the frame cap.
Keep both endpoints whenever the selected count is greater than one.
Already-aligned clips are returned unchanged. A cap/input shorter than
five frames selects only the first image, using H3's still-image path.
"""
if max_frames is not None and max_frames < 1:
raise ValueError("max_frames must be at least 1.")
count = snap_to_causal_grid(min(frames.shape[0], max_frames)
if max_frames is not None else frames.shape[0])
if count == frames.shape[0]:
return frames
indices = torch.linspace(0, frames.shape[0] - 1, count, device=frames.device).round().long()
return frames[indices]
def ensure_min_size(image, floor: int = 320):
"""Upscale (never downscale) so both spatial dims are >= ``floor`` px.
The MiniMax H3 VAE encodes with internal tiled_encode (~256px tiles). A
reference smaller than the tile size in one dimension can make the tiler
compute a zero-size edge tile, which crashes deep inside conv_in with a
cryptic 'Expected 4D or 5D... but got [1,3,1,0,W]' error. This applies
regardless of extraction mode ('encode' already resizes down to
ref_resolution but never guarantees a floor; 'training' now resizes to
the same cap, also without a floor), so it's a separate, unconditional
safety net right before encode.
"""
import comfy.utils
h, w = image.shape[1], image.shape[2]
if h >= floor and w >= floor:
return image
scale = floor / min(h, w)
tw = max(floor, round(w * scale / 32) * 32)
th = max(floor, round(h * scale / 32) * 32)
samples = image[..., :3].movedim(-1, 1)
samples = comfy.utils.common_upscale(samples, tw, th, "lanczos", "disabled")
return samples.movedim(1, -1)