Squashed 'vendor/dumas_minimax_h3_refmod/' content from commit 517e8c4
git-subtree-dir: vendor/dumas_minimax_h3_refmod git-subtree-split: 517e8c419a8584fbf69fceb487bc39d83e6a6ec8
This commit is contained in:
@@ -0,0 +1,237 @@
|
||||
"""Shared helpers for the RefMod pack: media loading and the refmods folder."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import ntpath
|
||||
import math
|
||||
import random
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import comfy.utils
|
||||
|
||||
IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif"}
|
||||
VIDEO_EXTS = {".mp4", ".webm", ".mov", ".mkv", ".avi", ".m4v"}
|
||||
|
||||
|
||||
def refmods_dir() -> str:
|
||||
"""First registered RefMod root, or the default; selection does not write."""
|
||||
import folder_paths
|
||||
try:
|
||||
roots = folder_paths.get_folder_paths("refmods")
|
||||
except KeyError:
|
||||
roots = []
|
||||
return roots[0] if roots else os.path.join(folder_paths.models_dir, "refmods")
|
||||
|
||||
|
||||
def list_media_files(folder: str) -> Tuple[List[str], List[str]]:
|
||||
"""(images, videos) directly under ``folder`` (top level only), sorted by name."""
|
||||
images, videos = [], []
|
||||
if os.path.isdir(folder):
|
||||
for fn in sorted(os.listdir(folder)):
|
||||
ext = os.path.splitext(fn)[1].lower()
|
||||
p = os.path.join(folder, fn)
|
||||
if os.path.isfile(p):
|
||||
if ext in IMAGE_EXTS:
|
||||
images.append(p)
|
||||
elif ext in VIDEO_EXTS:
|
||||
videos.append(p)
|
||||
return images, videos
|
||||
|
||||
|
||||
def load_image_file(path: str, max_edge: Optional[int] = None) -> torch.Tensor:
|
||||
"""Load one image file -> [1, H, W, 3] float32 in [0, 1]."""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
with Image.open(path) as img:
|
||||
img = img.convert("RGB")
|
||||
w, h = img.size
|
||||
if max_edge is not None:
|
||||
scale = min(1.0, max_edge / max(w, h))
|
||||
if scale < 1.0:
|
||||
img = img.resize((max(1, round(w * scale)), max(1, round(h * scale))),
|
||||
Image.LANCZOS)
|
||||
arr = torch.from_numpy(np.asarray(img).copy()).float() / 255.0
|
||||
return arr.unsqueeze(0) # [1, H, W, 3]
|
||||
|
||||
|
||||
def _sample_video_frames(frames, total, max_frames, max_edge):
|
||||
"""Uniform known-length sampling; bounded deterministic reservoir otherwise."""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
known = math.isfinite(total) and total > 0
|
||||
total = int(total) if known else 0
|
||||
targets = None
|
||||
if total > max_frames:
|
||||
targets = {round(i * (total - 1) / max(1, max_frames - 1)) for i in range(max_frames)}
|
||||
kept = []
|
||||
rng = random.Random(0)
|
||||
for index, frame in enumerate(frames):
|
||||
if targets is not None and index not in targets:
|
||||
continue
|
||||
# Treat underestimated counts as unknown after filling the reservoir.
|
||||
slot = len(kept) if len(kept) < max_frames else rng.randrange(index + 1)
|
||||
if slot >= max_frames:
|
||||
continue
|
||||
frame = np.asarray(frame)
|
||||
if max_edge is not None:
|
||||
h, w = frame.shape[:2]
|
||||
scale = min(1.0, max_edge / max(h, w))
|
||||
if scale < 1.0:
|
||||
frame = np.asarray(Image.fromarray(frame).resize(
|
||||
(max(1, round(w * scale)), max(1, round(h * scale))), Image.Resampling.BILINEAR))
|
||||
item = (index, frame.copy())
|
||||
if slot == len(kept):
|
||||
kept.append(item)
|
||||
else:
|
||||
kept[slot] = item
|
||||
if targets is not None and len(kept) == len(targets):
|
||||
break
|
||||
if not kept:
|
||||
raise ValueError("Video contains no decodable frames.")
|
||||
return torch.from_numpy(np.stack([frame for _, frame in sorted(kept)])).float() / 255.0
|
||||
|
||||
|
||||
def load_video_file(path: str, max_frames: int = 240,
|
||||
max_edge: Optional[int] = None) -> torch.Tensor:
|
||||
"""Load bounded, ordered RGB frames; release either decoder on every exit."""
|
||||
if max_frames < 1:
|
||||
raise ValueError("max_frames must be at least 1.")
|
||||
errors = []
|
||||
try:
|
||||
import cv2
|
||||
cap = cv2.VideoCapture(path)
|
||||
try:
|
||||
total = cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0
|
||||
def frames():
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
break
|
||||
yield cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
return _sample_video_frames(frames(), total, max_frames, max_edge)
|
||||
finally:
|
||||
cap.release()
|
||||
except (ImportError, OSError, ValueError, RuntimeError) as exc:
|
||||
errors.append(str(exc))
|
||||
try:
|
||||
import imageio.v2 as imageio
|
||||
reader = imageio.get_reader(path)
|
||||
try:
|
||||
total = reader.get_meta_data().get("nframes", 0) or 0
|
||||
return _sample_video_frames(iter(reader), total, max_frames, max_edge)
|
||||
finally:
|
||||
reader.close()
|
||||
except (ImportError, OSError, ValueError, RuntimeError) as exc:
|
||||
errors.append(str(exc))
|
||||
raise RuntimeError(f"Cannot decode video '{path}': " + "; ".join(errors))
|
||||
|
||||
|
||||
def mod_output_path(name, subfolder=""):
|
||||
relative = "/".join(part for part in (subfolder.replace("\\", "/"), name) if part)
|
||||
if (ntpath.splitdrive(relative)[0] or relative.startswith("/") or
|
||||
any(p in ("..", ".", "graph_presets", ".git", "__pycache__") for p in relative.split("/"))):
|
||||
raise ValueError("Use a relative RefMod subfolder without '..' or reserved directories.")
|
||||
root = os.path.realpath(refmods_dir())
|
||||
path = os.path.join(root, relative)
|
||||
if os.path.commonpath((root, os.path.realpath(path + ".safetensors"))) != root:
|
||||
raise ValueError("RefMod save path leaves the model folder.")
|
||||
return path
|
||||
|
||||
|
||||
def resize_ref(image, short_edge: int, canvas=None):
|
||||
"""Aspect-preserving downscale (never upscale) to ``short_edge`` px; dims to /32.
|
||||
|
||||
When several refs are stacked into one mod they must share a single spatial
|
||||
canvas, so ``canvas`` (tw, th) cover-crops each ref to it (like the official
|
||||
node's follower keyframes). Mirrors the official ref2video node: refs are
|
||||
resized before VAE encode, so the stored latent rides the same
|
||||
full-resolution path the model was trained with (the pooled path below is
|
||||
the cheap "thumbnail" alternative).
|
||||
"""
|
||||
h, w = image.shape[1], image.shape[2]
|
||||
if h <= 0 or w <= 0:
|
||||
raise ValueError(
|
||||
f"_resize_ref: source has an empty frame ({h}x{w}) before any "
|
||||
f"resize — the reference itself is invalid.")
|
||||
scale = min(1.0, short_edge / min(h, w))
|
||||
tw = max(32, round(w * scale / 32) * 32)
|
||||
th = max(32, round(h * scale / 32) * 32)
|
||||
crop = "disabled"
|
||||
if canvas is not None:
|
||||
tw, th = canvas
|
||||
crop = "center"
|
||||
if tw <= 0 or th <= 0:
|
||||
raise ValueError(
|
||||
f"_resize_ref: computed a zero-size resize target ({tw}x{th}) "
|
||||
f"for a {h}x{w} source (short_edge={short_edge}, canvas={canvas}). "
|
||||
f"This should be impossible — please report this shape.")
|
||||
samples = image[..., :3].movedim(-1, 1)
|
||||
samples = comfy.utils.common_upscale(samples, tw, th, "lanczos", crop)
|
||||
if samples.shape[2] <= 0 or samples.shape[3] <= 0:
|
||||
raise ValueError(
|
||||
f"_resize_ref: common_upscale produced an empty result "
|
||||
f"{tuple(samples.shape)} from a {h}x{w} source targeting "
|
||||
f"{tw}x{th} (crop={crop}, canvas={canvas}). This points to a bug "
|
||||
f"in comfy.utils.common_upscale for this input, not in RefMod's "
|
||||
f"own math.")
|
||||
return samples.movedim(1, -1)
|
||||
|
||||
|
||||
def snap_to_causal_grid(n_frames: int) -> int:
|
||||
"""Round down to H3's reference-video grid: 5, 22, 39, ... (17k+5).
|
||||
|
||||
The VAE encodes 17-frame chunks, pads the final chunk and drops three
|
||||
trailing latent tokens. Its effective video grid is not the encoder's
|
||||
standalone 4x temporal stride. RefMod keeps its single-image fallback
|
||||
when fewer than five frames are selected.
|
||||
"""
|
||||
if n_frames < 1:
|
||||
raise ValueError("A reference must contain at least one frame.")
|
||||
if n_frames < 5:
|
||||
return 1
|
||||
return ((n_frames - 5) // 17) * 17 + 5
|
||||
|
||||
|
||||
def sample_video_for_vae(frames: torch.Tensor, max_frames: Optional[int] = None) -> torch.Tensor:
|
||||
"""Sample the full clip onto H3's grid without exceeding the frame cap.
|
||||
|
||||
Keep both endpoints whenever the selected count is greater than one.
|
||||
Already-aligned clips are returned unchanged. A cap/input shorter than
|
||||
five frames selects only the first image, using H3's still-image path.
|
||||
"""
|
||||
if max_frames is not None and max_frames < 1:
|
||||
raise ValueError("max_frames must be at least 1.")
|
||||
count = snap_to_causal_grid(min(frames.shape[0], max_frames)
|
||||
if max_frames is not None else frames.shape[0])
|
||||
if count == frames.shape[0]:
|
||||
return frames
|
||||
indices = torch.linspace(0, frames.shape[0] - 1, count, device=frames.device).round().long()
|
||||
return frames[indices]
|
||||
|
||||
|
||||
|
||||
|
||||
def ensure_min_size(image, floor: int = 320):
|
||||
"""Upscale (never downscale) so both spatial dims are >= ``floor`` px.
|
||||
|
||||
The MiniMax H3 VAE encodes with internal tiled_encode (~256px tiles). A
|
||||
reference smaller than the tile size in one dimension can make the tiler
|
||||
compute a zero-size edge tile, which crashes deep inside conv_in with a
|
||||
cryptic 'Expected 4D or 5D... but got [1,3,1,0,W]' error. This applies
|
||||
regardless of extraction mode ('encode' already resizes down to
|
||||
ref_resolution but never guarantees a floor; 'training' now resizes to
|
||||
the same cap, also without a floor), so it's a separate, unconditional
|
||||
safety net right before encode.
|
||||
"""
|
||||
import comfy.utils
|
||||
h, w = image.shape[1], image.shape[2]
|
||||
if h >= floor and w >= floor:
|
||||
return image
|
||||
scale = floor / min(h, w)
|
||||
tw = max(floor, round(w * scale / 32) * 32)
|
||||
th = max(floor, round(h * scale / 32) * 32)
|
||||
samples = image[..., :3].movedim(-1, 1)
|
||||
samples = comfy.utils.common_upscale(samples, tw, th, "lanczos", "disabled")
|
||||
return samples.movedim(1, -1)
|
||||
Reference in New Issue
Block a user