diff --git a/H3_LONG_VIDEOS_GUIDE.md b/H3_LONG_VIDEOS_GUIDE.md index 11c6a59..10a78b6 100644 --- a/H3_LONG_VIDEOS_GUIDE.md +++ b/H3_LONG_VIDEOS_GUIDE.md @@ -1149,81 +1149,34 @@ These two belong together. If you change `shift_video`, you usually need to change `shift_audio` in proportion. -## Group 8: Detail Pass +## Group 8: Latent Upscale -This is the optional second pass. +This is the optional latent refinement stage, used before decode. -### `detail_pass` - -Enables the refinement pass. +The long-video node now expects a separate `Dumas H3 Latent Upscale Params` node for this stage. +Wire that node into the `latent_upscale_param` input when you want the shot to be upscaled and lightly +re-sampled before decode. What this really means: - the node renders the beat once -- then runs a second sampler pass over that result -- the goal is to polish, not to invent a whole different shot - -### `detail_sampler_name` - -Sampler for the refinement pass. - -### `detail_scheduler` - -Scheduler for the refinement pass. - -### `detail_steps` - -Extra steps for the refinement pass. - -What this really means: - -- more steps gives the second pass more opportunity to change the image -- that can help detail -- but after a point it stops being "cleanup" and starts becoming "rewrite" - -### `detail_denoise` - -How strongly the refinement pass is allowed to rewrite the beat. - -What this really means: - -- low denoise = polish what is already there -- high denoise = let the second pass substantially alter what is already there - -### How The Detail-Pass Settings Work Together - -The detail pass starts from the first-pass result and tries to polish it. - -Gentle settings: - -- low to medium `detail_steps` -- low `detail_denoise` - -Aggressive settings: - -- high `detail_steps` -- high `detail_denoise` - -Aggressive settings can improve texture, but they can also: - -- change faces -- pull away from references -- break continuity - -That is why this group should be read as one combined strength control: - -- `detail_pass` decides whether the second pass exists -- `detail_steps` decides how long it keeps working -- `detail_denoise` decides how free it is to change things -- `detail_sampler_name` and `detail_scheduler` shape how that rewrite behaves +- the sampled latent is upscaled in latent space to the target size +- the conditioning is rebuilt at that target size +- the node then runs a hard-coded 2-step refinement pass over the upscaled latent Good starting point: -- `detail_pass = on` -- `detail_sampler_name = euler` -- `detail_scheduler = beta` -- `detail_steps = 4` to `8` -- `detail_denoise = 0.20` to `0.35` +- use the `model` mode when you want the strongest latent detail recovery +- use the interpolation mode when you want a cheaper resize-only path +- keep the refinement denoise low to medium + +The important part is that this stage is still a latent pass, not a pixel-space resize: + +- it happens before decode +- it can change structure more than a normal image upscale +- it is the place to recover detail without adding another full detail-pass toggle + +If you do not wire the helper node, the long-video node skips latent upscale entirely and renders as before. ## Group 9: Performance, Decode, And Upscale diff --git a/README.md b/README.md index 61a9bd2..04dad5e 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,14 @@ - Prompt `` tags now map to the actual ref socket numbers you wire, even with gaps such as only `ref_2` and `ref_7` connected. - Character refs now contribute appearance and wardrobe context from the same structured object, while location refs contribute environment context from theirs. - The default ref2v bias is now stronger: `ref_mode` defaults to `auto ref2v` so untagged prompts condition every shot instead of only shot 1, and `ref_noise_aug` defaults to `0.95` rather than the upstream-literal `0.999`. + - `Dumas H3 Latent Upscale Params` provides the optional pre-decode latent refinement stage for the long-video node. - Per-shot directives now support `continuity:`, `ref_mode:`, `ref_noise_aug:`, `anchor_add:`, `soundscape:`, and `music:` in addition to the existing timing and wardrobe directives. +- `Dumas H3 Latent Upscale Params` + - Inputs: `mode`, `model_name`, `method`, `width`, `height`, `device`, `precision`, `refine_denoise` + - Output: `latent_upscale_param` + - Bundles the optional latent-space upscaler settings used by `Dumas H3 Long Videos` before decode, so the main node can rebuild conditioning at the target size and run a short refinement pass. + - `Dumas H3 Beat Prompt` - Inputs: authored through the custom front-end beat editor - Output: `prompt` diff --git a/__init__.py b/__init__.py index 5e20c38..2d34548 100644 --- a/__init__.py +++ b/__init__.py @@ -14,6 +14,10 @@ from .dumas_h3_longvideos import ( NODE_CLASS_MAPPINGS as H3_LONGVIDEO_NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS as H3_LONGVIDEO_NODE_DISPLAY_NAME_MAPPINGS, ) +from .dumas_h3_latent_upscale import ( + NODE_CLASS_MAPPINGS as H3_LATENT_UPSCALE_NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS as H3_LATENT_UPSCALE_NODE_DISPLAY_NAME_MAPPINGS, +) from .dumas_h3_shot_length import ( NODE_CLASS_MAPPINGS as H3_SHOT_LENGTH_NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS as H3_SHOT_LENGTH_NODE_DISPLAY_NAME_MAPPINGS, @@ -31,6 +35,7 @@ NODE_CLASS_MAPPINGS = {} NODE_CLASS_MAPPINGS.update(JSON_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(IMAGE_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(H3_LONGVIDEO_NODE_CLASS_MAPPINGS) +NODE_CLASS_MAPPINGS.update(H3_LATENT_UPSCALE_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(H3_SHOT_LENGTH_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(H3_INSPECTOR_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(H3_BEAT_PROMPT_NODE_CLASS_MAPPINGS) @@ -39,6 +44,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {} NODE_DISPLAY_NAME_MAPPINGS.update(JSON_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(IMAGE_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(H3_LONGVIDEO_NODE_DISPLAY_NAME_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(H3_LATENT_UPSCALE_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(H3_SHOT_LENGTH_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(H3_INSPECTOR_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(H3_BEAT_PROMPT_NODE_DISPLAY_NAME_MAPPINGS) diff --git a/dumas_h3_latent_upscale.py b/dumas_h3_latent_upscale.py new file mode 100644 index 0000000..e901b37 --- /dev/null +++ b/dumas_h3_latent_upscale.py @@ -0,0 +1,468 @@ +from functools import lru_cache +import glob +import math +import os +import re + +import folder_paths +import torch + +try: + import torch.nn as nn + import torch.nn.functional as F +except Exception: # pragma: no cover - import-time fallback for the test shim + nn = None + F = None + + +LATENTS_MEAN = [ + 0.858090341091156, -0.9606591463088989, 1.0661640167236328, -0.5090325474739075, + -0.2727581858634949, -1.3675414323806763, -0.2553254961967468, -0.26907554268836975, + -0.5376840829849243, -0.0464097298681736, 0.6657370328903198, 0.19690127670764923, + -0.5460608005523682, -0.4035342037677765, -0.23683024942874908, 0.25928452610969543, + -0.30133944749832153, 0.211341992020607, -1.1206848621368408, 0.3581933379173279, + -0.04225143790245056, 0.2604829967021942, 0.22864092886447906, 0.7056031823158264, +] +LATENTS_STD = [ + 1.2223774194717407, 1.2767263650894165, 1.6831774711608887, 1.7549455165863037, + 1.5636216402053833, 2.194143533706665, 0.9653137922286987, 1.0569885969161987, + 0.841948926448822, 0.7729952931404114, 1.8955937623977661, 0.946841835975647, + 0.7996809482574463, 0.44988900423049927, 0.7197399735450745, 0.6936293244361877, + 2.961095094680786, 2.7694199085235596, 3.0496184825897217, 2.1088054180265264, + 3.276226282119751, 3.1627357006073, 2.2816812992095947, 2.6127843856811523, +] + +_LATENT_UPSCALE_FOLDER = "latent_upscale_models" + + +def _models_dir(): + try: + if _LATENT_UPSCALE_FOLDER not in folder_paths.folder_names_and_paths: + folder_paths.add_model_folder_path( + _LATENT_UPSCALE_FOLDER, + os.path.join(folder_paths.models_dir, _LATENT_UPSCALE_FOLDER), + ) + return folder_paths.get_folder_paths(_LATENT_UPSCALE_FOLDER)[0] + except Exception: + return os.path.join(getattr(folder_paths, "models_dir", ""), _LATENT_UPSCALE_FOLDER) + + +def _scan_models(): + try: + model_dir = _models_dir() + files = [] + for ext in ("*.pth", "*.safetensors"): + files.extend(glob.glob(os.path.join(model_dir, ext))) + names = sorted(os.path.basename(path) for path in files) + return ["none"] + names if names else [f"(no upscale models found in: {model_dir})"] + except Exception: + return ["none"] + + +def _make_norm_tensors(device, dtype): + mean = torch.tensor(LATENTS_MEAN, dtype=dtype, device=device).view(1, -1, 1, 1, 1) + std = torch.tensor(LATENTS_STD, dtype=dtype, device=device).view(1, -1, 1, 1, 1) + return mean, std + + +if nn is not None: + + def _normalization(channels): + return nn.GroupNorm(32, channels) + + + def _zero_module(module): + for p in module.parameters(): + p.detach().zero_() + return module + + + class _AttnBlock3D(nn.Module): + def __init__(self, in_channels): + super().__init__() + self.norm = _normalization(in_channels) + self.q = nn.Conv3d(in_channels, in_channels, 1) + self.k = nn.Conv3d(in_channels, in_channels, 1) + self.v = nn.Conv3d(in_channels, in_channels, 1) + self.proj_out = nn.Conv3d(in_channels, in_channels, 1) + + def forward(self, x): + h = self.norm(x) + b, c, t, hh, w = h.shape + q = self.q(h).flatten(2).transpose(1, 2) + k = self.k(h).flatten(2).transpose(1, 2) + v = self.v(h).flatten(2).transpose(1, 2) + h = F.scaled_dot_product_attention(q, k, v) + h = h.transpose(1, 2).view(b, c, t, hh, w) + return x + self.proj_out(h) + + + class _ResBlockEmb3D(nn.Module): + def __init__(self, channels, emb_channels, dropout=0, out_channels=None): + super().__init__() + self.out_channels = out_channels or channels + self.in_layers = nn.Sequential( + _normalization(channels), nn.SiLU(), + nn.Conv3d(channels, self.out_channels, 3, padding=1), + ) + self.emb_layers = nn.Sequential( + nn.SiLU(), nn.Linear(emb_channels, 2 * self.out_channels), + ) + self.out_norm = _normalization(self.out_channels) + self.out_layers = nn.Sequential( + nn.SiLU(), nn.Dropout(p=dropout), + _zero_module(nn.Conv3d(self.out_channels, self.out_channels, 3, padding=1)), + ) + self.skip = ( + nn.Conv3d(channels, self.out_channels, 1) + if self.out_channels != channels else nn.Identity() + ) + + def forward(self, x, emb): + h = self.in_layers(x) + emb_out = self.emb_layers(emb).type(h.dtype) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + scale, shift = torch.chunk(emb_out, 2, dim=1) + h = self.out_norm(h) * (1 + scale) + shift + h = self.out_layers(h) + return self.skip(x) + h + + + class _TemporalConv(nn.Module): + def __init__(self, channels, kernel_size=5): + super().__init__() + padding = kernel_size // 2 + self.norm = _normalization(channels) + self.dwconv = nn.Conv3d( + channels, channels, kernel_size=(kernel_size, 1, 1), + padding=(padding, 0, 0), groups=channels, + ) + self.pwconv = nn.Conv3d(channels, channels, kernel_size=1) + nn.init.zeros_(self.pwconv.weight) + nn.init.zeros_(self.pwconv.bias) + + def forward(self, x): + identity = x + h = self.norm(x) + h = F.silu(h) + h = self.dwconv(h) + h = self.pwconv(h) + return identity + h + + + class _LatentResizer3D(nn.Module): + def __init__(self, in_channels=24, in_blocks=12, out_blocks=12, + channels=512, dropout=0.1, attn=False, + temporal_every=2, temporal_kernel=5): + super().__init__() + self.conv_in = nn.Conv3d(in_channels, channels, 3, padding=1) + embed_dim = 64 + self.embed = nn.Sequential( + nn.Linear(1, embed_dim), nn.SiLU(), nn.Linear(embed_dim, embed_dim)) + + self.in_blocks = nn.ModuleList() + for b in range(in_blocks): + if (b == 1 or b == in_blocks - 1) and attn: + self.in_blocks.append(_AttnBlock3D(channels)) + self.in_blocks.append(_ResBlockEmb3D(channels, embed_dim, dropout)) + if temporal_every > 0 and b % temporal_every == 0: + self.in_blocks.append(_TemporalConv(channels, temporal_kernel)) + + self.out_blocks = nn.ModuleList() + for b in range(out_blocks): + if (b == 1 or b == out_blocks - 1) and attn: + self.out_blocks.append(_AttnBlock3D(channels)) + self.out_blocks.append(_ResBlockEmb3D(channels, embed_dim, dropout)) + if temporal_every > 0 and b % temporal_every == 0: + self.out_blocks.append(_TemporalConv(channels, temporal_kernel)) + + self.norm_out = _normalization(channels) + self.conv_out = nn.Conv3d(channels, in_channels, 3, padding=1) + + def forward(self, x, scale=None, target_size=None): + if target_size is not None: + size = target_size + elif scale is not None: + size = tuple(int(round(s * scale)) for s in x.shape[-3:]) + else: + return x + + if size == x.shape[-3:]: + return x + + scale_emb = torch.tensor( + [scale - 1 if scale is not None else 0.0], + dtype=x.dtype, device=x.device).unsqueeze(0) + emb = self.embed(scale_emb) + + x = self.conv_in(x) + for block in self.in_blocks: + if isinstance(block, _ResBlockEmb3D): + x = block(x, emb.expand(x.shape[0], -1)) + else: + x = block(x) + + x = F.interpolate(x, size=size, mode="trilinear", align_corners=False) + + for block in self.out_blocks: + if isinstance(block, _ResBlockEmb3D): + x = block(x, emb.expand(x.shape[0], -1)) + else: + x = block(x) + + x = self.norm_out(x) + x = F.silu(x) + x = self.conv_out(x) + return x + +else: # pragma: no cover - import-time fallback for the test shim + _LatentResizer3D = None + + +_MODEL_CACHE = {} + + +def _load_raw_sd(path): + if path.endswith(".safetensors"): + from safetensors.torch import load_file + sd = load_file(path, device="cpu") + else: + sd = torch.load(path, map_location="cpu", weights_only=False) + if isinstance(sd, dict) and "model" in sd: + sd = sd["model"] + float8 = getattr(torch, "float8_e4m3fn", None) + if float8 is not None: + sd = {k: v.to(torch.float16) if getattr(v, "dtype", None) == float8 else v for k, v in sd.items()} + return sd + + +def _extract_upscaler_sd(sd): + if any(k.startswith("upscaler.") for k in sd): + return {k[len("upscaler."):]: v for k, v in sd.items() if k.startswith("upscaler.")} + return sd + + +def _detect_arch(sd): + cfg = { + "in_channels": 24, "in_blocks": 12, "out_blocks": 12, "channels": 512, + "dropout": 0.1, "attn": False, "temporal_every": 2, "temporal_kernel": 5, + } + conv_key = "conv_in.weight" + if conv_key in sd: + cfg["in_channels"] = sd[conv_key].shape[1] + cfg["channels"] = sd[conv_key].shape[0] + + in_ids, out_ids = set(), set() + temporal_in_indices, temporal_out_indices = set(), set() + for k in sd.keys(): + m = re.match(r"in_blocks\.(\d+)\.in_layers\.", k) + if m: + in_ids.add(int(m.group(1))) + m = re.match(r"out_blocks\.(\d+)\.in_layers\.", k) + if m: + out_ids.add(int(m.group(1))) + m = re.match(r"in_blocks\.(\d+)\.dwconv\.weight", k) + if m: + temporal_in_indices.add(int(m.group(1))) + m = re.match(r"out_blocks\.(\d+)\.dwconv\.weight", k) + if m: + temporal_out_indices.add(int(m.group(1))) + + if in_ids: + cfg["in_blocks"] = len(in_ids) + if out_ids: + cfg["out_blocks"] = len(out_ids) + + if temporal_in_indices or temporal_out_indices: + cfg["temporal_every"] = 2 + for k in sd.keys(): + if "dwconv.weight" in k and k.endswith("dwconv.weight"): + cfg["temporal_kernel"] = sd[k].shape[2] + break + else: + cfg["temporal_every"] = 0 + + cfg["attn"] = False + return cfg + + +def load_upscale_model(name, device, precision): + if _LatentResizer3D is None: + raise RuntimeError("latent upscaler requires torch.nn") + + cache_key = f"{name}::{device}::{precision}" + if cache_key in _MODEL_CACHE: + return _MODEL_CACHE[cache_key].to(device) + + path = os.path.join(_models_dir(), name) + if not os.path.exists(path): + raise FileNotFoundError(f"Model file not found: {path}") + + raw_sd = _load_raw_sd(path) + up_sd = _extract_upscaler_sd(raw_sd) + cfg = _detect_arch(up_sd) + if cfg["in_channels"] != 24: + raise ValueError( + f"Checkpoint '{name}' is not an H3 latent upscaler (expected 24 input channels, got {cfg['in_channels']})." + ) + + model = _LatentResizer3D( + in_channels=cfg["in_channels"], in_blocks=cfg["in_blocks"], out_blocks=cfg["out_blocks"], + channels=cfg["channels"], dropout=cfg["dropout"], attn=cfg["attn"], + temporal_every=cfg["temporal_every"], temporal_kernel=cfg["temporal_kernel"], + ) + model.load_state_dict(up_sd, strict=True) + dtype = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16}.get(precision, torch.float32) + model = model.to(device).eval().requires_grad_(False) + if dtype != torch.float32: + model = model.to(dtype) + + _MODEL_CACHE[cache_key] = model + return model + + +def unload_upscale_model(name, device, precision): + cache_key = f"{name}::{device}::{precision}" + model = _MODEL_CACHE.get(cache_key) + if model is not None and str(next(model.parameters()).device) != "cpu": + model.to("cpu") + if str(device) == "cuda" and hasattr(torch, "cuda"): + try: + torch.cuda.empty_cache() + except Exception: + pass + + +def _compute_upscale_target(width, height, h_in, w_in): + ds = 16 + w_px = float(width) + h_px = float(height) + eff = (w_px / (w_in * ds) + h_px / (h_in * ds)) / 2.0 + + w_px_f = round(w_px / ds) * ds + h_px_f = round(h_px / ds) * ds + w_out = max(1, int(w_px_f // ds)) + h_out = max(1, int(h_px_f // ds)) + return h_out, w_out, eff + + +def upscale_video_model(video, param): + model_name = param["model_name"] + width = int(param["width"]) + height = int(param["height"]) + device = param.get("device", "cuda") + precision = param.get("precision", "fp16") + + orig_dtype = video.dtype + dev = torch.device(device if (device == "cpu" or torch.cuda.is_available()) else "cpu") + compute_dtype = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16}[precision] + + _, c, t, h_in, w_in = video.shape + h_out, w_out, eff = _compute_upscale_target(width, height, h_in, w_in) + + if eff < 1.0 and (w_out < w_in or h_out < h_in): + raise ValueError("This model only supports upscaling (effective scale >= 1.0).") + if w_out == w_in and h_out == h_in: + return video, h_in, w_in + + if str(model_name).startswith("("): + raise ValueError("Please place H3 upscale model files into the latent_upscale_models directory") + + s = video.to(device=dev, dtype=compute_dtype, copy=True) + model = load_upscale_model(model_name, dev, precision) + norm_mean, norm_std = _make_norm_tensors(dev, compute_dtype) + + with torch.inference_mode(): + s = s.sub(norm_mean).div(norm_std) + out = model(s, scale=eff, target_size=(t, h_out, w_out)) + del s + out = out.mul(norm_std).add(norm_mean) + + out = out.to(device="cpu", dtype=orig_dtype) + unload_upscale_model(model_name, dev, precision) + return out, h_out, w_out + + +def upscale_video_interp(video, param): + method = str(param.get("method") or "bilinear") + width = int(param.get("width", 0) or 0) + height = int(param.get("height", 0) or 0) + _, c, t, h_in, w_in = video.shape + h_out, w_out, _ = _compute_upscale_target(width, height, h_in, w_in) + if h_out == h_in and w_out == w_in: + return video, h_in, w_in + video_bt = video.permute(0, 2, 1, 3, 4).reshape(-1, c, h_in, w_in) + up = F.interpolate(video_bt, size=(h_out, w_out), mode=method) + up = up.reshape(video.shape[0], t, c, h_out, w_out).permute(0, 2, 1, 3, 4).contiguous() + return up, h_out, w_out + + +def upscale_latent_video(video, param): + mode = str(param.get("mode") or "off") + if mode == "off": + return video, video.shape[-2], video.shape[-1] + if mode == "model": + return upscale_video_model(video, param) + return upscale_video_interp(video, param) + + +class H3LatentUpscaleParams: + CATEGORY = "Dumas/MiniMax" + FUNCTION = "build" + RETURN_TYPES = ("DUMAS_H3_LATENT_UPSCALE_PARAM",) + RETURN_NAMES = ("latent_upscale_param",) + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "mode": (["off", "model", "interp"], {"default": "off", + "tooltip": "Latent refinement mode. 'off' skips the stage, 'model' uses the H3 latent upscaler model, 'interp' uses model-free interpolation."}), + "model_name": (_scan_models(), { + "default": "none", + "tooltip": "H3 latent upscale checkpoint from models/latent_upscale_models, used when mode = model."}), + "method": (["nearest-exact", "bilinear", "area", "bicubic"], {"default": "bilinear", + "tooltip": "Interpolation method used when mode = interp."}), + "width": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 32, + "tooltip": "Target upscaled pixel width for the latent refinement stage. 0 keeps the original width and effectively disables the resize."}), + "height": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 32, + "tooltip": "Target upscaled pixel height for the latent refinement stage. 0 keeps the original height and effectively disables the resize."}), + "device": (["cuda", "cpu"], {"default": "cuda", + "tooltip": "Device used by the H3 latent upscaler model when mode = model."}), + "precision": (["fp16", "fp32", "bf16"], {"default": "fp16", + "tooltip": "Computation precision used by the H3 latent upscaler model when mode = model."}), + "refine_denoise": ("FLOAT", {"default": 0.25, "min": 0.0, "max": 1.0, "step": 0.01, + "tooltip": "How much the 2-step refinement pass may rewrite the upscaled latent. Lower = safer, higher = freer."}), + } + } + + def build(self, mode, model_name, method, width, height, device, precision, refine_denoise): + width = int(width) + height = int(height) + if mode == "off": + return ({"mode": "off", "width": width, "height": height, "refine_denoise": float(refine_denoise)},) + if width > 0: + width = int(round(width / 32.0)) * 32 + if height > 0: + height = int(round(height / 32.0)) * 32 + return ({ + "mode": mode, + "model_name": model_name, + "method": method, + "width": width, + "height": height, + "device": device, + "precision": precision, + "refine_denoise": float(refine_denoise), + },) + + +NODE_CLASS_MAPPINGS = {"DumasH3LatentUpscaleParams": H3LatentUpscaleParams} +NODE_DISPLAY_NAME_MAPPINGS = {"DumasH3LatentUpscaleParams": "Dumas H3 Latent Upscale Params"} + +__all__ = [ + "NODE_CLASS_MAPPINGS", + "NODE_DISPLAY_NAME_MAPPINGS", + "upscale_latent_video", +] diff --git a/dumas_h3_longvideos.py b/dumas_h3_longvideos.py index ce7960b..b13330b 100644 --- a/dumas_h3_longvideos.py +++ b/dumas_h3_longvideos.py @@ -30,11 +30,12 @@ video; each later paragraph = a scene beat), a shot length, and a resolution fro the VRAM-appropriate list. It splits the beats into shots that fit H3's ceiling and your VRAM, chains them, and returns the finished video + audio. -Requirements: H3 is CFG-free (cfg 1) and needs no negative prompt -- the node -makes an empty one internally. The main pass keeps denoise fixed at 1.0: a -partial denoise desyncs the joint audio/video schedule. An optional refinement -pass can use its own denoise later, before any upscale, while keeping the output -video-only. +Requirements: H3 is CFG-free (cfg 1) and needs no negative prompt -- the node +makes an empty one internally. The main pass keeps denoise fixed at 1.0: a +partial denoise desyncs the joint audio/video schedule. An optional latent +upscale stage can rebuild the conditioning at a larger target size, run a short +refinement pass, and keep the output video-only before the final pixel-space +upscale options. Verified against ComfyUI core (comfy_extras/nodes_minimax_h3.py, model_base.py, ldm/minimax/model.py, text_encoders/minimax.py, sd.py). @@ -57,28 +58,36 @@ import comfy.nested_tensor import comfy.model_management as mm import node_helpers -try: - from . import dumas_h3_overlay as _overlay - from . import dumas_image_nodes as _image_nodes -except ImportError: # loaded as a bare file (test_prompt_logic.py), not as a package - import importlib.util as _ilu - import os as _os - import sys as _sys - _spec = _ilu.spec_from_file_location( - "dumas_h3_overlay", - _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "dumas_h3_overlay.py"), - ) - _overlay = _ilu.module_from_spec(_spec) - _spec.loader.exec_module(_overlay) - _image_nodes = _sys.modules.get("dumas_image_nodes") - if _image_nodes is None: - _img_spec = _ilu.spec_from_file_location( - "dumas_image_nodes", - _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "dumas_image_nodes.py"), - ) - _image_nodes = _ilu.module_from_spec(_img_spec) - _sys.modules["dumas_image_nodes"] = _image_nodes - _img_spec.loader.exec_module(_image_nodes) +try: + from . import dumas_h3_overlay as _overlay + from .dumas_h3_latent_upscale import upscale_latent_video as _upscale_latent_video + from . import dumas_image_nodes as _image_nodes +except ImportError: # loaded as a bare file (test_prompt_logic.py), not as a package + import importlib.util as _ilu + import os as _os + import sys as _sys + _spec = _ilu.spec_from_file_location( + "dumas_h3_overlay", + _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "dumas_h3_overlay.py"), + ) + _overlay = _ilu.module_from_spec(_spec) + _spec.loader.exec_module(_overlay) + _latent_spec = _ilu.spec_from_file_location( + "dumas_h3_latent_upscale", + _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "dumas_h3_latent_upscale.py"), + ) + _latent_upscale = _ilu.module_from_spec(_latent_spec) + _latent_spec.loader.exec_module(_latent_upscale) + _upscale_latent_video = _latent_upscale.upscale_latent_video + _image_nodes = _sys.modules.get("dumas_image_nodes") + if _image_nodes is None: + _img_spec = _ilu.spec_from_file_location( + "dumas_image_nodes", + _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "dumas_image_nodes.py"), + ) + _image_nodes = _ilu.module_from_spec(_img_spec) + _sys.modules["dumas_image_nodes"] = _image_nodes + _img_spec.loader.exec_module(_image_nodes) AUDIO_LATENT_FPS = 40 GB = 1024 ** 3 @@ -272,16 +281,14 @@ def split_paragraphs(text, delimiter): # that carry no widget value can stay grouped in INPUT_TYPES without being listed. ADDED_WIDGETS = ( "beat_split", - "watermark_text", "watermark_position", "watermark_size", "watermark_opacity", - "watermark_margin", "intro_text", "intro_position", "intro_seconds", - "intro_fade", "intro_size", "overlay_font", "overlay_stroke", - "ref_mode", "ref_image_size", "ref_noise_aug", "auto_props", "prevent_nudity", - "exposed_terms", "anatomy_guard", "lock_restraints", "solidity_guard", - "motion_guard", "contact_guard", - "auto_soundscape", "allow_nonspeech_vocals", - "detail_pass", "detail_sampler_name", "detail_scheduler", - "detail_steps", "detail_denoise", -) + "watermark_text", "watermark_position", "watermark_size", "watermark_opacity", + "watermark_margin", "intro_text", "intro_position", "intro_seconds", + "intro_fade", "intro_size", "overlay_font", "overlay_stroke", + "ref_mode", "ref_image_size", "ref_noise_aug", "auto_props", "prevent_nudity", + "exposed_terms", "anatomy_guard", "lock_restraints", "solidity_guard", + "motion_guard", "contact_guard", + "auto_soundscape", "allow_nonspeech_vocals", +) NL = "\n" # Lines that CONFIGURE a beat rather than being one. They attach to the beat that @@ -4048,10 +4055,10 @@ def _latent_with_replaced_samples(template_latent, sampled_latent): return sampled_latent -def _video_only_refined_latent(base_latent, refined_latent): - """Keep the refined video latent, but preserve the original audio latent.""" - base = base_latent.get("samples") if isinstance(base_latent, dict) else None - refined = refined_latent.get("samples") if isinstance(refined_latent, dict) else None +def _video_only_refined_latent(base_latent, refined_latent): + """Keep the refined video latent, but preserve the original audio latent.""" + base = base_latent.get("samples") if isinstance(base_latent, dict) else None + refined = refined_latent.get("samples") if isinstance(refined_latent, dict) else None if base is None or refined is None: return refined_latent if not getattr(base, "is_nested", False) or not getattr(refined, "is_nested", False): @@ -4062,8 +4069,22 @@ def _video_only_refined_latent(base_latent, refined_latent): if len(base_parts) >= 2 and len(refined_parts) >= 1: return {"samples": comfy.nested_tensor.NestedTensor((refined_parts[0], base_parts[-1]))} except Exception: - return refined_latent - return refined_latent + return refined_latent + return refined_latent + + +def _nested_tensor_parts(samples): + if samples is None: + return () + parts = getattr(samples, "tensors", None) + if parts is not None: + return tuple(parts) + if hasattr(samples, "unbind"): + try: + return tuple(samples.unbind()) + except Exception: + return () + return () def _coerce_bool_flag(value): @@ -4093,25 +4114,25 @@ def _format_elapsed_seconds(seconds): def _format_timing_note(shot_timings): if not shot_timings: return "" - totals = { - "total": 0.0, - "retry_elapsed": 0.0, - "sample": 0.0, - "detail_sample": 0.0, - "decode_video": 0.0, - "decode_audio": 0.0, - "cleanup": 0.0, + totals = { + "total": 0.0, + "retry_elapsed": 0.0, + "sample": 0.0, + "latent_upscale_sample": 0.0, + "decode_video": 0.0, + "decode_audio": 0.0, + "cleanup": 0.0, "retries": 0, } slowest = None for shot in shot_timings: totals["total"] += float(shot.get("total", 0.0) or 0.0) - totals["retry_elapsed"] += float(shot.get("retry_elapsed", 0.0) or 0.0) - totals["sample"] += float(shot.get("sample", 0.0) or 0.0) - totals["detail_sample"] += float(shot.get("detail_sample", 0.0) or 0.0) - totals["decode_video"] += float(shot.get("decode_video", 0.0) or 0.0) - totals["decode_audio"] += float(shot.get("decode_audio", 0.0) or 0.0) - totals["cleanup"] += float(shot.get("cleanup", 0.0) or 0.0) + totals["retry_elapsed"] += float(shot.get("retry_elapsed", 0.0) or 0.0) + totals["sample"] += float(shot.get("sample", 0.0) or 0.0) + totals["latent_upscale_sample"] += float(shot.get("latent_upscale_sample", 0.0) or 0.0) + totals["decode_video"] += float(shot.get("decode_video", 0.0) or 0.0) + totals["decode_audio"] += float(shot.get("decode_audio", 0.0) or 0.0) + totals["cleanup"] += float(shot.get("cleanup", 0.0) or 0.0) totals["retries"] += max(0, int(shot.get("attempts", 1) or 1) - 1) if slowest is None or float(shot.get("total", 0.0) or 0.0) > float(slowest.get("total", 0.0) or 0.0): slowest = shot @@ -4124,8 +4145,8 @@ def _format_timing_note(shot_timings): ] if totals["retry_elapsed"]: pieces.append(f"retry elapsed {_format_elapsed_seconds(totals['retry_elapsed'])}") - if totals["detail_sample"]: - pieces.append(f"detail {_format_elapsed_seconds(totals['detail_sample'])}") + if totals["latent_upscale_sample"]: + pieces.append(f"latent upscale {_format_elapsed_seconds(totals['latent_upscale_sample'])}") if totals["retries"]: pieces.append(f"retries {totals['retries']}") if slowest is not None: @@ -6075,18 +6096,22 @@ class H3LongVideos: "doesn't cut to digital silence with a click. The silenced shots keep NO " "original audio at all -- fading the muted shot itself would leave this many " "ms of the gibberish audible at each end of every muted shot."}), - "decode_tile_frames": ("INT", {"default": 0, "min": 0, "max": 128, "step": 1, - "tooltip": "Temporal tiling for the VAE decode (tile_t). 0 = ComfyUI default, which " - "expands the WHOLE clip at once -- the single largest allocation in a run, " - "and the usual point where a big checkpoint tips into shared memory. Try 8-16 " - "if you spill during decode rather than sampling. Lower = less peak VRAM, " - "slightly slower."}), - "decode_tile_size": ("INT", {"default": 0, "min": 0, "max": 1024, "step": 32, - "tooltip": "Spatial tile size for the VAE decode (tile_x/tile_y). 0 = ComfyUI default. " - "Try 256 on a tight card at 1344x768."}), + "decode_tile_frames": ("INT", {"default": 0, "min": 0, "max": 128, "step": 1, + "tooltip": "Temporal tiling for the VAE decode (tile_t). 0 = ComfyUI default, which " + "expands the WHOLE clip at once -- the single largest allocation in a run, " + "and the usual point where a big checkpoint tips into shared memory. Try 8-16 " + "if you spill during decode rather than sampling. Lower = less peak VRAM, " + "slightly slower."}), + "decode_tile_size": ("INT", {"default": 0, "min": 0, "max": 1024, "step": 32, + "tooltip": "Spatial tile size for the VAE decode (tile_x/tile_y). 0 = ComfyUI default. " + "Try 256 on a tight card at 1344x768."}), + "latent_upscale_param": ("DUMAS_H3_LATENT_UPSCALE_PARAM", { + "tooltip": "Output of 'Dumas H3 Latent Upscale Params'. When connected, the first-pass " + "latent is upscaled and run through a hard-coded 2-step refinement pass before " + "decode. Leave unconnected to skip latent upscaling entirely."}), "upscale": (["off", "rtx", "model", "lanczos"], {"default": "off", - "tooltip": "Optional post-pass on the finished frames. 'rtx' = NVIDIA RTX Video Super " - "Resolution (Tensor Cores -- fastest and best for video; needs the " + "tooltip": "Optional post-pass on the finished frames. 'rtx' = NVIDIA RTX Video Super " + "Resolution (Tensor Cores -- fastest and best for video; needs the " "Nvidia_RTX_Nodes_ComfyUI pack, falls back automatically if absent). 'model' = " "a Real-ESRGAN/UltraSharp upscale model from upscale_model. 'lanczos' = plain " "resize. All of these ENHANCE/ENLARGE; for true detail reconstruction from a " @@ -6363,27 +6388,7 @@ class H3LongVideos: "the jacket, 'wardrobe: += sunglasses' adds one. TWO+ PEOPLE: name them -- " "'Maya = grey shorts, red jacket; Jon = navy overalls', then edit one at a " "time: 'wardrobe: Maya -= jacket' leaves Jon untouched."}), - "detail_pass": ("BOOLEAN", {"default": False, - "tooltip": "Run a second refinement sampler on each beat BEFORE any upscale. " - "It reuses the same conditioning and keeps the output video-only by " - "preserving the first pass's audio latent. Use it for detail cleanup, not " - "for huge rewrites: too many steps or too much denoise can pull identity or " - "continuity away from the main pass."}), - "detail_sampler_name": (comfy.samplers.KSampler.SAMPLERS, {"default": "euler", - "tooltip": "Sampler for the optional refinement pass. Euler is the maintained default " - "direction for this lane."}), - "detail_scheduler": (comfy.samplers.KSampler.SCHEDULERS, {"default": "beta", - "tooltip": "Scheduler for the optional refinement pass. Beta is the maintained default " - "direction for the H3 enhancement lane."}), - "detail_steps": ("INT", {"default": 8, "min": 1, "max": 200, - "tooltip": "Extra steps for the refinement pass only. Start around 4-8. More is not " - "automatically better; once the pass starts rewriting instead of polishing, " - "identity and continuity can drift."}), - "detail_denoise": ("FLOAT", {"default": 0.4, "min": 0.0, "max": 1.0, "step": 0.01, - "tooltip": "How hard the refinement pass is allowed to rewrite the beat latent. Start " - "around 0.20-0.35 for gentle cleanup; 0.4+ is stronger and can noticeably " - "change faces, motion or composition."}), - }, + }, # Read-only graph access, for SLA-LoRA detection: a LoRA's filename is # the only thing that identifies an SLA build, and the graph is the only # place it survives. Named 'graph'/'node_id' rather than the usual @@ -6404,15 +6409,14 @@ class H3LongVideos: opt[name] = opt.pop(name) # re-insert at the end, value unchanged return schema - def _render(self, model, clip, vae, audio_vae, negative, prompt, w, h, ln, fps, tiled, sa, - handoff, decode_tile_frames=0, decode_tile_size=0, - refs=None, ref_image_size="match", ref_noise_aug=None, silent=False, - detail_pass=False, detail_sampler_name="euler", detail_scheduler="beta", - detail_steps=8, detail_denoise=0.4, timing_sink=None): - timing = {"sample": 0.0, "detail_sample": 0.0, "decode_video": 0.0, "decode_audio": 0.0, "cleanup": 0.0} - positive, latent = _build_shot_conditioning(clip, vae, prompt, w, h, ln, fps, handoff, - ref_images=refs, ref_image_size=ref_image_size, - ref_noise_aug=ref_noise_aug, + def _render(self, model, clip, vae, audio_vae, negative, prompt, w, h, ln, fps, tiled, sa, + handoff, decode_tile_frames=0, decode_tile_size=0, + refs=None, ref_image_size="match", ref_noise_aug=None, silent=False, + latent_upscale_param=None, timing_sink=None): + timing = {"sample": 0.0, "latent_upscale_sample": 0.0, "decode_video": 0.0, "decode_audio": 0.0, "cleanup": 0.0} + positive, latent = _build_shot_conditioning(clip, vae, prompt, w, h, ln, fps, handoff, + ref_images=refs, ref_image_size=ref_image_size, + ref_noise_aug=ref_noise_aug, audio_vae=audio_vae, silent=silent) seed, steps, cfg, sn, sch, denoise = sa # Conditioning is built, so the text encoder and VAEs are dead weight for the @@ -6423,30 +6427,51 @@ class H3LongVideos: (out,) = nodes.common_ksampler(model, seed, steps, cfg, sn, sch, positive, negative, latent, denoise=denoise) timing["sample"] += time.perf_counter() - sample_start - except Exception as e: - # Mark WHERE this failed. `tiled` only affects the DECODE, so the caller's - # OOM retry cannot help an OOM raised here -- it just re-runs the whole - # sampling pass and fails the same way, which on a 362-frame shot is four - # more minutes for nothing. - if _is_oom(e): - e._h3_stage = "sampling" - raise - refined_out = out - detail_pass = _coerce_bool_flag(detail_pass) - if detail_pass: - detail_latent = _latent_with_replaced_samples(latent, out) + except Exception as e: + # Mark WHERE this failed. `tiled` only affects the DECODE, so the caller's + # OOM retry cannot help an OOM raised here -- it just re-runs the whole + # sampling pass and fails the same way, which on a 362-frame shot is four + # more minutes for nothing. + if _is_oom(e): + e._h3_stage = "sampling" + raise + refined_out = out + latent_upscale_param = latent_upscale_param or None + if ( + isinstance(latent_upscale_param, dict) + and str(latent_upscale_param.get("mode", "off")) != "off" + and int(latent_upscale_param.get("width", 0) or 0) > 0 + and int(latent_upscale_param.get("height", 0) or 0) > 0 + ): try: - detail_start = time.perf_counter() + latent_start = time.perf_counter() + out_samples = out["samples"] + parts = _nested_tensor_parts(out_samples) + if not getattr(out_samples, "is_nested", False) or len(parts) < 2: + raise RuntimeError("latent upscale expects a nested AV latent") + upscaled_video, up_h, up_w = _upscale_latent_video(parts[0], latent_upscale_param) + target_w = int(up_w) * 16 + target_h = int(up_h) * 16 + if target_w <= 0 or target_h <= 0: + raise RuntimeError("latent upscale target size must be positive") + if latent_upscale_param.get("mode") == "model" and str(latent_upscale_param.get("device", "cuda")) == "cuda" and hasattr(model, "clone_base_uuid"): + mm.unload_model_and_clones(model, unload_additional_models=False) + mm.soft_empty_cache() + upscale_cond, upscale_latent = _build_shot_conditioning( + clip, vae, prompt, target_w, target_h, ln, fps, handoff, + ref_images=refs, ref_image_size=ref_image_size, + ref_noise_aug=ref_noise_aug, audio_vae=audio_vae, silent=silent) + upscale_latent["samples"] = comfy.nested_tensor.NestedTensor( + (upscaled_video, parts[1])) (refined_out,) = nodes.common_ksampler( - model, seed, int(detail_steps), cfg, detail_sampler_name, detail_scheduler, - positive, negative, detail_latent, denoise=float(detail_denoise)) - timing["detail_sample"] += time.perf_counter() - detail_start + model, seed, 2, cfg, sn, sch, upscale_cond, negative, upscale_latent, + denoise=float(latent_upscale_param.get("refine_denoise", 0.25) or 0.25)) + timing["latent_upscale_sample"] += time.perf_counter() - latent_start + refined_out = _video_only_refined_latent(out, refined_out) except Exception as e: if _is_oom(e): e._h3_stage = "sampling" raise - refined_out = _video_only_refined_latent(out, refined_out) - del detail_latent # Keep a CPU copy of the sampled latent BEFORE decoding, for the `latent` # output. Latents are ~1000x smaller than the frames they decode to (a # 1344x768 124f shot is ~1.5MB against ~1.5GB), so carrying one per shot for @@ -6458,8 +6483,8 @@ class H3LongVideos: audio = _decode_audio(audio_vae, out) timing["decode_audio"] += time.perf_counter() - decode_audio_start # Audio is much smaller than the video decode. Drop the first-pass - # conditioning before the VAE work so the optional detail pass does not - # keep both sampled latents resident across the heaviest allocation. + # conditioning before the VAE work so the optional latent upscale pass does + # not keep both sampled latents resident across the heaviest allocation. del out, positive, latent video = _decode_video(vae, refined_out, tiled, free_first=model, tile_t=decode_tile_frames, tile_xy=decode_tile_size) @@ -6489,19 +6514,18 @@ class H3LongVideos: auto_soundscape="fill if blank", auto_silence_nonspeech=True, allow_nonspeech_vocals=False, subject_count_guard="auto", - upscale="off", upscale_model="none", - upscale_target_short_edge=0, upscale_batch=4, - mute_nonspeech_audio=True, mute_fade_ms=40, - watermark_text="", watermark_position="bottom-right", watermark_size=4.0, - watermark_opacity=0.75, watermark_margin=3.0, + upscale="off", upscale_model="none", + upscale_target_short_edge=0, upscale_batch=4, + mute_nonspeech_audio=True, mute_fade_ms=40, + watermark_text="", watermark_position="bottom-right", watermark_size=4.0, + watermark_opacity=0.75, watermark_margin=3.0, intro_text="", intro_position="center", intro_seconds=3.0, intro_fade=0.6, intro_size=9.0, overlay_font="arial.ttf", overlay_stroke=0, ref_1=None, ref_2=None, ref_3=None, ref_4=None, ref_5=None, ref_6=None, ref_7=None, ref_8=None, ref_9=None, ref_mode="auto ref2v", ref_image_size="match", ref_noise_aug=0.95, - detail_pass=False, detail_sampler_name="euler", detail_scheduler="beta", - detail_steps=8, detail_denoise=0.4, + latent_upscale_param=None, graph=None, node_id=None): # FIRST: detect a checkpoint swap since the previous execution and hard-flush. @@ -6562,17 +6586,26 @@ class H3LongVideos: # Patch the dual video/audio schedule onto the model here, so a missing # upstream ModelSamplingMiniMaxH3 can't silently produce gibberish audio. # Shifts come from the widgets (12/3 base default; MXFP8/turbo differ). - ms_note = "" - if apply_model_sampling: - model, ms_note = apply_h3_model_sampling(model, shift_video, shift_audio) - detail_note = "" - if detail_pass: - detail_note = (f" detail pass: {int(detail_steps)} step(s) via " - f"{detail_sampler_name}/{detail_scheduler} at denoise " - f"{float(detail_denoise):.2f}; video-only refinement keeps " - f"audio from the first pass") - - paras = split_paragraphs(prompt, "##") + ms_note = "" + if apply_model_sampling: + model, ms_note = apply_h3_model_sampling(model, shift_video, shift_audio) + latent_upscale_note = "" + if isinstance(latent_upscale_param, dict) and str(latent_upscale_param.get("mode", "off")) != "off": + target_w = int(latent_upscale_param.get("width", 0) or 0) + target_h = int(latent_upscale_param.get("height", 0) or 0) + if target_w > 0 and target_h > 0: + mode = str(latent_upscale_param.get("mode", "off")) + detail = f" via {mode}" + if mode == "model": + detail += f"/{latent_upscale_param.get('model_name', 'none')}" + else: + detail += f"/{latent_upscale_param.get('method', 'bilinear')}" + latent_upscale_note = ( + f" latent upscale: target {target_w}x{target_h}px{detail}; " + f"2-step refinement denoise {float(latent_upscale_param.get('refine_denoise', 0.25) or 0.25):.2f}" + ) + + paras = split_paragraphs(prompt, "##") if anchor_override.strip(): anchor, beat_paras = anchor_override.strip(), paras elif paras: @@ -6878,12 +6911,13 @@ class H3LongVideos: (f"PLAN (no render): {shape} = ~{total:g}s at {w}x{h}. " f"{len(beats) or 1} beat(s). decode {'tiled' if tiled else 'full'}. {vram_str}." + (f" {beats_note}." if beats_note else "") - + (" ANCHOR: " + "; ".join(anchor_hazards) + "." - if anchor_hazards else "") - + (f"{anatomy_note}." if anatomy_note else "") - + (f"{plan_audio}." if plan_audio else "") - + (" EXPOSURE -- " + "; ".join(wardrobe_notes) + "." - if wardrobe_notes else "") + + (" ANCHOR: " + "; ".join(anchor_hazards) + "." + if anchor_hazards else "") + + (f"{anatomy_note}." if anatomy_note else "") + + (f"{latent_upscale_note}." if latent_upscale_note else "") + + (f"{plan_audio}." if plan_audio else "") + + (" EXPOSURE -- " + "; ".join(wardrobe_notes) + "." + if wardrobe_notes else "") + (" OVERRIDES -- " + "; ".join(override_notes) + "." if override_notes else "") + (f"{plan_ref}." if plan_ref else "") @@ -7025,12 +7059,11 @@ class H3LongVideos: shot_attempts += 1 attempt_start = time.perf_counter() try: - frames, audio, shot_latent = self._render( - model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, - tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, - shot_refs, ref_image_size, shot_aug, shot_silent, - detail_pass, detail_sampler_name, detail_scheduler, - detail_steps, detail_denoise, timing_sink=shot_timing) + frames, audio, shot_latent = self._render( + model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, + tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, + shot_refs, ref_image_size, shot_aug, shot_silent, + latent_upscale_param=latent_upscale_param, timing_sink=shot_timing) break except (torch.cuda.OutOfMemoryError, RuntimeError) as e: shot_retry_elapsed += time.perf_counter() - attempt_start @@ -7048,12 +7081,11 @@ class H3LongVideos: try: shot_attempts += 1 attempt_start = time.perf_counter() - frames, audio, shot_latent = self._render( - model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, - tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, - shot_refs, ref_image_size, shot_aug, shot_silent, - detail_pass, detail_sampler_name, detail_scheduler, - detail_steps, detail_denoise, timing_sink=shot_timing) + frames, audio, shot_latent = self._render( + model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, + tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, + shot_refs, ref_image_size, shot_aug, shot_silent, + latent_upscale_param=latent_upscale_param, timing_sink=shot_timing) except (torch.cuda.OutOfMemoryError, RuntimeError) as e: shot_retry_elapsed += time.perf_counter() - attempt_start if _is_oom(e) and getattr(e, "_h3_stage", "") == "sampling": @@ -7068,12 +7100,11 @@ class H3LongVideos: mm.soft_empty_cache(True); tiled = True; backoff.append(f"shot {i+1}: tiled") shot_attempts += 1 attempt_start = time.perf_counter() - frames, audio, shot_latent = self._render( - model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, - tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, - shot_refs, ref_image_size, shot_aug, shot_silent, - detail_pass, detail_sampler_name, detail_scheduler, - detail_steps, detail_denoise, timing_sink=shot_timing) + frames, audio, shot_latent = self._render( + model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, + tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, + shot_refs, ref_image_size, shot_aug, shot_silent, + latent_upscale_param=latent_upscale_param, timing_sink=shot_timing) shot_retry_elapsed += time.perf_counter() - attempt_start shot_total = time.perf_counter() - shot_total_start @@ -7343,7 +7374,7 @@ class H3LongVideos: f"shot before them ended on dialogue." if mouth_settled else "") + (f"{anatomy_note}." if anatomy_note else "") + (f"{latent_note}." if latent_note else "") - + (f"{detail_note}." if detail_note else "") + + (f"{latent_upscale_note}." if latent_upscale_note else "") + (f" SLA LoRA '{os.path.basename(str(sla_name))}' paired with sparse attention." if sla_name and sparse_on else "") + (f" {beats_note}." if beats_note else "") diff --git a/js/h3_longvideos/index.js b/js/h3_longvideos/index.js index 4c30e34..aea53aa 100644 --- a/js/h3_longvideos/index.js +++ b/js/h3_longvideos/index.js @@ -53,11 +53,10 @@ const GROUPS = [ }, { id: "finish", - label: "Upscale/Detail", + label: "Upscale", defaultCollapsed: true, widgets: [ "upscale", "upscale_model", "upscale_target_short_edge", "upscale_batch", - "detail_pass", "detail_sampler_name", "detail_scheduler", "detail_steps", "detail_denoise", ], }, { diff --git a/tests/test_dumas_h3_longvideos.py b/tests/test_dumas_h3_longvideos.py index 8337616..1aea54b 100644 --- a/tests/test_dumas_h3_longvideos.py +++ b/tests/test_dumas_h3_longvideos.py @@ -24,6 +24,7 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): "PIL.Image", "folder_paths", "dumas_image_nodes", + "dumas_h3_latent_upscale", "dumas_h3_longvideos", ) } @@ -207,7 +208,7 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): "retry_elapsed": 1.2, "attempts": 2, "sample": 8.0, - "detail_sample": 0.5, + "latent_upscale_sample": 0.5, "decode_video": 2.1, "decode_audio": 0.4, "cleanup": 0.2, @@ -230,11 +231,11 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): self.assertIn("decode audio 0.7s", note) self.assertIn("cleanup 0.3s", note) self.assertIn("retry elapsed 1.2s", note) - self.assertIn("detail 0.5s", note) + self.assertIn("latent upscale 0.5s", note) self.assertIn("retries 1", note) self.assertIn("slowest shot 1 12.4s", note) - def test_detail_pass_refines_video_but_preserves_audio(self): + def test_latent_upscale_refines_video_but_preserves_audio(self): class FakeTensor: def __init__(self, name): self.name = name @@ -263,6 +264,8 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): original_decode_video = self.module._decode_video original_decode_audio = self.module._decode_audio original_cleanup = self.module._deep_cleanup + original_upscale = self.module._upscale_latent_video + original_copy_sample = self.module._copy_sample_latent original_nested = getattr(self.module.comfy.nested_tensor, "NestedTensor", None) try: self.module.comfy.nested_tensor.NestedTensor = FakeNestedTensor @@ -277,6 +280,8 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): {"samples": FakeNestedTensor((FakeTensor("basev"), FakeTensor("basea")))}, ) self.module._evict_all_but = lambda *_args, **_kwargs: None + self.module._upscale_latent_video = lambda video, param: (FakeTensor("upv"), 8, 16) + self.module._copy_sample_latent = lambda sampled: sampled["samples"].unbind() self.module._decode_video = lambda _vae, out_latent, *_args, **_kwargs: out_latent self.module._decode_audio = lambda _vae, out_latent: out_latent self.module._deep_cleanup = lambda: None @@ -298,18 +303,22 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): tiled=False, sa=(123, 20, 1.0, "res_multistep", "simple", 1.0), handoff=None, - detail_pass=True, - detail_sampler_name="euler", - detail_scheduler="beta", - detail_steps=5, - detail_denoise=0.4, + latent_upscale_param={ + "mode": "model", + "model_name": "upscale.safetensors", + "width": 256, + "height": 128, + "device": "cpu", + "precision": "fp16", + "refine_denoise": 0.4, + }, ) self.assertEqual(len(calls), 2) self.assertIsNot(calls[1][0][8], first_out) - self.assertIs(calls[1][0][8]["samples"], first_out["samples"]) - self.assertEqual(calls[1][0][4], "euler") - self.assertEqual(calls[1][0][5], "beta") + self.assertEqual(calls[1][0][8]["samples"].unbind()[0].name, "upv") + self.assertEqual(calls[1][0][4], "res_multistep") + self.assertEqual(calls[1][0][5], "simple") self.assertAlmostEqual(calls[1][1]["denoise"], 0.4) self.assertEqual(result[1], first_out) self.assertEqual(result[2][0].name, "v2") @@ -323,12 +332,14 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): self.module._decode_video = original_decode_video self.module._decode_audio = original_decode_audio self.module._deep_cleanup = original_cleanup + self.module._upscale_latent_video = original_upscale + self.module._copy_sample_latent = original_copy_sample if original_nested is None: delattr(self.module.comfy.nested_tensor, "NestedTensor") else: self.module.comfy.nested_tensor.NestedTensor = original_nested - def test_detail_pass_decodes_audio_before_video_and_cleans_up(self): + def test_latent_upscale_decodes_audio_before_video_and_cleans_up(self): class FakeTensor: def __init__(self, name): self.name = name @@ -357,12 +368,14 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): original_decode_video = self.module._decode_video original_decode_audio = self.module._decode_audio original_cleanup = self.module._deep_cleanup + original_upscale = self.module._upscale_latent_video + original_copy_sample = self.module._copy_sample_latent original_nested = getattr(self.module.comfy.nested_tensor, "NestedTensor", None) try: self.module.comfy.nested_tensor.NestedTensor = FakeNestedTensor def common_ksampler(*args, **kwargs): - order.append("detail_sample" if len(order) else "sample") + order.append("latent_upscale_sample" if len(order) else "sample") return (first_out if len([x for x in order if x.endswith("sample")]) == 1 else second_out,) def decode_audio(_vae, out_latent): @@ -385,6 +398,8 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): {"samples": FakeNestedTensor((FakeTensor("basev"), FakeTensor("basea")))}, ) self.module._evict_all_but = lambda *_args, **_kwargs: None + self.module._upscale_latent_video = lambda video, param: (FakeTensor("upv"), 8, 16) + self.module._copy_sample_latent = lambda sampled: sampled["samples"].unbind() self.module._decode_video = decode_video self.module._decode_audio = decode_audio self.module._deep_cleanup = cleanup @@ -406,15 +421,19 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): tiled=False, sa=(123, 20, 1.0, "res_multistep", "simple", 1.0), handoff=None, - detail_pass=True, - detail_sampler_name="euler", - detail_scheduler="beta", - detail_steps=5, - detail_denoise=0.4, + latent_upscale_param={ + "mode": "model", + "model_name": "upscale.safetensors", + "width": 256, + "height": 128, + "device": "cpu", + "precision": "fp16", + "refine_denoise": 0.4, + }, ) self.assertEqual(order[0], "sample") - self.assertEqual(order[1], "detail_sample") + self.assertEqual(order[1], "latent_upscale_sample") self.assertLess(order.index("audio"), order.index("video")) self.assertEqual(order[-1], "cleanup") finally: @@ -424,12 +443,14 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): self.module._decode_video = original_decode_video self.module._decode_audio = original_decode_audio self.module._deep_cleanup = original_cleanup + self.module._upscale_latent_video = original_upscale + self.module._copy_sample_latent = original_copy_sample if original_nested is None: delattr(self.module.comfy.nested_tensor, "NestedTensor") else: self.module.comfy.nested_tensor.NestedTensor = original_nested - def test_detail_pass_treats_falsey_strings_as_disabled(self): + def test_latent_upscale_off_skips_second_pass(self): calls = [] original_common_ksampler = self.module.nodes.common_ksampler original_build = self.module._build_shot_conditioning @@ -437,6 +458,8 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): original_decode_video = self.module._decode_video original_decode_audio = self.module._decode_audio original_cleanup = self.module._deep_cleanup + original_upscale = self.module._upscale_latent_video + original_copy_sample = self.module._copy_sample_latent try: self.module.nodes.common_ksampler = lambda *args, **kwargs: (calls.append((args, kwargs)) or {"samples": "latent"},) self.module._build_shot_conditioning = lambda *_args, **_kwargs: ("cond", {"samples": "base"}) @@ -444,6 +467,8 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): self.module._decode_video = lambda _vae, out_latent, *_args, **_kwargs: out_latent self.module._decode_audio = lambda _vae, out_latent: out_latent self.module._deep_cleanup = lambda: None + self.module._upscale_latent_video = lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("should not run")) + self.module._copy_sample_latent = lambda sampled: sampled self.module.H3LongVideos()._render( model=object(), @@ -459,7 +484,7 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): tiled=False, sa=(123, 20, 1.0, "res_multistep", "simple", 1.0), handoff=None, - detail_pass="false", + latent_upscale_param={"mode": "off"}, ) self.assertEqual(len(calls), 1) @@ -470,6 +495,8 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): self.module._decode_video = original_decode_video self.module._decode_audio = original_decode_audio self.module._deep_cleanup = original_cleanup + self.module._upscale_latent_video = original_upscale + self.module._copy_sample_latent = original_copy_sample def test_distribute_generations_canonicalizes_per_shot_audio_and_anchor_directives(self): generations = self.module.distribute_generations( @@ -736,6 +763,12 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): self.assertNotIn(f"ref_image_{index}", optional) self.assertNotIn("per_beat_length", optional) self.assertNotIn("cleanup_between_shots", optional) + self.assertNotIn("detail_pass", optional) + self.assertNotIn("detail_sampler_name", optional) + self.assertNotIn("detail_scheduler", optional) + self.assertNotIn("detail_steps", optional) + self.assertNotIn("detail_denoise", optional) + self.assertIn("latent_upscale_param", optional) def test_shot_seconds_tooltip_describes_ceiling_behavior(self): optional = self.module.H3LongVideos.INPUT_TYPES()["optional"] @@ -743,7 +776,7 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): self.assertIn("GLOBAL per-shot maximum", tooltip) self.assertIn("A beat's own `seconds:` directive can still ask for less", tooltip) - self.assertIn("honoring it; may spill to system RAM (slow) or OOM", tooltip) + self.assertIn("let the render fail instead of shrinking it", tooltip) def test_resolve_shot_frames_honors_forced_request_over_budget(self): original_estimate_shot_frames = self.module.estimate_shot_frames @@ -1061,6 +1094,18 @@ class DumasH3LongVideosHelperTests(unittest.TestCase): {"DumasH3LongVideos": "Dumas H3 Long Videos (FL2VA + REF2VA)"}, ) + def test_latent_upscale_params_node_is_exposed(self): + latent = importlib.import_module("dumas_h3_latent_upscale") + + self.assertEqual( + latent.NODE_CLASS_MAPPINGS, + {"DumasH3LatentUpscaleParams": latent.H3LatentUpscaleParams}, + ) + self.assertEqual( + latent.NODE_DISPLAY_NAME_MAPPINGS, + {"DumasH3LatentUpscaleParams": "Dumas H3 Latent Upscale Params"}, + ) + def test_compose_persistent_does_not_expand_ambiguous_plural_to_full_cast(self): active = self.module.parse_wardrobe( "Maya = she, red jacket\n"