Files
DumasNodes/dumas_h3_upscale_nodes.py

497 lines
20 KiB
Python

"""Per-clip tiled latent upscaling for MiniMax H3 Continuum.
Continuum exposes physical video groups as a LATENT list. The native
``MMH3UltimateUpscale`` node operates on one H3 latent at a time, so this
adapter deliberately applies it independently to each physical clip instead
of concatenating the clips into one larger VRAM allocation.
"""
from __future__ import annotations
from typing import Any
def _sampler_options() -> list[str]:
try:
import comfy.samplers
return list(comfy.samplers.KSampler.SAMPLERS)
except Exception: # pragma: no cover - exercised on a ComfyUI host
return ["euler", "euler_ancestral", "res_multistep", "sa_solver"]
def _scheduler_options() -> list[str]:
try:
import comfy.samplers
return list(comfy.samplers.KSampler.SCHEDULERS)
except Exception: # pragma: no cover - exercised on a ComfyUI host
return ["simple", "beta", "sgm_uniform", "karras", "exponential"]
def _continuum_helper(name: str):
"""Resolve a helper from the installed upstream H3 Continuum package."""
import sys
for module_name, module in list(sys.modules.items()):
if module is not None and (
module_name.endswith(".v3.second_pass")
or module_name.endswith(".v2.sampling")
or module_name.endswith(".state")
):
helper = getattr(module, name, None)
if helper is not None:
return helper
import importlib
for module_name in ("v3.second_pass", "v2.sampling", "state"):
try:
helper = getattr(importlib.import_module(module_name), name, None)
if helper is not None:
return helper
except Exception:
pass
raise RuntimeError(
"DUMAS H3 per-clip upscale requires the original "
"ComfyUI-H3-Continuum AV latent helpers."
)
def _native_node(name: str):
try:
import nodes
return (getattr(nodes, "NODE_CLASS_MAPPINGS", {}) or {}).get(name)
except Exception:
return None
def _unwrap_node_output(value: Any) -> Any:
"""Extract values returned by ComfyUI's newer NodeOutput API.
The v3 API wraps node results in ``NodeOutput`` while legacy nodes expect
the underlying tensor/object directly. Keep this adapter compatible with
both APIs without importing ComfyUI internals.
"""
result = getattr(value, "result", None)
if result is None and not hasattr(value, "result"):
return value
return result
def _unwrap_single_output(value: Any, name: str) -> Any:
marker = object()
result = getattr(value, "result", marker)
# Ordinary tuples are valid values for the lightweight test doubles and
# for some legacy ComfyUI helpers. Only unwrap an actual NodeOutput-like
# object, identified by its result property.
if result is marker:
return value
if isinstance(result, (tuple, list)):
if len(result) != 1:
raise ValueError(f"{name} returned {len(result)} outputs; expected one")
return result[0]
return result
def _call_native(node_class, kwargs: dict[str, Any]):
"""Invoke a ComfyUI node through its declared FUNCTION contract.
ComfyUI node classes do not have a universal Python method name. The live
MiniMax node currently declares its executor through ``FUNCTION`` rather
than exposing a method named ``upscale``.
"""
instance = node_class()
declared = getattr(node_class, "FUNCTION", None)
candidates = []
if isinstance(declared, str) and declared:
candidates.append(declared)
candidates.extend(("execute", "run", "process", "sample", "upscale"))
seen = set()
for name in candidates:
if name in seen:
continue
seen.add(name)
method = getattr(instance, name, None)
if callable(method):
return _unwrap_node_output(method(**kwargs))
raise RuntimeError(
"MMH3UltimateUpscale exposes no callable ComfyUI FUNCTION method "
f"(declared={declared!r})"
)
def _one(value: Any, name: str) -> Any:
"""Unwrap ComfyUI list inputs while rejecting ambiguous values."""
if isinstance(value, list):
if len(value) != 1:
raise ValueError(f"{name} must contain one shared value")
return value[0]
return value
def _clips(value: Any) -> list[Any]:
if isinstance(value, list):
return value
return [value]
def _per_clip(value: Any, count: int, name: str) -> list[Any]:
values = _clips(value)
if len(values) == 1:
return values * count
if len(values) != count:
raise ValueError(f"{name} has {len(values)} values for {count} clips")
return values
class DumasH3ContinuumTiledLatentUpscalePerClip:
"""Upscale each Continuum physical clip separately with tiled H3 latent work.
This is intentionally a thin adapter around ComfyUI's native
``MMH3UltimateUpscale`` implementation. It does not vendor or replace
the model-specific implementation; it only supplies an independent
sampler/scheduler/noise/CFG schedule for each clip and preserves list
boundaries so clips never become one oversized VRAM job.
"""
DESCRIPTION = (
"Apply native MiniMax H3 tiled latent upscaling independently to every "
"Continuum physical clip. Clips remain separate LATENT list items."
)
SEARCH_ALIASES = [
"H3 tiled upscale per clip",
"MiniMax H3 clip upscale",
"Continuum latent upscale",
]
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"video_latents": ("LATENT",),
"audio_latents": ("LATENT",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
"latent_upscale_param": ("H3_UPSCALE_PARAM",),
"spatial_split_param": ("H3_SPATIAL_PARAM",),
"upscale_sampler": (
_sampler_options(),
{"default": "euler_ancestral"},
),
"upscale_scheduler": (
_scheduler_options(),
{"default": "simple"},
),
"upscale_steps": ("INT", {"default": 3, "min": 1, "max": 50}),
"upscale_denoise": (
"FLOAT",
{"default": 0.2, "min": 0.0, "max": 1.0, "step": 0.01},
),
"upscale_cfg": (
"FLOAT",
{"default": 1.0, "min": 0.0, "max": 20.0, "step": 0.1},
),
"upscale_noise_seed": (
"INT",
{
"default": 363742558835919,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
},
),
},
"optional": {
"conditioning": ("CONDITIONING",),
"video_vae": ("VAE",),
"negative": ("CONDITIONING",),
"temporal_split_param": ("H3_TEMPORAL_PARAM",),
"upscale_sigmas": ("SIGMAS",),
},
}
INPUT_IS_LIST = True
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
RETURN_NAMES = ("refined_video_latents", "audio_latents", "updated_assembly_plan", "status")
OUTPUT_IS_LIST = (True, True, False, False)
FUNCTION = "upscale"
CATEGORY = "Dumas/MiniMax H3/Upscale"
def upscale(
self,
model,
clip,
video_latents,
audio_latents,
assembly_plan,
refine_context,
latent_upscale_param,
spatial_split_param,
upscale_sampler,
upscale_scheduler,
upscale_steps,
upscale_denoise,
upscale_cfg,
upscale_noise_seed,
conditioning=None,
video_vae=None,
negative=None,
temporal_split_param=None,
upscale_sigmas=None,
):
latents = _clips(video_latents)
if not latents:
raise ValueError("video_latents must contain at least one clip")
if conditioning is None:
# The Continuum sampler's refine context is a runtime contract, not
# a Core CONDITIONING value. Reuse Continuum's own bridge so each
# physical clip receives its captured prompt/reference conditioning
# and its context-aware model clone at the target geometry.
prepare_physical_refine_groups = _continuum_helper("prepare_physical_refine_groups")
prepared = prepare_physical_refine_groups(
model=_one(model, "model"),
clip=_one(clip, "clip"),
video_latents=latents,
assembly_plan=_one(assembly_plan, "assembly_plan"),
refine_context=_one(refine_context, "refine_context"),
video_vae=(
None if video_vae is None else _one(video_vae, "video_vae")
),
)
models = prepared["group_models"]
conditionings = prepared["conditioning"]
if len(models) != len(latents) or len(conditionings) != len(latents):
raise ValueError(
"refine context bridge did not produce one model and conditioning "
"value per physical clip"
)
else:
models = _per_clip(model, len(latents), "model")
conditionings = _per_clip(conditioning, len(latents), "conditioning")
negatives = (
[None] * len(latents)
if negative is None
else _per_clip(negative, len(latents), "negative")
)
temporal = (
[None] * len(latents)
if temporal_split_param is None
else _per_clip(temporal_split_param, len(latents), "temporal_split_param")
)
sigma_values = (
[None] * len(latents)
if upscale_sigmas is None
else _per_clip(upscale_sigmas, len(latents), "upscale_sigmas")
)
sampler_name = str(_one(upscale_sampler, "upscale_sampler"))
scheduler_name = str(_one(upscale_scheduler, "upscale_scheduler"))
steps = int(_one(upscale_steps, "upscale_steps"))
denoise = float(_one(upscale_denoise, "upscale_denoise"))
cfg = float(_one(upscale_cfg, "upscale_cfg"))
seed = int(_one(upscale_noise_seed, "upscale_noise_seed"))
upscale_param = _one(latent_upscale_param, "latent_upscale_param")
spatial_param = _one(spatial_split_param, "spatial_split_param")
native = _native_node("MMH3UltimateUpscale")
random_noise = _native_node("RandomNoise")
sampler_select = _native_node("KSamplerSelect")
scheduler = _native_node("BasicScheduler")
is_dumas_2d = isinstance(upscale_param, dict) and upscale_param.get("dumas_h3_2d")
missing = [
name
for name, value in (
("MMH3UltimateUpscale", native),
("RandomNoise", random_noise),
("KSamplerSelect", sampler_select),
("BasicScheduler", scheduler),
)
if value is None
]
if missing:
raise RuntimeError("Missing native ComfyUI nodes: " + ", ".join(missing))
outputs = []
video_outputs = []
reports = []
build_av_latent = _continuum_helper("latent_from_cpu")
extract_av_streams = _continuum_helper("extract_av_streams")
audio_values = _clips(audio_latents)
if len(audio_values) != len(latents):
raise ValueError(
"audio_latents must contain one first-pass audio latent per physical clip"
)
if is_dumas_2d:
from .dumas_h3_latent_upscale_2d import DumasMMH3LatentUpscale2D
processor = DumasMMH3LatentUpscale2D()
for index, (clip_latent, clip_model, clip_conditioning) in enumerate(
zip(latents, models, conditionings), start=1
):
samples = clip_latent.get("samples") if isinstance(clip_latent, dict) else None
if samples is None or getattr(samples, "ndim", None) != 5:
raise ValueError(f"clip {index} is not a 5-D H3 LATENT")
upscaled, report = processor.upscale(
{"samples": samples},
model_name=upscale_param["model_name"],
width=upscale_param["target_width"],
height=upscale_param["target_height"],
tile_size=upscale_param["tile_size"],
tile_overlap=upscale_param["tile_overlap"],
sharpness=upscale_param["sharpness"],
device=upscale_param["device"],
precision=upscale_param["precision"],
)
audio_latent = audio_values[index - 1]
audio_samples = audio_latent.get("samples")
nested_latent = build_av_latent(upscaled["samples"], audio_samples)
noise = _unwrap_single_output(
random_noise().get_noise(seed + index - 1), "RandomNoise"
)
sampler = _unwrap_single_output(
sampler_select().get_sampler(sampler_name), "KSamplerSelect"
)
sigmas = sigma_values[index - 1]
if sigmas is None:
sigmas = _unwrap_single_output(
scheduler().get_sigmas(
clip_model, scheduler_name, steps, denoise
),
"BasicScheduler",
)
refine_kwargs = {
"model": clip_model,
"conditioning": clip_conditioning,
"latent": nested_latent,
"noise": noise,
"sampler": sampler,
"sigmas": sigmas,
"cfg": cfg,
# The 2D model has already performed the resize. Passing no
# learned-upscale parameter makes the native node resample
# the target-size tiles only.
"latent_upscale_param": None,
"temporal_split_param": None,
"spatial_split_param": spatial_param,
}
if negatives[index - 1] is not None:
refine_kwargs["negative"] = negatives[index - 1]
refined = _call_native(native, refine_kwargs)
if not isinstance(refined, (tuple, list)) or not refined:
raise RuntimeError(f"native refinement returned no output for clip {index}")
output = refined[0]
outputs.append(output)
output_samples, _ = extract_av_streams(output)
video_outputs.append({"samples": output_samples})
reports.append(f"clip {index}: {report}; native tiled refinement applied")
audio_outputs = _clips(audio_latents)
update_geometry = _continuum_helper("update_second_pass_geometry")
updated_plan = update_geometry(
_one(assembly_plan, "assembly_plan"), video_outputs
)
return (
outputs, audio_outputs, updated_plan,
"H3 Continuum DUMAS 2D spatial latent upscale; "
+ "; ".join(reports),
)
for index, (clip_latent, clip_model, clip_conditioning) in enumerate(
zip(latents, models, conditionings), start=1
):
samples = clip_latent.get("samples") if isinstance(clip_latent, dict) else None
if samples is None or getattr(samples, "ndim", None) != 5:
raise ValueError(f"clip {index} is not a 5-D H3 LATENT")
audio_latent = audio_values[index - 1]
audio_samples = (
audio_latent.get("samples")
if isinstance(audio_latent, dict) else None
)
if audio_samples is None:
raise ValueError(f"audio clip {index} is not a LATENT mapping")
nested_latent = build_av_latent(samples, audio_samples)
noise = _unwrap_single_output(
random_noise().get_noise(seed + index - 1), "RandomNoise"
)
sampler = _unwrap_single_output(
sampler_select().get_sampler(sampler_name), "KSamplerSelect"
)
sigmas = sigma_values[index - 1]
if sigmas is None:
sigmas = _unwrap_single_output(
scheduler().get_sigmas(
clip_model, scheduler_name, steps, denoise
),
"BasicScheduler",
)
kwargs = {
"model": clip_model,
"conditioning": clip_conditioning,
"latent": nested_latent,
"noise": noise,
"sampler": sampler,
"sigmas": sigmas,
"cfg": cfg,
"latent_upscale_param": upscale_param,
"temporal_split_param": temporal[index - 1],
"spatial_split_param": spatial_param,
}
if negatives[index - 1] is not None:
kwargs["negative"] = negatives[index - 1]
result = _call_native(native, kwargs)
if not isinstance(result, (tuple, list)) or not result:
raise RuntimeError(f"native upscale returned no output for clip {index}")
output = result[0]
try:
output_samples, _ = extract_av_streams(output)
except Exception as exc:
raise RuntimeError(
f"native upscale returned an invalid AV latent for clip {index}: {exc}"
) from exc
if output_samples.shape[2] != samples.shape[2]:
raise RuntimeError(f"tiled upscale changed temporal length for clip {index}")
outputs.append(output)
# The native node returns an H3 AV latent (video + audio nested in
# ``samples``), while the Continuum geometry contract deliberately
# accepts plain video LATENT mappings. Keep the AV latent for the
# node output, but pass only the extracted video tensor to the
# geometry validator/update helper.
video_outputs.append({"samples": output_samples})
reports.append(
f"clip {index}: {tuple(samples.shape)} -> {tuple(output_samples.shape)}"
)
audio_outputs = _clips(audio_latents)
if len(audio_outputs) != len(outputs):
raise ValueError(
"audio_latents must contain one first-pass audio latent per physical clip"
)
update_geometry = _continuum_helper("update_second_pass_geometry")
updated_plan = update_geometry(
_one(assembly_plan, "assembly_plan"), video_outputs
)
return (
outputs,
audio_outputs,
updated_plan,
"H3 Continuum per-clip tiled upscale replacement; "
f"clips={len(outputs)}, sampler={sampler_name}, scheduler={scheduler_name}, "
f"steps={steps}, denoise={denoise:g}, cfg={cfg:g}, seed_base={seed}; "
"audio=first-pass passthrough; " + "; ".join(reports),
)
NODE_CLASS_MAPPINGS = {
"DumasH3ContinuumTiledLatentUpscalePerClip": DumasH3ContinuumTiledLatentUpscalePerClip,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"DumasH3ContinuumTiledLatentUpscalePerClip": (
"DUMAS H3 Continuum Tiled Latent Upscale (Per Clip)"
),
}