Add RefMod bridge for H3 Continuum
This commit is contained in:
@@ -104,6 +104,28 @@ behaviour. The picker shows matching preview images when present.
|
||||
The maintained MiniMax H3 RefMod reader must be installed alongside DUMAS;
|
||||
DUMAS does not duplicate that reader or override its node IDs.
|
||||
|
||||
### DUMAS H3 RefMods to Continuum
|
||||
|
||||
`DUMAS H3 RefMods to Continuum` is the minimal compatibility bridge for the
|
||||
upstream Continuum sampler. It decodes one visual frame from each selected
|
||||
RefMod with the H3 video VAE and emits Continuum's native nine-slot V3.9
|
||||
Reference Images bundle. Continuum then performs its normal resize, VAE encode,
|
||||
prompt presentation, and physical-group routing.
|
||||
|
||||
Connect the sequence prompt and choose `Prompt @R tags` to route each RefMod
|
||||
only to chunks whose prompt section contains its `@R1`–`@R9` tag. Choose `All
|
||||
chunks` to provide every decoded visual reference to every chunk. Audio-only
|
||||
RefMods are skipped because Continuum's image-reference socket is visual-only.
|
||||
The native image bundle has no per-reference strength field, so picker weights
|
||||
are intentionally not carried across this compatibility boundary.
|
||||
|
||||
This bridge intentionally avoids patching Continuum's internal conditioning
|
||||
runtime. Direct latent injection would need to alter its per-group reference
|
||||
contracts, storage fingerprints, and prompt-picture mapping, and would be a
|
||||
much larger and more fragile fork. The trade-off is one decode followed by
|
||||
Continuum's normal reference encode; pooled/training RefMods will therefore
|
||||
retain only the visual information present in their stored latent.
|
||||
|
||||
### DUMAS H3 Continuum Tiled Latent Upscale (Per Clip)
|
||||
|
||||
`DumasH3ContinuumTiledLatentUpscalePerClip` is a replacement for the upstream
|
||||
|
||||
@@ -9,6 +9,11 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
try:
|
||||
import torch
|
||||
except ImportError: # import-time test fallback; ComfyUI supplies torch at runtime
|
||||
torch = None
|
||||
|
||||
|
||||
def _registered_refmod_root():
|
||||
@@ -59,6 +64,65 @@ def _mod_path(mod):
|
||||
return str(getattr(mod, "path", "") or "")
|
||||
|
||||
|
||||
def _continuum_reference_bundle_type():
|
||||
"""Resolve Continuum's native V3.9 bundle without copying its code."""
|
||||
import importlib
|
||||
candidates = [
|
||||
module for name, module in list(sys.modules.items())
|
||||
if module is not None and name.endswith(".v3.reference_images_v39")
|
||||
]
|
||||
try:
|
||||
candidates.append(importlib.import_module("v3.reference_images_v39"))
|
||||
except Exception:
|
||||
pass
|
||||
for module in candidates:
|
||||
bundle = getattr(module, "ReferenceImagesV39Bundle", None)
|
||||
if bundle is not None:
|
||||
return bundle
|
||||
raise RuntimeError(
|
||||
"DUMAS RefMod-to-Continuum requires the original "
|
||||
"ComfyUI-H3-Continuum v3.reference_images_v39 module."
|
||||
)
|
||||
|
||||
|
||||
def _decode_refmod_image(video_vae, mod):
|
||||
if torch is None:
|
||||
raise RuntimeError("DUMAS RefMod-to-Continuum requires PyTorch/ComfyUI.")
|
||||
latent = getattr(mod, "latent", None)
|
||||
if getattr(mod, "kind", None) == "audio" or latent is None:
|
||||
return None
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5:
|
||||
raise ValueError(f"RefMod '{getattr(mod, 'name', 'unknown')}' has no visual H3 latent.")
|
||||
with torch.no_grad():
|
||||
decoded = video_vae.decode(latent[:, :, :1])
|
||||
if not torch.is_tensor(decoded):
|
||||
raise ValueError("The connected H3 video VAE returned a non-tensor image.")
|
||||
if decoded.ndim == 5:
|
||||
decoded = decoded.reshape(-1, decoded.shape[-3], decoded.shape[-2], decoded.shape[-1])
|
||||
if decoded.ndim != 4 or decoded.shape[0] < 1 or decoded.shape[-1] < 3:
|
||||
raise ValueError(f"Decoded RefMod image has unexpected shape {tuple(decoded.shape)}.")
|
||||
return decoded[:1, :, :, :3].clamp(0.0, 1.0).float().cpu()
|
||||
|
||||
|
||||
def _prompt_selectors(prompt, count):
|
||||
"""Return R-slot selectors from chunk headers and @R tags."""
|
||||
if not prompt:
|
||||
return ("all",) * 9
|
||||
text = str(prompt)
|
||||
markers = list(re.finditer(r"\[[^\]]*?-[^\]]*?s\]", text, re.IGNORECASE))
|
||||
if not markers:
|
||||
return ("all",) * 9
|
||||
selectors = []
|
||||
for ref_index in range(1, 10):
|
||||
chunks = []
|
||||
for chunk_index, marker in enumerate(markers[:count], start=1):
|
||||
end = markers[chunk_index].start() if chunk_index < len(markers) else len(text)
|
||||
if re.search(rf"@R{ref_index}(?![A-Za-z0-9_])", text[marker.start():end], re.IGNORECASE):
|
||||
chunks.append(str(chunk_index))
|
||||
selectors.append(",".join(chunks) or "off")
|
||||
return tuple(selectors)
|
||||
|
||||
|
||||
class DumasH3RefModSave:
|
||||
"""Save upstream MiniMax H3 RefMods using an explicit DUMAS policy.
|
||||
|
||||
@@ -163,10 +227,82 @@ class DumasH3RefModSave:
|
||||
return {"ui": {"text": saved}, "result": (mods, report)}
|
||||
|
||||
|
||||
class DumasH3RefModsToContinuum:
|
||||
"""Decode visual RefMods into Continuum's native V3.9 image bundle."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"mods": ("H3_REF_MODS",),
|
||||
"video_vae": ("VAE",),
|
||||
"routing": (["All chunks", "Prompt @R tags"], {"default": "Prompt @R tags"}),
|
||||
"chunk_count": ("INT", {"default": 1, "min": 1, "max": 16}),
|
||||
},
|
||||
"optional": {
|
||||
"sequence_prompt": ("STRING", {"default": "", "multiline": True}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("H3_CONTINUUM_REFERENCE_IMAGES_V39", "STRING")
|
||||
RETURN_NAMES = ("reference_images", "info")
|
||||
FUNCTION = "convert"
|
||||
CATEGORY = "Dumas/MiniMax H3/RefMod"
|
||||
DESCRIPTION = (
|
||||
"Decode visual H3 RefMods and feed them through Continuum's native "
|
||||
"V3.9 Reference Images input. Connect the sequence prompt to route @R "
|
||||
"tags per chunk."
|
||||
)
|
||||
|
||||
def convert(self, mods, video_vae, routing="Prompt @R tags", chunk_count=1,
|
||||
sequence_prompt=""):
|
||||
if not mods:
|
||||
raise ValueError("The H3 RefMod bundle is empty.")
|
||||
if routing not in ("All chunks", "Prompt @R tags"):
|
||||
raise ValueError(f"Unknown RefMod routing mode: {routing!r}")
|
||||
images, skipped_audio, ignored_strength = [], [], []
|
||||
seen = set()
|
||||
for mod, _strength in mods:
|
||||
if getattr(mod, "kind", None) == "audio":
|
||||
skipped_audio.append(getattr(mod, "name", "audio"))
|
||||
continue
|
||||
source_key = _mod_path(mod) or (getattr(mod, "name", "refmod"), id(mod))
|
||||
if source_key in seen:
|
||||
continue
|
||||
seen.add(source_key)
|
||||
strength = float(_strength)
|
||||
if strength <= 0:
|
||||
continue
|
||||
if abs(strength - 1.0) > 1e-6:
|
||||
ignored_strength.append(getattr(mod, "name", "refmod"))
|
||||
if len(images) >= 9:
|
||||
raise ValueError("Continuum V3.9 supports at most nine RefMod image slots.")
|
||||
image = _decode_refmod_image(video_vae, mod)
|
||||
if image is not None:
|
||||
images.append(image)
|
||||
if not images:
|
||||
raise ValueError("No visual RefMods were found; audio-only RefMods cannot feed image references.")
|
||||
selectors = (("all",) * 9 if routing == "All chunks"
|
||||
else _prompt_selectors(sequence_prompt, int(chunk_count)))
|
||||
slots = tuple(images) + (None,) * (9 - len(images))
|
||||
bundle = _continuum_reference_bundle_type()(
|
||||
images=slots, reference_use="Per chunk", selectors=selectors,
|
||||
)
|
||||
details = [f"decoded {len(images)} visual RefMod(s) into Continuum slots R1-R{len(images)}"]
|
||||
details.append("routing=all chunks" if routing == "All chunks" else "routing=prompt @R tags")
|
||||
if skipped_audio:
|
||||
details.append("skipped audio-only: " + ", ".join(skipped_audio))
|
||||
if ignored_strength:
|
||||
details.append("native image routing ignores weights: " + ", ".join(ignored_strength))
|
||||
return bundle, "; ".join(details)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"DumasH3RefModSave": DumasH3RefModSave,
|
||||
"DumasH3RefModsToContinuum": DumasH3RefModsToContinuum,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"DumasH3RefModSave": "DUMAS H3 RefMod Save",
|
||||
"DumasH3RefModsToContinuum": "DUMAS H3 RefMods to Continuum",
|
||||
}
|
||||
|
||||
@@ -4,6 +4,10 @@ import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
try:
|
||||
import torch
|
||||
except ImportError:
|
||||
torch = None
|
||||
|
||||
|
||||
class FakeMod:
|
||||
@@ -17,6 +21,17 @@ class FakeMod:
|
||||
return path + ".safetensors"
|
||||
|
||||
|
||||
class FakeVisualMod:
|
||||
kind = "video"
|
||||
name = "hero"
|
||||
latent = torch.zeros(1, 24, 1, 2, 2) if torch is not None else None
|
||||
|
||||
|
||||
class FakeAudioMod:
|
||||
kind = "audio"
|
||||
name = "voice"
|
||||
|
||||
|
||||
class DumasRefModSaveTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
@@ -63,6 +78,41 @@ class DumasRefModSaveTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "input_folder"):
|
||||
node.save([ (FakeMod("hero"), 1.0) ], save_location="Input folder")
|
||||
|
||||
@unittest.skipUnless(torch is not None, "PyTorch is only available in the ComfyUI runtime")
|
||||
def test_refmods_to_continuum_decodes_and_routes_prompt_tags(self):
|
||||
bundle_module = types.SimpleNamespace(
|
||||
ReferenceImagesV39Bundle=lambda **kwargs: kwargs)
|
||||
sys.modules["fake.v3.reference_images_v39"] = bundle_module
|
||||
try:
|
||||
class VAE:
|
||||
def decode(self, latent):
|
||||
return torch.ones(1, 1, 4, 3)
|
||||
|
||||
bundle, info = self.module.DumasH3RefModsToContinuum().convert(
|
||||
[(FakeVisualMod(), 1.0)], VAE(), routing="Prompt @R tags", chunk_count=2,
|
||||
sequence_prompt="[0-4s]\n@R1 action\n\n[4-8s]\ncontinue")
|
||||
self.assertEqual(bundle["reference_use"], "Per chunk")
|
||||
self.assertEqual(bundle["selectors"][0], "1")
|
||||
self.assertIn("decoded 1 visual", info)
|
||||
finally:
|
||||
sys.modules.pop("fake.v3.reference_images_v39", None)
|
||||
|
||||
@unittest.skipUnless(torch is not None, "PyTorch is only available in the ComfyUI runtime")
|
||||
def test_refmods_to_continuum_skips_audio_only_rows(self):
|
||||
bundle_module = types.SimpleNamespace(
|
||||
ReferenceImagesV39Bundle=lambda **kwargs: kwargs)
|
||||
sys.modules["fake.v3.reference_images_v39"] = bundle_module
|
||||
try:
|
||||
class VAE:
|
||||
def decode(self, latent):
|
||||
return torch.ones(1, 1, 4, 3)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "No visual RefMods"):
|
||||
self.module.DumasH3RefModsToContinuum().convert(
|
||||
[(FakeAudioMod(), 1.0)], VAE(), routing="All chunks")
|
||||
finally:
|
||||
sys.modules.pop("fake.v3.reference_images_v39", None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user