Add RefMod bridge for H3 Continuum
This commit is contained in:
@@ -104,6 +104,28 @@ behaviour. The picker shows matching preview images when present.
|
|||||||
The maintained MiniMax H3 RefMod reader must be installed alongside DUMAS;
|
The maintained MiniMax H3 RefMod reader must be installed alongside DUMAS;
|
||||||
DUMAS does not duplicate that reader or override its node IDs.
|
DUMAS does not duplicate that reader or override its node IDs.
|
||||||
|
|
||||||
|
### DUMAS H3 RefMods to Continuum
|
||||||
|
|
||||||
|
`DUMAS H3 RefMods to Continuum` is the minimal compatibility bridge for the
|
||||||
|
upstream Continuum sampler. It decodes one visual frame from each selected
|
||||||
|
RefMod with the H3 video VAE and emits Continuum's native nine-slot V3.9
|
||||||
|
Reference Images bundle. Continuum then performs its normal resize, VAE encode,
|
||||||
|
prompt presentation, and physical-group routing.
|
||||||
|
|
||||||
|
Connect the sequence prompt and choose `Prompt @R tags` to route each RefMod
|
||||||
|
only to chunks whose prompt section contains its `@R1`–`@R9` tag. Choose `All
|
||||||
|
chunks` to provide every decoded visual reference to every chunk. Audio-only
|
||||||
|
RefMods are skipped because Continuum's image-reference socket is visual-only.
|
||||||
|
The native image bundle has no per-reference strength field, so picker weights
|
||||||
|
are intentionally not carried across this compatibility boundary.
|
||||||
|
|
||||||
|
This bridge intentionally avoids patching Continuum's internal conditioning
|
||||||
|
runtime. Direct latent injection would need to alter its per-group reference
|
||||||
|
contracts, storage fingerprints, and prompt-picture mapping, and would be a
|
||||||
|
much larger and more fragile fork. The trade-off is one decode followed by
|
||||||
|
Continuum's normal reference encode; pooled/training RefMods will therefore
|
||||||
|
retain only the visual information present in their stored latent.
|
||||||
|
|
||||||
### DUMAS H3 Continuum Tiled Latent Upscale (Per Clip)
|
### DUMAS H3 Continuum Tiled Latent Upscale (Per Clip)
|
||||||
|
|
||||||
`DumasH3ContinuumTiledLatentUpscalePerClip` is a replacement for the upstream
|
`DumasH3ContinuumTiledLatentUpscalePerClip` is a replacement for the upstream
|
||||||
|
|||||||
@@ -9,6 +9,11 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
|
try:
|
||||||
|
import torch
|
||||||
|
except ImportError: # import-time test fallback; ComfyUI supplies torch at runtime
|
||||||
|
torch = None
|
||||||
|
|
||||||
|
|
||||||
def _registered_refmod_root():
|
def _registered_refmod_root():
|
||||||
@@ -59,6 +64,65 @@ def _mod_path(mod):
|
|||||||
return str(getattr(mod, "path", "") or "")
|
return str(getattr(mod, "path", "") or "")
|
||||||
|
|
||||||
|
|
||||||
|
def _continuum_reference_bundle_type():
|
||||||
|
"""Resolve Continuum's native V3.9 bundle without copying its code."""
|
||||||
|
import importlib
|
||||||
|
candidates = [
|
||||||
|
module for name, module in list(sys.modules.items())
|
||||||
|
if module is not None and name.endswith(".v3.reference_images_v39")
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
candidates.append(importlib.import_module("v3.reference_images_v39"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
for module in candidates:
|
||||||
|
bundle = getattr(module, "ReferenceImagesV39Bundle", None)
|
||||||
|
if bundle is not None:
|
||||||
|
return bundle
|
||||||
|
raise RuntimeError(
|
||||||
|
"DUMAS RefMod-to-Continuum requires the original "
|
||||||
|
"ComfyUI-H3-Continuum v3.reference_images_v39 module."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_refmod_image(video_vae, mod):
|
||||||
|
if torch is None:
|
||||||
|
raise RuntimeError("DUMAS RefMod-to-Continuum requires PyTorch/ComfyUI.")
|
||||||
|
latent = getattr(mod, "latent", None)
|
||||||
|
if getattr(mod, "kind", None) == "audio" or latent is None:
|
||||||
|
return None
|
||||||
|
if not torch.is_tensor(latent) or latent.ndim != 5:
|
||||||
|
raise ValueError(f"RefMod '{getattr(mod, 'name', 'unknown')}' has no visual H3 latent.")
|
||||||
|
with torch.no_grad():
|
||||||
|
decoded = video_vae.decode(latent[:, :, :1])
|
||||||
|
if not torch.is_tensor(decoded):
|
||||||
|
raise ValueError("The connected H3 video VAE returned a non-tensor image.")
|
||||||
|
if decoded.ndim == 5:
|
||||||
|
decoded = decoded.reshape(-1, decoded.shape[-3], decoded.shape[-2], decoded.shape[-1])
|
||||||
|
if decoded.ndim != 4 or decoded.shape[0] < 1 or decoded.shape[-1] < 3:
|
||||||
|
raise ValueError(f"Decoded RefMod image has unexpected shape {tuple(decoded.shape)}.")
|
||||||
|
return decoded[:1, :, :, :3].clamp(0.0, 1.0).float().cpu()
|
||||||
|
|
||||||
|
|
||||||
|
def _prompt_selectors(prompt, count):
|
||||||
|
"""Return R-slot selectors from chunk headers and @R tags."""
|
||||||
|
if not prompt:
|
||||||
|
return ("all",) * 9
|
||||||
|
text = str(prompt)
|
||||||
|
markers = list(re.finditer(r"\[[^\]]*?-[^\]]*?s\]", text, re.IGNORECASE))
|
||||||
|
if not markers:
|
||||||
|
return ("all",) * 9
|
||||||
|
selectors = []
|
||||||
|
for ref_index in range(1, 10):
|
||||||
|
chunks = []
|
||||||
|
for chunk_index, marker in enumerate(markers[:count], start=1):
|
||||||
|
end = markers[chunk_index].start() if chunk_index < len(markers) else len(text)
|
||||||
|
if re.search(rf"@R{ref_index}(?![A-Za-z0-9_])", text[marker.start():end], re.IGNORECASE):
|
||||||
|
chunks.append(str(chunk_index))
|
||||||
|
selectors.append(",".join(chunks) or "off")
|
||||||
|
return tuple(selectors)
|
||||||
|
|
||||||
|
|
||||||
class DumasH3RefModSave:
|
class DumasH3RefModSave:
|
||||||
"""Save upstream MiniMax H3 RefMods using an explicit DUMAS policy.
|
"""Save upstream MiniMax H3 RefMods using an explicit DUMAS policy.
|
||||||
|
|
||||||
@@ -163,10 +227,82 @@ class DumasH3RefModSave:
|
|||||||
return {"ui": {"text": saved}, "result": (mods, report)}
|
return {"ui": {"text": saved}, "result": (mods, report)}
|
||||||
|
|
||||||
|
|
||||||
|
class DumasH3RefModsToContinuum:
|
||||||
|
"""Decode visual RefMods into Continuum's native V3.9 image bundle."""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(cls):
|
||||||
|
return {
|
||||||
|
"required": {
|
||||||
|
"mods": ("H3_REF_MODS",),
|
||||||
|
"video_vae": ("VAE",),
|
||||||
|
"routing": (["All chunks", "Prompt @R tags"], {"default": "Prompt @R tags"}),
|
||||||
|
"chunk_count": ("INT", {"default": 1, "min": 1, "max": 16}),
|
||||||
|
},
|
||||||
|
"optional": {
|
||||||
|
"sequence_prompt": ("STRING", {"default": "", "multiline": True}),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
RETURN_TYPES = ("H3_CONTINUUM_REFERENCE_IMAGES_V39", "STRING")
|
||||||
|
RETURN_NAMES = ("reference_images", "info")
|
||||||
|
FUNCTION = "convert"
|
||||||
|
CATEGORY = "Dumas/MiniMax H3/RefMod"
|
||||||
|
DESCRIPTION = (
|
||||||
|
"Decode visual H3 RefMods and feed them through Continuum's native "
|
||||||
|
"V3.9 Reference Images input. Connect the sequence prompt to route @R "
|
||||||
|
"tags per chunk."
|
||||||
|
)
|
||||||
|
|
||||||
|
def convert(self, mods, video_vae, routing="Prompt @R tags", chunk_count=1,
|
||||||
|
sequence_prompt=""):
|
||||||
|
if not mods:
|
||||||
|
raise ValueError("The H3 RefMod bundle is empty.")
|
||||||
|
if routing not in ("All chunks", "Prompt @R tags"):
|
||||||
|
raise ValueError(f"Unknown RefMod routing mode: {routing!r}")
|
||||||
|
images, skipped_audio, ignored_strength = [], [], []
|
||||||
|
seen = set()
|
||||||
|
for mod, _strength in mods:
|
||||||
|
if getattr(mod, "kind", None) == "audio":
|
||||||
|
skipped_audio.append(getattr(mod, "name", "audio"))
|
||||||
|
continue
|
||||||
|
source_key = _mod_path(mod) or (getattr(mod, "name", "refmod"), id(mod))
|
||||||
|
if source_key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(source_key)
|
||||||
|
strength = float(_strength)
|
||||||
|
if strength <= 0:
|
||||||
|
continue
|
||||||
|
if abs(strength - 1.0) > 1e-6:
|
||||||
|
ignored_strength.append(getattr(mod, "name", "refmod"))
|
||||||
|
if len(images) >= 9:
|
||||||
|
raise ValueError("Continuum V3.9 supports at most nine RefMod image slots.")
|
||||||
|
image = _decode_refmod_image(video_vae, mod)
|
||||||
|
if image is not None:
|
||||||
|
images.append(image)
|
||||||
|
if not images:
|
||||||
|
raise ValueError("No visual RefMods were found; audio-only RefMods cannot feed image references.")
|
||||||
|
selectors = (("all",) * 9 if routing == "All chunks"
|
||||||
|
else _prompt_selectors(sequence_prompt, int(chunk_count)))
|
||||||
|
slots = tuple(images) + (None,) * (9 - len(images))
|
||||||
|
bundle = _continuum_reference_bundle_type()(
|
||||||
|
images=slots, reference_use="Per chunk", selectors=selectors,
|
||||||
|
)
|
||||||
|
details = [f"decoded {len(images)} visual RefMod(s) into Continuum slots R1-R{len(images)}"]
|
||||||
|
details.append("routing=all chunks" if routing == "All chunks" else "routing=prompt @R tags")
|
||||||
|
if skipped_audio:
|
||||||
|
details.append("skipped audio-only: " + ", ".join(skipped_audio))
|
||||||
|
if ignored_strength:
|
||||||
|
details.append("native image routing ignores weights: " + ", ".join(ignored_strength))
|
||||||
|
return bundle, "; ".join(details)
|
||||||
|
|
||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"DumasH3RefModSave": DumasH3RefModSave,
|
"DumasH3RefModSave": DumasH3RefModSave,
|
||||||
|
"DumasH3RefModsToContinuum": DumasH3RefModsToContinuum,
|
||||||
}
|
}
|
||||||
|
|
||||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
"DumasH3RefModSave": "DUMAS H3 RefMod Save",
|
"DumasH3RefModSave": "DUMAS H3 RefMod Save",
|
||||||
|
"DumasH3RefModsToContinuum": "DUMAS H3 RefMods to Continuum",
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ import sys
|
|||||||
import tempfile
|
import tempfile
|
||||||
import types
|
import types
|
||||||
import unittest
|
import unittest
|
||||||
|
try:
|
||||||
|
import torch
|
||||||
|
except ImportError:
|
||||||
|
torch = None
|
||||||
|
|
||||||
|
|
||||||
class FakeMod:
|
class FakeMod:
|
||||||
@@ -17,6 +21,17 @@ class FakeMod:
|
|||||||
return path + ".safetensors"
|
return path + ".safetensors"
|
||||||
|
|
||||||
|
|
||||||
|
class FakeVisualMod:
|
||||||
|
kind = "video"
|
||||||
|
name = "hero"
|
||||||
|
latent = torch.zeros(1, 24, 1, 2, 2) if torch is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
class FakeAudioMod:
|
||||||
|
kind = "audio"
|
||||||
|
name = "voice"
|
||||||
|
|
||||||
|
|
||||||
class DumasRefModSaveTests(unittest.TestCase):
|
class DumasRefModSaveTests(unittest.TestCase):
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
@@ -63,6 +78,41 @@ class DumasRefModSaveTests(unittest.TestCase):
|
|||||||
with self.assertRaisesRegex(ValueError, "input_folder"):
|
with self.assertRaisesRegex(ValueError, "input_folder"):
|
||||||
node.save([ (FakeMod("hero"), 1.0) ], save_location="Input folder")
|
node.save([ (FakeMod("hero"), 1.0) ], save_location="Input folder")
|
||||||
|
|
||||||
|
@unittest.skipUnless(torch is not None, "PyTorch is only available in the ComfyUI runtime")
|
||||||
|
def test_refmods_to_continuum_decodes_and_routes_prompt_tags(self):
|
||||||
|
bundle_module = types.SimpleNamespace(
|
||||||
|
ReferenceImagesV39Bundle=lambda **kwargs: kwargs)
|
||||||
|
sys.modules["fake.v3.reference_images_v39"] = bundle_module
|
||||||
|
try:
|
||||||
|
class VAE:
|
||||||
|
def decode(self, latent):
|
||||||
|
return torch.ones(1, 1, 4, 3)
|
||||||
|
|
||||||
|
bundle, info = self.module.DumasH3RefModsToContinuum().convert(
|
||||||
|
[(FakeVisualMod(), 1.0)], VAE(), routing="Prompt @R tags", chunk_count=2,
|
||||||
|
sequence_prompt="[0-4s]\n@R1 action\n\n[4-8s]\ncontinue")
|
||||||
|
self.assertEqual(bundle["reference_use"], "Per chunk")
|
||||||
|
self.assertEqual(bundle["selectors"][0], "1")
|
||||||
|
self.assertIn("decoded 1 visual", info)
|
||||||
|
finally:
|
||||||
|
sys.modules.pop("fake.v3.reference_images_v39", None)
|
||||||
|
|
||||||
|
@unittest.skipUnless(torch is not None, "PyTorch is only available in the ComfyUI runtime")
|
||||||
|
def test_refmods_to_continuum_skips_audio_only_rows(self):
|
||||||
|
bundle_module = types.SimpleNamespace(
|
||||||
|
ReferenceImagesV39Bundle=lambda **kwargs: kwargs)
|
||||||
|
sys.modules["fake.v3.reference_images_v39"] = bundle_module
|
||||||
|
try:
|
||||||
|
class VAE:
|
||||||
|
def decode(self, latent):
|
||||||
|
return torch.ones(1, 1, 4, 3)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "No visual RefMods"):
|
||||||
|
self.module.DumasH3RefModsToContinuum().convert(
|
||||||
|
[(FakeAudioMod(), 1.0)], VAE(), routing="All chunks")
|
||||||
|
finally:
|
||||||
|
sys.modules.pop("fake.v3.reference_images_v39", None)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user