Route RefMods through H3 prompt builder
This commit is contained in:
@@ -65,6 +65,12 @@ the chunk count, the shared chunk duration, and a per-chunk reference bundle.
|
|||||||
Reference images are inserted as `@R1` through `@R9`; routing is based on the
|
Reference images are inserted as `@R1` through `@R9`; routing is based on the
|
||||||
shared directive, soundscape, music, and each chunk's full prompt text.
|
shared directive, soundscape, music, and each chunk's full prompt text.
|
||||||
|
|
||||||
|
The builder also accepts an optional `H3_REF_MODS` input. Visual RefMods are
|
||||||
|
assigned after any connected image-reference slots, automatically described in
|
||||||
|
the generated shared directive, and emitted together with their slot offset.
|
||||||
|
Connect those two outputs to `DUMAS H3 RefMods to Continuum` so the bridge can
|
||||||
|
merge RefMods and ordinary image references into one Continuum bundle.
|
||||||
|
|
||||||
The prompt builder intentionally uses the `ReferenceImagesV39Bundle` class
|
The prompt builder intentionally uses the `ReferenceImagesV39Bundle` class
|
||||||
from the installed upstream H3 Continuum pack. Install that pack alongside
|
from the installed upstream H3 Continuum pack. Install that pack alongside
|
||||||
DumasNodes; DumasNodes does not copy or replace the upstream reference-image
|
DumasNodes; DumasNodes does not copy or replace the upstream reference-image
|
||||||
|
|||||||
+61
-11
@@ -33,6 +33,34 @@ def _continuum_reference_bundle_type():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _visual_refmods(mods):
|
||||||
|
"""Return positive-weight visual RefMods in stable picker order."""
|
||||||
|
result = []
|
||||||
|
seen = set()
|
||||||
|
for mod, strength in (mods or []):
|
||||||
|
if getattr(mod, "kind", None) == "audio" or float(strength) <= 0:
|
||||||
|
continue
|
||||||
|
key = str(getattr(mod, "path", "") or "") or (getattr(mod, "name", "refmod"), id(mod))
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
result.append((mod, float(strength)))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _refmod_directive(refmods, start_slot):
|
||||||
|
"""Build conservative default definitions for RefMods in the prompt."""
|
||||||
|
lines = []
|
||||||
|
for offset, (mod, _strength) in enumerate(refmods):
|
||||||
|
slot = start_slot + offset
|
||||||
|
name = str(getattr(mod, "name", "visual reference") or "visual reference").strip()
|
||||||
|
lines.append(
|
||||||
|
f"@R{slot} is the {name} visual reference; preserve its identity, "
|
||||||
|
"appearance, and important physical details when it is used."
|
||||||
|
)
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
MAX_CLIPS = 16
|
MAX_CLIPS = 16
|
||||||
DEFAULT_CLIP_SECONDS = 4.0
|
DEFAULT_CLIP_SECONDS = 4.0
|
||||||
|
|
||||||
@@ -162,8 +190,8 @@ class DumasH3SequencePromptBuilder:
|
|||||||
"the node emits one absolute Timeline prompt for Sequence Prompt."
|
"the node emits one absolute Timeline prompt for Sequence Prompt."
|
||||||
)
|
)
|
||||||
CATEGORY = "Dumas/MiniMax H3/Prompt"
|
CATEGORY = "Dumas/MiniMax H3/Prompt"
|
||||||
RETURN_TYPES = ("STRING", "INT", "FLOAT", REFERENCE_IMAGES_V39_TYPE)
|
RETURN_TYPES = ("STRING", "INT", "FLOAT", REFERENCE_IMAGES_V39_TYPE, "H3_REF_MODS", "INT")
|
||||||
RETURN_NAMES = ("sequence_prompt", "chunk_count", "chunk_duration", "reference_images")
|
RETURN_NAMES = ("sequence_prompt", "chunk_count", "chunk_duration", "reference_images", "ref_mods", "refmod_offset")
|
||||||
FUNCTION = "build"
|
FUNCTION = "build"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -179,11 +207,17 @@ class DumasH3SequencePromptBuilder:
|
|||||||
"music": ("STRING", {"default": "", "multiline": True}),
|
"music": ("STRING", {"default": "", "multiline": True}),
|
||||||
},
|
},
|
||||||
"optional": {
|
"optional": {
|
||||||
f"reference_image_{index}": ("IMAGE", {
|
**{
|
||||||
"display_name": f"Reference Image {index} (@R{index})",
|
f"reference_image_{index}": ("IMAGE", {
|
||||||
"tooltip": f"Fixed Reference Image {index}; use @R{index} in a clip prompt.",
|
"display_name": f"Reference Image {index} (@R{index})",
|
||||||
}) for index in range(1, 10)
|
"tooltip": f"Fixed Reference Image {index}; use @R{index} in a clip prompt.",
|
||||||
}
|
}) for index in range(1, 10)
|
||||||
|
},
|
||||||
|
"ref_mods": ("H3_REF_MODS", {
|
||||||
|
"display_name": "RefMods (assign @R slots after images)",
|
||||||
|
"tooltip": "Visual RefMods are described in the prompt and emitted for the Continuum bridge.",
|
||||||
|
}),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
def build(
|
def build(
|
||||||
@@ -202,21 +236,35 @@ class DumasH3SequencePromptBuilder:
|
|||||||
default_seconds=float(default_chunk_seconds),
|
default_seconds=float(default_chunk_seconds),
|
||||||
)
|
)
|
||||||
selectors = []
|
selectors = []
|
||||||
directive = str(shared_directive or "").strip()
|
routing_directive = str(shared_directive or "").strip()
|
||||||
|
directive = routing_directive
|
||||||
sound = str(soundscape or "").strip() or "N/A"
|
sound = str(soundscape or "").strip() or "N/A"
|
||||||
music_text = str(music or "").strip() or "N/A"
|
music_text = str(music or "").strip() or "N/A"
|
||||||
|
connected_image_slots = [
|
||||||
|
index for index in range(1, 10)
|
||||||
|
if kwargs.get(f"reference_image_{index}") is not None
|
||||||
|
]
|
||||||
|
refmods = _visual_refmods(kwargs.get("ref_mods"))
|
||||||
|
refmod_offset = max(connected_image_slots, default=0)
|
||||||
|
if refmod_offset + len(refmods) > 9:
|
||||||
|
raise ValueError("The prompt builder supports at most nine combined image and RefMod slots.")
|
||||||
|
generated = _refmod_directive(refmods, refmod_offset + 1)
|
||||||
|
if generated:
|
||||||
|
directive = "\n\n".join(part for part in (directive, generated) if part)
|
||||||
for reference_index in range(1, 10):
|
for reference_index in range(1, 10):
|
||||||
chunks = []
|
chunks = []
|
||||||
for chunk_index, clip in enumerate(sequence["clips"], start=1):
|
for chunk_index, clip in enumerate(sequence["clips"], start=1):
|
||||||
# Match against the same per-chunk content emitted by
|
# Match against the same per-chunk content emitted by
|
||||||
# format_sequence_prompt, not only the editable shot lines.
|
# format_sequence_prompt, not only the editable shot lines.
|
||||||
text = "\n".join([
|
text = "\n".join([
|
||||||
directive,
|
routing_directive,
|
||||||
*(str(slot.get("text", "")) for slot in clip["slots"]),
|
*(str(slot.get("text", "")) for slot in clip["slots"]),
|
||||||
f"overall_soundscape: {sound}",
|
f"overall_soundscape: {sound}",
|
||||||
f"non_diegetic_music: {music_text}",
|
f"non_diegetic_music: {music_text}",
|
||||||
])
|
])
|
||||||
if kwargs.get(f"reference_image_{reference_index}") is not None and re.search(
|
available = kwargs.get(f"reference_image_{reference_index}") is not None
|
||||||
|
available = available or refmod_offset < reference_index <= refmod_offset + len(refmods)
|
||||||
|
if available and re.search(
|
||||||
rf"@(R{reference_index}|image{reference_index})(?![A-Za-z0-9_])",
|
rf"@(R{reference_index}|image{reference_index})(?![A-Za-z0-9_])",
|
||||||
text,
|
text,
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
@@ -231,7 +279,7 @@ class DumasH3SequencePromptBuilder:
|
|||||||
)
|
)
|
||||||
return (
|
return (
|
||||||
format_sequence_prompt(
|
format_sequence_prompt(
|
||||||
shared_directive=shared_directive,
|
shared_directive=directive,
|
||||||
sequence=sequence,
|
sequence=sequence,
|
||||||
soundscape=soundscape,
|
soundscape=soundscape,
|
||||||
music=music,
|
music=music,
|
||||||
@@ -239,6 +287,8 @@ class DumasH3SequencePromptBuilder:
|
|||||||
len(sequence["clips"]),
|
len(sequence["clips"]),
|
||||||
float(default_chunk_seconds),
|
float(default_chunk_seconds),
|
||||||
reference_bundle,
|
reference_bundle,
|
||||||
|
kwargs.get("ref_mods") or [],
|
||||||
|
refmod_offset,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -241,6 +241,8 @@ class DumasH3RefModsToContinuum:
|
|||||||
},
|
},
|
||||||
"optional": {
|
"optional": {
|
||||||
"sequence_prompt": ("STRING", {"default": "", "multiline": True}),
|
"sequence_prompt": ("STRING", {"default": "", "multiline": True}),
|
||||||
|
"base_reference_images": ("H3_CONTINUUM_REFERENCE_IMAGES_V39",),
|
||||||
|
"reference_offset": ("INT", {"default": 0, "min": 0, "max": 8}),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -255,12 +257,22 @@ class DumasH3RefModsToContinuum:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def convert(self, mods, video_vae, routing="Prompt @R tags", chunk_count=1,
|
def convert(self, mods, video_vae, routing="Prompt @R tags", chunk_count=1,
|
||||||
sequence_prompt=""):
|
sequence_prompt="", base_reference_images=None, reference_offset=0):
|
||||||
if not mods:
|
if not mods and base_reference_images is None:
|
||||||
raise ValueError("The H3 RefMod bundle is empty.")
|
raise ValueError("The H3 RefMod bundle is empty.")
|
||||||
if routing not in ("All chunks", "Prompt @R tags"):
|
if routing not in ("All chunks", "Prompt @R tags"):
|
||||||
raise ValueError(f"Unknown RefMod routing mode: {routing!r}")
|
raise ValueError(f"Unknown RefMod routing mode: {routing!r}")
|
||||||
images, skipped_audio, ignored_strength = [], [], []
|
images, skipped_audio, ignored_strength = [], [], []
|
||||||
|
offset = int(reference_offset)
|
||||||
|
if not 0 <= offset <= 8:
|
||||||
|
raise ValueError("reference_offset must be between 0 and 8.")
|
||||||
|
if base_reference_images is not None:
|
||||||
|
base_images = getattr(base_reference_images, "images", None)
|
||||||
|
if not isinstance(base_images, (tuple, list)) or len(base_images) != 9:
|
||||||
|
raise ValueError("base_reference_images is not a valid Continuum V3.9 bundle.")
|
||||||
|
slots = list(base_images)
|
||||||
|
else:
|
||||||
|
slots = [None] * 9
|
||||||
seen = set()
|
seen = set()
|
||||||
for mod, _strength in mods:
|
for mod, _strength in mods:
|
||||||
if getattr(mod, "kind", None) == "audio":
|
if getattr(mod, "kind", None) == "audio":
|
||||||
@@ -275,20 +287,25 @@ class DumasH3RefModsToContinuum:
|
|||||||
continue
|
continue
|
||||||
if abs(strength - 1.0) > 1e-6:
|
if abs(strength - 1.0) > 1e-6:
|
||||||
ignored_strength.append(getattr(mod, "name", "refmod"))
|
ignored_strength.append(getattr(mod, "name", "refmod"))
|
||||||
if len(images) >= 9:
|
target = offset + len(images)
|
||||||
|
if target >= 9:
|
||||||
raise ValueError("Continuum V3.9 supports at most nine RefMod image slots.")
|
raise ValueError("Continuum V3.9 supports at most nine RefMod image slots.")
|
||||||
image = _decode_refmod_image(video_vae, mod)
|
image = _decode_refmod_image(video_vae, mod)
|
||||||
if image is not None:
|
if image is not None:
|
||||||
images.append(image)
|
images.append(image)
|
||||||
if not images:
|
if slots[offset + len(images) - 1] is not None:
|
||||||
|
raise ValueError("A RefMod slot overlaps an existing image reference.")
|
||||||
|
slots[offset + len(images) - 1] = image
|
||||||
|
if not images and base_reference_images is None:
|
||||||
raise ValueError("No visual RefMods were found; audio-only RefMods cannot feed image references.")
|
raise ValueError("No visual RefMods were found; audio-only RefMods cannot feed image references.")
|
||||||
selectors = (("all",) * 9 if routing == "All chunks"
|
selectors = (("all",) * 9 if routing == "All chunks"
|
||||||
else _prompt_selectors(sequence_prompt, int(chunk_count)))
|
else _prompt_selectors(sequence_prompt, int(chunk_count)))
|
||||||
slots = tuple(images) + (None,) * (9 - len(images))
|
|
||||||
bundle = _continuum_reference_bundle_type()(
|
bundle = _continuum_reference_bundle_type()(
|
||||||
images=slots, reference_use="Per chunk", selectors=selectors,
|
images=tuple(slots), reference_use="Per chunk", selectors=selectors,
|
||||||
)
|
)
|
||||||
details = [f"decoded {len(images)} visual RefMod(s) into Continuum slots R1-R{len(images)}"]
|
details = [f"decoded {len(images)} visual RefMod(s) into Continuum slots starting at R{offset + 1}"]
|
||||||
|
if base_reference_images is not None:
|
||||||
|
details.append("merged with builder image references")
|
||||||
details.append("routing=all chunks" if routing == "All chunks" else "routing=prompt @R tags")
|
details.append("routing=all chunks" if routing == "All chunks" else "routing=prompt @R tags")
|
||||||
if skipped_audio:
|
if skipped_audio:
|
||||||
details.append("skipped audio-only: " + ", ".join(skipped_audio))
|
details.append("skipped audio-only: " + ", ".join(skipped_audio))
|
||||||
|
|||||||
@@ -62,6 +62,12 @@ class FakeReferenceBundle:
|
|||||||
selectors: tuple
|
selectors: tuple
|
||||||
|
|
||||||
|
|
||||||
|
class FakeRefMod:
|
||||||
|
kind = "video"
|
||||||
|
name = "hero_ref"
|
||||||
|
path = ""
|
||||||
|
|
||||||
|
|
||||||
class H3NodeTests(unittest.TestCase):
|
class H3NodeTests(unittest.TestCase):
|
||||||
def test_prompt_schema_and_full_chunk_reference_routing(self):
|
def test_prompt_schema_and_full_chunk_reference_routing(self):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -88,6 +94,23 @@ class H3NodeTests(unittest.TestCase):
|
|||||||
finally:
|
finally:
|
||||||
sys.modules.pop(reference_module.__name__, None)
|
sys.modules.pop(reference_module.__name__, None)
|
||||||
|
|
||||||
|
def test_prompt_builder_defines_connected_refmods_and_emits_them(self):
|
||||||
|
reference_module = types.ModuleType("upstream.v3.reference_images_v39")
|
||||||
|
reference_module.ReferenceImagesV39Bundle = FakeReferenceBundle
|
||||||
|
sys.modules[reference_module.__name__] = reference_module
|
||||||
|
try:
|
||||||
|
result = PROMPT.DumasH3SequencePromptBuilder().build(
|
||||||
|
2, 4, "", '{"clips":[{"slots":[{"start":0,"end":4,"text":"Use @R1."}]},'
|
||||||
|
'{"slots":[{"start":0,"end":4,"text":"Continue with @R1."}]}]}',
|
||||||
|
"", "", ref_mods=[(FakeRefMod(), 1.0)],
|
||||||
|
)
|
||||||
|
self.assertEqual(result[4][0][0].name, "hero_ref")
|
||||||
|
self.assertEqual(result[5], 0)
|
||||||
|
self.assertIn("@R1 is the hero_ref visual reference", result[0])
|
||||||
|
self.assertEqual(result[3].selectors[0], "1,2")
|
||||||
|
finally:
|
||||||
|
sys.modules.pop(reference_module.__name__, None)
|
||||||
|
|
||||||
def test_per_clip_upscale_uses_connected_sigmas(self):
|
def test_per_clip_upscale_uses_connected_sigmas(self):
|
||||||
original = UPSCALE._native_node
|
original = UPSCALE._native_node
|
||||||
original_helper = UPSCALE._continuum_helper
|
original_helper = UPSCALE._continuum_helper
|
||||||
|
|||||||
Reference in New Issue
Block a user