diff --git a/README.md b/README.md index af457c4..55f7966 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,22 @@ - Outputs: `plan`, `image1`..`image9`, `connected_images` - Reads back the nine optional images for a selected MiniMax H3 plan scene, for example by connecting the current `clip_index`. +- `Dumas H3 Long Videos (FL2VA + REF2VA)` + - Inputs: H3 model stack, prompt socket, optional `first_frame`, optional `ref_image_1`..`ref_image_4`, plus the upstream long-video control surface for pacing, continuity, audio, overlays, and guards + - Outputs: `images`, `audio`, `info`, `script`, `frames_per_shot`, `total_frames`, `shots`, `video_seconds`, `fps`, `fps_int`, `latent`, `soundscape` + - First-pass Dumas port of the `MiniMax-H3-Longvideos` sampler, brought in as a local starting point for long-form H3 chaining work. + - Keeps the upstream split-beats / handoff / ref-routing behavior close to source so future Dumas-specific improvements can be compared against a known baseline. + +- `Dumas H3 Shot Length` + - Inputs: `shot_seconds`, `fps`, optional `cap_to_h3_max` + - Outputs: `seconds`, `frames`, `info` + - Emits one H3-safe shot length as both seconds and a grid-aligned frame count for wiring into the long-video sampler and preview helpers. + +- `Dumas H3 Model Inspector` + - Input: `model` + - Outputs: `format`, `report` + - Reports the detected H3 base precision / quant format and the relevant compute-capability hints for the current card. + - `Dumas Character Helper` - Inputs: `image1`, `image2`, `image1_picture_id`, `image2_picture_id`, `character_id`, `name`, `alias`, `gender`, `age`, `nationality`, `occupation`, `height_feet`, `height_inches`, `accent`, `general` - Outputs: `image1`, `image2`, `character_text` diff --git a/__init__.py b/__init__.py index 02d4188..b9693db 100644 --- a/__init__.py +++ b/__init__.py @@ -10,14 +10,32 @@ from .dumas_json_nodes import ( NODE_CLASS_MAPPINGS as JSON_NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS as JSON_NODE_DISPLAY_NAME_MAPPINGS, ) +from .dumas_h3_longvideos import ( + NODE_CLASS_MAPPINGS as H3_LONGVIDEO_NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS as H3_LONGVIDEO_NODE_DISPLAY_NAME_MAPPINGS, +) +from .dumas_h3_shot_length import ( + NODE_CLASS_MAPPINGS as H3_SHOT_LENGTH_NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS as H3_SHOT_LENGTH_NODE_DISPLAY_NAME_MAPPINGS, +) +from .dumas_h3_inspector import ( + NODE_CLASS_MAPPINGS as H3_INSPECTOR_NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS as H3_INSPECTOR_NODE_DISPLAY_NAME_MAPPINGS, +) NODE_CLASS_MAPPINGS = {} NODE_CLASS_MAPPINGS.update(JSON_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(IMAGE_NODE_CLASS_MAPPINGS) +NODE_CLASS_MAPPINGS.update(H3_LONGVIDEO_NODE_CLASS_MAPPINGS) +NODE_CLASS_MAPPINGS.update(H3_SHOT_LENGTH_NODE_CLASS_MAPPINGS) +NODE_CLASS_MAPPINGS.update(H3_INSPECTOR_NODE_CLASS_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS = {} NODE_DISPLAY_NAME_MAPPINGS.update(JSON_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(IMAGE_NODE_DISPLAY_NAME_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(H3_LONGVIDEO_NODE_DISPLAY_NAME_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(H3_SHOT_LENGTH_NODE_DISPLAY_NAME_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(H3_INSPECTOR_NODE_DISPLAY_NAME_MAPPINGS) WEB_DIRECTORY = "./js" diff --git a/dumas_h3_inspector.py b/dumas_h3_inspector.py new file mode 100644 index 0000000..55a98f9 --- /dev/null +++ b/dumas_h3_inspector.py @@ -0,0 +1,177 @@ +""" +H3 Model Inspector (detect the base precision / quant format) +============================================================== +Reads the loaded MODEL and reports which precision/quant format the H3 DiT is +stored in: BF16, FP8 (e4m3 / e5m2), INT8 (+ convrot), NVFP4, MXFP8, +ConvRot-W4A4, W4A8, or a mix. Report-only — a manual hint you read and act on. + +WHY IT'S FUTURE-PROOF (incl. MXFP8 "once one comes out") +-------------------------------------------------------- +It doesn't sniff dtypes and guess. ComfyUI tags every quantized layer at load +with module.quant_format, using fixed strings it already recognizes: + nvfp4, mxfp8, float8_e4m3fn, float8_e5m2, int8_tensorwise, convrot_w4a4, + asym_w4a8_int8 (see comfy/ops.py). +This node reads that tag. MXFP8 is already a recognized format in ComfyUI +(comfy/ops.py + comfy/float.py + model_management.supports_mxfp8_compute), so +the day someone ships an MXFP8 H3 checkpoint, this node labels it correctly +with no change. Any brand-new tag lands under "other: " instead of +crashing, so it degrades gracefully. + +It also reports whether YOUR card can run NVFP4 / MXFP8 natively +(model_management.supports_nvfp4_compute / supports_mxfp8_compute). + +NOT detected: pruned vs full (that's an architecture axis — factorized AdaLN — +not a quant format). Reported as a caveat, not guessed. + +INSTALL: drop into ComfyUI/custom_nodes/, restart. +Node: MiniMax-H3 -> Model Inspector. +""" + +# ---- pure helpers (no torch; unit-testable) ------------------------------- +_FRIENDLY = { + "float8_e4m3fn": "FP8 (e4m3)", + "float8_e5m2": "FP8 (e5m2)", + "mxfp8": "MXFP8", + "nvfp4": "NVFP4", + "int8_tensorwise": "INT8", + "int8_tensorwise+convrot": "INT8 convrot", + "convrot_w4a4": "ConvRot W4A4 (int4)", + "asym_w4a8_int8": "W4A8 (int4/int8)", + "bf16": "BF16", + "fp16": "FP16", +} + +# implication note per format, tied to a Blackwell 16GB context +_IMPLICATION = { + "NVFP4": "native on Blackwell (sm_120); half the size of INT8.", + "MXFP8": "needs Blackwell + torch >= 2.10 for native compute.", + "FP8 (e4m3)": "fp8 storage; runs on Ada/Blackwell.", + "FP8 (e5m2)": "fp8 storage; runs on Ada/Blackwell.", + "INT8": "int8 storage.", + "INT8 convrot": "int8+ConvRot — needs working sm_120 kernels (absent on some 50-series setups).", + "ConvRot W4A4 (int4)": "4-bit ConvRot; requires the matching custom nodes/branch.", + "W4A8 (int4/int8)": "4-bit weight / 8-bit activation.", + "BF16": "full precision; largest footprint, cleanest LoRA apply.", + "FP16": "half precision.", +} + + +def friendly(fmt): + return _FRIENDLY.get(fmt, f"other: {fmt}") + + +def summarize(counts): + """counts: {raw_format: n}. Returns (label, per_format_summary_lines). + Label = the dominant NON-bf16/fp16 quant format if any (the main blocks), + else the dominant plain dtype.""" + quant = {k: v for k, v in counts.items() if k not in ("bf16", "fp16")} + lines = [] + for raw, n in sorted(counts.items(), key=lambda kv: -kv[1]): + lines.append(f" {friendly(raw)}: {n} layer(s)") + if quant: + top = max(quant.items(), key=lambda kv: kv[1])[0] + label = friendly(top) + elif counts: + top = max(counts.items(), key=lambda kv: kv[1])[0] + label = friendly(top) + else: + label = "unknown" + return label, lines + + +# ---- ComfyUI node --------------------------------------------------------- +def _detect(model): + """Walk the DiT modules, tally quant_format tags (and dtype for the rest). + Returns (label, counts, report_lines). Imports torch/mm lazily so the pure + helpers above stay importable without a ComfyUI runtime.""" + import torch + import comfy.model_management as mm + + # locate the diffusion model inside the ModelPatcher + dm = getattr(getattr(model, "model", None), "diffusion_model", None) + if dm is None: + dm = getattr(model, "model", None) or model + + def dtype_label(dt): + return { + torch.bfloat16: "bf16", torch.float16: "fp16", + torch.float8_e4m3fn: "float8_e4m3fn", torch.float8_e5m2: "float8_e5m2", + torch.int8: "int8_tensorwise", + }.get(dt, str(dt).replace("torch.", "")) + + counts = {} + if hasattr(dm, "modules"): + for m in dm.modules(): + fmt = getattr(m, "quant_format", None) + if fmt is not None: + # distinguish int8 convrot via the packed weight's params + if fmt == "int8_tensorwise": + params = getattr(getattr(m, "weight", None), "_params", None) + if getattr(params, "convrot", False): + fmt = "int8_tensorwise+convrot" + counts[fmt] = counts.get(fmt, 0) + 1 + continue + w = getattr(m, "weight", None) + if w is not None and hasattr(w, "dtype"): + counts[dtype_label(w.dtype)] = counts.get(dtype_label(w.dtype), 0) + 1 + + label, lines = summarize(counts) + + # hardware capability for the relevant 4-bit/8-bit formats + try: + nv = mm.supports_nvfp4_compute() + except Exception: + nv = None + try: + mx = mm.supports_mxfp8_compute() + except Exception: + mx = None + + report = [f"Detected base precision: {label}"] + report += lines + impl = _IMPLICATION.get(label) + if impl: + report.append(f" -> {impl}") + report.append(f" card supports NVFP4 compute: {nv}; MXFP8 compute: {mx}") + if label == "MXFP8" and mx is False: + report.append(" WARNING: MXFP8 file but this card/torch can't run it natively.") + if label == "NVFP4" and nv is False: + report.append(" WARNING: NVFP4 file but this card can't run it natively.") + report.append(" (pruned-vs-full is a separate architecture axis; not detected here.)") + return label, counts, "\n".join(report) + + +class H3ModelInspector: + CATEGORY = "Dumas/MiniMax" + FUNCTION = "inspect" + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("format", "report") + OUTPUT_NODE = True + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"model": ("MODEL",)}} + + def inspect(self, model): + label, _counts, report = _detect(model) + return (label, report) + + +NODE_CLASS_MAPPINGS = {"DumasH3ModelInspector": H3ModelInspector} +NODE_DISPLAY_NAME_MAPPINGS = {"DumasH3ModelInspector": "Dumas H3 Model Inspector"} +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] + + +if __name__ == "__main__": + # exercise the pure aggregation logic with mocked layer tallies + cases = { + "NVFP4 file (200 main + bf16 rest)": {"nvfp4": 200, "bf16": 132}, + "INT8 convrot": {"int8_tensorwise+convrot": 170, "bf16": 30}, + "plain bf16": {"bf16": 340}, + "FP8": {"float8_e4m3fn": 200, "bf16": 140}, + "MXFP8 (future file)": {"mxfp8": 200, "bf16": 132}, + "some unknown new tag": {"fp6_e3m2": 200, "bf16": 132}, + } + for name, counts in cases.items(): + label, lines = summarize(counts) + print(f"{name:38s} -> {label}") diff --git a/dumas_h3_longvideos.py b/dumas_h3_longvideos.py new file mode 100644 index 0000000..cb4dce4 --- /dev/null +++ b/dumas_h3_longvideos.py @@ -0,0 +1,6445 @@ +""" +H3 Long Videos (one prompt + one length -> long video+audio) +============================================================= +One node covering both of H3's conditioning tasks: + + * FL2VA -- a first frame (or the previous shot's last frame) anchors the shot. + This is what drives the chain: each shot continues the one before it. + * REF2VA -- reference images condition the shot on what a character LOOKS like, + independent of any frame. + +Connect nothing to ref_image_* and it behaves exactly as the FL2VA node always +did. Connect a reference and `ref_mode` decides which shots use it. + +THE ONE RULE: a shot carries EITHER references or the last-frame handoff, never +both. They are two task conditionings competing for the same cond_video_latents +slot in comfy/model_base.py -- the refs branch overwrites what the keyframe branch +wrote, while the packed layout still reserves rows for both, so a shot given both +hands the DiT fewer latents than it has condition rows. `ref_mode` chooses: + + first shot -- refs establish the cast in shot 1, every later shot + uses the handoff. Continuity unbroken. + every shot -- strongest identity; no handoff, so beats meet as + CUTS rather than as one continuous take. + every shot + handoff ref -- refs every shot, plus the previous shot's last + frame as one more reference. Continuity returns as + a soft signal. + +You give it a prompt (first paragraph = the look/character kept across the whole +video; each later paragraph = a scene beat), a shot length, and a resolution from +the VRAM-appropriate list. It splits the beats into shots that fit H3's ceiling +and your VRAM, chains them, and returns the finished video + audio. + +Requirements: H3 is CFG-free (cfg 1) and needs no negative prompt -- the node +makes an empty one internally. denoise is fixed at 1.0: a partial denoise desyncs +the joint audio/video schedule. + +Verified against ComfyUI core (comfy_extras/nodes_minimax_h3.py, model_base.py, +ldm/minimax/model.py, text_encoders/minimax.py, sd.py). +""" + +import gc +import json +import logging +import math +import os +import re +import torch + +import nodes +import comfy.utils +import comfy.samplers +import comfy.nested_tensor +import comfy.model_management as mm +import node_helpers + +try: + from . import dumas_h3_overlay as _overlay +except ImportError: # loaded as a bare file (test_prompt_logic.py), not as a package + import importlib.util as _ilu + import os as _os + _spec = _ilu.spec_from_file_location( + "dumas_h3_overlay", + _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "dumas_h3_overlay.py"), + ) + _overlay = _ilu.module_from_spec(_spec) + _spec.loader.exec_module(_overlay) + +AUDIO_LATENT_FPS = 40 +GB = 1024 ** 3 +H3_MAX_FRAMES = 362 +# H3's temporal grid is FIXED at 24 fps -- comfy_extras/nodes_minimax_h3.py hard-codes +# FPS = 24, and the audio latent length is derived from frame_count / 24. The model +# emits 24 fps content no matter what any node asks for, so every seconds<->frames +# conversion here MUST use 24. Treating it as a variable is what made a requested +# 10s shot render 124 frames (~5.2s of real time) when the widget said 12. +H3_FPS = 24 +MIN_SHOT_FRAMES = 124 # internal VRAM floor (~5s @24fps) + + +# --- H3 geometry ----------------------------------------------------------- +def align_frame_count(n): + while n % 17 != 5: + n += 1 + return n + + +def align_frame_count_nearest(n): + """Nearest 17k+5 grid point, not the next one up. + + align_frame_count always rounds UP, which is right when honoring a length that + was ASKED for -- never give back less than requested. It is wrong for an + ESTIMATE: the grid steps 17 frames (~0.7s), so rounding up added up to 0.7s to + every content-sized shot and pushed a 9.5s estimate out to 10.1s. Pacing leans + short on purpose; rounding should not quietly lean the other way.""" + n = max(5, int(n)) + lo = n - ((n - 5) % 17) + hi = lo + 17 + return lo if (n - lo) <= (hi - n) else hi + + +def video_latent_t(fc): + return 2 if fc <= 5 else ((fc - 5) // 17) * 5 + 2 + + +def temporal_shape(length, fps=H3_FPS): + """`fps` is accepted for call-site compatibility but deliberately IGNORED: the + audio latent must line up with 24 fps video or the shot's sound is stretched + against its picture.""" + fc = align_frame_count(max(5, length)) + return fc, video_latent_t(fc), round(fc / H3_FPS * AUDIO_LATENT_FPS) + + +def res_down(w, h, factor=0.85, mult=32): + snap = lambda v: max(mult, round(v * factor / mult) * mult) + return snap(w), snap(h) + + +# --- native 768p canvas per ratio (H3-Base renders at 768 short edge) ------ +# H3-Base's native/trained resolution is 768 on the short edge; rendering below +# it softens the whole frame (faces worst). So resolution is ALWAYS kept native +# and never traded down for VRAM -- when the card is tight, SHOT LENGTH shrinks +# instead (see estimate_shot_frames). The 768*1344 area cap means very wide +# ratios (21:9) land just under 768 short edge natively. +NATIVE_RES = { + "16:9": (1344, 768), + "9:16": (768, 1344), + "4:3": (1024, 768), + "3:4": (768, 1024), + "1:1": (768, 768), + "21:9": (1536, 672), + "9:21": (672, 1536), +} +# 512-short-edge "fast" tier: ~4x fewer pixels than native, for the generate-low- +# then-upscale (LTX 2.3) workflow. Best for close/medium shots -- H3 distorts faces +# on WIDE shots at any resolution, so keep faces reasonably large in frame. +FAST_RES = { + "16:9": (896, 512), + "9:16": (512, 896), + "4:3": (704, 512), + "3:4": (512, 704), + "1:1": (512, 512), + "21:9": (1184, 512), +} +# 640-short-edge "balanced" tier: a middle ground between fast 512 and native 768. +MID_RES = { + "16:9": (1152, 640), + "9:16": (640, 1152), + "4:3": (864, 640), + "3:4": (640, 864), + "1:1": (640, 640), + "21:9": (1504, 640), +} +NATIVE_PIXELS = 1344 * 768 # ~1MP reference for the VRAM/length budget +# Shot-length budget fit (see estimate_shot_frames). Measured anchors on a 16GB +# card: at 1344x768 with the pruned NVFP4 DiT, 243f fits (~2.7GB spare) and 362f +# overflowed by ~4.3GB -> slope (362-243)/(7.0-2.7) ~= 27.7 frames per GB. The +# baseline absorbs the part of the latent that fits in already-counted space. +# Refit against BOTH measured points at once: 243f must be reachable at the 640p / +# 13.6GB case (~1.1GB scaled spare) and 362f must NOT be until ~7GB. That gives +# slope (362-243)/(7.0-1.12) ~= 20.2 f/GB with a 10.91GB baseline. The native +# NVFP4 case then lands at 260f -- above the 243f measured safe and below the 362f +# measured overflow, i.e. consistent with both rather than fitted to either. +FRAMES_PER_GB = 20.2 +# Fraction of free VRAM held back for transient activation peaks during sampling +# (the steady-state latent is not the high-water mark). Prevents the node from +# picking a length that fits on paper but spills into shared memory mid-shot. +SPIKE_RESERVE = 0.12 +FRAMES_BASELINE_GB = 10.91 + + +def resolution_options(): + """ASPECT RATIOS only. `megapixels` decides the size. + + Shape and size are independent, and the widgets now say so. This list used to + carry three short-edge tiers per ratio (native 768 / balanced 640 / fast 512), + which baked a SIZE into every label -- and once megapixels existed those labels + lied whenever it was on. Nothing is lost: the tiers were three points on the + megapixel axis (~0.98 / ~0.70 / ~0.44MP), and a continuous control reaches them + and everything between. MID_RES and FAST_RES are kept only as the reference + anchors documented on them. + + NATIVE_RES still supplies each ratio's exact shape, which is what makes 1.00MP + reproduce H3's own sizes: the ratio NAMES are approximations -- 1344x768 is + 1.750, i.e. 7:4, NOT 16:9 (1.778) -- so scaling runs from the real dimensions + rather than the nominal ratio.""" + return list(NATIVE_RES) + + +def parse_resolution(choice): + """The chosen ratio's reference dimensions, which `megapixels` then scales. + + Accepts a bare ratio ("16:9") and ALSO the old "16:9 - 1344x768 (native)" form, + so a workflow saved before this list was simplified still resolves to the right + shape instead of silently falling back to the first entry. Unrecognized input + gives 16:9.""" + text = (choice or "").strip() + if text in NATIVE_RES: + return NATIVE_RES[text] + m = re.search(r"(\d+)\s*x\s*(\d+)", text) # legacy label carried its size + if m: + return int(m.group(1)), int(m.group(2)) + for r in NATIVE_RES: # legacy label led with the ratio + if text.startswith(r): + return NATIVE_RES[r] + return NATIVE_RES["16:9"] + + +# --- sizing by pixel budget ------------------------------------------------- +# Cost and training-distribution match are functions of TOKEN COUNT -- +# (h/16)*(w/16)*frames -- which tracks total pixels, not the short edge. A +# short-edge target makes two aspect ratios look comparable when they are not: +# +# 1:1 768x768 short edge 768, reads native -> 0.56 MP, 43% under +# 21:9 1536x672 short edge 672, reads sub-native -> 0.98 MP, full budget +# +# Scaling from the PRESET's own dimensions (rather than from a nominal ratio) is +# what makes 1.00MP reproduce each preset's native size exactly. That distinction +# is real: 1344x768 is 1.750, i.e. 7:4 -- NOT 16:9, which is 1.778 -- and +# 1536x672 is 16:7, not 21:9. Computing from a nominal 16:9 lands on 1376x768 and +# never reproduces the native. +# +# This does NOT change the sigma schedule. H3's shift is a fixed 12.0 in its model +# config with no resolution-dependent term, unlike Flux/SD3 dynamic shifting. +MP_UNIT = 1024 * 1024 # 1 MP == 1024x1024, matching ComfyUI's own convention +RES_MULTIPLE = 32 # every shipped preset is a multiple of 32 + + +def scale_to_megapixels(w, h, mp, multiple=RES_MULTIPLE): + """Resize (w, h) to hit `mp` megapixels, keeping the aspect ratio. + + Constant-area square root, snapped to `multiple` on both axes -- which is what + H3's patchified latent grid needs. Snapping moves the real area off the request + slightly, so callers report the ACHIEVED value: a readout of what was asked for + hides what was produced. mp <= 0 means "leave the preset alone".""" + if not mp or mp <= 0 or w <= 0 or h <= 0: + return int(w), int(h) + multiple = max(1, int(multiple)) + scale = math.sqrt((float(mp) * MP_UNIT) / float(w * h)) + nw = max(multiple, int(round(w * scale / multiple)) * multiple) + nh = max(multiple, int(round(h * scale / multiple)) * multiple) + return nw, nh + + +# --- prompt parsing + auto time distribution ------------------------------- +def split_paragraphs(text, delimiter): + raw = text.replace("\r\n", "\n").strip() + if not raw: + return [] + raw = re.sub(r"(?m)^\s*" + re.escape(delimiter) + r"\s*$", "\n\n", raw) + return [p.strip() for p in re.split(r"\n\s*\n", raw) if p.strip()] + + +# Widgets added after the node's original 36-widget layout. Kept LAST in +# INPUT_TYPES so a workflow saved before they existed still maps its stored values +# onto the right widgets (ComfyUI matches them by position, not by name). +# APPEND to this tuple when adding a widget; never insert into the middle. +ADDED_WIDGETS = ( + "beat_split", "per_beat_length", + "watermark_text", "watermark_position", "watermark_size", "watermark_opacity", + "watermark_margin", "intro_text", "intro_position", "intro_seconds", + "intro_fade", "intro_size", "overlay_font", "overlay_stroke", + "ref_mode", "ref_image_size", "ref_noise_aug", "auto_props", "prevent_nudity", + "exposed_terms", "anatomy_guard", "lock_restraints", "solidity_guard", + "motion_guard", "contact_guard", + "auto_soundscape", "allow_nonspeech_vocals", +) + +NL = "\n" +# Lines that CONFIGURE a beat rather than being one. They attach to the beat that +# follows them, so a line-split never turns "wardrobe: ..." into its own shot. +DIRECTIVE_KEYS = ("wardrobe", "seconds", "duration", "exit", "enter", + "overall_soundscape", "non_diegetic_music", "soundscape", "music") + + +def is_directive_line(line): + return bool(re.match(r"\s*(" + "|".join(DIRECTIVE_KEYS) + r")\s*:", line or "", re.I)) + + +def expand_beats(paras, mode="auto"): + """Turn the prompt's beat PARAGRAPHS into the final beat list. Returns + (beats, note). + + Beats are separated by a BLANK line (or a '##' line). That is unambiguous, but + it is also the single easiest thing to get wrong in a textarea: six beats typed + on six consecutive lines are one paragraph, so they render as ONE shot with six + actions crammed into it -- which reads as characters moving at triple speed, not + as a splitting problem. + + mode: + 'auto' -- blank lines first; any paragraph still holding more than one + content line is then split one beat per line, and says so. + 'each line' -- every content line is its own beat. Same result as 'auto'; kept + so the intent can be stated explicitly. + + There is deliberately no strict blank-lines-only mode any more. It was the ONE + setting that could silently lose beats: six beats typed as two blocks of three + rendered as two shots, with no note to say why, because the split note is only + written when a paragraph is actually split. Nothing else on the node can change + the beat count, so removing that option removes the whole failure class. A + workflow that still stores 'blank line' falls through to 'auto' below. + + Directive lines ('wardrobe:', 'seconds:', 'exit:' ...) are never beats of their + own: they attach to the next content line, or to the previous beat if they + trail the paragraph.""" + # Any unrecognized mode means AUTO, never "do nothing". An earlier version fell + # through an if/elif with no else and silently DROPPED every multi-line paragraph + # -- six beats arrived as two shots with four beats simply gone. A stale value on + # this widget (including 'blank line' from a workflow saved before it was removed) + # is enough to trigger it, so the safe branch has to be the default. + if mode not in ("auto", "each line"): + mode = "auto" + out, split_from = [], 0 + for p in paras: + lines = [ln for ln in (p or "").splitlines() if ln.strip()] + content = [ln for ln in lines if not is_directive_line(ln)] + if len(content) <= 1: + out.append(p) + else: + split_from += 1 + pending = [] + for ln in lines: + if is_directive_line(ln): + # Hold it for the NEXT content line: a directive reads as a header + # for the beat it introduces ("wardrobe: -= jacket" / "she shrugs + # it off"). Only if nothing follows does it fall back to the beat + # above, handled after the loop. + pending.append(ln) + continue + out.append(NL.join(pending + [ln])) + pending = [] + if pending: # directives with no beat after them + if out: + out[-1] = out[-1] + NL + NL.join(pending) + else: + out.append(NL.join(pending)) + note = "" + if split_from and mode == "auto": + note = (f"{split_from} paragraph(s) held several lines and were split one beat per LINE " + f"-> {len(out)} beats. Separate beats with a BLANK line (or a '##' line) to control " + f"this yourself") + return out, note + + +def _garment_side(side): + """Is `side` one garment, for the purpose of splitting an 'A and B' entry? + + Qualifies when its head noun is a garment the zone tables know, or when it is + long enough to be a described item rather than a bare adjective. The zone + vocabulary is looked up at call time because it is defined further down.""" + words = side.split() + if not words: + return False + vocab = _ZONE_LOWER | _ZONE_UPPER | _ZONE_BOTH + return _item_head(side) in vocab or len(words) >= 2 + + +def _split_conjoined(item): + """Split 'a black jacket and blue jeans' into two tracked garments. + + A sheet entry joined by 'and' was kept as ONE item, and _item_name() truncates + at 'and' -- so "small white t-shirt and shiny white lace thong" became the item + "small white t-shirt" with head `t-shirt`, and the thong was not tracked at all. + "pulls down the thong" then matched nothing, the removal silently did not fire, + and the whole compound string -- thong included -- was re-stamped into the + character's parenthetical on every later shot. The prompt kept saying she was + wearing it, so the model kept putting it back. + + Only split when BOTH sides read as garments and at least one is a garment the + zone tables recognise. That last condition is what keeps colour pairs and + ordinary attributes intact: "black and white dress" has 'black' on the left, + which is neither a known garment nor two words, and "blonde hair and blue eyes" + has no garment on either side. Not splitting is always safe -- the left-hand + garment stays tracked through _item_name()'s truncation either way -- so this + is purely additive: it can only start tracking a garment that was invisible.""" + parts = [p.strip(" .;") for p in re.split(r"\s+and\s+", item or "", flags=re.I)] + parts = [p for p in parts if p] + if len(parts) < 2 or not all(_garment_side(p) for p in parts): + return [item] + vocab = _ZONE_LOWER | _ZONE_UPPER | _ZONE_BOTH + if not any(_item_head(p) in vocab for p in parts): + return [item] + return parts + + +def _split_items(s): + """Split a description into attribute items on commas AND sentence ends. + + A sheet written naturally ends clauses with a period -- "wearing a black t-shirt + and jeans. Mouth closed." -- and treating that as ONE item drags a whole sentence + into the inline parenthetical, which then reads as its own statement about a + person rather than an attribute of the pronoun. Splitting on '.' as well keeps + each item a short attribute. + + Garments joined by 'and' are then separated so each is independently removable; + see _split_conjoined().""" + out = [] + for i in re.split(r"[,.;]", s or ""): + i = i.strip(" .;") + if i: + out += _split_conjoined(i) + return out + + +def _norm_name(name): + """Normalize a person key: trim whitespace and a trailing ':' so 'Kristy:' + and 'Kristy' are the same person (the colon is natural to type because it's + how the sheet renders back).""" + return name.strip().rstrip(":").strip() + + +def _split_name(part): + """Split 'Name = items' or 'Name: items' into (name, items_str). The name is + bound by '=' or a leading 'Name:' (the token before ':' must have no comma, + so a plain clothing list like 'grey shorts, red jacket' stays unnamed). + Returns ('', part) when there's no name binder.""" + if "=" in part: + name, desc = part.split("=", 1) + return _norm_name(name), desc + if ":" in part: + head, tail = part.split(":", 1) + if "," not in head: # a name won't contain a comma + return head.strip(), tail + return "", part + + +def _entries(text): + """Split a wardrobe sheet into per-person entries. Accepts BOTH ';' and + NEWLINES as separators -- character_memory is a multiline box, so one person + per line is the natural way to write it, and silently mis-parsing that (folding + the next person into the previous one's item list) breaks name lookup and makes + removals fail to match. Also tolerates a leading '-' bullet per line.""" + parts = [] + for chunk in re.split(r"[;\n\r]+", text or ""): + chunk = chunk.strip().lstrip("-*\u2022 ").strip() + if chunk: + parts.append(chunk) + return parts + + +def parse_wardrobe(text): + """Parse an INITIAL wardrobe sheet into an ordered {name: [items]} dict, so + people are tracked independently and individual garments can be added or + removed later. Entries split on ';' OR newlines; each is 'Name = a, b' OR + 'Name: a, b' (colon works too). An entry with no name binder is the single + unnamed subject under '' (one-person, backward-compat).""" + out = {} + for part in _entries(text): + name, desc = _split_name(part) + out[name] = _split_items(desc) + return out + + +def apply_wardrobe_change(active, text): + """Apply a per-beat 'wardrobe:' directive so you DON'T restate the whole + outfit to change one thing. Entries split on ';'; each targets one person + (or the unnamed subject) with an operator: + Name = a, b replace that person's whole outfit + Name += c, d ADD items + Name -= jacket REMOVE items whose text contains any given token + The Name may be written with or without a trailing colon ('Maya' or 'Maya:' + both work). Bare forms (no Name) target the single unnamed subject: '= a,b', + '+= hat', '-= jacket'. Names not mentioned are left untouched. So dropping a + jacket is just 'wardrobe: Maya -= jacket' -- one token, nothing re-typed.""" + active = {k: list(v) for k, v in active.items()} + for part in _entries(text): + if "+=" in part: + name, val, op = (*part.split("+=", 1), "+") + elif "-=" in part: + name, val, op = (*part.split("-=", 1), "-") + elif "=" in part: + name, val, op = (*part.split("=", 1), "=") + else: + name, val = _split_name(part); op = "=" # bare or 'Name: items' -> replace + name = _norm_name(name) + items = _split_items(val) + cur = active.get(name, []) + if op == "=": + active[name] = items + elif op == "+": + active[name] = cur + [i for i in items if i.lower() not in (c.lower() for c in cur)] + elif op == "-": + toks = [t.lower() for t in items] + active[name] = [it for it in cur if not any(t in it.lower() for t in toks)] + return active + + +_PRO = {"she": "she", "her": "she", "hers": "she", + "he": "he", "him": "he", "his": "he", + "they": "they", "them": "they", "their": "they", "theirs": "they"} +_GENDER = {"woman": "she", "women": "she", "female": "she", "girl": "she", "lady": "she", + "man": "he", "male": "he", "boy": "he", "guy": "he", "gentleman": "he"} + + +def _pronoun_of(items): + """A person's pronoun, from an explicit token in their sheet ('she') or a + gender word in their description ('woman'). None if undeclared/undetectable.""" + for it in items: + if it.strip().lower() in _PRO: + return _PRO[it.strip().lower()] + for it in items: + for w in re.findall(r"[a-z]+", it.lower()): + if w in _GENDER: + return _GENDER[w] + return None + + +_PERSON_NOUNS = (r"woman|women|man|men|girl|boy|guy|lady|gentleman|person|people|" + r"female|male|figure|character|adult|teen|teenager") + + +def _deposition(desc, name=None): + """Turn a description into pure ATTRIBUTES, removing any subject-introducing + noun phrase. + + A description written naturally -- "a woman with silver hair", "a young woman", + "Kristy is a tall woman" -- renders inline as `She (a woman with silver hair)`. + That is TWO subject nouns in one clause ("She" and "a woman"), which + text-to-video reads as two people: character duplication, visible from the very + first shot and independent of resolution. Attributes alone -- "silver hair" -- + bind to the pronoun instead of competing with it. + + Strips: a leading article + optional adjectives + person noun (keeping any + following "with/in ..." attributes), a copula phrase ("Kristy is a tall + woman"), and a bare repeat of the character's own name.""" + d = (desc or "").strip() + if not d: + return d + if name: + d = re.sub(r"\b" + re.escape(name) + r"\b\s*(?:is|,)?\s*", "", d, flags=re.I) + # "a young woman with silver hair" -> "silver hair"; "a tall woman" -> "tall" + m = re.match(r"^\s*(?:an?|the)\s+((?:[\w\-]+\s+){0,3}?)(?:" + _PERSON_NOUNS + r")\b" + r"(?:\s+(?:with|in|wearing)\s+)?(.*)$", d, re.I) + if m: + adjectives, rest = m.group(1).strip(), m.group(2).strip() + d = (rest if rest else adjectives) or adjectives + # "wearing a black t-shirt" -> "black t-shirt": inside a parenthetical the verb + # reads as a separate predicate about a subject, not an attribute of the pronoun + d = re.sub(r"^\s*(?:wearing|dressed in|dressed|clad in|in)\s+", "", d, flags=re.I) + d = re.sub(r"^\s*(?:an?|the)\s+", "", d) + # a bare person noun left on its own carries no attribute -> drop it + if re.fullmatch(r"\s*(?:an?|the)?\s*(?:" + _PERSON_NOUNS + r")\s*", d, re.I): + return "" + return re.sub(r"\s{2,}", " ", d).strip(" ,") + + +# Mouth/lip state items in a character sheet ("mouth closed", "lips together"). +# Users add these to force mouths shut on action shots, which works -- but they are +# re-stamped into EVERY shot, so on a beat with real quoted dialogue the prompt +# tells the model to keep the mouth closed AND to speak. Dropped on speaking shots +# only; kept everywhere else so the forced-closed behaviour is preserved. + + +def _is_mouth_state(item): + it = (item or "").strip().lower() + if not it: + return False + return bool(re.search(r"\b(?:mouth|lips|jaw)\b", it) and + re.search(r"\b(?:closed|shut|together|still|sealed|not\s+talking|no\s+talking)\b", it)) + + +def _clean_items(items, name=None, drop_mouth_state=False): + """Drop bare pronoun tokens, de-position any noun-phrase descriptions so a + parenthetical never introduces a second subject, and -- on shots that contain + real quoted dialogue -- drop mouth-state items so the sheet does not order a + closed mouth in the same breath as a spoken line.""" + out = [] + for it in items: + if it.strip().lower() in _PRO: + continue + if drop_mouth_state and _is_mouth_state(it): + continue + d = _deposition(it, name) + if d: + out.append(d) + return out + + + +def _pron_map(active): + """{pronoun: [names]} for resolving a bare 'she'/'he' to a person.""" + out = {} + for n, items in active.items(): + if not n: + continue + p = _pronoun_of(items) + if p: + out.setdefault(p, []).append(n) + return out + + +def _resolve_subject(word, names, pron_map, single): + """Map a subject token (a name or a pronoun) to a tracked person, or None if + ambiguous. In a one-person scene any pronoun maps to that person.""" + wl = word.lower() + for n in names: + if n.lower() == wl: + return n + if wl in _PRO: + want = _PRO[wl] + cands = pron_map.get(want, []) + if len(cands) == 1: + return cands[0] + if cands: + return None # ambiguous: two people share this pronoun + # No candidate with this pronoun. Only fall back to the lone remaining + # person if their pronoun is UNDECLARED -- never map 'he' onto a declared + # 'she' (which happened once the 'he' character had left the scene). + if single and names and not any(names[0] in v for v in pron_map.values()): + return names[0] + return None + return None + + +# Words that start a DETAIL trailing off a garment rather than continuing its name: +# "red leather jacket WITH silver zippers", "boots WITH steel buckles". +# NOTE: "down" is deliberately absent -- it is a material ("a puffy down jacket"), +# not a position, and cutting there would leave "puffy" as the garment name. +_ITEM_DETAIL = re.compile( + r"\b(?:with|without|featuring|showing|bearing|in|on|over|under|across|that|which|" + r"and|plus|sporting|carrying|covered|around|about|at|through|along|behind|" + r"beneath|beside|near)\b") + + +def _item_name(item): + """The part of a wardrobe item that NAMES the garment, without its detail. + + "red leather jacket with a white circular chest patch" -> "red leather jacket". + Used both to match a removal and to refer to the garment in generated prose: + the detail belongs in the description that is stamped every shot, not in a + sentence whose only job is to say the thing came off.""" + il = (item or "").strip() + cut = _ITEM_DETAIL.search(il.lower()) + name = il[:cut.start()].strip(" ,") if cut and cut.start() > 0 else il + # Drop a leading determiner: sheets are written both ways ("a diaper", "diaper"), + # and the generated prose supplies its own article -- "the a diaper underneath". + name = re.sub(r"^(?:a|an|the|her|his|their|its|my|your)\s+", "", name, flags=re.I) + return name or il + + +def _item_head(item): + """The garment's own head noun, ignoring any detail trailing off it. + + Taking the LAST word was wrong the moment an item carried detail: the head of + "red leather jacket with silver zippers" came out as `zippers`, of "bomber + jacket with a white logo on the chest" as `chest`. "takes off her red jacket" + then matched nothing and the removal SILENTLY did not fire -- the garment stayed + on the sheet and got re-stamped into every later shot. Detailed wardrobe entries + are normal (logos, zippers, torn knees), so the head has to be read from the + part of the phrase that names the garment.""" + words = re.findall(r"[a-z\-]+", _item_name(item).lower()) + if not words: + words = re.findall(r"[a-z\-]+", (item or "").lower()) + return words[-1] if words else "" + + +def _item_mentioned(item, window): + """Does `window` refer to this wardrobe item? Matches the whole phrase, or the + item's head noun, tolerant of singular/plural ('boots' vs 'boot') -- the strict + exact-substring test missed 'takes off her boots' when the sheet said 'boots' + and vice versa. Ignores generic colour/size adjectives so 'red jacket' is still + matched by 'her jacket', and trailing detail so 'red jacket with silver zippers' + is still matched by 'her jacket'.""" + w = window.lower() + il = item.lower().strip() + if not il: + return False + if il in w: + return True + head = _item_head(il) + if not head: + return False + for form in {head, head.rstrip("s"), head + "s", head + "es"}: + if form and re.search(r"\b" + re.escape(form) + r"\b", w): + return True + return False + + +# Where a removal verb's OBJECT ends. Anything past one of these belongs to a +# different phrase -- what was revealed, where the garment was put, what happened +# next -- and must not be treated as something that also came off. +_OBJECT_STOP = re.compile( + r"[,.;:!?]" + r"|\b(?:revealing|reveals?|showing|shows?|exposing|exposes?|leaving|leaves?|wearing|" + r"wears?|underneath|beneath|under|over|on|onto|in|into|to|from|at|by|beside|near|" + r"next|behind|against|while|as|before|after|then|toward|towards)\b" + r"|\b(?:puts?|pulls?|slips?|throws?|zips?|buttons?|laces?)\s+(?:on|into)\b") + + +def _removal_object_spans(text, m): + """(forward, backward) -- the two places a removal verb's object can sit. + + FORWARD is the normal case ("takes off her red jacket"). It runs from the verb + to the first phrase boundary and INCLUDES the matched cue, because the put-away + patterns ("hangs her jacket on a hook") carry the garment inside the match. It + deliberately does not extend past a boundary: a fixed ~68-character window used + to sweep up any garment sitting near the removal, so "drops it on the bench next + to her boots" removed the boots and "over her black tank top" removed the top. + + BACKWARD covers the phrasings that put the garment FIRST -- "her jacket slips + off her shoulders", "her dress falls to the ground", "the jacket is off now". + It is tried only when the forward span turns up nothing, and it stops at the + previous clause boundary so it reaches the subject and no further.""" + tail = text[m.end():m.end() + 60] + cut = _OBJECT_STOP.search(tail) + forward = m.group(0) + (tail[:cut.start()] if cut else tail) + head = text[max(0, m.start() - 45):m.start()] + bcut = None + for b in re.finditer(r"[,.;:!?]|\b(?:and|then|while|as|before|after)\b", head): + bcut = b + backward = head[bcut.end():] if bcut else head + return forward, backward + + +# Items that STAY ON until something explicitly says otherwise. +# +# A garment coming off by itself is usually what the prose meant. A restraint is +# not: it is a plot state that the scene establishes and only the scene ends. Left +# to the ordinary removal detector these came off far too easily, and often by +# accident -- "steps out of her jacket and the chain falls away" removed the ankle +# chain as a side effect of a beat about a jacket, because the removal window +# reaches any tracked item near the cue. +# +# Deliberately narrow. Only nouns that are unambiguously a restraint: `chain`, +# `collar`, `strap` and `belt` are all left OUT, because they are jewellery, a +# shirt part, a dress part and a garment at least as often. A compound like +# "ankle chain" or "chain restraint" is caught by its qualifier instead. +_RESTRAINT_HEADS = { + "handcuff", "handcuffs", "cuff", "cuffs", "shackle", "shackles", + "manacle", "manacles", "fetter", "fetters", "iron", "irons", + "restraint", "restraints", "binding", "bindings", "bond", "bonds", + "gag", "blindfold", "hood", "muzzle", "harness", "leash", "hobble", + "strait-jacket", "straitjacket", "spreader", "zip-tie", "ziptie", +} +# ...and the qualifiers that turn an ambiguous noun into a restraint. Body parts +# and binding participles ONLY. +# +# Materials are deliberately absent. "leather" would make a leather belt a +# restraint, "steel" a steel watch strap. And a word may not appear in BOTH lists: +# `chain` and `rope` were in each at first, so they qualified themselves and a bare +# "gold chain" came out a restraint. +_RESTRAINT_QUALIFIER = re.compile( + r"\b(?:ankle|wrist|leg|arm|thumb|neck|waist|" + r"chained|shackled|cuffed|bound|tied|locked|padlocked|restraining)\b", re.I) +_RESTRAINT_NOUN = re.compile( + r"\b(?:chain|chains|rope|ropes|cord|cords|tie|ties|strap|straps|collar|collars|" + r"band|bands|belt|belts|cuff|cuffs)\b", re.I) +# Two shapes the head-noun/qualifier rules above both miss: +# * compounds whose HEAD noun is innocent -- "spreader bar" resolves to `bar`, +# which is furniture far more often than hardware, so only the word SPREADER +# identifies it; +# * a binding participle fastened straight onto a body part -- "bound wrists", +# "shackled ankles", "tied hands" -- where there is no equipment noun at all. +# Only the STRONG participles qualify here, never a bare body part: the qualifier +# list already carries ankle/wrist/etc. for the equipment route, and letting a +# body part qualify itself would make "waist tie" on a dress unremovable. +_RESTRAINT_SPREADER = re.compile(r"\bspreaders?\b", re.I) +_RESTRAINT_PARTICIPLE = re.compile( + r"\b(?:chained|shackled|cuffed|bound|tied|locked|padlocked)\b", re.I) +_BODY_PART_TOKEN = re.compile( + r"\b(?:ankles?|wrists?|legs?|arms?|thumbs?|knees?|elbows?|hands?|feet|foot|" + r"fingers?|neck|waist)\b", re.I) + + +def is_restraint(item): + """True when this wardrobe item is a physical restraint rather than clothing. + + Either an unambiguous restraint noun on its own ("handcuffs", "shackles"), an + ambiguous one carrying a restraint qualifier ("ankle chain", "leather wrist + straps"), a compound named by its modifier ("spreader bar" -- the head noun + `bar` says nothing), or a binding participle on a body part ("bound wrists"). + A bare "chain" or "collar" is NOT a restraint -- it is jewellery or a shirt + part far more often, and a false positive here means a garment that can never + be taken off.""" + name = _item_name(item or "").lower() + if not name: + return False + if _item_head(item) in _RESTRAINT_HEADS: + return True + if _RESTRAINT_NOUN.search(name) and _RESTRAINT_QUALIFIER.search(name): + # ...unless the phrase resolves to an actual BODY-COVERING GARMENT -- + # "waist tie dress", "tie-front blouse with a neck strap" -- where the + # equipment word is a feature of the clothing rather than hardware. + # Clothing wins here, or the garment can never come off; genuine + # restraints ("ankle chain", "wrist cuffs") cover no zone at all. + if not garment_zones(item): + return True + if _RESTRAINT_SPREADER.search(name): + return True + return bool(_RESTRAINT_PARTICIPLE.search(name) and _BODY_PART_TOKEN.search(name)) + + +# What a restraint DOES, once it is on. Keeping the item in the wardrobe list only +# says it exists; nothing there says the body cannot move freely, so H3 renders a +# cuffed character walking with their arms swinging. The restraint is present and +# doing nothing -- which reads as it having broken. +# +# Stated as a POSITIVE physical state, the same as the mouth state and the limb +# count. "cannot move her arms" is a negation and a weak cue; "her wrists stay +# together in front of her" describes a pose the model can actually render. +# +# Keyed by the body region the restraint binds, so two wrist restraints produce one +# clause rather than two competing ones. +_RESTRAINT_EFFECT = { + "wrists": "the wrists stay bound close together, the arms moving as one and never " + "swinging apart", + "ankles": "the ankles stay bound close together, steps short and shuffling, the legs " + "never striding apart", + "mouth": "the mouth stays covered and the jaw still", + "eyes": "the eyes stay covered, the head turning toward sound rather than sight", + "body": "the body stays held by the restraint, movement limited and tethered", +} +# Which region each restraint binds. Checked against the item's full name, so +# "ankle chain" and "leg irons" both reach `ankles`. +_RESTRAINT_REGION = ( + ("mouth", re.compile(r"\b(?:gag|gagged|muzzle|muzzled)\b", re.I)), + ("eyes", re.compile(r"\b(?:blindfold|blindfolded|hood|hooded)\b", re.I)), + ("ankles", re.compile(r"\b(?:ankle|ankles|leg|legs|hobble|feet|foot)\b", re.I)), + ("wrists", re.compile(r"\b(?:handcuff|handcuffs|cuff|cuffs|wrist|wrists|manacle|" + r"manacles|thumb|arm|arms)\b", re.I)), +) + +# HOW a wrist restraint holds changes what the body can do, and the default effect +# text only covers ONE of those ways. "the wrists stay bound close together, the +# arms moving as one" is true of wrist-to-wrist cuffs and false of everything else: +# a character chained to a headboard has arms held APART at fixed anchor points, +# hands cuffed behind the back have arms folded behind, cuffed above the head has +# arms raised. Telling the model two contradictory things about the same wrists -- +# scene says spread, guard says together -- is exactly how the cuffs end up +# rendered broken: neither sentence holds, and the hardware loses. +# +# A tether needs a FASTENING verb before it trusts a target noun ("walks to the +# table" must not read as chained to the table); resistance verbs qualify only with +# "against", which is how pushing on an anchor point reads. +_RESTRAINT_TETHER = re.compile( + r"\b(?:(?:cuff(?:ed|s)?|chain(?:ed|s)?|ti(?:ed|es)|lock(?:ed|s)?|shackle(?:d|s)?|" + r"fasten(?:ed|s)?|strap(?:ped|s)?|bolt(?:ed|s)?|secur(?:ed|es)|hitch(?:ed)?|" + r"bind(?:s|ing)?|bound)\b[^.]{0,40}?\b(?:to|around)\s+(?:the|a|an|her|his|their|its)?\s*" + r"|(?:strain\w*|pull\w*|tug\w*|yank\w*)\b[^.]{0,30}?\bagainst\s+the\s+)" + r"(bed|bedpost|headboard|wall|post|pole|frame|ring|hook|ceiling|rack|table|bench|" + r"chair|doorframe|beam|pipe|rail|floor|tree|cross|stainless steel bar)\b", re.I) +_RESTRAINT_POSE = ( + ("behind", re.compile(r"\bbehind\s+(?:her|his|their|the)?\s*back\b", re.I)), + ("overhead", + re.compile(r"\b(?:above|over)\s+(?:her|his|their|the)?\s*head\b|\boverhead\b", re.I)), + ("spread", + re.compile(r"\bspread[- ]?eagl\w*\b|\b(?:wrists|arms)\s+(?:held\s+)?spread\b", re.I)), +) + + +def _detect_restraint_usage(text): + """Read HOW a restraint is being used out of one block of prose. + + Returns {"tether": anchor noun or None, "pose": pose name or None}. A tether + outranks a pose (the current shot said the anchor explicitly); poses are read + in _RESTRAINT_POSE order so 'hands cuffed behind her back' wins over a stray + 'legs spread'. Nothing found -> both None.""" + t = text or "" + m = _RESTRAINT_TETHER.search(t) + if m: + return {"tether": m.group(1).lower(), "pose": None} + for pose, rx in _RESTRAINT_POSE: + if rx.search(t): + return {"tether": None, "pose": pose} + return {"tether": None, "pose": None} + + +def _restraint_effect_text(region, text, usage=None): + """The effect sentence for one bound region, adapted to HOW the restraint holds. + + `text` is the person's own wardrobe entries plus this shot's beat -- both are + places the user states the attachment ("wrist cuffs to the bed frame") or the + pose ("hands cuffed behind her back"). When this shot says nothing, `usage` + carries what an EARLIER shot established for this person -- restraint use + persists until the prompt changes it, exactly like the wardrobe it belongs + to. Without either, the default wording stays correct for plain + wrist-to-wrist cuffs.""" + base = _RESTRAINT_EFFECT[region] + if region != "wrists": + return base + found = _detect_restraint_usage(text) + stored = usage or {} + # What this shot SAYS replaces what an earlier shot established, wholesale -- + # mixing a live pose with a remembered tether would blend two setups into the + # kind of contradiction this clause exists to prevent. + if found["tether"]: + anchor, pose = found["tether"], None + elif found["pose"]: + anchor, pose = None, found["pose"] + else: + anchor, pose = stored.get("tether"), stored.get("pose") + if anchor: + return ("the cuffs stay locked closed around the wrists and fastened to the " + f"{anchor}, the chain between them taut, the arms held where they are " + "secured") + if pose == "behind": + return ("the wrists stay bound close together behind the back, the arms " + "held there and moving as one") + if pose == "overhead": + return ("the wrists stay bound close together above the head, the arms held " + "up and moving as one") + if pose == "spread": + return ("the wrists stay bound apart at two fixed points, the arms held wide " + "and never coming closer together") + return base + + +# The failure mode where the HARDWARE itself gives up: H3 renders an open cuff, a +# snapped link, or a strap turned to ribbon mid-struggle. Nothing in the per-region +# effects says the equipment keeps its state, so state it once, positively -- at cfg +# 1 a negation ("does not break") would only name the breaking. "Full tension" was +# tried here and read as strain: the model rendered maximum-pull struggle to match +# the words, which is the exact failure this sentence exists to stop. +_RESTRAINT_HARDWARE = (" Every restraint stays whole and closed, fastened exactly " + "as it was put on.") + + +def restraint_regions(items): + """Body regions currently bound, for everything this person is wearing. + + A restraint with no region of its own ("shackles", "fetters", "restraints") + falls back to `body`, which says movement is limited without claiming to know + which limb. Guessing a specific limb there would be worse than saying less.""" + out = [] + for it in items or []: + if not is_restraint(it): + continue + name = _item_name(it) + for region, rx in _RESTRAINT_REGION: + if rx.search(name): + if region not in out: + out.append(region) + break + else: + if "body" not in out: + out.append("body") + return out + + +# What a bared zone keeps doing while the body moves. The marker in the item list +# says the zone IS bare; this says it STAYS bare through a change of view, which is +# the moment the clothed prior gets its chance -- a turn presents a surface the model +# has no evidence for, and its default for an undescribed body is a dressed one. +# +# Names no garment. Saying which garment is off puts that garment in the prompt, and +# a mention is a presence cue -- the same reason removed items are scrubbed from the +# anchor rather than negated in it. +# IMPERSONAL -- no name, no pronoun, no subject at all. An earlier version said +# "She is uncovered there and stays that way", which is a SECOND reference to someone +# the shot has already introduced, and a second mention is what renders a second +# figure. That put it straight up against the subject-count guard. The anchor-prose +# garments are stated impersonally for the same reason: it summons nobody. +# +# The wording also has to avoid the MARKER phrases themselves ("bare below the waist", +# "bare chest", "bare breasts"). Those strings are what identifies a marker in an item +# list, and repeating one verbatim in the prose makes a test for "is the generic marker +# still being used" -- or anyone reading the prompt -- unable to tell prose from state. +_BARE_PERSIST = { + "lower": "Bared skin from the waist down stays bared as the body turns, the same " + "from the front, the side and behind.", + "upper": "Bared skin above the waist stays bared as the body turns, the same from " + "the front, the side and behind.", +} + + +def bare_persist_clause(bare_zones, active, body): + """State that an already-bared zone stays bare through a change of angle. + + Only for people actually in the shot, on the same presence gate as the restraint + and mouth states: describing an uncovered body in a shot that person is not in + would summon them into it. + + Gated upstream by whatever allowed the marker in the first place -- this reads + the markers that are already on the person, so prevent_nudity, exposed_terms and + a sheet declaration all keep exactly the authority they had.""" + zones = [] + for name, zs in (bare_zones or {}).items(): + if name and not person_in_shot(body, name, active): + continue + for z in zs or (): + if z in _BARE_PERSIST and z not in zones: + zones.append(z) + if not zones: + return "" + # One sentence per bared ZONE, not per person: two people bared the same way need + # the state said once. Saying it per person would reintroduce the extra subject + # references this is written to avoid. + return " " + " ".join(_BARE_PERSIST[z] for z in zones) + + +def _guard_fires(mode, cued): + """Shared off/auto/on gate for the per-shot guards. + + 'off' never speaks, 'on' always does, 'auto' only when the shot gave it + something to talk about -- a cue in the beat, or a list of objects it found. + Three guards had this same three-line shape written out separately.""" + if mode == "off": + return False + return mode == "on" or bool(cued) + + +def contact_clause(body, n_present, mode="auto"): + """Geometry for two bodies in contact: whose limbs, no interpenetration, fixed roles. + + Needs TWO people. One body cannot be misaligned against another, and stating a + two-body arrangement in a one-person shot would invite the second person in -- + the same presence-cue failure every other per-shot state is gated against. + + 'auto' fires on a contact cue in the beat. 'on' states it whenever two or more + people are in the shot, which is worth having if bodies drift apart or merge in + beats that do not name the contact explicitly.""" + if n_present < 2 or not _guard_fires(mode, _CONTACT_CUE.search(body or "")): + return "" + return CONTACT_STATE + + +def motion_clause(body, mode="auto"): + """State that a pose is reached by travelling to it. + + 'auto' speaks only on a beat that actually moves someone, which is where a snap + can happen -- a beat where nobody changes orientation has no path to describe and + would just be paying for a sentence. + + Impersonal, like the solidity and bare-persistence states: it names no one, so it + adds no second reference to anybody already in the shot.""" + if not _guard_fires(mode, _MOTION_CUE.search(body or "")): + return "" + return MOTION_STATE + + +def _restrained_present(active, body): + """True when anyone PRESENT in this shot is currently carrying a restraint. + + Holds the generic free-travel motion clause back from bound bodies: that + sentence tells a figure to turn through every position on the way, which a + cuffed body cannot obey -- and the model settles the contradiction by letting + the restraints fail so it can. The bound parts carry their own continuity + instead ('the arms moving as one'), which is all a restrained figure should + promise.""" + return any(restraint_regions(active.get(nm)) and person_in_shot(body, nm, active) + for nm in active) + + +def solid_things_in(text): + """Solid objects named in this text, in order of appearance. + + Deduplicated by head noun so "table" and "tables" are not said twice, but the + surface form is what comes back: "stairs" singularises to "stair", which is not + what anyone calls them, and inherently-plural nouns are common here (stairs, + steps, shelves, rocks).""" + out, seen = [], set() + for m in _SOLID_NOUNS.finditer(text or ""): + w = m.group(0).lower() + head = w[:-1] if w.endswith("s") and not w.endswith("ss") else w + if head not in seen: + seen.add(head) + out.append(w) + return out + + +def solidity_clause(body, persistent="", mode="auto"): + """Assert that the objects in this shot occupy space and stop bodies. + + 'auto' only speaks when the shot actually names something solid, so a beat with + nothing to collide with pays nothing. 'on' always states it. + + The named list matters more than the general sentence: the failure is a body + passing through one PARTICULAR thing the shot established, and naming it again + is what keeps it in the frame as an obstacle rather than as scenery.""" + if mode == "off": + return "" + # BEAT first, then the identity block. The list is trimmed to three, and the + # object that matters is the one this shot's action involves -- scanning the + # anchor first filled the quota with set dressing and cut the stairs the + # character is actually climbing. + found = solid_things_in(body) + for w in solid_things_in(persistent): + if w not in found: + found.append(w) + if not _guard_fires(mode, found): + return "" + out = SOLIDITY_STATE + if found: + # A bare list, so no verb has to agree with a noun that may be plural + # ("the stairs is solid"). Three at most -- past that it reads as an + # inventory and starts competing with the beat for attention. + out += (" Solid, and occupying real space here: " + + ", ".join("the " + n for n in found[:3]) + ".") + return out + + +def _restraint_about(sentence, name, active): + """Is this prose sentence about THIS bound person, rather than someone else? + + The beat is shared but the restraint wording is per person: 'Mara is cuffed + to the headboard. Jon watches.' must not fasten JON to anything -- and before + persistence that mistake washed out a shot later; now it would stick. A + sentence naming another tracked person is skipped unless it also names this + person or uses the pronoun their sheet declares. Unattributed sentences + ("the chain rattles") stay relevant to everyone.""" + named_here = re.search(r"\b" + re.escape(name) + r"\b", sentence, re.I) + others = [n for n in active if n and n != name + and re.search(r"\b" + re.escape(n) + r"\b", sentence, re.I)] + if not others: + return True + if named_here: + return True + pronouns = [i.lower() for i in (active.get(name) or []) + if i.lower() in ("she", "he", "they")] + return any(re.search(r"\b" + re.escape(p) + r"\b", sentence, re.I) + for p in pronouns) + + +def restraint_clause(active, body, lock_restraints=True, usage=None): + """State what each restrained person's body cannot do, for the people in shot. + + Only for someone actually referenced in the beat -- describing a bound body in + a shot that person is not in would summon them into it, which is the same + failure the mouth state and the limb count are both gated against. + + `usage` is an optional dict carried ACROSS shots by distribute_generations: + when a shot states how the restraints are used (a tether anchor or a pose) it + is remembered per person and reused on later shots that only say "she strains" + -- otherwise those shots would fall back to wording that CONTRADICTS the + established attachment, which is exactly how cuffs render broken. Entries are + pruned upstream in distribute_generations the moment the person's sheet no + longer carries a restraint (freed, uncuffed, item removed), so a re-bound + character starts fresh.""" + if not lock_restraints: + return "" + bits = [] + for name, items in (active or {}).items(): + regions = restraint_regions(items) + if not regions: + continue + if name and not person_in_shot(body, name, active): + continue + subj = _subject_term(name, active) if name else "the subject" + # `their` rather than a repeated name: naming someone twice in a shot is + # what renders them twice. The attachment/pose scan reads BOTH the person's + # own wardrobe entries and this shot's prose -- but only the prose that is + # ABOUT them (one person's tether must not fasten a second restrained + # character to the same anchor), and never quoted dialogue: speech + # describes a restraint, it does not attach one. + masked, _qspans = _mask_quotes(body or "") + sentences = [s for s in re.split(r"(?<=[.!?])\s+", masked) if s.strip()] + if name: + sentences = [s for s in sentences if _restraint_about(s, name, active)] + text = " ".join(items) + " " + " ".join(sentences) + found = _detect_restraint_usage(text) + if usage is not None and (found["tether"] or found["pose"]): + usage[name] = found + effects = "; ".join(_restraint_effect_text(r, text, + usage.get(name) if usage else None) + for r in regions) + bits.append(f"{subj} is physically restrained -- {effects}") + if not bits: + return "" + return (" " + ". ".join(b[0].upper() + b[1:] for b in bits) + "." + + _RESTRAINT_HARDWARE) + + +def auto_wardrobe_removals(active, body, lock_restraints=True): + """Infer clothing REMOVALS from a beat's own action text, so you don't have + to write a 'wardrobe:' line at all -- "she takes off her jacket" drops the + jacket by itself. + + SAFE BY DESIGN: a removal only fires on an item the character is ALREADY + wearing. Non-garment objects match nothing, so "the plane takes off down the + runway" removes nothing. The subject can be a NAME or a PRONOUN: with two + people, declare a pronoun per person ('Maya = she, ...; Jon = he, ...') and + 'she takes off her jacket' attributes to Maya. In a one-person scene any + pronoun maps to that person. If the subject is ambiguous (two same-pronoun + people, no name), the item is dropped from whoever wears it. Explicit + 'wardrobe: -=' always overrides.""" + if not body: + return active + # Quoted speech is an INSTRUCTION, not an action. 'Mom says: "take off your + # thong"' used to strip the garment in the shot where it is merely asked for, + # one shot before the character does it -- so the shot that performs the + # removal no longer knew the garment was on, and the removal never got its + # direction clause. Negation made it worse: "do not take off your jacket" + # removed the jacket. Detect removals from narration only; the beat that + # actually stages it is unquoted. + text = " " + _mask_quotes(body)[0].lower() + " " + active = {k: list(v) for k, v in active.items()} + names = [n for n in active if n] + pron_map = _pron_map(active) + single = len(names) == 1 + + remove_cue = re.compile( + # verb ... off / out of / aside / away (covers "takes off", "steps out of") + r"\b(?:takes?|took|taken|taking|pulls?|pulled|peels?|peeled|strips?|stripped|" + r"slips?|slipped|shrugs?|shrugged|tears?|tore|yanks?|yanked|casts?|kicks?|" + r"throws?|threw|tosses|tossed|hangs?|hung|drops?|dropped|sets?|set|puts?|put|" + # ...and the ones you get OUT OF rather than take off + r"steps?|stepped|stepping|wriggles?|wriggled|wiggles?|wiggled|squirms?|squirmed|" + r"struggles?|struggled|struggling|" + r"slides?|slid|sliding|climbs?|climbed|works?|worked|eases?|eased|shakes?|shook)\b" + r"[\w\s\']{0,20}?\b(?:off|out of|aside|away|down)\b" + # the garment itself is the subject: "her dress falls to the ground", + # "the jacket pools at her feet". Matched backward to the subject. + r"|\b(?:falls?|fell|falling|slides?|slid|slips?|slipped|drops?|dropped|" + r"pools?|pooled|tumbles?|tumbled|crumples?|crumpled)\s+" + r"(?:to|onto|on to|down|off|open|away|around|at)\b" + # "lets her jacket fall", "lets it drop" + r"|\blets?\b[\w\s\']{0,20}?\b(?:fall|falls|drop|drops|slide|slides|slip|slips)\b" + # standalone removal verbs + r"|\b(?:removes?|removed|removing|sheds?|shed|shedding|discards?|discarded|" + r"ditch(?:es|ed)?|doffs?|doffed|unbuttons?|unzips?|unzipped|unbuckles?|" + r"undoes|undid|undone|unlaces?|unlaced|unhooks?|unhooked|unfastens?|unfastened|" + r"unclasps?|unclasped|unsnaps?|unsnapped|unties?|untied|unwraps?|unwrapped|" + r"hangs? up|hung up)\b" + # " is off / are off" + r"|\bis off\b|\bare off\b" + # put-away phrasings: "hangs her jacket on a hook", "drapes it over a chair" + r"|\b(?:hangs?|hung|drapes?|draped|slings?|slung|drops?|dropped|tosses|tossed|" + r"throws?|threw|leaves?|left|sets?|set|lays?|laid|places?|placed)\b[\w\s\']{0,20}?" + r"\b(?:on|over|across)\s+(?:a|an|the)\b") + + subj_tokens = [re.escape(n.lower()) for n in names] + list(_PRO.keys()) + subj_re = re.compile(r"\b(" + "|".join(subj_tokens) + r")\b") if subj_tokens else None + + def nearest_subject(pos): + best, bp = None, -1 + if subj_re: + for mm in subj_re.finditer(text): + if 0 <= mm.start() < pos and mm.start() > bp: + person = _resolve_subject(mm.group(1), names, pron_map, single) + if person is not None: + bp, best = mm.start(), person + return best + + # Donning phrases must never trigger a removal ("pulls on a jacket", "puts on + # her boots", "slips into her coat") -- the wrong direction is far worse than a + # miss, since it would strip clothing the character just put ON. + don_re = re.compile(r"\b(?:pulls?|puts?|slips?|throws?|shrugs?|zips?|buttons?|laces?|" + r"pulled|put|slipped|threw|shrugged)\b\s+(?:on|into)\b") + don_spans = [(d.start(), d.end() + 45) for d in don_re.finditer(text)] + + for m in remove_cue.finditer(text): + if any(a <= m.start() <= b for a, b in don_spans): + continue + forward, backward = _removal_object_spans(text, m) + tgt = nearest_subject(m.start()) + # Try the verb's forward object first; only if that names nothing tracked do + # we look BACK to the subject, which is where "her jacket falls to the + # ground" and "her jacket slips off her shoulders" put the garment. + for window in (forward, backward): + if not window.strip(): + continue + hit = False + for name in ([tgt] if tgt else list(active.keys())): + for it in list(active.get(name, [])): + if it.strip().lower() in _PRO: + continue + # A restraint is a plot state, not a garment. Prose never takes + # one off; only an explicit 'wardrobe: Name -= handcuffs' does. + if lock_restraints and is_restraint(it): + continue + if _item_mentioned(it, window): + active[name] = [x for x in active[name] if x != it] + hit = True + if hit: + break + return active + + + + +def _scrub_removed(text, removed): + """Delete phrases for removed garments from PERSISTENT text (the anchor), so a + garment written into the anchor prose -- e.g. 'A woman in a red flight jacket' + -- can't re-apply itself on every shot after the character takes it off. Removes + the item phrase plus a leading connector ('in a', 'wearing a', 'with a') and + tidies the leftover punctuation. Case-insensitive; leaves everything else alone.""" + if not text or not removed: + return text + changed = False + for item in sorted(removed, key=len, reverse=True): + item = item.strip() + if not item: + continue + pat = (r"(?:,\s*)?\b(?:wearing|dressed in|in|with)?\s*(?:a|an|the|her|his|their)?\s*" + + re.escape(item) + r"\b") + new_text = re.sub(pat, "", text, flags=re.I) + if new_text != text: + changed = True + text = new_text + if not changed: + # Nothing was actually scrubbed from this text, so leave it EXACTLY as the + # user wrote it. The tidy-up below repairs punctuation left by a removal; + # running it unconditionally silently rewrote untouched prose (e.g. + # "hangar and airfield" -> "hangar, airfield") on every shot after any + # unrelated garment removal. + return text.strip() + text = re.sub(r"\s{2,}", " ", text) + text = re.sub(r"\s+([,.])", r"\1", text) + text = re.sub(r"(,\s*){2,}", ", ", text) + text = re.sub(r",\s*\.", ".", text) + # tidy connectors left dangling by a removed phrase: "silver hair and , in a + # hangar" / "silver hair and hangar" -> "silver hair, in a hangar" + text = re.sub(r"\s+and\s*,", ",", text) + text = re.sub(r"\s+and\s+(in|at|on|with|under|beside)\b", r", \1", text) + text = re.sub(r"\s+and\s+(?=[a-z]+\s*,)", ", ", text) + text = re.sub(r"\s+and\s*$", "", text) + text = re.sub(r"^\s*(?:and|,)\s+", "", text) # leading dangling connector + text = re.sub(r"^\s*and\b", "", text) + text = re.sub(r"\s{2,}", " ", text) + text = re.sub(r"(,\s*){2,}", ", ", text) + return text.strip(" ,") + + +def _strip_people_from_anchor(anchor_id, active): + """Remove tracked people from the ANCHOR prose. + + The anchor is stamped into every shot, so if it also introduces a character -- + by name ("Kristy stands by the plane") or by description ("a woman with silver + hair in a red jacket") -- that character is asserted TWICE per shot: once by the + anchor and once by the beat's own inline binding. Text-to-video reads two + introductions of one subject as two subjects, which is the character-duplication + users see. The anchor should carry scene and style only; who is present is + decided per beat. + + Removes (a) any tracked NAME plus its clause, and (b) a person-phrase whose + description overlaps a tracked person's items (gender word + shared descriptors). + Leaves everything else -- setting, lighting, lens, mood -- untouched.""" + if not anchor_id: + return anchor_id + txt = anchor_id + for name in [n for n in active if n]: + # drop a whole sentence that names this person, else just the name token + sentences = re.split(r'(?<=[.!?])\s+', txt) + kept = [c for c in sentences if not re.search(r"\b" + re.escape(name) + r"\b", c, re.I)] + if len(kept) != len(sentences): + txt = " ".join(kept) + else: + txt = re.sub(r"\b" + re.escape(name) + r"\b", "", txt, flags=re.I) + # description overlap: "a woman with silver hair in a red jacket" + for name, items in active.items(): + descs = [i for i in _clean_items(items, name) if len(i.split()) <= 4] + if not descs: + continue + pron = _pronoun_of(items) + nouns = {"she": r"(?:woman|women|girl|lady)", "he": r"(?:man|men|boy|guy)"}.get(pron, r"(?:person|figure)") + # match the person phrase up to a sentence end / conjunction, so trailing + # "with X in a Y" clauses go with it instead of leaving fragments behind + pat = re.compile(r"(?:,\s*)?\b(?:a|an|the)\s+(?:[\w\-]+\s+){0,3}" + nouns + + r"(?:(?!\.|\band\b).)*", re.I) + for mm_ in list(pat.finditer(txt)): + phrase = mm_.group(0) + if sum(1 for d in descs if d.lower() in phrase.lower()) >= 1: + txt = txt.replace(phrase, "") + # Generic, UNNAMED person references in the anchor ("the camera follows the + # subject", "moves toward the person", "tracks the figure") are stamped into + # every shot alongside the named cast, so the model renders an extra body that + # matches nobody -- the phantom third person. Camera-direction wording is the + # usual way these creep in. Rewrite them to refer to the framing, not a body. + txt = re.sub(r"\b(?:the|a|an)\s+(?:main\s+|central\s+)?" + r"(?:subject|person|figure|character|model|individual|protagonist)\b", + "the scene", txt, flags=re.I) + txt = re.sub(r"\b(?:the|a|an)\s+(?:subjects|people|figures|characters)\b", + "the scene", txt, flags=re.I) + txt = re.sub(r"\s{2,}", " ", txt) + txt = re.sub(r"\s+([,.])", r"\1", txt) + txt = re.sub(r"(,\s*){2,}", ", ", txt) + txt = re.sub(r"\.{2,}", ".", txt) # "Warm light.." -> "Warm light." + txt = re.sub(r"^[\s.,]+", "", txt) # leading ". " left by a removed clause + txt = re.sub(r"^\s*(?:and|,)\s+", "", txt) + txt = re.sub(r"\s+and\s*$", "", txt) + txt = re.sub(r"\s+\.", ".", txt) + return txt.strip(" ,.").strip() + ("." if txt.strip(" ,") else "") + + +# Pronoun by grammatical case: (subject, object, possessive). +_PRON_CASES = {"she": ("she", "her", "her"), + "he": ("he", "him", "his"), + "they": ("they", "them", "their")} +# A name right after one of these is an OBJECT ("walks over to Dan"), so it takes the +# object form. Anything else mid-sentence is treated as an object too, since that is +# where a bare name usually lands ("hands Dan a wrench", "asks Dan"). +_OBJECT_PREPS = ("to", "with", "at", "for", "from", "toward", "towards", "behind", + "beside", "near", "of", "on", "onto", "into", "over", "under", "past", + "by", "about", "around", "beneath", "against", "alongside", "opposite", + "between", "upon", "across", "after", "before", "beyond", "through") +# ...and after one of these (or a sentence end) it is a SUBJECT ("and Dan takes it"). +_SUBJECT_LEADS = ("and", "then", "but", "so", "as", "while", "when", "until", "because", + "if", "though", "although", "where", "who") + + +def _mask_quotes(text): + """Hide double-quoted spans behind placeholders so a rewrite cannot touch the + spoken words. Returns (masked_text, spans).""" + spans = [] + + def grab(m): + spans.append(m.group(0)) + return "\x00%d\x00" % (len(spans) - 1) + + return re.sub(r'["“][^"”]*["”]', grab, text), spans + + +def _unmask_quotes(text, spans): + return re.sub(r"\x00(\d+)\x00", lambda m: spans[int(m.group(1))], text) + + +def dedupe_person_mentions(body, active): + """Replace the SECOND and later mentions of a tracked person's name inside one + beat with the right pronoun. + + Naming one person twice in a shot is the single most reliable way to make + text-to-video render them twice -- "Kristy finds Dan ... she walks over to Dan" + puts two Dans in frame. Binding the description once (compose_persistent) fixes + the description, not the name itself, so the bare repeat still duplicates. + + Only fires where the result is unambiguous: + * the person's pronoun must be known (declared in their sheet, or a gender word + in their description) -- an undeclared person is left exactly as written; + * no OTHER person in the shot may share that pronoun, or 'he' could not be + traced back to the right one; + * words inside double quotes are never touched -- a name in a spoken line is + dialogue ("Kristy, over here"), not a second reference to stage. + The FIRST mention always survives, so the description still has a name to bind to + and the reader can still tell who the shot is about.""" + if not body: + return body + present = [n for n in active if n and active[n]] + if not present: + return body + by_pron = {} + for n in present: + p = _pronoun_of(active[n]) + if p: + by_pron.setdefault(p, []).append(n) + + masked, spans = _mask_quotes(body) + for n in present: + p = _pronoun_of(active[n]) + if not p or len(by_pron.get(p, [])) != 1: + continue # undeclared pronoun, or two people share it + subj, obj, poss = _PRON_CASES[p] + hits = list(re.finditer(r"\b" + re.escape(n) + r"(?:'s|’s)?\b", masked, re.I)) + if len(hits) < 2: + continue + for m in reversed(hits[1:]): # right to left, so earlier offsets stay valid + token = m.group(0) + raw_before = masked[:m.start()] + before = raw_before.rstrip() + prev = re.search(r"([A-Za-z']+)\s*$", before) + prev = prev.group(1).lower() if prev else "" + if token.endswith("s") and ("'" in token or "’" in token): + rep = poss + elif prev in _OBJECT_PREPS: + rep = obj + elif (not before) or before[-1] in ".!?;:" or prev in _SUBJECT_LEADS: + rep = subj + else: + rep = obj + # capitalize only at a real sentence/line start + if (not before) or before[-1] in ".!?" or raw_before.rstrip(" \t").endswith("\n"): + rep = rep.capitalize() + masked = masked[:m.start()] + rep + masked[m.end():] + return _unmask_quotes(masked, spans) + + +def compose_persistent(body, active, anchor_id, removed=None, departed=None, + count_subjects=False, speaking=False, front_load=False, + count_auto=False, silence_nonspeech=True): + """Assemble one shot's text WITHOUT duplicating subjects. + + Each present person's description is injected as a parenthetical at the FIRST + reference to them in the beat -- whether that reference is their NAME or a + resolvable PRONOUN ('she'/'he'). So 'she takes off her jacket' becomes 'she + (silver hair, grey shorts) takes off her jacket': described once, no name, no + duplicate subject. A person not referenced at all (by name or pronoun) is + omitted from that shot. The unnamed single subject is prepended as before. + + Pronoun tokens declared in a person's sheet ('Maya = she, ...') are used to + resolve 'she'/'he' but are stripped from the shown description. Keep the + anchor to scene/style with NO names.""" + count_prefix = "" # set when the count clause is front-loaded (LoRA runs) + listeners = set() # bound people keeping their mouths shut while another speaks + departed = set(departed or ()) + # A character who has LEFT the scene is never described again -- not even if a + # later pronoun could resolve to them. This is what stops an exited character + # being silently re-summoned into a later shot. + active = {k: v for k, v in active.items() if k not in departed} + # Collapse repeat NAME mentions to pronouns before anything is measured or bound: + # naming one person twice in a shot renders them twice, and the refs below must be + # computed against the text that will actually be emitted. + body = dedupe_person_mentions(body, active) + named = [n for n in active if n and active[n]] + unnamed = active.get("", []) + anchor_id = _scrub_removed(anchor_id, removed) + # Keep tracked people OUT of the always-on anchor: the beat binds them inline, + # so leaving them here too introduces each character twice per shot. + anchor_id = _strip_people_from_anchor(anchor_id, active) + # An UNNAMED sheet is emitted as a bare list in front of the beat, so it has to + # be closed off as its own sentence. Without the period it ran straight into the + # action -- "...red jacket, blue jeans, black boots Kristy walks around the + # garage" -- where "black boots Kristy" reads as one noun phrase. A NAMED sheet + # never had this problem: it binds as a parenthetical at the person's first + # mention instead of being prepended. + unnamed_txt = ", ".join(unnamed) if unnamed else "" + if unnamed_txt and unnamed_txt[-1] not in ".!?": + unnamed_txt += "." + prefix_bits = [x for x in (anchor_id, unnamed_txt) if x] + + if named: + names = list(named) + pron_map = _pron_map(active) + single = len(names) == 1 + low = body.lower() + + # first reference position for each present person (name first, else pronoun) + refs = {} + for n in names: + m = re.search(r"\b" + re.escape(n.lower()) + r"\b", low) + if m: + refs[n] = m.end() + for m in re.finditer(r"\b(she|he|they|her|him|them|his|their)\b", low): + person = _resolve_subject(m.group(1), names, pron_map, single) + if person and person not in refs: + refs[person] = m.end() + + # Nobody bound individually, but the beat addresses the cast in the plural: + # bind everyone with a roll-call in FRONT of the beat rather than rewriting + # the sentence. Prepending keeps the author's prose exactly as written -- + # 'they' can mean a subset, and expanding it in place would assert a cast + # list the author did not write. + # Who is LISTENING here? Only decidable when every attributed speaker is one + # of this shot's bound people and at least one bound person is not speaking. + # An unattributed quote ('she says' with two people on screen) names nobody, + # so nothing is constrained -- guessing would gag the wrong mouth. + listeners = set() + if speaking and silence_nonspeech and len(names) >= 2 and not unnamed: + speakers = _speakers_in(body, names) + if speakers: + bound_names = set(refs) or ( + {n for n in names} if _PLURAL_CAST.search(body) else set()) + if speakers & bound_names: + listeners = bound_names - speakers + + # The listener keeps their sheet's mouth-state items ("mouth closed", "lips + # together"): dropping them, as a speaking shot used to do for everyone, + # left the listening mouth with nothing holding it shut. + def _drop_mouth(n): + return speaking and n not in listeners + + roll_call = "" + if not refs and len(names) > 1 and _PLURAL_CAST.search(body): + bits = [] + for n in names: + desc = ", ".join(_clean_items(active[n], n, drop_mouth_state=_drop_mouth(n))) + bits.append(f"{n} ({desc})" if desc else n) + roll_call = ((", ".join(bits[:-1]) + " and " + bits[-1]) + + (" are both in this shot." if len(bits) == 2 + else " are all in this shot.")) + refs = {n: 0 for n in names} # for the subject count below + + if refs: + # inject from rightmost position first so earlier indices stay valid + if not roll_call: + for n in sorted(refs, key=lambda k: refs[k], reverse=True): + desc = ", ".join(_clean_items(active[n], n, drop_mouth_state=_drop_mouth(n))) + if desc: + pos = refs[n] + body = body[:pos] + f" ({desc})" + body[pos:] + else: + prefix_bits.append(roll_call) + # An EXPLICIT SUBJECT COUNT is the strongest prompt-side defence against + # the model rendering a character twice. Duplication gets much more + # likely below the model's native resolution: fewer pixels per subject + # pushes the sample away from the training distribution and the figure + # gets tiled. Stating the count (and "no other people") gives the model + # a hard target instead of leaving the number implicit. + # + # 'auto' also fires whenever THIS shot binds two or more people, whatever + # the resolution: two figures in frame is where tiling and merging + # happen even at native size, and the clause is the cheapest thing that + # holds the count down. + if count_subjects or (count_auto and len(refs) >= 2): + n_people = len(refs) + word = {1: "one", 2: "two", 3: "three", 4: "four", + 5: "five", 6: "six"}.get(n_people, str(n_people)) + noun = "person" if n_people == 1 else "people" + clause = (f"Exactly {word} {noun} in this shot, no duplicates, " + f"no other people in frame, no extra bodies, " + f"no repeated figures, no crowd. ") + if front_load: + # A distilled LoRA settles composition in its first step or two, + # so the count must be the FIRST thing in the prompt -- ahead of + # scene and style -- not buried after the anchor. + count_prefix = clause + clause = "" + else: + count_prefix = "" + body = clause + body + # If NOBODY is referenced by name or pronoun, this is a scenery/cutaway beat + # ("the hangar doors roll open"). Emit no people at all: the old grouped + # 'Kristy: ... Jon: ...' prefix both re-introduced names (the duplication + # pattern) and forced absent characters into shots they don't belong in. + + if listeners: + # The listening side of a dialogue shot gets the same physical mouth state + # a silent shot gets -- stated per person, positively, because at cfg 1 + # nothing suppresses; it can only be told what to do instead. + quiet = [] + for n in sorted(listeners): + term = _subject_term(n, active) + quiet.append(f"{term[0].upper() + term[1:]} stays silent through the line, " + "mouth closed, lips together, jaw still.") + body = body.rstrip(". ") + "." if body.strip() else body + body = body + " " + " ".join(quiet) + + prefix = " ".join(prefix_bits) + out = (prefix + " " + body).strip() if prefix else body.strip() + return (count_prefix + out).strip() + + +def extract_wardrobe(body): + """Pull a 'wardrobe: ...' directive line out of a beat body. Returns + (clean_body, wardrobe_or_None). The directive is a whole line starting with + 'wardrobe:' (case-insensitive), placed INSIDE a beat (not as its own blank- + line-separated paragraph, which would become its own shot). It's removed + from the body so the literal 'wardrobe:' text isn't stamped as an action.""" + kept, wardrobe = [], None + for ln in body.split("\n"): + if re.match(r"\s*wardrobe\s*:", ln, re.I): + wardrobe = ln.split(":", 1)[1].strip() + else: + kept.append(ln) + return "\n".join(kept).strip(), wardrobe + + +# --- anchor hazards --------------------------------------------------------- +# The anchor is stamped into EVERY shot, so anything in it has to be true of every +# shot. Four kinds of thing are not, and each fails in its own way. +_ANCHOR_PERSON = re.compile( + r"\b(skin|pores?|complexion|freckles?|stubble|face|facial|eyes?|lips?|mouth|" + r"hairs?|figure|portrait|subject|person|people|man|woman|men|women|girl|boy|" + r"he|she|him|her|his|hers|they|them|their)\b", re.I) +_ANCHOR_APPARATUS = re.compile( + r"\b(camera|camcorder|lens|sensor|tripod|gimbal|steadicam|dolly|crane|drone|" + r"handheld|hand-held|iphone|phone|gopro|dslr|webcam|filming|filmed|crew|" + r"operator|documentary|selfie|pov|point of view|shot on|taken with)\b", re.I) +_ANCHOR_FRAMING = re.compile( + r"\b(medium shot|close-?ups?|wide shot|long shot|full shot|two shot|" + r"over the shoulder|low angle|high angle|aerial|overhead shot|establishing shot)\b", re.I) +# A reflective surface shows whoever is in frame, and H3 renders that reflection as +# a SECOND figure standing in the room -- the same duplicated-subject failure a +# sub-native render produces, but caused by the scene text itself. Because it lives +# in the anchor, the mirror is present and doubling people on EVERY shot. +_ANCHOR_MIRROR = re.compile( + r"\b(mirrors?|mirrored|reflections?|reflexions?|reflective|reflecting)\b", re.I) + + +# Scene words -> the ambience they imply. Read from the ANCHOR, because the +# soundscape is global: it is stamped on every shot, so it has to describe the +# PLACE rather than what happens in any one beat. +# +# NOT ONE ENTRY NAMES A HUMAN SOUND. No chatter, no murmur, no crowd, no +# announcements, no footsteps-of-people. This node spends most of its silence +# machinery stopping H3 inventing a voice, and an auto-generated soundscape that +# says "busy cafe" would hand it one on every shot. A bar gets glassware and room +# tone; a station gets hall reverb and machinery. +_SOUNDSCAPE_CUES = ( + # weather and time first -- they layer over whatever the place is + (r"\bthunder|thunderstorm|lightning\b", "heavy rain and distant thunder"), + (r"\brain(y|ing|fall)?\b", "steady rain"), + (r"\bsnow(y|ing)?|blizzard\b", "muffled snowfall and thin wind"), + (r"\bfog|mist(y)?\b", "still muffled air"), + (r"\b(?:wind|windy|windswept|gale)\b", "gusting wind"), + # interiors + (r"\bhangar|warehouse|factory|silo\b", "cavernous interior, long reverb, distant metal ticks"), + (r"\bworkshop|garage|shed\b", "close interior room tone, faint metal clinks"), + (r"\bkitchen\b", "quiet room tone, faint appliance hum"), + (r"\bbathroom|shower\b", "tiled room tone, faint water drip"), + (r"\bcorridor|hallway|stairwell\b", "narrow interior reverb"), + (r"\bchurch|cathedral|chapel\b", "large stone reverb"), + (r"\bcave|tunnel|mine\b", "deep echoing reverb, dripping water"), + (r"\bbar\b|\bpub\b|\bcafe|\bdiner|restaurant\b", "low room tone, faint glassware and cutlery"), + (r"\bstation|platform|terminal|airport\b", "large hall reverb, distant machinery"), + (r"\boffice\b", "quiet room tone, faint ventilation hum"), + (r"\bhospital|clinic|ward\b", "quiet corridor tone, faint equipment beeps"), + # exteriors + (r"\bbeach|shore|ocean|\bsea\b|coast\b", "waves breaking, sea wind, distant gulls"), + (r"\bharbour|harbor|dock|pier|marina\b", "water lapping, hull and rigging creak, gulls"), + (r"\bforest|woods|jungle\b", "wind in leaves, birdsong"), + (r"\briver|stream|creek|waterfall\b", "running water"), + (r"\bdesert|dunes\b", "dry wind and drifting grit"), + (r"\bmountain|cliff|ridge\b", "high open wind"), + (r"\bfield|meadow|farm|barn|pasture\b", "open wind, insects, distant birds"), + (r"\brooftop|roof\b", "open wind, distant city hum"), + (r"\balley|alleyway\b", "close urban reverb, distant traffic"), + (r"\bcity|street|urban|downtown|sidewalk|pavement\b", + "distant traffic hum, occasional horn"), + (r"\bhighway|motorway|freeway\b", "passing vehicles and tyre noise"), + (r"\bpark|garden\b", "wind in trees, distant birds"), + # things that make noise wherever they are + (r"\bcampfire|bonfire|fireplace|fire\b", "crackling fire"), + (r"\baircraft|airplane|propeller|\bplane\b", "propeller drone"), + (r"\btrain|railway|locomotive\b", "rail rumble"), + (r"\bboat|ship|yacht\b", "hull creak and water"), + (r"\bvan\b|\bcar\b|\btruck\b|engine\b", "engine idle and road noise"), + (r"\bgenerator|machinery|turbine\b", "low machine hum"), + (r"\bnight|midnight|nocturnal\b", "night air, faint insects"), +) +# Tried only when nothing above matched: a generic interior is better than silence, +# but naming it alongside a specific one ("kitchen ... and also a room") is noise. +_SOUNDSCAPE_FALLBACK = ( + (r"\b(?:bedroom|living room|apartment|house|home|room|indoors|interior)\b", + "quiet indoor room tone"), + (r"\b(?:outside|outdoors|exterior)\b", "open outdoor air, faint distant wind"), +) + +# Camera and style vocabulary, stripped before matching. "shallow depth of FIELD" +# is not a meadow, and an anchor is mostly camera language -- exactly the words a +# naive scan trips over. +_CAMERA_WORDS = re.compile( + r"\b(?:depth of field|field of view|shallow focus|deep focus|focal length|" + r"\d+\s*mm|f/\d+(?:\.\d+)?|bokeh|anamorphic|handheld|dolly|steadicam|" + r"colour grade|color grade|film grain|motion blur|golden hour)\b", re.I) + +_SOUNDSCAPE_MAX = 4 # more than this reads as a sound-effects list, not a bed + + +def derive_soundscape(anchor, beats=()): + """An ambient bed inferred from the scene, or "" when nothing matches. + + Reads the ANCHOR first, since that is the permanent scene and the soundscape is + global. Beats are read only as a fallback -- a location often appears in the + first beat rather than the anchor -- and never for one-off actions, which would + stamp a single shot's noise onto the whole chain. + + Capped, ordered, de-duplicated. Weather layers before place, because "rain" over + "a city street" is the useful order to read.""" + for source in (anchor or "", " ".join(beats or ())): + low = _CAMERA_WORDS.sub(" ", source.lower()) + seen, out = set(), [] + for rx, phrase in _SOUNDSCAPE_CUES: + if len(out) >= _SOUNDSCAPE_MAX: + break + if phrase not in seen and re.search(rx, low, re.I): + seen.add(phrase) + out.append(phrase) + if not out: + for rx, phrase in _SOUNDSCAPE_FALLBACK: + if re.search(rx, low, re.I): + out.append(phrase) + break + if out: + return ", ".join(out) # the anchor described the place; stop there + return "" + + +def anchor_warnings(anchor): + """Things in the anchor that will misfire because it repeats on every shot. + + Pure text, no model. Each of these has cost a real render: face words put a face + in an empty establishing frame, apparatus words render the equipment, framing + pins every shot to one size, and clothing here is immutable so a removal can + never stick.""" + a = (anchor or "").strip() + if not a: + return [] + out = [] + def found(rx): + return sorted({m.group(0).lower() for m in rx.finditer(a)}) + p = found(_ANCHOR_PERSON) + if p: + out.append(f"person/face words in the anchor ({', '.join(p)}) -- the anchor is stamped on " + f"EVERY shot, so these arrive in shots with nobody in them and can render a " + f"face in an empty frame; move them to character_memory, which is only emitted " + f"where that person appears") + q = found(_ANCHOR_APPARATUS) + if q: + out.append(f"camera/apparatus words in the anchor ({', '.join(q)}) -- naming the equipment " + f"can render the equipment, or someone holding it; describe the IMAGE instead " + f"('shallow depth of field' rather than '35mm lens', 'fine grain' rather than " + f"'sensor grain')") + f = found(_ANCHOR_FRAMING) + if f: + out.append(f"framing in the anchor ({', '.join(f)}) -- this pins every shot to that size; " + f"put framing in the beats so it can change shot to shot") + mr = found(_ANCHOR_MIRROR) + if mr: + out.append(f"mirror/reflection words in the anchor ({', '.join(mr)}) -- H3 renders a " + f"reflection as a SECOND figure standing in the room, so this duplicates " + f"whoever is on screen on EVERY shot; drop it from the anchor, or make each " + f"shot that shows one say what is in the mirror and why") + garments = sorted({w.lower() for w in re.findall(r"[A-Za-z][\w\-]*", a) + if garment_zones(w)}) + if garments: + out.append(f"clothing in the anchor ({', '.join(garments)}) -- the anchor is immutable, so " + f"it re-applies the garment on every shot and a removal cannot stick; put " + f"clothing in character_memory, the only channel that can change mid-chain") + return out + + +def anchor_contributes_nothing(anchor, char_memory=""): + """True when the paragraph about to be consumed as the identity anchor would add + NOTHING to any shot -- i.e. taking it as the anchor silently DELETES it. + + The anchor is stamped into every shot, so _strip_people_from_anchor removes any + sentence that names a tracked character (otherwise that character is introduced + twice per shot and the model renders them twice). A first paragraph that is + *itself* an action beat about a tracked person -- "Kristy walks around in a garage + looking for engine parts." -- is therefore stripped to nothing: the user loses that + shot AND the only scene text they wrote, with just a mild note to say so. + + Returns False whenever the paragraph carries something real: a 'wardrobe:' line (it + seeds the wardrobe channel), or prose that survives the strip. So a normal + identity/scene anchor is never touched, and with no character_memory nothing is + tracked, nothing is stripped, and this cannot fire.""" + anchor_id, anchor_wardrobe = extract_wardrobe((anchor or "").strip()) + if anchor_wardrobe: # seeds the wardrobe channel -> it matters + return False + if not anchor_id.strip(): + return False + active = parse_wardrobe((char_memory or "").strip()) + return not _strip_people_from_anchor(anchor_id, active).strip(" .,") + + +# A sentence that STAGES something: a name or pronoun subject followed by a verb +# ("Kristy walks", "She finds", "Dan answers"). An anchor is scene and style -- noun +# phrases and lists ("An open 4 bay car garage.", "natural lighting, flat lighting") -- +# and does not match. +_ACTION_SENT = re.compile(r"^\s*(?:He|She|They|[A-Z][a-z]+)" + r"(?:\s+and\s+(?:[A-Z][a-z]+|he|she|they))?" + r"\s+[a-z]+(?:s|ed|ing)\b") + + +def anchor_is_action_beat(anchor, later_paras=()): + """True when the paragraph about to be consumed as the anchor is plainly a BEAT. + + anchor_contributes_nothing() only catches this when the character is tracked in + character_memory -- with no sheet, nothing is tracked, nothing is stripped, and an + action paragraph sails through to become the anchor. That is the common case: a + prompt written as three beats, no character sheet, renders as two shots with the + first beat demoted to a header stamped on the other two. + + Fires only when EVERY sentence stages an action AND the subject recurs later, so a + mixed paragraph ("Kristy stands by the plane. A cinematic hangar, warm light.") + keeps its scene text and stays an anchor, and a style list never matches at all. A + pronoun subject is accepted on its own -- 'She walks in.' cannot be scene text.""" + body, wardrobe = extract_wardrobe((anchor or "").strip()) + if wardrobe: # seeds the wardrobe channel -> it matters + return False + body = body.strip() + if not body: + return False + sents = [s for s in re.split(r"(?<=[.!?])\s+", body) if s.strip()] + if not sents or not all(_ACTION_SENT.match(s) or has_speech(s) for s in sents): + return False + m = re.match(r"\s*([A-Za-z]+)", body) + if not m: + return False + subj = m.group(1).lower() + if subj in ("he", "she", "they"): + return True + # A NAME is only a beat subject if the prompt goes on using it. This is what keeps + # a one-word style lead ("Cinematic lighting, warm tones.") from reading as an + # action: 'cinematic' never comes back as a subject in the beats. + later = " ".join(later_paras or ()).lower() + return bool(re.search(r"\b" + re.escape(subj) + r"\b", later)) + + +# --- props: objects that must survive the shot boundary --------------------- +# Nouns that are never a prop worth carrying: parts of the frame, parts of a body, +# and abstractions. Binding these would produce "the same ground from the previous +# shot", which is noise at best. +_NOT_A_PROP = { + "ground", "air", "sky", "floor", "ceiling", "background", "foreground", "distance", + "camera", "frame", "shot", "scene", "screen", "view", "angle", "light", "lighting", + "shadow", "sun", "moment", "time", "day", "night", "morning", "evening", "way", + # anatomy -- a body part is never a prop to carry between shots. Without these, + # "reveals a nipple" was tracked as an object and a later mention got the full + # continuity treatment: "exactly one nipple in this shot, the same nipple from + # the previous shot, in the same place". + "head", "face", "eyes", "eye", "hand", "hands", "arm", "arms", "leg", "legs", + "body", "hair", "mouth", "jaw", "lips", "shoulder", "shoulders", "chest", "torso", + "waist", "hip", "hips", "knee", "knees", "foot", "feet", "ankle", "wrist", "neck", + "throat", "stomach", "belly", "navel", "skin", "thigh", "thighs", "breast", + "breasts", "nipple", "nipples", "genitals", "genitalia", "penis", "vagina", + "vulva", "groin", "crotch", "buttock", "buttocks", "backside", "bottom", + # frame-relative words and bare determiners/pronouns, which are never objects + "back", "front", "side", "top", "edge", "middle", "end", "left", "right", + "centre", "center", "the", "a", "an", "it", "this", "that", "these", "those", + "they", "them", "her", "his", "him", "its", "one", "other", "another", "something", +} +# Where a prop's NAME stops and its circumstance begins: "a van PARKED in the bay", +# "a barn WITH a red roof". Same idea as _ITEM_DETAIL for garments. +_PROP_TAIL = re.compile( + r"\b(?:parked|standing|sitting|leaning|lying|resting|waiting|stopped|covered|" + r"filled|loaded|painted|marked|in|on|at|by|near|beside|behind|under|over|with|" + r"and|that|which|down|across|toward|towards|from|" + # a VERB ends the noun phrase too: "a bare breast CATCHES the light" was being + # read four words deep and keyed on the trailing "the". + r"is|are|was|were|has|have|had|catches|catch|presses|press|hangs|hang|rests|" + r"sits|sit|stands|stand|lies|lie|falls|fall|moves|move|shows|show|reveals|reveal|" + r"appears|appear|becomes|become|looks|look|seems|seem|glints|glows|shines)\b") + + +def introduced_props(text): + """{noun: phrase} for objects this text introduces INDEFINITELY -- "a white van" + -> {"van": "white van"}. These are the things a later shot can refer back to.""" + out = {} + body = _PICTURE_TAG.sub(" ", text or "") + # Scan from each article WITHOUT consuming what follows it: a greedy match over + # "a van and a truck" would swallow the truck's own article and lose it. + # Case-insensitive, because beats routinely OPEN with the object -- "A white van + # sits outside." -- and a case-sensitive scan silently tracked nothing there, + # while bind_props/dedupe_prop_mentions/repeated_props below are all re.I. + for m in re.finditer(r"\b(?:a|an)\s+", body, re.I): + ahead = body[m.end():] + ahead = re.split(r"[,.;:!?]", ahead)[0] + words_ahead = re.findall(r"[A-Za-z][\w\-]*", ahead)[:4] + phrase = " ".join(words_ahead) + cut = _PROP_TAIL.search(phrase.lower()) + if cut and cut.start() > 0: + phrase = phrase[:cut.start()].strip() + elif cut: + continue # starts with a tail word -- not a name + words = phrase.split() + if not words: + continue + noun = words[-1].lower() + if noun in _NOT_A_PROP or len(noun) < 3: + continue + out.setdefault(noun, phrase.strip()) + return out + + +def bind_props(body, props): + """Rewrite the FIRST definite reference to each carried prop so it names the + object instead of assuming one. Returns (body, [nouns bound]). + + "the van" in a later shot has no antecedent -- each shot is its own generation, + and nothing in that prompt describes a van. The model invents one, which is how + a second van appears in the frame while the first is still there. Naming it and + asserting it is the SAME one is what binds the two shots together.""" + if not props or not body: + return body, [] + masked, spans = _mask_quotes(body) + bound = [] + for noun, phrase in props.items(): + pat = re.compile(r"\bthe\s+(" + re.escape(noun) + r")\b", re.I) + m = pat.search(masked) + if not m: + continue + masked = masked[:m.start()] + f"the same {phrase}" + masked[m.end():] + bound.append(noun) + return _unmask_quotes(masked, spans), bound + + +def dedupe_prop_mentions(body, nouns): + """Collapse repeat mentions of the SAME object within one beat. + + "drives a van ... gets out of the van ... walks to the back of the van" is three + vans in one prompt, and repetition is how a video model ends up rendering three. + The node already does exactly this for people -- naming someone twice in a beat + is the most reliable way to get two of them -- and an object is no different. + + Only fires when a single carried object is in play, so "it" cannot be ambiguous, + and never inside quoted speech.""" + if not body or len(nouns) != 1: + return body, 0 + noun = nouns[0] + masked, spans = _mask_quotes(body) + hits = list(re.finditer(r"\b(?:the|that|this)\s+" + re.escape(noun) + r"\b", masked, re.I)) + if len(hits) < 2: + return body, 0 + # keep the first definite mention; the rest become pronouns + for m in reversed(hits[1:]): + masked = masked[:m.start()] + "it" + masked[m.end():] + return _unmask_quotes(masked, spans), len(hits) - 1 + + +def repeated_props(body, own): + """Objects a beat introduces and then refers back to definitely, in the SAME + beat -- the case a cross-shot carry never sees.""" + out = [] + for noun in own: + if re.search(r"\b(?:the|that|this)\s+" + re.escape(noun) + r"\b", body or "", re.I): + out.append(noun) + return out + + +def prop_count_clause(nouns): + """Positive count for objects, matching the subject-count guard's shape.""" + if not nouns: + return "" + bits = [f"exactly one {n} in this shot" for n in nouns] + s = ", ".join(bits) + return " " + s[0].upper() + s[1:] + "." + + +def prop_continuity_clause(bound, props): + """One short sentence pinning a carried prop to the previous shot's object. + + Re-describing it is not enough on its own: "a white van" in shot 1 and "a white + van" in shot 2 are two white vans. The clause states identity with the previous + shot, and states the COUNT positively. + + It deliberately does NOT say "no second van". Naming the unwanted thing is how + "she is no longer wearing the red jacket" put the jacket back on: to a video + model a mention is a presence cue and a negation is weak, so "no second van" + puts a second van in the text. The subject-count guard already had this right + for people -- it leads with "Exactly one person" -- and props follow the same + shape.""" + if not bound: + return "" + bits = [] + for noun in bound: + phrase = props.get(noun, noun) + bits.append(f"exactly one {noun} in this shot, the same {phrase} from the previous shot, " + f"in the same place") + s = "; ".join(bits) + return " " + s[0].upper() + s[1:] + "." + + +# Cues that the quoted words are PRINTED in the scene rather than spoken aloud. +# Kept to things that are unambiguously written surfaces or reading/marking verbs. +_WRITTEN_CUE = re.compile( + r"\b(?:reads?|reading|marked|labell?ed|titled|captioned|headlined|written|" + r"printed|engraved|stamped|embroidered|scrawled|painted|spells?|spelled|" + r"signs?|posters?|banner|placard|plaque|sticker|label|headline|" + r"caption|graffiti|screen|display|monitor|billboard|tattoo|note|letter|" + r"envelope|book|page|menu|ticket|receipt|badge|nameplate)\b", re.I) + +# ...and the verbs that mean someone said it out loud. +_SPOKEN_CUE = re.compile( + r"\b(?:says?|said|saying|asks?|asked|asking|replies|replied|answers?|answered|" + r"shouts?|shouted|yells?|yelled|calls?|called|whispers?|whispered|murmurs?|" + r"murmured|mutters?|muttered|adds?|added|tells?|told|cries|cried|barks?|" + r"barked|snaps?|snapped|breathes?|offers?|insists?|repeats?|begins?|continues?|" + r"declares?|announces?|responds?|responded|urges?|warns?|pleads?|laughs?)\b", + re.I) + + +def has_speech(body): + """True only if a beat contains ACTUAL scripted speech -- double-quoted words + or an explicit ... tag. Bare speech VERBS ('calls out', 'tells', 'says' + with no quoted line) deliberately do NOT count: unscripted speech is exactly + what H3 fills with gibberish, so those beats get silenced too. If you want + someone to speak, quote the line: She says, "Ready for departure." + Apostrophes/single quotes never count (they'd false-fire on "she's"). + + WRITTEN text in quotes is not speech. A sign, a label, a headline -- 'reads the + sign marked "EXIT"' -- used to make the whole shot count as dialogue, so it got + neither the lips-closed clause nor the no-voice soundscape and the characters + stood there opening their mouths. Nothing in the beat was ever spoken.""" + if not body: + return False + if re.search(r".*?", body, re.S): + return True + for m in re.finditer(r'["\u201c\u201d].+?["\u201c\u201d]', body, re.S): + # Look at what introduces this quote, and let the NEAREST cue decide. + # "Mara reads the sign, then says 'we go left'" is speech: 'says' sits + # closer to the quote than 'reads' does. Comparing presence rather than + # position got that backwards. + lead = body[max(0, m.start() - 60):m.start()].lower() + written = [x.end() for x in _WRITTEN_CUE.finditer(lead)] + spoken = [x.end() for x in _SPOKEN_CUE.finditer(lead)] + if written and (not spoken or written[-1] > spoken[-1]): + continue # printed in the scene, nobody said it + return True + return False + + +def _spoken_quotes(body): + """Each double-quoted span that IS speech, skipping printed text -- the same + nearest-cue rule has_speech uses, but returning the spans so each one can be + attributed to whoever introduced it.""" + body = body or "" + out = [] + for m in re.finditer(r'["\u201c\u201d](.+?)["\u201c\u201d]', body, re.S): + lead = body[max(0, m.start() - 60):m.start()].lower() + written = [x.end() for x in _WRITTEN_CUE.finditer(lead)] + spoken = [x.end() for x in _SPOKEN_CUE.finditer(lead)] + if written and (not spoken or written[-1] > spoken[-1]): + continue + out.append(m) + return out + + +def _speakers_in(body, names): + """Which tracked names are ATTRIBUTED to a spoken line in this beat. + + Attribution needs the name ADJACENT to a speech verb -- 'Jon says:' before + the quote, or 'said Jon' just after the close. Presence anywhere in a window + is not enough: in '"Open it." Mara steps back.' Mara sits right after the + quote and would be credited with saying it, and in 'Mara steps back. Jon + says: "..."' she sits inside any naive look-back window too -- so the lead + is cut at the last sentence boundary and the name must sit within a few + words of the verb on either side.""" + found = set() + text = body or "" + for m in _spoken_quotes(text): + lead = re.split(r"[.!?\u2026\n;]", + text[max(0, m.start() - 60):m.start()])[-1].lower() + tail = re.split(r"[.!?\u2026\n;]", text[m.end():m.end() + 45])[0].lower() + for frag in (lead, tail): + cues = list(_SPOKEN_CUE.finditer(frag)) + if not cues: + continue + for n in names: + if not n: + continue + esc = re.escape(n.lower()) + for nm in re.finditer(rf"\b{esc}\b", frag): + for c in cues: + if nm.end() <= c.start() and c.start() - nm.end() <= 16: + found.add(n) # 'Jon says: ' + elif nm.start() >= c.end() and nm.start() - c.end() <= 4: + found.add(n) # 'said Jon' + return found + + +def _emphasis_quotes(body): + """Quoted spans that LOOK like emphasis rather than dialogue: a single word, + no terminal punctuation inside the quotes, and no speech verb introducing + them. She gave him a "look" is prose; He whispers "now" is a real line -- + the spoken cue keeps genuine single-word dialogue out of this list.""" + out = [] + for m in _spoken_quotes(body): + inner = m.group(1).strip() + words = inner.split() + if len(words) == 1 and not re.search(r"[.!?\u2026]\s*$", inner): + lead = (body or "")[max(0, m.start() - 60):m.start()].lower() + if not _SPOKEN_CUE.search(lead): + out.append(inner) + return out + + +# Silence is stated as a described PHYSICAL STATE, which H3 follows far better than +# an appended negation -- but NOT in the leading position it used to occupy. +# +# Opening every silent shot with "mouth closed, lips together, jaw still" put face +# anatomy in the first tokens the model reads, and a distilled LoRA fixes +# composition in its first step or two. The result was a face rendered at the start +# of shots -- including scenery shots with nobody in them at all, which is where it +# was unmistakable. The mouth state now follows the action instead of preceding it, +# and it is skipped entirely on a shot with no people, where "everyone is silent +# with their mouth closed" describes nobody and only invites a face. +# +# The audio half of the babble fix does not depend on this: the no-voice soundscape +# line and mute_nonspeech_audio both still apply. +# Extra limbs, duplicated hands, a third arm. There is exactly one lever for this: +# H3 is CFG-FREE at cfg 1, and comfy/samplers.py:610 sets uncond_ = None at that +# scale, so the negative prompt is NEVER EVALUATED. "extra limbs, deformed hands" +# in a negative does nothing at all on this model. +# +# So it has to be said POSITIVELY, in the same shape as the subject count clause +# that already stops duplicate people. Stating a number gives the model a target; +# negating one just puts the word in the prompt, and on this model a mention is a +# presence cue -- which is why this says what the body HAS and never what it lacks. +# +# Placed per-shot, never in the anchor. Anchor body words are what burned a face +# into the opening frames of every shot, found by bisection, and limb words there +# would carry the same risk on every shot including scenery. +# How many frames of grace a mouth gets to close before its frame is handed on. +# ~125ms at 24fps -- the tail of a syllable. Applied ONLY at a dialogue -> silence +# boundary, where the next shot's keyframe would otherwise be an open mouth +# mid-word, and where no amount of lips-closed text can outvote it. +MOUTH_SETTLE_FRAMES = 3 + +ANATOMY_STATE = (" Each person has one head, two arms, two hands with five fingers on each hand, " + "and two legs with two feet. Each arm joins the body at one shoulder and runs " + "shoulder to elbow to wrist to hand; each leg joins at one hip and runs hip to " + "knee to ankle to foot. The parts stack in order: head on the neck, neck on the " + "shoulders, arms hanging along the sides of the torso, legs under the hips. Every " + "limb moves only with the person it belongs to. Between the legs there is one groin.") + +# Two bodies in physical contact. Position-AGNOSTIC on purpose: a dictionary of named +# positions would be endless, and the model already knows more names than any list +# would hold. What it gets wrong is not the name, it is the geometry -- so what gets +# stated is the geometry, and these invariants hold for every arrangement: +# +# ownership limbs belong to the person they are attached to. Two overlapping +# bodies is exactly when an arm gets reassigned to the wrong torso, +# or grown a second time. +# separation they meet AT the skin. Bodies passing into each other is the +# solidity failure again, between two bodies rather than a body and +# a table, and the same positive phrasing is the only lever. +# stable roles whoever is above stays above. Positions morph mid-shot because +# nothing says the arrangement is fixed. +# support weight rests somewhere real, which is what stops a body floating in +# a pose that nothing is holding up. +_CONTACT_CUE = re.compile( + r"\b(?:straddl\w+|astride|mount\w+|on top of|underneath|beneath|" + r"embrac\w+|entwin\w+|intertwin\w+|wrapped around|arms around|legs around|" + r"holds?|holding|grips?|gripping|clutch\w+|press\w+ against|pinned|" + r"lies? on|lying on|lies? under|lying under|lies? beside|lying beside|" + r"kneels? (?:behind|before|between|in front of)|sits? on|sitting on|" + r"behind her|behind him|behind them|face to face|facing each other|" + r"against (?:her|him|them)|body to body|skin to skin|" + r"in (?:her|his|their) lap|carries|carrying|lifts?|lifting|" + r"leans? (?:on|against|over)|bent over|" + r"sex|intercourse|making love|coupling|position)\b", re.I) + +CONTACT_STATE = ( + " Two bodies in contact form ONE fixed arrangement. Each person keeps their own head, " + "two arms and two legs, each limb joined to the body it belongs to and moving with that " + "person. The bodies meet at the surface of the skin and rest against each other there, " + "each keeping its own solid volume. Whoever is above stays above, whoever is below stays " + "below, and whoever is behind stays behind, the same arrangement held for the whole shot " + "and read the same from every angle the camera takes. The weight rests on whatever is " + "supporting it, and both bodies stay in proportion to each other.") + +# Beats where a body changes orientation or position -- the moments a pose can be +# reached without the frames in between. A head that arrives at a new angle without +# passing through the intermediate ones is the "neck snap": not a wrong pose, a +# missing path between two right ones. +_MOTION_CUE = re.compile( + r"\b(?:turn|turns|turning|turned|spin|spins|spinning|pivot|pivots|" + r"look|looks|looking|looked|glance|glances|glancing|face|faces|facing|" + r"nod|nods|nodding|shake|shakes|shaking|tilt|tilts|tilting|" + r"lean|leans|leaning|bend|bends|bending|swing|swings|swinging|" + r"walk|walks|walking|run|runs|running|step|steps|stepping|" + r"rise|rises|rising|stand|stands|standing|sit|sits|sitting|" + r"reach|reaches|reaching|raise|raises|raising|lower|lowers|" + r"kneel|kneels|climb|climbs|climbing|follow|follows|following|" + r"approach|approaches|enter|enters|exit|exits|leave|leaves|" + # High-jerk motion -- struggling, pulling, twisting. These are exactly the + # beats where a limb arrives without its path or spasm-renders, and 'auto' + # used to stay silent through all of them because a struggle is not an + # orientation change. A restrained character's beats are almost entirely + # made of these, which is where cuffs visibly tear. + r"struggl\w*|pull\w*|yank\w*|jerk\w*|tug\w*|twist\w*|writh\w*|thrash\w*|" + r"flail\w*|squirm\w*|strain\w*|crawl\w*|danc\w*|push\w*|stagger\w*|" + r"stumbl\w*|sway\w*|trembl\w*|kick\w*|lung(e|es|ing))\b", re.I) + +# Positive throughout, for the same reason as the solidity state: the negative is +# never evaluated at cfg 1, and "the head does not snap round" names a head snapping +# round. What is missing in a snap is the PATH, so the path is what gets stated. +MOTION_STATE = (" Movement is continuous and carries its own weight: the head and body turn " + "through every position on the way, at one steady speed, the neck following " + "the shoulders and the shoulders following the hips, so each pose is reached " + "by travelling to it.") + + +# Things a body has to stop at. Deliberately excludes anything genuinely passable -- +# a curtain, a beaded screen, smoke -- because asserting those are solid would be +# wrong, and excludes vague ones ("edge", "side") that would fire on prose. +_SOLID_NOUNS = re.compile( + r"\b(?:wall|walls|door|doors|doorway|gate|fence|railing|rail|banister|" + r"table|tables|desk|desks|counter|countertop|bench|workbench|" + r"chair|chairs|stool|couch|sofa|armchair|bed|beds|bunk|" + # No "chest" or "trunk": both are furniture AND body parts, and in a node that + # tracks bared zones the body reading is the likelier one -- "the chest is solid + # and occupies real space" was calling a person's chest a piece of furniture. + # "wardrobe" stays: the garment sense is a collection, never a thing in frame. + r"crate|crates|box|boxes|barrel|cabinet|dresser|wardrobe|" + r"shelf|shelves|bookcase|pillar|column|post|beam|" + r"stair|stairs|staircase|step|steps|ladder|" + r"window|windowpane|pane|windshield|" + r"van|car|truck|bike|motorcycle|trailer|" + r"floor|ground|ceiling|" + r"rock|rocks|boulder|tree|trees|" + r"barrier|partition|roller door|shutter)\b", re.I) + +# Positive throughout. "Does not walk through the wall" names walking through a wall, +# and a mention is a presence cue -- the same reason a removed garment came back when +# the prompt said it was gone. Say what the bodies DO instead. +SOLIDITY_STATE = (" Solid things stay solid: a body stops where it meets a surface, feet rest on " + "the floor, hands press against what they touch, and anyone crossing the space " + "walks around the furniture rather than across it.") + +LIPS_CLOSED_STATE = (" Everyone in this shot is silent with their mouth closed and lips together, " + "jaw still, not talking.") +LIPS_CLOSED_TAIL = " No speech, no dialogue, no lip movement, no mouth movement." + +# The lips-closed clause constrains the PICTURE only. H3 generates audio from its +# own fields, and an ABSENT `overall_soundscape:` leaves that branch unconditioned +# -- which is exactly when it invents speech-like babble under a silent shot. So a +# silenced shot always gets a soundscape line, and it says no voices outright. +NO_VOICE_SOUNDSCAPE = ("ambient background sound and room tone only, no voices, no speech, " + "no talking, no whispering, no singing, no vocal sounds") +NO_VOICE_CLAUSE = ", no voices, no speech, no talking, no vocal sounds" +NO_VOICE_SPEECH_SOUNDSCAPE = ("ambient background sound and room tone only, no speech, no dialogue, " + "no talking, no singing, no whispering, no spoken words") +NO_VOICE_SPEECH_CLAUSE = ", no speech, no dialogue, no talking, no singing, no whispering, no spoken words" + + +# A beat that refers to the cast only in the PLURAL ("they face each other") used +# to bind nobody: _resolve_subject() maps a pronoun to ONE person, and 'they' with +# two people resolves to neither. The shot then silently described no one -- losing +# both characters' descriptions and, after a removal, their exposure markers, so a +# stripped character quietly went back to being unmarked. +# +# Bare 'them'/'their' are deliberately NOT here. They refer to objects at least as +# often as to people -- "she steps out of them" is a garment, "light floods through +# them" is a pair of doors -- and this fires only when nobody was bound by name or +# singular pronoun, which is exactly the scenery-beat case that must stay empty. +_PLURAL_CAST = re.compile( + r"\b(?:they|themselves|both|each other|one another|" + r"the two of them|all of them)\b", re.I) + + +def person_referenced(body, name, active): + """Is this person actually in the beat -- by name, or by a pronoun that resolves + to them? Used to keep a wardrobe statement out of a shot they aren't in: saying + "she is no longer wearing the jacket" in a shot about someone else SUMMONS her + into it, which is the duplication failure the whole builder exists to avoid.""" + low = (body or "").lower() + if name and re.search(r"\b" + re.escape(name.lower()) + r"\b", low): + return True + names = [n for n in active if n] + pron_map = _pron_map(active) + single = len(names) == 1 + for m in re.finditer(r"\b(she|he|they|her|him|them|his|their)\b", low): + if _resolve_subject(m.group(1), names, pron_map, single) == name: + return True + return False + + +def person_in_shot(body, name, active, departed=()): + """Is this person IN this shot -- by name, by a resolvable pronoun, or as part + of a cast addressed in the plural? + + The single presence test. person_referenced() alone is not it: it resolves a + pronoun to ONE person, so 'they' and 'both of them' answer False for everybody, + and any clause gated on it silently skips a beat that binds the whole cast. + + That exact bug has now been written twice -- once in the mouth-state gate + (a plural beat got no lips-closed clause, so those shots babbled) and again in + the restraint clause (a plural beat dropped the physical constraint, so the + restraints appeared to break). Both are gated on this function now, so a third + caller cannot rediscover it.""" + if person_referenced(body, name, active): + return True + present = [n for n in (active or {}) if n and n not in (departed or ())] + return len(present) > 1 and bool(_PLURAL_CAST.search(body or "")) + + +def _subject_term(name, active): + """How to refer to a person in a generated clause: their declared PRONOUN when + it identifies them uniquely, otherwise their name. Pronoun-first is the rule the + whole builder follows -- a bare name is a fresh introduction, and introducing + someone twice in a shot is what makes the model render them twice.""" + pron = _pronoun_of(active.get(name, [])) + if pron: + holders = [n for n in active if n and _pronoun_of(active[n]) == pron] + if len(holders) == 1: + return pron + return name + + +# Which part of the body a garment covers. Only two zones matter here, and the +# question each answers is strictly "is this part of the body still COVERED?" -- a +# removal that empties a zone is the one that renders as nudity. +# +# That question, not "is this clothing?", decides what belongs in these sets. A +# garter belt, stockings, hold-ups, socks, gloves and a scarf are all clothing and +# all deliberately absent: they leave the zone bare, so counting them as cover +# would SUPPRESS the exposure warning exactly when it is needed. Anything not +# listed maps to no zone, which is the safe default -- it can never mask a warning, +# it can only fail to volunteer an under-layer. +_ZONE_LOWER = { + # trousers and their families + "pants", "trousers", "jeans", "denims", "slacks", "chinos", "khakis", "cargos", + "cords", "corduroys", "joggers", "sweatpants", "sweats", "trackpants", + "breeches", "jodhpurs", "capris", "culottes", "bloomers", "harems", + # skirts + "skirt", "miniskirt", "midiskirt", "maxiskirt", "kilt", "sarong", "lungi", "dhoti", + # shorts + "shorts", "boardshorts", "trunks", "speedos", "hotpants", + # legwear that DOES cover the pelvis + "leggings", "jeggings", "treggings", "tights", "pantyhose", "pantihose", + # underwear and lingerie bottoms + "briefs", "boxers", "boxershorts", "panties", "knickers", "underpants", + "underwear", "undies", "drawers", "thong", "g-string", "gstring", "tanga", + "boyshorts", "boyshort", "hipsters", "jockstrap", "loincloth", "bottoms", + # nappies + "diaper", "diapers", "nappy", "nappies", "pull-ups", "pullups", "pull-up", +} +_ZONE_UPPER = { + # shirts and tops + "shirt", "t-shirt", "tshirt", "tee", "top", "crop-top", "croptop", "tanktop", + "tank", "blouse", "jersey", "polo", "henley", "turtleneck", "flannel", "smock", + "tunic", "kurta", "halter", "bandeau", "tube-top", + # knitwear and outerwear + "sweater", "jumper", "hoodie", "sweatshirt", "pullover", "cardigan", "jacket", + "coat", "blazer", "vest", "waistcoat", "gilet", "anorak", "parka", "windbreaker", + "bomber", "peacoat", "trenchcoat", "raincoat", "poncho", "shawl", "cape", "cloak", + "thermal", "thermals", + # lingerie and underlayers for the torso + "bra", "brassiere", "bralette", "bustier", "corset", "basque", "camisole", "cami", + "singlet", "undershirt", "brasiere", +} +_ZONE_BOTH = { + # one-piece garments covering torso AND pelvis + "dress", "gown", "frock", "pinafore", "jumpsuit", "romper", "playsuit", "catsuit", + "unitard", "leotard", "bodysuit", "bodystocking", "onesie", "overalls", + "dungarees", "coveralls", "boilersuit", "snowsuit", "wetsuit", "drysuit", + "robe", "bathrobe", "housecoat", "dressinggown", "kimono", "kaftan", "caftan", + "abaya", "sari", "saree", "toga", + # sleepwear and lingerie one-pieces + "nightgown", "nightie", "nightdress", "negligee", "babydoll", "teddy", "slip", + "chemise", "pyjamas", "pajamas", "pjs", + # swimwear counted as a set + "swimsuit", "swimming-costume", "maillot", "bikini", "tankini", "monokini", + # a suit is jacket + trousers + "suit", "tracksuit", +} + + +def garment_zones(item): + """The body zones a garment covers: {'lower'}, {'upper'}, both, or empty. + + Empty means it is not body covering at all (hat, boots, a scar, hair colour), so + removing it can never expose anything.""" + head = _item_head(item) + for form in (head, head.rstrip("s"), head + "s"): + if form in _ZONE_BOTH: + return {"upper", "lower"} + if form in _ZONE_LOWER: + return {"lower"} + if form in _ZONE_UPPER: + return {"upper"} + return set() + + +# Once a zone has been stripped, its state has to be STATED in every later shot. +# Deleting the garment is only a silence, and a video model's default prior is a +# clothed person, so silence gets them dressed again a shot or two later -- the same +# reason "no longer wearing the red jacket" was not enough on its own. These read as +# a physical description, not as a negation, and they live in the wardrobe channel +# so they persist and clear exactly like a garment. +_BARE_MARK = {"lower": "bare below the waist", "upper": "bare chest"} +# The upper-zone default has to follow the person. "bare chest" on a woman is both +# odd phrasing and a weak cue -- it describes a male torso, and H3 renders roughly +# what the words describe. The lower default stays neutral: it is a position on the +# body, not an anatomy, and naming anatomy there is exactly what exposed_terms is +# for. A person with no declared pronoun keeps the neutral wording. +_BARE_MARK_BY_PRON = { + "she": {"lower": "bare below the waist", "upper": "bare breasts"}, + "he": {"lower": "bare below the waist", "upper": "bare chest"}, +} + + +# A character can START bare rather than becoming bare. Until this existed the +# exposure marker only fired on a REMOVAL, so someone naked from shot 1 was never +# marked and exposed_terms never applied to them. +# +# This has to be DECLARED, never inferred. Absence of clothing in a sheet means the +# author did not enumerate it -- "Jon = he, 35, bald" is an ordinary +# under-specified sheet, not a naked man -- and inferring nudity from a short sheet +# would put it in scenes nobody asked for. Only these explicit tokens count. +_DECLARED_BARE = { + "nude": {"lower", "upper"}, "naked": {"lower", "upper"}, + "fully nude": {"lower", "upper"}, "fully naked": {"lower", "upper"}, + "completely nude": {"lower", "upper"}, "completely naked": {"lower", "upper"}, + "undressed": {"lower", "upper"}, "unclothed": {"lower", "upper"}, + "bottomless": {"lower"}, "topless": {"upper"}, + "waist down nude": {"lower"}, "nude below the waist": {"lower"}, + "bare chested": {"upper"}, "barechested": {"upper"}, "bare-chested": {"upper"}, +} + + +def declared_bare_zones(items): + """Zones a person's sheet says are bare from the outset, and the tokens saying so. + + Returns (zones, tokens). The tokens are returned so the caller can drop them + from the description: they are replaced by the exposure marker, and leaving both + in would state the same fact twice in one parenthetical.""" + zones, tokens = set(), [] + for it in items or []: + key = _item_name(it).strip().lower() + z = _DECLARED_BARE.get(key) + if z is None: + z = _DECLARED_BARE.get(re.sub(r"[^a-z ]", "", (it or "").strip().lower())) + if z: + zones |= z + tokens.append(it) + return zones, tokens + + +def parse_exposed_terms(text): + """Per-person text for a stripped zone: {key: {zone: phrase}}. + + Written like the wardrobe sheet, so there is one syntax to learn: + + she = visible vulva, mvagina + he = visible penis, mpenis + Mara upper = bare breasts + + A key is a PRONOUN (applies to everyone declaring it) or a NAME (which wins + over the pronoun). Without a trailing 'upper' the entry describes the lower + zone. Empty means the generic wording is used.""" + out = {} + for line in (text or "").splitlines(): + line = line.strip() + if not line or "=" not in line: + continue + key, val = line.split("=", 1) + key, val = key.strip(), val.strip() + if not key or not val: + continue + zone = "lower" + parts = key.split() + if len(parts) > 1 and parts[-1].lower() in ("upper", "lower"): + zone = parts[-1].lower() + key = " ".join(parts[:-1]) + out.setdefault(_norm_name(key).lower(), {})[zone] = val + return out + + +def exposed_mark(zone, name, items, terms): + """The phrase to stamp for a stripped `zone` on this person. + + Name beats pronoun, pronoun beats the person's own default, which beats the + neutral one -- so one setup covers a whole cast, and a single character can + still be given their own wording.""" + pron = _pronoun_of(items or []) + if terms: + by_name = terms.get((name or "").strip().lower(), {}) + if zone in by_name: + return by_name[zone] + if pron: + by_pron = terms.get(pron.lower(), {}) + if zone in by_pron: + return by_pron[zone] + # Configuring only the lower zone is the common case ('she = vagina'), and the + # upper zone still has to be worded for the right body rather than defaulting + # to a male torso. + return _BARE_MARK_BY_PRON.get(pron or "", _BARE_MARK)[zone] + + +def bare_state_items(items, stripped_zones, marks=None): + """Markers to add / remove so a stripped zone keeps saying it is stripped. + + Returns (add, drop). A zone that something covers again -- because a garment was + put back on -- drops its marker, which is what "unless requested" means.""" + marks = marks or dict(_BARE_MARK) + # Every phrase this function could have stamped, so a marker is recognised for + # removal even if the configured wording changed between runs -- including the + # per-pronoun defaults, or a marker stamped as 'bare breasts' on one run could + # not be dropped on the next. + known = set(_BARE_MARK.values()) | set(marks.values()) + for d in _BARE_MARK_BY_PRON.values(): + known |= set(d.values()) + add, drop = [], [] + for zone in _BARE_MARK: + mark = marks.get(zone, _BARE_MARK[zone]) + present = mark in items + covered = bool(remaining_cover([i for i in items if i not in known], {zone})) + if zone in stripped_zones and not covered and not present: + add.append(mark) + elif (covered or zone not in stripped_zones) and present: + drop.append(mark) + return add, drop + + +def remaining_cover(items, zones): + """Items still worn that cover any of `zones` -- what is underneath.""" + return [i for i in items if garment_zones(i) & zones] + + +def _is_plural_garment(item): + """Garments that take a plural verb: overalls, jeans, boots, gloves, shorts. + A head noun ending in a DOUBLE s (dress, harness) is singular, which is what + separates them from a real plural. + + Read from the garment's own head, not the item's last word: "red jacket with + silver zippers" ends on a plural detail while the garment itself is singular, + which produced "the red jacket ... ARE off and she is not wearing THEM".""" + head = _item_head(item) + return bool(head) and head.endswith("s") and not head.endswith("ss") + + +def takes_off_clause(pairs, active=None): + """The DIRECTION of a removal, stated in the shot that performs it. + + A removal is the one wardrobe change with a failure mode of its own: the motion + is symmetric. The same frames played backwards are a person putting the garment + ON, and both readings satisfy "takes off her red jacket" equally well. The model + picks whichever the rest of the conditioning supports -- and when the shot's own + description still listed the garment as worn, backwards was the reading that + matched. The removal rendered in reverse and the jacket came back. + + So the end state is stated explicitly, and the reverse is ruled out by name. + Said ONCE, in the removal shot only: every later shot simply describes what the + person is wearing now, and never names the garment again -- to a video model a + mention is a presence cue, and a negation is a weak one, so "no longer wearing + the red jacket" in the NEXT shot was itself enough to put it back on.""" + active = active or {} + by = {} + for name, item in pairs: + item = (item or "").strip() + if item and item not in by.setdefault(name or "", []): + by[name or ""].append(item) + bits, still = [], [] + for name, items in by.items(): + # Refer to the garment by NAME, not by its full sheet entry. The detail + # (logo, zippers, torn knee) is already stamped in the description every + # shot; repeating it twice inside a sentence that only has to say the thing + # came off buries the instruction in 22 words of wardrobe. + what = " and ".join(_item_name(i) for i in items) + # "the navy overalls IS off" reads as a mistake to the encoder that has to + # parse this. Garments like overalls/jeans/boots are grammatically plural, + # as is any list of more than one. + plural = len(items) > 1 or any(_is_plural_garment(i) for i in items) + # `pron` is the OBJECT form ("takes them off"), `subj_pron` the SUBJECT form + # ("they are off") -- the impersonal branch needs the latter. + verb, pron, subj_pron = ("are", "them", "they") if plural else ("is", "it", "it") + subj = _subject_term(name, active) if name else "" + # Name the garment ONCE. It was named twice here and again in the beat prose, + # which made it the most-referenced thing in the shot -- and a garment + # referenced that often gets rendered as a prominent object, picked up and + # handled by whoever is nearby. Same rule as people: repeat the reference, + # get the thing repeated. Saying where it ENDS UP is what stops it lingering + # in someone's hands. + if subj: + bits.append(f"{subj} takes the {what} off during this shot; by the last frame " + f"{subj_pron} {verb} off, dropped away out of frame, and {subj.lower()} is " + f"no longer wearing {pron}") + else: + bits.append(f"the {what} {'come' if plural else 'comes'} off during this shot; by the " + f"last frame {subj_pron} {verb} off and dropped away out of frame") + # SAY WHAT IS STILL ON. The clause is five statements about clothing coming + # off; without this, nothing in it says the body is still covered, and the + # model completes the obvious continuation -- shorts worn UNDER trousers were + # listed once in a distant parenthetical and simply not rendered. Naming the + # under-layer here, in the same breath as the removal, is what keeps it on. + zones = set() + for i in items: + zones |= garment_zones(i) + under = remaining_cover(active.get(name, []), zones) if zones else [] + if under: + worn = " and ".join(_item_name(u) for u in under) + who = (_subject_term(name, active).lower() if name else "the character") + still.append(f"the {worn} underneath {'stay' if len(under) > 1 or _is_plural_garment(under[0]) else 'stays'} " + f"on and {who} is still wearing {'them' if len(under) > 1 or _is_plural_garment(under[0]) else 'it'}") + if not bits: + return "" + s = "; ".join(bits) + if still: + s += ". " + ("; ".join(still)).capitalize() + # The anti-reverse instruction is the point of the clause, so it is not left + # implicit in the end-state description. Worded without a pronoun so it needs no + # agreement with whatever came off. + return (s[0].upper() + s[1:] + + ". The motion runs one way only: the clothing comes off and is never put back on, " + "never re-worn, and the action never plays in reverse.") + + +# Words that can never be part of the garment phrase itself. +_GARMENT_LEAD = {"off", "out", "of", "aside", "away", "down", "up", "the", "a", "an", + "her", "his", "their", "its", "it", "them", "then"} +# Words that END a garment phrase: a conjunction, a new preposition, or a new +# article all start something that is no longer the garment. +_GARMENT_END = {"and", "or", "but", "then", "on", "onto", "over", "into", "in", "to", + "from", "at", "by", "with", "under", "beside", "as", "while", "before", + "after", "a", "an", "the", "she", "he", "they", "her", "his", "their"} +# A person noun is never part of a garment phrase -- scrubbing one deletes the +# CHARACTER from the anchor and leaves the clothing behind. +_PERSON_NOUN = {"woman", "women", "man", "men", "girl", "boy", "guy", "lady", "person", + "people", "figure", "child", "kid", "teen", "teenager", "male", "female"} + + +def removed_phrase_items(body, anchor_id): + """Garments named in a REMOVAL phrase in this beat that also appear in the + anchor prose. Covers the case where the item was never in the wardrobe channel + at all -- e.g. the anchor says 'a woman in a red flight jacket' and the beat + says 'she takes off her jacket'. Without this the anchor would keep re-applying + it forever. Returns the anchor phrases to scrub. + + The phrase is read to its HEAD NOUN, not to the first word after the verb. The + earlier version stopped at the first non-stop word, so "takes off her red + jacket" yielded 'red' -- and matching 'red' with its preceding words in the + anchor produced 'A woman in a red', which scrubbed the PERSON out of + 'A woman in a red jacket' and left 'jacket'. The garment survived, the + character vanished, and clothing removal looked completely broken.""" + if not body or not anchor_id: + return [] + verb = re.compile(r"\b(takes?|took|taking|pulls?|pulled|peels?|peeled|strips?|stripped|" + r"slips?|slipped|shrugs?|shrugged|removes?|removed|sheds?|shed|discards?|" + r"ditch(?:es|ed)?|doffs?|unbuttons?|unzips?)\b", re.I) + out = [] + for m in verb.finditer(body): + # Stop at punctuation: "shrugs off his overalls, a flight suit underneath" + # must not drag the second clause into the garment. + tail = re.split(r"[,.;:!?]", body[m.end():m.end() + 60])[0] + words = re.findall(r"[A-Za-z][A-Za-z\-]*", tail) + i = 0 + while i < len(words) and words[i].lower() in _GARMENT_LEAD: + i += 1 + phrase = [] + while i < len(words) and words[i].lower() not in _GARMENT_END and len(phrase) < 4: + phrase.append(words[i]) + i += 1 + if not phrase: + continue + head = phrase[-1] + if head.lower() in _PERSON_NOUN: # "takes off after the man" -- not clothing + continue + # Take the head noun with its adjectives out of the anchor, then trim any + # leading word that belongs to the SENTENCE rather than to the garment. + am = re.search(r"((?:[A-Za-z\-]+\s+){0,2}" + re.escape(head) + r")\b", anchor_id, re.I) + if not am: + continue + toks = am.group(1).split() + while len(toks) > 1 and toks[0].lower() in (_GARMENT_END | _PERSON_NOUN | _GARMENT_LEAD): + toks.pop(0) + if toks and toks[-1].lower() not in _PERSON_NOUN: + out.append(" ".join(toks)) + return out + + +def extract_directive(body, key): + """Pull a ': ...' line out of a beat body. Returns (clean_body, value|None).""" + kept, val = [], None + for ln in body.split("\n"): + if re.match(r"\s*" + key + r"\s*:", ln, re.I): + val = ln.split(":", 1)[1].strip() + else: + kept.append(ln) + return "\n".join(kept).strip(), val + + +# "walks out OF THE BARN" is emerging INTO the scene, not leaving it -- and a false +# exit is the expensive error: the character is stripped from every later shot and +# only an explicit 'enter:' brings them back. So "out of " is never an +# exit unless the somewhere is the frame itself. +_EMERGENCE_TAIL = re.compile( + r"^\s+of\s+(?:the\s+|a\s+|an\s+|his\s+|her\s+|their\s+|its\s+)?" + r"(?!frame\b|shot\b|view\b|screen\b|scene\b|camera\b|sight\b|there\b|here\b)\w+", re.I) + + +def _is_emergence(text, m): + """True when this exit cue is really someone coming OUT OF a place into view. + + The cue match ends at "out"/"off", so what decides it is what FOLLOWS: "walks + out | of the barn" is emergence, "walks out | of frame" is departure, and a + bare "walks out" has nothing after it and stays an exit.""" + return bool(_EMERGENCE_TAIL.match(text[m.end():])) + + +# One list of departure phrasings for both readers of it: detect_exits(), which +# decides WHO left, and departed_phrase_people(), which scrubs their description +# out of the anchor. It was written out twice; the two copies had to agree or a +# character could be marked departed while the anchor kept describing them. +_EXIT_CUE = re.compile( + r"\b(?:leaves?|left|leaving|exits?|exited|departs?|departed|" + r"walks? (?:out|off|away)|walked (?:out|off|away)|steps? (?:out|off|away)|" + r"stepped (?:out|off|away)|drives? (?:off|away)|drove (?:off|away)|" + r"rides? (?:off|away)|runs? (?:out|off)|ran (?:out|off)|" + r"disappears?|vanishes?|is gone|are gone|out of frame|off screen|off-screen)\b", re.I) + + +def detect_exits(body, active, departed): + """Names of characters who LEAVE in this beat, so they don't reappear later. + Matches an exit phrase ('leaves', 'walks out', 'exits', 'drives off', 'steps + out of frame', 'is gone') attributed to the nearest preceding subject (name or + resolvable pronoun). Gated on tracked people, so 'the plane leaves' -- not a + tracked person -- departs nobody.""" + if not body: + return [] + text = " " + body.lower() + " " + names = [n for n in active if n and n not in departed] + if not names: + return [] + pron_map = _pron_map({k: v for k, v in active.items() if k not in departed}) + single = len(names) == 1 + + subj_tokens = [re.escape(n.lower()) for n in names] + list(_PRO.keys()) + subj_re = re.compile(r"\b(" + "|".join(subj_tokens) + r")\b") + + out = [] + for m in _EXIT_CUE.finditer(text): + if _is_emergence(text, m): + continue + best, bp = None, -1 + for sm in subj_re.finditer(text): + if 0 <= sm.start() < m.start() and sm.start() > bp: + person = _resolve_subject(sm.group(1), names, pron_map, single) + if person is not None: + bp, best = sm.start(), person + if best: + out.append(best) + return out + + +def departed_phrase_people(body, anchor_id): + """Anchor phrases for people who LEAVE in this beat but were never declared in + the character channel -- e.g. the anchor says 'a woman with silver hair and a + bald man in navy overalls' and the beat says 'he walks out'. Without this the + anchor keeps re-asserting them into every later shot. + + Resolves the departing subject from the pronoun/noun before the exit cue, then + finds the matching person-phrase in the anchor by gender word ('man'/'woman'/ + 'boy'/'girl'/etc.) and returns that whole phrase (with its trailing + prepositional clause, e.g. 'a bald man in navy overalls') for scrubbing. + Returns [] when nothing matches, so non-person exits ('the plane leaves') + remove nobody.""" + if not body or not anchor_id: + return [] + want = {"she": ("woman", "women", "girl", "lady", "female"), + "he": ("man", "men", "boy", "guy", "gentleman", "male")} + out = [] + for m in _EXIT_CUE.finditer(body): + head = body[:m.start()] + pm = None + for p in re.finditer(r"\b(she|he|her|him|his|the\s+\w+)\b", head, re.I): + pm = p.group(1).lower() + if not pm: + continue + key = _PRO.get(pm.split()[-1]) + nouns = want.get(key, ()) + if not nouns: + continue + for noun in nouns: + # the person phrase: optional article/adjectives + noun + an immediate + # clothing clause only ('a bald man in navy overalls'). The clause must + # not run past a comma, so a following scene phrase ('..., in a hangar') + # is left intact. + am = re.search(r"((?:a|an|the)\s+(?:[\w\-]+\s+){0,3}" + noun + + r"(?:\s+(?:in|with|wearing)\s+(?:a\s+|an\s+|the\s+)?" + r"(?:[\w\-]+\s+){0,2}[\w\-]+)?)(?=\s*(?:,|\.|$|\band\b))", + anchor_id, re.I) + if am: + out.append(am.group(1).strip().rstrip(",")) + break + return out + + +# Natural speech runs ~2.3-2.8 words/sec in film dialogue; 2.5 is a safe middle. +# Used only to WARN that a line looks too long for the shot it sits in. +WORDS_PER_SEC = 2.5 + + +# A spoken line needs a beat of air before and after it inside the same shot -- +# the mouth opens late and the last syllable must not land on the cut. +SPEECH_PAD_SEC = 1.0 +# ...and a two-hander needs a hand-off between turns. Two people trading three +# lines is not the same screen time as one person saying all three back to back: +# the camera/mouth has to switch subject between each. +TURN_GAP_SEC = 0.5 + + +# --- content-aware shot length --------------------------------------------- +# A beat's screen time is estimated from how many ACTIONS it stages, not from its +# word count. Word count measures how wordy you were; clause count measures how +# much has to happen. +# +# The estimate is deliberately biased SHORT, because the two errors are not +# symmetric. A shot that ends before the action finishes hands a mid-motion frame +# to the next shot, which is exactly what the handoff chain is built to continue. +# A shot that outlasts its action leaves the model seconds it was told nothing +# about, and the cheapest filler for a symmetric action (taking a jacket off, a +# door opening, sitting down) is to run it BACKWARDS -- which returns to the start +# state and makes the clip loopable. Too long is unrecoverable; too short is not. +BEAT_BASE_SEC = 2.0 # setup/settle time every shot needs regardless of content +SECONDS_PER_ACTION = 2.5 # screen time for one staged action clause +MIN_CONTENT_FRAMES = 73 # ~3.0s: the shortest shot that can hold one action +# Clause separators: a new coordinated verb phrase starts a new action. +_CLAUSE_SPLIT = (r"(?:[.!?;]+|,?\s+(?:and then|then|and|before|after|while|as|until)\s+" + r"|,\s+(?=[a-z]+ing\b))") + + +def action_clauses(beat): + """How many distinct staged actions a beat contains. + + "takes off her red jacket and drops it on the workbench" is two; "walks the + length of the garage, checking every bench, then stops at the far wall" is + three. Quoted speech is excluded -- that time is counted by dialogue_seconds.""" + body, _ = extract_wardrobe((beat or "").strip()) + body = re.sub(r'["“][^"”]*["”]', " ", body) + body = " ".join(ln for ln in body.splitlines() if not is_directive_line(ln)) + parts = [p.strip() for p in re.split(_CLAUSE_SPLIT, body) if p and p.strip()] + # A fragment of one word is a leftover ("it", "her"), not an action of its own. + return sum(1 for p in parts if len(p.split()) >= 2) + + +def estimate_beat_seconds(beat): + """Screen time this beat needs, from its own content. 0.0 when it has none. + + Action and dialogue OVERLAP rather than add -- people talk while they move -- + so the estimate is the larger of the two, not their sum.""" + n = action_clauses(beat) + action = (BEAT_BASE_SEC + SECONDS_PER_ACTION * n) if n else 0.0 + return max(action, dialogue_seconds(beat)) + + +def dialogue_spans(beat): + """Word count of each double-quoted span in a beat, in order. Length of the + returned list is the number of speaking TURNS -- the multi-character case.""" + body, _ = extract_wardrobe((beat or "").strip()) + return [len(q.split()) for q in re.findall(r'["\u201c]([^"\u201d]+)["\u201d]', body) if q.split()] + + +def dialogue_words(beat): + """Words inside double quotes in a beat -- the only speech H3 actually renders.""" + return sum(dialogue_spans(beat)) + + +def dialogue_seconds(beat, pad=True): + """Screen time this beat's dialogue needs, 0.0 when the beat has none. + + Counts every turn, so a two-character exchange is sized from the WHOLE + exchange plus a gap between turns -- not from the longest single line. + `pad` controls only the head/tail air; turn gaps are always counted because + they are time the shot genuinely has to contain.""" + spans = dialogue_spans(beat) + if not spans: + return 0.0 + return (sum(spans) / WORDS_PER_SEC + + TURN_GAP_SEC * (len(spans) - 1) + + (SPEECH_PAD_SEC if pad else 0.0)) + + +def beat_seconds_directive(beat): + """Explicit per-beat length: a 'seconds: 8' (or 'duration: 8') line in the beat. + Returns the float, or None when the beat doesn't set one.""" + for key in ("seconds", "duration"): + _, val = extract_directive((beat or ""), key) + if val: + m = re.search(r"([0-9]*\.?[0-9]+)", val) + if m: + try: + v = float(m.group(1)) + except ValueError: + continue + if v > 0: + return v + return None + + +def plan_beat_frames(beats, fps, budget, per_beat=True): + """Per-beat shot lengths in frames. Returns (lengths, notes). + + `budget` is the CEILING -- the VRAM budget, or a forced shot_seconds already + clamped to it. Per-beat sizing can only ever make a shot shorter than that + ceiling, never longer. Priority per beat: + + 1. an explicit 'seconds: N' line in the beat -- always honored, down to + H3's real 5-frame minimum, because you stated a duration outright; + 2. its own content -- action clauses and quoted dialogue (see + estimate_beat_seconds), floored at MIN_CONTENT_FRAMES so a shot always + has room for one action; + 3. with per_beat off, the ceiling, exactly as before. + + Why estimate at all, when action prose has no *reliable* duration? Because the + alternative is not "no guess" -- it is "guess the maximum", which is what giving + every beat the ceiling does. A 3-second action in a 12-second shot leaves nine + seconds the model was told nothing about, and it fills them by repeating or + REVERSING the action. Leaning short costs an unfinished action that the next + shot continues from the handoff frame; leaning long costs a jacket that takes + itself off and puts itself back on.""" + beats = beats if beats else [""] + # MIN_SHOT_FRAMES is the floor of the *VRAM budget* -- the shortest shot the node + # falls back to when it has to guess with no information at all. It must not raise + # a length that came from you or from the beat's own content: `max(floor, ...)` + # silently turned every request below ~5.2s into 124f, so 1s/2s/3s/4s all rendered + # identically and both the widget and the `seconds:` directive looked broken. + cap = max(5, int(budget)) + content_floor = align_frame_count(MIN_CONTENT_FRAMES) + out, notes = [], [] + fps = max(1, int(fps)) + for i, b in enumerate(beats, 1): + want, src, floor = beat_seconds_directive(b), "seconds:", 5 + snap = align_frame_count # a stated length is never rounded DOWN + if want is None: + want = estimate_beat_seconds(b) if per_beat else 0.0 + src, floor, snap = "content", content_floor, align_frame_count_nearest + if want <= 0: # no signal -> the ceiling + out.append(cap) + continue + n = min(cap, max(floor, snap(int(round(want * fps))))) + out.append(n) + if n != cap: + notes.append(f"shot {i}: {n}f (~{n / fps:.1f}s, from {src})") + return out, notes + + +def pacing_warnings(beats, lengths, fps): + """Beats whose content is far too thin for the length they were given. + + Pure arithmetic, no model involved: it cannot know that "walks across the + tarmac" is 2s or 12s, but it can see 12 words sitting in a 12-second shot and + say so BEFORE the render, instead of leaving you to discover it as an action + that repeats or plays backwards.""" + out = [] + fps = max(1, int(fps)) + for i, (b, n) in enumerate(zip(beats or [], lengths or []), 1): + if beat_seconds_directive(b): # you stated it; not the node's business + continue + need = estimate_beat_seconds(b) + have = n / fps + if need and have > need * 1.8 and have - need >= 3.0: + out.append(f"shot {i}: ~{need:.1f}s of content in a {have:.1f}s shot " + f"({action_clauses(b)} action(s), {dialogue_words(b)} spoken words)") + return out + + +def dialogue_fit_warnings(beats, seconds_per_shot): + """Flag beats whose quoted dialogue is unlikely to fit the shot length. + + The VRAM budget caps SHOTS (never resolution, since rendering below native + softens the frame). That is the right trade for picture quality, but it is blind + to dialogue: a line written for a 10s shot gets cut off mid-sentence in a 7s one. + Audio cannot span the handoff either -- each shot generates its own -- so a + truncated line is simply lost, not continued. + + seconds_per_shot takes a single value or a per-shot list (per-beat sizing). + Returns a list like ["shot 3: ~6.4s of dialogue in a 5.2s shot"] so the user can + shorten the line, or choose a lower resolution tier to buy the duration back.""" + out = [] + for i, b in enumerate(beats or [], 1): + if isinstance(seconds_per_shot, (list, tuple)): + if i > len(seconds_per_shot): + break + sec = seconds_per_shot[i - 1] + else: + sec = seconds_per_shot + need = dialogue_seconds(b, pad=False) + if not need: + continue + if need > sec * 0.92: # leave a little room to breathe + out.append(f"shot {i}: ~{need:.1f}s of dialogue in a {sec:.1f}s shot") + return out + + +def continuity_warnings(gens): + """Shots that describe NOBODY while people are still in the story. + + The chain hands each shot the previous one's last decoded frame. A scenery beat + describes no one, so the frame it produces has no one in it -- and the next + shot has to re-establish every character from an empty room. That is a cohesion + break, and it is invisible in the prompt: the text of both shots is individually + correct, which is why chains lose their people in the middle rather than + degrading steadily. + + Only flagged when people appear on BOTH sides. A scenery beat that opens or + closes a chain hands its frame to nobody, so it costs nothing.""" + if len(gens or []) < 3: + return [] + # A bound person shows up as an inline parenthetical of real description. + peopled = [] + for g in gens: + body = (g or "").split("\n")[0] + body = re.sub(r"^\[Generation \d+\]\s*", "", body) + body = re.sub(r"^Exactly [a-z]+ (?:person|people)[^.]*\.\s*", "", body) + peopled.append(bool(re.search(r"\([^)]{6,}\)", body))) + out = [] + for i in range(1, len(peopled) - 1): + if not peopled[i] and peopled[i - 1] and any(peopled[i + 1:]): + out.append( + f"shot {i + 1} describes nobody, between shots that do -- it hands shot " + f"{i + 2} a frame with no people in it, so every character has to be " + f"re-established from an empty room. Give it someone ('Dom watches from " + f"the doorway'), or move it to the start or end of the chain") + return out + + +def dialogue_filler_warnings(beats, seconds_per_shot): + """Dialogue shots with far more time than their line, which H3 fills with speech. + + dialogue_fit_warnings covers the opposite error -- a line too long for its shot, + which gets truncated. This is the one that produces BABBLE: a two-second line in + a ten-second shot leaves eight seconds of audio the model was told nothing + about, and the audio branch keeps talking to fill them. mute_nonspeech_audio + cannot help, because a shot with a scripted line is deliberately left audible. + + Same vacuum as an over-long action beat, one channel across.""" + out = [] + for i, b in enumerate(beats or [], 1): + sec = (seconds_per_shot[i - 1] if isinstance(seconds_per_shot, (list, tuple)) + else seconds_per_shot) if not isinstance(seconds_per_shot, (list, tuple)) \ + or i <= len(seconds_per_shot) else None + if sec is None: + break + spoken = dialogue_seconds(b, pad=False) + if not spoken: + continue + gap = sec - spoken + if gap >= 3.0 and sec > spoken * 2: + out.append(f"shot {i}: {spoken:.1f}s of dialogue in a {sec:.1f}s shot -- {gap:.1f}s of " + f"unscripted audio the model will fill with more speech") + return out + + +def speech_flags(beats): + """Per-beat: does it contain scripted (quoted) dialogue? Same rule the prompt + builder uses to decide silencing, exposed so the renderer can also MUTE the + audio of non-speech shots -- a deterministic fix when H3 vocalizes anyway.""" + out = [] + for b in (beats if beats else [""]): + body, _ = extract_wardrobe((b or "").strip()) + out.append(has_speech(body)) + return out + + +def distribute_generations(anchor, beats, gs, music="", char_memory="", auto_wardrobe=True, + auto_silence_nonspeech=True, allow_nonspeech_vocals=False, count_subjects=False, front_load=False, + notes_out=None, auto_props=True, prevent_nudity=True, + exposed_terms="", strip_out=None, anatomy_guard=False, + anatomy_auto=False, + lock_restraints=True, solidity_guard="auto", + motion_guard="auto", contact_guard="auto", count_auto=False): + """One beat = one shot. Stamp the permanent identity into each beat. Total + video length is (number of shots) x (per-shot length), computed by the + caller -- never divided out of a total, so beat count always equals shot count. + + WARDROBE LIVES IN ONE MUTABLE, PER-PERSON CHANNEL so it can be changed or + removed, and so multiple people are tracked independently. The channel is + seeded from character_memory, or from a 'wardrobe:' line in the anchor; + whatever anchor prose REMAINS after pulling that line is permanent identity, + stamped every shot. Clothing must NEVER be baked into the permanent anchor + prose (the anchor is immutable and would re-assert a garment you tried to + remove) -- keep identity in the prose, all clothing in this channel. + + auto_wardrobe (default on): removals are inferred from each beat's own action + text, so "she takes off her jacket" drops the jacket with no directive. It's + gated on tracked items, so non-garment objects ("the plane takes off") never + fire. Additions/swaps still use an explicit 'wardrobe: += ...' line, which + also overrides the auto-detection. + + Multi-person syntax: 'wardrobe: Maya = grey shorts, red jacket; Jon = navy + overalls' (a colon works too). A per-beat 'wardrobe:' line updates only the + names it mentions; one unnamed subject works as before. + + The two audio sections are appended after the visual timeline, in H3's + documented field order: + * `overall_soundscape:` -- ambient/environmental sound (rain, room tone). + * `non_diegetic_music:` -- background score not part of the scene. + Both are global (stamped on every shot). Dialogue and diegetic sound belong + in the beat body / timeline, NOT in either of these.""" + beats = beats if beats else [""] + anchor_id, anchor_wardrobe = extract_wardrobe((anchor or "").strip()) + seed = (char_memory or "").strip() or (anchor_wardrobe or "") + active = parse_wardrobe(seed) # {name: [items]}, mutable, per-person + removed = [] # garments taken off -> also scrubbed from the anchor + departed = set() # characters who left the scene -> never reappear + props = {} # objects introduced so far -> their phrase + stripped = {} # person -> body zones stripped so far + # person -> how their restraints are used (tether anchor / pose), as stated by + # whichever shot said it. A restraint is part of the wardrobe channel, and its + # USE persists with the wardrobe: shot 5 saying "she strains" must keep shot + # 2's "cuffed to the headboard", not fall back to wording that contradicts it. + restraint_usage = {} + exposed = parse_exposed_terms(exposed_terms) + # Every person key that existed at any point, so an entry naming someone who is + # only introduced later by a 'wardrobe: Name = ...' directive is not called + # unmatched. Checked after the loop, once the full cast is known. + seen_names = {k for k in active if k} + blocks = [] + for gi, b in enumerate(beats, 1): + body, wardrobe_change = extract_wardrobe((b or "").strip()) + body, _ = extract_directive(body, "seconds") # shot length, not prose + body, _ = extract_directive(body, "duration") # ditto (alias) + body, exit_directive = extract_directive(body, "exit") # explicit 'exit: Jon' + body, enter_directive = extract_directive(body, "enter") # explicit 'enter: Jon' (undo) + if enter_directive: + for nm in _entries(enter_directive): + departed.discard(_norm_name(nm)) + # Naming a departed character again is intent to have them BACK. Without + # this they stayed departed, so the beat carried their bare NAME with no + # description while everyone else kept theirs -- and the described character + # absorbed the action. A PRONOUN still cannot re-summon anyone: "he waves" + # after someone left is ambiguous, a name is not. Use 'exit: Name' again to + # send them back out. + if departed: + named_here, _spans = _mask_quotes(body) + for nm in list(departed): + if nm and re.search(r"\b" + re.escape(nm) + r"\b", named_here, re.I): + departed.discard(nm) + body = body or "continue the action, same subject" + # Props introduced in an EARLIER beat: bind the first definite reference to + # them, so "the van" in shot 2 means the van from shot 1 instead of an + # invented one standing next to it. Garments are excluded -- they have their + # own channel, and "the same red jacket" would fight a removal. + worn_nouns = {_item_head(i) for v in active.values() for i in v} + own_props = {n: p for n, p in introduced_props(body).items() if n not in worn_nouns} + carried = {n: p for n, p in props.items() + if n not in worn_nouns and n not in own_props} + body, bound_props = bind_props(body, carried) if auto_props else (body, []) + # An object introduced AND referred back to inside one beat never reaches the + # cross-shot carry -- but that is the case that duplicates hardest, because + # the repetition is all in one prompt. Collapse the repeats the way repeated + # NAMES are collapsed, and state the count. + here_again = repeated_props(body, own_props) if auto_props else [] + if here_again: + body, _ = dedupe_prop_mentions(body, here_again) + off_now = [] # (person, garment) coming off in THIS shot + + def _drop(before, after): + """Record what `after` no longer has, for both removal paths.""" + for k, v in before.items(): + gone = [it for it in v if it not in after.get(k, [])] + removed.extend(gone) + off_now.extend((k, it) for it in gone) + return after + + if wardrobe_change is not None: + before = {k: list(v) for k, v in active.items()} + # explicit: takes effect THIS shot + active = _drop(before, apply_wardrobe_change(active, wardrobe_change)) + # Auto-removals are resolved BEFORE the shot is composed, so the garment is + # already out of the person's description in the very shot that takes it off. + # + # It used to be deferred to the next shot, on the reasoning that the shot + # SHOWING the removal should still show the garment. That produced a shot + # whose description says "wearing a red jacket" while its verb says "takes off + # her red jacket" -- and the cheapest way for the model to satisfy both is to + # run the motion the OTHER way, ending with the jacket on. The video played + # the removal in reverse. + # + # The start state does not need the description: for every shot after the + # first it is pinned by the handoff keyframe, which shows the garment still + # worn. So the keyframe carries the START state and the prompt carries the + # END state, and the direction between them is stated outright below. + if auto_wardrobe: + before = {k: list(v) for k, v in active.items()} + active = _drop(before, auto_wardrobe_removals(active, body, lock_restraints)) + # A garment that lives ONLY in the anchor prose (never in the wardrobe + # channel): the removal phrase names it, so scrub it from the anchor or + # the anchor re-applies it forever. + anchor_gone = removed_phrase_items(body, anchor_id) + # Same rule on the anchor side: a restraint named in the anchor prose is + # not scrubbed by a removal phrase either, or it would vanish from every + # later shot without anything having asked for it. + if lock_restraints: + anchor_gone = [it for it in anchor_gone if not is_restraint(it)] + removed += anchor_gone + # Voice the anchor-side removal only when the channel didn't already cover + # it, or the same jacket is announced twice. + if anchor_gone and not off_now: + off_now += [("", it) for it in anchor_gone] + # Restraint USE follows the wardrobe channel: the moment a sheet no longer + # carries the restraint, its recorded use is forgotten -- otherwise a freed + # and later RE-bound character would inherit an attachment nobody has + # restated. Pruned HERE rather than inside restraint_clause, because the + # clause only runs on shots with someone in them, while removals -- and + # therefore forgetting -- must happen whatever the shot shows. + for _nm in list(restraint_usage.keys()): + if not restraint_regions(active.get(_nm)): + del restraint_usage[_nm] + # Record which zones this person has been stripped in, then keep the state + # STATED in every later shot. Deleting the garment is only a silence, and a + # video model's default is a clothed person -- so silence puts the clothes + # back on a shot or two later. The marker clears by itself if a garment + # covering that zone is put back on, which is the "unless requested" half. + for nm, it in off_now: + z = garment_zones(it) + if z: + stripped.setdefault(nm, set()).update(z) + # A sheet can DECLARE a zone bare from the outset ('Jon = he, 35, nude'), + # which is the start-naked case: there is no removal to trigger on, so + # without this the marker never fires and exposed_terms never reaches them. + # The token itself is swapped out for the marker so the fact is stated once. + declared = {} + for nm in list(active): + zones, tokens = declared_bare_zones(active.get(nm, [])) + if zones: + declared[nm] = zones + stripped.setdefault(nm, set()).update(zones) + for tok in tokens: + active[nm].remove(tok) + bare_now = {} + for nm in list(active): + marks = {z: exposed_mark(z, nm, active.get(nm, []), exposed) + for z in ("lower", "upper")} + add, drop = bare_state_items(active.get(nm, []), stripped.get(nm, set()), marks) + # prevent_nudity gates the ASSERTION, not the removal. Deleting a garment + # only leaves the zone undescribed, and a video model's default prior is a + # clothed person -- so it dresses them again. It is this marker that makes + # the prompt SAY the body is bare, which is the thing that renders. The + # garment still comes off either way; without the marker the model simply + # covers what nobody described. `info` still reports the empty zone. + # + # Filling in exposed_terms IS the intent, so it overrides the guard for the + # people it names. Requiring both switches was a footgun: the terms sat + # there looking configured and did nothing. + # Declaring nudity in the sheet is as explicit as filling in + # exposed_terms, so it overrides the guard for that person the same way. + if prevent_nudity and not exposed and nm not in declared: + add = [] + stripped_here = any(n == nm for n, _ in off_now) + # The handoff reset is recorded INDEPENDENTLY of the nudity gate above: + # that gate decides whether the bare state gets STATED, but whether or + # not it is stated, this shot's last frame shows the removal in + # progress -- and letting the next shot continue from that frame is how + # the garment reappears. Recording lived inside the `add` loop, so with + # prevent_nudity at its default the reset silently never fired and + # every removal was followed by a stale frame. + if (stripped_here and stripped.get(nm) + and strip_out is not None and gi not in strip_out): + strip_out.append(gi) + for mark in add: + active[nm].append(mark) + # This shot newly bared a zone by REMOVING something. The NEXT shot + # must not continue from its last frame: that frame is the removal in + # progress, and a picture of the garment still being worn beats any + # sentence saying it is off. A zone that was declared bare from the + # start has no such frame -- nothing came off -- so it must NOT cost + # the next shot its handoff. + if stripped_here and strip_out is not None and gi not in strip_out: + strip_out.append(gi) + for mark in drop: + active[nm].remove(mark) + # Which zones are marked bare AFTER this shot's adds and drops. Read off + # the markers themselves, so every gate that governs them -- prevent_nudity, + # exposed_terms, a sheet declaration -- governs this too, with no second + # decision to keep in step. + zones_bare = [z for z, mk in marks.items() if mk in active[nm]] + if zones_bare: + bare_now[nm] = zones_bare + persistent = compose_persistent(body, active, anchor_id, removed, departed, count_subjects, + speaking=has_speech(body), front_load=front_load, + count_auto=count_auto, + silence_nonspeech=bool(auto_silence_nonspeech)) + # State the DIRECTION of the change, in the shot that performs it. Only for + # people actually in this shot; an anchor-prose garment is stated + # impersonally, so it summons nobody. + speak_off = [(n, it) for n, it in off_now + if not n or person_referenced(body, n, active)] + off_clause = takes_off_clause(speak_off, active) + prop_clause = (prop_continuity_clause(bound_props, carried) + + prop_count_clause([n for n in here_again if n not in bound_props])) + # A removal that leaves a body zone with NOTHING on it is the one the node + # cannot write its way out of: there is no under-layer to name, so the model + # renders bare skin. Say so before the render rather than after it. + if notes_out is not None: + for nm, it in off_now: + zones = garment_zones(it) + if zones and not remaining_cover(active.get(nm, []), zones): + who = nm or "the character" + notes_out.append( + f"shot {gi}: removing the {_item_name(it)} leaves {who} with nothing on the " + f"{'/'.join(sorted(zones))} body -- H3 will render bare skin there. Add an " + f"under-layer to character_memory (e.g. 'grey shorts') if that is not intended") + if off_clause: + persistent = persistent.rstrip(". ") + ". " + off_clause + if prop_clause: + persistent = persistent.rstrip(". ") + "." + prop_clause + # Silence non-speech shots: a shot with no scripted dialogue gets an explicit + # lips-closed / no-speech clause, so H3 doesn't animate a mouth or fill it with + # gibberish before (or between) actual dialogue. Shots WITH quoted dialogue are + # left alone so the speech renders. + # Two different silences, and they are NOT the same condition: + # no_speech -> the AUDIO constraint. A shot with no scripted line must not + # be given an unconditioned audio branch, whether or not + # anyone is on screen; an empty room still babbles. + # mouth_state-> the PICTURE constraint. Only meaningful when someone is + # there to have a mouth. On a scenery beat it describes + # nobody and can only invite a face into an empty frame. + no_speech = bool(auto_silence_nonspeech and not has_speech(body)) + allow_vocals = bool(allow_nonspeech_vocals and not has_speech(body)) + # A quoted single word with no speech verb is usually EMPHASIS, not a line -- + # but it still flips the shot to "speaking": every mouth goes free and the + # audio stays unmuted, which is how a character ends up mouthing prompt + # fragments. Report it rather than guess; the fix is one edit either way. + if notes_out is not None and auto_silence_nonspeech and not no_speech: + for q in _emphasis_quotes(body): + notes_out.append( + f'shot {gi}: "{q}" reads as emphasis, not dialogue -- this shot keeps every ' + f'mouth free because of it. If nobody says it aloud, drop the quotes; if ' + f'someone does, attribute the line (she says, "...") so only their mouth moves') + # Must agree with compose_persistent()'s binding, including the PLURAL case. + # It did not: person_referenced() resolves a pronoun to one person, so + # 'they'/'both of them' answered False for everyone, and a beat that the + # roll-call had just described in full counted as having nobody in it. Those + # shots got no mouth constraint at all -- two people on screen, nothing + # saying their lips are closed -- which is exactly a shot that opens mouths + # at random. + present_names = [n for n in active if n and n not in departed] + people_here = (bool(active.get("")) + or any(person_in_shot(body, n, active, departed) + for n in present_names)) + # EVERY per-shot state below is gated on someone being in the shot, for one + # reason: describing a body in a frame that has none can only invite one in. + # That gate used to be written out six times, along with the same + # rstrip-and-append; the ORDER is the only thing that differed, so the order + # is data here and the mechanics happen once. + # + # The order is deliberate, and each entry says why it sits where it does. + # `ident` is the identity block BEFORE any of these were appended: the + # solidity scan reads it for set dressing, and passing the growing text would + # let one guard's wording become another guard's input. + if people_here: + ident = persistent + n_in_shot = max(sum(1 for n in present_names + if person_in_shot(body, n, active, departed)), + 1 if active.get("") else 0) + states = [ + # the mouth, before anything describes the body it is in + (no_speech and not allow_vocals, lambda: LIPS_CLOSED_STATE + LIPS_CLOSED_TAIL), + # the limb COUNT, before anything constrains those limbs. 'auto' + # also fires wherever TWO or more bodies share the frame: spare + # limbs are grown where bodies meet or move together, whatever + # the resolution. + (anatomy_guard or (anatomy_auto and n_in_shot >= 2), + lambda: ANATOMY_STATE), + # what the restraints DO -- limits on limbs already established + (lock_restraints, lambda: restraint_clause(active, body, lock_restraints, + usage=restraint_usage)), + # a bared zone stays bared once the body turns to a surface the shot + # has not shown; the model's default for undescribed skin is clothed + (True, lambda: bare_persist_clause(bare_now, active, body)), + # two bodies arranged, BEFORE being told to hold together while moving + (True, lambda: contact_clause(body, n_in_shot, contact_guard)), + # then how that body moves -- unless someone in the frame is bound: + # the free-travel sentence outvotes the binding, and H3 settles + # that fight by rendering the restraints giving up. + (True, lambda: ("" if _restrained_present(active, body) + else motion_clause(body, motion_guard))), + # and last, what it cannot move through + (True, lambda: solidity_clause(body, ident, solidity_guard)), + ] + for want, produce in states: + clause = produce() if want else "" + if clause: + persistent = persistent.rstrip(". ") + "." + clause + silent_shot = no_speech and not allow_vocals + block = f"[Generation {gi}] {persistent}".strip() + # A silenced shot ALWAYS gets a soundscape line. Leaving the field out is + # what let H3 improvise a voice track under a shot whose picture was already + # told to keep its mouth shut -- the babble the lips-closed clause cannot + # reach, because it only constrains the frames. + if "soundscape:" not in block.lower(): + if gs: + if silent_shot: + block += f"\noverall_soundscape: {gs}{NO_VOICE_CLAUSE}" + elif no_speech: + block += f"\noverall_soundscape: {gs}{NO_VOICE_SPEECH_CLAUSE}" + else: + block += f"\noverall_soundscape: {gs}" + elif silent_shot: + block += f"\noverall_soundscape: {NO_VOICE_SOUNDSCAPE}" + elif no_speech: + block += f"\noverall_soundscape: {NO_VOICE_SPEECH_SOUNDSCAPE}" + # Music is OPT-IN: a blank field emits the spec's silence token N/A on every + # shot, so H3 doesn't improvise a score. (Soundscape is NOT forced to N/A -- + # per the spec it takes N/A only when total silence is explicitly wanted, so a + # blank soundscape still lets H3 provide ambient sound.) + if "non_diegetic_music:" not in block.lower(): + block += f"\nnon_diegetic_music: {music if music else 'N/A'}" + blocks.append(block.strip()) + # Exits stay DEFERRED, unlike removals: a character has to be visible in the + # shot that shows them leaving, and the frame they leave in is the shot's own + # subject -- there is no reverse-motion trap, because "walks out" ending with + # them present would contradict the beat itself, not just a description. + if exit_directive: + for nm in _entries(exit_directive): + departed.add(_norm_name(nm)) + departed.update(detect_exits(body, active, departed)) + # Plain-text case: a person described only in the anchor prose (never in the + # character channel) can't be "departed" by name -- scrub their phrase from + # the anchor instead, exactly as removed garments are scrubbed. + removed += departed_phrase_people(body, anchor_id) + # Anything this beat introduces indefinitely becomes referable later. + for _n, _ph in introduced_props(body).items(): + props.setdefault(_n, _ph) + seen_names |= {k for k in active if k} + # An exposed_terms key that matches neither a character nor a usable pronoun + # never fires, and it fails SILENTLY: the lookup falls through to the pronoun, + # then to the default wording, so a typo'd name looks configured and does + # nothing. Only the three canonical pronouns work as pronoun keys, because + # _pronoun_of() normalizes to those -- an object form like 'her' is dead + # config for the same reason and is worth the same warning. + if exposed and notes_out is not None: + known = {n.lower() for n in seen_names} | set(_PRO.values()) + for key in exposed: + if key in known: + continue + hint = "" + if key in _PRO: # 'her'/'him'/'his'/'them'... + hint = f" -- use '{_PRO[key]}' for the pronoun form" + elif seen_names: + hint = " -- known characters: " + ", ".join(sorted(seen_names)) + # Echo the key as the user typed it: parse_exposed_terms() lowercases, + # and quoting something back in different case than they wrote reads + # like a different entry. + m = re.search(r"^\s*(" + re.escape(key) + r")\b", exposed_terms or "", + re.I | re.M) + notes_out.append( + f"exposed_terms entry '{m.group(1) if m else key}' matches no character " + f"and no pronoun, so it never applies{hint}") + return blocks + + +# --- VRAM helpers ---------------------------------------------------------- +def vram_gb(device=None): + try: + dev = device or mm.get_torch_device() + total, free = mm.get_total_memory(dev) / GB, mm.get_free_memory(dev) / GB + if total > 0: + return round(total, 2), round(free, 2) + except Exception: + pass + try: + if torch.cuda.is_available(): + fb, tb = torch.cuda.mem_get_info() + return round(tb / GB, 2), round(fb / GB, 2) + except Exception: + pass + return 0.0, 0.0 + + +def _deep_cleanup(): + """Release VRAM + RAM between shots so a long chain doesn't accumulate and OOM. + Runs a Python GC pass (frees dereferenced tensors / CPU buffers), then hands + ComfyUI its aggressive cache purge, then empties the CUDA allocator's cached + blocks and IPC handles. Cheap relative to sampling; called once per beat.""" + gc.collect() + try: + mm.soft_empty_cache(True) # aggressive (unload_all_models path) + except TypeError: + mm.soft_empty_cache() + try: + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception: + pass + + +def dit_resident_gb(model): + """Loaded model weight size in GB, using ComfyUI's OWN accounting so the + figure matches how ComfyUI itself budgets VRAM and counts packed/quantized + dtypes correctly. The old raw numel*element_size sum over DiT params + over-counts NVFP4/FP8 (it reads unpacked shapes + scale tensors) -- that's + what produced the impossible 61.7GB reading. Falls back progressively.""" + # 1) ModelPatcher.model_size() -- authoritative, same call ComfyUI budgets with + try: + sz = model.model_size() + if sz and sz > 0: + return round(sz / GB, 2) + except Exception: + pass + # 2) model_management.module_size on the inner model + try: + inner = getattr(model, "model", None) + if inner is not None and hasattr(mm, "module_size"): + sz = mm.module_size(inner) + if sz and sz > 0: + return round(sz / GB, 2) + except Exception: + pass + # 3) last resort: raw param sum (over-counts quant, but non-zero) + dm = getattr(getattr(model, "model", None), "diffusion_model", None) + if dm is not None and hasattr(dm, "parameters"): + try: + return round(sum(p.numel() * p.element_size() for p in dm.parameters()) / GB, 2) + except Exception: + pass + return 0.0 + + +def estimate_shot_frames(total_gb, resident_gb, headroom_gb, pixels=None, free_gb=None): + """Largest grid-aligned shot length the card can attempt. + + Budgets from CARD CAPACITY minus measured weight size -- deliberately NOT from + instantaneous free VRAM. Free VRAM is read at one moment during graph execution, + and whatever is resident right then (the checkpoint, the text encoder, a LoRA's + adapters, another node's leftovers) makes it read far lower than the memory + actually available across the render. That produced a real failure: a 13.6GB + checkpoint at 640p floored to 124f/5s even though a forced 10s shot ran fine, + peaking at 15.2GB on a 15.9GB card and settling at 11.2GB. Capacity minus + weights is stable regardless of when the node happens to run. + + Model-agnostic: the only model-dependent input is resident_gb (ComfyUI's own + accounting), so NVFP4 / FP8 / INT8 / GGUF / BF16 all flow through the same + arithmetic -- a heavier checkpoint leaves less room and yields shorter shots. + The rest is the latent + activations, which scale with pixels x frames, so + `pixels` normalizes any resolution back to the native reference. + + Continuous fit, anchored to MEASURED points on a 16GB card: + 1344x768, ~11.7GB NVFP4 -> 243f fits; 362f overflowed by ~4.3GB + 640p, ~13.6GB HQ -> 10s (243f) fits, peak 15.2GB + free_gb is still accepted (callers pass it) but is used only as a sanity floor: + if the card is genuinely almost full right now, don't promise a long shot.""" + floor = align_frame_count(MIN_SHOT_FRAMES) + if total_gb <= 0: + return floor + avail = total_gb - resident_gb - headroom_gb + if resident_gb >= total_gb: + # STREAMING REGIME. model_size() reports the whole checkpoint, but a checkpoint + # larger than the card is never all resident: ComfyUI streams it, so the weight + # figure is NOT what occupies VRAM and cannot be subtracted from capacity. Doing + # that arithmetic anyway drove the budget deeply negative and floored every shot + # to 124f/~5s on a card that was demonstrably not running out -- a 44.3GB MXFP8 + # build on a 15.9GB card sampled 243f at 768x768 without exceeding VRAM. + # + # There is no meaningful "capacity minus weights" here, so budget from the LIVE + # free reading instead: it measures what is actually unoccupied right now, which + # in this regime is the only number that means anything. Without a reading there + # is nothing to go on, so fall back to the floor. + if not free_gb or free_gb <= 0: + return floor + avail = max(0.0, free_gb * (1.0 - SPIKE_RESERVE) - headroom_gb) + # avail <= 0 here means the weights FIT but the safety headroom eats what is left. + # That is not the same thing, and it used to floor every shot to 124f/~5s no matter + # what -- including at the fast 512 tier, where a frame costs a quarter as much. + # Two dialogue beats came out at ~5s each on a card that could hold far more. The + # baseline term below already represents the latent that fits in space the weight + # accounting has covered, so let the arithmetic run instead of bailing out. + if avail > 0 and pixels and pixels > 0: + # Lower res -> effectively more room. Only ever applied to a POSITIVE surplus: + # a deficit is weights that do not fit, which no resolution can shrink, and + # scaling it would perversely make lower resolutions look worse. + avail *= NATIVE_PIXELS / float(pixels) + frames = FRAMES_PER_GB * (avail + FRAMES_BASELINE_GB) + # Sanity floor from a LIVE reading: capacity-minus-weights is the right basis + # (see above), but if the card is genuinely almost empty right now -- another + # app holding VRAM, a model that failed to unload -- do not promise a long + # shot on paper. Only ever REDUCES the estimate; it can never raise it, so a + # momentarily low reading during model load can't floor the budget the way + # budgeting from free_gb directly used to. + if free_gb is not None and free_gb > 0: + live = FRAMES_PER_GB * ((free_gb * (1.0 - SPIKE_RESERVE)) + FRAMES_BASELINE_GB) + if pixels and pixels > 0: + live = FRAMES_PER_GB * (((free_gb * (1.0 - SPIKE_RESERVE)) + * (NATIVE_PIXELS / float(pixels))) + FRAMES_BASELINE_GB) + frames = min(frames, live) + frames = max(MIN_SHOT_FRAMES, min(H3_MAX_FRAMES, int(frames))) + return max(floor, align_frame_count(min(H3_MAX_FRAMES, frames))) + + +def resolve_shot_frames(shot_seconds, fps, total_gb, resident_gb, headroom_gb, + allow_oversize=False, pixels=None, free_gb=None): + """Returns (frames, note). + + Auto mode (shot_seconds <= 0): frames = the VRAM budget estimate (resolution- + scaled). Forced mode: the requested length is clamped DOWN to the budget + unless allow_oversize is set. When VRAM is unknown the request is honored.""" + budget = estimate_shot_frames(total_gb, resident_gb, headroom_gb, pixels, free_gb) + if not (shot_seconds and float(shot_seconds) > 0): + return budget, "" + requested = align_frame_count(min(H3_MAX_FRAMES, max(5, round(float(shot_seconds) * fps)))) + if total_gb <= 0 or requested <= budget: + return requested, "" + if allow_oversize: + return requested, (f"OVERSIZE: {requested}f requested vs {budget}f budget -- honoring it; " + f"may spill to system RAM (slow) or OOM") + return budget, (f"requested {requested}f (~{requested/max(1,fps):.1f}s) exceeds the ~{budget}f VRAM " + f"budget -- clamped to {budget}f (~{budget/max(1,fps):.1f}s). Set allow_oversize_shots to override") + + +def _is_oom(e): + return isinstance(e, torch.cuda.OutOfMemoryError) or "out of memory" in str(e).lower() + + +# --- conditioning + decode ------------------------------------------------- +def _resize(image, width, height, crop): + s = image[..., :3].movedim(-1, 1) + s = comfy.utils.common_upscale(s, width, height, "lanczos", crop) + return s.movedim(1, -1) + + +def _empty_av_latent(width, height, length, fps, batch_size=1): + fc, lt, at = temporal_shape(length, fps) + video = torch.zeros([batch_size, 24, lt, height // 16, width // 16], device=mm.intermediate_device()) + audio = torch.zeros([batch_size, 32, 2, at], device=mm.intermediate_device()) + return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, fc + + +def _decode_video(vae, out_latent, tiled, free_first=None, tile_t=None, tile_xy=None): + """Decode the video latent. If `free_first` is the diffusion model, unload it + first: sampling is finished, and the ~5GB video VAE needs the room. Leaving the + DiT (plus resident bypass-LoRA adapters) on the card while the VAE loads is a + second ratchet -- ComfyUI would otherwise evict reactively, after spilling.""" + if free_first is not None: + try: + mm.free_memory(1e30, mm.get_torch_device(), keep_loaded=[]) + except Exception: + pass + latent = out_latent["samples"] + if latent.is_nested: + latent = latent.unbind()[0] + if tiled: + # Temporal + spatial tiling. Without tile_t the VAE expands the WHOLE latent + # clip at once, which on a 243-frame 1344x768 shot is the single largest + # allocation in the run -- and on an unpruned checkpoint that is already + # streaming, it is what tips the card over. Decoding in temporal chunks + # trades a little speed for a much lower peak; None keeps ComfyUI's defaults. + args = {} + if tile_t: + args["tile_t"] = int(tile_t) + args["overlap_t"] = max(1, int(tile_t) // 8) + if tile_xy: + args["tile_x"] = int(tile_xy) + args["tile_y"] = int(tile_xy) + try: + imgs = vae.decode_tiled(latent, **args) if args else vae.decode_tiled(latent) + except TypeError: + imgs = vae.decode_tiled(latent) # older signature without tile_t + else: + imgs = vae.decode(latent) + if len(imgs.shape) == 5: + imgs = imgs.reshape(-1, imgs.shape[-3], imgs.shape[-2], imgs.shape[-1]) + return imgs + + +# One second of encoded silence, made once per process and tiled to any length. +# Module-level so it outlives the node instance, which ComfyUI recreates per run. +_SILENT_UNIT = {"lat": None} + + +def _silent_audio_latent(audio_vae, frame_count, fps): + """A keyframe audio latent of actual SILENCE, or None if it cannot be made. + + H3 is a JOINT model: the mouth follows the audio branch. On a shot with no + scripted line the branch is otherwise unconditioned, and an unconditioned audio + branch invents a voice -- which the picture then lip-syncs to. The lips-closed + sentence is arguing with a stream that has already decided someone is talking. + + Seeding the keyframe's audio channel with encoded silence anchors that stream + instead. comfy/ldm/minimax/audio_vae.py encode() takes stereo [B, 2, L] at + 32 kHz and returns [B, 32, 2, T] on the same 40 Hz grid temporal_shape() uses. + + Everything here is defensive. The shape is CHECKED against what the layout + expects rather than assumed, and any failure returns None so the shot falls + back to today's behaviour instead of breaking the render.""" + try: + sr = int(getattr(audio_vae, "audio_sample_rate", 0) or 0) + if sr <= 0: + return None + _, _, want_t = temporal_shape(frame_count, fps) + if want_t <= 0: + return None + unit = _SILENT_UNIT.get("lat") + if unit is None: + # CHANNELS LAST. comfy.sd.VAE.encode() does `pixel_samples.movedim(-1, 1)` + # before handing off, so the audio VAE -- which wants [B, 2, L] -- must be + # given [B, L, 2]. Passing [B, 2, L] raises inside the encoder, and the + # first version did exactly that: swallowed by the guard below, so the + # whole layer silently did nothing. + # + # ONE SECOND, encoded ONCE. Silence is homogeneous, so the result tiles + # along time -- and encoding a full 15s shot instead cost a VAE pass big + # enough to OOM mid-render on a 16GB card, where the failure again + # degraded silently to no conditioning at all. + enc = audio_vae.encode(torch.zeros((1, sr, 2))) + if enc is None or enc.dim() != 4 or enc.shape[1] != 32 or enc.shape[-1] < 3: + return None + # ONE STEADY FRAME, from the MIDDLE. The encoder's zero-padding leaves + # heavy edge artifacts -- measured deviation 0.351 at the first and last + # frames against 0.002 in the interior, ~170x -- and tiling the whole + # second therefore stamped a spike every 40 latent frames, which at 40Hz + # is once per SECOND. That is a metronome in the audio conditioning of a + # joint audio-video model, and the picture lip-syncs to it. Repeating a + # single interior frame gives conditioning that is genuinely constant. + mid = enc.shape[-1] // 2 + _SILENT_UNIT["lat"] = enc[..., mid:mid + 1].detach().to("cpu").clone() + unit = _SILENT_UNIT["lat"] + if unit.shape[-1] != 1: + return None + return unit.repeat(1, 1, 1, want_t).clone() + except Exception: + return None # never fail a render for a nicety + + +def _decode_audio(audio_vae, out_latent): + latent = out_latent["samples"] + if latent.is_nested: + latent = latent.unbind()[-1] + audio = audio_vae.decode(latent).movedim(-1, 1) + std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0 + std[std < 1.0] = 1.0 + audio = audio / std + sr = getattr(audio_vae, "audio_sample_rate_output", getattr(audio_vae, "audio_sample_rate", 44100)) + return {"waveform": audio, "sample_rate": sr} + + +# --- ref2va reference conditioning ---------------------------------------- +# H3's reference pipeline encodes a reference image at up to a 2048 short edge. +# Reference rows ride through EVERY sampling step, so this is also the setting +# that decides how much the references cost per step. +REF_IMAGE_SHORT_EDGE = 2048 +CANVAS_MULTIPLE = 32 + + +def ref_image_canvas(w, h, gen_w, gen_h, mode="match"): + """Pure: the (width, height) a reference image is encoded at. + + 'match' scales it (DOWN only, aspect kept) to the generation's pixel area, so a + reference costs about as much as one frame of the shot. 'max' goes to the + reference pipeline's 2048 short edge for the best identity fidelity, which on a + long chain is several times slower because the rows are re-attended every step + of every shot. Never upscales: a small reference stays small.""" + w, h = max(1, int(w)), max(1, int(h)) + if mode == "max": + scale = min(1.0, REF_IMAGE_SHORT_EDGE / min(w, h)) + else: + scale = min(1.0, math.sqrt((int(gen_w) * int(gen_h)) / float(w * h))) + snap = lambda v: max(CANVAS_MULTIPLE, round(v * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) + return snap(w), snap(h) + + +def _build_ref_images(vae, images, gen_w, gen_h, mode="match"): + """(tokenizer items, DiT blocks) for a list of reference IMAGE tensors. + + The tokenizer labels each one `:` itself, in the order given here -- + so the roster the prompt refers to is decided by input order, not by anything + written in the prompt.""" + items, blocks = [], [] + for img in images: + if img is None: + continue + h, w = int(img.shape[1]), int(img.shape[2]) + tw, th = ref_image_canvas(w, h, gen_w, gen_h, mode) + resized = _resize(img[:1], tw, th, "disabled") + items.append({"type": "image", "data": resized}) + blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16, + "latent": vae.encode(resized)}) + return items, blocks + + +def _build_shot_conditioning(clip, vae, prompt, width, height, length, fps, handoff, + ref_images=None, ref_image_size="match", ref_noise_aug=None, + audio_vae=None, silent=False): + latent, fc = _empty_av_latent(width, height, length, fps) + refs = [r for r in (ref_images or []) if r is not None] + if refs: + # ref2va: this shot is reference-conditioned rather than keyframe-conditioned, + # and run() decides which per shot. A tagged shot is handed the previous + # frame as an extra REFERENCE so a tag never reads as a cut. + # + # WHY it is either/or is now historical. On ComfyUI 0.30 the two channels + # could not ride together: model_base.py filled ONE `cond_video_latents` + # list and the refs branch OVERWROTE whatever the keyframe branch had put + # there, while PackedLayout still laid out rows for both -- so the row count + # and the latent count disagreed, landing as a shape error deep in the DiT + # or feeding keyframe rows a reference's latent. + # + # ComfyUI 0.31+ concatenates instead: + # payload["cond_video_latents"] = payload.get("cond_video_latents", []) + [...] + # and PackedLayout appends keyframe segments before ref segments, so the two + # orders agree and both channels coexist. A tagged shot therefore takes its + # references AND a real keyframe: the keyframe ANCHORS the first frame, + # which is what continuity needs, while a reference only supplies identity. + # Passing the handoff as a reference (the 0.30 workaround) asked the model to + # look like the previous frame rather than to start from it. + items, blocks = _build_ref_images(vae, refs, width, height, ref_image_size) + # The handoff has to reach the TEXT ENCODER as well as the DiT. minimax.py's + # tokenize_with_weights is EITHER/OR (:158-184): passing minimax_ref_items + # makes it ignore `images` entirely. So on this path the VLM saw only the + # identity references and never the previous frame -- it was told the location + # in words and given a latent anchor at frame 0, but nothing showed it where + # the shot left off, so it re-imagined the scenery. Same place, new scene: + # the keyframe alone anchors the first frame without describing it. + # + # The keyframe-only path has always shown it as `` via images=, so + # this is the same convention. Appended AFTER the references, leaving their + # numbers -- which the prompt's tags refer to -- untouched. + hand_img = None + if handoff is not None and keyframe_rides_with_refs(ref_noise_aug): + hand_img = _resize(handoff[:1], width, height, "disabled") + items = items + [{"type": "image", "data": hand_img}] + tokens = clip.tokenize(prompt, minimax_ref_items=items) + cond = clip.encode_from_tokens_scheduled(tokens) + vals = {} + if blocks: + vals["minimax_refs"] = blocks + # How CLEAN the reference is presented as. The DiT both blends the + # condition latent with noise at (1 - aug) and labels those rows with a + # timestep of max(t_video, aug) -- so the model default of 0.999 hands it + # a finished, noise-free image, which is an invitation to REPRODUCE it + # rather than to take an identity from it. Lower says "approximate". + # + # It is a payload-level value, so it reaches the keyframe rows too. That + # is why the handoff stays out of the ref channel: a keyframe softened to + # 0.90 would stop anchoring, and continuity is exactly what it is for. + if ref_noise_aug is not None: + vals["minimax_visual_cond_noise_aug"] = float(ref_noise_aug) + # Enforced here as well as in run(): one aug covers every cond latent, so a + # softened reference would soften the anchor. Refusing at the source means no + # caller can assemble that combination by accident. + kfs = [] + if hand_img is not None: + # Same tensor the tokenizer was given, encoded once: the VLM is shown the + # frame and the DiT is anchored to it, which is what a seamless cut needs. + kfs.append({"resolved_frame_index": 0, "latent": vae.encode(hand_img)}) + # A reference-conditioned shot needs the silence anchor just as much as a + # keyframe-conditioned one, and it used to get NOTHING: `silent` was only + # honoured on the keyframe-only path below, so wiring any ref_image made the + # whole mechanism dead code and non-dialogue shots babbled again. + # + # It has to hang on a KEYFRAME, not a ref block. model_base.py:2174 does put a + # ref's audio_latent into cond_audio_latents, but PackedLayout only emits + # ref_audio rows for blocks of kind audio/video/video_audio (model.py:379-399) + # -- an "image" ref gets none -- so the latent list and the row layout would + # disagree and land as a shape error inside the DiT. + kfs = _attach_silence(kfs, audio_vae, fc, fps, silent) + if kfs: + vals["minimax_keyframes"] = kfs + vals["minimax_frame_count"] = fc + if vals: + cond = node_helpers.conditioning_set_values(cond, vals) + return cond, latent + # (fall through to the keyframe-only path below) + images, keyframes = [], [] + if handoff is not None: + img = _resize(handoff[:1], width, height, "disabled") + images.append(img) + keyframes.append({"resolved_frame_index": 0, "image": img}) + tokens = clip.tokenize(prompt, images=images) + cond = clip.encode_from_tokens_scheduled(tokens) + for kf in keyframes: + kf["latent"] = vae.encode(kf.pop("image")) + # Outside the `if keyframes:` it used to sit inside. A shot with no handoff -- + # the FIRST shot of every chain -- has no keyframe, so the silence anchor was + # skipped there too, on exactly the shot that sets the tone for the rest. + keyframes = _attach_silence(keyframes, audio_vae, fc, fps, silent) + if keyframes: + cond = node_helpers.conditioning_set_values(cond, {"minimax_keyframes": keyframes, "minimax_frame_count": fc}) + return cond, latent + + +def _attach_silence(keyframes, audio_vae, fc, fps, silent): + """Anchor this shot's audio channel to real silence. Returns the keyframe list. + + H3 is JOINT -- the mouth follows the audio branch -- so an unconditioned audio + stream invents a voice and the picture lip-syncs to it. A "lips closed" sentence + in the prompt cannot outvote a stream that has already decided someone is + speaking; this conditions the stream itself. + + Hangs on a keyframe because that is the only carrier PackedLayout emits matching + `cond_audio` rows for (model.py:354-361). When the shot has no keyframe -- the + first shot of a chain, or a ref-conditioned shot -- an AUDIO-ONLY keyframe is + appended: model_base.py:2168 filters video latents on `latent` and audio on + `audio_latent` independently, and model.py:345 skips the video segment when + `latent` is absent, so a dict carrying only audio produces exactly one cond_audio + segment and no video rows. + """ + if not silent or audio_vae is None: + return keyframes + sil = _silent_audio_latent(audio_vae, fc, fps) + if sil is None: + return keyframes + if keyframes: + keyframes[0]["audio_latent"] = sil # one bed, on the first carrier only + else: + keyframes = [{"resolved_frame_index": 0, "audio_latent": sil}] + return keyframes + + +# Below this, softening the references would soften the handoff KEYFRAME with them. +# visual_cond_noise_aug is a single payload value applied to every cond video latent +# (ldm/minimax/model.py:502-510) and it labels both segments the same way (:584: +# "cond": max(t_v, vis_aug), "ref_img": max(t_v, vis_aug)). There is no per-channel +# control, so a lowered ref_noise_aug would blend the anchor frame with noise and +# push its timestep up -- destroying exactly the continuity the keyframe is for. +KEYFRAME_SAFE_AUG = 0.99 + + +def keyframe_rides_with_refs(ref_noise_aug): + """Can a tagged shot carry its handoff as a real KEYFRAME alongside references? + + Yes at the default aug, where the keyframe passes through essentially untouched. + No once the user has softened the references, because the same value would soften + the keyframe -- there the handoff falls back to riding as an extra reference, + which is weaker for continuity but leaves nothing to compromise.""" + return ref_noise_aug is None or float(ref_noise_aug) >= KEYFRAME_SAFE_AUG + + +# , , -- all the ways people write the tag. +_PICTURE_TAG = re.compile(r"<\s*picture[\s_\-]*(\d+)\s*>", re.I) + + +def picture_tags(text): + """The reference slots a shot's text asks for, in ascending order.""" + return sorted({int(m.group(1)) for m in _PICTURE_TAG.finditer(text or "")}) + + +def resolve_tagged_refs(text, ref_list): + """(rewritten text, images, dropped) for the tags in ONE shot. + + The tokenizer numbers references by their position in the list it is handed, so + a shot that uses only would receive that image labelled + and the text would point at nothing. The tags are therefore + RENUMBERED per shot to match what that shot actually carries: slot 2 alone + becomes , slots 2 and 4 become and . + + A tag naming a slot with no image connected refers to nothing at all, so it is + removed from the text rather than left to confuse the encoder, and reported.""" + wanted = picture_tags(text) + live = [n for n in wanted if 1 <= n <= len(ref_list or [])] + dropped = [n for n in wanted if n not in live] + renumber = {old: new for new, old in enumerate(live, 1)} + + def sub(m): + n = int(m.group(1)) + return f"" if n in renumber else "" + + out = _PICTURE_TAG.sub(sub, text or "") + if dropped: # tidy the gap a removed tag leaves behind + out = re.sub(r"\s+([,.;:])", r"\1", out) + out = re.sub(r"(,\s*){2,}", ", ", out) + out = re.sub(r"\s{2,}", " ", out) + return out.strip(), [ref_list[n - 1] for n in live], dropped + + +def shot_references(ref_list, ref_mode, shot_index, handoff): + """Pure: which reference images shot `shot_index` is conditioned on, or [] when + the shot should use the keyframe handoff instead. + + NOTE: on ComfyUI 0.31+ a shot CAN carry both channels, and run() adds the handoff + as a real keyframe alongside whatever this returns (unless ref_noise_aug has been + lowered, which would soften the anchor too). The descriptions below are what each + mode contributes to the REFERENCE channel; they no longer describe a shot's whole + conditioning, and 'no handoff at all' is no longer a consequence of picking one: + + 'first shot' -- references establish the cast in shot 1; every later shot + uses the last-frame handoff. Continuity is unbroken and the + look propagates down the chain, but only through the frames. + 'every shot' -- every shot is ref-conditioned. Strongest identity, and no + handoff at all, so shots meet as CUTS rather than as one + continuous take. + 'every shot + handoff ref' + -- every shot is ref-conditioned AND the previous shot's last + frame is appended as one more reference. Continuity comes + back as a soft signal (the model is shown where the last + shot ended rather than told to start exactly there), and it + stays a single ref2va task, so nothing conflicts.""" + if not ref_list: + return [] + if ref_mode == "first shot": + return list(ref_list) if shot_index == 0 else [] + if ref_mode == "every shot": + return list(ref_list) + if ref_mode == "every shot + handoff ref": + return list(ref_list) + ([handoff] if handoff is not None else []) + return list(ref_list) if shot_index == 0 else [] # unknown value -> safest + + +# --- text-encoder / DiT compatibility ------------------------------------- +# H3's DiT accepts text conditioning at exactly two widths (comfy/ldm/minimax/ +# model.py, preprocess_text_embeds): +# * text_dim -- raw encoder states, projected by condition_proj (5120 on +# stock H3: Qwen3-VL-32B truncated to 50 layers) +# * hidden_size-- states already refined to DiT width (5376), passed through +# Anything else dies deep inside ComfyUI as a bare "mat1 and mat2 shapes cannot +# be multiplied (156x6144 and 5120x5376)", which reads like a bug in this node. +# Check the width up front and name what is actually wrong. +_TE_HIDDEN = { + 5120: "Qwen3-VL-32B truncated to 50 layers -- the H3 text encoder", + 5376: "text embeds already refined to DiT width", + 4096: "Qwen3-VL-8B / T5-XXL -- not an H3 encoder", + 3584: "Qwen2.5-VL-7B -- not an H3 encoder", + 2560: "Qwen3-VL-4B -- not an H3 encoder", + 2048: "Qwen3-VL-2B / Qwen3-30B-A3B -- not an H3 encoder", +} + + +def _te_name(dim): + return _TE_HIDDEN.get(dim, "not a width any H3 encoder produces") + + +def text_encoder_mismatch_note(got, accepted): + """Pure: message for conditioning of width `got` fed to a DiT that accepts + the widths in `accepted`, or None if it fits / nothing is known. Torch-free + so the tests can drive it.""" + ok = sorted({int(a) for a in (accepted or ()) if a}) + if not got or not ok or int(got) in ok: + return None + got = int(got) + return ( + f"H3 Long Videos: the CLIP input does not match this diffusion model. Its " + f"conditioning is {got}-dim ({_te_name(got)}), but this H3 DiT only accepts " + + " or ".join(f"{a} ({_te_name(a)})" for a in ok) + ". Check, in this order: " + f"(1) the CLIPLoader feeding 'clip' is set to the MiniMax-H3 type -- the same " + f"file loaded under another type gives a different width; (2) the encoder file " + f"is the H3 one that shipped with your H3 checkpoint, not another Qwen3-VL; " + f"(3) no upstream node replaced the conditioning between the encoder and this " + f"node. Nothing was rendered." + ) + + +def _dit_text_widths(model): + """The text widths this DiT accepts: (condition_proj.in_features, + hidden_size). Reads module attributes, not weight.shape -- a quantized or + packed weight has a misleading shape and would fake a mismatch. Missing + values are dropped, so a model this can't introspect yields ().""" + m = getattr(model, "model", model) + dm = getattr(m, "diffusion_model", None) + proj = getattr(dm, "condition_proj", None) + out = [] + for n in (getattr(proj, "in_features", None), getattr(dm, "hidden_size", None)): + if isinstance(n, int) and n > 0: + out.append(n) + return tuple(out) + + +def _cond_embed_dim(cond): + """Width of an encoded conditioning's embedding tensor, or None.""" + try: + return int(cond[0][0].shape[-1]) + except Exception: + return None + + +# --- quantization kernels --------------------------------------------------- +# The kernels are NOT something this node installs or calls. comfy_kitchen (imported +# as `ck` in comfy/quant_ops.py) is a compiled package shipped with ComfyUI, and +# comfy/ops.py routes every quantized Linear through it -- ck.int8_linear() and +# friends -- whenever the loaded weights carry a quant_format. Sampling runs the +# model, so the node gets that path for free and cannot opt in or out of it. +# +# What the node CAN do is notice when the path is not there, because that failure +# is silent. If the CUDA backend is disabled (torch built against CUDA < 13 +# triggers ck.registry.disable("cuda")) or comfy_kitchen fails to import, ComfyUI +# logs one line at startup and then quietly runs a slower, lower-fidelity route. +# Nothing errors, and the first sign is soft output -- which is indistinguishable +# from a dozen other causes unless something says so. +# +# Every format the checkpoint might use, mapped to the comfy-kitchen capability +# that serves it. The names come from ck.list_backends()[...]["capabilities"]. +_QUANT_CAPABILITY = { + "int8_tensorwise": "int8_linear", + "int8_tensorwise+convrot": "int8_linear", + "convrot_w4a4": "convrot_w4a4_linear", + "asym_w4a8_int8": "w4a8_int8_linear", + "nvfp4": "scaled_mm_nvfp4", + "mxfp8": "scaled_mm_mxfp8", +} + + +def quant_format_of(model): + """The dominant quant format of the loaded DiT, or "" when it is unquantized. + + Same detection the inspector node uses: a module's `quant_format` tag, with + int8 split by whether its packed weight carries convrot.""" + try: + dm = getattr(getattr(model, "model", None), "diffusion_model", None) + if dm is None or not hasattr(dm, "modules"): + return "" + counts = {} + for m in dm.modules(): + fmt = getattr(m, "quant_format", None) + if fmt is None: + continue + if fmt == "int8_tensorwise": + params = getattr(getattr(m, "weight", None), "_params", None) + if getattr(params, "convrot", False): + fmt = "int8_tensorwise+convrot" + counts[fmt] = counts.get(fmt, 0) + 1 + return max(counts.items(), key=lambda kv: kv[1])[0] if counts else "" + except Exception: + return "" + + +def kernel_backend_note(model): + """Warn when the loaded checkpoint's quant format has no accelerated backend. + + Silent on an unquantized checkpoint (nothing to accelerate) and silent when a + capable backend is present, so it only speaks when something is actually + wrong.""" + fmt = quant_format_of(model) + if not fmt: + return "" + want = _QUANT_CAPABILITY.get(fmt) + try: + import comfy_kitchen as ck + except Exception as e: + return (f"the checkpoint is {fmt} but comfy_kitchen failed to import " + f"({type(e).__name__}) -- ComfyUI is running the slow dequantize " + f"fallback, which is lower fidelity as well as slower") + try: + backends = ck.list_backends() + except Exception: + return "" # cannot introspect; never block on that + live = [name for name, b in backends.items() + if b.get("available") and not b.get("disabled") + and (want is None or want in (b.get("capabilities") or ()))] + if live: + return "" + disabled = [f"{name} ({b.get('unavailable_reason') or 'disabled'})" + for name, b in backends.items() if not b.get("available") or b.get("disabled")] + return (f"the checkpoint is {fmt} but no comfy-kitchen backend offers " + f"'{want or fmt}' -- " + ("; ".join(disabled) if disabled else "none available") + + ". ComfyUI falls back to a dequantize path: slower, and lower fidelity. " + "Check the startup log for 'Found comfy_kitchen backend', and that torch " + "is built against CUDA 13+ (cu130), which is what keeps the CUDA backend " + "enabled") + + +# --- flow-shift vs step count ---------------------------------------------- +# The shift maps timesteps onto sigmas, and the 'simple' scheduler then samples +# that curve at evenly spaced INDICES. At a high shift the curve is steep at the +# low-sigma end, so the last interval swallows most of the run: +# +# 4 steps, shift 12 -> 2.7% / 5.0% / 12.3% / 80.0% +# 4 steps, shift 3 -> 10.0% / 15.0% / 25.0% / 50.0% +# 20 steps, shift 12 -> worst single step 38.7% +# +# 12 is the right default -- it is what H3's own model config declares, and at ~20 +# steps it is well balanced. It only misbehaves when a distill LoRA drops the step +# count under it, and then it does so invisibly: nothing errors, the picture just +# comes back soft and painterly because one enormous final jump cannot resolve fine +# detail. Cheap to detect, so detect it. +def shot_latent_cells(w, h, frames, fps): + """Latent cells in one shot: what sampling VRAM actually scales with. + + Not a byte figure -- the constant depends on the quantisation path -- but it is + exactly linear in both shot length and area, so ratios between settings are + right even though the absolute number is not a prediction.""" + _, lt, _ = temporal_shape(frames, fps) + return max(1, int(lt)) * max(1, w // 16) * max(1, h // 16) + + +def sampling_oom_help(w, h, frames, fps, megapixels=0.0): + """What to change, in this shot's own numbers, after a SAMPLING OOM. + + Tiling is a decode setting and cannot help here, so the generic "try tiling" + advice is worse than useless -- it costs another full sampling pass before + failing the same way. Give the two levers that do change sampling cost, each + priced from the shot that just failed.""" + now = shot_latent_cells(w, h, frames, fps) + secs = frames / float(fps or 24) + out = [f"This is a SAMPLING out-of-memory, not a decode one, so tiled decode " + f"cannot help it. The shot is {w}x{h} x {frames}f (~{secs:.1f}s) = " + f"{now:,} latent cells, and sampling cost scales linearly with that."] + opts = [] + for cut in (10.0, 7.0): + if cut < secs - 0.4: + f2 = align_frame_count(int(round(cut * (fps or 24)))) + opts.append(f"shot_seconds {cut:g} ({f2}f) is " + f"{100 - shot_latent_cells(w, h, f2, fps) * 100 // now}% smaller") + if megapixels: + for mp in (0.5, 0.35): + if mp < megapixels - 0.02: + w2, h2 = scale_to_megapixels(w, h, mp) + opts.append(f"megapixels {mp:g} ({w2}x{h2}) is " + f"{100 - shot_latent_cells(w2, h2, frames, fps) * 100 // now}% smaller") + if opts: + out.append("Options: " + "; ".join(opts) + ".") + out.append("Shot length is the stronger lever on a chain, because every shot pays it. " + "H3's own cap is 362 frames and this shot is at or near it.") + return " ".join(out) + + +def flow_step_shares(shift, steps, timesteps=1000): + """Fraction of total denoising each sampler step performs. + + Mirrors comfy.model_sampling.ModelSamplingDiscreteFlow (time_snr_shift) plus + comfy.samplers.simple_scheduler, which indexes the precomputed sigma table + linearly. Returns [] when the inputs cannot form a schedule.""" + steps = int(steps) + if steps < 2 or shift is None or float(shift) <= 0: + return [] + a = float(shift) + table = [a * t / (1.0 + (a - 1.0) * t) + for t in ((i + 1) / timesteps for i in range(timesteps))] + stride = len(table) / steps + sig = [table[-(1 + int(x * stride))] for x in range(steps)] + [0.0] + deltas = [sig[i] - sig[i + 1] for i in range(steps)] + total = sum(deltas) + if total <= 0: + return [] + return [d / total for d in deltas] + + +# On ComfyUI 0.31+ the two shifts are COUPLED, and that is new. ModelSamplingAV +# carries the audio latent on the VIDEO schedule scaled by +# +# audio_scale = shift_video / shift_audio (12 / 3 = 4.0 by default) +# +# and that ratio drives process_latent_in, process_latent_out, the minimax payload +# and the DiT forward (comfy/model_base.py 2141/2144/2181, ldm/minimax/model.py:530). +# Collapse it to 1.0 by setting the two shifts equal and the audio pipeline loses +# the scaling it is built around -- which comes back as babble or silence. +# +# Before 0.31 the audio velocity was scaled by a derivative instead, so the two +# shifts were effectively independent and lowering shift_video alone was harmless. +# It is not harmless now. +H3_AUDIO_SCALE = 4.0 # the model's own 12/3 + + +def audio_scale_note(shift_video, shift_audio): + """Warn when the video/audio shift RATIO has drifted from what H3 expects.""" + try: + sv, sa = float(shift_video), float(shift_audio) + except (TypeError, ValueError): + return "" + if sa <= 0 or sv <= 0: + return "" + ratio = sv / sa + if ratio >= 1.75: + return "" + return (f"shift_video {sv:g} / shift_audio {sa:g} gives audio_scale {ratio:.2f}, " + f"against the {H3_AUDIO_SCALE:g} this model is built around. The audio latent " + f"rides the VIDEO schedule scaled by that ratio, so flattening it toward 1.0 " + f"breaks the audio branch -- babble or silence. Keep the two shifts about " + f"{H3_AUDIO_SCALE:g}:1 apart: for shift_video {sv:g}, use shift_audio " + f"{max(0.25, sv / H3_AUDIO_SCALE):g}") + + +def schedule_balance_note(shift, steps, scheduler, worst_allowed=0.55): + """Warn when one sampler step carries most of the denoising. + + Only for the 'simple' scheduler, because that is the curve this reproduces -- + reporting these numbers for a scheduler that spaces sigmas differently would be + making them up. The suggestion is searched rather than guessed: the lowest + shift whose worst step falls under the threshold.""" + if str(scheduler) != "simple": + return "" + shares = flow_step_shares(shift, steps) + if not shares: + return "" + worst = max(shares) + if worst <= worst_allowed: + return "" + better = "" + for cand in (6.0, 5.0, 4.0, 3.0, 2.5, 2.0, 1.5, 1.0): + cand_shares = flow_step_shares(cand, steps) + if cand_shares and max(cand_shares) <= worst_allowed: + # shift_audio has to come down WITH it. The two are coupled on 0.31+: + # audio_scale = shift_video / shift_audio, and lowering only the video + # shift flattens that ratio toward 1.0, which breaks the audio branch. + # Suggesting a bare shift_video here is what produced babble. + better = (f"; shift_video {cand:g} spreads it to " + + "/".join(f"{s * 100:.0f}%" for s in cand_shares) + + f" -- lower shift_audio to {max(0.25, cand / H3_AUDIO_SCALE):g} " + f"at the same time, or the audio breaks") + break + return (f"shift_video {float(shift):g} at {int(steps)} steps puts " + + "/".join(f"{s * 100:.0f}%" for s in shares) + + f" of the denoising into each step -- one step doing {worst * 100:.0f}% " + f"cannot resolve fine detail, which renders as soft, painterly output" + + better) + + +def check_text_encoder(model, cond): + """Raise a readable RuntimeError when clip and model disagree. Silent when + either side can't be read -- never block a run on a failed introspection.""" + note = text_encoder_mismatch_note(_cond_embed_dim(cond), _dit_text_widths(model)) + if note: + raise RuntimeError(note) + + +def _is_audio_vae(v): + """True when v looks like the H3 audio VAE (DAC/BigVGAN), False when it looks + like a video/image VAE, None when it can't be told. The video VAEs carry a + 3-tuple upscale_ratio (t, y, x); the audio VAE carries a scalar and reports + latent_dim 2 with an audio_sample_rate.""" + ur = getattr(v, "upscale_ratio", None) + if isinstance(ur, (tuple, list)): + return False + if getattr(v, "audio_sample_rate", None) or getattr(v, "audio_sample_rate_output", None): + return True + if isinstance(ur, (int, float)) and getattr(v, "latent_dim", None) == 2: + return True + return None + + +def check_audio_vae_loaded(audio_vae): + """Catch an UNCONVERTED audio VAE checkpoint. + + comfy/ldm/minimax/audio_vae.py loads a checkpoint whose weight-norm has been + folded into plain "*.weight" tensors. Feed it the raw upstream file (172 + weight_g/weight_v pairs, no latents_mean/latents_std) and load_state_dict + reports the misses as a WARNING, not an error: every weight-normed conv keeps + its random init and the two normalization buffers stay torch.empty(), i.e. + uninitialized memory. Decoding then multiplies the latents by garbage and the + audio comes out as noise -- with nothing in the log at render time to say why. + + latents_std is the cheapest tell: it is a real per-channel scale, so a + non-finite or absurd value means the buffer was never filled.""" + m = getattr(audio_vae, "first_stage_model", None) + mean, std = getattr(m, "latents_mean", None), getattr(m, "latents_std", None) + if mean is None or std is None: + return + try: + bad = (not torch.isfinite(mean).all() or not torch.isfinite(std).all() + or float(std.min()) <= 0.0 or float(std.max()) > 1e3 + or float(mean.abs().max()) > 1e3) + except Exception: + return # never block a render on a failed introspection + if bad: + raise RuntimeError( + "the audio VAE loaded but its weights are NOT initialized -- this is the raw " + "upstream MiniMax-H3 audio checkpoint (weight_g/weight_v weight-norm pairs, no " + "latents_mean/latents_std). ComfyUI's loader needs the CONVERTED file, with " + "weight-norm folded into plain '*.weight' tensors. Look for the 'Missing VAE keys' " + "warning in the log when the VAE loaded. Download the repackaged H3 audio VAE from " + "the Comfy-Org release; rendering with this one produces noise, not speech.") + + +def check_vae_wiring(vae, audio_vae): + """Catch the commonest miswire -- the video VAE dropped into BOTH VAE inputs. + Without this the run samples a whole shot, decodes the video fine, then dies + deep inside comfy/sd.py with 'IndexError: tuple index out of range' when the + video memory estimator indexes shape[4] of the 4-D audio latent.""" + if _is_audio_vae(audio_vae) is False: + raise RuntimeError( + "audio_vae is a video/image VAE, not the H3 audio VAE. Load the audio " + "autoencoder (the DAC/BigVGAN one shipped with MiniMax-H3, e.g. " + "minimax_h3_audio_vae.safetensors) in its own VAELoader and wire that " + "into 'audio_vae'; the video VAE belongs on 'vae' only.") + check_audio_vae_loaded(audio_vae) + if _is_audio_vae(vae) is True: + raise RuntimeError( + "vae is the H3 audio VAE -- the video and audio VAE inputs are swapped. " + "Wire the video VAE into 'vae' and the audio VAE into 'audio_vae'.") + + +def _find_h3_sampling_node(): + """Locate the H3 sigma-shift node under ANY registered name. It was renamed + to 'ModelSamplingMiniMaxH3' in a later patch (kijai PR #15243); older 0.30.x + builds register it under a different id, so exact-key lookup misses it. Try + the known names, then fuzzy-scan all node mappings for the H3 model-sampling + node. Returns (class, key) or (None, None).""" + maps = getattr(nodes, "NODE_CLASS_MAPPINGS", {}) or {} + for key in ("ModelSamplingMiniMaxH3", "ModelSamplingMinimaxH3", "ModelSamplingMinimax", "ModelSamplingH3"): + if key in maps: + return maps[key], key + for k, v in maps.items(): + kl = k.lower() + if "sampl" in kl and (("minimax" in kl and "h3" in kl) or ("h3" in kl and "shift" in kl)): + return v, k + for k, v in maps.items(): + kl = k.lower() + if ("minimax" in kl or "h3" in kl) and ("shift" in kl or "sampling" in kl): + return v, k + return None, None + + +def _call_node(cls, model, shift_video, shift_audio): + """Call the H3 sampling node whether it uses the V1 (INPUT_TYPES/FUNCTION) or + V3 (define_schema/execute) API, mapping the shift args by name.""" + inst = cls() + # V1 API + if hasattr(cls, "INPUT_TYPES") and getattr(cls, "FUNCTION", None): + req = cls.INPUT_TYPES().get("required", {}) + kwargs = {} + for name in req: + low = name.lower() + if low == "model": + kwargs[name] = model + elif "video" in low: + kwargs[name] = float(shift_video) + elif "audio" in low: + kwargs[name] = float(shift_audio) + out = getattr(inst, cls.FUNCTION)(**kwargs) + # A V3 node exposes INPUT_TYPES and a truthy FUNCTION ('EXECUTE_NORMALIZED') + # for compatibility, so this branch runs on 0.31+ too -- and there it returns + # a NodeOutput, not a tuple. Without the unwrap the caller got the wrapper + # object where a MODEL belonged. Unreachable today because the direct patch + # succeeds first, which is exactly why it went unnoticed. + out = getattr(out, "result", out) + return out[0] if isinstance(out, (tuple, list)) else out + # V3 API: an execute()/patch() classmethod taking model + shift kwargs + fn = None + for cand in ("execute", "patch", "apply"): + if hasattr(inst, cand): + fn = getattr(inst, cand); break + if fn is None: + raise RuntimeError("unknown node API") + out = fn(model=model, shift_video=float(shift_video), shift_audio=float(shift_audio)) + out = getattr(out, "result", out) # V3 NodeOutput + return out[0] if isinstance(out, (tuple, list)) else out + + +def _direct_model_sampling(model, shift_video, shift_audio): + """Fallback that sets the shift on the model's own model_sampling object + without any node -- version-tolerant and V3-proof, since it uses model-level + APIs (get_model_object / set_parameters / add_object_patch) rather than + calling a node. Copies the sampling object so the base model isn't mutated, + and applies audio_shift only if the installed set_parameters accepts it.""" + import inspect, copy + m = model.clone() + # deepcopy, not copy: model_sampling is an nn.Module, and a SHALLOW copy shares + # its `_buffers` dict with the original. set_parameters() re-registers `sigmas` + # into that shared dict, so a shallow copy silently rewrites the BASE model's + # sigma table -- the very thing this copy exists to prevent. Our own run reads + # the patched object either way, but ComfyUI caches the model across queue + # runs, so the damage outlives this execution and reaches anything else holding + # that model. The buffer is ~1000 floats; the deepcopy is free. + ms = copy.deepcopy(m.get_model_object("model_sampling")) + sig = inspect.signature(ms.set_parameters) + kwargs = {} + if "shift" in sig.parameters: + kwargs["shift"] = float(shift_video) + if "audio_shift" in sig.parameters: + # NOTE: on ComfyUI 0.31 the audio latent is carried on the video schedule + # scaled by audio_scale = shift_video / shift_audio (12/3 = 4.0), applied in + # process_latent_in and undone in process_latent_out. Forcing that ratio to + # 1.0 (audio_shift == shift_video) as a "legacy 0.30" emulation produces + # SILENT output -- the model needs the scaling -- so it is not offered. + kwargs["audio_shift"] = float(shift_audio) + if not kwargs: + raise RuntimeError("set_parameters takes no shift") + ms.set_parameters(**kwargs) + m.add_object_patch("model_sampling", ms) + return m + + +def apply_h3_model_sampling(model, shift_video, shift_audio): + """Apply H3's dual video/audio flow schedule from INSIDE the node so a missing + upstream patch can't silently gibberish the audio. + + On ComfyUI 0.31+ the H3 nodes are V3-schema and don't live in the legacy + NODE_CLASS_MAPPINGS the old way -- AND the model already defaults to the correct + FLOW_AV schedule (12/3) at load. So the reliable path here is a DIRECT model- + level patch (works regardless of node API); the node call is only a secondary. + Order: direct model_sampling patch -> node under any name (V1/V3) -> give up with + an informative, non-alarming note. Shifts aren't hardcoded (12/3 base, ~8 video + for low-step MXFP8, ~4-6 audio for turbo).""" + try: + return _direct_model_sampling(model, shift_video, shift_audio), \ + f"model_sampling video {shift_video:g}/audio {shift_audio:g} (direct)" + except Exception: + pass + cls, key = _find_h3_sampling_node() + if cls is not None: + try: + return _call_node(cls, model, shift_video, shift_audio), \ + f"model_sampling video {shift_video:g}/audio {shift_audio:g} (via {key})" + except Exception: + pass + return model, (f"model_sampling not explicitly set (video {shift_video:g}/audio {shift_audio:g}); " + "on ComfyUI 0.30+ the model already defaults to the correct schedule, so this is " + "usually harmless -- only set shift_video/audio explicitly if you're on a low-step " + "MXFP8/turbo profile and the audio sounds wrong") + + +def _upscale_model_list(): + """Filenames in models/upscale_models, plus 'none'. Read fresh at INPUT_TYPES + time so newly-added models show up on a graph reload.""" + try: + import folder_paths + return ["none"] + list(folder_paths.get_filename_list("upscale_models")) + except Exception: + return ["none"] + + +def _invoke_node(cls, **kwargs): + """Call a registered ComfyUI node (V1 FUNCTION or V3 execute) with kwargs and + return its first output. Used to reuse ComfyUI's own upscale-model loader/apply + so we don't reimplement spandrel loading or tiled scaling.""" + inst = cls() + fn = None + if getattr(cls, "FUNCTION", None) and hasattr(inst, cls.FUNCTION): + fn = getattr(inst, cls.FUNCTION) + else: + for cand in ("execute", "upscale", "load_model", "load"): + if hasattr(inst, cand): + fn = getattr(inst, cand); break + if fn is None: + raise RuntimeError("no callable entrypoint") + out = fn(**kwargs) + out = getattr(out, "result", out) + return out[0] if isinstance(out, (tuple, list)) else out + + +def _find_node(substrings): + """Find a registered node whose key contains all of `substrings` (lowercased).""" + maps = getattr(nodes, "NODE_CLASS_MAPPINGS", {}) or {} + for k, v in maps.items(): + kl = k.lower() + if all(s in kl for s in substrings): + return v + return None + + +def _resize_short_edge(frames, target, method="lanczos"): + """Resize a [B,H,W,C] frame batch so its short edge == target (keeping aspect, + snapped to /32). Plain high-quality resize -- enlarges, doesn't add detail.""" + b, h, w, c = frames.shape + if min(h, w) == target: + return frames + if h <= w: + nh = target; nw = max(32, int(round(target * w / h / 32) * 32)) + else: + nw = target; nh = max(32, int(round(target * h / w / 32) * 32)) + s = frames.movedim(-1, 1) + s = comfy.utils.common_upscale(s, nw, nh, method, "disabled") + return s.movedim(1, -1) + + +def _upscale_frames(frames, mode, model_name, target_short_edge, batch=4): + """Optional post-pass upscale of the finished frames (on CPU). + mode 'model' : run a ComfyUI upscale model (Real-ESRGAN/UltraSharp class) + via the registered loader+apply nodes, chunked with cleanup + so 2000+ frames don't OOM; then fit to target short edge. + mode 'rtx' : NVIDIA RTX Video Super Resolution (Tensor Cores; fastest, + best quality for video -- needs Nvidia_RTX_Nodes_ComfyUI). + mode 'lanczos' : plain high-quality resize to the target short edge. + Any failure falls back to lanczos (or the raw frames), so it never breaks a + render. Returns (frames, note). NOTE: this SHARPENS/ENLARGES; it does not + reconstruct video detail the way a second-model (LTX 2.3) pass does.""" + if mode == "off" or frames is None or getattr(frames, "shape", [0])[0] == 0: + return frames, "" + note = "" + if mode == "rtx": + # NVIDIA RTX Video Super Resolution (Comfy-Org/Nvidia_RTX_Nodes_ComfyUI). + # Runs on RTX Tensor Cores -- far faster than ESRGAN-class models and + # generally cleaner on video, though like them it enhances/enlarges rather + # than reconstructing detail (an LTX 2.3 re-generation does that). + try: + rtx = (_find_node(["rtx", "video", "super"]) or _find_node(["rtxvideosuperresolution"]) + or _find_node(["rtx", "upscale"])) + if rtx is None: + raise RuntimeError("RTX node not installed (Nvidia_RTX_Nodes_ComfyUI)") + scale = 2 + if target_short_edge and int(target_short_edge) > 0: + cur = min(frames.shape[1], frames.shape[2]) + if cur > 0: + scale = max(1, min(4, int(round(int(target_short_edge) / cur)))) + out = [] + n = frames.shape[0] + step = max(1, int(batch)) + for st in range(0, n, step): + part = frames[st:st + step] + res = None + for kw in ({"image": part, "scale": scale}, {"images": part, "scale": scale}, + {"image": part, "scale_factor": scale}, {"image": part}): + try: + res = _invoke_node(rtx, **kw); break + except TypeError: + continue + if res is None: + raise RuntimeError("RTX node signature not recognized") + out.append(res.detach().to("cpu")) + del res, part + _deep_cleanup() + frames = torch.cat(out, dim=0) + note = f"RTX Video Super Resolution x{scale}" + if target_short_edge and int(target_short_edge) > 0: + frames = _resize_short_edge(frames, int(target_short_edge)) + note += f"; fit to {int(target_short_edge)}px short edge" + return frames, note + except Exception as e: + mode = "model" + note = f"RTX upscale unavailable ({e}); fell back to model/lanczos" + if mode == "model" and model_name and model_name != "none": + try: + loader = _find_node(["upscale", "model", "load"]) or _find_node(["loadupscalemodel"]) + applier = _find_node(["imageupscale", "model"]) or _find_node(["upscaleimageusingmodel"]) + if loader is None or applier is None: + raise RuntimeError("upscale-model nodes not found") + up_model = _invoke_node(loader, model_name=model_name) + out = [] + n = frames.shape[0] + for s in range(0, n, max(1, int(batch))): + part = frames[s:s + max(1, int(batch))] + res = _invoke_node(applier, upscale_model=up_model, image=part) + out.append(res.detach().to("cpu")) + del res, part + _deep_cleanup() + frames = torch.cat(out, dim=0) + note = f"upscaled with {model_name}" + except Exception as e: + mode = "lanczos" + note = f"model upscale unavailable ({e}); used lanczos" + if target_short_edge and int(target_short_edge) > 0: + try: + frames = _resize_short_edge(frames, int(target_short_edge)) + note = (note + "; " if note else "") + f"fit to {int(target_short_edge)}px short edge" + except Exception as e: + note = (note + "; " if note else "") + f"resize failed ({e})" + elif mode == "lanczos" and not note: + note = "lanczos selected but no target set -> unchanged" + return frames, note + + +def lora_active(model): + """True if a LoRA is applied to this model, by either mechanism. + + Stock LoraLoaderModelOnly folds deltas in as ModelPatcher weight *patches*; + bypass LoRAs (turbo packs) register *injections* / wrappers instead. Detecting + both matters because a distilled LoRA compresses ~20 steps into 4-8, so the + model commits to global composition -- including HOW MANY PEOPLE are in frame -- + within the first step or two and then reinforces that choice rather than + revising it. That is why turbo LoRAs duplicate subjects even when the prompt is + clean, and why the subject-count guard has to be forced on for them regardless + of resolution.""" + try: + if getattr(model, "patches", None): + return True + for attr in ("injections", "wrappers"): + d = getattr(model, attr, None) or {} + if any(len(v) for v in d.values()): + return True + except Exception: + pass + return False + + +def lora_overhead_gb(model): + """Extra VRAM a bypass-LoRA holds resident during sampling. + + A bypass LoRA (e.g. the MiniMax-H3 Turbo LoRA) does NOT fold into the weights: + it keeps every low-rank A/B pair live in bf16 and adds lora(x) in activation + space each forward. With ~208 adapters plus per-adapter activations that is a + real, measurable chunk the budget must not spend on frames -- otherwise the + node picks a shot length that fits the base model and then overflows once the + adapters and their activations land. Returns an estimate in GB (0 if none).""" + try: + injections = getattr(model, "injections", None) or {} + n_inj = sum(len(v) for v in injections.values()) + n_hooks = 0 + for v in injections.values(): + for inj in v: + n_hooks += len(getattr(inj, "hooks", ()) or ()) + wrappers = getattr(model, "wrappers", None) or {} + n_wrap = sum(len(w) for w in wrappers.values()) + if not (n_inj or n_hooks or n_wrap): + return 0.0 + # low-rank deltas are small individually; the cost that matters is the + # per-adapter activation working set during the forward pass. + return round(max(0.6, 0.004 * max(n_hooks, 1)), 2) + except Exception: + return 0.0 + + +# --- SLA (sparse-linear attention) pairing ---------------------------------- +# An SLA LoRA is a turbo LoRA fine-tuned WITH sparse attention in the loop, so the +# weights have already adapted to the approximation. That is the whole benefit: you +# get the sparse-attention speedup without the quality collapse. The two halves are +# a matched pair and only work together -- +# +# sparse attention ON + ordinary LoRA -> the model sees an attention map it was +# never trained on. Long-range coherence is what sparsity drops first, and in +# a video DiT that shows up as the SAME PERSON RENDERED TWICE. +# SLA LoRA + sparse attention OFF -> the weights are pre-compensating for +# sparsity that isn't there. You pay its quality cost and get no speedup. +# +# Neither half is inferable from the model object. Verified against the actual +# file: minimax_h3_fl2v_turbo_4step_v0.1_768p_sla_comfyui_bf16.safetensors carries +# NO SLA marker -- not in its 624 tensor names, not in its 9 metadata keys. It is +# byte-shape-identical to any un-resized rank-128 H3 turbo LoRA. The only place the +# word appears is the FILENAME, so that is what has to be read. +# +# Matches 'sla' as a DELIMITED token, so 'slack', 'translate', 'isla' and 'SLAYER' +# do not fire. A LoRA that ends in '_sla' for some unrelated reason would warn +# spuriously -- the cost is one wrong line in `info`, never a changed render. +_SLA_NAME = re.compile(r"(?:^|[^a-z])sla(?:[^a-z]|$)", re.I) + + +def sparse_attention_active(model): + """True when an attention-override patch (Sol-Attn and friends) is on `model`. + + ComfyUI carries these in model_options['transformer_options'], which our node + receives already patched because the patch node ran upstream of us.""" + try: + opts = getattr(model, "model_options", None) or {} + tro = opts.get("transformer_options", {}) or {} + return tro.get("optimized_attention_override") is not None + except Exception: + return False + + +def upstream_lora_names(graph, node_id, _seen=None): + """LoRA filenames on the MODEL chain feeding this node, nearest last. + + Walks the workflow graph backwards from our own `model` input. Reading the + filename is the only way to identify an SLA LoRA (see above), and the graph is + the only place the filename survives -- ComfyUI stashes a LoRA's safetensors + metadata on the patcher but never its name. + + Deliberately follows ONLY model-carrying inputs, so a LoRA wired into some + other branch of the workflow is not mistaken for one that is affecting us.""" + if graph is None or node_id is None: + return [] + _seen = set() if _seen is None else _seen + nid = str(node_id) + if nid in _seen: + return [] + _seen.add(nid) + try: + node = graph.get_node(nid) + except Exception: + return [] + names, inputs = [], (node.get("inputs") or {}) + for key, val in inputs.items(): + # a link is [upstream_node_id, output_slot]; anything else is a widget value + if isinstance(val, (list, tuple)) and len(val) == 2 and not isinstance(val[1], (list, dict)): + if "model" in str(key).lower(): + names += upstream_lora_names(graph, val[0], _seen) + else: + names += lora_names_in_widget(key, val) + return names + + +# Filenames a stacked loader uses for an EMPTY slot. +_LORA_EMPTY = {"", "none", "null", "no lora", "-", "undefined"} + + +def lora_names_in_widget(key, val, _depth=0): + """LoRA filenames inside one widget value, however the loader packs them. + + A one-LoRA-per-widget loader puts a bare filename under a key containing + "lora", and that was all this understood. Stacked loaders do not: DaSiWa packs + every LoRA into ONE json string under `stack_data`, and rgthree's Power Lora + Loader stores a dict per slot. Neither has "lora" in the key it is filed under, + so a chain carrying four LoRAs read as carrying none. + + Disabled slots and zero-strength entries are skipped: a LoRA that is switched + off is not affecting this render and should not be reported as if it were.""" + if _depth > 6: # cyclic or absurdly nested + return [] + if isinstance(val, str): + s = val.strip() + if s[:1] in "[{": # a packed json payload + try: + return lora_names_in_widget(key, json.loads(s), _depth + 1) + except (ValueError, TypeError): + return [] + if s.lower() in _LORA_EMPTY: + return [] + return [s] if "lora" in str(key).lower() else [] + if isinstance(val, dict): + for flag in ("on", "enabled", "enable", "active"): + if flag in val and not val[flag]: + return [] # slot switched off + for f in ("str", "strength", "strength_model", "model_strength", "weight"): + if f in val: + try: + if abs(float(val[f])) < 1e-6: + return [] # at 0 it changes nothing + except (TypeError, ValueError): + pass + break + out = [] + for k, v in val.items(): + out += lora_names_in_widget(k, v, _depth + 1) + return out + if isinstance(val, (list, tuple)): + out = [] + for v in val: + out += lora_names_in_widget(key, v, _depth + 1) + return out + return [] + + +def sla_pairing(model, graph, node_id): + """(sla_lora_name|None, sparse_on, note) -- how the two halves line up. + + `note` is the warning when they are mismatched, and it is worth a warning + rather than a silent default because both mismatches cost a full render.""" + sparse = sparse_attention_active(model) + loras = upstream_lora_names(graph, node_id) + sla = next((n for n in reversed(loras) if _SLA_NAME.search(os.path.basename(str(n)))), None) + note = "" + if sla and not sparse: + note = (f"SLA LoRA '{os.path.basename(str(sla))}' is loaded but NO sparse-attention patch " + f"is active -- it was fine-tuned WITH sparse attention, so on dense attention you " + f"pay its quality cost and get none of its speed. Add the Sol-Attn patch node " + f"between the loader and this node, or load the non-SLA turbo LoRA instead") + elif sparse and loras and not sla: + note = (f"sparse attention is ON but the LoRA on this chain " + f"({os.path.basename(str(loras[-1]))}) is not an SLA build -- the model was never " + f"trained against the approximated attention map, and the first thing sparsity " + f"drops is long-range coherence, which renders as the SAME PERSON TWICE. Load the " + f"'_sla_' turbo LoRA, or turn the sparse-attention patch off") + elif sparse and not loras: + note = ("sparse attention is ON with no LoRA on this chain -- base H3 was not trained " + "against an approximated attention map, so expect duplicated subjects. Pair it " + "with an SLA turbo LoRA") + return sla, sparse, note + + +# --- what a LoRA actually declares about itself ------------------------------ +# Verified against all six LoRAs in this install before any of it was written, +# because most of what a "LoRA scanner" sounds like it should do is not in the +# files. What IS there: +# +# base model metadata: 'base_model' (Comfy-Org turbo builds) or +# 'ss_base_model_version' (ai-toolkit builds, e.g. 'minimax_h3') +# strength metadata: 'training_scale' / 'baked_scale', both 1.0 +# step count FILENAME only ('..._turbo_4step_...') -- no metadata field +# training res FILENAME only ('..._768p_...'), plus 'ss_resolution' on kohya +# builds, which none of these are +# +# What is NOT there, checked file by file: +# +# trigger words no 'ss_tag_frequency' in any of them. The ai-toolkit LoRAs carry +# nine metadata keys and none is a caption or tag list, so a +# trigger like 'mpenis' cannot be discovered -- it stays a manual +# exposed_terms entry. +# sampler/cfg/ no field for these exists in any LoRA metadata standard, and +# scheduler none of these files has one. Reporting them would be invention. +# +# Everything below therefore either quotes metadata or says the filename said it. +# Nothing here overrides a widget: a render must stay reproducible from what the +# graph shows. +_LORA_STEPS = re.compile(r"(?:^|[^a-z0-9])(\d{1,2})[ _-]?step", re.I) + +_LORA_RES = re.compile(r"(?:^|[^a-z0-9])(\d{3,4})p(?:[^a-z0-9]|$)", re.I) + + +def lora_declared(model): + """The nearest LoRA's own metadata dict, as ComfyUI stashed it, or {}. + + comfy.sd.load_lora_for_models() calls set_attachments('lora_metadata', ...) on + the patcher, so this is the LoRA's real safetensors header -- not a guess. With + several stacked, the attachment holds the last one applied.""" + try: + get = getattr(model, "get_attachment", None) + return (get("lora_metadata") if get else None) or {} + except Exception: + return {} + + +def lora_hint_notes(model, graph, node_id, steps, short_edge): + """Warnings where a LoRA's declared training disagrees with this run. + + Each note says WHERE the number came from, because the two sources are not + equally trustworthy: metadata is what the trainer wrote, a filename is a naming + convention that anyone can break by renaming the file.""" + notes = [] + names = upstream_lora_names(graph, node_id) + if not names: + return notes + md = lora_declared(model) + + # --- base model: the one check that is pure metadata --- + # ComfyUI's set_attachments('lora_metadata', ...) is overwritten by each loader + # in turn, so on a stack this describes the LAST one applied only. Attributed to + # that file by name rather than to "your LoRA", so the note cannot be read as a + # claim about the others. + base = str(md.get("base_model") or md.get("ss_base_model_version") or "") + nearest = os.path.basename(str(names[-1])) + if base and "minimax" not in base.lower() and "h3" not in base.lower(): + notes.append(f"'{nearest}' declares base_model '{base}', which is not MiniMax-H3 -- " + f"it will apply as noise on an H3 DiT") + + # Filename-derived checks run over EVERY LoRA on the chain. Only the nearest + # one's metadata is reachable, but every one of their names is -- and on a stack + # the step-count LoRA is usually NOT the nearest, so checking one file silently + # skipped the very LoRA whose step count the sampler has to match. + for raw in names: + name = os.path.basename(str(raw)) + + # --- step count: filename convention only --- + m = _LORA_STEPS.search(name) + if m: + want = int(m.group(1)) + if want and int(steps) != want: + notes.append(f"'{name}' is named as a {want}-step LoRA but steps={int(steps)} " + f"(from the filename, not metadata)" + + ("; a distill LoRA run past its step count re-noises a composition it " + "already settled" if int(steps) > want else + "; under its step count the distill has not finished resolving")) + + # --- training resolution: filename convention only --- + m = _LORA_RES.search(name) + if m and short_edge: + want = int(m.group(1)) + if short_edge > want * 1.34: + notes.append(f"'{name}' is named for {want}p but the short edge here is " + f"{int(short_edge)}px (from the filename, not metadata); well above a " + f"LoRA's training resolution H3 tends to tile the figure") + elif short_edge * 1.34 < want: + notes.append(f"'{name}' is named for {want}p but the short edge here is " + f"{int(short_edge)}px (from the filename, not metadata); well below " + f"it the LoRA's detail work has nothing to land on") + + # --- stacking a distill/turbo LoRA with others --- + # The genuine conflict, and the one that reads as "a LoRA fight". A distill LoRA + # rewrites the sampling trajectory: it settles global composition in the first + # step or two of a 4-8 step schedule. A subject LoRA trained against BASE H3 at + # a normal step count contributes deltas calibrated for a schedule that no + # longer exists, and at full strength those land in exactly the steps the + # distill is using to fix composition. Order does not matter -- ComfyUI sums the + # patches -- so strength is the only lever. + distill = [os.path.basename(str(n)) for n in names + if _LORA_STEPS.search(os.path.basename(str(n))) + or re.search(r"turbo|distill", os.path.basename(str(n)), re.I)] + if distill and len(names) > 1: + others = [os.path.basename(str(n)) for n in names + if os.path.basename(str(n)) not in distill] + if others: + notes.append( + f"{len(names)} LoRAs on this chain, one of them a distill/turbo build " + f"('{distill[0]}'). It settles composition in its first step or two, and the " + f"deltas from {', '.join(f'{o!r}' for o in others[:3])} land in those same " + f"steps -- lower the SUBJECT LoRA strengths first (0.6-0.8), not the distill's. " + f"Stacking order does not matter; ComfyUI sums the patches") + return notes + + +# Fingerprint of the model used by the previous run, so a checkpoint swap can be +# detected between queue executions. Module-level: it must outlive the node +# instance, which ComfyUI recreates per execution. +_LAST_MODEL_FP = {"fp": None} + + +def model_fingerprint(model): + """A cheap, stable identity for the loaded DiT: (quant format, layer count, + weight bytes, class name). Changes whenever the checkpoint changes -- a + different quant, a pruned-vs-full build, or a different model entirely -- while + staying identical across shots of the same run. Deliberately avoids hashing + weights, which would cost more than the flush it guards.""" + try: + dm = getattr(getattr(model, "model", None), "diffusion_model", None) + fmts, n = {}, 0 + if dm is not None and hasattr(dm, "modules"): + for mod in dm.modules(): + n += 1 + f = getattr(mod, "quant_format", None) + if f: + fmts[f] = fmts.get(f, 0) + 1 + top = max(fmts.items(), key=lambda kv: kv[1])[0] if fmts else "none" + size = 0 + try: + size = int(model.model_size()) + except Exception: + pass + cls = type(dm).__name__ if dm is not None else "unknown" + return (top, n, size, cls) + except Exception: + return None + + +def flush_for_model_change(model): + """Detect a checkpoint swap since the last run and, if one happened, hard-flush + GPU state before doing anything else. + + Why this matters: ComfyUI keeps previously-loaded models in current_loaded_models + and only evicts reactively. Swapping checkpoints mid-session (e.g. NVFP4 -> FP8 -> + MXFP8 while comparing quality) leaves the OLD DiT resident alongside the new one, + plus any hooks/injections a previous LoRA installed and stale cached allocator + blocks sized for the old model's layers. The result is a card that is already + half full before the first shot samples -- which looks exactly like the node + over-spilling, when in fact the budget was computed against memory the previous + checkpoint never released. + + Returns a note for `info` when a change was detected (empty string otherwise).""" + fp = model_fingerprint(model) + prev = _LAST_MODEL_FP.get("fp") + _LAST_MODEL_FP["fp"] = fp + if prev is None or fp is None or prev == fp: + return "" + try: + mm.unload_all_models() # drop every resident model, not just the cache + except Exception: + pass + # Never let a cleanup failure abort the run: the flush is best-effort hygiene, + # and a partially-flushed card is still better than raising here. + for _ in range(2): # 2nd pass frees blocks released by the 1st + try: + _deep_cleanup() + except Exception: + pass + old_fmt, _n, old_sz, _c = prev + new_fmt = fp[0] + return (f"model changed since last run ({old_fmt} ~{old_sz / GB:.1f}GB -> {new_fmt} " + f"~{fp[2] / GB:.1f}GB): flushed all resident models and VRAM caches") + + +def quant_accel_note(model): + """Report the loaded DiT's quant format and whether THIS card can run it on + tensor cores natively -- so a silent fall back to emulated/upcast math shows up + in `info` instead of just looking like slow output. + + The node itself never sets dtypes, never autocasts and never rebuilds modules: + it delegates sampling to ComfyUI's common_ksampler, and its only model patch is + a schedule-object patch (add_object_patch on 'model_sampling'). So NVFP4/MXFP8 + tensor-core acceleration is entirely ComfyUI's dispatch on the quantized layers + -- which is what we want: nothing here can disturb it. This is a read-only + check.""" + try: + import comfy.model_management as _mm + dm = getattr(getattr(model, "model", None), "diffusion_model", None) + fmts = {} + if dm is not None and hasattr(dm, "modules"): + for mod in dm.modules(): + f = getattr(mod, "quant_format", None) + if f: + fmts[f] = fmts.get(f, 0) + 1 + if not fmts: + return "" + top = max(fmts.items(), key=lambda kv: kv[1])[0] + native = None + if "nvfp4" in top: + native = getattr(_mm, "supports_nvfp4_compute", lambda: None)() + elif "mxfp8" in top: + native = getattr(_mm, "supports_mxfp8_compute", lambda: None)() + if native is True: + return f"{top}: native tensor-core compute" + if native is False: + return (f"WARNING {top}: this card/torch cannot run it natively -- weights are being " + f"upcast, so you pay full-precision compute with none of the speedup") + return f"{top} weights" + except Exception: + return "" + + + + +def _evict_all_but(keep_model): + """Unload every model EXCEPT the diffusion model from the GPU. + + This is the fix for VRAM ratcheting across a long chain. soft_empty_cache() + only drops the CUDA allocator's cached blocks -- it does NOT unload models, so + ComfyUI keeps the Qwen3-VL text encoder (~14.6GB) and both VAEs resident in + current_loaded_models alongside the DiT. Each shot re-encodes the prompt + (text encoder), encodes the handoff keyframe (video VAE), then samples (DiT), + so all three compete for the card; ComfyUI only evicts reactively, i.e. AFTER + it has already spilled. With a bypass LoRA also holding 208 bf16 adapters + resident there is no room left, and every shot leaves the card fuller. + + Freeing them explicitly, right after conditioning is built and before + sampling, keeps only what the sampler actually needs on the GPU.""" + try: + keep = [] + for lm in list(getattr(mm, "current_loaded_models", [])): + try: + if lm.model is keep_model or getattr(lm, "model", None) is getattr(keep_model, "model", None): + keep.append(lm) + except Exception: + pass + mm.free_memory(1e30, mm.get_torch_device(), keep_loaded=keep) + except Exception: + try: + mm.soft_empty_cache(True) + except Exception: + pass + + + + + + +class H3LongVideos: + CATEGORY = "Dumas/MiniMax" + FUNCTION = "run" + # fps is emitted as BOTH types on purpose: ComfyUI does not coerce between them, + # and the nodes that want a frame rate are split -- CreateVideo / SaveWEBM / + # VHS Video Combine take a FLOAT, while plenty of utility nodes take an INT. + # Wiring the wrong one is a red link, not a runtime error, so both are offered. + # LATENT is APPENDED, never inserted: ComfyUI stores a link by output SLOT + # INDEX, so adding at the end leaves every existing wire pointing at the same + # output. Inserting mid-list would silently re-target them. + # APPEND to these, never insert. A workflow stores an output link by SLOT INDEX, + # so a new type in the middle silently re-points every link after it. + RETURN_TYPES = ("IMAGE", "AUDIO", "STRING", "STRING", "INT", "INT", "INT", "FLOAT", "FLOAT", "INT", + "LATENT", "STRING") + RETURN_NAMES = ("images", "audio", "info", "script", "frames_per_shot", "total_frames", + "shots", "video_seconds", "fps", "fps_int", + "latent", "soundscape") + + @classmethod + def IS_CHANGED(cls, plan_only=False, **kwargs): + """Force a re-run for the PLAN, leave a real render cacheable. + + Without an IS_CHANGED, ComfyUI keys this node's cache on its inputs alone, so + re-queueing with the same widgets returns the previous outputs untouched -- and + `info` is an output. That reads as "info doesn't update on each run", and it is + actively misleading here, because both the info AND the chosen shot length now + depend on LIVE FREE VRAM, which is not an input: the cached answer describes a + card state that may no longer exist. + + plan_only is near-instant, so it always recomputes -- a stale plan is worse than + no plan. A real render still respects the cache (returning NaN there would + re-sample for minutes every time the graph is queued); change the seed, or any + widget, to force one.""" + if plan_only: + return float("nan") # NaN != NaN -> never matches the cached signature + return False + + @classmethod + def INPUT_TYPES(cls): + schema = { + "required": { + "model": ("MODEL",), "clip": ("CLIP",), "vae": ("VAE",), + "audio_vae": ("VAE",), + "prompt": ("STRING", {"multiline": True, "forceInput": True, "default": + "A woman with short silver hair and a scar over her left eyebrow. Warm " + "late-afternoon light, cinematic, 2K.\n" + "wardrobe: weathered red flight jacket, grey cargo shorts, black boots\n\n" + "walks across the tarmac toward a small propeller plane.\n\n" + "climbs in and flips the switches; the propeller spins.\n\n" + "taxis down the grass runway, the tail lifting.\n\n" + "the plane leaves the ground; wide shot banking against the sky.", + "tooltip": "This IS the integrated_multimodal_description (the visual/action " + "timeline). First paragraph = PERMANENT IDENTITY kept across the whole " + "video (hair, face, build) -- put NO clothing in this prose, or it can't " + "be changed later. Put clothing on a 'wardrobe:' line (in the first " + "paragraph and/or the character_memory field); it's the only channel that " + "can be changed/removed mid-chain. Each later paragraph = one scene beat. " + "Put dialogue and 'lips closed' beats in the beat bodies."}), + "resolution": (resolution_options(), { + "tooltip": "ASPECT RATIO only -- `megapixels` decides the size. The two are " + "independent, so changing shape does not change cost. Each ratio uses " + "H3's own dimensions as its reference, which is why 1.00MP reproduces " + "the model's native sizes exactly (16:9 -> 1344x768, 21:9 -> 1536x672)."}), + # No off-switch any more: `resolution` is a bare ratio, so there are no + # preset dimensions to fall back to. The floor keeps every result legal. + "megapixels": ("FLOAT", {"default": 1.0, "min": 0.10, "max": 4.0, "step": 0.01, + "tooltip": "Pixel BUDGET, applied to the preset's aspect ratio. 1.00 = 1024x1024 " + "worth of pixels, the same convention as ComfyUI's Scale Image to Total " + "Pixels. START at 1.00: every NATIVE preset reproduces its own size there " + "(1344x768, 1536x672, ...), then step down -- 0.83 gives a 704 short edge, " + "0.65 gives 640 -- for speed, VRAM and longer shots. Cost and training fit " + "track TOTAL PIXELS, not the short edge: 1:1 768x768 reads as native by " + "short edge but is only 0.56MP, while 21:9 1536x672 reads as sub-native at " + "a full 0.98MP. Snapped to multiples of 32; `info` reports the size and MP " + "actually produced. Set 0 to use the preset's own dimensions instead."}), + # Base H3 (NVFP4/FP8, no distill LoRA) needs ~20 steps with res_multistep+simple. + # 6-8 steps only makes sense WITH a working 4-step distill/turbo LoRA or an MXFP8 + # checkpoint tuned for low steps -- at 6-8 on the bare base model the frame comes + # out soft/under-formed (faces worst). Default is the safe base value. + "steps": ("INT", {"default": 20, "min": 1, "max": 200, + "tooltip": "Base H3 wants ~20 (res_multistep + simple). Drop to 6-8 ONLY with a " + "working distill/turbo LoRA or a low-step MXFP8 checkpoint -- on the " + "bare base model, low steps are the #1 cause of soft output."}), + "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 30.0, "step": 0.1}), + "sampler_name": (comfy.samplers.KSampler.SAMPLERS, {"default": "res_multistep"}), + "scheduler": (comfy.samplers.KSampler.SCHEDULERS, {"default": "simple"}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "control_after_generate": True}), + }, + "optional": { + "first_frame": ("IMAGE",), + # ref2va inputs. Order matters and is the ONLY thing that decides the + # roster: the tokenizer labels these .. in the + # order they appear here, then appends the prompt. Refer to them by + # those tags in the prompt if you want a reference bound to a named + # character ("Kristy, , walks in"). + "ref_image_1": ("IMAGE", {"tooltip": "Reference image -- identity/appearance " + "carried into the shots. Which shots receive it is set by ref_mode (or " + "tags in the beats); a referenced shot ALSO carries the previous frame as its " + "keyframe, so taking a reference never costs continuity."}), + "ref_image_2": ("IMAGE", {"tooltip": "Reference image ."}), + "ref_image_3": ("IMAGE", {"tooltip": "Reference image ."}), + "ref_image_4": ("IMAGE", {"tooltip": "Reference image ."}), + "plan_only": ("BOOLEAN", {"default": False, + "tooltip": "Preview the shot split WITHOUT rendering. Uses THIS node's own settings (no " + "second node, no duplicate entry): returns the plan in 'info' and the " + "shots/frames/seconds outputs near-instantly. Turn off to render for real."}), + "fps": ("INT", {"default": 24, "min": 1, "max": 60, + "tooltip": "DISPLAY ONLY -- H3 always renders 24 fps. The model's frame grid and its " + "audio latent are both defined against 24, so this node computes every " + "duration at 24 regardless of what you set here. Set your video-save node " + "to 24 as well, or the clip plays at the wrong speed."}), + "global_soundscape": ("STRING", {"multiline": True, "forceInput": True, "default": "", + "tooltip": "AMBIENT/environmental sound only (rain, room tone, footsteps, engines). " + "Appended to every shot as overall_soundscape. NOT for dialogue -- speech " + "and lip timing live in the prompt beats. Leave blank for no ambient bed."}), + "non_diegetic_music": ("STRING", {"multiline": True, "forceInput": True, "default": "", + "tooltip": "Background SCORE only -- genre, mood, instrumentation, tempo -- music that " + "is NOT part of the scene. Music is OPT-IN: leave this BLANK and the node " + "emits 'non_diegetic_music: N/A' on every shot so H3 adds no score (fixes " + "unwanted music). Fill it in to request a specific score. Not for music a " + "character plays/hears (that's diegetic; put it in the beat)."}), + "apply_model_sampling": ("BOOLEAN", {"default": True, + "tooltip": "Patch ModelSamplingMiniMaxH3 (the dual video/audio schedule) inside the " + "node so you don't have to wire it upstream. Without it, H3's audio comes " + "out as gibberish. Turn OFF only if you patch it yourself upstream."}), + "shift_video": ("FLOAT", {"default": 12.0, "min": 1.0, "max": 32.0, "step": 0.01, + "round": 0.01, + "tooltip": "Video flow shift. 12 = base H3 (correct default). A low-step MXFP8 " + "checkpoint wants ~8. Only used when apply_model_sampling is on."}), + "shift_audio": ("FLOAT", {"default": 3.0, "min": 0.25, "max": 16.0, "step": 0.01, + "round": 0.01, + "tooltip": "Audio flow shift. 3 = base H3. COUPLED to shift_video on ComfyUI " + "0.31+: the audio latent rides the video schedule scaled by " + "audio_scale = shift_video / shift_audio (12/3 = 4). Flattening that " + "ratio toward 1.0 breaks the audio branch -- babble or silence -- so " + "if you lower shift_video, lower this by the same factor. Only used " + "when apply_model_sampling is on."}), + "trim_seam": ("BOOLEAN", {"default": True, + "tooltip": "Drop the FIRST frame of every shot after the first.\n\n" + "That frame is the model's own reproduction of the handoff -- the " + "last frame of the previous shot, which it was anchored to. Keeping " + "it shows the same moment twice and reads as a stutter.\n\n" + "So at a working seam the last frame of one shot and the first of " + "the next are NOT identical: they are one frame of normal motion " + "apart, which is what continuous footage looks like. Turn this off " + "only to inspect how closely the anchor was reproduced."}), + "vary_seed_per_shot": ("BOOLEAN", {"default": False, + "tooltip": "Give each shot its own seed (seed+1, seed+2, ...) instead of one " + "seed for the whole chain.\n\n" + "OFF by default, because this node builds a CONTINUOUS TAKE. The " + "seed sets the noise field every shot is sampled from, and that " + "field is what fixes the stochastic detail -- grain, micro-texture, " + "the exact rendering of surfaces the prompt never names. Change it " + "between shots and all of that resets at the boundary, which reads " + "as a cut even when the keyframe anchors the frame and the location " + "is unchanged: the same room, rendered afresh.\n\n" + "Shots still differ with one seed -- each has its own beat text and " + "its own handoff keyframe. Turn this ON only when you WANT the " + "beats to look separately shot, or when repeated beats are coming " + "out too alike."}), + "handoff_offset": ("INT", {"default": 0, "min": 0, "max": 12, "step": 1, + "tooltip": "End each shot this many frames early and hand THAT frame to the next " + "shot instead of the literal last frame. Set 2-4 if chained shots open " + "with moving/talking mouths -- it avoids seeding the next shot with a " + "mid-word open-mouth pose. Trims the matching audio tail too. 0 = last frame."}), + "shot_seconds": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 15.1, "step": 0.5, + "forceInput": True, + "tooltip": "Length of EACH shot in seconds, taken from a connected input -- " + "H3 Shot Length is the intended source, since it also reports the " + "matching frame count on the 17k+5 grid.\n\n" + "Leave it UNCONNECTED for auto: the largest shot that fits at the " + "chosen size, which is what a 0 in the old widget did. One " + "paragraph = one shot, so total video = (paragraph count) x this. " + "Max ~15s."}), + "allow_oversize_shots": ("BOOLEAN", {"default": False, + "tooltip": "OFF (default): a forced shot_seconds that won't fit VRAM is clamped DOWN to " + "what fits, and the clamp is reported in info. ON: honor the requested length " + "even if it exceeds the budget -- the render may spill into system RAM (slow) " + "or OOM. Only affects forced shot_seconds, not auto."}), + "vram_headroom_gb": ("FLOAT", {"default": 1.5, "min": 0.0, "max": 32.0, "step": 0.5}), + "allow_res_backoff": ("BOOLEAN", {"default": True, + "tooltip": "If VRAM is tight, step resolution down instead of failing."}), + # ON by default: the prompt-side clauses ASK H3 not to vocalize (and now + # condition the soundscape field too), but asking is not a guarantee -- + # babble under a silent shot was the one artifact that survived both. + # Muting is the only deterministic answer, so it is the default and the + # trade-off (that shot's ambience goes too) is stated in `info`. + "mute_nonspeech_audio": ("BOOLEAN", {"default": True, + "tooltip": "DETERMINISTIC gibberish fix: FULLY silence the audio of any shot that has no " + "scripted dialogue (no double-quoted line). Prompt-level silencing asks H3 " + "not to babble; this guarantees it. TRADE-OFF: it also removes that shot's " + "generated ambience/SFX, so lay a continuous ambient bed under the video in " + "post. Shots WITH quoted dialogue keep their audio untouched."}), + "mute_fade_ms": ("INT", {"default": 40, "min": 0, "max": 500, "step": 10, + "tooltip": "Fade applied to the AUDIBLE shots that border a silenced one, so audio " + "doesn't cut to digital silence with a click. The silenced shots keep NO " + "original audio at all -- fading the muted shot itself would leave this many " + "ms of the gibberish audible at each end of every muted shot."}), + "decode_tile_frames": ("INT", {"default": 0, "min": 0, "max": 128, "step": 1, + "tooltip": "Temporal tiling for the VAE decode (tile_t). 0 = ComfyUI default, which " + "expands the WHOLE clip at once -- the single largest allocation in a run, " + "and the usual point where a big checkpoint tips into shared memory. Try 8-16 " + "if you spill during decode rather than sampling. Lower = less peak VRAM, " + "slightly slower."}), + "decode_tile_size": ("INT", {"default": 0, "min": 0, "max": 1024, "step": 32, + "tooltip": "Spatial tile size for the VAE decode (tile_x/tile_y). 0 = ComfyUI default. " + "Try 256 on a tight card at 1344x768."}), + "cleanup_between_shots": ("BOOLEAN", {"default": True, + "tooltip": "Between beats, move each shot's decoded video+audio to system RAM and run " + "a full VRAM+RAM purge (GC + CUDA cache), so a long chain doesn't accumulate " + "on the GPU and OOM. Recommended on 16GB. Turn off only on a big card where " + "you want to skip the per-shot cleanup cost."}), + "upscale": (["off", "rtx", "model", "lanczos"], {"default": "off", + "tooltip": "Optional post-pass on the finished frames. 'rtx' = NVIDIA RTX Video Super " + "Resolution (Tensor Cores -- fastest and best for video; needs the " + "Nvidia_RTX_Nodes_ComfyUI pack, falls back automatically if absent). 'model' = " + "a Real-ESRGAN/UltraSharp upscale model from upscale_model. 'lanczos' = plain " + "resize. All of these ENHANCE/ENLARGE; for true detail reconstruction from a " + "low-res render, use a separate LTX 2.3 upscale pass."}), + "upscale_model": (_upscale_model_list(), { + "tooltip": "Upscale model from models/upscale_models (used when upscale = model)."}), + "upscale_target_short_edge": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 32, + "tooltip": "Fit the result's short edge to this many px (0 = keep the model's native " + "factor / no resize). E.g. generate 512 fast, set 768 to land at native size."}), + "upscale_batch": ("INT", {"default": 4, "min": 1, "max": 64, + "tooltip": "Frames per chunk for the model upscale (lower = less VRAM, slower)."}), + "watermark_text": ("STRING", {"default": "", + "tooltip": "Composited with PIL onto every finished frame -- NOT rendered by the " + "model and NOT added to the prompt. White glyphs on a transparent layer, " + "alpha-blended over the video, so only the letters land on the picture. " + "Applied AFTER any upscale, so the text is crisp at final resolution. " + "Leave empty for none."}), + "watermark_position": (["bottom-right", "bottom-left", "bottom-center", + "top-right", "top-left", "top-center", "center"], + {"default": "bottom-right"}), + "watermark_size": ("FLOAT", {"default": 4.0, "min": 0.5, "max": 40.0, "step": 0.5, + "tooltip": "Cap height as a percentage of the frame's SHORT edge, so the mark keeps " + "its apparent size across portrait and landscape presets alike."}), + "watermark_opacity": ("FLOAT", {"default": 0.75, "min": 0.0, "max": 1.0, "step": 0.05, + "tooltip": "Multiplies the white text alpha. 1.0 = solid white; 0.75 reads as a " + "watermark without burying the picture under it."}), + "watermark_margin": ("FLOAT", {"default": 3.0, "min": 0.0, "max": 25.0, "step": 0.5, + "tooltip": "Inset from the frame edge, as a percentage of the SHORT edge."}), + "intro_text": ("STRING", {"multiline": True, "forceInput": True, "default": "", + "tooltip": "Title composited over the OPENING frames -- white on transparent, so the " + "first shot plays underneath it rather than being replaced by a card. " + "Multi-line is centered as a block. Holds for intro_seconds, then fades " + "out over intro_fade. Also PIL, never the model."}), + "intro_position": (["center", "lower-third", "top-center", "bottom-center"], + {"default": "center"}), + "intro_seconds": ("FLOAT", {"default": 3.0, "min": 0.0, "max": 30.0, "step": 0.5, + "tooltip": "How long the title stays at full opacity before the fade starts."}), + "intro_fade": ("FLOAT", {"default": 0.6, "min": 0.0, "max": 10.0, "step": 0.1, + "tooltip": "Linear fade-out length after the hold. 0 = hard cut."}), + "intro_size": ("FLOAT", {"default": 9.0, "min": 0.5, "max": 40.0, "step": 0.5, + "tooltip": "Title cap height as a percentage of the frame's SHORT edge."}), + "overlay_font": ("STRING", {"default": "arial.ttf", + "tooltip": "TrueType font for BOTH overlays: a bare name resolved against the system " + "font folder (arial.ttf, arialbd.ttf, segoeui.ttf) or a full path to a " + ".ttf/.otf file. Falls back to the first font that loads if this one fails."}), + "overlay_stroke": ("INT", {"default": 0, "min": 0, "max": 20, + "tooltip": "Black outline thickness in pixels around the white text. 0 keeps it pure " + "white as asked; 2-3 makes it survive a bright sky or a white wall."}), + "ref_mode": (["where tagged", "first shot", "every shot", "every shot + handoff ref"], + {"default": "where tagged", + "tooltip": "Which shots the ref_image inputs condition. A shot carries EITHER " + "references or the last-frame handoff, never both. 'where tagged' " + "(default): write in the beat where that character " + "appears and ONLY that shot gets the reference -- every other shot " + "keeps its handoff. This is the precise option: the other modes go by " + "shot NUMBER and are blind to who is actually in the shot, so a " + "character who first appears in shot 2 gets nothing while an empty " + "establishing shot 1 gets a portrait pushed into it. Tags are " + "renumbered per shot, so alone still resolves. With refs " + "connected but no tags anywhere, falls back to first shot rather than " + "silently doing nothing. 'first shot' / 'every shot' / 'every shot + " + "handoff ref' go purely by position. Ignored when no ref_image is " + "connected."}), + "ref_noise_aug": ("FLOAT", {"default": 0.999, "min": 0.50, "max": 1.0, "step": 0.005, + "tooltip": "How CLEAN each reference is presented to the model. 0.999 (H3's own " + "default) hands it a finished, noise-free image -- which invites the " + "model to REPRODUCE the reference in the opening frames instead of just " + "taking an identity from it. Lower values blend the condition with " + "noise and label it as approximate, so it informs the face without " + "being copied: try 0.95, then 0.90. Too low (below ~0.8) and the " + "reference stops holding identity at all. Applies ONLY to " + "ref-conditioned shots -- the last-frame handoff is never weakened, or " + "continuity would break."}), + "ref_image_size": (["match", "max"], {"default": "match", + "tooltip": "How large each reference is encoded. 'match' scales it down to the " + "generation's pixel area -- a reference then costs about one frame per " + "step. 'max' uses the reference pipeline's 2048 short edge for the best " + "identity fidelity, but reference rows are re-attended EVERY step of " + "EVERY ref-conditioned shot, so on a long chain it is several times " + "slower. Neither ever upscales a small reference."}), + "beat_split": (["auto", "each line"], {"default": "auto", + "tooltip": "How the prompt box becomes beats. Beats are meant to be separated by a " + "BLANK line (or a '##' line) -- but six beats typed on six consecutive " + "lines are ONE paragraph, so they would render as one shot with six actions " + "crammed into it, which looks like everyone is moving at triple speed. " + "auto (default): blank lines first, then any paragraph still holding " + "several lines is split one beat per LINE, and the info output says so. " + "'each line': every line is its own beat -- same result, stated explicitly. " + "Neither can lose a beat. (The old strict 'blank line' option was REMOVED: " + "it was the only setting that could silently collapse beats, and a stored " + "value of it now reads as 'auto'.) Directive lines (wardrobe:, seconds:, exit:) " + "are never beats -- they attach to the beat that follows them."}), + "anchor_override": ("STRING", {"multiline": True, "forceInput": True, "default": "", + "tooltip": "Set the persistent look explicitly instead of using the first paragraph. " + "When this is filled in, EVERY paragraph of the prompt box is a beat/shot -- " + "nothing is consumed as the identity anchor. Put the permanent identity here " + "(hair, face, build, age) and the clothing in character_memory."}), + "per_beat_length": ("BOOLEAN", {"default": True, + "tooltip": "PACING. Size each shot from what its beat actually stages, instead of giving " + "every shot the same length. ON (default): a beat's time is ~2s of setup plus " + "~2.5s per action clause, or its spoken line, whichever is longer -- so 'she " + "takes off her jacket and drops it on the bench' gets ~7s and a three-part " + "beat gets more. OFF: every shot gets the full ceiling. WHY IT MATTERS: a 3s " + "action in a 12s shot leaves 9 seconds the model was told nothing about, and " + "it fills them by repeating or REVERSING the action -- which is why clothing " + "comes off and goes back on. The estimate leans SHORT on purpose: an " + "unfinished action is continued by the next shot from the handoff frame, " + "while an overlong one is unrecoverable. Never exceeds the ceiling " + "(shot_seconds or the VRAM budget) and always lands on the 17n+5 grid. " + "Override any single beat with 'seconds: 8' on its own line inside that " + "paragraph -- that wins over everything, including this toggle."}), + "auto_soundscape": (["off", "fill if blank", "always"], {"default": "fill if blank", + "tooltip": "Build the ambient bed from the scene instead of typing one. Reads the " + "ANCHOR (the soundscape is global, so it must describe the PLACE, not " + "one beat's action), falling back to the beats when the anchor is pure " + "camera language. 'A disused aircraft hangar' -> cavernous interior, " + "long reverb, distant metal ticks. Weather layers first: rain, wind, " + "snow, fog. NO human sounds are ever generated -- no chatter, crowd or " + "announcements -- because an ambient bed that implies voices is how H3 " + "starts talking.\n\n" + "'fill if blank' derives one only when the global_soundscape input " + "is unconnected or empty, so connecting your own text is enough to " + "keep it. 'always' derives even when you HAVE connected one, " + "overriding it -- deliberate, for comparing your bed against a " + "derived one without unwiring. 'off' never derives.\n\n" + "Whichever fires, the bed actually used comes out on the " + "`soundscape` output, so you can read it and wire it back into the " + "input to pin it."}), + "lock_restraints": ("BOOLEAN", {"default": True, + "tooltip": "Physical restraints stay ON until something explicitly removes them. " + "Handcuffs, shackles, manacles, fetters, irons, gags, blindfolds, " + "harnesses, leashes, plus qualified forms like 'ankle chain' or " + "'leather wrist straps'. Without this they come off like any garment, " + "and often by ACCIDENT -- 'steps out of her jacket and the chain falls " + "away' removed the ankle chain as a side effect of a jacket beat. To " + "take one off, say so directly: 'wardrobe: Mara -= handcuffs'. Bare " + "'chain', 'collar', 'strap' and 'belt' are NOT treated as restraints; " + "they are jewellery, a shirt part, a dress part and a garment at least " + "as often."}), + "contact_guard": (["off", "auto", "on"], {"default": "auto", + "tooltip": "Keep two bodies in contact correctly aligned -- any position, " + "not a list of named ones.\n\n" + "Position-agnostic on purpose. The model already knows more " + "position names than a dictionary could hold; what it gets wrong " + "is the GEOMETRY, so the geometry is what gets stated, and these " + "hold for every arrangement:\n" + " - OWNERSHIP: each person keeps their own head, two arms and two " + "legs, each joined to the body it belongs to. Overlapping bodies " + "is exactly when an arm gets reassigned to the wrong torso.\n" + " - SEPARATION: they meet at the surface of the skin, each keeping " + "its own volume, rather than passing into one another.\n" + " - STABLE ROLES: whoever is above stays above, below stays below, " + "behind stays behind, for the whole shot and from every camera " + "angle. Positions morph mid-shot because nothing fixes them.\n" + " - SUPPORT: weight rests on whatever is holding it, and the two " + "bodies stay in proportion.\n\n" + "Needs TWO people in the shot -- one body cannot be misaligned " + "against another, and saying otherwise would invite a second person " + "in. 'auto' fires on a contact cue in the beat; 'on' states it " + "whenever two or more people are present.\n\n" + "Describe the arrangement in the beat itself in RELATIVE terms " + "(who is above, behind, facing whom) rather than by a position " + "name alone -- this guard holds a stated arrangement together, it " + "cannot infer one you did not state."}), + "motion_guard": (["off", "auto", "on"], {"default": "auto", + "tooltip": "Stop poses being reached without the frames in between -- the " + "head arriving at a new angle with no path to it (a 'neck snap'), " + "a body teleporting between two positions.\n\n" + "What is missing in a snap is the PATH, not the pose, so the path " + "is what gets stated: movement travels through every position on " + "the way, at one steady speed, the neck following the shoulders " + "and the shoulders following the hips. Positive, because at cfg 1 " + "H3 is CFG-free and the negative is never evaluated -- and 'the " + "head does not snap round' in the positive names a head snapping " + "round.\n\n" + "'auto' speaks only on a beat that actually moves someone (turns, " + "looks, walks, leans, reaches... and the high-jerk ones -- struggles, " + "pulls, twists, writhes -- where a limb most often arrives without its " + "path), since a beat where nobody changes orientation has no path to " + "describe. 'on' states it every shot. Names nobody, so it adds no " + "second reference to anyone already in frame.\n\n" + "A snap right after a cut is a different thing: that is the model " + "leaving the keyframe pose. handoff_offset helps there."}), + "solidity_guard": (["off", "auto", "on"], {"default": "auto", + "tooltip": "Keep bodies from passing through objects. States that the solid " + "things in the shot occupy space and that bodies stop at surfaces.\n\n" + "Stated POSITIVELY, and it has to be: H3 is CFG-free at cfg 1, so a " + "negative prompt is never evaluated, and 'does not walk through the " + "wall' in the positive names walking through a wall -- a mention is a " + "presence cue. It says what bodies DO instead: stop at the surface, " + "rest on the floor, press against what they touch, go around the " + "furniture.\n\n" + "'auto' speaks only when the shot actually names something solid " + "(walls, doors, tables, stairs, vehicles, crates, trees...), reading " + "BOTH the beat and the identity block, since the set is usually " + "described in the anchor. 'on' states it every shot. Only ever " + "applied to a shot with someone in it -- a body is needed before one " + "can pass through anything."}), + "anatomy_guard": (["off", "auto", "on"], {"default": "auto", + "tooltip": "State each person's limb COUNT positively, to stop spare arms, " + "duplicated hands and the third leg. H3 is CFG-free at cfg 1, so a " + "NEGATIVE prompt is never evaluated -- 'extra limbs' in a negative does " + "nothing. Naming a number gives the model a target instead; negating one " + "only puts the word in the prompt. Added per-shot and only where people " + "are actually present, never in the anchor (anchor body words are what " + "burn a face into every opening frame). 'auto' = on below 768 short edge " + "OR when a LoRA is applied, and also on ANY shot holding two or more " + "people -- spare limbs are grown where bodies meet and move together. " + "Costs ~90 tokens on shots with people."}), + "exposed_terms": ("STRING", {"multiline": True, "forceInput": True, "default": "", + "tooltip": "What a stripped body zone is CALLED, per character, so it persists " + "automatically instead of being typed into every beat. Same syntax as " + "character_memory -- a PRONOUN covers everyone who declares it, a NAME " + "overrides it, and a trailing 'upper' targets the chest. " + "For example -- 'she = visible vulva, mvagina' / " + "'he = visible penis, mpenis' / 'Mara upper = bare breasts'. " + "Once a removal empties that zone the phrase is stamped into every " + "later shot that person is in, and clears by itself when something " + "covers the zone again. Put LoRA trigger words here too. Requires " + "prevent_nudity OFF; empty falls back to 'bare below the waist'."}), + "prevent_nudity": ("BOOLEAN", {"default": True, + "tooltip": "Never let the prompt ASSERT that a body is bare. A removal still " + "happens either way -- this gates the sentence, not the garment. " + "Deleting the last item covering a zone only leaves it undescribed, " + "and a video model's default is a clothed person, so it covers what " + "nobody described. With this OFF the node states the state outright " + "('bare below the waist') and keeps stating it until something covers " + "that zone again, which is what makes a strip actually stick. ON is " + "the safe default; turn it OFF only when nudity is intended. Either " + "way info reports which zone a removal left uncovered."}), + "auto_props": ("BOOLEAN", {"default": True, + "tooltip": "Carry OBJECTS across shots. Each shot is a separate generation, so " + "'the van' in shot 2 has no antecedent -- nothing in that prompt " + "describes a van, and the model invents one, which is how a second van " + "appears while the first is still in frame. With this on, an object " + "introduced indefinitely ('a white van') is bound on its first definite " + "reference in any later beat ('the van' -> 'the same white van') and a " + "short clause pins it to the previous shot: one van only, no second van. " + "Only the FIRST mention per shot is expanded, quoted dialogue is never " + "rewritten, worn garments are excluded (they have the wardrobe channel), " + "and frame/body nouns (the ground, the light, the hand) are never " + "carried."}), + "auto_wardrobe": ("BOOLEAN", {"default": True, + "tooltip": "Read clothing REMOVALS straight from your beat prose -- 'she takes off her " + "jacket' drops the jacket with no 'wardrobe:' line needed. Safe: only fires " + "on items the character is already wearing, so 'the plane takes off' does " + "nothing. Additions/swaps still use 'wardrobe: += ...' (which overrides). " + "Turn OFF to control wardrobe only via explicit 'wardrobe:' lines."}), + "subject_count_guard": (["auto", "on", "off"], {"default": "auto", + "tooltip": "Anti-duplication: prepend an explicit subject count to each shot " + "(\"Exactly two people in this shot, no duplicates, no other people in " + "frame\"). Character duplication gets much more likely BELOW the model's " + "native 768 short edge -- fewer pixels per subject pushes the sample out of " + "the training distribution and the figure gets tiled. A LoRA causes it too: a " + "distilled LoRA fixes composition (including how many people are in frame) " + "in its first step or two, so it duplicates even at native size -- there the " + "count is moved to the FRONT of the prompt so it binds before the scene. " + "'auto' = on when the short edge is under 768 OR a LoRA is applied, and also " + "on ANY shot holding two or more people -- multi-figure frames are where " + "duplication happens even at native size; " + "'on' always; 'off' never."}), + "auto_silence_nonspeech": ("BOOLEAN", {"default": True, + "tooltip": "Stop mouths moving / gibberish audio on shots with no dialogue. Any beat " + "with no scripted speech gets an explicit 'lips closed, no dialogue' clause, " + "so H3 doesn't animate or vocalize a mouth before real dialogue. Beats with " + "quoted dialogue (\"...\") are left alone. To make someone speak, put the " + "words in double quotes. Turn OFF to manage lip state yourself."}), + "allow_nonspeech_vocals": ("BOOLEAN", {"default": False, + "tooltip": "Allow non-speech vocal sounds (screams, sobs, gasps, moans) on " + "shots with no dialogue. When ON, the node skips the lips-closed " + "clause and softens the no-voice soundscape to ban only speech, " + "dialogue and singing -- not screams, sobs, or other distress " + "vocalizations. Audio is also left unmuted on those shots. Turn ON " + "when your scene contains distress sounds that H3 would otherwise " + "suppress. Keep auto_silence_nonspeech ON for shots that should be " + "truly silent."}), + "character_memory": ("STRING", {"multiline": True, "forceInput": True, "default": "", + "tooltip": "Optional dedicated wardrobe channel (same role as a 'wardrobe:' line in " + "the first paragraph -- use whichever you prefer; this field wins if both " + "are set). Re-stamped into every shot so clothing holds even when the " + "camera crops it out. IMPORTANT: this is the ONLY place clothing should " + "live -- keep it out of the anchor prose, or a removal won't stick because " + "the immutable anchor keeps re-adding it. To change/remove an item " + "mid-chain, put 'wardrobe: ' inside the beat where it " + "changes; omit the removed item from the new sheet and it stays gone. " + "WRITE ATTRIBUTES, NOT NOUN PHRASES: 'silver hair, 27, red jacket' -- NOT " + "'a woman with silver hair'. A noun phrase renders as 'She (a woman with...)', " + "i.e. two subjects in one clause, which causes character duplication. The node " + "strips them automatically, but writing attributes directly is cleaner. " + "ONE-TOKEN EDITS (no restating the outfit): 'wardrobe: -= jacket' removes " + "the jacket, 'wardrobe: += sunglasses' adds one. TWO+ PEOPLE: name them -- " + "'Maya = grey shorts, red jacket; Jon = navy overalls', then edit one at a " + "time: 'wardrobe: Maya -= jacket' leaves Jon untouched."}), + }, + # Read-only graph access, for SLA-LoRA detection: a LoRA's filename is + # the only thing that identifies an SLA build, and the graph is the only + # place it survives. Named 'graph'/'node_id' rather than the usual + # 'prompt' because this node already has a `prompt` widget -- ComfyUI + # passes hidden inputs by parameter name, so "prompt": "PROMPT" would + # overwrite the user's text with the workflow dict. Hidden inputs carry + # no widget, so they cannot shift saved widget positions. + "hidden": {"graph": "DYNPROMPT", "node_id": "UNIQUE_ID"}, + } + # ComfyUI restores a saved graph's widget values POSITIONALLY, from a flat + # widgets_values array. A widget inserted in the MIDDLE therefore shifts every + # value after it onto the wrong widget in every workflow saved before it + # existed -- silently, with no error. So widgets added after v1 are forced to + # the END here, leaving the original order byte-for-byte intact. + opt = schema["optional"] + for name in ADDED_WIDGETS: + if name in opt: + opt[name] = opt.pop(name) # re-insert at the end, value unchanged + return schema + + def _render(self, model, clip, vae, audio_vae, negative, prompt, w, h, ln, fps, tiled, sa, + handoff, decode_tile_frames=0, decode_tile_size=0, + refs=None, ref_image_size="match", ref_noise_aug=None, silent=False): + positive, latent = _build_shot_conditioning(clip, vae, prompt, w, h, ln, fps, handoff, + ref_images=refs, ref_image_size=ref_image_size, + ref_noise_aug=ref_noise_aug, + audio_vae=audio_vae, silent=silent) + seed, steps, cfg, sn, sch, denoise = sa + # Conditioning is built, so the text encoder and VAEs are dead weight for the + # whole sampling loop -- evict them and keep only the DiT on the card. + _evict_all_but(model) + try: + (out,) = nodes.common_ksampler(model, seed, steps, cfg, sn, sch, positive, negative, + latent, denoise=denoise) + except Exception as e: + # Mark WHERE this failed. `tiled` only affects the DECODE, so the caller's + # OOM retry cannot help an OOM raised here -- it just re-runs the whole + # sampling pass and fails the same way, which on a 362-frame shot is four + # more minutes for nothing. + if _is_oom(e): + e._h3_stage = "sampling" + raise + # Keep a CPU copy of the sampled latent BEFORE decoding, for the `latent` + # output. Latents are ~1000x smaller than the frames they decode to (a + # 1344x768 124f shot is ~1.5MB against ~1.5GB), so carrying one per shot for + # the whole chain is free. Detached and moved off the card immediately, for + # the same reason the decoded frames are. + raw = out.get("samples") if isinstance(out, dict) else None + shot_latent = None + if raw is not None: + try: + parts = raw.unbind() if hasattr(raw, "unbind") else None + shot_latent = ([t.detach().to("cpu", copy=True) for t in parts] + if parts else raw.detach().to("cpu", copy=True)) + except Exception: + shot_latent = None # never fail a render for the sake of an output + video = _decode_video(vae, out, tiled, free_first=model, + tile_t=decode_tile_frames, tile_xy=decode_tile_size) + audio = _decode_audio(audio_vae, out) + del out, positive, latent + _deep_cleanup() + return video, audio, shot_latent + + def run(self, model, clip, vae, audio_vae, prompt, resolution, + steps, cfg, sampler_name, scheduler, seed, + megapixels=1.0, + first_frame=None, fps=24, plan_only=False, + global_soundscape="", non_diegetic_music="", apply_model_sampling=True, + shift_video=12.0, shift_audio=3.0, trim_seam=True, vary_seed_per_shot=False, + handoff_offset=0, vram_headroom_gb=1.5, allow_res_backoff=True, + decode_tile_frames=0, decode_tile_size=0, + cleanup_between_shots=True, + anchor_override="", shot_seconds=0.0, allow_oversize_shots=False, + per_beat_length=True, beat_split="auto", + character_memory="", auto_wardrobe=True, auto_props=True, prevent_nudity=True, + exposed_terms="", anatomy_guard="auto", lock_restraints=True, + solidity_guard="auto", motion_guard="auto", contact_guard="auto", + auto_soundscape="fill if blank", + auto_silence_nonspeech=True, allow_nonspeech_vocals=False, + subject_count_guard="auto", + upscale="off", upscale_model="none", + upscale_target_short_edge=0, upscale_batch=4, + mute_nonspeech_audio=True, mute_fade_ms=40, + watermark_text="", watermark_position="bottom-right", watermark_size=4.0, + watermark_opacity=0.75, watermark_margin=3.0, + intro_text="", intro_position="center", intro_seconds=3.0, intro_fade=0.6, + intro_size=9.0, overlay_font="arial.ttf", overlay_stroke=0, + ref_image_1=None, ref_image_2=None, ref_image_3=None, ref_image_4=None, + ref_mode="where tagged", ref_image_size="match", ref_noise_aug=0.999, + graph=None, node_id=None): + + # FIRST: detect a checkpoint swap since the previous execution and hard-flush. + # A stale resident model from a different checkpoint would otherwise poison + # every VRAM measurement below (and leave old hooks/allocator blocks behind), + # so this must run before the schedule patch and before vram_gb(). + swap_note = flush_for_model_change(model) + + # Cheap wiring preflight: a video VAE on the audio_vae socket only blows up + # after a full shot has been sampled and decoded, so reject it up front. + check_vae_wiring(vae, audio_vae) + + # H3 renders 24 fps, always. Honor the widget only as a warning: a lower value + # used to silently shorten every shot (10s -> 124f -> 5.2s of real time). + fps_note = ("" if int(fps) == H3_FPS else + f"fps widget is {int(fps)} but H3 always renders {H3_FPS} fps -- all durations " + f"computed at {H3_FPS}; set your video-save node to {H3_FPS} too") + fps = H3_FPS + w, h = parse_resolution(resolution) + # A pixel budget overrides the preset's SIZE while keeping its aspect ratio, + # so the dropdown chooses the shape and this chooses how big. Scaling from + # the preset's own dimensions is what makes 1.00MP reproduce each native + # size exactly -- the preset NAMES are approximations (1344x768 is 7:4, not + # 16:9), so computing from a nominal ratio would not. + # + # Reported as ACHIEVED, not requested: snapping to the 32-grid moves the + # real area, and echoing the asked-for figure would print a number the + # render never used. + mp_note = "" + if megapixels and float(megapixels) > 0: + nw, nh = scale_to_megapixels(w, h, float(megapixels)) + if (nw, nh) != (w, h): + mp_note = (f"megapixels {float(megapixels):.2f} -> {nw}x{nh} " + f"({nw * nh / MP_UNIT:.3f}MP actual; preset was {w}x{h} " + f"@ {w * h / MP_UNIT:.3f}MP)") + w, h = nw, nh + else: + mp_note = (f"megapixels {float(megapixels):.2f} -> {w}x{h} " + f"({w * h / MP_UNIT:.3f}MP), the preset's own size") + # H3 is CFG-free (cfg 1): the sampler skips the negative, but common_ksampler + # still needs a conditioning object, so build an empty one from the clip. + negative = clip.encode_from_tokens_scheduled(clip.tokenize("")) + # Cheapest possible preflight: this empty encode already went through the + # text encoder, so compare its width to the DiT's before anything expensive. + check_text_encoder(model, negative) + + # Patch the dual video/audio schedule onto the model here, so a missing + # upstream ModelSamplingMiniMaxH3 can't silently produce gibberish audio. + # Shifts come from the widgets (12/3 base default; MXFP8/turbo differ). + ms_note = "" + if apply_model_sampling: + model, ms_note = apply_h3_model_sampling(model, shift_video, shift_audio) + + paras = split_paragraphs(prompt, "##") + if anchor_override.strip(): + anchor, beat_paras = anchor_override.strip(), paras + elif paras: + anchor, beat_paras = paras[0], paras[1:] + else: + anchor, beat_paras = "", [] + # A first paragraph that would be stripped to nothing is not an anchor -- it is an + # action beat about a tracked character, and consuming it deletes that shot + # outright (the sentence names the character, so it gets removed from the always-on + # anchor to avoid introducing them twice). Keep it as a BEAT and say so loudly, + # rather than losing a shot and the scene text along with it. + anchor_note = "" + if (not anchor_override.strip()) and paras and \ + (anchor_contributes_nothing(anchor, character_memory.strip()) + or anchor_is_action_beat(anchor, paras[1:])): + preview = " ".join(anchor.split())[:60] + anchor, beat_paras = "", paras + anchor_note = ( + f'WARNING: paragraph 1 ("{preview}...") reads as an action beat about a tracked ' + f'character, not an identity anchor -- consuming it would have deleted that shot ' + f'entirely, so it was KEPT AS A BEAT. There is now no persistent scene text: put ' + f'the setting and style (with NO character names) in anchor_override.') + # The anchor repeats on every shot, so what is IN it matters more than its + # length. These have each cost a render: face words putting a face in an empty + # establishing frame, apparatus words rendering the equipment (or someone + # holding it), framing pinning every shot, clothing that no removal can strip. + anchor_hazards = anchor_warnings(anchor) + # Anchor extraction happens on PARAGRAPHS first, so a line-split can never + # eat into the identity block; only the beat paragraphs are expanded. + beats, split_note = expand_beats(beat_paras, beat_split) + # Build the ambient bed from the scene when asked. Done here, where the + # anchor and beats are both parsed, and BEFORE distribute_generations stamps + # the soundscape onto every shot. + sound_note = "" + if auto_soundscape != "off": + typed = global_soundscape.strip() + if auto_soundscape == "always" or not typed: + derived = derive_soundscape(anchor, beats) + if derived and derived != typed: + sound_note = (f"soundscape {'replaced with' if typed else 'built from'} the " + f"scene: '{derived}'" + + (f" (yours: '{typed}')" if typed else "") + + ". Set auto_soundscape to 'off' to keep your own") + global_soundscape = derived + beats_note = (f"{len(beats)} beat(s) -> {len(beats)} shot(s) from {len(paras)} paragraph(s)" + + ("" if anchor_override.strip() else + "; paragraph 1 was consumed as the identity anchor (fill anchor_override " + "to make EVERY paragraph a beat)") + + (f". {split_note}" if split_note else "")) + + total_gb, free_gb = vram_gb() + resident_gb = dit_resident_gb(model) + # Weights larger than the card means ComfyUI must stream them: NO shot + # length or resolution avoids spilling into shared/system memory, so say so + # plainly rather than letting it look like a tuning problem. + streaming = total_gb > 0 and resident_gb > 0 and resident_gb > total_gb + lora_gb = lora_overhead_gb(model) + eff_headroom = vram_headroom_gb + lora_gb + ln, ln_note = resolve_shot_frames(shot_seconds, fps, total_gb, resident_gb, + eff_headroom, allow_oversize_shots, w * h, free_gb) + if lora_gb: + ln_note = ((ln_note + " ") if ln_note else "") + ( + f"reserved ~{lora_gb:.1f}GB for bypass-LoRA adapters (they stay resident in bf16 " + f"rather than folding into the weights)") + # Hitting the internal floor means the budget arithmetic gave up, and every shot + # comes out ~5s regardless of what the beats need. That looked like the node + # ignoring the prompt; say what actually ran out and what moves the number. + if ln <= align_frame_count(MIN_SHOT_FRAMES) and total_gb > 0 and not ( + shot_seconds and float(shot_seconds) > 0): + ln_note = ((ln_note + " ") if ln_note else "") + ( + f"SHOT LENGTH IS AT THE {ln}f (~{ln / fps:.1f}s) FLOOR -- every shot will be this " + f"long whatever the beat asks for. " + + (f"No live free-VRAM reading was available, so there was nothing to budget from " + f"(weights ~{resident_gb:.1f}GB stream and cannot be subtracted from the " + f"{total_gb:.1f}GB card)." + if streaming else + f"Weights ~{resident_gb:.1f}GB + headroom ~{eff_headroom:.1f}GB leave nothing of " + f"the {total_gb:.1f}GB card for the latent.") + + f" Free right now: ~{free_gb:.1f}GB. Lower vram_headroom_gb, drop to the " + f"balanced/fast resolution tier, or close other GPU apps") + accel_note = quant_accel_note(model) + if streaming: + ln_note = ((ln_note + " ") if ln_note else "") + ( + f"weights (~{resident_gb:.1f}GB) exceed VRAM (~{total_gb:.1f}GB), so they stream rather " + f"than sitting on the card -- that figure is NOT subtracted from the budget, which is " + f"built from the ~{free_gb:.1f}GB actually free instead") + tiled = total_gb > 0 and (total_gb - resident_gb) < 20 + + # Sub-native renders duplicate subjects far more often, so default the guard on + # there and leave native renders alone (the extra clause costs prompt budget). + lora_on = lora_active(model) + # An SLA LoRA and a sparse-attention patch are a matched pair; either one + # alone costs a full render to discover. Computed here so plan_only reports + # it too -- that is the point of catching it, before anything is sampled. + sla_name, sparse_on, sla_note = sla_pairing(model, graph, node_id) + # Same idea, wider: where a LoRA's own declared training disagrees with this + # run. Reports only -- never overrides a widget, so the render stays + # reproducible from what the graph shows. + hint_notes = lora_hint_notes(model, graph, node_id, steps, min(w, h)) + # The flow shift is right for base H3 at ~20 steps and wrong for a distill + # at 4-8, and it fails silently -- soft output, no error. Computed here so + # plan_only reports it before anything is sampled. + sched_note = (schedule_balance_note(shift_video, steps, scheduler) + if apply_model_sampling else "") + # The quantization kernels are ComfyUI's, not this node's -- but losing them + # is silent, and the symptom (soft output) looks like a dozen other causes. + kernel_note = kernel_backend_note(model) + audio_ratio_note = (audio_scale_note(shift_video, shift_audio) + if apply_model_sampling else "") + # One ordered list, emitted once per output. These used to be six separate + # `+ (f" X -- {note}." if note else "")` fragments repeated at BOTH the plan + # and the render site -- twelve lines that had to stay in sync by hand. They + # did not: `audio_note` collided with the mute-reporting variable of the same + # name, so the shift-ratio warning reached plan_only and never a real render, + # while the mute note was printed twice under the wrong label. + # 'auto' fires below native resolution AND whenever a LoRA is applied: a + # distilled LoRA fixes the subject count in its first step or two, so the + # count has to be stated even at native size. + anatomy_on = (anatomy_guard == "on" or + (anatomy_guard == "auto" and (min(w, h) < 768 or lora_on))) + # 'auto' additionally states the limb count on any multi-person shot, + # whatever the resolution -- the same reasoning as count_auto above. + anatomy_auto = (anatomy_guard == "auto") + count_subjects = (subject_count_guard == "on" or + (subject_count_guard == "auto" and (min(w, h) < 768 or lora_on))) + # `ln` is the CEILING (VRAM budget, or a forced shot_seconds). Each beat gets + # its own length UNDER that ceiling -- including when shot_seconds is forced, + # which now means "no shot longer than this" rather than "every shot exactly + # this". Forcing a length used to DISABLE per-beat sizing entirely, which is + # why a plan made with a forced length disagreed with the auto render: two + # different code paths for the same question. + lens, len_notes = plan_beat_frames(beats, fps, ln, per_beat=bool(per_beat_length)) + secs = [n / fps for n in lens] + if len_notes: + n_short = sum(1 for n in lens if n < ln) + ln_note = ((ln_note + " ") if ln_note else "") + ( + f"per-beat pacing sized {n_short} of {len(lens)} shot(s) under the {ln}f " + f"(~{ln / fps:.1f}s) ceiling from their own content: " + "; ".join(len_notes) + + ". Turn per_beat_length OFF to give every shot the full ceiling") + # With pacing OFF, every beat gets the ceiling whether it has anything to fill + # it with or not -- so say which beats are too thin for the length they got. + # This is the failure that reads as an action repeating or playing backwards. + pace_warnings = pacing_warnings(beats, lens, fps) + if pace_warnings: + ln_note = ((ln_note + " ") if ln_note else "") + ( + "THIN BEATS -- the model must invent the remaining time, which it fills by " + "repeating or REVERSING the action: " + "; ".join(pace_warnings) + + ". Add a second clause to the beat, set 'seconds:' on it, or turn " + "per_beat_length ON to size shots from their content") + fit_warnings = dialogue_fit_warnings(beats, secs) + # The opposite error, and the one that babbles: far more shot than line. + filler_warnings = dialogue_filler_warnings(beats, secs) + if filler_warnings: + ln_note = ((ln_note + " ") if ln_note else "") + ( + "BABBLE RISK -- " + "; ".join(filler_warnings) + + ". Turn per_beat_length ON to size these shots from their line, or set " + "'seconds:' on the beat") + wardrobe_notes = [] + strip_shots = [] # shots that newly bared a zone -> the NEXT shot starts fresh + gens = distribute_generations(anchor, beats, global_soundscape.strip(), + non_diegetic_music.strip(), character_memory.strip(), + auto_wardrobe, auto_silence_nonspeech, allow_nonspeech_vocals, count_subjects, + lora_on, notes_out=wardrobe_notes, auto_props=auto_props, + prevent_nudity=prevent_nudity, + exposed_terms=exposed_terms, strip_out=strip_shots, + anatomy_guard=anatomy_on, + anatomy_auto=anatomy_auto, + lock_restraints=lock_restraints, + solidity_guard=solidity_guard, + motion_guard=motion_guard, + contact_guard=contact_guard, + count_auto=(subject_count_guard == "auto")) + + # A scenery beat mid-chain hands the next shot a frame with no people in + # it. Both prompts are individually correct, so this is invisible without + # looking at the sequence -- which is why chains lose their cast in the + # middle rather than degrading steadily. + cohesion_notes = continuity_warnings(gens) + # A per-shot seed resets the noise field -- and with it the grain, the + # micro-texture and every surface detail the prompt never names -- at each + # boundary. That reads as a cut even when the keyframe anchors the frame and + # the location is unchanged, and nothing else reported it. + if vary_seed_per_shot and len(gens) > 1: + cohesion_notes.append( + f"vary_seed_per_shot is ON, so the {len(gens)} shots sample from seeds " + f"{seed}..{seed + len(gens) - 1} rather than one field. Stochastic detail " + f"resets at every boundary, which looks like a cut in a continuous take. " + f"Turn it off unless the beats are meant to look separately shot") + preflight = [("SLA", sla_note), + ("LORA HINTS", "; ".join(hint_notes)), + ("", mp_note), + ("SCHEDULE", sched_note), + ("KERNELS", kernel_note), + ("AUDIO", audio_ratio_note), + ("CONTINUITY", "; ".join(cohesion_notes)), + ("SOUND", sound_note)] + preflight_txt = "".join(f"{(lbl + ' -- ') if lbl else ''}{txt}. " + for lbl, txt in preflight if txt) + + if plan_only: + # Preview the split using THIS node's own settings -- no render, near-instant. + shots = len(gens) + plan_lens = (lens + [ln] * shots)[:shots] + total = round(sum(plan_lens) / fps, 2) + uniform = len(set(plan_lens)) == 1 + shape = (f"{shots} shot(s) x {plan_lens[0]}f (~{plan_lens[0] / fps:g}s each)" if uniform + else f"{shots} shot(s), {sum(plan_lens)}f total: " + + ", ".join(f"{n}f/~{n / fps:.1f}s" for n in plan_lens)) + vram_str = f"{total_gb:.1f}GB total / {resident_gb:.1f}GB weights / {free_gb:.1f}GB free" if total_gb else "VRAM unknown" + # Same dialogue/audio accounting the render reports, so the plan says up + # front which shots will come back silent instead of surprising you after. + n_silent = sum(1 for f in speech_flags(beats) if not f) + plan_audio = "" + if n_silent: + if allow_nonspeech_vocals: + plan_audio = (f" {n_silent} of {shots} shot(s) have no quoted dialogue -> " + f"non-speech vocals allowed (speech still suppressed)") + else: + plan_audio = (f" {n_silent} of {shots} shot(s) have no quoted dialogue -> " + + ("AUDIO-MUTED (ambience goes too)" if mute_nonspeech_audio + else "prompt/soundscape silencing only")) + # Same reference accounting the render reports: which shots lose the + # handoff is a composition decision, so it belongs in the preview. + n_refs = len([r for r in (ref_image_1, ref_image_2, ref_image_3, ref_image_4) + if r is not None]) + plan_ref = "" + if n_refs: + # Mirror the render's placement exactly: 'where tagged' reads the + # prompts and falls back to first shot when nothing is tagged -- + # reporting by ref_mode alone described shots the render never gave + # references to. + if ref_mode == "where tagged" and any(picture_tags(g) for g in gens): + on = [n + 1 for n, g in enumerate(gens) if picture_tags(g)] + how = "placed by tags" + else: + mode_eff = "first shot" if ref_mode == "where tagged" else ref_mode + on = [n + 1 for n in range(shots) + if shot_references([1] * n_refs, mode_eff, n, 1 if n else None)] + how = (f"ref_mode '{mode_eff}'" + + (" -- no tags found anywhere" if ref_mode == "where tagged" else "")) + plan_ref = (f" ref2va: {n_refs} reference image(s) at '{ref_image_size}' on shot(s) " + f"{','.join(str(n) for n in on) or 'none'} ({how}) -> those shots keep " + f"the previous frame as their keyframe too, unless ref_noise_aug was lowered") + plan = ((anchor_note + " ") if anchor_note else "") + \ + preflight_txt + \ + (("DIALOGUE MAY BE CUT OFF -- " + "; ".join(fit_warnings) + ". ") if fit_warnings else "") + \ + (f"PLAN (no render): {shape} = ~{total:g}s at {w}x{h}. " + f"{len(beats) or 1} beat(s). decode {'tiled' if tiled else 'full'}. {vram_str}." + + (f" {beats_note}." if beats_note else "") + + (" ANCHOR: " + "; ".join(anchor_hazards) + "." + if anchor_hazards else "") + + (f"{plan_audio}." if plan_audio else "") + + (" EXPOSURE -- " + "; ".join(wardrobe_notes) + "." + if wardrobe_notes else "") + + (f"{plan_ref}." if plan_ref else "") + + (f" {fps_note}." if fps_note else "") + + (f" {ln_note}." if ln_note else "")) + ph_img = torch.zeros((1, 64, 64, 3)) + ph_audio = {"waveform": torch.zeros((1, 2, 1)), "sample_rate": 44100} + # plan_only samples nothing, so there is no latent to hand out. Emit a + # correctly-SHAPED empty one rather than None: a downstream LATENT input + # would choke on None, and this keeps the preview wireable exactly like + # a real run. + return (ph_img, ph_audio, plan, "\n---\n".join(gens), max(plan_lens), + sum(plan_lens), shots, total, float(fps), int(fps), + _empty_av_latent(w, h, 5, fps)[0], global_soundscape) + + spk = speech_flags(beats) # which shots have real (quoted) dialogue + vram_trace = [] # free VRAM after each shot + muted_flags = [] # which shots were audio-silenced + hoff = max(0, int(handoff_offset)) + backoff, video_chunks, audio_chunks = [], [], [] + latent_chunks = [] # per-shot sampled latents, pre-decode + mouth_settled = [] # shots seeded from a settled (closed) mouth + handoff, sr = first_frame, None + ref_list = [r for r in (ref_image_1, ref_image_2, ref_image_3, ref_image_4) if r is not None] + ref_shots = [] # which shots ended up ref-conditioned + ref_missing = [] # tags naming an unconnected slot + ref_carried = [] # tagged shots that kept continuity as an extra ref + ref_keyframed = [] # tagged shots that kept it as a real keyframe + # 'where tagged' reads the prompt instead of counting shots. If references are + # connected but nothing is tagged anywhere, fall back to first-shot placement + # rather than silently conditioning nothing at all. + tag_driven = bool(ref_list) and ref_mode == "where tagged" and any( + picture_tags(g) for g in gens) + if ref_list and ref_mode == "where tagged" and not tag_driven: + ref_mode = "first shot" + if cleanup_between_shots: + _deep_cleanup() # start the first (heaviest) shot with max free VRAM + + shot_lens = (lens + [ln] * len(gens))[:len(gens)] + for i, gen_prompt in enumerate(gens): + # denoise is fixed at 1.0 (partial denoise desyncs the joint AV schedule). + sa = (seed + i if vary_seed_per_shot else seed, steps, cfg, sampler_name, scheduler, 1.0) + ln_i = shot_lens[i] # this beat's own length (<= the VRAM ceiling) + # Which conditioning channels this shot carries is decided here; see + # _build_shot_conditioning for how they are packed. On ComfyUI 0.31+ a + # shot may carry BOTH references and a keyframe. + carry_keyframe = False # tagged shot keeps its handoff as a keyframe + if tag_driven: + # The prompt itself says where each reference belongs: the shot whose + # text names gets image N, renumbered to match what that + # shot actually carries. Every untagged shot keeps its handoff. + gen_prompt, shot_refs, dropped = resolve_tagged_refs(gen_prompt, ref_list) + for n in dropped: + if n not in ref_missing: + ref_missing.append(n) + # A tagged shot keeps its continuity, but by which channel depends on + # ref_noise_aug. On ComfyUI 0.31+ refs and keyframes coexist, so the + # handoff can be a REAL keyframe -- it anchors the first frame, which + # is what continuity means. Once references are softened that same + # aug would soften the keyframe too, so there it falls back to riding + # as an extra reference (the pre-0.31 workaround): weaker, but it + # leaves no anchor to compromise. Appended AFTER the tagged images, so + # their numbers are untouched. + if shot_refs and handoff is not None: + if keyframe_rides_with_refs(ref_noise_aug): + carry_keyframe = True + ref_keyframed.append(i + 1) + else: + shot_refs = shot_refs + [handoff] + ref_carried.append(i + 1) + else: + shot_refs = shot_references(ref_list, ref_mode, i, handoff) + # ComfyUI 0.31+ lets references and a keyframe ride TOGETHER, and only + # the tagged branch above was ever updated for it. Everywhere else a + # ref-conditioned shot still dropped its handoff, as 0.30 required: + # 'every shot' -> no keyframe at all, so consecutive + # shots meet as CUTS + # 'every shot + handoff ref' -> the handoff demoted to a soft + # reference ("look like this") rather + # than an anchor ("start from this") + # 'first shot', shot 0 -> the start_image was ignored outright + # In each case the last frame of a shot does not become the first frame + # of the next, which is exactly the reported symptom. + if shot_refs and handoff is not None and keyframe_rides_with_refs(ref_noise_aug): + carry_keyframe = True + ref_keyframed.append(i + 1) + # It is anchoring as a keyframe now, so the SAME frame repeated in + # the ref channel would only spend rows saying it twice -- and say + # it more weakly. + shot_refs = [r for r in shot_refs if r is not handoff] + elif (shot_refs and handoff is not None + and ref_mode == "every shot + handoff ref"): + ref_carried.append(i + 1) # softened refs: the 0.30 fallback + # A shot that follows a strip starts FRESH. Continuing from a frame that + # still shows the garment is how it reappears -- the picture outvotes the + # text every time. Costs a cut exactly where the state changes, which is + # where a cut belongs anyway. + # No scripted line -> anchor this shot's audio branch to silence. + shot_silent = bool(auto_silence_nonspeech and not allow_nonspeech_vocals and i < len(spk) and not spk[i]) + after_strip = i in strip_shots # strip_shots is 1-based, i is 0-based + shot_handoff = (None if after_strip + else handoff if (carry_keyframe or not shot_refs) else None) + if shot_refs: + ref_shots.append(i + 1) + if i == 0: + while True: + try: + frames, audio, shot_latent = self._render(model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, + shot_refs, ref_image_size, ref_noise_aug, shot_silent) + break + except (torch.cuda.OutOfMemoryError, RuntimeError) as e: + if not _is_oom(e): + raise + mm.soft_empty_cache(True) + if not tiled: + tiled = True; backoff.append("tiled decode") + elif allow_res_backoff and min(w, h) > 384: + nw, nh = res_down(w, h); backoff.append(f"res->{nw}x{nh}"); w, h = nw, nh + else: + raise RuntimeError("H3 Long Videos: not enough VRAM even at the smallest size. " + "Pick a smaller resolution, close other GPU apps, or use a smaller quant.") + else: + try: + frames, audio, shot_latent = self._render(model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, + shot_refs, ref_image_size, ref_noise_aug, shot_silent) + except (torch.cuda.OutOfMemoryError, RuntimeError) as e: + if _is_oom(e) and getattr(e, "_h3_stage", "") == "sampling": + # Retrying with tiles would re-run the whole sampling pass and + # fail identically. Fail now, and say what actually shrinks it. + raise RuntimeError( + f"H3 Long Videos: shot {i + 1} of {len(gens)} ran out of VRAM " + f"while sampling. " + sampling_oom_help(w, h, ln_i, fps, megapixels) + ) from e + if not _is_oom(e) or tiled: + raise + mm.soft_empty_cache(True); tiled = True; backoff.append(f"shot {i+1}: tiled") + frames, audio, shot_latent = self._render(model, clip, vae, audio_vae, negative, gen_prompt, w, h, ln_i, fps, tiled, sa, shot_handoff, decode_tile_frames, decode_tile_size, + shot_refs, ref_image_size, ref_noise_aug, shot_silent) + + if shot_latent is not None: + latent_chunks.append(shot_latent) + sr = audio["sample_rate"]; wav = audio["waveform"] + + # End the shot `hoff` frames early so the frame handed to the NEXT shot + # isn't the literal last frame (which may catch an open, mid-word mouth + # and make the next shot start "talking"). Drop the matching audio tail + # so this shot's A/V stays aligned. Skipped if the shot is too short. + # ...but ONLY when there IS a next shot. On the final shot the trim hands its + # frames to nobody, so it just deletes the tail of the finished video -- on a + # single-shot run that is the whole point of handoff_offset applied to the one + # thing it cannot help (243f requested came back as 231 frames). + # A dialogue shot handing its last frame to a SILENT shot is the one + # boundary where this matters, and it is where the prompt cannot help. + # The next shot's lips-closed clause is a sentence; its keyframe is a + # PICTURE of an open mouth mid-word, and a picture outvotes a sentence -- + # the same thing that made removed garments come back. So the mouth gets + # a moment to close before the frame is taken, automatically, at exactly + # that transition. 3 frames is ~125ms at 24fps: the tail of a syllable. + # + # Only when the user has not set their own offset, and only speech -> + # silence. Silence -> silence needs nothing, and silence -> speech wants + # the literal last frame so the mouth is already in place. + shot_hoff = hoff + auto_settle = (not hoff and auto_silence_nonspeech and not allow_nonspeech_vocals + and i < len(gens) - 1 and i < len(spk) - 1 + and spk[i] and not spk[i + 1]) + if auto_settle: + shot_hoff = MOUTH_SETTLE_FRAMES + if (i + 1) not in mouth_settled: + mouth_settled.append(i + 1) + if shot_hoff and i < len(gens) - 1 and frames.shape[0] > shot_hoff + 1: + cut = round(shot_hoff * sr / fps) + frames = frames[:-shot_hoff] + if cut: + wav = wav[..., :max(0, wav.shape[-1] - cut)] + + # Keep only a CPU copy of the handoff keyframe (re-encoded next shot), + # and move this shot's decoded video+audio to CPU/RAM immediately so + # they DON'T pile up in VRAM across the chain -- the main long-run OOM. + if cleanup_between_shots: + handoff = frames[-1:].detach().contiguous().to("cpu", copy=True) + else: + handoff = frames[-1:].clone() + if trim_seam and i > 0: + frames = frames[1:]; wav = wav[..., max(0, round(sr / fps)):] + + # Deterministic gibberish fix: a non-dialogue shot is silenced COMPLETELY. + # + # The earlier version faded the first/last `mute_fade_ms` from full + # volume, which left ~20ms of the original audio audible at BOTH ends of + # every muted shot -- on a 10-shot chain that is 20 short bursts of the + # very gibberish the setting exists to remove. The fade belongs on the + # NEIGHBOURING audible shots instead (applied after the loop), not on the + # silent one, so nothing of the muted shot survives. + muted_this_shot = bool(mute_nonspeech_audio and not allow_nonspeech_vocals and i < len(spk) and not spk[i]) + if muted_this_shot: + wav = torch.zeros_like(wav) + muted_flags.append(muted_this_shot) + + if cleanup_between_shots: + # .contiguous() forces a real copy: after trim_seam / handoff_offset + # these are SLICES of the decoded GPU tensor, and a view keeps the + # whole parent allocation alive even after .to("cpu"). Without it the + # previous shot's full decode is pinned while the next shot samples, + # which is the VRAM ratchet across a long chain. + frames_out = frames.detach().contiguous().to("cpu", copy=True) + wav_out = wav.detach().contiguous().to("cpu", copy=True) + video_chunks.append(frames_out); audio_chunks.append(wav_out) + # drop every GPU reference from this shot, then purge VRAM + RAM + del frames, wav, audio, frames_out, wav_out + _deep_cleanup() + else: + video_chunks.append(frames); audio_chunks.append(wav) + mm.soft_empty_cache() + # trace free VRAM after each shot: a falling series means something is + # still accumulating; a flat one means the chain is stable. + vram_trace.append(round(vram_gb()[1], 2)) + + # Fade the EDGES OF AUDIBLE chunks that border a silenced one, so audio does + # not cut to digital silence with a click. The silenced shots stay fully + # silent; only the audible neighbours are ramped. + if mute_nonspeech_audio and any(muted_flags): + fade = max(0, int(sr * int(mute_fade_ms) / 1000)) if sr else 0 + for idx, chunk in enumerate(audio_chunks): + if idx >= len(muted_flags) or muted_flags[idx] or not fade: + continue + n_s = chunk.shape[-1] + if n_s <= 2 * fade: + continue + prev_muted = idx > 0 and muted_flags[idx - 1] + next_muted = idx + 1 < len(muted_flags) and muted_flags[idx + 1] + if prev_muted: + ramp = torch.linspace(0.0, 1.0, fade, device=chunk.device, dtype=chunk.dtype) + chunk[..., :fade] *= ramp + if next_muted: + ramp = torch.linspace(1.0, 0.0, fade, device=chunk.device, dtype=chunk.dtype) + chunk[..., n_s - fade:] *= ramp + + all_frames = torch.cat(video_chunks, dim=0) + all_audio = torch.cat(audio_chunks, dim=-1) + + # --- the `latent` output ------------------------------------------------ + # The sampled latents, joined on the temporal axis. This is NOT the latent + # form of `images`, and the difference is not cosmetic: + # + # * trim_seam and handoff_offset cut DECODED frames. H3 compresses time, + # so one pixel frame is not one latent step and those cuts have no exact + # latent equivalent -- the seam frames trim_seam removes are still here. + # * the overlap fade and any post-pass upscale are pixel-space too. + # + # So decoding this yourself gives a slightly longer video with the seams + # intact. On a SINGLE-shot run none of those apply and it is exact, which is + # the case that matters for testing a latent upscaler. + latent_note = "" + latent_out = {"samples": _empty_av_latent(w, h, 5, fps)[0]["samples"]} + if latent_chunks: + try: + if all(isinstance(c, list) and len(c) == 2 for c in latent_chunks): + vids = [c[0] for c in latent_chunks] + auds = [c[1] for c in latent_chunks] + # A mid-chain resolution backoff makes the shots un-concatenable. + if len({tuple(v.shape[1:2] + v.shape[3:]) for v in vids}) == 1: + latent_out = {"samples": comfy.nested_tensor.NestedTensor( + (torch.cat(vids, dim=2), torch.cat(auds, dim=-1)))} + if len(latent_chunks) > 1: + latent_note = (f" latent: {len(latent_chunks)} shot(s) joined on the " + f"time axis -- PRE-trim, so it holds the seam frames " + f"trim_seam drops from `images` and is longer by " + f"{len(latent_chunks) - 1} frame(s)") + else: + latent_note = " latent: single shot, exact match for `images`" + else: + latent_out = {"samples": comfy.nested_tensor.NestedTensor( + (vids[-1], auds[-1]))} + latent_note = (" latent: shots differ in size after a resolution backoff, " + "so only the LAST shot's latent is emitted") + except Exception as e: + latent_note = f" latent: could not be assembled ({type(e).__name__})" + + # Optional post-pass upscale of the finished frames (safe: any failure + # falls back to lanczos / raw frames and never breaks the render). + up_note = "" + if upscale != "off": + _deep_cleanup() + all_frames, up_note = _upscale_frames(all_frames, upscale, upscale_model, + upscale_target_short_edge, upscale_batch) + + # Text overlays LAST -- after the upscale, so glyphs are rasterized at the + # final pixel size instead of being interpolated up along with the picture. + all_frames, ov_note = _overlay.apply_overlays( + all_frames, fps, watermark_text, watermark_position, watermark_size, + watermark_opacity, watermark_margin, intro_text, intro_seconds, + intro_fade, intro_size, intro_position, overlay_font, overlay_stroke) + + script = "\n---\n".join(gens) + actual = all_frames.shape[0] / fps + uniform_len = len(set(shot_lens)) == 1 + shape_str = (f"{len(gens)} shot(s) x {shot_lens[0]}f (~{shot_lens[0] / fps:.1f}s each) " + f"= ~{sum(shot_lens) / fps:.1f}s" if uniform_len else + f"{len(gens)} shot(s), per-beat " + + ", ".join(f"{n}f/~{n / fps:.1f}s" for n in shot_lens) + + f" = ~{sum(shot_lens) / fps:.1f}s") + vram_str = f"{total_gb:.1f}GB total / {resident_gb:.1f}GB weights / {free_gb:.1f}GB free" if total_gb else "VRAM unknown" + # Say how many shots the trim actually touched: on a single-shot run it is none, + # which explains the frame count instead of leaving it looking like a shortfall. + hoff_str = (f" handoff -{hoff}f on {max(0, len(gens) - 1)} of {len(gens)} shot(s)" + f"{' (last shot keeps its tail)' if len(gens) else ''}." if hoff else "") + # Say what was done about babble on non-dialogue shots, and what it cost. Both + # states need reporting: muting is silent about the ambience it removes, and + # NOT muting is silent about the babble it may leave in. + n_silent = sum(1 for f in spk if not f) + n_muted = sum(1 for f in muted_flags if f) + if n_muted: + audio_note = (f" {n_muted} of {len(gens)} shot(s) have no quoted dialogue and were AUDIO-MUTED " + f"(mute_nonspeech_audio) -- that also removes their generated ambience, so lay an " + f"ambient bed under the video in post, or untick it to keep H3's own") + elif n_silent: + audio_note = (f" {n_silent} of {len(gens)} shot(s) have no quoted dialogue: silenced in the prompt " + f"and soundscape only. If any of them still vocalize, tick mute_nonspeech_audio " + f"for a guaranteed fix") + else: + audio_note = "" + # Which shots actually took the reference channel, and what they gave up for + # it. Silence here would leave "why did shot 2 cut instead of continuing?" + # unanswerable from the output alone. + if ref_missing: + ref_note_missing = (f" named in the prompt " + f"but no image is connected to that ref_image input -- the tag(s) were " + f"dropped from the text") + else: + ref_note_missing = "" + if ref_list and ref_shots: + kept = [n for n in range(1, len(gens) + 1) if n not in ref_shots] + ref_placement = "placed by tags" if tag_driven else f"ref_mode '{ref_mode}'" + ref_note = (f" ref2va: {len(ref_list)} reference image(s) at '{ref_image_size}' on shot(s) " + f"{','.join(str(n) for n in ref_shots)} " + f"({ref_placement})" + + (f", ref_noise_aug {ref_noise_aug:.3f}" if ref_noise_aug is not None + and float(ref_noise_aug) < 0.999 else "") + + (f"; shot(s) {','.join(str(n) for n in kept)} keep the handoff" if kept + else "") + + (f"; shot(s) {','.join(str(n) for n in ref_keyframed)} carry the previous " + f"frame as a real KEYFRAME alongside their references, so they anchor " + f"rather than cut" if ref_keyframed else "") + + (f"; the previous frame rides along as an extra reference on shot(s) " + f"{','.join(str(n) for n in ref_carried)} -- weaker than a keyframe, but " + f"ref_noise_aug below {KEYFRAME_SAFE_AUG:g} would soften a keyframe too " + f"(one aug covers every cond latent)" if ref_carried else "") + + ("" if (ref_keyframed or ref_carried or kept) + else ", so every cut between beats is a CUT, not a continuous take") + + ref_note_missing) + elif ref_list: + ref_note = (f" ref2va: {len(ref_list)} reference image(s) connected but ref_mode " + f"'{ref_mode}' applied them to no shot") + else: + ref_note = "" + info = ((anchor_note + " ") if anchor_note else "") + \ + (f"{shape_str} at {w}x{h}; {all_frames.shape[0]} frames (~{actual:.1f}s actual). " + f"decode {'tiled' if tiled else 'full'}. {vram_str}.{hoff_str}" + + (" DIALOGUE MAY BE CUT OFF -- " + "; ".join(fit_warnings) + + ". Shorten the line, or pick a lower resolution tier to keep the duration." + if fit_warnings else "") + + ((" subject-count guard ON (" + + ("sub-native resolution" if min(w, h) < 768 else "") + + ("; " if min(w, h) < 768 and lora_on else "") + + ("LoRA active -- count front-loaded so it binds before the scene" + if lora_on else "") + + ").") if count_subjects else "") + + ((" " + preflight_txt.strip()) if preflight_txt else "") + + (f" MOUTH -- shot(s) {','.join(str(n) for n in mouth_settled)} were seeded " + f"from a settled mouth ({MOUTH_SETTLE_FRAMES}f before the cut), because the " + f"shot before them ended on dialogue." if mouth_settled else "") + + (f"{latent_note}." if latent_note else "") + + (f" SLA LoRA '{os.path.basename(str(sla_name))}' paired with sparse attention." + if sla_name and sparse_on else "") + + (f" {beats_note}." if beats_note else "") + + (f"{audio_note}." if audio_note else "") + + (" EXPOSURE -- " + "; ".join(wardrobe_notes) + "." + if wardrobe_notes else "") + + (f"{ref_note}." if ref_note else "") + + (f" {fps_note}." if fps_note else "") + + (f" {swap_note}." if swap_note else "") + + (f" free VRAM/shot: {vram_trace}." if len(vram_trace) > 1 else "") + + (f" {accel_note}." if accel_note else "") + + (f" {ms_note}." if ms_note else "") + + (f" {ln_note}." if ln_note else "") + + (f" {up_note}." if up_note else "") + + (f" {ov_note}." if ov_note else "") + + (f" Adjusted: {'; '.join(backoff)}." if backoff else "")) + # frames_per_shot is a single INT for a now-variable series: report the LONGEST + # shot, which is what a downstream consumer must be able to hold. + # `global_soundscape` is the soundscape ACTUALLY used: the derivation above + # reassigns it, so this is the derived bed when auto_soundscape fired and + # your own text when it did not. Emitting it means you can read what was + # generated, and feed it straight back into the widget-input to pin it. + return (all_frames, {"waveform": all_audio, "sample_rate": sr}, info, script, + max(shot_lens), all_frames.shape[0], len(gens), round(actual, 2), + float(fps), int(fps), latent_out, global_soundscape) + + +# REF2VA registers under its OWN key. The FL2VA pack one directory up keeps +# "H3LongVideosFL2VA" and the legacy "H3LongVideosV1" alias; ComfyUI builds one +# flat registry, so repeating either here would silently overwrite that node -- +# same name in the search, no way to tell which copy a workflow is running. +# ONE node, three registration keys. ComfyUI stores the key verbatim in every saved +# workflow, so all three must keep resolving or existing graphs load as red "missing +# node" boxes: "H3LongVideosV1" is the original name, "H3LongVideosFL2VA" the rename, +# and "H3LongVideosREF2VA" the separate reference node that has now been folded in. +# They are aliases onto the same class -- there is no second implementation. +NODE_CLASS_MAPPINGS = { + "DumasH3LongVideos": H3LongVideos, + "DumasH3LongVideosFL2VA": H3LongVideos, + "DumasH3LongVideosREF2VA": H3LongVideos, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "DumasH3LongVideos": "Dumas H3 Long Videos (FL2VA + REF2VA)", + "DumasH3LongVideosFL2VA": "Dumas H3 Long Videos (FL2VA + REF2VA)", + "DumasH3LongVideosREF2VA": "Dumas H3 Long Videos (FL2VA + REF2VA)", +} +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/dumas_h3_overlay.py b/dumas_h3_overlay.py new file mode 100644 index 0000000..44df13b --- /dev/null +++ b/dumas_h3_overlay.py @@ -0,0 +1,255 @@ +""" +PIL text overlays for H3 Long Videos -- watermark and intro title. + +Text is COMPOSITED onto the decoded frames, never asked of the model. H3 (like +every video diffusion model) renders text as plausible-looking letterforms that +drift, warp and re-spell themselves frame to frame; a watermark that changes +shape every frame is worse than none. Compositing gives pixel-identical text on +every frame at zero sampling cost, and keeps the words out of the prompt where +they would otherwise steal conditioning from the actual shot. + +Both overlays are WHITE text drawn on a fully transparent RGBA layer, then +alpha-blended over the video -- so only the glyphs themselves land on the frame +and the picture shows through everywhere else. + +Everything here is best-effort: any failure returns the frames untouched with a +note, because a cosmetic overlay must never lose a finished render. +""" + +import torch + +BLEND_CHUNK = 64 # frames blended per slice -- bounds peak RAM on long chains + +# Auto-fit: text is wrapped, then shrunk in FIT_SHRINK steps until the block fits +# inside the margins. MIN_FONT_PX is the point below which the text would be +# unreadable anyway, so the loop stops there and lets PIL clip rather than spin. +MIN_FONT_PX = 8 +FIT_SHRINK = 0.92 +FIT_STEPS = 48 + +# Fonts to try when the requested one cannot be loaded. PIL resolves bare names +# against the system font directory, so "arial.ttf" works on Windows as-is. +FONT_FALLBACKS = ("arial.ttf", "segoeui.ttf", "DejaVuSans.ttf", "LiberationSans-Regular.ttf") + +# Anchor -> (x, y) as a fraction of the free space: 0 = hard against the left/top +# margin, 1 = hard against the right/bottom, 0.5 = centered. +POSITIONS = { + "bottom-right": (1.0, 1.0), + "bottom-left": (0.0, 1.0), + "bottom-center": (0.5, 1.0), + "top-right": (1.0, 0.0), + "top-left": (0.0, 0.0), + "top-center": (0.5, 0.0), + "center": (0.5, 0.5), + "lower-third": (0.5, 0.72), +} + + +def _load_font(name, px): + """A truetype font at px, falling back through the known-present faces and + finally to PIL's bitmap default (which ignores size -- ugly, but never fatal).""" + from PIL import ImageFont + px = max(8, int(px)) + for cand in ([name] if name else []) + list(FONT_FALLBACKS): + try: + return ImageFont.truetype(cand, px) + except Exception: + continue + return ImageFont.load_default() + + +def _measure(draw, text, font, stroke_px, spacing): + """(x0, y0, x1, y1) of a multi-line block, tolerant of older Pillow builds.""" + try: + return draw.multiline_textbbox((0, 0), text, font=font, align="center", + stroke_width=stroke_px, spacing=spacing) + except TypeError: # older Pillow: no stroke/spacing kwargs + return draw.multiline_textbbox((0, 0), text, font=font, align="center") + + +def _wrap(draw, text, font, max_w, stroke_px, spacing): + """Greedy word-wrap every hard line to max_w. A single word wider than the + frame cannot be broken -- the shrink loop in render_text_layer handles that.""" + out = [] + for hard in text.split("\n"): + words = hard.split() + if not words: + out.append("") + continue + cur = words[0] + for wd in words[1:]: + trial = cur + " " + wd + b = _measure(draw, trial, font, stroke_px, spacing) + if b[2] - b[0] <= max_w: + cur = trial + else: + out.append(cur) + cur = wd + out.append(cur) + return "\n".join(out) + + +def _fit(draw, text, font_name, px, max_w, max_h, stroke_px, line_spacing, wrap=True): + """Largest size at or below px whose wrapped block fits (max_w, max_h). + + Without this, a title is drawn at the requested size and whatever runs past the + frame is simply CLIPPED by PIL -- silently, with no error and no note. That is + the whole "overlays don't work at other resolutions" failure: the size is a + percentage, so the same text that fits 1344x768 overflows a 512-wide portrait + canvas and loses its outer characters.""" + px = max(MIN_FONT_PX, int(px)) + for _ in range(FIT_STEPS): + font = _load_font(font_name, px) + spacing = int(max(0.0, px * (line_spacing - 1.0))) + fitted = _wrap(draw, text, font, max_w, stroke_px, spacing) if wrap else text + box = _measure(draw, fitted, font, stroke_px, spacing) + if (box[2] - box[0] <= max_w and box[3] - box[1] <= max_h) or px <= MIN_FONT_PX: + return font, fitted, box, spacing, px + px = max(MIN_FONT_PX, int(px * FIT_SHRINK)) + return font, fitted, box, spacing, px + + +def render_text_layer(width, height, text, font_px, position="bottom-right", + margin_pct=3.0, font_name="", stroke_px=0, line_spacing=1.15, + wrap=True): + """White text on a transparent RGBA canvas the size of one frame. + + The block is WRAPPED and SHRUNK until it fits inside the margins, so the same + settings render legibly on every supported preset -- portrait canvases and the + 512 tier included -- instead of being clipped at the frame edge. + + Returns (rgb, alpha, bbox): rgb [H,W,3] float 0..1, alpha [H,W,1] float 0..1 + (zero everywhere except the glyphs and their optional stroke), and the tight + (x0, y0, x1, y1) box of non-transparent pixels so the blend only has to touch + the region the text actually occupies. None when there is nothing to draw.""" + from PIL import Image, ImageDraw + import numpy as np + + text = (text or "").strip() + if not text: + return None + + img = Image.new("RGBA", (int(width), int(height)), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + stroke_px = max(0, int(stroke_px)) + + margin = int(min(width, height) * max(0.0, margin_pct) / 100.0) + ax, ay = POSITIONS.get(position, POSITIONS["bottom-right"]) + + # Measure first, so the block is placed by its real size rather than a guess -- + # and fit it to the space the margins actually leave. + max_w = max(1, int(width) - 2 * margin) + max_h = max(1, int(height) - 2 * margin) + font, text, box, spacing, font_px = _fit(draw, text, font_name, font_px, max_w, max_h, + stroke_px, line_spacing, wrap) + tw, th = box[2] - box[0], box[3] - box[1] + + free_w = max(0, int(width) - 2 * margin - tw) + free_h = max(0, int(height) - 2 * margin - th) + x = margin + free_w * ax - box[0] + y = margin + free_h * ay - box[1] + + kwargs = dict(font=font, fill=(255, 255, 255, 255), align="center") + if stroke_px: + kwargs.update(stroke_width=stroke_px, stroke_fill=(0, 0, 0, 255)) + try: + draw.multiline_text((x, y), text, spacing=spacing, **kwargs) + except TypeError: + draw.multiline_text((x, y), text, **kwargs) + + arr = np.asarray(img, dtype=np.float32) / 255.0 # [H, W, 4] + alpha = arr[..., 3:4] + if not alpha.any(): + return None + # Tight bbox of drawn pixels: blending a whole 1344x768 frame for a corner + # watermark would cost ~50x more work on a 3000-frame chain. + ys, xs = np.nonzero(alpha[..., 0] > 0.0) + bbox = (int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1) + return (torch.from_numpy(arr[..., :3].copy()), + torch.from_numpy(alpha.copy()), + bbox) + + +def blend_layer(frames, layer, frame_alpha=None, opacity=1.0): + """Alpha-composite a rendered layer over frames [N,H,W,3] in 0..1, in place. + + frame_alpha is an optional per-frame multiplier (length N) -- that is what + makes an intro title hold and then fade instead of sitting on the whole + video. Frames whose multiplier is 0 are skipped entirely.""" + if layer is None: + return frames + rgb, alpha, (x0, y0, x1, y1) = layer + n = frames.shape[0] + if frame_alpha is None: + frame_alpha = torch.ones(n, dtype=torch.float32) + frame_alpha = frame_alpha.to(torch.float32).clamp(0.0, 1.0) * float(opacity) + + a_crop = alpha[y0:y1, x0:x1, :].to(frames.dtype) + c_crop = rgb[y0:y1, x0:x1, :].to(frames.dtype) + live = (frame_alpha > 0).nonzero().flatten().tolist() + for s in range(0, len(live), BLEND_CHUNK): + idx = live[s:s + BLEND_CHUNK] + fa = frame_alpha[idx].to(frames.dtype).view(-1, 1, 1, 1) + sub = frames[idx, y0:y1, x0:x1, :] + a = a_crop * fa + frames[idx, y0:y1, x0:x1, :] = sub * (1.0 - a) + c_crop * a + return frames + + +def hold_fade_alpha(total_frames, hold_frames, fade_frames): + """Per-frame opacity for an intro: full through hold_frames, then a linear + ramp to zero over fade_frames, then nothing. Returns a length-N tensor.""" + a = torch.zeros(int(total_frames), dtype=torch.float32) + hold = max(0, min(int(hold_frames), int(total_frames))) + a[:hold] = 1.0 + fade = max(0, min(int(fade_frames), int(total_frames) - hold)) + if fade: + a[hold:hold + fade] = torch.linspace(1.0, 0.0, fade + 2)[1:-1] + return a + + +def apply_overlays(frames, fps, watermark="", wm_position="bottom-right", wm_size_pct=4.0, + wm_opacity=0.75, wm_margin_pct=3.0, intro="", intro_seconds=3.0, + intro_fade=0.6, intro_size_pct=9.0, intro_position="center", + font_name="", stroke_px=0): + """Composite the watermark (every frame) and the intro title (first seconds + only, then faded out). Returns (frames, note). Never raises -- a cosmetic + overlay must not be able to destroy a finished render.""" + notes = [] + if frames is None or frames.ndim != 4 or frames.shape[0] == 0: + return frames, "" + n, h, w = frames.shape[0], frames.shape[1], frames.shape[2] + frames = frames.contiguous() + # Size from the SHORT edge, not the height. Height is the long edge on every + # portrait preset, so a height-based percentage drew 9:16 text ~1.75x larger + # than the same setting at 16:9 -- on the canvas with the LEAST room for it. + # The short edge makes one setting mean the same apparent size at every ratio. + short = min(int(w), int(h)) + + if (watermark or "").strip(): + try: + layer = render_text_layer(w, h, watermark, short * max(0.5, wm_size_pct) / 100.0, + wm_position, wm_margin_pct, font_name, stroke_px) + if layer is not None: + blend_layer(frames, layer, None, wm_opacity) + notes.append(f"watermark composited ({wm_position}, {wm_opacity:.0%})") + except Exception as e: + notes.append(f"watermark skipped ({type(e).__name__}: {e})") + + if (intro or "").strip(): + try: + layer = render_text_layer(w, h, intro, short * max(0.5, intro_size_pct) / 100.0, + intro_position, 6.0, font_name, stroke_px) + if layer is not None: + hold = round(max(0.0, float(intro_seconds)) * fps) + fade = round(max(0.0, float(intro_fade)) * fps) + fa = hold_fade_alpha(n, hold, fade) + if fa.max() > 0: + blend_layer(frames, layer, fa, 1.0) + notes.append(f"intro title composited ({hold}f hold + {fade}f fade)") + else: + notes.append("intro title skipped (no hold or fade frames)") + except Exception as e: + notes.append(f"intro title skipped ({type(e).__name__}: {e})") + + return frames, "; ".join(notes) diff --git a/dumas_h3_shot_length.py b/dumas_h3_shot_length.py new file mode 100644 index 0000000..18af5c7 --- /dev/null +++ b/dumas_h3_shot_length.py @@ -0,0 +1,66 @@ +""" +H3 Shot Length (single model-free source for shot length) +========================================================== +Holds ONE shot-length value and emits it as both seconds and a grid-aligned +H3 frame count. Because it never reads the model, it can sit upstream of +Kijai's Model Preview Override without creating a cycle -- unlike the preview +node or the sampler, which read the model to compute their frame counts and so +cannot feed anything that produces the model. + +Wire: + H3 Shot Length (seconds) -> H3 Long Videos FL2VA (shot_seconds) + H3 Shot Length (frames) -> Model Preview Override (preview_frames) + +One value, entered once here, drives both -- no manual re-entry, no cycle. + +Note: this is a FIXED shot length you choose. The sampler's *auto* (VRAM- +picked) length can't be used for preview_frames, because computing it requires +reading the model, which is the very dependency that creates the loop. Set the +sampler's shot_seconds from this node's `seconds` output so the two agree. +""" + +H3_MAX_FRAMES = 362 + + +def align_up_grid(n): + n = max(5, int(n)) + while n % 17 != 5: + n += 1 + return n + + +class H3ShotLength: + CATEGORY = "Dumas/MiniMax" + FUNCTION = "emit" + RETURN_TYPES = ("FLOAT", "INT", "STRING") + RETURN_NAMES = ("seconds", "frames", "info") + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "shot_seconds": ("FLOAT", {"default": 5.0, "min": 0.2, "max": 15.1, "step": 0.5, + "tooltip": "Length of each shot. Feeds the sampler's shot_seconds AND (as frames) " + "the preview override. Max ~15s (362 frames)."}), + "fps": ("INT", {"default": 24, "min": 1, "max": 60}), + }, + "optional": { + "cap_to_h3_max": ("BOOLEAN", {"default": True, + "tooltip": "Clamp frames to 362 (~15s), H3's single-clip maximum."}), + }, + } + + def emit(self, shot_seconds, fps, cap_to_h3_max=True): + fps = max(1, int(fps)) + frames = align_up_grid(round(float(shot_seconds) * fps)) + capped = cap_to_h3_max and frames > H3_MAX_FRAMES + if capped: + frames = H3_MAX_FRAMES + info = (f"{shot_seconds:g}s/shot @ {fps}fps -> {frames} frames" + f"{' (capped 362)' if capped else ''}") + return (round(float(shot_seconds), 3), frames, info) + + +NODE_CLASS_MAPPINGS = {"DumasH3ShotLength": H3ShotLength} +NODE_DISPLAY_NAME_MAPPINGS = {"DumasH3ShotLength": "Dumas H3 Shot Length"} +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]