Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bfaa5fa733 | ||
|
|
9923a3e417 | ||
|
|
ea6fae7e56 |
@@ -35,20 +35,23 @@
|
||||
|
||||
- `Dumas H3 Long Videos`
|
||||
- Inputs/outputs: the current upstream `MiniMax-H3-Longvideos` sampler surface, exposed under the existing `DumasH3LongVideos` key for saved Dumas workflows.
|
||||
- The local Dumas prompt-engineering fork has been removed from this node. Long Videos now wraps the upstream sampler/engine directly so it can track the source project again.
|
||||
- The local Dumas prompt-engineering fork has been removed from this node. Long Videos now wraps the upstream sampler/engine/runtime/audio/conditioning/shot-plan modules directly so it can track the source project again.
|
||||
- Upstream compatibility keys `H3LongVideos`, `H3LongVideosFL2VA`, `H3LongVideosV1`, and `H3LongVideosREF2VA` are also registered to the same class.
|
||||
- The old Dumas browser widget grouping script is disabled for this node because it targeted controls that no longer exist on the upstream sampler.
|
||||
- `handoff_frames` extends the upstream last-frame handoff: `1` keeps the current single keyframe behavior; higher values keep that final-frame keyframe and add earlier tail frames from the previous shot as claimed reference context for the next beat.
|
||||
- Upstream license text is included in [`H3_LONGVIDEOS_UPSTREAM_LICENSE.txt`](./H3_LONGVIDEOS_UPSTREAM_LICENSE.txt).
|
||||
|
||||
- `Dumas H3 Latent Upscale Params`
|
||||
- Inputs: `mode`, `model_name`, `method`, `width`, `height`, `device`, `precision`, `sampler_name`, `scheduler`, `steps`, `denoise`, `megapixels`, `tile_width`, `tile_height`, `overlap`, `fade_width`, `fade_height`, `overlap_mode`, `overlap_blend`, `tile_size_mode`, `grid_rows`, `grid_cols`, `spatial_w_overlap`, `spatial_h_overlap`, `min_tile_size`, `masked_area_noise`, `brightness_match`, `dynamic_fade`, `dynamic_fade_min`, `chunk_length`, `temporal_overlap`, `resize_conditioning`, `anchor_strength`
|
||||
- Output: `latent_upscale_param`
|
||||
- Bundles the optional latent-space upscaler settings used by `Dumas H3 Long Videos` before decode, so the main node can rebuild conditioning at the target size and run a short refinement pass with your chosen sampler, scheduler, step count, denoise, and the full upstream spatial split controls.
|
||||
- Legacy helper from the abandoned Dumas Long Videos fork. The current upstream-backed `Dumas H3 Long Videos` node does not consume this socket; it uses the upstream latent-upscale controls on the Long Videos node itself.
|
||||
|
||||
- `Dumas H3 Beat Prompt`
|
||||
- Inputs: authored through the custom front-end beat editor
|
||||
- Output: `prompt`
|
||||
- Builds one H3 prompt block per beat, with quick controls for per-shot timing, continuity, ref behavior, anchor additions, soundscape, and music while staying compatible with direct text editing.
|
||||
- Builds an upstream-compatible Long Videos prompt: optional scene paragraph, optional character sheet, then one blank-line-separated textbox per beat.
|
||||
- Per-beat helpers only emit upstream-supported state directives: `remove:` / `removed:` / `off:` and `add:` / `wear:` / `wearing:`.
|
||||
- Old Dumas-only beat directives such as `seconds:`, `continuity:`, `ref_mode:`, `ref_noise_aug:`, `anchor_add:`, `soundscape:`, and `music:` are stripped from the generated prompt so they are not sent to the upstream node as visible text.
|
||||
|
||||
- `Dumas H3 Prompt Curator`
|
||||
- Inputs: `action_prompt`, `anatomy_guard`, `subject_count_guard`, optional `anchor`, optional `soundscape`, optional `bgm`, optional `ref_1` through `ref_9`
|
||||
|
||||
+55
-8
@@ -2,11 +2,46 @@ import json
|
||||
|
||||
|
||||
_DEFAULT_BEAT = "Describe this beat."
|
||||
_DEFAULT_STATE = {"beats": [{"text": _DEFAULT_BEAT}]}
|
||||
_DEFAULT_STATE = {"scene": "", "character_sheet": "", "beats": [{"text": _DEFAULT_BEAT}]}
|
||||
_LEGACY_DIRECTIVE_PREFIXES = (
|
||||
"seconds",
|
||||
"duration",
|
||||
"continuity",
|
||||
"ref_mode",
|
||||
"ref_noise_aug",
|
||||
"anchor_add",
|
||||
"overall_soundscape",
|
||||
"soundscape",
|
||||
"non_diegetic_music",
|
||||
"music",
|
||||
"wardrobe",
|
||||
"enter",
|
||||
"exit",
|
||||
)
|
||||
|
||||
|
||||
def _clone_default_state():
|
||||
return {"beats": [{"text": _DEFAULT_BEAT}]}
|
||||
return {
|
||||
"scene": "",
|
||||
"character_sheet": "",
|
||||
"beats": [{"text": _DEFAULT_BEAT}],
|
||||
}
|
||||
|
||||
|
||||
def _strip_legacy_directives(text):
|
||||
"""Remove directives from the abandoned Dumas Long Videos fork.
|
||||
|
||||
The upstream Long Videos node sends unknown field labels to the model as text,
|
||||
so this builder strips the old managed controls rather than emitting prompts
|
||||
that ask H3 to draw labels such as "seconds:" or "music:" in the frame.
|
||||
"""
|
||||
kept = []
|
||||
for line in str(text or "").splitlines():
|
||||
lowered = line.strip().lower()
|
||||
if any(lowered.startswith(f"{name}:") for name in _LEGACY_DIRECTIVE_PREFIXES):
|
||||
continue
|
||||
kept.append(line)
|
||||
return "\n".join(kept).strip()
|
||||
|
||||
|
||||
def _parse_beat_prompt_state(value):
|
||||
@@ -21,6 +56,8 @@ def _parse_beat_prompt_state(value):
|
||||
except Exception:
|
||||
return _clone_default_state()
|
||||
|
||||
scene = str(raw.get("scene") or "")
|
||||
character_sheet = str(raw.get("character_sheet") or "")
|
||||
beats = []
|
||||
for item in list(raw.get("beats") or []):
|
||||
if isinstance(item, dict):
|
||||
@@ -30,15 +67,25 @@ def _parse_beat_prompt_state(value):
|
||||
beats.append({"text": text})
|
||||
|
||||
if not beats:
|
||||
return _clone_default_state()
|
||||
return {"beats": beats}
|
||||
beats = [{"text": _DEFAULT_BEAT}]
|
||||
return {
|
||||
"scene": scene,
|
||||
"character_sheet": character_sheet,
|
||||
"beats": beats,
|
||||
}
|
||||
|
||||
|
||||
def _assemble_beat_prompt(state):
|
||||
parsed = _parse_beat_prompt_state(state)
|
||||
chunks = []
|
||||
scene = str(parsed.get("scene") or "").strip()
|
||||
if scene:
|
||||
chunks.append(scene)
|
||||
character_sheet = str(parsed.get("character_sheet") or "").strip()
|
||||
if character_sheet:
|
||||
chunks.append(character_sheet)
|
||||
for beat in parsed["beats"]:
|
||||
text = str(beat.get("text") or "").strip()
|
||||
text = _strip_legacy_directives(beat.get("text") or "")
|
||||
if text:
|
||||
chunks.append(text)
|
||||
return "\n\n".join(chunks)
|
||||
@@ -46,9 +93,9 @@ def _assemble_beat_prompt(state):
|
||||
|
||||
class DumasH3BeatPromptNode:
|
||||
DESCRIPTION = (
|
||||
"Build a MiniMax H3 prompt from one textbox per beat, with a front-end beat "
|
||||
"editor that can append directive examples and expose per-shot controls for "
|
||||
"timing, continuity, ref behavior, anchor additions, soundscape, and music."
|
||||
"Build an upstream MiniMax H3 Long Videos prompt: optional scene paragraph, "
|
||||
"optional character sheet, then one blank-line-separated textbox per beat. "
|
||||
"Per-beat helpers only emit directives the upstream node understands."
|
||||
)
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("prompt",)
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos
|
||||
# Copyright (c) 2026 Smite79. All rights reserved.
|
||||
# Redistribution, in whole or in part, requires written permission.
|
||||
# This notice may not be removed or altered. See LICENSE.
|
||||
"""Audio policy shared by conditioning and soundtrack assembly."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import comfy.nested_tensor
|
||||
from h3_runtime import temporal_shape
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ShotAudio:
|
||||
speech: bool
|
||||
sounded: bool
|
||||
voiced_only: bool
|
||||
silence_enabled: bool
|
||||
lead_seconds: float
|
||||
latent_fps: int
|
||||
# The tail. Everything after the line's expected end is pinned the way the lead
|
||||
# pins everything before its start. All three default off, so a ShotAudio built
|
||||
# the old way -- six positional arguments -- behaves exactly the old way.
|
||||
line_seconds: float = 0.0 # planner's estimate of the spoken line
|
||||
tail_seconds: float = 0.0 # free audio kept after that estimate; 0 = no tail pin
|
||||
frame_count: int = 0 # the shot in pixel frames; the audio T comes from it
|
||||
|
||||
@property
|
||||
def pinned(self):
|
||||
return self.silence_enabled and not self.speech and not self.sounded
|
||||
|
||||
@property
|
||||
def lead_frames(self):
|
||||
if not self.speech or self.lead_seconds <= 0:
|
||||
return 0
|
||||
return round(self.lead_seconds * self.latent_fps)
|
||||
|
||||
@property
|
||||
def tail_frames(self):
|
||||
"""Audio latent frames pinned at the END of a dialogue shot.
|
||||
|
||||
The lead pins the opening so the line cannot start early; nothing pinned the
|
||||
close, and a 2s line in a 9s shot left 7s of open branch in a shot the model
|
||||
knows has a voice in it -- which is where speech carries on past the line, or
|
||||
doubles it. The free span is lead + the line's estimate + tail_seconds; the
|
||||
rest is held at encoded silence. The model chooses WHEN to speak, so the
|
||||
margin is the author's dial: a clipped word costs more than a second of babble.
|
||||
Off unless the shot speaks, the margin is set, and at least half a second would
|
||||
be pinned -- a sliver is not worth the risk of clipping."""
|
||||
if (not self.speech or self.tail_seconds <= 0 or self.line_seconds <= 0
|
||||
or self.frame_count <= 0):
|
||||
return 0
|
||||
total = temporal_shape(self.frame_count)[2]
|
||||
free = self.lead_frames + round((self.line_seconds + self.tail_seconds) * self.latent_fps)
|
||||
tail = total - free
|
||||
return tail if tail >= round(0.5 * self.latent_fps) else 0
|
||||
|
||||
|
||||
_SILENT_UNIT = {"lat": None, "key": None}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NOTHING HERE IS SYNTHESISED ANY MORE. Removed on the report: "Just get rid of
|
||||
# the ambient sounds all together. They sound horrid. Go back to the model's
|
||||
# natural audio."
|
||||
#
|
||||
# What was here built the soundtrack's non-vocal half out of shaped noise: a room
|
||||
# tone from the scene's own wording (synth_ambient, over a table of recipes, with
|
||||
# plain_bed under it as a floor) and 21 foley recipes laid into the shots whose
|
||||
# audio branch is pinned to silence (foley_for, over _hits/_band/_room and later
|
||||
# _contact/_flow/_creak, timed off the picture's own movement for footsteps).
|
||||
#
|
||||
# It went in because a shot pinned to silence cannot get audio from the model at
|
||||
# all -- prompt text never opens a branch -- so auto_sound was writing sounds into
|
||||
# prompts that could not make them. That reasoning was sound and the thing it built
|
||||
# still did not pass: reported first as footsteps sounding like heartbeats and a
|
||||
# bathroom that tapped, and then, once both of those measured clean, as horrid
|
||||
# anyway. Synthesis that measures right and sounds wrong is the end of that road.
|
||||
#
|
||||
# So the audio is the model's, whole. H3 is a joint model and the audio branch is
|
||||
# where its sound comes from; the prompt still describes what a shot sounds like,
|
||||
# which is the half that was always doing the real work.
|
||||
#
|
||||
# The consequence, which is real and is reported in info rather than left to be
|
||||
# discovered: a shot with no line and no sound you wrote is pinned to silence and
|
||||
# is now SILENT. The pin is not a bug and is deliberately untouched -- it is what
|
||||
# stops a free branch filling itself with babble and a face lip-syncing to it.
|
||||
# Write the sound into the beat to open the branch on purpose, or wire a recording
|
||||
# to ambient_audio, which is played under the finished track and conditions
|
||||
# nothing. mix_ambient below is that path, and it is all that is left here.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _seamless_loop(x, n, sr):
|
||||
"""[C, M] -> [C, n], looped with a crossfade so the join does not click.
|
||||
|
||||
Plain tiling puts a discontinuity at every repeat, once per loop length. In a
|
||||
bed that is meant to sit under everything unnoticed, a regular click is the one
|
||||
thing that gets noticed -- the same objection that made the silence latent
|
||||
ping-pong its interior rather than tile it. Here the material is real audio
|
||||
being PLAYED rather than a latent being conditioned on, so it cannot be
|
||||
reversed: a room tone read backwards is fine, but footsteps are not. Crossfade
|
||||
instead, which works on both."""
|
||||
m = int(x.shape[-1])
|
||||
if m <= 0:
|
||||
return None
|
||||
if m >= n:
|
||||
return x[..., :n]
|
||||
fade = min(int(0.25 * sr), m // 4)
|
||||
if fade < 1:
|
||||
reps = -(-n // m)
|
||||
return x.repeat(1, reps)[..., :n]
|
||||
# OVERLAP-ADD the tail onto the head, and shorten the unit by the overlap. The
|
||||
# unit then runs x[m-fade] .. x[m-fade-1], so tiling it steps between samples
|
||||
# that were adjacent in the source and there is no discontinuity anywhere.
|
||||
#
|
||||
# Measured, because the obvious construction is wrong: appending the crossfade
|
||||
# to the END of a full-length unit leaves it finishing on x[fade-1] while the
|
||||
# next repeat starts on x[0], which are not adjacent -- a 2s tone that does not
|
||||
# divide evenly gave a 64x jump at the join, worse than plain tiling's 41x.
|
||||
t = torch.linspace(0.0, 1.0, fade, dtype=x.dtype, device=x.device)
|
||||
head = x[..., :fade] * t + x[..., m - fade:] * (1.0 - t)
|
||||
unit = torch.cat([head, x[..., fade:m - fade]], dim=-1)
|
||||
if int(unit.shape[-1]) < 1:
|
||||
reps = -(-n // m)
|
||||
return x.repeat(1, reps)[..., :n]
|
||||
reps = -(-n // int(unit.shape[-1]))
|
||||
return unit.repeat(1, reps)[..., :n]
|
||||
|
||||
|
||||
def mix_ambient(audio, sr, bed, level):
|
||||
"""Lay an ambient bed UNDER a finished soundtrack. -> (waveform, note).
|
||||
|
||||
The bed is PLAYED, not conditioned on: it is the file, at the level asked for,
|
||||
under whatever the model generated. That is the whole reason to do it here
|
||||
rather than in the sampler -- ambience needs no cooperation from a joint model,
|
||||
has nothing to lip-sync to, and so cannot put a voice in a wordless shot. The
|
||||
conditioning path can only steer the branch toward something bed-LIKE, and on a
|
||||
shot with a line it competes with the line.
|
||||
|
||||
Defensive throughout, like the silence latent: any failure returns the audio
|
||||
untouched with a note saying so, because a bed is a nicety and a render is not.
|
||||
"""
|
||||
try:
|
||||
if audio is None or bed is None or float(level or 0.0) <= 0.0:
|
||||
return audio, ""
|
||||
w = bed.get("waveform") if isinstance(bed, dict) else None
|
||||
if w is None or not int(getattr(w, "ndim", 0)):
|
||||
return audio, ("ambient_audio is wired but carries no waveform, so nothing "
|
||||
"was laid under the soundtrack")
|
||||
w = w[0] if w.dim() == 3 else w # [B, C, M] -> [C, M]
|
||||
if w.dim() != 2 or w.shape[-1] < 2:
|
||||
return audio, ("ambient_audio is too short to loop, so nothing was laid "
|
||||
"under the soundtrack")
|
||||
w = w.detach().to(dtype=audio.dtype, device=audio.device)
|
||||
b_sr = int((bed.get("sample_rate") if isinstance(bed, dict) else 0) or 0)
|
||||
# RESAMPLE, or the bed plays at the wrong speed and pitch. Linear is coarse
|
||||
# for music and inaudible on a room tone, which is what this input is for.
|
||||
resampled = ""
|
||||
if b_sr > 0 and b_sr != int(sr):
|
||||
want = max(2, int(round(w.shape[-1] * float(sr) / float(b_sr))))
|
||||
w = torch.nn.functional.interpolate(
|
||||
w.unsqueeze(0), size=want, mode="linear", align_corners=False)[0]
|
||||
resampled = f", resampled from {b_sr} Hz"
|
||||
ch = int(audio.shape[1])
|
||||
if int(w.shape[0]) != ch:
|
||||
w = (w.mean(dim=0, keepdim=True).repeat(ch, 1) if int(w.shape[0]) > ch
|
||||
else w[:1].repeat(ch, 1))
|
||||
n = int(audio.shape[-1])
|
||||
loop = _seamless_loop(w, n, int(sr))
|
||||
if loop is None:
|
||||
return audio, ""
|
||||
out = audio + loop.unsqueeze(0) * float(level)
|
||||
# NORMALISE rather than clip. Clipping a bed that pushed a loud line over
|
||||
# the top distorts the LINE, which is the thing worth keeping.
|
||||
peak = float(out.abs().max())
|
||||
gain = ""
|
||||
if peak > 1.0:
|
||||
out = out / peak
|
||||
gain = f", and the mix was scaled by {1.0 / peak:.2f} to stop it clipping"
|
||||
secs = w.shape[-1] / float(sr)
|
||||
return out, (f"an ambient bed was laid under the whole soundtrack at level "
|
||||
f"{float(level):.2f} -- {secs:.1f}s of audio{resampled}, looped "
|
||||
f"with a crossfade so the join does not click{gain}. It is your "
|
||||
f"file, played under what the model generated: it conditions "
|
||||
f"nothing, so it cannot put a voice in a wordless shot the way "
|
||||
f"an inferred bed did. Shots pinned to silence keep their silent "
|
||||
f"conditioning and get the bed on top, which is what makes a "
|
||||
f"wordless shot sound like a room instead of a mute")
|
||||
except Exception as exc:
|
||||
return audio, (f"the ambient bed could not be mixed ({type(exc).__name__}), so "
|
||||
f"the soundtrack is unchanged")
|
||||
|
||||
|
||||
_SILENCE_STATUS = {"asked": 0, "applied": 0, "why": ""}
|
||||
|
||||
|
||||
_SILENT_SECONDS = 2
|
||||
|
||||
|
||||
_SILENT_EDGE = 4
|
||||
|
||||
|
||||
def _silent_audio_latent(audio_vae, frame_count, fps):
|
||||
"""A keyframe audio latent of actual SILENCE, or None if it cannot be made.
|
||||
|
||||
H3 is a JOINT model: the mouth follows the audio branch. On a shot with no
|
||||
scripted line the branch is otherwise unconditioned, and an unconditioned audio
|
||||
branch invents a voice -- which the picture then lip-syncs to. The lips-closed
|
||||
sentence is arguing with a stream that has already decided someone is talking.
|
||||
|
||||
REBUILT 2026-09-05, from measurements against the real VAE rather than from
|
||||
reasoning. The previous version encoded one second, kept a SINGLE interior
|
||||
frame and repeated it, on the argument that silence is homogeneous. It is not,
|
||||
in latent space: encoded silence has genuine frame-to-frame variation (delta
|
||||
mean 0.002-0.004, max 0.021), and a repeated frame has a delta of exactly
|
||||
0.000000. That is a flat signal no encoder produces, and a model handed
|
||||
conditioning outside its own distribution has every reason to disregard it --
|
||||
which is an audio branch back to inventing a voice, with the report saying
|
||||
silence went on.
|
||||
|
||||
The fix that version was avoiding is real too: tiling the whole encoded second
|
||||
end to end leaves a 25x spike at each join (0.554 against 0.022), once per
|
||||
second, which is a metronome in the conditioning of a joint model.
|
||||
|
||||
So: encode two seconds, drop the padded ends, and PING-PONG the interior --
|
||||
forward, reversed, forward. Every join repeats a frame, so there is no seam,
|
||||
and the interior statistics are the encoder's own. Measured over a 9s shot:
|
||||
|
||||
one frame repeated peak 0.000686 delta mean 0.000000 max 0.000000
|
||||
whole 2s tiled peak 0.000314 delta mean 0.017451 max 0.554715
|
||||
interior ping-pong peak 0.000566 delta mean 0.002039 max 0.021159
|
||||
|
||||
where the encoder's own interior is mean 0.0021, max 0.0212. Decoded peak
|
||||
0.000566 on a +/-1.0 scale is about -65 dBFS: silence.
|
||||
|
||||
Everything here stays defensive. Shapes are CHECKED against what the layout
|
||||
expects rather than assumed, and any failure returns None so the shot falls
|
||||
back to an unconditioned branch instead of breaking the render -- the caller
|
||||
reports when that happens, so it is no longer a silent failure.
|
||||
"""
|
||||
try:
|
||||
sr = int(getattr(audio_vae, "audio_sample_rate", 0) or 0)
|
||||
if sr <= 0:
|
||||
return None
|
||||
_, _, want_t = temporal_shape(frame_count, fps)
|
||||
if want_t <= 0:
|
||||
return None
|
||||
key = (id(audio_vae), sr)
|
||||
block = _SILENT_UNIT.get("lat") if _SILENT_UNIT.get("key") == key else None
|
||||
if block is None:
|
||||
# CHANNELS LAST. comfy.sd.VAE.encode() does `pixel_samples.movedim(-1, 1)`
|
||||
# before handing off, so the audio VAE -- which wants [B, 2, L] -- must be
|
||||
# given [B, L, 2]. Passing [B, 2, L] raises inside the encoder, and an
|
||||
# early version did exactly that: swallowed by the guard below, so the
|
||||
# whole layer silently did nothing.
|
||||
#
|
||||
# Two seconds, encoded ONCE and cached. Encoding a full 15s shot instead
|
||||
# cost a VAE pass big enough to OOM mid-render on a 16GB card, where the
|
||||
# failure again degraded silently to no conditioning at all.
|
||||
enc = audio_vae.encode(torch.zeros((1, sr * _SILENT_SECONDS, 2)))
|
||||
if enc is None or enc.dim() != 4 or enc.shape[1] != 32:
|
||||
return None
|
||||
if enc.shape[-1] <= 2 * _SILENT_EDGE + 1:
|
||||
return None
|
||||
block = enc[..., _SILENT_EDGE:-_SILENT_EDGE].detach().to("cpu").clone()
|
||||
_SILENT_UNIT["lat"] = block
|
||||
_SILENT_UNIT["key"] = key
|
||||
n = block.shape[-1]
|
||||
if n < 1:
|
||||
return None
|
||||
# Forward, reversed, forward... Each join repeats a frame, so the seam that
|
||||
# plain tiling leaves is gone while the interior variation is the encoder's.
|
||||
pieces, have, i = [], 0, 0
|
||||
while have < want_t:
|
||||
piece = block if i % 2 == 0 else torch.flip(block, dims=[-1])
|
||||
pieces.append(piece)
|
||||
have += n
|
||||
i += 1
|
||||
out = torch.cat(pieces, dim=-1)[..., :want_t].clone()
|
||||
if out.shape[-1] != want_t:
|
||||
return None
|
||||
return out
|
||||
except Exception:
|
||||
return None # never fail a render for a nicety
|
||||
|
||||
|
||||
def _pin_audio_silence(latent, silence, lead_frames=None, tail_frames=0):
|
||||
"""Start target audio at encoded silence and preserve the requested span(s).
|
||||
|
||||
lead_frames None pins the whole shot. Otherwise the first lead_frames and the
|
||||
last tail_frames are held at silence and the span between is left to the model
|
||||
-- that is where the line goes. The tail is clipped to what the lead leaves, so
|
||||
the two can never overlap. Nothing pinned at all is a no-op, reported as False
|
||||
so the caller does not count it as applied."""
|
||||
try:
|
||||
video, audio = latent["samples"].unbind()
|
||||
silence = silence.to(device=audio.device, dtype=audio.dtype)
|
||||
if silence.shape != audio.shape:
|
||||
return False
|
||||
audio_mask = torch.ones_like(audio[:, :1])
|
||||
if lead_frames is None:
|
||||
audio_mask.zero_()
|
||||
else:
|
||||
t = audio.shape[-1]
|
||||
n = min(t, max(0, int(lead_frames)))
|
||||
m = min(t - n, max(0, int(tail_frames or 0)))
|
||||
if n <= 0 and m <= 0:
|
||||
return False
|
||||
if n > 0:
|
||||
audio_mask[..., :n] = 0
|
||||
if m > 0:
|
||||
audio_mask[..., t - m:] = 0
|
||||
latent["samples"] = comfy.nested_tensor.NestedTensor((video, silence))
|
||||
latent["noise_mask"] = comfy.nested_tensor.NestedTensor(
|
||||
(torch.ones_like(video[:, :1]), audio_mask))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,310 @@
|
||||
# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos
|
||||
# Copyright (c) 2026 Smite79. All rights reserved.
|
||||
# Redistribution, in whole or in part, requires written permission.
|
||||
# This notice may not be removed or altered. See LICENSE.
|
||||
"""Decisions about which pictures may condition a shot."""
|
||||
|
||||
import torch
|
||||
import node_helpers
|
||||
from h3_runtime import (H3_FPS, AUDIO_LATENT_FPS, _empty_av_latent, _resize, ref_image_canvas,
|
||||
frame_levels)
|
||||
from h3_audio import _SILENCE_STATUS, _silent_audio_latent, _pin_audio_silence
|
||||
|
||||
|
||||
def may_carry_room(previous_cast, current_cast, tagged_names):
|
||||
"""A previous frame is safe as a reference only when it adds no subject."""
|
||||
previous = [name for name in (previous_cast or ()) if name]
|
||||
current = set(current_cast or ())
|
||||
tagged = set(tagged_names or ())
|
||||
return bool(previous) and all(name in current for name in previous) \
|
||||
and not any(name in tagged for name in previous)
|
||||
|
||||
|
||||
def may_carry_frame(previous_cast, current_cast, tagged_names):
|
||||
"""A previous frame of the SAME room is safe as a reference claimed with everyone in it.
|
||||
|
||||
Unlike may_carry_room, somebody this shot does not describe may be in it: the claim
|
||||
names them, and they are still in that room. Refused only for an empty frame, or
|
||||
one holding somebody whose own portrait also rides this shot -- two pictures of one
|
||||
person is how a second one gets drawn."""
|
||||
previous = [name for name in (previous_cast or ()) if name]
|
||||
current = set(current_cast or ())
|
||||
tagged = set(tagged_names or ())
|
||||
return bool(previous) and not any(name in tagged and name in current
|
||||
for name in previous)
|
||||
|
||||
|
||||
def recoverable_subject(cast, tagged_names, returning_names, captured):
|
||||
"""Return the sole safe recovered subject, or an empty string."""
|
||||
people = [name for name in (cast or ()) if name]
|
||||
if len(people) != 1:
|
||||
return ""
|
||||
name = people[0]
|
||||
return name if (name not in set(tagged_names or ())
|
||||
and name in set(returning_names or ())
|
||||
and captured.get(name) is not None) else ""
|
||||
|
||||
|
||||
KEYFRAME_SAFE_AUG = 0.99 # below this, a ref aug would soften the keyframe too
|
||||
|
||||
# What ONE boundary is allowed to claim it measured. Wider than any real per-pass drift,
|
||||
# narrow enough that a bad frame -- a flash, a cut to black, a frame the model lost --
|
||||
# cannot swing the estimate. The median across boundaries does the real rejecting.
|
||||
LEVEL_GAIN_CAP = 0.12 # in log-gain, so +-12.7% of contrast
|
||||
LEVEL_OFFSET_CAP = 0.05
|
||||
# The within-shot term is believed only when boundaries AGREE on its sign, and even then
|
||||
# only this far: within-shot change is often the author's (a light switched off), so it is
|
||||
# the half of the signal that cannot be trusted on its own.
|
||||
LEVEL_SHOT_GAIN_CAP = 0.015
|
||||
LEVEL_SHOT_OFFSET_CAP = 0.010
|
||||
LEVEL_AGREE = 2.0 / 3.0
|
||||
LEVEL_MIN_OBS = 3
|
||||
# What the correction may do to one handoff, whatever it measured. A cut should not carry
|
||||
# a visible grade step: shot N's last frame reaches the video uncorrected while N+1 is
|
||||
# sampled from a corrected keyframe, so an uncapped correction trades burn-in for a pop at
|
||||
# every join -- the same class of complaint, differently shaped.
|
||||
LEVEL_GAIN_LO, LEVEL_GAIN_HI = 0.80, 1.25
|
||||
LEVEL_OFFSET_BOUND = 0.02
|
||||
# Below this a frame is too flat for a contrast RATIO to mean anything.
|
||||
LEVEL_MIN_SIGMA = 0.01
|
||||
|
||||
|
||||
class HandoffLevels:
|
||||
"""Takes the grade the chain adds to itself back out of the handoff.
|
||||
|
||||
THE MEASUREMENT, which is the whole reason this needs no scene list. At every
|
||||
boundary the render holds two pictures that are SUPPOSED to be the same frame: K,
|
||||
the handoff it gave the shot, and R, frame one of what came back -- the model's own
|
||||
reproduction of K, from a keyframe labelled sigma 0.001. Nothing was asked to change
|
||||
between them, so everything separating them is the chain's own doing and none of it
|
||||
is the author's intent. That is the one difference in the loop that can be corrected
|
||||
without guessing at anybody's lighting, and R costs nothing to look at: it is the
|
||||
frame trim_seam throws away.
|
||||
|
||||
A beat that walks into a darker room moves K, and R follows it there. So the level is
|
||||
never anchored, never compared to shot 1, and never compared to a target -- only K
|
||||
against its own reproduction, boundary by boundary.
|
||||
|
||||
WHAT IT WILL NOT FIX. Clipping already baked into earlier shots, because the VAE
|
||||
clamps every decode and headroom spent is gone. Softening, which is a different
|
||||
measurement and a different cause. Anything spatial -- ghosting, local burn, identity
|
||||
drift. A tone curve with a knee in it, since this is affine per channel; the residual
|
||||
in the report is how that would show itself. The first boundary, which has nothing to
|
||||
measure yet. And a deliberate monotone move -- a film that dims every single beat --
|
||||
loses a bounded, reported fraction of itself."""
|
||||
|
||||
def __init__(self):
|
||||
self._bg, self._bo = [], [] # per boundary: K -> R, the chain's own drift
|
||||
self._sg, self._so = [], [] # per shot: R -> last frame, believed only on agreement
|
||||
self.applied = [] # (gain, offset) actually used, for the report
|
||||
|
||||
def observe(self, given, repro, last=None, pre_up_last=None):
|
||||
"""Record one boundary. given is the keyframe this shot got, repro is frame one
|
||||
of what it produced, last is its final frame, pre_up_last the handoff it hands on.
|
||||
|
||||
last/pre_up_last are how the pre-upscale handoff and the post-upscale output are
|
||||
put in the same frame of reference: their difference IS the pipeline's own offset,
|
||||
measured on one frame that went through both, so it can be subtracted from the
|
||||
K->R reading instead of being mistaken for drift. With latent_upscale off they are
|
||||
the same frame and the term is zero."""
|
||||
gm, gs = frame_levels(given)
|
||||
rm, rs = frame_levels(repro)
|
||||
if gm is None or rm is None:
|
||||
return False
|
||||
if float(gs.min()) < LEVEL_MIN_SIGMA or float(rs.min()) < LEVEL_MIN_SIGMA:
|
||||
return False
|
||||
ug = torch.zeros(3)
|
||||
uo = torch.zeros(3)
|
||||
lm, ls = frame_levels(last) if last is not None else (None, None)
|
||||
if pre_up_last is not None and lm is not None:
|
||||
pm, ps = frame_levels(pre_up_last)
|
||||
if pm is not None and float(ps.min()) >= LEVEL_MIN_SIGMA:
|
||||
ug = torch.log(ls / ps)
|
||||
uo = lm - pm
|
||||
self._bg.append((torch.log(rs / gs) - ug).clamp(-LEVEL_GAIN_CAP, LEVEL_GAIN_CAP))
|
||||
self._bo.append((rm - gm - uo).clamp(-LEVEL_OFFSET_CAP, LEVEL_OFFSET_CAP))
|
||||
if lm is not None and float(ls.min()) >= LEVEL_MIN_SIGMA:
|
||||
self._sg.append(torch.log(ls / rs))
|
||||
self._so.append(lm - rm)
|
||||
return True
|
||||
|
||||
def _agreed(self, rows, cap):
|
||||
"""The median of rows, but only per channel where at least LEVEL_AGREE of them
|
||||
share its sign. A within-shot change the boundaries disagree about is content, not
|
||||
drift, and content must not be corrected."""
|
||||
out = torch.zeros(3)
|
||||
if len(rows) < LEVEL_MIN_OBS:
|
||||
return out
|
||||
st = torch.stack(rows)
|
||||
med = st.median(dim=0).values
|
||||
agree = ((st * med.sign().unsqueeze(0)) > 0).float().mean(dim=0)
|
||||
keep = agree >= LEVEL_AGREE
|
||||
return torch.where(keep, med.clamp(-cap, cap), out)
|
||||
|
||||
def estimate(self):
|
||||
"""(gain_log, offset) the chain is drifting by per boundary, per channel."""
|
||||
if not self._bg:
|
||||
return None, None
|
||||
g = torch.stack(self._bg).median(dim=0).values + self._agreed(self._sg, LEVEL_SHOT_GAIN_CAP)
|
||||
o = torch.stack(self._bo).median(dim=0).values + self._agreed(self._so, LEVEL_SHOT_OFFSET_CAP)
|
||||
return g, o
|
||||
|
||||
def gains(self, strength):
|
||||
"""(gain, offset) as 3-vectors, or (None, None) when there is nothing worth doing.
|
||||
|
||||
Separate from note() because more than one frame leaves a shot -- the handoff,
|
||||
and any face captured for a return several shots later -- and they have to carry
|
||||
the SAME grade. A recovered face arriving at a different exposure from the shot
|
||||
around it would be a new bug of exactly the kind this is fixing."""
|
||||
g, o = self.estimate()
|
||||
if g is None or strength <= 0:
|
||||
return None, None
|
||||
gain = torch.exp(-float(strength) * g).clamp(LEVEL_GAIN_LO, LEVEL_GAIN_HI)
|
||||
off = (-float(strength) * o).clamp(-LEVEL_OFFSET_BOUND, LEVEL_OFFSET_BOUND)
|
||||
# The next thing this frame meets is an 8-bit quantisation, so a correction under
|
||||
# 1/255 would be erased on the way there. Claiming it would be worse than silence.
|
||||
if float((gain - 1.0).abs().max()) < 1e-3 and float(off.abs().max()) < 1.0 / 255.0:
|
||||
return None, None
|
||||
return gain, off
|
||||
|
||||
def note(self, gain, off):
|
||||
"""Record what was applied, and say it in one clause."""
|
||||
self.applied.append((gain.clone(), off.clone()))
|
||||
return (f"gain {'/'.join(f'{float(v):.3f}' for v in gain)} "
|
||||
f"level {'/'.join(f'{float(v):+.4f}' for v in off)}")
|
||||
|
||||
def _keyframe_latent(vae, hand_img):
|
||||
"""The keyframe latent for this shot: an ENCODE of the previous shot's last frame.
|
||||
|
||||
This was briefly an optimisation -- pass the previous shot's own latent straight
|
||||
through and skip a VAE round trip per boundary. It was wrong, and it degraded
|
||||
every shot after the first.
|
||||
|
||||
A keyframe is ONE pixel frame, and H3's grid puts that at 5f -> TWO latent
|
||||
frames. Slicing [:, :, -1:] off a finished shot hands over one. Worse, the video
|
||||
VAE is causal: the last latent of a 72-frame sequence encodes its temporal
|
||||
context, not a standalone opening frame, so even at the right count it does not
|
||||
mean what a keyframe means. The spatial-size guard could not see either problem.
|
||||
|
||||
The round trip is real but it is one lossy step on a correctly formed anchor,
|
||||
which beats a cheap malformed one."""
|
||||
return vae.encode(hand_img)
|
||||
|
||||
|
||||
def _build_ref_images(vae, images, gen_w, gen_h, mode="match"):
|
||||
"""(tokenizer items, DiT blocks) for a list of reference IMAGE tensors.
|
||||
|
||||
The tokenizer labels each one `<Picture N>:` itself, in the order given here --
|
||||
so the roster the prompt refers to is decided by input order, not by anything
|
||||
written in the prompt."""
|
||||
items, blocks = [], []
|
||||
for img in images:
|
||||
if img is None:
|
||||
continue
|
||||
h, w = int(img.shape[1]), int(img.shape[2])
|
||||
tw, th = ref_image_canvas(w, h, gen_w, gen_h, mode)
|
||||
resized = _resize(img[:1], tw, th, "disabled")
|
||||
items.append({"type": "image", "data": resized})
|
||||
blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16,
|
||||
"latent": vae.encode(resized)})
|
||||
return items, blocks
|
||||
|
||||
|
||||
def build_conditioning(clip, vae, audio_vae, prompt, width, height, length,
|
||||
handoff=None, refs=None,
|
||||
ref_noise_aug=0.999, silent=False, ref_image_size="match",
|
||||
handoff_as_ref=False, speech_lead_seconds=0.0, speech_tail_frames=0):
|
||||
"""Encode prompt, identity references, keyframe, and audio constraints for a shot."""
|
||||
latent, fc = _empty_av_latent(width, height, length, H3_FPS)
|
||||
refs = [r for r in (refs or []) if r is not None]
|
||||
|
||||
hand_img = None
|
||||
if handoff is not None:
|
||||
hand_img = _resize(handoff[:1], width, height, "disabled")
|
||||
|
||||
# REFERENCES AND THE KEYFRAME RIDE TOGETHER. This is the arrangement the node
|
||||
# had before I broke it, and the reason is in ComfyUI's own layout:
|
||||
#
|
||||
# model_base.py:2183-2191 cond_video_latents = keyframe latents THEN ref latents
|
||||
# model.py PackedLayout emits keyframe "cond" segments THEN ref "ref_img" ones
|
||||
#
|
||||
# The two orders agree, so both channels coexist. A shot takes its references AND
|
||||
# a real keyframe: the keyframe ANCHORS the first frame, which is what continuity
|
||||
# needs, while a reference only supplies identity. They are not alternatives.
|
||||
#
|
||||
# I had read "<Picture 1>" as MEANING the first frame on fl2va, and rearranged the
|
||||
# roster around that. It does not. Which image is the first frame is decided by
|
||||
# resolved_frame_index in minimax_keyframes, not by a label's number -- the labels
|
||||
# are only how the images are shown to the VLM, and what they have to line up with
|
||||
# is the <Picture N> tags in the prompt.
|
||||
#
|
||||
# So references come FIRST and keep slots 1..N, which is what a sheet line's
|
||||
# `Name: <Picture 1>, ...` points at, and the handoff is appended AFTER them where
|
||||
# it disturbs no numbering. It has to be in the list at all because
|
||||
# tokenize_with_weights is either/or: passing minimax_ref_items makes it ignore
|
||||
# `images` outright, so leaving the handoff out means the VLM is never shown where
|
||||
# the shot left off and re-imagines the scenery -- same place, new room.
|
||||
keyframe_ok = ref_noise_aug is None or float(ref_noise_aug) >= KEYFRAME_SAFE_AUG
|
||||
# One aug covers every visual condition row, references AND the keyframe. Below
|
||||
# KEYFRAME_SAFE_AUG the keyframe latent would be noised and labelled at the wrong
|
||||
# timestep, so the handoff stops being an anchor and rides as an extra reference
|
||||
# instead: weaker continuity, but nothing pretending to anchor while carrying noise.
|
||||
# ...or because the caller asked for it. A shot that introduces somebody already
|
||||
# in position wants the room this picture carries and NOT the first frame it
|
||||
# would force, and that is a demotion the aug knows nothing about.
|
||||
carry_as_ref = bool(hand_img is not None
|
||||
and (handoff_as_ref or (refs and not keyframe_ok)))
|
||||
|
||||
enc_refs = refs + ([hand_img] if carry_as_ref else [])
|
||||
items, blocks = ([], [])
|
||||
if enc_refs:
|
||||
items, blocks = _build_ref_images(vae, enc_refs, width, height, ref_image_size)
|
||||
if hand_img is not None and not carry_as_ref:
|
||||
items = items + [{"type": "image", "data": hand_img}]
|
||||
|
||||
if items:
|
||||
tokens = clip.tokenize(prompt, minimax_ref_items=items)
|
||||
else:
|
||||
tokens = clip.tokenize(prompt)
|
||||
cond = clip.encode_from_tokens_scheduled(tokens)
|
||||
|
||||
vals = {}
|
||||
if blocks:
|
||||
vals["minimax_refs"] = blocks
|
||||
# How CLEAN the references are shown. One aug covers every conditioning
|
||||
# latent, keyframe included -- which is why softening references below
|
||||
# KEYFRAME_SAFE_AUG would soften the anchor too.
|
||||
if ref_noise_aug is not None:
|
||||
vals["minimax_visual_cond_noise_aug"] = float(ref_noise_aug)
|
||||
|
||||
kfs = []
|
||||
if hand_img is not None and not carry_as_ref:
|
||||
kfs.append({"resolved_frame_index": 0,
|
||||
"latent": _keyframe_latent(vae, hand_img)})
|
||||
# Audio keyframes are extra conditioning rows in H3's PackedLayout. Pin the
|
||||
# generated target stream instead, so the joint model also sees a quiet mouth.
|
||||
# A dialogue shot pins its opening (the lead) and, past the line's estimated end,
|
||||
# its close (the tail); the span between is the model's.
|
||||
if silent or float(speech_lead_seconds or 0.0) > 0.0 or int(speech_tail_frames or 0) > 0:
|
||||
_SILENCE_STATUS["asked"] += 1
|
||||
if audio_vae is None:
|
||||
_SILENCE_STATUS["why"] = "no audio VAE is wired to the node"
|
||||
else:
|
||||
sil = _silent_audio_latent(audio_vae, fc, H3_FPS)
|
||||
if sil is None:
|
||||
_SILENCE_STATUS["why"] = ("the audio VAE would not encode a silent "
|
||||
"second -- the wrong VAE is on the "
|
||||
"audio_vae input")
|
||||
else:
|
||||
lead = None if silent else round(float(speech_lead_seconds) *
|
||||
AUDIO_LATENT_FPS)
|
||||
tail = 0 if silent else int(speech_tail_frames or 0)
|
||||
if _pin_audio_silence(latent, sil, lead, tail):
|
||||
_SILENCE_STATUS["applied"] += 1
|
||||
else:
|
||||
_SILENCE_STATUS["why"] = "the silent latent did not match the shot"
|
||||
if kfs:
|
||||
vals["minimax_keyframes"] = kfs
|
||||
if vals:
|
||||
cond = node_helpers.conditioning_set_values(cond, vals)
|
||||
return cond, latent, fc, carry_as_ref
|
||||
+286
-67
@@ -66,21 +66,8 @@ HARDWARE = (
|
||||
(r"cuffs?|cuffed", "cuffs", "wrists"),
|
||||
(r"tape", "tape", "wrists"),
|
||||
)
|
||||
# Material and colour survive because they decide what the thing looks like:
|
||||
# "steel collar" must not come back as "collar" two shots later.
|
||||
# WHAT THE AUTHOR CALLED IT. This decides how much of the wording survives into
|
||||
# the guard clauses, and the guard is what every shot after the first repeats --
|
||||
# so a word missing here is a word the model stops hearing.
|
||||
#
|
||||
# It was twenty-odd words, and "a mirrored steel collar" came back as "steel
|
||||
# collar" while "a brushed nickel collar" came back as "collar". A bare "collar"
|
||||
# repeated once a shot is a bare collar, and the prior for that is a black
|
||||
# leather one -- which is exactly what was reported.
|
||||
#
|
||||
# Hyphenated compounds pass whole ("mirror-finish", "chrome-plated"), so an
|
||||
# unusual finish survives without being listed. Bare participles are NOT
|
||||
# accepted: "-ed" is a verb far more often than a modifier, and capturing one
|
||||
# would put an action into the name of the thing.
|
||||
# Preserve visual modifiers in continuity text. Hyphenated compounds pass whole;
|
||||
# arbitrary participles do not, because they are more often verbs than modifiers.
|
||||
_ADJ = (r"(?:[A-Za-z]+-[A-Za-z]+|"
|
||||
# materials
|
||||
r"steel|stainless|iron|metal|metallic|nickel|chrome|chromed|brass|"
|
||||
@@ -131,8 +118,26 @@ PART_VARIES = frozenset({"chain", "rope", "straps", "tape"})
|
||||
# outlives the beat that caused it, and the clause that says so has to be
|
||||
# writable from any later shot.
|
||||
REGION_OF = (
|
||||
# UNDERWEAR IS IN HERE TOO, on both halves of the body. The torso row has
|
||||
# listed a bra since the day it was written -- that is the report it exists
|
||||
# for, "a bra coming back on somebody topless" -- and the leg row never got
|
||||
# its counterpart, so region_of("thong") answered "". A garment that cannot
|
||||
# be placed latches no bare region, so underwear coming off said nothing
|
||||
# about the hips in that shot or in any shot after it, and an unspecified
|
||||
# region is filled by the model's own prior -- which for a hip is underwear.
|
||||
#
|
||||
# Worse, a beat saying somebody is NAKED looks each worn garment's region up
|
||||
# to take it off the body, so the one garment it could not place stayed
|
||||
# "worn" in the state while the text said she was nude. Reported as a thong
|
||||
# restored a beat after she undressed to get in the shower.
|
||||
#
|
||||
# No hardware. A chastity belt is in the layering vocabulary, but it is a
|
||||
# restraint: it is latched and held by its own mechanism, and a bare region
|
||||
# read off it would argue with that.
|
||||
(r"shorts|trousers|jeans|slacks|chinos|skirt|kilt|leggings|joggers|tights|"
|
||||
r"pantyhose|jeggings|culottes|tracksuit\s+bottoms", "legs",
|
||||
r"pantyhose|jeggings|culottes|tracksuit\s+bottoms|"
|
||||
r"panties|knickers|thong|g-?string|briefs|boxers|underwear|undies|"
|
||||
r"jockstrap|loincloth", "legs",
|
||||
"The legs are bare from the hip down"),
|
||||
(r"socks|stockings|hold-?ups|boots|shoes|trainers|sneakers|sandals|heels",
|
||||
"feet", "The feet and ankles are bare"),
|
||||
@@ -214,16 +219,34 @@ RELEASE_VERB = (
|
||||
# the engine knew a cell and a warehouse, the sampler did not, so a scene set in
|
||||
# either was a room to one reader and nowhere to the other. Same fault the
|
||||
# garment lists had, waiting to be reported.
|
||||
# MULTI-WORD ROOMS COME BEFORE BARE "room". _MOD is non-greedy, so it tries no
|
||||
# modifier first and the longest place wins -- but only if the long form is here to
|
||||
# win with. Without "locker\s+room", "heads to the locker room" read its destination
|
||||
# as "room": _MOD swallowed "locker" and the capture took what was left, so the film
|
||||
# was reported as entering "room" and the shot was told to arrive in one.
|
||||
#
|
||||
# A gym, a locker room and a court were in no list at all, which is worse than vague:
|
||||
# with the ORIGIN unknown, travel_anchor emitted nothing, so a beat walking out of a
|
||||
# gym was never told to walk and the set simply changed under the characters.
|
||||
PLACES = (r"hallway|hall|corridor|passage|landing|stairwell|staircase|stairs|"
|
||||
r"steps|bedroom|bathroom|washroom|kitchen|living\s+room|lounge|"
|
||||
r"dining\s+room|study|office|garage|basement|cellar|attic|loft|porch|"
|
||||
r"dining\s+room|locker\s+rooms?|changing\s+rooms?|dressing\s+rooms?|"
|
||||
r"waiting\s+rooms?|utility\s+rooms?|gymnasium|gym|classroom|library|"
|
||||
r"cafeteria|canteen|reception|laundry|pantry|sauna|balcony|terrace|"
|
||||
r"rooftop|elevator|court|pool|showers|shower|store|shop|studio|"
|
||||
r"study|office|garage|basement|cellar|attic|loft|porch|"
|
||||
r"veranda|garden|yard|driveway|street|alley|car\s?park|lobby|foyer|"
|
||||
r"doorway|cell|warehouse|barn|shed|van|truck|room")
|
||||
# Place words that are also ordinary verbs or everyday nouns. A reader with a
|
||||
# preposition in front of it ("in the study") can tell which sense is meant; the
|
||||
# free-text one cannot, and "she steps out", "they study the map" and "he lands
|
||||
# badly" are all commoner than the rooms they collide with.
|
||||
PLACE_ALSO_A_VERB = {"steps", "landing", "study", "lounge", "garage", "porch"}
|
||||
# "bar" and "lift" are deliberately NOT places in this file at all: bars are
|
||||
# restraint hardware here ("chained to the bars") and lifting is what happens to a
|
||||
# garment or a body, so behind a preposition they would both read as journeys.
|
||||
PLACE_ALSO_A_VERB = {"steps", "landing", "study", "lounge", "garage", "porch",
|
||||
"court", "pool", "shower", "showers", "store", "shop",
|
||||
"studio", "reception"}
|
||||
# A room is usually described, not just named -- "the tiled bathroom", "the long
|
||||
# hallway". Up to three adjectives, non-greedy so the NEAREST room still wins.
|
||||
_ROOM_MOD = (r"(?:(?!(?:of|the|an?|and|or|to|in|into|from|with|on|at|by|for|her|"
|
||||
@@ -272,15 +295,21 @@ _GAP = r"(?:\s+\S+){0,4}?\s+"
|
||||
TAKES_OFF = (r"(?:takes?|took|taking|pulls?|pulled|peels?|peeled|strips?|"
|
||||
r"stripped|shrugs?|slips?|slipped|steps?|gets?|got|kicks?|"
|
||||
r"kicked)" + _GAP + r"(?:off|out\s+of)\b"
|
||||
r"|\b(?:removes?|removed|removing|discards?|discarded|"
|
||||
r"undresses|undressed|unbuttons?|unzips?|unzipped)")
|
||||
r"|\b(?:removes?|removed|removing|discards?|discarded|sheds?|shedding|"
|
||||
r"undresses|undressed)")
|
||||
PUTS_ON = (r"(?:puts?|putting|pulls?|pulled|slips?|slipped|tugs?|tugged|"
|
||||
r"steps?|stepped|climbs?|climbed|gets?|got|wriggles?)" + _GAP +
|
||||
r"(?:on|into|back\s+on)\b"
|
||||
r"|\b(?:dresses?\s+in|dressed\s+in|buttons?|zips?\s+up|fastens?)")
|
||||
# UNZIPPING A JACKET LEAVES IT ON. These were removals, so "Owen unzips his jacket"
|
||||
# took the jacket out of every later shot and called his chest bare -- and "rolls up
|
||||
# his sleeves" a beat later rolled the sleeves of a shirt that had gone with it. They
|
||||
# open a garment; a beat that also takes it off says so ("and takes it off").
|
||||
DISPLACES = (r"(?:pulls?|pulled|pushes?|pushed|tugs?|tugged|hikes?|hiked|"
|
||||
r"rolls?|rolled|lifts?|lifted|yanks?|yanked|shoves?|shoved)"
|
||||
+ _GAP + r"(?:aside|up|down|open)\b")
|
||||
+ _GAP + r"(?:aside|up|down|open)\b"
|
||||
r"|\b(?:unzips?|unzipped|unbuttons?|unbuttoned|unfastens?|unfastened|"
|
||||
r"undoes|undid)\b")
|
||||
|
||||
# POSTURES and the _POSTURE list built from it used to live here. Nothing read
|
||||
# _POSTURE -- it was a second, dead copy of the posture vocabulary, and it had
|
||||
@@ -440,6 +469,17 @@ def _outside_speech(text):
|
||||
return _SPOKEN_SPAN.sub(" ", text or "")
|
||||
|
||||
|
||||
def staged_text(text):
|
||||
"""What a beat STAGES: not what anybody says, and not what the narration asks.
|
||||
|
||||
A narrated question is the same kind of thing as a line of speech. "Maya waits by
|
||||
the door. Will he come?" asks whether he will, which is to say he is not there --
|
||||
and the "he" read as him being present, so Will was described into the shot of
|
||||
her waiting for him. Removed for deciding who is in the shot, as speech is."""
|
||||
staged = _outside_speech(text)
|
||||
return " ".join(s for s in re.split(r"(?<=[.!?])\s+", staged) if not s.rstrip().endswith("?"))
|
||||
|
||||
|
||||
# THREE modifiers, not two: "mirrored stainless steel collar" is three words and
|
||||
# a noun, and the third was the first to be dropped.
|
||||
_HW_ONE = _rx(r"\b(" + _ADJ + r"(?:\s+" + _ADJ + r"){0,2}\s+)?("
|
||||
@@ -478,6 +518,14 @@ _GARMENT_ONE = _rx(r"\b(" + _ADJ + r"(?:\s+" + _ADJ + r"){0,2}\s+)?("
|
||||
_TAKES_OFF = _rx(r"\b" + TAKES_OFF + r"\b")
|
||||
_PUTS_ON = _rx(r"\b" + PUTS_ON + r"\b")
|
||||
_DISPLACES = _rx(r"\b" + DISPLACES + r"\b")
|
||||
# ...and the sentence can still finish the job after the garment is named. "Kate
|
||||
# unzips the denim skirt and steps out of it": the unzip opens it, the rest of the
|
||||
# sentence takes it off, and the removal verb comes after the item where the reader
|
||||
# above does not look.
|
||||
_OPENS_GARMENT = _rx(r"\b(?:unzips?|unzipped|unbuttons?|unbuttoned|unfastens?|unfastened|"
|
||||
r"undoes|undid|unhooks?|unhooked|unclasps?|unclasped)\b")
|
||||
_COMPLETES_OFF = _rx(r"\b(?:off|out\s+of|away|lets?\s+(?:it|them)\s+(?:fall|drop|slide)|"
|
||||
r"drops?\s+(?:it|them)|falls?\s+(?:to|down|away|off))\b")
|
||||
_MOVES = _rx(r"\b(?:walks?|walked|walking|goes|go|went|going|runs?|ran|running|"
|
||||
r"steps?|stepped|stepping|moves?|moved|moving|enters?|entered|"
|
||||
r"leaves?|left|leaving|crosses|crossed|crossing|climbs?|climbed|"
|
||||
@@ -595,11 +643,6 @@ def nudity_in(text):
|
||||
return out
|
||||
|
||||
|
||||
def bare_sentence(region):
|
||||
"""How to say a region is bare, or "" for one with no wording."""
|
||||
return next((s for _rx, r, s in _REGION_RX if r == region), "")
|
||||
|
||||
|
||||
def _bare_on(p, regions):
|
||||
for r in ([regions] if isinstance(regions, str) else regions):
|
||||
if r and r not in p.bare:
|
||||
@@ -653,11 +696,6 @@ def _nearest_part(parts, at, ats):
|
||||
return ""
|
||||
|
||||
|
||||
def hardware_in(text):
|
||||
"""Every piece of hardware named, as (canonical, part, as-written)."""
|
||||
return [(c, p, w) for c, p, w, _at in hardware_spans(text)]
|
||||
|
||||
|
||||
def position_spans(text):
|
||||
"""Every limb position named, as (name, at)."""
|
||||
out = []
|
||||
@@ -705,9 +743,15 @@ def place_in(text):
|
||||
|
||||
Behind a preposition, so a room has to be somewhere somebody IS. "Ana looks
|
||||
at the door" names no room -- and a door is not on the list in any case."""
|
||||
m = _PLACE_IN.search(text or "")
|
||||
text = text or ""
|
||||
m = _PLACE_IN.search(text)
|
||||
if not m:
|
||||
return ""
|
||||
clause = re.split(r"[.;!?]", text[:m.end()])[-1]
|
||||
present = re.search(r"\b(?:is|are|was|were|stands?|sits?|waits?|lies?|remains?)\b",
|
||||
clause, re.I)
|
||||
if not _MOVES.search(clause) and not present:
|
||||
return ""
|
||||
got = re.sub(r"\s+", " ", m.group(1).lower()).strip()
|
||||
# A BARE "room" NAMES NOWHERE. "Ana walks into the room" says she goes
|
||||
# inside, not which room -- and taking it as a place produced "The shot is
|
||||
@@ -730,6 +774,18 @@ def garments_in(text):
|
||||
return out
|
||||
|
||||
|
||||
_CLAUSE_BOUNDARY = re.compile(r"[,;]|\b(?:and|while)\b", re.I)
|
||||
|
||||
|
||||
def _clause_at(text, at, boundaries=None):
|
||||
"""Return the clause containing character offset `at` and its start offset."""
|
||||
if boundaries is None:
|
||||
boundaries = list(_CLAUSE_BOUNDARY.finditer(text or ""))
|
||||
lo = max((m.end() for m in boundaries if m.end() <= at), default=0)
|
||||
hi = min((m.start() for m in boundaries if m.start() > at), default=len(text))
|
||||
return text[lo:hi], lo
|
||||
|
||||
|
||||
|
||||
# Hardware, not clothing. Taking clothes off does not unlock anything, so these
|
||||
# are kept out of the garment answer -- the standing rule is that hardware is
|
||||
@@ -740,6 +796,32 @@ _NOT_CLOTHING = re.compile(
|
||||
r"straitjacket|spreader|hogtie|clamps?|clips?)$", re.I)
|
||||
_PHRASE_ONE = _rx(r"\b(?:" + GARMENT_PHRASES + r")\b")
|
||||
_WORD_ONE = _rx(r"^(?:" + GARMENT_WORDS + r")s?$")
|
||||
# The same list with NO optional plural, which is what says whether a trailing "s"
|
||||
# belongs to the word or was added to it. The vocabulary is clean on this: garments that
|
||||
# are inherently plural are listed only in the plural (boots, jeans, shorts, panties,
|
||||
# tights, leggings, socks, knickers, trousers, gloves) and the rest only in the singular
|
||||
# (skirt, vest, top), so "stem is itself a garment" is an exact test and not a guess.
|
||||
_WORD_EXACT = _rx(r"^(?:" + GARMENT_WORDS + r")$")
|
||||
|
||||
|
||||
def singular_garment(word):
|
||||
"""A garment word as the VOCABULARY spells it, so two readers cannot disagree.
|
||||
|
||||
Reported: several women take their skirts off and the skirts are back in the next
|
||||
beat. "their skirts" yields the token "skirts" while the sheet says "a denim
|
||||
skirt", and every reader downstream looks the token up in the sheet -- the scrub
|
||||
by pattern, infer_removals by entry head -- so a plural garment matched nothing
|
||||
and the removal silently did nothing at all. One woman undressing wrote "her
|
||||
skirt" and worked; the moment the subject went plural so did the garment.
|
||||
|
||||
A trailing "s" comes off only when the stem is ITSELF a garment word, which is an
|
||||
exact test here and not a guess: the vocabulary lists inherently plural garments
|
||||
only in the plural (boots, jeans, shorts, panties, tights, leggings, socks,
|
||||
knickers, trousers, gloves) and the rest only in the singular."""
|
||||
low = str(word or "").lower().strip("-")
|
||||
if low.endswith("s") and _WORD_EXACT.match(low[:-1]):
|
||||
return low[:-1]
|
||||
return low
|
||||
|
||||
|
||||
def garment_words(text):
|
||||
@@ -761,12 +843,60 @@ def garment_words(text):
|
||||
text = _PHRASE_ONE.sub(" ", text)
|
||||
for word in re.findall(r"\b[\w-]{3,}\b", text):
|
||||
low = word.lower().strip("-")
|
||||
if low in out or _NOT_CLOTHING.match(low):
|
||||
if _NOT_CLOTHING.match(low):
|
||||
continue
|
||||
if _WORD_ONE.match(low):
|
||||
if not _WORD_ONE.match(low):
|
||||
continue
|
||||
# The token the sheet wrote, not the one the beat happened to inflect. See
|
||||
# singular_garment -- shared with infer_removals so the two cannot disagree.
|
||||
low = singular_garment(low)
|
||||
if low not in out:
|
||||
out.append(low)
|
||||
return out
|
||||
|
||||
# A POSTURE DENIED IS NOT A POSTURE TAKEN.
|
||||
#
|
||||
# Reported: a woman chained by the ankles and forced into a squat stood up anyway.
|
||||
# The chain clauses were all correct -- the metal "already drawn to its full length,
|
||||
# so the position it fixes is the position that keeps" was on every shot after the
|
||||
# squat. What sat beside it was the posture latch saying she was STANDING, because
|
||||
# "She cannot stand." matched `stand` and nothing looked at the `cannot`. The latch
|
||||
# then carried that forward, so every later shot asserted, flatly and positively,
|
||||
# the one thing the chains were there to prevent. At cfg 1 a positive statement wins.
|
||||
#
|
||||
# The same shape as _in_a_request, which already suppresses a posture that is ASKED
|
||||
# for rather than taken. Attempts are here too: "tries to stand", "struggles to get
|
||||
# up", "strains to rise" are all bodies that have NOT got there, and reading them as
|
||||
# arrival is the same error in a friendlier disguise.
|
||||
#
|
||||
# A SHORT WINDOW on purpose -- the five words before the verb. The cue always sits
|
||||
# immediately in front of it ("cannot stand", "no longer able to stand"), and a wider
|
||||
# reach would let a `cannot` from a different clause silence a real posture.
|
||||
_POSTURE_DENIED = _rx(
|
||||
r"\b(?:cannot|can\s*not|can['\u2019]?t|could\s*not|could\s*n['\u2019]?t|"
|
||||
r"unable|never|not\s+able|no\s+longer\s+able|"
|
||||
r"does\s*n['\u2019]?t|does\s+not|do\s*n['\u2019]?t|did\s*n['\u2019]?t|did\s+not|"
|
||||
r"will\s+not|wo\s*n['\u2019]?t|fail(?:s|ed|ing)?|"
|
||||
r"tr(?:y|ies|ied|ying)|attempt(?:s|ed|ing)?|struggl(?:e|es|ed|ing)|"
|
||||
r"strain(?:s|ed|ing)?|fight(?:s|ing)?|want(?:s|ed)?|need(?:s|ed)?|"
|
||||
r"told|ordered|asked|begs?|begged)\b")
|
||||
|
||||
|
||||
def denied_posture(text, at):
|
||||
"""Is the posture verb at `at` negated, or only attempted, by what precedes it?"""
|
||||
before = str(text or "")[:max(0, int(at))]
|
||||
# A cue belongs to its OWN clause. Without stopping at the boundary, "McKenna
|
||||
# cannot kneel, so she sits" reached back past the comma and silenced the sitting
|
||||
# -- suppressing a posture the beat plainly states, which is the same class of
|
||||
# error in the other direction.
|
||||
cut = 0
|
||||
for _m in re.finditer(r"[,;:.!?]|\b(?:so|and|but|then|yet|while|as|before|after)\b",
|
||||
before, re.I):
|
||||
cut = _m.end()
|
||||
window = " ".join(re.findall(r"[\w'\u2019]+", before[cut:])[-5:])
|
||||
return bool(_POSTURE_DENIED.search(window))
|
||||
|
||||
|
||||
def posture_in(text):
|
||||
"""The posture this beat puts a body in. '' when it does not.
|
||||
|
||||
@@ -775,7 +905,7 @@ def posture_in(text):
|
||||
for that, but off the same vocabulary."""
|
||||
t = text or ""
|
||||
hits = sorted((m.start(), name) for name, rx in _POSTURE_OF
|
||||
for m in [rx.search(t)] if m)
|
||||
for m in rx.finditer(t) if not denied_posture(t, m.start()))
|
||||
return hits[0][1] if hits else ""
|
||||
|
||||
|
||||
@@ -820,7 +950,7 @@ _TRAILING_VERB = (r"take[sn]?|took|taking|pull(?:s|ed|ing)?|peel(?:s|ed|ing)?|"
|
||||
r"toss(?:es|ed)?|throw[s]?|threw|kick(?:s|ed|ing)?|"
|
||||
r"slide[s]?|slid|wriggle[sd]?|wiggle[sd]?")
|
||||
# ...and verbs that are a removal on their own, needing no particle.
|
||||
_UNDO_VERB = (r"remove[sd]?|removing|undress(?:es|ed)?|unzip(?:s|ped)?|"
|
||||
_UNDO_VERB = (r"remove[sd]?|removing|undress(?:es|ed)?|shed(?:s|ding)?|unzip(?:s|ped)?|"
|
||||
r"unbutton(?:s|ed)?|unhook(?:s|ed)?|unclasp(?:s|ed)?|unfasten(?:s|ed)?|"
|
||||
# Hardware comes off by being UNDONE, and these were missing: a beat
|
||||
# saying "unlocks the belt" left it described as worn for the rest of
|
||||
@@ -830,11 +960,26 @@ _UNDO_VERB = (r"remove[sd]?|removing|undress(?:es|ed)?|unzip(?:s|ped)?|"
|
||||
r"undo(?:es)?|undid")
|
||||
|
||||
|
||||
# WHERE A GARMENT ENDS UP ONCE IT IS OFF. A thing on the floor is not on a body,
|
||||
# and this is how a beat says so -- by DESTINATION, not by verb. Two readers need
|
||||
# the same list, for opposite reasons: the removal reader to call it a removal,
|
||||
# the restore reader to stop calling it one. "Lets the skirt fall" drops a lifted
|
||||
# skirt back over her legs; "lets the thong fall to the floor" is the thong coming
|
||||
# off, and the only difference between those two sentences is this list.
|
||||
FLOOR = (r"floor|ground|tiles?|tiling|lino|mat|bath\s*mat|rug|carpet|deck|boards|"
|
||||
r"concrete|grass|sand|bed|sofa|couch|chair|seat|stool|bench|basket|"
|
||||
r"hamper|laundry|pile|heap")
|
||||
TO_THE_FLOOR = (r"(?:to|on|onto|into|in)\s+(?:the|a|an|her|his|their)?\s*"
|
||||
r"(?:" + FLOOR + r")\b")
|
||||
_LANDS_OFF = _rx(r"\s*(?:fall(?:s|ing)?|drop(?:s|ping)?|land(?:s|ing)?)?\s*"
|
||||
+ TO_THE_FLOOR)
|
||||
|
||||
_DISPLACE_WAY = (r"back\s+up|back\s+down|down|up|aside|open|back|"
|
||||
r"off\s+(?:one|her|his|their)\s+shoulders?")
|
||||
_DISPLACE = re.compile(
|
||||
r"\b(?:" + _STRIP_VERB + r"|push(?:es|ed|ing)?|shove[sd]?|roll(?:s|ed|ing)?|"
|
||||
r"hitch(?:es|ed)?|hike[sd]?|open(?:s|ed)?|undo(?:es)?|unzip(?:s|ped)?|"
|
||||
r"hitch(?:es|ed)?|hike[sd]?|open(?:s|ed)?|undo(?:es)?|undid|unzip(?:s|ped)?|"
|
||||
r"unbutton(?:s|ed)?|unfasten(?:s|ed)?|unhook(?:s|ed)?|unclasp(?:s|ed)?|"
|
||||
# LIFTING A SKIRT IS DISPLACING IT, and none of these were here. Asked
|
||||
# for directly: "when the skirt has been lifted up to show the chastity
|
||||
# belt, that's when it should be shown". Lifting was not read as moving
|
||||
@@ -874,14 +1019,23 @@ def scene_name_for(head, scene):
|
||||
# a reference is pinning, so it is the worst one to describe loosely.
|
||||
item = re.sub(r"<\s*picture\s+\d+\s*>", " ", item, flags=re.I)
|
||||
item = re.sub(r"\s+", " ", item).strip()
|
||||
if not item or item.split()[-1].lower() != head:
|
||||
# ONE ENTRY CAN HOLD SEVERAL GARMENTS, and the name is the garment's own
|
||||
# part of it. "navy jacket over a white shirt" ends in "shirt", so the
|
||||
# whole entry came back as the shirt's name and "the navy jacket over a
|
||||
# white shirt open" was said about a shirt being unbuttoned. Split on the
|
||||
# words that join garments, and cut a "with ..." tail, which describes a
|
||||
# garment rather than naming it.
|
||||
for part in re.split(r"\s+(?:over|under|beneath|underneath|on\s+top\s+of|and)\s+",
|
||||
item, flags=re.I):
|
||||
part = re.split(r"\s+with\s+", part, flags=re.I)[0].strip()
|
||||
if not part or part.split()[-1].lower() != head:
|
||||
continue
|
||||
# Drop a leading article or possessive; they are not description.
|
||||
item = re.sub(r"^(?:a|an|the|her|his|their|its)\s+", "", item, flags=re.I)
|
||||
# The longest entry wins: a sheet that names it twice described it most
|
||||
# fully once, and the fuller name is the one worth carrying.
|
||||
if len(item) > len(best):
|
||||
best = item
|
||||
part = re.sub(r"^(?:a|an|the|her|his|their|its)\s+", "", part, flags=re.I)
|
||||
# The longest entry wins: a sheet that names it twice described it
|
||||
# most fully once, and the fuller name is the one worth carrying.
|
||||
if len(part) > len(best):
|
||||
best = part
|
||||
# The author's OWN capitalisation. Lowercasing turned "PVC" into "pvc" and
|
||||
# "Shiny white crop top" into all-lowercase -- a different token sequence than
|
||||
# was written, for a brand or material name that is capitalised for a reason.
|
||||
@@ -910,6 +1064,13 @@ def displaced_garments(beat, scene):
|
||||
if not way and re.match(r"\s*(?:lift|rais|hoist|gather|bunch)", m.group(0),
|
||||
re.I):
|
||||
way = "up"
|
||||
# ...and UNDOING a garment opens it. "Owen unzips his jacket" was a removal,
|
||||
# then (once it was not) nothing at all -- the jacket went back to being
|
||||
# described closed on the next shot, which is a jacket zipping itself up
|
||||
# across a cut.
|
||||
if not way and re.match(r"\s*(?:unzip|unbutton|unfasten|unhook|unclasp|undo|undid)",
|
||||
m.group(0), re.I):
|
||||
way = "open"
|
||||
if not way or not thing or thing in seen:
|
||||
continue
|
||||
# The garment has to be one the scene already dresses them in, and the head
|
||||
@@ -987,9 +1148,26 @@ def restored_garments(beat, scene):
|
||||
return []
|
||||
out, low = [], scene.lower()
|
||||
for m in _PUT_BACK_NAMED.finditer(beat):
|
||||
# ...UNLESS IT LANDS ON THE FLOOR. These verbs are the restore vocabulary
|
||||
# because that is what people write for a lifted skirt -- let fall, drop,
|
||||
# lower, let go of -- and the identical words take a garment OFF when the
|
||||
# sentence says where it lands. "Lets the thong fall to the floor" was read
|
||||
# as putting the thong back on: the author's removal, enacted backwards, and
|
||||
# from there the sheet described it as worn for the rest of the film.
|
||||
# Reported as a thong restored after she undressed. See FLOOR.
|
||||
if _LANDS_OFF.match(beat[m.end():]):
|
||||
continue
|
||||
thing = re.sub(r"\s+", " ", (m.group(1) or "")).strip().lower()
|
||||
if not thing:
|
||||
continue
|
||||
# ...AND THE CAPTURE CANNOT RUN THROUGH A PREPOSITION. The group takes
|
||||
# spaces so a sheet's "long grey skirt" comes back whole, and on "drops the
|
||||
# thong on the floor" it swallowed "thong on the floor" instead -- head
|
||||
# "floor" -- so the restore was keyed to the ROOM. The scene named a wet
|
||||
# floor, scene_name_for handed back "tiled bathroom with a wet floor", and
|
||||
# the beat was recorded as putting the bathroom back on.
|
||||
if re.search(r"\b(?:on|onto|to|into|in|at|over|under|from|with|and)\b", thing):
|
||||
continue
|
||||
head = thing.split()[-1]
|
||||
if len(head) < 3 or head not in low:
|
||||
continue
|
||||
@@ -1197,6 +1375,10 @@ def _alias_at(word, staged):
|
||||
return fallback
|
||||
|
||||
|
||||
_AUX_FOLLOWER = re.compile(r"\s+(?:I|you|he|she|we|they|it|there|this|that|anyone|someone|"
|
||||
r"everyone|anybody|somebody)\b")
|
||||
|
||||
|
||||
def names_in(beat, cast):
|
||||
"""Names this beat STAGES, in the order the sentence puts them.
|
||||
|
||||
@@ -1208,14 +1390,23 @@ def names_in(beat, cast):
|
||||
Speech-stripped, for the same reason it is everywhere else -- "McKenna, where
|
||||
are you?" is how absence gets written, and reading it as presence put a whole
|
||||
sheet entry into a shot the person is not in."""
|
||||
staged = _outside_speech(beat or "")
|
||||
staged = staged_text(beat or "")
|
||||
names = [str(n) for n in (cast or []) if n]
|
||||
hits, found = [], set()
|
||||
for n in names:
|
||||
m = re.search(r"\b" + re.escape(n) + r"\b", staged)
|
||||
if m:
|
||||
# A NAME THAT IS ALSO A WORD. "Will he come?" and "May I come in?" open a
|
||||
# sentence with the name followed by who the question is about, and read as
|
||||
# the person they staged Will and May -- a second character in a shot about
|
||||
# somebody waiting for them. A name followed straight away by a subject
|
||||
# pronoun at the start of a sentence is the verb; the next use can still be
|
||||
# the person ("Will opens the gate").
|
||||
for m in re.finditer(r"\b" + re.escape(n) + r"\b", staged):
|
||||
_opens = re.search(r"(?:^|[.!?]\s*[\"'\u201c]?)\s*$", staged[:m.start()])
|
||||
if _opens and _AUX_FOLLOWER.match(staged[m.end():]):
|
||||
continue
|
||||
hits.append((m.start(), n))
|
||||
found.add(n)
|
||||
break
|
||||
# A SHEET NAME IS OFTEN LONGER THAN WHAT THE BEATS CALL HER. "Mistress Vale"
|
||||
# on the sheet and "the Mistress" in every beat matched nothing, so her line
|
||||
# was in no shot at all and the model invented her from scratch each time.
|
||||
@@ -1415,20 +1606,35 @@ class SceneState:
|
||||
subject = who[0] if who else next(iter(list(self.people) or list(cast)
|
||||
or [""]))
|
||||
|
||||
hw = hardware_in(beat)
|
||||
applying = bool(hw) and bool(_APPLY.search(beat)) and not _RELEASE.search(beat)
|
||||
spans = hardware_spans(beat)
|
||||
garments = list(_GARMENT_ONE.finditer(beat))
|
||||
boundaries = list(_CLAUSE_BOUNDARY.finditer(beat)) if spans or garments else []
|
||||
applying = bool(spans) and bool(_APPLY.search(beat))
|
||||
releasing = bool(_RELEASE.search(beat))
|
||||
|
||||
if applying:
|
||||
wearer = _wearer(beat, who, subject)
|
||||
p = self.person(wearer)
|
||||
if applying or releasing:
|
||||
# MODIFIERS BIND TO THE NEAREST ITEM. "handcuffs her wrists behind
|
||||
# her back and locks a steel collar around her neck, chained to the
|
||||
# wall" carries two modifiers and two items; giving both modifiers
|
||||
# to both items produced handcuffs chained to a wall they were never
|
||||
# near, and a collar held behind a back.
|
||||
spans = hardware_spans(beat)
|
||||
for canon, part, written, at in spans:
|
||||
clause, lo = _clause_at(beat, at, boundaries)
|
||||
item_at = at - lo
|
||||
apply_at = max((m.start() for m in _APPLY.finditer(clause)
|
||||
if m.start() <= item_at), default=-1)
|
||||
release_at = max((m.start() for m in _RELEASE.finditer(clause)
|
||||
if m.start() <= item_at), default=-1)
|
||||
if apply_at < 0 and release_at < 0:
|
||||
continue
|
||||
local_who = names_in(clause, cast)
|
||||
wearer = _wearer(clause, local_who or who, subject)
|
||||
p = self.person(wearer)
|
||||
if release_at >= 0:
|
||||
keys = [k for k in list(p.hardware) if k[0] == canon]
|
||||
for key in keys:
|
||||
changed["released"].append((wearer, p.hardware.pop(key)))
|
||||
continue
|
||||
# KEYED BY THE PAIR. A chain on the ankles and a chain on the
|
||||
# wrists are two restraints; keyed by name alone the second
|
||||
# overwrote the first and one of them was never drawn again.
|
||||
@@ -1446,7 +1652,7 @@ class SceneState:
|
||||
# Its anchor needs no transferring either: with the tether gone from
|
||||
# the spans, the anchor binds to the nearest remaining item, which
|
||||
# is the one it was always describing.
|
||||
elif releasing:
|
||||
if releasing and not spans:
|
||||
# Whoever is actually wearing it. "The guard unlocks the handcuffs"
|
||||
# names only the agent, and taking the subject there tried to
|
||||
# release hardware from the man holding the key.
|
||||
@@ -1457,12 +1663,7 @@ class SceneState:
|
||||
# Released by NAME, whatever part it is on: an unlocking beat says
|
||||
# "unlocks the chain", not which of two chains, and matching the
|
||||
# pair left one fastened forever.
|
||||
_kinds = {c for c, _pt, _w in hw}
|
||||
named = [k for k in list(p.hardware) if k[0] in _kinds]
|
||||
if named:
|
||||
for key in named:
|
||||
changed["released"].append((wearer, p.hardware.pop(key)))
|
||||
elif re.search(r"\b(?:them|it|her|him|everything|all\s+of\s+it)\b",
|
||||
if re.search(r"\b(?:them|it|her|him|everything|all\s+of\s+it)\b",
|
||||
beat, re.I):
|
||||
# "the guard releases her" names no item, so all of it comes off.
|
||||
while p.hardware:
|
||||
@@ -1472,12 +1673,26 @@ class SceneState:
|
||||
# to be named -- a bare "she undresses" says nothing about which garment,
|
||||
# and guessing is how a garment came off a beat before the beat that
|
||||
# took it off.
|
||||
wearer_g = _wearer(beat, who, subject) if len(who) > 1 else subject
|
||||
if wearer_g:
|
||||
p = self.person(wearer_g)
|
||||
for g in garments_in(beat):
|
||||
if subject:
|
||||
for m in garments:
|
||||
g = f"{(m.group(1) or '').strip()} {m.group(2)}".strip().lower()
|
||||
key = _garment_key(g)
|
||||
if _TAKES_OFF.search(beat):
|
||||
clause, lo = _clause_at(beat, m.start(), boundaries)
|
||||
item_at = m.start() - lo
|
||||
actions = [(x.start(), "off") for x in _TAKES_OFF.finditer(clause)
|
||||
if x.start() <= item_at]
|
||||
actions += [(x.start(), "on") for x in _PUTS_ON.finditer(clause)
|
||||
if x.start() <= item_at]
|
||||
actions += [(x.start(), "aside") for x in _DISPLACES.finditer(clause)
|
||||
if x.start() <= item_at]
|
||||
action = max(actions, default=(-1, ""))[1]
|
||||
if action == "aside" and _OPENS_GARMENT.search(clause[:item_at]):
|
||||
if _COMPLETES_OFF.search(re.split(r"[.;!?]", beat[m.end():])[0]):
|
||||
action = "off"
|
||||
local_who = names_in(clause, cast)
|
||||
wearer_g = _wearer(clause, local_who, subject)
|
||||
p = self.person(wearer_g)
|
||||
if action == "off":
|
||||
if key not in [_garment_key(x) for x in p.removed]:
|
||||
p.removed.append(g)
|
||||
changed["removed"].append((wearer_g, g))
|
||||
@@ -1485,7 +1700,7 @@ class SceneState:
|
||||
p.displaced = [x for x in p.displaced
|
||||
if _garment_key(x) != key]
|
||||
_bare_on(p, region_of(g))
|
||||
elif _PUTS_ON.search(beat):
|
||||
elif action == "on":
|
||||
if key not in [_garment_key(x) for x in p.worn]:
|
||||
p.worn.append(g)
|
||||
changed["worn"].append((wearer_g, g))
|
||||
@@ -1496,7 +1711,7 @@ class SceneState:
|
||||
# dresses is told for the rest of the film that the region is
|
||||
# bare, over the garment she just put on.
|
||||
_bare_off(p, region_of(g))
|
||||
elif _DISPLACES.search(beat):
|
||||
elif action == "aside":
|
||||
if key not in [_garment_key(x) for x in p.displaced]:
|
||||
p.displaced.append(g)
|
||||
changed["displaced"].append((wearer_g, g))
|
||||
@@ -1506,7 +1721,11 @@ class SceneState:
|
||||
# ever said what was on the chest.
|
||||
_nude = nudity_in(beat)
|
||||
if _nude:
|
||||
for n in (who or ([subject] if subject else [])):
|
||||
nude_at = min((m.start() for rx, _regions in _NUDITY_RX
|
||||
for m in rx.finditer(beat)), default=len(beat))
|
||||
located = [(abs(beat.find(n) - nude_at), n) for n in who if beat.find(n) >= 0]
|
||||
owners = [min(located)[1]] if located else ([subject] if subject else [])
|
||||
for n in owners:
|
||||
q = self.person(n)
|
||||
_bare_on(q, _nude)
|
||||
# ...and it takes the garments OFF. Saying somebody is topless
|
||||
|
||||
@@ -0,0 +1,739 @@
|
||||
# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos
|
||||
# Copyright (c) 2026 Smite79. All rights reserved.
|
||||
# Redistribution, in whole or in part, requires written permission.
|
||||
# This notice may not be removed or altered. See LICENSE.
|
||||
"""Sampling, decoding, resizing, memory handling, and frame assembly."""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import nodes
|
||||
import comfy.utils
|
||||
import comfy.sample
|
||||
import comfy.samplers
|
||||
import comfy.nested_tensor
|
||||
import comfy.model_management as mm
|
||||
import latent_preview
|
||||
|
||||
|
||||
class FrameAccumulator:
|
||||
"""Build the final frame tensor once, retaining overflow only when necessary."""
|
||||
|
||||
def __init__(self, capacity, dtype, store_on_cpu):
|
||||
self.capacity = int(capacity)
|
||||
self.dtype = dtype
|
||||
self.store_on_cpu = bool(store_on_cpu)
|
||||
self.tensor = None
|
||||
self.used = 0
|
||||
self.overflow = []
|
||||
|
||||
def add(self, frames):
|
||||
count = int(frames.shape[0])
|
||||
if self.tensor is None and count:
|
||||
device = torch.device("cpu") if self.store_on_cpu else frames.device
|
||||
self.tensor = torch.empty(
|
||||
(max(count, self.capacity),) + tuple(frames.shape[1:]),
|
||||
dtype=self.dtype, device=device)
|
||||
if (not self.overflow and self.tensor is not None
|
||||
and self.used + count <= self.tensor.shape[0]):
|
||||
self.tensor[self.used:self.used + count].copy_(frames)
|
||||
self.used += count
|
||||
return
|
||||
self.overflow.append(frames.to("cpu", self.dtype, copy=True)
|
||||
if self.store_on_cpu else frames)
|
||||
|
||||
def release(self):
|
||||
"""Drop every tensor held, now, rather than whenever the collector gets to it.
|
||||
|
||||
On an interrupt the render unwinds through frames the collector tears down in
|
||||
its own order, and a large video buffer freed after the models it was sized
|
||||
against have already gone is a free the allocator cannot explain. Deliberately
|
||||
does NOT empty the cache: that is another CUDA call, and if the context is
|
||||
already in a sticky error state it is one more thing to abort inside."""
|
||||
self.tensor = None
|
||||
self.overflow = []
|
||||
self.used = 0
|
||||
|
||||
def finish(self):
|
||||
if not self.overflow:
|
||||
if self.tensor is None:
|
||||
return torch.cat(self.overflow, dim=0)
|
||||
if self.used == self.tensor.shape[0]:
|
||||
out = self.tensor
|
||||
else:
|
||||
# COMPACT, never a slice. A slice of a larger buffer keeps the WHOLE
|
||||
# buffer's storage alive, which is the retention this class exists to
|
||||
# prevent -- and test_the_chain_is_never_held_twice measures exactly
|
||||
# that, demanding no unused bytes behind the returned tensor.
|
||||
#
|
||||
# There is slack because the capacity is now an upper bound: it can no
|
||||
# longer assume trim_seam drops a frame at every seam, since a shot that
|
||||
# opens on no keyframe keeps its first frame. Over-allocating by at most
|
||||
# one frame per seam and compacting once is the bounded cost. The
|
||||
# alternative -- an exact guess that can be too small -- drops into the
|
||||
# overflow list, which with cleanup_between_shots off holds every shot's
|
||||
# decoded frames live on the GPU until the end of the run.
|
||||
out = torch.empty((self.used,) + tuple(self.tensor.shape[1:]),
|
||||
dtype=self.dtype, device=self.tensor.device)
|
||||
out.copy_(self.tensor[:self.used])
|
||||
self.tensor = None
|
||||
return out
|
||||
|
||||
extra = sum(int(piece.shape[0]) for piece in self.overflow)
|
||||
reference = self.tensor if self.tensor is not None else self.overflow[0]
|
||||
out = torch.empty((self.used + extra,) + tuple(reference.shape[1:]),
|
||||
dtype=self.dtype, device=reference.device)
|
||||
if self.tensor is not None and self.used:
|
||||
out[:self.used].copy_(self.tensor[:self.used])
|
||||
at = self.used
|
||||
while self.overflow:
|
||||
piece = self.overflow.pop(0)
|
||||
count = int(piece.shape[0])
|
||||
out[at:at + count].copy_(piece)
|
||||
at += count
|
||||
self.tensor = None
|
||||
return out
|
||||
|
||||
|
||||
H3_FPS = 24 # H3 renders 24 fps, always
|
||||
|
||||
|
||||
AUDIO_LATENT_FPS = 40 # audio latent frames per second
|
||||
|
||||
|
||||
AUTO_TILE_T = 8 # temporal chunk for a tiled decode
|
||||
|
||||
|
||||
MAX_FRAMES = 362 # H3's own ceiling (~15s)
|
||||
|
||||
|
||||
CANVAS_MULTIPLE = 32
|
||||
|
||||
|
||||
REF_IMAGE_SHORT_EDGE = 2048
|
||||
|
||||
|
||||
def align_frame_count(n):
|
||||
"""Up to the next valid H3 frame count. The grid is 17k+5."""
|
||||
n = max(5, int(n))
|
||||
while n % 17 != 5:
|
||||
n += 1
|
||||
return min(n, MAX_FRAMES)
|
||||
|
||||
|
||||
def video_latent_t(fc):
|
||||
return 2 if fc <= 5 else ((fc - 5) // 17) * 5 + 2
|
||||
|
||||
|
||||
def temporal_shape(length, fps=H3_FPS):
|
||||
"""(frame count, video latent frames, audio latent frames) for a shot.
|
||||
|
||||
`fps` is accepted but deliberately IGNORED: the audio latent has to line up
|
||||
with 24 fps video or the shot's sound is stretched against its picture."""
|
||||
fc = align_frame_count(length)
|
||||
return fc, video_latent_t(fc), round(fc / H3_FPS * AUDIO_LATENT_FPS)
|
||||
|
||||
|
||||
def ref_image_canvas(w, h, gen_w, gen_h, mode="match"):
|
||||
"""Pure: the (width, height) a reference image is encoded at.
|
||||
|
||||
'match' scales it (DOWN only, aspect kept) to the generation's pixel area, so a
|
||||
reference costs about as much as one frame of the shot. 'max' goes to the
|
||||
reference pipeline's 2048 short edge for the best identity fidelity, which on a
|
||||
long chain is several times slower because the rows are re-attended every step
|
||||
of every shot. Never upscales: a small reference stays small."""
|
||||
w, h = max(1, int(w)), max(1, int(h))
|
||||
if mode == "max":
|
||||
scale = min(1.0, REF_IMAGE_SHORT_EDGE / min(w, h))
|
||||
else:
|
||||
scale = min(1.0, math.sqrt((int(gen_w) * int(gen_h)) / float(w * h)))
|
||||
snap = lambda v: max(CANVAS_MULTIPLE, round(v * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)
|
||||
return snap(w), snap(h)
|
||||
|
||||
|
||||
def _resize(image, width, height, crop):
|
||||
s = image[..., :3].movedim(-1, 1)
|
||||
s = comfy.utils.common_upscale(s, width, height, "lanczos", crop)
|
||||
return s.movedim(1, -1)
|
||||
|
||||
|
||||
def _empty_av_latent(width, height, length, fps, batch_size=1):
|
||||
fc, lt, at = temporal_shape(length, fps)
|
||||
video = torch.zeros([batch_size, 24, lt, height // 16, width // 16], device=mm.intermediate_device())
|
||||
audio = torch.zeros([batch_size, 32, 2, at], device=mm.intermediate_device())
|
||||
return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, fc
|
||||
|
||||
|
||||
def _auto_tile_t(n_latent_frames, requested=None):
|
||||
"""Temporal tile for a tiled decode. An explicit value wins.
|
||||
|
||||
The decode_tile_frames widget is gone, so this is where the value comes from
|
||||
now. It has to come from somewhere: ComfyUI's decode_tiled_3d defaults tile_t
|
||||
to 999, i.e. SPATIAL tiles only, and expanding the whole clip's time axis at
|
||||
once is the single largest allocation in a run. A "tiled" decode that keeps the
|
||||
full temporal extent barely lowers the peak, so the OOM retry that switches
|
||||
tiling on was, without this, retrying with almost the same footprint."""
|
||||
if requested:
|
||||
return int(requested)
|
||||
n = int(n_latent_frames or 0)
|
||||
return AUTO_TILE_T if n > AUTO_TILE_T else None
|
||||
|
||||
|
||||
def _decode_video(vae, out_latent, tiled, free_first=None, tile_t=None, tile_xy=None,
|
||||
keep=()):
|
||||
"""Decode the video latent.
|
||||
|
||||
`free_first` is the diffusion model: sampling is finished, and the video VAE
|
||||
needs the room for THIS decode -- the free runs immediately before it, not to
|
||||
make room for the next shot. On a card where the DiT is most of the VRAM, the
|
||||
decode does not fit until it goes.
|
||||
|
||||
`keep` is what must NOT be evicted on the way. It was `keep_loaded=[]`, which
|
||||
unloaded every resident model -- including the video VAE, which ComfyUI then
|
||||
reloaded three lines later to run the decode. An evict-and-reload of the thing
|
||||
about to be used, once per shot, on every card. Peak VRAM is identical either
|
||||
way, since the VAE has to be resident to decode; the round trip was pure cost.
|
||||
|
||||
memory_required is ASKED FOR HONESTLY, which it was not. It was 1e30, and
|
||||
free_memory computes `memory_to_free = memory_required - get_free_memory(device)`
|
||||
(model_management.py:887), so 1e30 means "unload everything not in keep_loaded",
|
||||
every shot, in full -- skipping partially_unload entirely.
|
||||
|
||||
What that evicts is the DiT, three lines before the next shot needs it again. On
|
||||
a machine whose RAM is already full of finished frames there is nowhere for it to
|
||||
go but disk, so the reload is a read from the drive, once per shot. Reported as
|
||||
thrashing that slows the preload, and it is exactly that: the same weights being
|
||||
read back at every boundary.
|
||||
|
||||
The VAE knows what its own decode costs -- ComfyUI sizes it with
|
||||
memory_used_decode and uses that number everywhere else. Asked for that instead,
|
||||
a card with headroom frees NOTHING and the DiT simply stays. A card without
|
||||
headroom frees what it needs and no more, which is what partially_unload is for.
|
||||
1e30 remains the fallback for a VAE that cannot estimate itself."""
|
||||
latent = out_latent["samples"]
|
||||
if latent.is_nested:
|
||||
latent = latent.unbind()[0]
|
||||
if free_first is not None:
|
||||
try:
|
||||
mm.free_memory(_decode_headroom(vae, latent), mm.get_torch_device(),
|
||||
keep_loaded=_resident(keep or (vae,)))
|
||||
except Exception:
|
||||
pass
|
||||
# A VAE THAT ALREADY TILES DOES NOT NEED TO BE ASKED TO, AND ASKING COSTS 3x.
|
||||
#
|
||||
# MiniMaxH3VideoVAE.decode_tiled is, in full:
|
||||
#
|
||||
# def decode_tiled(self, z, **kwargs):
|
||||
# return self.decode(z)
|
||||
#
|
||||
# Every tile_t/overlap_t/tile_x/tile_y this function computes is discarded, so
|
||||
# the tiling the widget promises is not happening here -- the model tiles
|
||||
# internally either way (256px spatial, 17-frame temporal), which is why
|
||||
# comfy/sd.py sets handles_tiling on it.
|
||||
#
|
||||
# What the detour costs is the OUTPUT BUFFER. comfy's VAE.decode preallocates
|
||||
# ONE result at vae_output_dtype and hands it to the model as output_buffer=,
|
||||
# and MiniMaxH3VideoVAE.decode_temporal writes finalized chunks straight into
|
||||
# it. Going through decode_tiled instead reaches _decode_tiled_owned, which
|
||||
# calls the model with output_buffer=None -- so decode_temporal allocates its
|
||||
# own at torch.float32 -- and then makes an fp16 `copy=True` of that. Two
|
||||
# buffers, the larger of them at double width:
|
||||
#
|
||||
# tiled : fp32 2.60GB + fp16 copy 1.30GB = 3.90GB per shot
|
||||
# decode: one preallocated fp16 = 1.30GB per shot
|
||||
#
|
||||
# at 362 frames of 1056x608. Every shot, on the node's own default.
|
||||
#
|
||||
# So: when the VAE owns its tiling AND can be written into, the un-tiled call IS
|
||||
# the tiled one, minus the copies. Anything else keeps the old path -- this is a
|
||||
# detour around a detour, not a claim that tiling is useless.
|
||||
_owns_tiling = bool(getattr(vae, "handles_tiling", False) and getattr(
|
||||
getattr(vae, "first_stage_model", None), "comfy_has_chunked_io", False))
|
||||
if tiled and _owns_tiling:
|
||||
imgs = vae.decode(latent)
|
||||
elif tiled:
|
||||
# Temporal + spatial tiling. Without tile_t the VAE expands the WHOLE latent
|
||||
# clip at once, which on a 243-frame 1344x768 shot is the single largest
|
||||
# allocation in the run -- and on an unpruned checkpoint that is already
|
||||
# streaming, it is what tips the card over. Decoding in temporal chunks
|
||||
# trades a little speed for a much lower peak; None keeps ComfyUI's defaults.
|
||||
args = {}
|
||||
tile_t = _auto_tile_t(latent.shape[2] if latent.ndim >= 5 else 0, tile_t)
|
||||
if tile_t:
|
||||
args["tile_t"] = int(tile_t)
|
||||
args["overlap_t"] = max(1, int(tile_t) // 8)
|
||||
if tile_xy:
|
||||
args["tile_x"] = int(tile_xy)
|
||||
args["tile_y"] = int(tile_xy)
|
||||
try:
|
||||
imgs = vae.decode_tiled(latent, **args) if args else vae.decode_tiled(latent)
|
||||
except TypeError:
|
||||
imgs = vae.decode_tiled(latent) # older signature without tile_t
|
||||
else:
|
||||
imgs = vae.decode(latent)
|
||||
if len(imgs.shape) == 5:
|
||||
imgs = imgs.reshape(-1, imgs.shape[-3], imgs.shape[-2], imgs.shape[-1])
|
||||
return imgs
|
||||
|
||||
|
||||
def _decode_audio(audio_vae, out_latent):
|
||||
latent = out_latent["samples"]
|
||||
if latent.is_nested:
|
||||
latent = latent.unbind()[-1]
|
||||
audio = audio_vae.decode(latent).movedim(-1, 1)
|
||||
std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0
|
||||
std[std < 1.0] = 1.0
|
||||
audio = audio / std
|
||||
sr = getattr(audio_vae, "audio_sample_rate_output", getattr(audio_vae, "audio_sample_rate", 44100))
|
||||
return {"waveform": audio, "sample_rate": sr}
|
||||
|
||||
|
||||
def _is_oom(e):
|
||||
return isinstance(e, torch.cuda.OutOfMemoryError) or "out of memory" in str(e).lower()
|
||||
|
||||
|
||||
def _deep_cleanup():
|
||||
"""Release cached VRAM between shots so a long chain does not accumulate and OOM.
|
||||
|
||||
It unloads NOTHING. soft_empty_cache(force) ignores `force` in current ComfyUI
|
||||
(model_management.py:2050) -- the body only reaches empty_cache() and
|
||||
ipc_collect() -- so this drops cached blocks, not models. The `True` is kept
|
||||
only for older builds that read it; the older comment here claimed this took an
|
||||
unload_all_models path, and it does not."""
|
||||
try:
|
||||
mm.soft_empty_cache(True)
|
||||
except TypeError:
|
||||
mm.soft_empty_cache()
|
||||
try:
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.ipc_collect()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
DECODE_HEADROOM = 1.25 # over ComfyUI's own estimate, for working allocations
|
||||
|
||||
|
||||
SAMPLE_HEADROOM = 1.35 # likewise for sampling, which is the longer stretch
|
||||
|
||||
|
||||
def _decode_headroom(vae, latent):
|
||||
"""VRAM this decode actually needs, by the VAE's own estimate. 1e30 if unknown.
|
||||
|
||||
ComfyUI sizes every VAE with memory_used_decode and uses that number itself, so
|
||||
it is the honest figure to hand free_memory. The alternative -- and what was here
|
||||
-- is 1e30, which means "unload everything" and evicts the DiT before every
|
||||
decode, three lines before the next shot reloads it.
|
||||
|
||||
1e30 on failure rather than 0: a bad estimate that frees too little turns a slow
|
||||
render into an OOM, and a wrong guess should fall back to the behaviour that has
|
||||
been running, not to no freeing at all."""
|
||||
try:
|
||||
dtype = getattr(vae, "vae_dtype", None) or latent.dtype
|
||||
need = float(vae.memory_used_decode(tuple(latent.shape), dtype))
|
||||
if need > 0:
|
||||
return need * DECODE_HEADROOM
|
||||
except Exception:
|
||||
pass
|
||||
return 1e30
|
||||
|
||||
|
||||
def _resident(models):
|
||||
"""The LoadedModel entries ComfyUI currently holds for `models`.
|
||||
|
||||
That is the form free_memory's keep_loaded wants: it compares against the
|
||||
entries in current_loaded_models, not against the ModelPatcher objects a node
|
||||
is holding. Anything not matched is simply not kept, so a model that is not
|
||||
resident costs nothing here."""
|
||||
out = []
|
||||
for lm in list(getattr(mm, "current_loaded_models", [])):
|
||||
for m in models or ():
|
||||
if m is None:
|
||||
continue
|
||||
try:
|
||||
if lm.model is m or getattr(lm, "model", None) is getattr(m, "model", None):
|
||||
if lm not in out:
|
||||
out.append(lm)
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def _image_out_dtype():
|
||||
"""The dtype ComfyUI itself hands between nodes on THIS install.
|
||||
|
||||
The join used to end in a hard-coded .float(), commented "back to what every
|
||||
downstream node expects". That was true when it was written and is not a
|
||||
constant: ComfyUI has --fp16-intermediates, and on an install running it the
|
||||
VAE's own decode already returns fp16 -- VAE.vae_output_dtype() IS
|
||||
model_management.intermediate_dtype() (comfy/sd.py) -- as do EmptyLatentImage
|
||||
and the rest of nodes.py. So on that install the node was taking frames the
|
||||
VAE handed it in fp16, widening them to fp32 nothing had asked for, and
|
||||
handing them to nodes whose own convention is fp16.
|
||||
|
||||
It is the largest thing this node holds, so the widening is not free: the
|
||||
2580-frame chain costed at the join is 9.3GB as fp16 and 18.5GB as fp32,
|
||||
against 44.6GB of staged weights on a 62GB machine -- which is the difference
|
||||
between the render finishing and the OOM killer taking the server. Reported as
|
||||
exactly that, twice.
|
||||
|
||||
Asked, not assumed, and never widened: whatever ComfyUI says it wants between
|
||||
nodes is what the chain is built in. An install with the flag off is told
|
||||
float32 and gets float32, byte for byte what it got before. Older builds have
|
||||
no intermediate_dtype at all, so the fallback is the old constant."""
|
||||
try:
|
||||
return mm.intermediate_dtype()
|
||||
except Exception:
|
||||
return torch.float32
|
||||
|
||||
|
||||
def _evict_all_but(keep_model, latent=None):
|
||||
"""Unload every model EXCEPT the diffusion model from the GPU.
|
||||
|
||||
This is the fix for VRAM ratcheting across a long chain. soft_empty_cache()
|
||||
only drops the CUDA allocator's cached blocks -- it does NOT unload models, so
|
||||
ComfyUI keeps the Qwen3-VL text encoder (~14.6GB) and both VAEs resident in
|
||||
current_loaded_models alongside the DiT. Each shot re-encodes the prompt
|
||||
(text encoder), encodes the handoff keyframe (video VAE), then samples (DiT),
|
||||
so all three compete for the card.
|
||||
|
||||
ComfyUI does free ahead of each load -- load_models_gpu() calls free_memory()
|
||||
for what it is about to need (model_management.py:975), so the weight path is
|
||||
not purely reactive. What it cannot size for is a long chain's ACTIVATIONS on
|
||||
a card where the DiT is most of the VRAM. Freeing explicitly, right after
|
||||
conditioning is built and before sampling, keeps only what the sampler needs.
|
||||
|
||||
ASKED FOR HONESTLY, and this is the expensive one. free_memory computes
|
||||
`memory_to_free = memory_required - get_free_memory(device)`, so 1e30 meant
|
||||
"unload everything but the DiT" on every shot, unconditionally -- on a 48GB card
|
||||
with room for all of it as readily as on a 16GB one. What it unloads is the
|
||||
~14.6GB text encoder and both VAEs, and the next shot re-encodes the prompt and
|
||||
the handoff keyframe, so all three come straight back. On a machine whose RAM is
|
||||
already full of finished frames they come back from DISK, once per shot, which is
|
||||
the thrashing this was reported as.
|
||||
|
||||
The DiT can size its own activations -- memory_required(shape) is what ComfyUI
|
||||
itself calls before a load -- so ask for that. A card with room frees nothing and
|
||||
keeps the encoder resident; a card without frees exactly as much as it must.
|
||||
1e30 stays the fallback, because a bad estimate that frees too little turns a
|
||||
slow render into an OOM."""
|
||||
need = 1e30
|
||||
try:
|
||||
if latent is not None:
|
||||
shape = latent["samples"].shape if isinstance(latent, dict) else latent.shape
|
||||
need = float(keep_model.model.memory_required(tuple(shape))) * SAMPLE_HEADROOM
|
||||
if not (need > 0):
|
||||
need = 1e30
|
||||
except Exception:
|
||||
need = 1e30
|
||||
try:
|
||||
mm.free_memory(need, mm.get_torch_device(),
|
||||
keep_loaded=_resident([keep_model]))
|
||||
except Exception:
|
||||
try:
|
||||
mm.soft_empty_cache(True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _sample_on_sigmas(model, seed, cfg, sampler_name, positive, negative, latent, sigmas):
|
||||
"""common_ksampler, driven by an EXTERNAL sigma schedule.
|
||||
|
||||
common_ksampler derives its sigmas from (sampler_name, scheduler, steps, denoise)
|
||||
and takes no schedule argument, so a schedule computed anywhere else cannot
|
||||
reach it. Under PDD that is fatal rather than merely inconvenient: the heads
|
||||
accept only their nine trained boundaries, and re-deriving the grid from
|
||||
widgets means hitting it by coincidence and losing it again the moment a step
|
||||
count changes.
|
||||
|
||||
Mirrors nodes.common_ksampler's noise / mask / callback handling exactly -- the
|
||||
only substitution is comfy.sample.sample_custom for comfy.sample.sample."""
|
||||
latent_image = latent["samples"]
|
||||
latent_image = comfy.sample.fix_empty_latent_channels(
|
||||
model, latent_image,
|
||||
latent.get("downscale_ratio_spacial", None),
|
||||
latent.get("downscale_ratio_temporal", None))
|
||||
noise = comfy.sample.prepare_noise(latent_image, seed, latent.get("batch_index"))
|
||||
# `steps` here only sizes the progress bar -- the schedule is `sigmas`, whose
|
||||
# step count is one less than its length (the trailing 0.0 is an endpoint).
|
||||
callback = latent_preview.prepare_callback(model, max(len(sigmas) - 1, 1))
|
||||
samples = comfy.sample.sample_custom(
|
||||
model, noise, cfg, comfy.samplers.sampler_object(sampler_name), sigmas,
|
||||
positive, negative, latent_image,
|
||||
noise_mask=latent.get("noise_mask"), callback=callback,
|
||||
disable_pbar=not comfy.utils.PROGRESS_BAR_ENABLED, seed=seed)
|
||||
out = latent.copy()
|
||||
out.pop("downscale_ratio_spacial", None)
|
||||
out.pop("downscale_ratio_temporal", None)
|
||||
out["samples"] = samples
|
||||
return out
|
||||
|
||||
|
||||
RESIZE_CHUNK = 32
|
||||
|
||||
|
||||
def _stream_chunks(total):
|
||||
"""A collector that writes upscaled chunks into ONE destination as they land.
|
||||
|
||||
Both chunk loops in _upscale_frames used `out.append(...)` then
|
||||
`frames = torch.cat(out, dim=0)`. That is the shape the finished-chain join was
|
||||
rebuilt to stop, at a LARGER size: the list holds the whole upscaled chain and
|
||||
the cat allocates a second one, both live at the cat, and `out` is a local that
|
||||
is never cleared -- so it survives the cat, survives the trailing resize, and is
|
||||
still bound at the return. Meanwhile the CALLER's pre-upscale chain cannot be
|
||||
dropped either, because `part = frames[s:s+batch]` is a view into it.
|
||||
|
||||
At 2580 frames of 1056x608 that is 9.26GB per copy per doubling: 37GB x2 at 2x,
|
||||
and 148GB x2 with the RealESRGAN_x4plus that is sitting in models/upscale_models.
|
||||
Preallocating from the first chunk and copying into it removes exactly one of
|
||||
those two, and drops the list at the same time.
|
||||
|
||||
The destination is sized from the FIRST chunk, so the model's scale factor does
|
||||
not have to be known in advance, and the frame count is the caller's own -- an
|
||||
upscaler changes width and height, never the number of frames."""
|
||||
state = {"dst": None, "at": 0}
|
||||
|
||||
def put(piece):
|
||||
if state["dst"] is None:
|
||||
state["dst"] = torch.empty((int(total),) + tuple(piece.shape[1:]),
|
||||
dtype=piece.dtype, device=piece.device)
|
||||
k = int(piece.shape[0])
|
||||
end = min(state["at"] + k, state["dst"].shape[0])
|
||||
if end > state["at"]:
|
||||
state["dst"][state["at"]:end].copy_(piece[:end - state["at"]])
|
||||
state["at"] = end
|
||||
|
||||
def done():
|
||||
d, at = state["dst"], state["at"]
|
||||
if d is None:
|
||||
return None
|
||||
return d if at == d.shape[0] else d[:at]
|
||||
|
||||
return put, done
|
||||
|
||||
|
||||
def _resize_short_edge(frames, target, method="lanczos", chunk=0):
|
||||
"""Resize a [B,H,W,C] frame batch so its short edge == target (keeping aspect,
|
||||
snapped to /32). Plain high-quality resize -- enlarges, doesn't add detail.
|
||||
|
||||
IN CHUNKS, BECAUSE LANCZOS IS FOUR FULL-LENGTH COPIES. The whole chain went
|
||||
into one common_upscale call, and comfy.utils.lanczos is three successive list
|
||||
comprehensions over every frame at once:
|
||||
|
||||
images = [Image.fromarray(...) for image in samples] # N at source size
|
||||
images = [image.resize(...) for image in images] # N at target size
|
||||
images = [torch.from_numpy(np.array(im).astype(np.float32)/255.) ...]
|
||||
result = torch.stack(images)
|
||||
return result.to(samples.device, samples.dtype)
|
||||
|
||||
A comprehension builds the new list completely before rebinding the name, so at
|
||||
each rebind BOTH are live; then torch.stack allocates a full copy while its list
|
||||
still exists, and .to() allocates the result while the stack still exists. Note
|
||||
the astype(np.float32): the input is fp16 but the two largest transients are at
|
||||
DOUBLE its width. At 2580 frames to a 1080 short edge that peaked around 147GB
|
||||
to produce a 29GB result, and it fires on a DOWNSCALE too.
|
||||
|
||||
Chunked, the peak is the result plus one chunk's worth of that machinery. It is
|
||||
bit-identical: PIL resizes each frame independently, so per-chunk and per-chain
|
||||
give the same pixels. The early return for an already-correct size is kept, so
|
||||
the common no-op case still allocates nothing."""
|
||||
b, h, w, c = frames.shape
|
||||
if min(h, w) == target:
|
||||
return frames
|
||||
if h <= w:
|
||||
nh = target; nw = max(32, int(round(target * w / h / 32) * 32))
|
||||
else:
|
||||
nw = target; nh = max(32, int(round(target * h / w / 32) * 32))
|
||||
step = max(1, int(chunk) or RESIZE_CHUNK)
|
||||
out = torch.empty((b, nh, nw, c), dtype=frames.dtype, device=frames.device)
|
||||
for i in range(0, b, step):
|
||||
part = comfy.utils.common_upscale(
|
||||
frames[i:i + step].movedim(-1, 1), nw, nh, method, "disabled")
|
||||
out[i:i + step].copy_(part.movedim(1, -1))
|
||||
del part
|
||||
return out
|
||||
|
||||
|
||||
def _upscale_frames(frames, mode, model_name, target_short_edge, batch=4):
|
||||
"""Optional post-pass upscale of the finished frames (on CPU).
|
||||
mode 'model' : run a ComfyUI upscale model (Real-ESRGAN/UltraSharp class)
|
||||
via the registered loader+apply nodes, chunked with cleanup
|
||||
so 2000+ frames don't OOM; then fit to target short edge.
|
||||
mode 'rtx' : NVIDIA RTX Video Super Resolution (Tensor Cores; fastest,
|
||||
best quality for video -- needs Nvidia_RTX_Nodes_ComfyUI).
|
||||
mode 'lanczos' : plain high-quality resize to the target short edge.
|
||||
Any failure falls back to lanczos (or the raw frames), so it never breaks a
|
||||
render. Returns (frames, note). NOTE: this SHARPENS/ENLARGES; it does not
|
||||
reconstruct video detail the way a second-model (LTX 2.3) pass does."""
|
||||
if mode == "off" or frames is None or getattr(frames, "shape", [0])[0] == 0:
|
||||
return frames, ""
|
||||
note = ""
|
||||
if mode == "rtx":
|
||||
# NVIDIA RTX Video Super Resolution (Comfy-Org/Nvidia_RTX_Nodes_ComfyUI).
|
||||
# Runs on RTX Tensor Cores -- far faster than ESRGAN-class models and
|
||||
# generally cleaner on video, though like them it enhances/enlarges rather
|
||||
# than reconstructing detail (an LTX 2.3 re-generation does that).
|
||||
try:
|
||||
rtx = (_find_node(["rtx", "video", "super"]) or _find_node(["rtxvideosuperresolution"])
|
||||
or _find_node(["rtx", "upscale"]))
|
||||
if rtx is None:
|
||||
raise RuntimeError("RTX node not installed (Nvidia_RTX_Nodes_ComfyUI)")
|
||||
scale = 2
|
||||
if target_short_edge and int(target_short_edge) > 0:
|
||||
cur = min(frames.shape[1], frames.shape[2])
|
||||
if cur > 0:
|
||||
scale = max(1, min(4, int(round(int(target_short_edge) / cur))))
|
||||
_put, _done = _stream_chunks(frames.shape[0])
|
||||
n = frames.shape[0]
|
||||
step = max(1, int(batch))
|
||||
for st in range(0, n, step):
|
||||
part = frames[st:st + step]
|
||||
res = None
|
||||
for kw in ({"image": part, "scale": scale}, {"images": part, "scale": scale},
|
||||
{"image": part, "scale_factor": scale}, {"image": part}):
|
||||
try:
|
||||
res = _invoke_node(rtx, **kw); break
|
||||
except TypeError:
|
||||
continue
|
||||
if res is None:
|
||||
raise RuntimeError("RTX node signature not recognized")
|
||||
_put(res.detach().to("cpu"))
|
||||
del res, part
|
||||
_deep_cleanup()
|
||||
frames = _done()
|
||||
note = f"RTX Video Super Resolution x{scale}"
|
||||
if target_short_edge and int(target_short_edge) > 0:
|
||||
frames = _resize_short_edge(frames, int(target_short_edge))
|
||||
note += f"; fit to {int(target_short_edge)}px short edge"
|
||||
return frames, note
|
||||
except Exception as e:
|
||||
mode = "model"
|
||||
note = f"RTX upscale unavailable ({e}); fell back to model/lanczos"
|
||||
if mode == "model" and model_name and model_name != "none":
|
||||
try:
|
||||
loader = _find_node(["upscale", "model", "load"]) or _find_node(["loadupscalemodel"])
|
||||
applier = _find_node(["imageupscale", "model"]) or _find_node(["upscaleimageusingmodel"])
|
||||
if loader is None or applier is None:
|
||||
raise RuntimeError("upscale-model nodes not found")
|
||||
up_model = _invoke_node(loader, model_name=model_name)
|
||||
_put, _done = _stream_chunks(frames.shape[0])
|
||||
n = frames.shape[0]
|
||||
for s in range(0, n, max(1, int(batch))):
|
||||
part = frames[s:s + max(1, int(batch))]
|
||||
res = _invoke_node(applier, upscale_model=up_model, image=part)
|
||||
_put(res.detach().to("cpu"))
|
||||
del res, part
|
||||
_deep_cleanup()
|
||||
frames = _done()
|
||||
note = f"upscaled with {model_name}"
|
||||
except Exception as e:
|
||||
mode = "lanczos"
|
||||
note = f"model upscale unavailable ({e}); used lanczos"
|
||||
if target_short_edge and int(target_short_edge) > 0:
|
||||
try:
|
||||
frames = _resize_short_edge(frames, int(target_short_edge))
|
||||
note = (note + "; " if note else "") + f"fit to {int(target_short_edge)}px short edge"
|
||||
except Exception as e:
|
||||
note = (note + "; " if note else "") + f"resize failed ({e})"
|
||||
elif mode == "lanczos" and not note:
|
||||
note = "lanczos selected but no target set -> unchanged"
|
||||
return frames, note
|
||||
|
||||
|
||||
def _find_node(substrings):
|
||||
"""Find a registered node whose key contains all of `substrings` (lowercased)."""
|
||||
maps = getattr(nodes, "NODE_CLASS_MAPPINGS", {}) or {}
|
||||
for k, v in maps.items():
|
||||
kl = k.lower()
|
||||
if all(s in kl for s in substrings):
|
||||
return v
|
||||
return None
|
||||
|
||||
|
||||
def _invoke_node(cls, **kwargs):
|
||||
"""Call a registered ComfyUI node (V1 FUNCTION or V3 execute) with kwargs and
|
||||
return its first output. Used to reuse ComfyUI's own upscale-model loader/apply
|
||||
so we don't reimplement spandrel loading or tiled scaling."""
|
||||
inst = cls()
|
||||
fn = None
|
||||
if getattr(cls, "FUNCTION", None) and hasattr(inst, cls.FUNCTION):
|
||||
fn = getattr(inst, cls.FUNCTION)
|
||||
else:
|
||||
for cand in ("execute", "upscale", "load_model", "load"):
|
||||
if hasattr(inst, cand):
|
||||
fn = getattr(inst, cand); break
|
||||
if fn is None:
|
||||
raise RuntimeError("no callable entrypoint")
|
||||
out = fn(**kwargs)
|
||||
out = getattr(out, "result", out)
|
||||
return out[0] if isinstance(out, (tuple, list)) else out
|
||||
|
||||
|
||||
# --- THE GRADE THE CHAIN ADDS TO ITSELF -------------------------------------
|
||||
# Every shot boundary decodes a shot, hands its LAST frame over, and re-encodes that as
|
||||
# the next shot's keyframe. The distill reproduces the keyframe faithfully enough to
|
||||
# inherit whatever is already in it and SYNTHESISES frame 0 rather than copying it, so
|
||||
# its own bias lands on top: S_next = a*S + b, a near 1, b above 0. Linear at best,
|
||||
# geometric at worst, invisible shot to shot. And the VAE hard-clips every decode to
|
||||
# 0..1, which makes the expansion a RATCHET -- headroom spent is not recoverable, so it
|
||||
# shows as crushed blacks and blown highlights rather than merely as more contrast.
|
||||
#
|
||||
# These two are the measurement and the correction. Both work per colour channel,
|
||||
# because the clip is per channel: the VAE un-whitens with ImageNet stds before it
|
||||
# clamps, so the 0..1 rails sit at different distances in each channel and the blue
|
||||
# floor and red ceiling bite first. A single luma number would miss the colour half.
|
||||
LEVEL_POOL = 64 # cells per axis the level statistics are measured on
|
||||
|
||||
|
||||
def frame_levels(img):
|
||||
"""(mean, std) per colour channel for one frame, as 3-vectors, or (None, None).
|
||||
|
||||
Area-pooled to LEVEL_POOL first, so a pre-upscale frame and an upscaled one can be
|
||||
compared: pooling measures the PICTURE's levels rather than its resolution. Measured
|
||||
across a 2x resize, std agrees to 0.28% on picture-like content -- and to only 15%
|
||||
on pure noise, because pooling cannot preserve variance that lives entirely at the
|
||||
pixel scale. Real frames are the former, and whatever residual there is cancels
|
||||
anyway: the caller measures the same pipeline difference separately and subtracts it.
|
||||
|
||||
float32 throughout, deliberately: these frames are fp16 under
|
||||
--fp16-intermediates, and an fp16 mean accumulated over a 1344x768 frame biases
|
||||
badly enough to matter at the sizes being corrected here."""
|
||||
x = img
|
||||
if x.dim() == 4:
|
||||
x = x[0]
|
||||
if x.dim() != 3 or int(x.shape[-1]) < 3:
|
||||
return None, None
|
||||
if int(x.shape[0]) < 2 or int(x.shape[1]) < 2:
|
||||
return None, None
|
||||
x = x[..., :3].float().permute(2, 0, 1).unsqueeze(0)
|
||||
p = torch.nn.functional.adaptive_avg_pool2d(x, LEVEL_POOL)[0].reshape(3, -1)
|
||||
return p.mean(dim=1), p.std(dim=1)
|
||||
|
||||
|
||||
# The per-shot motion envelope lived here, measured so the built footsteps could be
|
||||
# timed off the picture. Nothing is built any more -- see the note at the top of
|
||||
# audio.py -- so there is nothing left to time, and a measurement with no reader is a
|
||||
# measurement that rots. Removed with the synthesiser it served.
|
||||
|
||||
|
||||
def apply_levels(img, gain, offset):
|
||||
"""Rescale a frame's contrast and level about its OWN per-channel mean.
|
||||
|
||||
The pivot is the frame's own mean and never a target. That is the whole reason this
|
||||
can run on any scene: a beat that walks into a darker room keeps its darkness,
|
||||
because nothing here knows or cares what the level is -- only how much the last
|
||||
boundary expanded it. Anchoring to shot 1 instead would cancel every deliberate
|
||||
lighting change in the film, which is the opposite failure.
|
||||
|
||||
Clamped into 0..1 because the next thing that happens to this frame is an 8-bit
|
||||
quantisation (comfy.utils.common_upscale goes through a uint8 PIL round trip even
|
||||
at the same size), so there is no headroom outside the range to borrow from."""
|
||||
x = img.float()
|
||||
c = min(3, int(x.shape[-1]))
|
||||
m = x[..., :c].reshape(-1, c).mean(dim=0)
|
||||
g = gain[:c].to(device=x.device, dtype=x.dtype)
|
||||
o = offset[:c].to(device=x.device, dtype=x.dtype)
|
||||
y = x.clone()
|
||||
y[..., :c] = ((x[..., :c] - m) * g + m + o).clamp(0.0, 1.0)
|
||||
return y.to(img.dtype)
|
||||
@@ -0,0 +1,109 @@
|
||||
# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos
|
||||
# Copyright (c) 2026 Smite79. All rights reserved.
|
||||
# Redistribution, in whole or in part, requires written permission.
|
||||
# This notice may not be removed or altered. See LICENSE.
|
||||
"""Per-shot records passed from prompt planning to rendering."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class Shot:
|
||||
prompt: str
|
||||
cast: list[str]
|
||||
speech: bool
|
||||
sounded: bool
|
||||
voiced_only: bool
|
||||
events: list[str]
|
||||
frame_count: int = 0
|
||||
refs: list[object] = field(default_factory=list)
|
||||
line_seconds: float = 0.0 # the planner's estimate of the spoken line, words / WORDS_PER_SEC
|
||||
|
||||
|
||||
@dataclass
|
||||
class ShotPlan:
|
||||
shots: list[Shot] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def prompts(self):
|
||||
return [shot.prompt for shot in self.shots]
|
||||
|
||||
def add(self, prompt, cast, speech, sounded, voiced_only, events):
|
||||
shot = Shot(prompt, list(cast or ()), bool(speech), bool(sounded),
|
||||
bool(voiced_only), list(events or ()))
|
||||
self.shots.append(shot)
|
||||
|
||||
def set_frame_counts(self, counts):
|
||||
counts = [int(n) for n in counts]
|
||||
if len(counts) != len(self.shots) or any(n <= 0 for n in counts):
|
||||
raise ValueError("frame counts must be positive and match the planned shots")
|
||||
for shot, count in zip(self.shots, counts):
|
||||
shot.frame_count = count
|
||||
|
||||
def validate(self):
|
||||
if any(shot.frame_count <= 0 for shot in self.shots):
|
||||
raise ValueError("each shot needs a positive frame count before rendering")
|
||||
return self
|
||||
|
||||
def __len__(self):
|
||||
return len(self.shots)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PreparedVideo:
|
||||
"""Resolved inputs consumed by the render stage; model/tensor handles are shared."""
|
||||
_placed_shots: object
|
||||
_first_is_plate: object
|
||||
_returns: object
|
||||
_soft_landing: object
|
||||
_tagged_names: object
|
||||
ambient_audio: object
|
||||
ambient_level: float
|
||||
apply_model_sampling: bool
|
||||
audio_vae: object
|
||||
auto_sound: bool
|
||||
bared_shots: object
|
||||
cfg: float
|
||||
cleanup_between_shots: bool
|
||||
clip: object
|
||||
first_frame: object
|
||||
foley_level: float
|
||||
h: int
|
||||
latent_upscale: str
|
||||
latent_upscale_scale: float
|
||||
megapixels: float
|
||||
model: object
|
||||
moved_shots: object
|
||||
negative: object
|
||||
notes: list[str]
|
||||
plan: ShotPlan
|
||||
ref_noise_aug: float | None
|
||||
restart_after_removal: bool
|
||||
revealed_shots: object
|
||||
sampler_name: str
|
||||
scheduler: str
|
||||
seed: int
|
||||
shift_audio: float
|
||||
shift_video: float
|
||||
sigmas: object
|
||||
silence_nonspeech: bool
|
||||
speech_lead_seconds: float
|
||||
speech_tail_seconds: float
|
||||
hold_levels: float
|
||||
handoff_frames: int
|
||||
staging_shots: object
|
||||
steps: int
|
||||
stripped_shots: object
|
||||
cut_shots: object
|
||||
tiled_decode: bool
|
||||
trim_seam: bool
|
||||
upscale: str
|
||||
upscale_batch: int
|
||||
upscale_model: str
|
||||
upscale_target_short_edge: int
|
||||
vae: object
|
||||
w: int
|
||||
shot_rooms: object = None # {0-based shot: (room it opens in, room it ends in)}
|
||||
hardware_changed: object = None # 1-based shots that put hardware on or take it off
|
||||
shot_frames: object = None # {0-based shot: (who its frames show, who is still there at its end)}
|
||||
reentry_shots: object = None # {0-based shot: who walks in while the keyframe still has them}
|
||||
+5377
-2111
File diff suppressed because it is too large
Load Diff
+74
-100
@@ -8,32 +8,15 @@ const DEFAULT_W = 520;
|
||||
const DEFAULT_H = 340;
|
||||
const DEFAULT_BEAT = "Describe this beat.";
|
||||
const STATE_PROPERTY = "dumas_h3_beat_prompt_state";
|
||||
const CONTINUITY_OPTIONS = ["", "soft carry", "hard cut", "keyframe carry", "handoff ref"];
|
||||
const REF_MODE_OPTIONS = ["", "auto ref2v", "where tagged", "first shot", "every shot", "every shot + handoff ref"];
|
||||
const MANAGED_DIRECTIVES = {
|
||||
seconds: ["seconds", "duration"],
|
||||
continuity: ["continuity"],
|
||||
ref_mode: ["ref_mode"],
|
||||
ref_noise_aug: ["ref_noise_aug"],
|
||||
anchor_add: ["anchor_add"],
|
||||
overall_soundscape: ["overall_soundscape", "soundscape"],
|
||||
non_diegetic_music: ["non_diegetic_music", "music"],
|
||||
remove: ["remove", "removed", "off"],
|
||||
add: ["add", "wear", "wearing"],
|
||||
};
|
||||
const DIRECTIVE_EXAMPLES = [
|
||||
["wardrobe set", "wardrobe: Maya = grey shorts, red jacket"],
|
||||
["wardrobe add", "wardrobe: Maya += red jacket"],
|
||||
["wardrobe remove", "wardrobe: Maya -= red jacket"],
|
||||
["seconds", "seconds: 8"],
|
||||
["exit", "exit: Maya"],
|
||||
["enter", "enter: Jon"],
|
||||
["continuity", "continuity: hard cut"],
|
||||
["ref_mode", "ref_mode: every shot"],
|
||||
["ref_noise_aug", "ref_noise_aug: 0.92"],
|
||||
["anchor_add", "anchor_add: harsh sodium-vapor spill, wet pavement, long-lens compression"],
|
||||
["overall_soundscape", "overall_soundscape: soft rain, distant traffic"],
|
||||
["non_diegetic_music", "non_diegetic_music: tense analog synth pulse"],
|
||||
["soundscape", "soundscape: fluorescent room tone, faint HVAC hum"],
|
||||
["music", "music: low ominous cello and sparse percussion"],
|
||||
["remove", "remove: red jacket"],
|
||||
["off", "off: steel collar"],
|
||||
["add", "add: white shirt underneath"],
|
||||
["wearing", "wearing: black coat"],
|
||||
];
|
||||
|
||||
function injectCSS() {
|
||||
@@ -183,7 +166,7 @@ function injectCSS() {
|
||||
}
|
||||
|
||||
function defaultState() {
|
||||
return { beats: [{ text: DEFAULT_BEAT }] };
|
||||
return { scene: "", character_sheet: "", beats: [{ text: DEFAULT_BEAT }] };
|
||||
}
|
||||
|
||||
function normalizeState(value) {
|
||||
@@ -200,7 +183,11 @@ function normalizeState(value) {
|
||||
const normalized = beats.map((beat) => ({
|
||||
text: typeof beat?.text === "string" ? beat.text : String(beat?.text || ""),
|
||||
}));
|
||||
return normalized.length ? { beats: normalized } : defaultState();
|
||||
return {
|
||||
scene: typeof parsed.scene === "string" ? parsed.scene : String(parsed.scene || ""),
|
||||
character_sheet: typeof parsed.character_sheet === "string" ? parsed.character_sheet : String(parsed.character_sheet || ""),
|
||||
beats: normalized.length ? normalized : [{ text: DEFAULT_BEAT }],
|
||||
};
|
||||
}
|
||||
|
||||
function readState(node) {
|
||||
@@ -321,6 +308,51 @@ function renderUI(node) {
|
||||
node._dh3bpRenderedState = JSON.stringify(state);
|
||||
ui.list.innerHTML = "";
|
||||
|
||||
const buildTopTextarea = ({ labelText, placeholder, value, onInput }) => {
|
||||
const card = document.createElement("div");
|
||||
card.className = "dh3bp-beat";
|
||||
|
||||
const label = document.createElement("div");
|
||||
label.className = "dh3bp-label";
|
||||
label.textContent = labelText;
|
||||
|
||||
const textarea = document.createElement("textarea");
|
||||
textarea.className = "dh3bp-text";
|
||||
textarea.placeholder = placeholder;
|
||||
textarea.value = value || "";
|
||||
textarea.addEventListener("input", () => {
|
||||
onInput(textarea.value);
|
||||
updateTextareaHeight(textarea);
|
||||
});
|
||||
textarea.addEventListener("keydown", stopCanvasKeyboard);
|
||||
|
||||
card.append(label, textarea);
|
||||
updateTextareaHeight(textarea);
|
||||
return card;
|
||||
};
|
||||
|
||||
ui.list.appendChild(buildTopTextarea({
|
||||
labelText: "Scene paragraph",
|
||||
placeholder: "Optional. Persistent location, lighting, camera, tone. Leave empty if you wire the Long Videos anchor input.",
|
||||
value: state.scene,
|
||||
onInput: (value) => {
|
||||
const next = readState(node);
|
||||
next.scene = value;
|
||||
writeState(node, next);
|
||||
},
|
||||
}));
|
||||
|
||||
ui.list.appendChild(buildTopTextarea({
|
||||
labelText: "Character sheet",
|
||||
placeholder: "Optional. One character per line, e.g. Maya: 27, she, silver hair, red jacket, the woman in <Picture 1>.",
|
||||
value: state.character_sheet,
|
||||
onInput: (value) => {
|
||||
const next = readState(node);
|
||||
next.character_sheet = value;
|
||||
writeState(node, next);
|
||||
},
|
||||
}));
|
||||
|
||||
state.beats.forEach((beat, index) => {
|
||||
const card = document.createElement("div");
|
||||
card.className = "dh3bp-beat";
|
||||
@@ -379,85 +411,27 @@ function renderUI(node) {
|
||||
return wrap;
|
||||
};
|
||||
|
||||
const secondsInput = document.createElement("input");
|
||||
secondsInput.className = "dh3bp-input";
|
||||
secondsInput.type = "text";
|
||||
secondsInput.placeholder = "8";
|
||||
secondsInput.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.seconds);
|
||||
secondsInput.addEventListener("input", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "seconds", MANAGED_DIRECTIVES.seconds, secondsInput.value));
|
||||
const removeInput = document.createElement("input");
|
||||
removeInput.className = "dh3bp-input";
|
||||
removeInput.type = "text";
|
||||
removeInput.placeholder = "red jacket";
|
||||
removeInput.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.remove);
|
||||
removeInput.addEventListener("input", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "remove", MANAGED_DIRECTIVES.remove, removeInput.value));
|
||||
});
|
||||
|
||||
const continuitySelect = document.createElement("select");
|
||||
continuitySelect.className = "dh3bp-select";
|
||||
CONTINUITY_OPTIONS.forEach((value) => {
|
||||
const option = document.createElement("option");
|
||||
option.value = value;
|
||||
option.textContent = value || "Default";
|
||||
continuitySelect.appendChild(option);
|
||||
});
|
||||
continuitySelect.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.continuity);
|
||||
continuitySelect.addEventListener("change", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "continuity", MANAGED_DIRECTIVES.continuity, continuitySelect.value));
|
||||
});
|
||||
|
||||
const refModeSelect = document.createElement("select");
|
||||
refModeSelect.className = "dh3bp-select";
|
||||
REF_MODE_OPTIONS.forEach((value) => {
|
||||
const option = document.createElement("option");
|
||||
option.value = value;
|
||||
option.textContent = value || "Global";
|
||||
refModeSelect.appendChild(option);
|
||||
});
|
||||
refModeSelect.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.ref_mode);
|
||||
refModeSelect.addEventListener("change", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "ref_mode", MANAGED_DIRECTIVES.ref_mode, refModeSelect.value));
|
||||
});
|
||||
|
||||
const refNoiseInput = document.createElement("input");
|
||||
refNoiseInput.className = "dh3bp-input";
|
||||
refNoiseInput.type = "text";
|
||||
refNoiseInput.placeholder = "0.95";
|
||||
refNoiseInput.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.ref_noise_aug);
|
||||
refNoiseInput.addEventListener("input", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "ref_noise_aug", MANAGED_DIRECTIVES.ref_noise_aug, refNoiseInput.value));
|
||||
});
|
||||
|
||||
const anchorInput = document.createElement("input");
|
||||
anchorInput.className = "dh3bp-input";
|
||||
anchorInput.type = "text";
|
||||
anchorInput.placeholder = "extra per-shot style treatment";
|
||||
anchorInput.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.anchor_add);
|
||||
anchorInput.addEventListener("input", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "anchor_add", MANAGED_DIRECTIVES.anchor_add, anchorInput.value));
|
||||
});
|
||||
|
||||
const soundscapeInput = document.createElement("input");
|
||||
soundscapeInput.className = "dh3bp-input";
|
||||
soundscapeInput.type = "text";
|
||||
soundscapeInput.placeholder = "faint traffic, loose sign rattle";
|
||||
soundscapeInput.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.overall_soundscape);
|
||||
soundscapeInput.addEventListener("input", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "overall_soundscape", MANAGED_DIRECTIVES.overall_soundscape, soundscapeInput.value));
|
||||
});
|
||||
|
||||
const musicInput = document.createElement("input");
|
||||
musicInput.className = "dh3bp-input";
|
||||
musicInput.type = "text";
|
||||
musicInput.placeholder = "low pulsing synth tension";
|
||||
musicInput.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.non_diegetic_music);
|
||||
musicInput.addEventListener("input", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "non_diegetic_music", MANAGED_DIRECTIVES.non_diegetic_music, musicInput.value));
|
||||
const addInput = document.createElement("input");
|
||||
addInput.className = "dh3bp-input";
|
||||
addInput.type = "text";
|
||||
addInput.placeholder = "white shirt underneath";
|
||||
addInput.value = readDirectiveValue(beat.text, MANAGED_DIRECTIVES.add);
|
||||
addInput.addEventListener("input", () => {
|
||||
applyTextUpdate(setDirectiveValue(textarea.value, "add", MANAGED_DIRECTIVES.add, addInput.value));
|
||||
});
|
||||
|
||||
controls.append(
|
||||
buildField({ labelText: "Seconds", input: secondsInput }),
|
||||
buildField({ labelText: "Continuity", input: continuitySelect }),
|
||||
buildField({ labelText: "Ref Mode", input: refModeSelect }),
|
||||
buildField({ labelText: "Ref Noise Aug", input: refNoiseInput }),
|
||||
buildField({ labelText: "Anchor Add", className: "dh3bp-control-wide", input: anchorInput }),
|
||||
buildField({ labelText: "Shot Soundscape", className: "dh3bp-control-wide", input: soundscapeInput }),
|
||||
buildField({ labelText: "Shot Music", className: "dh3bp-control-wide", input: musicInput }),
|
||||
buildField({ labelText: "Remove from memory", input: removeInput }),
|
||||
buildField({ labelText: "Add to memory", input: addInput }),
|
||||
);
|
||||
|
||||
const directives = document.createElement("div");
|
||||
@@ -501,7 +475,7 @@ function setupNode(node) {
|
||||
title.textContent = "Beat Prompt Builder";
|
||||
const subtitle = document.createElement("div");
|
||||
subtitle.className = "dh3bp-subtitle";
|
||||
subtitle.textContent = "One textbox per H3 beat, plus per-shot controls for timing, ref behavior, continuity, anchor adds, and audio directives.";
|
||||
subtitle.textContent = "Upstream Long Videos format: optional scene, optional character sheet, then one blank-line-separated beat per shot.";
|
||||
titleWrap.append(title, subtitle);
|
||||
|
||||
const addButton = document.createElement("button");
|
||||
|
||||
@@ -11,35 +11,62 @@ class DumasH3BeatPromptTests(unittest.TestCase):
|
||||
state = self.module._parse_beat_prompt_state("not json")
|
||||
self.assertEqual(
|
||||
state,
|
||||
{"beats": [{"text": "Describe this beat."}]},
|
||||
{
|
||||
"scene": "",
|
||||
"character_sheet": "",
|
||||
"beats": [{"text": "Describe this beat."}],
|
||||
},
|
||||
)
|
||||
|
||||
def test_assemble_prompt_joins_beats_with_blank_lines(self):
|
||||
def test_assemble_prompt_outputs_upstream_sections(self):
|
||||
prompt = self.module._assemble_beat_prompt(
|
||||
{
|
||||
"scene": "A rainy kitchen at night.",
|
||||
"character_sheet": "Maya: 27, she, red jacket, silver hair.",
|
||||
"beats": [
|
||||
{"text": "A woman enters the room."},
|
||||
{"text": "wardrobe: Maya = red jacket\nShe sits at the table."},
|
||||
{"text": "Maya enters the room."},
|
||||
{"text": "remove: red jacket\nadd: white shirt underneath\nShe sits at the table."},
|
||||
{"text": " "},
|
||||
{"text": "music: low synth pulse"},
|
||||
]
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
prompt,
|
||||
(
|
||||
"A woman enters the room.\n\n"
|
||||
"wardrobe: Maya = red jacket\nShe sits at the table.\n\n"
|
||||
"music: low synth pulse"
|
||||
"A rainy kitchen at night.\n\n"
|
||||
"Maya: 27, she, red jacket, silver hair.\n\n"
|
||||
"Maya enters the room.\n\n"
|
||||
"remove: red jacket\nadd: white shirt underneath\nShe sits at the table."
|
||||
),
|
||||
)
|
||||
|
||||
def test_assemble_prompt_strips_old_dumas_directives(self):
|
||||
prompt = self.module._assemble_beat_prompt(
|
||||
{
|
||||
"beats": [
|
||||
{
|
||||
"text": (
|
||||
"seconds: 8\n"
|
||||
"continuity: hard cut\n"
|
||||
"ref_mode: every shot\n"
|
||||
"soundscape: soft rain\n"
|
||||
"music: low synth\n"
|
||||
"Maya opens the cupboard.\n"
|
||||
"remove: red jacket"
|
||||
)
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(prompt, "Maya opens the cupboard.\nremove: red jacket")
|
||||
|
||||
def test_node_build_prompt_uses_hidden_state(self):
|
||||
node = self.module.DumasH3BeatPromptNode()
|
||||
result = node.build_prompt(
|
||||
'{"beats":[{"text":"Beat one"},{"text":"Beat two"}]}'
|
||||
'{"scene":"Scene","character_sheet":"Maya: 27, she","beats":[{"text":"Beat one"},{"text":"Beat two"}]}'
|
||||
)
|
||||
self.assertEqual(result, ("Beat one\n\nBeat two",))
|
||||
self.assertEqual(result, ("Scene\n\nMaya: 27, she\n\nBeat one\n\nBeat two",))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -23,6 +23,16 @@ class DumasH3LongVideosUpstreamWrapperTests(unittest.TestCase):
|
||||
"folder_paths",
|
||||
"dumas_h3_longvideos",
|
||||
"dumas_h3_longvideos_upstream",
|
||||
"dumas_h3_longvideos_engine",
|
||||
"dumas_h3_longvideos_shot_plan",
|
||||
"dumas_h3_longvideos_runtime",
|
||||
"dumas_h3_longvideos_audio",
|
||||
"dumas_h3_longvideos_conditioning",
|
||||
"h3_engine",
|
||||
"h3_shot_plan",
|
||||
"h3_runtime",
|
||||
"h3_audio",
|
||||
"h3_conditioning",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -112,8 +122,20 @@ class DumasH3LongVideosUpstreamWrapperTests(unittest.TestCase):
|
||||
self.assertIn("first_frame", schema["optional"])
|
||||
self.assertIn("ref_image_1", schema["optional"])
|
||||
self.assertIn("latent_upscale", schema["optional"])
|
||||
self.assertIn("speech_tail_seconds", schema["optional"])
|
||||
self.assertIn("hold_camera", schema["optional"])
|
||||
self.assertIn("verbatim", schema["optional"])
|
||||
self.assertIn("handoff_frames", schema["optional"])
|
||||
self.assertEqual(schema["optional"]["handoff_frames"][1]["default"], 1)
|
||||
self.assertEqual(node_cls.RETURN_NAMES[0:4], ("images", "audio", "info", "script"))
|
||||
|
||||
def test_handoff_context_claim_names_reference_range(self):
|
||||
upstream = importlib.import_module("dumas_h3_longvideos_upstream")
|
||||
|
||||
self.assertIn("<Picture 2> through <Picture 22>", upstream.handoff_context_claim(2, 22))
|
||||
self.assertIn("no new subjects", upstream.handoff_context_claim(2, 22))
|
||||
self.assertIn("<Picture 5>", upstream.handoff_context_claim(5, 5))
|
||||
|
||||
|
||||
class _NullContext:
|
||||
def __enter__(self):
|
||||
|
||||
Reference in New Issue
Block a user